diff --git a/.env.example b/.env.example index 06ec842..b05a114 100644 --- a/.env.example +++ b/.env.example @@ -17,6 +17,29 @@ API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 # Write path of POST message: function = one SQL call (migration 0003), multi = lab3 baseline path. MESSAGE_WRITE_PATH=function +# Lab 4: private S3 bucket (SeaweedFS service `s3`). The API gets ITS OWN key pair (Read+Write on +# this bucket only) as Docker secrets; admin/reconciler pairs go only to s3-init/reconciler. +S3_ENDPOINT=http://s3:8333 +S3_REGION=us-east-1 +S3_BUCKET=messenger-attachments +S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id +S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +# Contract limits (may be lowered, never raised above 10485760 / 40000000). +MAX_UPLOAD_BYTES=10485760 +MAX_IMAGE_PIXELS=40000000 +# Per replica: bodies buffered at once (RAM bound), wait for a slot, body deadline, image checks at once. +UPLOAD_CONCURRENCY=4 +UPLOAD_QUEUE_TIMEOUT_SECONDS=5 +UPLOAD_READ_TIMEOUT_SECONDS=30 +IMAGE_VERIFY_TIMEOUT_SECONDS=10 +IMAGE_VERIFY_CONCURRENCY=1 +JPEG_MAX_MEMORY_MB=256 +S3_TIMEOUT_SECONDS=3 +# Janitor: 'pending' intents older than this are failed uploads (object + row removed). +PENDING_GRACE_SECONDS=900 +JANITOR_INTERVAL_SECONDS=60 +# Level 4 failure experiments only (after_s3_put, after_ready_commit); keep empty. +FAULT_POINTS= # Additional settings of this implementation (defaults shown; all optional): LOG_LEVEL=info DB_POOL_MIN=1 @@ -30,7 +53,9 @@ DRAIN_SECONDS=3 SHUTDOWN_TIMEOUT_SECONDS=8 # CPU/RAM limits (docker compose deploy.resources.limits); API_* are per replica. API_CPUS=1.0 -API_MEMORY=256m +API_MEMORY=512m +S3_CPUS=1.0 +S3_MEMORY=512m PG_CPUS=2.0 PG_MEMORY=1g VALKEY_CPUS=0.5 diff --git a/Makefile b/Makefile index 148df02..3e68a6d 100644 --- a/Makefile +++ b/Makefile @@ -29,9 +29,9 @@ test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) # --------------------------------------------------------------------------- -# Student solution (lab 3, Python). On Windows: PowerShell/Git Bash with GNU make, public +# Student solution (lab 4, Python). On Windows: PowerShell/Git Bash with GNU make, public # tests need PYTHONUTF8=1, and the hooks are started through scripts/*.cmd (Git Bash). -.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series +.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series reconcile reconcile-apply bench-memory bench-reconcile secrets: $(PYTHON) scripts/init_secrets.py @@ -80,3 +80,18 @@ unit-test: explain: $(PYTHON) scripts/explain_history.py --output evidence/explain-history.txt + +# Track 2: reconciler of S3 objects vs attachment rows (dry-run by default, read-only). +reconcile: + docker compose run --rm reconciler + +reconcile-apply: + docker compose run --rm reconciler --apply + +# Level 4: peak RAM of the replicas under 16 simultaneous uploads (evidence/memory/). +bench-memory: + $(PYTHON) bench/upload_memory.py --output evidence/memory + +# Track 2: cost of one reconciler pass for 1k/10k/50k attachments (evidence/reconcile/). +bench-reconcile: + $(PYTHON) bench/reconcile_cost.py --output evidence/reconcile diff --git a/bench/reconcile_cost.py b/bench/reconcile_cost.py new file mode 100644 index 0000000..81255f3 --- /dev/null +++ b/bench/reconcile_cost.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Track 2 experiment: cost of one reconciler pass as a function of the number of attachments. + +Hypothesis: a pass is O(N) with ~N/1000 S3 list requests + N/1000 DB pages and constant memory +(merge-join of two key-ordered streams); --verify-checksums adds reading every byte (O(total size)). + +Method: synthetic, CONSISTENT attachments (row 'ready' + 1 KiB object with the matching size and +SHA-256, filename 'bench-reconcile') are added in steps up to the largest N; after each step the +reconciler runs in dry-run 3 times (median reported), plus page-size and checksum variants. +Everything synthetic is deleted at the end (objects first, then rows). + + python bench/reconcile_cost.py --steps 1000 10000 50000 --output evidence/reconcile +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import statistics +import subprocess +import sys +import time +import uuid +from pathlib import Path + +import httpx2 as httpx + +ROOT = Path(__file__).resolve().parents[1] +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080") +PASSWORD = "Bench-password-Ω-12345" +OBJECT = bytes(range(256)) * 4 # 1 KiB, the same content for every object +SHA = hashlib.sha256(OBJECT).hexdigest() + +PUT_SCRIPT = """ +import asyncio, sys +from messenger.config import Settings, s3_settings +from messenger.storage import Storage +DATA = bytes(range(256)) * 4 +async def main(): + s = Storage(Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings())) + await s.open() + keys = sys.stdin.read().split() + sem = asyncio.Semaphore(32) + async def one(k): + async with sem: + await s.{op}(k{args}) + await asyncio.gather(*(one(k) for k in keys)) + await s.close() + print(len(keys)) +asyncio.run(main()) +""" + + +def compose(*args: str, stdin: str | None = None, timeout: float = 1800, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run(["docker", "compose", *args], cwd=ROOT, input=stdin, capture_output=True, text=True, + encoding="utf-8", timeout=timeout, check=check) + + +def psql(sql: str) -> str: + return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-v", + "ON_ERROR_STOP=1", "-c", sql).stdout.strip() + + +def s3_bulk(op: str, keys: list[str]) -> None: + script = PUT_SCRIPT.replace("{op}", op).replace("{args}", ", DATA, 'image/png', {}" if op == "put" else "") + compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "python", "reconciler", "-c", script, stdin="\n".join(keys)) + + +def owner_and_chat() -> tuple[str, str]: + import base64 + c = httpx.Client(base_url=BASE, timeout=30) + name = "rc_" + uuid.uuid4().hex[:10] + c.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status() + token = c.post("/api/v1/auth/sessions", headers={ + "Authorization": "Basic " + base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()}).json()["token"] + me = c.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json() + chat = c.post("/api/v1/chats", headers={"Authorization": "Bearer " + token}, + json={"title": "bench", "member_ids": [me["id"]]}).json() + return me["id"], chat["id"] + + +def add(n: int, owner: str, chat: str) -> None: + keys = ["o/" + uuid.uuid4().hex for _ in range(n)] + for i in range(0, n, 5000): + part = keys[i:i + 5000] + values = ",".join(f"('{uuid.uuid4()}','{k}')" for k in part) + psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, " + "object_key, status, created_at, updated_at) " + f"SELECT v.id::uuid, '{chat}', '{owner}', 'bench-reconcile', 'image/png', {len(OBJECT)}, " + f"decode('{SHA}', 'hex'), 1, 1, v.k, 'ready', clock_timestamp() - interval '2 hours', " + f"clock_timestamp() - interval '2 hours' FROM (VALUES {values}) AS v(id, k)") + s3_bulk("put", keys) + + +def reconcile(*args: str) -> dict: + out = compose("run", "--rm", "--no-deps", "-T", "reconciler", *args).stdout + return json.loads(out) + + +def measure(label: str, args: list[str], repeats: int) -> dict: + runs = [reconcile(*args) for _ in range(repeats)] + durations = [r["duration_seconds"] for r in runs] + last = runs[-1] + return {"label": label, "args": args, "runs": repeats, "duration_s_median": statistics.median(durations), + "duration_s_all": durations, "objects": last["scanned"]["objects"], "rows": last["scanned"]["rows"], + "findings": {k: v for k, v in last["findings"].items() if v}, "cost": last["cost"]} + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--steps", type=int, nargs="+", default=[1000, 10000, 50000]) + p.add_argument("--repeats", type=int, default=3) + p.add_argument("--output", default="evidence/reconcile") + args = p.parse_args() + out = ROOT / args.output + out.mkdir(parents=True, exist_ok=True) + owner, chat = owner_and_chat() + present = 0 + results = [] + steps = sorted(args.steps) + variant_step = steps[min(1, len(steps) - 1)] + try: + for target in sorted(args.steps): + started = time.perf_counter() + add(target - present, owner, chat) + present = target + print(f"populated {present} synthetic attachments in {time.perf_counter() - started:.1f} s", flush=True) + results.append({"synthetic": present, **measure("dry-run, page 1000", [], args.repeats)}) + print(json.dumps(results[-1], ensure_ascii=False), flush=True) + if present == variant_step: # page-size and checksum variants at one size + for label, extra in (("dry-run, page 100", ["--page-size", "100"]), + ("dry-run, --verify-checksums", ["--verify-checksums"])): + results.append({"synthetic": present, **measure(label, extra, args.repeats)}) + print(json.dumps(results[-1], ensure_ascii=False), flush=True) + finally: + keys = psql("SELECT object_key FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = " + f"'{chat}'").split() + if keys: + s3_bulk("delete", keys) + psql(f"DELETE FROM attachments WHERE filename = 'bench-reconcile' AND chat_id = '{chat}'") + print(f"removed {len(keys)} synthetic attachments", flush=True) + meta = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(), + "commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, + text=True).stdout.strip(), + "object_bytes": len(OBJECT), "reconciler_limits": "cpus 1.0, memory 256m (compose)", + "note": "the bucket/table also contain the attachments of earlier test runs (counted in objects/rows)"} + (out / "cost.json").write_text(json.dumps({"meta": meta, "results": results}, ensure_ascii=False, indent=1), + encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bench/upload_memory.py b/bench/upload_memory.py new file mode 100644 index 0000000..fa13206 --- /dev/null +++ b/bench/upload_memory.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Level 4 experiment: peak memory of the API replicas under >= 10 simultaneous uploads. + +Hypothesis: the memory of one replica is bounded by UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES (bodies +waiting for a slot stay in TCP buffers, not in the process) + one image check at a time +(IMAGE_VERIFY_CONCURRENCY), independent of how many clients upload at once. + +Method (repeatable, the stand must be up): +* for every scenario the api replicas are RECREATED with the scenario's settings, so the cgroup + counter memory.peak starts from an idle process (it never decreases during a container's life); +* idle memory.current of each replica is read; then N clients upload at the same moment through + the LB (a barrier releases them together), each a different file; +* during the run memory.current of every replica is sampled every 50 ms inside the container; + afterwards memory.peak (cgroup v2: all processes of the container incl. djpeg) and VmHWM of + PID 1 (the Python server alone) are read; +* the normal configuration is restored at the end. + + python bench/upload_memory.py --output evidence/memory +""" +from __future__ import annotations + +import argparse +import io +import json +import os +import platform +import statistics +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +import httpx2 as httpx +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "student-tests")) +import imagegen # noqa: E402 + +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080") +PASSWORD = "Bench-password-Ω-12345" +MiB = 1024 * 1024 + + +def sh(*args: str, env: dict | None = None, timeout: float = 300) -> str: + return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=True, timeout=timeout, + env={**os.environ, **(env or {})}).stdout + + +def replicas() -> list[str]: + return sh("docker", "compose", "ps", "-q", "api").split() + + +def cg(cid: str, name: str) -> int: + return int(sh("docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}").split()[0]) + + +def vmhwm(cid: str) -> int: + for line in sh("docker", "exec", cid, "cat", "/proc/1/status").splitlines(): + if line.startswith("VmHWM:"): + return int(line.split()[1]) * 1024 + return 0 + + +def recreate(env: dict) -> None: + sh("docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "--wait", "api", env=env, timeout=300) + sh(sys.executable, "scripts/lbctl.py", "wait-up", str(len(replicas())), "--timeout", "90") # LB sees them UP + time.sleep(3) + + +class Sampler(threading.Thread): + """memory.current of one container every 50 ms, read by a shell loop inside it.""" + + def __init__(self, cid: str): + super().__init__(daemon=True) + self.cid = cid + self.values: list[int] = [] + self.proc = subprocess.Popen(["docker", "exec", cid, "sh", "-c", + "while :; do cat /sys/fs/cgroup/memory.current; sleep 0.05; done"], + stdout=subprocess.PIPE, text=True) + + def run(self) -> None: + for line in self.proc.stdout: + if line.strip().isdigit(): + self.values.append(int(line)) + + def stop(self) -> None: + self.proc.kill() + + +def actor(client: httpx.Client) -> tuple[str, str]: + name = "mem_" + uuid.uuid4().hex[:10] + client.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status() + import base64 + basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode() + token = client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}).json()["token"] + me = client.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json() + chat = client.post("/api/v1/chats", headers={"Authorization": "Bearer " + token}, + json={"title": "mem", "member_ids": [me["id"]]}).json() + return token, chat["id"] + + +def big_jpeg(width: int, height: int, seed: int) -> bytes: + """Large progressive JPEG (smooth content keeps it < 10 MiB): the costliest file to check.""" + image = Image.linear_gradient("L").resize((width, height)).convert("RGB") + image.putpixel((seed % width, 0), (seed % 255, 0, 0)) + out = io.BytesIO() + image.save(out, "JPEG", quality=90, progressive=True, subsampling=0) + return out.getvalue() + + +def run_scenario(name: str, env: dict, files: list[tuple[bytes, str]]) -> dict: + recreate(env) + ids = replicas() + idle = {cid: cg(cid, "memory.current") for cid in ids} + client = httpx.Client(base_url=BASE, timeout=120) + token, chat_id = actor(client) + barrier = threading.Barrier(len(files)) + results: list[tuple[int, float, str]] = [None] * len(files) # type: ignore[list-item] + + def upload(i: int, data: bytes, ctype: str) -> None: + c = httpx.Client(base_url=BASE, timeout=120) + barrier.wait() + started = time.perf_counter() + r = c.post(f"/api/v1/chats/{chat_id}/attachments?filename=f{i}", content=data, + headers={"Authorization": "Bearer " + token, "Content-Type": ctype}) + results[i] = (r.status_code, time.perf_counter() - started, r.headers.get("x-instance-id", "-")) + c.close() + + samplers = [Sampler(cid) for cid in ids] + for s in samplers: + s.start() + time.sleep(0.5) + threads = [threading.Thread(target=upload, args=(i, d, t)) for i, (d, t) in enumerate(files)] + wall = time.perf_counter() + for t in threads: + t.start() + for t in threads: + t.join() + wall = time.perf_counter() - wall + time.sleep(0.5) + for s in samplers: + s.stop() + per_replica = {} + for cid, s in zip(ids, samplers): + hostname = sh("docker", "inspect", "-f", "{{.Config.Hostname}}", cid).strip() + per_replica[hostname] = { + "idle_mib": round(idle[cid] / MiB, 1), + "peak_mib": round(cg(cid, "memory.peak") / MiB, 1), + "sampled_max_mib": round(max(s.values or [0]) / MiB, 1), + "python_vmhwm_mib": round(vmhwm(cid) / MiB, 1), + "uploads_served": sum(1 for r in results if r[2] == hostname), + "limit_mib": round(cg(cid, "memory.max") / MiB, 1), + } + per_replica[hostname]["peak_minus_idle_mib"] = round(per_replica[hostname]["peak_mib"] + - per_replica[hostname]["idle_mib"], 1) + lat = sorted(r[1] for r in results if r[0] == 201) + codes: dict[str, int] = {} + for r in results: + codes[str(r[0])] = codes.get(str(r[0]), 0) + 1 + summary = { + "scenario": name, "settings": env, "clients": len(files), + "file_mib_each": round(statistics.mean(len(d) for d, _ in files) / MiB, 2), + "types": sorted({t for _, t in files}), "status_codes": codes, "wall_seconds": round(wall, 2), + "latency_s": {"p50": round(lat[len(lat) // 2], 2), "max": round(lat[-1], 2)} if lat else None, + "replicas": per_replica, + } + print(json.dumps(summary, ensure_ascii=False), flush=True) + return summary + + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--clients", type=int, default=16) + p.add_argument("--output", default="evidence/memory") + args = p.parse_args() + out = ROOT / args.output + out.mkdir(parents=True, exist_ok=True) + print("generating files ...", flush=True) + pngs = [(imagegen.png_of_size(int(9.5 * MiB)), "image/png") for _ in range(args.clients)] + jpegs = [(big_jpeg(8000, 5000, i), "image/jpeg") for i in range(args.clients)] + base = {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"} + scenarios = [ + ("png-9.5MiB x16, slots=4 (default)", base, pngs), + ("png-9.5MiB x16, slots=16 (no bound)", {"UPLOAD_CONCURRENCY": "16", "IMAGE_VERIFY_CONCURRENCY": "1"}, pngs), + ("jpeg-40Mpx-progressive x16, slots=4, verify=1 (default)", base, jpegs), + ("jpeg-40Mpx-progressive x16, slots=4, verify=4", {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "4"}, jpegs), + ] + results = [] + try: + for name, env, files in scenarios: + results.append(run_scenario(name, env, files)) + finally: + recreate({}) # back to the compose defaults + meta = { + "date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(), + "commit": sh("git", "rev-parse", "--short", "HEAD").strip(), + "api_limits": sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", + replicas()[0]).strip(), + "jpeg_file_mib": round(len(jpegs[0][0]) / MiB, 2), "png_file_mib": round(len(pngs[0][0]) / MiB, 2), + } + (out / "summary.json").write_text(json.dumps({"meta": meta, "scenarios": results}, ensure_ascii=False, indent=1), + encoding="utf-8") + print(json.dumps(meta, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/compose.yaml b/compose.yaml index 580fd27..4d427ce 100644 --- a/compose.yaml +++ b/compose.yaml @@ -42,6 +42,7 @@ x-api-env: &api-env UPLOAD_QUEUE_TIMEOUT_SECONDS: ${UPLOAD_QUEUE_TIMEOUT_SECONDS:-5} UPLOAD_READ_TIMEOUT_SECONDS: ${UPLOAD_READ_TIMEOUT_SECONDS:-30} IMAGE_VERIFY_TIMEOUT_SECONDS: ${IMAGE_VERIFY_TIMEOUT_SECONDS:-10} + IMAGE_VERIFY_CONCURRENCY: ${IMAGE_VERIFY_CONCURRENCY:-1} JPEG_MAX_MEMORY_MB: ${JPEG_MAX_MEMORY_MB:-256} PENDING_GRACE_SECONDS: ${PENDING_GRACE_SECONDS:-900} JANITOR_INTERVAL_SECONDS: ${JANITOR_INTERVAL_SECONDS:-60} diff --git a/migrations/0004_attachments.sql b/migrations/0004_attachments.sql index 55f69b3..efc1d5f 100644 --- a/migrations/0004_attachments.sql +++ b/migrations/0004_attachments.sql @@ -50,7 +50,14 @@ CREATE TABLE attachments ( -- Janitor: stale intents, oldest first; tiny partial index (only rows of unfinished uploads). CREATE INDEX attachments_pending_created ON attachments (created_at) WHERE status = 'pending'; --- Which attachments a message references, in the order of attachment_ids. +-- attachment_ids of a message, twice on purpose (both written in the message's transaction): +-- * messages.attachment_ids (ordered array): the history page reads it from the same row, so the +-- keyset range scan of messages stays ONE index scan, without a per-row sub-query; +-- * message_attachments: foreign keys (a referenced attachment can never be deleted) and the +-- reverse lookup "is this attachment used?" (janitor/reconciler) by an index on attachment_id. +ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}'; +ALTER TABLE messages ADD CONSTRAINT messages_attachment_ids_len CHECK (cardinality(attachment_ids) <= 10); + CREATE TABLE message_attachments ( message_id uuid NOT NULL REFERENCES messages (id), position smallint NOT NULL, @@ -115,8 +122,8 @@ BEGIN IF FOUND THEN UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond') WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created; - INSERT INTO messages (id, chat_id, sender_id, text, created_at) - VALUES (p_message, p_chat, p_user, p_text, v_created); + INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) + VALUES (p_message, p_chat, p_user, p_text, v_created, p_attachments); INSERT INTO message_attachments (message_id, position, attachment_id) SELECT p_message, (a.ord - 1)::smallint, a.id FROM unnest(p_attachments) WITH ORDINALITY AS a(id, ord); @@ -127,8 +134,7 @@ BEGIN RETURN QUERY SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at, - ARRAY(SELECT ma.attachment_id FROM message_attachments ma - WHERE ma.message_id = m.id ORDER BY ma.position) + m.attachment_ids FROM idempotency_keys k JOIN messages m ON m.id = k.message_id WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key; IF NOT FOUND THEN diff --git a/src/messenger/app.py b/src/messenger/app.py index cc32393..9d04791 100644 --- a/src/messenger/app.py +++ b/src/messenger/app.py @@ -389,7 +389,7 @@ async def upload_attachment(request: Request) -> JSONResponse: except asyncio.TimeoutError: raise attachments.Busy("Слишком много одновременных загрузок, повторите позже") from None try: - upload = await attachments.read_image(reader, settings, declared) + upload = await attachments.read_image(reader, settings, declared, request.app.state.verify_slots) finally: slots.release() except ApiError: @@ -518,7 +518,8 @@ def create_app(settings: Settings | None = None) -> Starlette: "db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds, "instance_id": settings.instance_id, "drain_s": settings.drain_seconds, "message_write_path": settings.message_write_path, "s3_bucket": settings.s3_bucket, - "upload_concurrency": settings.upload_concurrency, "pending_grace_s": settings.pending_grace_seconds, + "upload_concurrency": settings.upload_concurrency, "verify_concurrency": settings.image_verify_concurrency, + "pending_grace_s": settings.pending_grace_seconds, "fault_points": sorted(settings.fault_points)}}) tasks = [asyncio.create_task(purge_loop())] if settings.janitor_interval_seconds > 0: @@ -567,5 +568,6 @@ def create_app(settings: Settings | None = None) -> Starlette: app.state.sessions = sessions app.state.storage = storage app.state.upload_slots = asyncio.Semaphore(settings.upload_concurrency) + app.state.verify_slots = asyncio.Semaphore(settings.image_verify_concurrency) app.state.draining = False return app diff --git a/src/messenger/attachments.py b/src/messenger/attachments.py index fd9804f..f6a95e5 100644 --- a/src/messenger/attachments.py +++ b/src/messenger/attachments.py @@ -227,7 +227,7 @@ def declared_length(request: Request, limit: int) -> int | None: return length -async def read_image(reader: BodyReader, settings: Settings, declared: str) -> Upload: +async def read_image(reader: BodyReader, settings: Settings, declared: str, verify_slots: asyncio.Semaphore) -> Upload: """Body -> verified image. Memory: at most MAX_UPLOAD_BYTES per upload (bytearray); the format and the pixel count are known after the first bytes, so a picture that is too big or not a PNG/JPEG is rejected without reading/decoding the rest.""" @@ -260,8 +260,9 @@ async def read_image(reader: BodyReader, settings: Settings, declared: str) -> U data = bytes(buffer) del buffer try: - await images.verify(data, info, timeout=settings.image_verify_timeout_seconds, - jpeg_max_memory_mb=settings.jpeg_max_memory_mb) + async with verify_slots: + await images.verify(data, info, timeout=settings.image_verify_timeout_seconds, + jpeg_max_memory_mb=settings.jpeg_max_memory_mb) except images.ImageError as exc: raise _image_error(exc) from None except asyncio.TimeoutError: diff --git a/src/messenger/config.py b/src/messenger/config.py index 1e3c1dc..1772442 100644 --- a/src/messenger/config.py +++ b/src/messenger/config.py @@ -130,6 +130,7 @@ class Settings: upload_queue_timeout_seconds: float = 5.0 upload_read_timeout_seconds: float = 30.0 image_verify_timeout_seconds: float = 10.0 + image_verify_concurrency: int = 1 jpeg_max_memory_mb: int = 256 pending_grace_seconds: int = 900 janitor_interval_seconds: float = 60.0 @@ -223,6 +224,9 @@ def s3_settings(*, credentials_required: bool = True) -> dict: upload_queue_timeout_seconds=_float("UPLOAD_QUEUE_TIMEOUT_SECONDS", 5.0, 0.0, 60.0), upload_read_timeout_seconds=_float("UPLOAD_READ_TIMEOUT_SECONDS", 30.0, 1.0, 300.0), image_verify_timeout_seconds=_float("IMAGE_VERIFY_TIMEOUT_SECONDS", 10.0, 1.0, 120.0), + # Full checks running at once per replica: a strict JPEG decode of a 40 Mpx progressive + # file needs up to ~120-240 MB of coefficient buffers (capped by JPEG_MAX_MEMORY_MB). + image_verify_concurrency=_int("IMAGE_VERIFY_CONCURRENCY", 1, 1, 16), jpeg_max_memory_mb=_int("JPEG_MAX_MEMORY_MB", 256, 16, 2048), # An intent row still `pending` after this long belongs to a failed upload: the janitor # deletes its object and the row. Far above the longest possible upload (~45 s). diff --git a/src/messenger/reconcile.py b/src/messenger/reconcile.py new file mode 100644 index 0000000..3e07569 --- /dev/null +++ b/src/messenger/reconcile.py @@ -0,0 +1,430 @@ +"""Reconciler of the two stores (lab 4, track 2): S3 objects vs attachment rows in PostgreSQL. + + docker compose run --rm reconciler # dry-run: report only (default) + docker compose run --rm reconciler --apply # fix what is safe to fix + docker compose run --rm reconciler --verify-checksums # also read every object (SHA-256) + +How it walks: both sides are read as key-ordered streams and merge-joined - ListObjectsV2 +pages (prefix o/, byte order) against keyset pages of `attachments ORDER BY object_key` +(COLLATE "C" = the same byte order, unique index). Memory does not depend on the bucket size; +cost is ~N/1000 S3 list requests + N/page DB pages (measured: docs/CONSISTENCY.md). + +What it finds and what --apply does (nothing else is ever changed): + + kind condition --apply + orphan_object object, no row, older than --grace delete the object + stale_intent row 'pending' older than --grace (+/- object) delete object, then row + missing_object row 'ready', object absent, older than --grace quarantine the row + size_mismatch row 'ready', object size != size_bytes quarantine the row + checksum_mismatch row 'ready', SHA-256 differs (--verify-checksums) quarantine the row + restored_object row quarantined, object present and correct again remove the quarantine + fresh_* the same, but younger than --grace nothing (in-flight upload) + +Safety: +* dry-run is read-only BY CONSTRUCTION: the PostgreSQL session runs with + default_transaction_read_only=on and the S3 wrapper has no write methods, so even a bug in + the planner cannot modify a store; +* ready rows are never deleted - a used (or unused) ready attachment can only be quarantined, + which is reversible; objects that have a row are never deleted unless the row is a stale + intent; anything younger than --grace is never touched; +* every action re-checks its condition right before acting, under a row lock (FOR UPDATE) + where a row exists, so the live API racing with the reconciler cannot be hurt; +* object first, row second: an interrupted run leaves a state the next run repairs; + re-running is idempotent (the second run finds nothing to do); +* safety valve: --apply refuses to delete more than --max-orphan-fraction of the scanned + objects (pointing it at the wrong/empty database would otherwise wipe the bucket). +""" +from __future__ import annotations + +import argparse +import asyncio +import datetime as dt +import hashlib +import json +import sys +import time +from collections import Counter +from collections.abc import AsyncIterator +from dataclasses import dataclass, field + +import psycopg + +from .config import ConfigError, Settings, _float, _secret, s3_settings +from .errors import Unavailable +from .storage import OBJECT_PREFIX, ListedObject, ObjectMissing, Storage + +KINDS = ("orphan_object", "stale_intent", "missing_object", "size_mismatch", "checksum_mismatch", + "restored_object", "fresh_orphan_object", "fresh_intent", "fresh_missing_object") +MIN_SAFE_GRACE = 60.0 # > the longest possible upload (queue 5 s + body 30 s + verify 10 s + S3) +ACTION_OF = {"orphan_object": "delete_object", "stale_intent": "delete_intent", "missing_object": "quarantine", + "size_mismatch": "quarantine", "checksum_mismatch": "quarantine", "restored_object": "unquarantine"} + + +def log(text: str) -> None: + print(f"reconcile: {text}", file=sys.stderr, flush=True) + + +@dataclass(frozen=True, slots=True) +class Row: + id: str + key: str + status: str + size: int + sha256: bytes + created_at: dt.datetime + updated_at: dt.datetime + lost_reason: str | None + used: bool + + +@dataclass(slots=True) +class Finding: + kind: str + key: str + attachment_id: str | None = None + used: bool = False + detail: str = "" + result: str = "planned" # planned | applied | skipped: | failed: + + +@dataclass +class Stats: + objects: int = 0 + rows: int = 0 + consistent: int = 0 + s3_list_requests: int = 0 + s3_head_requests: int = 0 + s3_get_bytes: int = 0 + db_pages: int = 0 + + +class ReadOnlyStorage: + """The dry-run view of S3: listing and reading only - there is no delete to call.""" + + def __init__(self, storage: Storage): + self._s = storage + + async def list_page(self, *args, **kwargs): + return await self._s.list_page(*args, **kwargs) + + async def head(self, key: str): + return await self._s.head(key) + + async def read_all(self, key: str, limit: int) -> bytes: + return await self._s.read_all(key, limit) + + +class Reconciler: + def __init__(self, conn: psycopg.AsyncConnection, storage: Storage | ReadOnlyStorage, *, grace: float, + page_size: int, verify: bool, apply: bool): + self.conn = conn + self.s3 = storage + self.grace = grace + self.page_size = page_size + self.verify = verify + self.apply = apply + self.stats = Stats() + self.findings: list[Finding] = [] + + # ---------------------------------------------------------- streams + async def objects(self) -> AsyncIterator[ListedObject]: + after = None + while True: + page, more = await self.s3.list_page(OBJECT_PREFIX, after, self.page_size) + self.stats.s3_list_requests += 1 + for item in page: + self.stats.objects += 1 + yield item + if not more or not page: + return + after = page[-1].key + + async def rows(self) -> AsyncIterator[Row]: + after = "" + while True: + cur = await self.conn.execute( + "SELECT a.id, a.object_key, a.status, a.size_bytes, a.sha256, a.created_at, a.updated_at, " + "a.lost_reason, EXISTS (SELECT 1 FROM message_attachments ma WHERE ma.attachment_id = a.id) " + "FROM attachments a WHERE a.object_key > %s ORDER BY a.object_key LIMIT %s", + (after, self.page_size)) + page = await cur.fetchall() + self.stats.db_pages += 1 + for r in page: + self.stats.rows += 1 + yield Row(str(r[0]), r[1], r[2], r[3], bytes(r[4]), r[5], r[6], r[7], r[8]) + if len(page) < self.page_size: + return + after = page[-1][1] + + # ---------------------------------------------------------- planning (read-only) + async def scan(self) -> None: + cur = await self.conn.execute("SELECT clock_timestamp()") + self.db_now = (await cur.fetchone())[0] + self.s3_now = time.time() + objects, rows = self.objects(), self.rows() + obj = await anext(objects, None) + row = await anext(rows, None) + while obj is not None or row is not None: + if row is None or (obj is not None and obj.key < row.key): + self.object_only(obj) + obj = await anext(objects, None) + elif obj is None or row.key < obj.key: + self.row_only(row) + row = await anext(rows, None) + else: + await self.both(obj, row) + obj = await anext(objects, None) + row = await anext(rows, None) + + def row_old(self, row: Row) -> bool: + return (self.db_now - max(row.created_at, row.updated_at)).total_seconds() > self.grace + + def object_only(self, obj: ListedObject) -> None: + old = self.s3_now - obj.last_modified_ts > self.grace + self.add(Finding("orphan_object" if old else "fresh_orphan_object", obj.key)) + + def row_only(self, row: Row) -> None: + if row.status == "pending": + self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id, + detail="object absent")) + elif row.lost_reason is not None: + self.stats.consistent += 1 # already quarantined, still missing: nothing new + else: + self.add(Finding("missing_object" if self.row_old(row) else "fresh_missing_object", row.key, row.id, + used=row.used)) + + async def both(self, obj: ListedObject, row: Row) -> None: + if row.status == "pending": + self.add(Finding("stale_intent" if self.row_old(row) else "fresh_intent", row.key, row.id, + detail="object present")) + return + problem = await self.object_problem(row, obj.size) + if problem: + if row.lost_reason is None: + self.add(Finding(problem, row.key, row.id, used=row.used)) + else: + self.stats.consistent += 1 # quarantined and still broken + elif row.lost_reason is not None: + self.add(Finding("restored_object", row.key, row.id, used=row.used)) + else: + self.stats.consistent += 1 + + async def object_problem(self, row: Row, size: int) -> str | None: + if size != row.size: + return "size_mismatch" + if self.verify: + data = await self.s3.read_all(row.key, row.size) + self.stats.s3_get_bytes += len(data) + if hashlib.sha256(data).digest() != row.sha256: + return "checksum_mismatch" + return None + + def add(self, finding: Finding) -> None: + if finding.kind not in ACTION_OF: + finding.result = "not_actionable" + self.findings.append(finding) + + # ---------------------------------------------------------- actions (--apply only) + async def run_actions(self, max_orphan_fraction: float) -> None: + orphans = sum(1 for f in self.findings if f.kind == "orphan_object") + if self.stats.objects and orphans > max_orphan_fraction * self.stats.objects: + raise SafetyValve(f"{orphans} of {self.stats.objects} objects look orphaned " + f"(> {max_orphan_fraction:.0%}); wrong database? nothing was changed") + handlers = {"delete_object": self.delete_object, "delete_intent": self.delete_intent, + "quarantine": self.quarantine, "unquarantine": self.unquarantine} + for finding in self.findings: + action = ACTION_OF.get(finding.kind) + if action is None: + continue + try: + finding.result = await handlers[action](finding) + except (Unavailable, psycopg.OperationalError) as exc: + finding.result = f"failed:{type(exc).__name__}" + raise # a store is down: stop; the next run continues from the committed state + + async def delete_object(self, f: Finding) -> str: + cur = await self.conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (f.key,)) + if await cur.fetchone(): + return "skipped:row_appeared" + try: + head = await self.s3.head(f.key) + self.stats.s3_head_requests += 1 + except ObjectMissing: + return "skipped:already_gone" + if time.time() - head.last_modified_ts <= self.grace: + return "skipped:object_rewritten_recently" + # No row names this key and the object is older than the grace period. Rows are always + # written BEFORE their object and keys are random and never reused, so no upload can + # still claim it. + await self.s3.delete(f.key) + return "applied" + + async def delete_intent(self, f: Finding) -> str: + async with self.conn.transaction(): + cur = await self.conn.execute( + "SELECT 1 FROM attachments WHERE id = %s AND status = 'pending' " + "AND greatest(created_at, updated_at) < clock_timestamp() - make_interval(secs => %s) FOR UPDATE", + (f.attachment_id, self.grace)) + if await cur.fetchone() is None: + return "skipped:no_longer_stale_intent" + await self.s3.delete(f.key) # object first (idempotent), then the row + await self.conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'pending'", + (f.attachment_id,)) + return "applied" + + async def quarantine(self, f: Finding) -> str: + async with self.conn.transaction(): + cur = await self.conn.execute( + "SELECT size_bytes, sha256 FROM attachments WHERE id = %s AND status = 'ready' " + "AND lost_at IS NULL FOR UPDATE", (f.attachment_id,)) + row = await cur.fetchone() + if row is None: + return "skipped:row_changed" + reason = await self.current_problem(f.key, row[0], bytes(row[1])) + if reason is None: + return "skipped:object_ok_now" + await self.conn.execute( + "UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = %s WHERE id = %s", + (reason, f.attachment_id)) + return "applied" + + async def unquarantine(self, f: Finding) -> str: + async with self.conn.transaction(): + cur = await self.conn.execute( + "SELECT size_bytes, sha256 FROM attachments WHERE id = %s AND lost_at IS NOT NULL FOR UPDATE", + (f.attachment_id,)) + row = await cur.fetchone() + if row is None: + return "skipped:row_changed" + if await self.current_problem(f.key, row[0], bytes(row[1])) is not None: + return "skipped:object_still_bad" + await self.conn.execute("UPDATE attachments SET lost_at = NULL, lost_reason = NULL WHERE id = %s", + (f.attachment_id,)) + return "applied" + + async def current_problem(self, key: str, size: int, sha256: bytes) -> str | None: + """Fresh look at the object right before changing the row (the listing may be stale).""" + try: + head = await self.s3.head(key) + self.stats.s3_head_requests += 1 + except ObjectMissing: + return "missing_object" + if head.size != size: + return "size_mismatch" + if self.verify: + data = await self.s3.read_all(key, size) + self.stats.s3_get_bytes += len(data) + if hashlib.sha256(data).digest() != sha256: + return "checksum_mismatch" + return None + + # ---------------------------------------------------------- report + def report(self, mode: str, seconds: float, error: str | None, sample: int) -> dict: + kinds = Counter(f.kind for f in self.findings) + results = Counter(f.result.split(":")[0] for f in self.findings if f.kind in ACTION_OF) + samples = {} + for f in self.findings: + items = samples.setdefault(f.kind, []) + if len(items) < sample: + items.append({"key": f.key, "attachment_id": f.attachment_id, "used": f.used, + "result": f.result, **({"detail": f.detail} if f.detail else {})}) + return { + "mode": mode, "grace_seconds": self.grace, "verify_checksums": self.verify, + "duration_seconds": round(seconds, 3), "error": error, + "scanned": {"objects": self.stats.objects, "rows": self.stats.rows, "consistent": self.stats.consistent}, + "findings": {k: kinds.get(k, 0) for k in KINDS}, + "used_attachments_in_findings": sum(1 for f in self.findings if f.used), + "actions": {"planned": sum(1 for f in self.findings if f.kind in ACTION_OF), + "applied": results.get("applied", 0), "skipped": results.get("skipped", 0), + "failed": results.get("failed", 0)}, + "cost": {"s3_list_requests": self.stats.s3_list_requests, "s3_head_requests": self.stats.s3_head_requests, + "s3_get_bytes": self.stats.s3_get_bytes, "db_pages": self.stats.db_pages, + "page_size": self.page_size}, + "samples": samples, + } + + +class SafetyValve(Exception): + pass + + +def arguments(argv: list[str] | None = None) -> argparse.Namespace: + p = argparse.ArgumentParser(prog="python -m messenger.reconcile", description=__doc__.split("\n")[0]) + p.add_argument("--apply", action="store_true", help="fix findings (default: dry-run, read-only)") + p.add_argument("--grace", type=float, default=3600.0, + help="seconds; anything younger is never touched (default 3600, >= PENDING_GRACE_SECONDS)") + p.add_argument("--allow-small-grace", action="store_true", + help="permit --grace below 60 s (tests only: a grace shorter than an upload could remove the " + "intent of an upload that is still running; that upload then fails with 503)") + p.add_argument("--page-size", type=int, default=1000, help="S3 list / DB page size (1..1000)") + p.add_argument("--verify-checksums", action="store_true", help="read every ready object and compare SHA-256") + p.add_argument("--samples", type=int, default=20, help="keys listed per finding kind in the report") + p.add_argument("--max-orphan-fraction", type=float, default=0.5, + help="--apply refuses if more than this share of objects look orphaned (default 0.5)") + args = p.parse_args(argv) + if not 1 <= args.page_size <= 1000: + p.error("--page-size must be 1..1000") + if args.grace < 1 or (args.grace < MIN_SAFE_GRACE and not args.allow_small_grace): + p.error(f"--grace must be >= {MIN_SAFE_GRACE:.0f} s (or >= 1 s with --allow-small-grace)") + if not 0 < args.max_orphan_fraction <= 1: + p.error("--max-orphan-fraction must be in (0, 1]") + return args + + +async def run(args: argparse.Namespace) -> int: + url = _secret("DATABASE_URL", required=True) + base = s3_settings() + settings = Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024, + database_url=url, **base) + timeout_ms = int(_float("DB_TIMEOUT_SECONDS", 10.0, 1.0, 120.0) * 1000) + options = f"-c statement_timeout={timeout_ms} -c lock_timeout={timeout_ms // 2}" + if not args.apply: + options += " -c default_transaction_read_only=on" # dry-run: PostgreSQL itself refuses writes + mode = "apply" if args.apply else "dry-run" + storage = Storage(settings) + await storage.open() + started = time.perf_counter() + error = None + code = 0 + try: + conn = await psycopg.AsyncConnection.connect(url, autocommit=True, connect_timeout=5, options=options, + application_name=f"messenger-reconcile-{mode}") + except psycopg.OperationalError as exc: + await storage.close() + error = f"store_unavailable: database {type(exc).__name__}" + print(json.dumps({"mode": mode, "error": error}, ensure_ascii=False)) + log(error) + return 1 + rec = Reconciler(conn, storage if args.apply else ReadOnlyStorage(storage), grace=args.grace, + page_size=args.page_size, verify=args.verify_checksums, apply=args.apply) + try: + log(f"{mode}: scanning bucket {settings.s3_bucket} (prefix {OBJECT_PREFIX}) against table attachments") + await rec.scan() + log(f"scan done: {rec.stats.objects} objects, {rec.stats.rows} rows, {len(rec.findings)} findings") + if args.apply: + await rec.run_actions(args.max_orphan_fraction) + except SafetyValve as exc: + error, code = f"safety_valve: {exc}", 3 + except (Unavailable, psycopg.OperationalError) as exc: + error, code = f"store_unavailable: {type(exc).__name__}", 1 + finally: + await conn.close() + await storage.close() + print(json.dumps(rec.report(mode, time.perf_counter() - started, error, args.samples), ensure_ascii=False, + indent=1)) + if error: + log(error) + return code + + +def main(argv: list[str] | None = None) -> int: + args = arguments(argv) + try: + return asyncio.run(run(args)) + except ConfigError as exc: + print(f"configuration error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/messenger/repo.py b/src/messenger/repo.py index d1b8aba..05791c0 100644 --- a/src/messenger/repo.py +++ b/src/messenger/repo.py @@ -219,8 +219,9 @@ async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID, (chat_id,)) (created_at,) = await cur.fetchone() await conn.execute( - "INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)", - (message_id, chat_id, user_id, _b(text), created_at)) + "INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) " + "VALUES (%s, %s, %s, %s, %s, %s::uuid[])", + (message_id, chat_id, user_id, _b(text), created_at, attachment_ids)) if attachment_ids: await conn.execute( "INSERT INTO message_attachments (message_id, position, attachment_id) " @@ -266,9 +267,7 @@ CLAIM_KEY = ( "WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => %s) " "RETURNING 1") STORED_RESULT = ( - "SELECT k.request_hash, m.id, m.text, m.created_at, " - "ARRAY(SELECT ma.attachment_id FROM message_attachments ma WHERE ma.message_id = m.id ORDER BY ma.position) " - "FROM idempotency_keys k " + "SELECT k.request_hash, m.id, m.text, m.created_at, m.attachment_ids FROM idempotency_keys k " "JOIN messages m ON m.id = k.message_id " "WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s") @@ -351,14 +350,13 @@ async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 50 # Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR # would push the cursor into a Filter and late pages would scan the whole prefix. -# attachment_ids: a correlated sub-select per returned row (at most limit + 1 = 101 lookups on the -# primary key of message_attachments), so the keyset range scan itself stays unchanged. -HISTORY_COLUMNS = ("SELECT m.id, m.sender_id, m.text, m.created_at, " - "ARRAY(SELECT ma.attachment_id FROM message_attachments ma " - "WHERE ma.message_id = m.id ORDER BY ma.position) FROM messages m ") -HISTORY_FIRST = HISTORY_COLUMNS + "WHERE m.chat_id = %s ORDER BY m.created_at, m.id LIMIT %s" -HISTORY_AFTER = (HISTORY_COLUMNS + - "WHERE m.chat_id = %s AND (m.created_at, m.id) > (%s, %s) ORDER BY m.created_at, m.id LIMIT %s") +# attachment_ids is a column of the message row (migration 0004): still one index range scan. +HISTORY_FIRST = ( + "SELECT id, sender_id, text, created_at, attachment_ids FROM messages " + "WHERE chat_id = %s ORDER BY created_at, id LIMIT %s") +HISTORY_AFTER = ( + "SELECT id, sender_id, text, created_at, attachment_ids FROM messages " + "WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s") async def list_messages(db: Database, chat_id: str, after: SortKey | None, limit: int) -> tuple[list[Message], bool]: diff --git a/src/messenger/storage.py b/src/messenger/storage.py index bf922d1..f9d42d1 100644 --- a/src/messenger/storage.py +++ b/src/messenger/storage.py @@ -44,6 +44,7 @@ class ObjectHead: size: int content_type: str metadata: dict[str, str] + last_modified_ts: float # POSIX seconds @dataclass(frozen=True, slots=True) @@ -116,7 +117,7 @@ class Storage: async def head(self, key: str) -> ObjectHead: response = await self._call("head", self.client.head_object(Bucket=self.bucket, Key=key)) return ObjectHead(int(response["ContentLength"]), response.get("ContentType", ""), - dict(response.get("Metadata", {}))) + dict(response.get("Metadata", {})), response["LastModified"].timestamp()) async def open_stream(self, key: str) -> tuple[int, AsyncIterator[bytes]]: """Start a GET: (Content-Length, async iterator of chunks). The headers arrive under the diff --git a/student-tests/conftest.py b/student-tests/conftest.py index 68b9e15..e1efdbb 100644 --- a/student-tests/conftest.py +++ b/student-tests/conftest.py @@ -21,6 +21,7 @@ import sys import time import uuid from pathlib import Path +from urllib.parse import quote import httpx2 as httpx import pytest @@ -53,9 +54,9 @@ def basic(username: str, password: str) -> str: # ------------------------------------------------------------------ docker / psql -def compose(*args: str, check: bool = True, timeout: float = 120) -> subprocess.CompletedProcess: +def compose(*args: str, check: bool = True, timeout: float = 120, stdin: str | None = None) -> subprocess.CompletedProcess: return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True, - encoding="utf-8", check=check, timeout=timeout) + encoding="utf-8", check=check, timeout=timeout, input=stdin) def psql(sql: str) -> str: @@ -234,6 +235,24 @@ class Api: def send(self, actor: dict, chat: dict, text: str, key: str | None = None) -> dict: return self.post_message(actor, chat, {"text": text}, key=key).json() + def upload(self, actor: dict, chat: dict, data, expected=201, *, content_type: str | None = "image/png", + filename: str | None = "photo.png", query: str | None = None, headers: dict | None = None): + """POST raw bytes to /chats/{id}/attachments. `query` replaces the whole query string + (to send hand-made percent-encoding); `data` may be bytes or an iterator (chunked).""" + h = dict(headers or {}) + if content_type is not None: + h["Content-Type"] = content_type + if query is None: + query = "" if filename is None else "filename=" + quote(filename, safe="") + path = f"/api/v1/chats/{chat['id']}/attachments" + (f"?{query}" if query else "") + if isinstance(data, (bytes, bytearray)): + return self.call("POST", path, expected, token=actor["token"], raw=bytes(data), headers=h, + schema="Attachment" if expected == 201 else None) + h["Authorization"] = "Bearer " + actor["token"] + response = self.client.post(path, content=data, headers=h) + assert response.status_code == expected, (response.status_code, response.text[:300]) + return response + def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]: """Walk all pages; returns (items, number_of_pages).""" items, pages, cursor = [], 0, None @@ -281,6 +300,76 @@ def run_hook(name: str, timeout: float = 180, **env: str) -> subprocess.Complete timeout=timeout, env={**os.environ, **env}) +def s3_exec(body: str, service: str = "api", stdin: str | None = None) -> str: + """Run `body` (async Python, `s` = messenger.storage.Storage) INSIDE a container that holds S3 + credentials: `api` (Read+Write, the API's own key pair) or `reconciler` (+List). The S3 port is + not published on the host; this is how the tests inject faults directly into the bucket.""" + script = ("import asyncio, base64, json, sys\n" + "from messenger.config import Settings, s3_settings\n" + "from messenger.storage import Storage, ObjectMissing\n" + "async def main():\n" + " st = Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings())\n" + " s = Storage(st)\n" + " await s.open()\n" + " try:\n" + "".join(" " + line + "\n" for line in body.splitlines()) + + " finally:\n" + " await s.close()\n" + "asyncio.run(main())\n") + if service == "api": + cmd = ["exec", "-T", "api", "python", "-c", script] + else: + cmd = ["run", "--rm", "--no-deps", "-T", "--entrypoint", "python", service, "-c", script] + return compose(*cmd, timeout=120, stdin=stdin).stdout.strip() + + +def s3_put(key: str, data: bytes) -> None: + """Write raw bytes under `key` (base64 over stdin: Windows limits command lines to 32 KiB).""" + s3_exec(f"await s.put({key!r}, base64.b64decode(sys.stdin.read()), 'image/png', {{}})", + stdin=base64.b64encode(data).decode()) + + +def s3_delete(key: str) -> None: + s3_exec(f"await s.delete({key!r})") + + +def s3_size(key: str) -> int | None: + out = s3_exec(f"try:\n print((await s.head({key!r})).size)\nexcept ObjectMissing:\n print(-1)") + size = int(out.splitlines()[-1]) + return None if size < 0 else size + + +def s3_keys() -> set[str]: + """Every object key under o/ (needs List: runs with the reconciler's key pair).""" + out = s3_exec("keys, after = [], None\n" + "while True:\n" + " page, more = await s.list_page('o/', after, 1000)\n" + " keys += [o.key for o in page]\n" + " if not more or not page:\n" + " break\n" + " after = page[-1].key\n" + "print(json.dumps(keys))", service="reconciler") + return set(json.loads(out.splitlines()[-1])) + + +def reconcile(*args: str, expect_code: int = 0) -> dict: + """Run the reconciler CLI (compose service, profile tools) and return its JSON report.""" + # --no-deps: never let `compose run` start a store that a test has stopped on purpose. + result = compose("run", "--rm", "--no-deps", "-T", "reconciler", "--samples", "100000", *args, check=False, + timeout=300) + assert result.returncode == expect_code, (result.returncode, result.stderr[-2000:]) + return json.loads(result.stdout) + + +def findings_by_key(report: dict) -> dict[str, dict]: + return {item["key"]: dict(item, kind=kind) for kind, items in report["samples"].items() for item in items} + + +def attachment_row(attachment_id: str) -> dict | None: + out = psql("SELECT json_build_object('status', status, 'object_key', object_key, 'size', size_bytes, " + "'lost_reason', lost_reason) FROM attachments WHERE id = '%s'" % attachment_id) + return json.loads(out) if out else None + + def probe_inside(replica: dict, path: str) -> int: """HTTP status of `path` asked inside the replica container itself (bypassing the LB).""" code = ("import urllib.request as u, urllib.error as e\n" diff --git a/student-tests/imagegen.py b/student-tests/imagegen.py new file mode 100644 index 0000000..0026fea --- /dev/null +++ b/student-tests/imagegen.py @@ -0,0 +1,93 @@ +"""Synthetic images for the lab 4 tests: valid PNG/JPEG of any size, and the broken files the +reviewer used against other solutions (truncated, corrupted IDAT with a valid CRC, JPEG without +scan data, decompression-bomb headers, MIME spoofing). Nothing big is stored in Git: every file +is generated in memory.""" +from __future__ import annotations + +import io +import os +import struct +import zlib + +from PIL import Image + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" + + +def chunk(ctype: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + ctype + data + struct.pack(">I", zlib.crc32(ctype + data)) + + +def ihdr(width: int, height: int, depth: int = 8, color: int = 2, interlace: int = 0) -> bytes: + return chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace)) + + +def png(width: int = 64, height: int = 48, mode: str = "RGB", noise: bool = False, **save) -> bytes: + if noise: + channels = len(mode) + image = Image.frombytes(mode, (width, height), os.urandom(width * height * channels)) + else: + image = Image.new(mode, (width, height), (40, 120, 200) if mode == "RGB" else 128) + out = io.BytesIO() + image.save(out, "PNG", **save) + return out.getvalue() + + +def png_of_size(target: int) -> bytes: + """A valid PNG of (about, at most) `target` bytes: random pixels do not compress.""" + side = int((target / 3) ** 0.5) + data = png(side, side, noise=True, compress_level=1) + while len(data) > target: + side -= 8 + data = png(side, side, noise=True, compress_level=1) + return data + + +def png_exactly(size: int) -> bytes: + """A valid PNG of EXACTLY `size` bytes: noise image + an ancillary padding chunk.""" + base = png_of_size(size - 64) + pad = size - len(base) - 12 + assert pad >= 0 + return base[:-12] + chunk(b"tEXt", b"pad\x00" + b"x" * (pad - 4)) + base[-12:] + + +def jpeg(width: int = 64, height: int = 48, progressive: bool = False, noise: bool = False, quality: int = 85) -> bytes: + if noise: + image = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)) + else: + image = Image.new("RGB", (width, height), (200, 30, 30)) + out = io.BytesIO() + image.save(out, "JPEG", progressive=progressive, quality=quality) + return out.getvalue() + + +# ------------------------------------------------------------------ broken files +def png_corrupt_idat() -> bytes: + """Valid signature, IHDR and CRCs; the IDAT payload is 86 bytes of garbage (reviewer's case).""" + return PNG_SIGNATURE + ihdr(20, 20) + chunk(b"IDAT", bytes(range(86))) + chunk(b"IEND", b"") + + +def png_short_pixels() -> bytes: + """A complete, CRC-correct PNG whose zlib stream holds fewer rows than the header promises.""" + raw = b"".join(b"\x00" + bytes(60) for _ in range(19)) + return PNG_SIGNATURE + ihdr(20, 20) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"") + + +def png_bomb_header(width: int = 20000, height: int = 10000) -> bytes: + """Header of a huge picture: must be refused (413) from the first 33 bytes.""" + raw_row = b"\x00" + bytes(width * 3) + body = zlib.compress(raw_row * 4, 9) # a few rows only: the decoder must never get that far + return PNG_SIGNATURE + ihdr(width, height) + chunk(b"IDAT", body) + chunk(b"IEND", b"") + + +def jpeg_without_scan_data() -> bytes: + data = jpeg(128, 96) + sos = data.index(b"\xff\xda") + length = struct.unpack(">H", data[sos + 2:sos + 4])[0] + return data[:sos + 2 + length] + b"\xff\xd9" + + +def jpeg_bomb_header() -> bytes: + """SOF0 announcing 20000 x 20000 = 400 Mpx.""" + return (b"\xff\xd8\xff\xc0\x00\x11\x08" + struct.pack(">HH", 20000, 20000) + + b"\x03\x01\x22\x00\x02\x11\x01\x03\x11\x01" + b"\xff\xd9") diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..8aaa015 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,268 @@ +"""Attachments through the load balancer (lab 4, levels 3 and 4): upload/download, ACL, limits, +content checks, filename rules and messages with attachment_ids. + +Negative inputs include every case the reviewer used against other lab 4 solutions: corrupted +IDAT with valid CRCs, the first 33 bytes of the fixture, JPEG without scan data, a 20000x10000 +header (DecompressionBomb), exactly 10 MiB through the LB, C1 control U+0085 and a whitespace +filename, text=null / missing text / attachment_ids=null in a message.""" +from __future__ import annotations + +import hashlib +import subprocess +import time +import uuid +from pathlib import Path + +import pytest + +import imagegen +from conftest import attachment_row, compose, s3_size + +pytestmark = pytest.mark.stack + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() +MiB = 1024 * 1024 + + +@pytest.fixture() +def trio(api): + """alice + bob share a chat; eve is an outsider with her own chat.""" + alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve") + return alice, bob, eve, api.chat(alice, bob), api.chat(eve) + + +def content(api, actor, attachment_id, expected=200, query=""): + return api.call("GET", f"/api/v1/attachments/{attachment_id}/content{query}", expected, token=actor["token"]) + + +# ------------------------------------------------------------------ positive path +def test_png_and_jpeg_roundtrip_byte_for_byte(api, trio): + alice, bob, _, chat, _ = trio + jpg = imagegen.jpeg(321, 123, noise=True) + for data, ctype in ((SAMPLE, "image/png"), (jpg, "image/jpeg")): + att = api.upload(alice, chat, data, content_type=ctype, filename="кот 😺.bin").json() + assert att["status"] == "ready" and att["variants"] == [] and att["error_code"] is None + assert att["chat_id"] == chat["id"] and att["owner_id"] == alice["id"] + assert att["size_bytes"] == len(data) and att["sha256"] == hashlib.sha256(data).hexdigest() + assert att["content_type"] == ctype and att["filename"] == "кот 😺.bin" + meta = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"], schema="Attachment").json() + assert meta == att + got = content(api, bob, att["id"]) + assert got.content == data and got.headers["content-type"] == ctype + assert got.headers["content-length"] == str(len(data)) + assert got.headers["x-content-type-options"] == "nosniff" + assert got.headers["content-disposition"].startswith("attachment;") + assert "location" not in got.headers # bytes through the API, never a redirect to S3 + + +def test_metadata_reveals_no_storage_details(api, trio): + alice, _, _, chat, _ = trio + att = api.upload(alice, chat, SAMPLE).json() + row = attachment_row(att["id"]) + assert row["status"] == "ready" and row["object_key"].startswith("o/") + text = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=alice["token"]).text + assert row["object_key"] not in text and "messenger-attachments" not in text and "s3" not in text.lower() + # The key is random: neither the id nor the filename is part of it. + assert att["id"].replace("-", "") not in row["object_key"] and "photo" not in row["object_key"] + assert s3_size(row["object_key"]) == len(SAMPLE) # the bytes ARE in the bucket + + +def test_original_is_not_stored_on_api_containers(api, trio): + """The API keeps no file of the original: no recently written file of that size anywhere in + the replica's filesystem (the bytes live only in the bucket).""" + alice, _, _, chat, _ = trio + data = imagegen.png_of_size(300_000) + api.upload(alice, chat, data) + size = len(data) + for cid in compose("ps", "-q", "api").stdout.split(): + out = subprocess.run( + ["docker", "exec", cid, "sh", "-c", f"find / -xdev -type f -size {size}c -mmin -10 2>/dev/null | head"], + capture_output=True, text=True, timeout=60).stdout.strip() + assert out == "", out + + +# ------------------------------------------------------------------ access control +def test_outsider_and_unknown_get_404(api, trio): + alice, _, eve, chat, _ = trio + att = api.upload(alice, chat, SAMPLE).json() + for suffix in ("", "/content"): + api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"]) + api.call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"]) + api.call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"]) + api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401) + # Upload into a foreign chat: 404 before anything about the body is looked at. + api.upload(eve, chat, SAMPLE, 404) + api.upload(eve, chat, b"garbage", 404, content_type="text/plain") + + +def test_attachment_ids_are_checked_per_chat(api, trio): + alice, bob, eve, chat, eve_chat = trio + other = api.chat(alice) # alice's second chat + att_other = api.upload(alice, other, SAMPLE).json() + att_eve = api.upload(eve, eve_chat, SAMPLE).json() + # Alice is a member of both chats, but a file is bound to its chat. + api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_other["id"]]}, 404) + api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404) + api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404) + # Eve cannot post into alice's chat even with her own file: membership first. + api.post_message(eve, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404) + # Any member may reference an attachment of the chat. + att = api.upload(alice, chat, SAMPLE).json() + msg = api.post_message(bob, chat, {"text": "", "attachment_ids": [att["id"]]}).json() + assert msg["attachment_ids"] == [att["id"]] and msg["sender_id"] == bob["id"] + + +# ------------------------------------------------------------------ messages with attachments +def test_message_with_attachments_and_history(api, trio): + alice, bob, _, chat, _ = trio + a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2)) + m1 = api.post_message(alice, chat, {"text": "", "attachment_ids": [a2, a1]}).json() + m2 = api.post_message(alice, chat, {"text": " ", "attachment_ids": [a1]}).json() # blank text + file: ok + m3 = api.post_message(alice, chat, {"text": "без файлов"}).json() + assert m1["attachment_ids"] == [a2, a1] and m1["text"] == "" + assert m2["text"] == " " and m3["attachment_ids"] == [] + items, _ = api.history(bob, chat, 2) + assert [m["attachment_ids"] for m in items] == [[a2, a1], [a1], []] + + +def test_idempotent_replay_with_attachments(api, trio): + alice, _, _, chat, _ = trio + a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2)) + key = uuid.uuid4().hex + first = api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a1, a2]}, key=key).json() + raw = ('{"attachment_ids":["%s","%s"],"text":"файлы"}' % (a1, a2)).encode() + assert api.post_message(alice, chat, None, raw=raw, key=key).json() == first # key order irrelevant + api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a2, a1]}, 409, key=key) # other body + api.post_message(alice, chat, {"text": "файлы"}, 409, key=key) + # A failed attachment check does not burn the key: the same key works with a valid body later. + key2 = uuid.uuid4().hex + api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404, key=key2) + api.post_message(alice, chat, {"text": "x", "attachment_ids": [a1]}, key=key2) + + +@pytest.mark.parametrize("raw", [ + b'{"attachment_ids":["%(a)s"]}', # text missing + b'{"text":null,"attachment_ids":["%(a)s"]}', # text null + b'{"text":1,"attachment_ids":["%(a)s"]}', + b'{"text":"x","attachment_ids":null}', # attachment_ids null + b'{"text":"x","attachment_ids":"%(a)s"}', + b'{"text":"x","attachment_ids":{}}', + b'{"text":"x","attachment_ids":["%(a)s","%(a)s"]}', # duplicates + b'{"text":"x","attachment_ids":["%(a)s",1]}', + b'{"text":"x","attachment_ids":["not-a-uuid"]}', + b'{"text":"","attachment_ids":[]}', # neither text nor files + b'{"text":" "}', + b'{"text":"x","attachment_ids":[],"sender_id":"%(a)s"}', +], ids=lambda raw: raw.decode()[:40]) +def test_message_body_validation(api, trio, raw): + alice, _, _, chat, _ = trio + a = api.upload(alice, chat, SAMPLE).json()["id"] + api.post_message(alice, chat, None, 400, raw=raw % {b"a": a.encode()}) + + +def test_at_most_ten_attachments(api, trio): + alice, _, _, chat, _ = trio + ids = [api.upload(alice, chat, imagegen.png(4, 4)).json()["id"] for _ in range(11)] + assert len(api.post_message(alice, chat, {"text": "", "attachment_ids": ids[:10]}).json()["attachment_ids"]) == 10 + api.post_message(alice, chat, {"text": "", "attachment_ids": ids}, 400) + + +# ------------------------------------------------------------------ limits +def test_size_limit_through_the_load_balancer(api, trio): + alice, bob, _, chat, _ = trio + exact = imagegen.png_exactly(10 * MiB) + assert len(exact) == 10 * MiB + att = api.upload(alice, chat, exact).json() # exactly the limit: accepted via LB + assert content(api, bob, att["id"]).content == exact + started = time.monotonic() + api.upload(alice, chat, exact + b"\x00", 413) # Content-Length over the limit + assert time.monotonic() - started < 5 + over = exact + bytes(1024) + + def chunks(): # no Content-Length: chunked body + for i in range(0, len(over), 256 * 1024): + yield over[i:i + 256 * 1024] + error = api.upload(alice, chat, chunks(), 413).json() + assert error["error"]["code"] == "payload_too_large" + + +def test_pixel_limit_before_decoding(api, trio): + alice, _, _, chat, _ = trio + for data, ctype in ((imagegen.png_bomb_header(), "image/png"), (imagegen.jpeg_bomb_header(), "image/jpeg"), + (imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), "image/png")): + started = time.monotonic() + assert api.upload(alice, chat, data, 413, content_type=ctype).json()["error"]["code"] == "payload_too_large" + assert time.monotonic() - started < 2 # refused from the header, nothing decoded + + +def test_exactly_40_megapixels_is_accepted(api, trio): + alice, _, _, chat, _ = trio + data = imagegen.png(8000, 5000, compress_level=9) # 40 000 000 px, ~120 KB file + att = api.upload(alice, chat, data).json() + assert att["size_bytes"] == len(data) + + +# ------------------------------------------------------------------ content checks +BROKEN = { + "png: first 33 bytes of the fixture": (SAMPLE[:33], "image/png"), + "png: truncated": (SAMPLE[:-20], "image/png"), + "png: corrupt IDAT, valid CRC": (imagegen.png_corrupt_idat(), "image/png"), + "png: fewer rows than IHDR": (imagegen.png_short_pixels(), "image/png"), + "png: flipped byte": (SAMPLE[:700] + bytes([SAMPLE[700] ^ 0x5A]) + SAMPLE[701:], "image/png"), + "png: bytes after IEND": (SAMPLE + b"", "image/png"), + "jpeg: no scan data after SOS": (imagegen.jpeg_without_scan_data(), "image/jpeg"), + "jpeg: truncated": (imagegen.jpeg(400, 300, noise=True)[:5000], "image/jpeg"), + "mime: png bytes declared jpeg": (SAMPLE, "image/jpeg"), + "mime: jpeg bytes declared png": (imagegen.jpeg(), "image/png"), + "mime: gif declared png": (b"GIF89a" + bytes(100), "image/png"), + "mime: html declared png": (b"", "image/png"), + "type: text/plain": (SAMPLE, "text/plain"), + "type: octet-stream": (SAMPLE, "application/octet-stream"), + "type: multipart": (SAMPLE, "multipart/form-data; boundary=x"), + "type: missing": (SAMPLE, None), +} + + +@pytest.mark.parametrize("name", list(BROKEN)) +def test_bad_content_is_415(api, trio, name): + alice, _, _, chat, _ = trio + data, ctype = BROKEN[name] + error = api.upload(alice, chat, data, 415, content_type=ctype).json() + assert error["error"]["code"] == "unsupported_media_type" + + +def test_empty_body_is_400(api, trio): + alice, _, _, chat, _ = trio + api.upload(alice, chat, b"", 400) + + +# ------------------------------------------------------------------ filename +@pytest.mark.parametrize("query", [ + "", "filename=", "filename=%20%20%20", "filename=.", "filename=..", "filename=a%2Fb.png", "filename=a%5Cb.png", + "filename=a%00b.png", "filename=c1%C2%85.png", "filename=tab%09.png", "filename=photo%E2%80%AEgnp.exe", + "filename=%FF%FE.png", "filename=a.png&filename=b.png", "filename=" + "a" * 256, +], ids=lambda q: q[:40] or "missing") +def test_bad_filename_is_400(api, trio, query): + alice, _, _, chat, _ = trio + api.upload(alice, chat, SAMPLE, 400, query=query) + + +def test_unicode_filename_is_kept_and_safely_exposed(api, trio): + alice, _, _, chat, _ = trio + name = 'отчёт "Q3"; ok.png' + att = api.upload(alice, chat, SAMPLE, filename=name).json() + assert att["filename"] == name + disposition = content(api, alice, att["id"]).headers["content-disposition"] + assert '"' not in disposition.split('filename="')[1].split('";')[0] # quotes neutralised in the fallback + assert "filename*=UTF-8''" in disposition + + +# ------------------------------------------------------------------ variants +def test_variant_parameter(api, trio): + alice, _, eve, chat, _ = trio + att = api.upload(alice, chat, SAMPLE).json() + assert content(api, alice, att["id"], query="?variant=original").content == SAMPLE + content(api, alice, att["id"], 404, query="?variant=thumbnail") # schema name, not produced in lab 4 + content(api, alice, att["id"], 400, query="?variant=huge") + content(api, eve, att["id"], 404, query="?variant=thumbnail") # outsider: 404, nothing revealed diff --git a/student-tests/test_balancing.py b/student-tests/test_balancing.py index c3d14a1..01f3d6e 100644 --- a/student-tests/test_balancing.py +++ b/student-tests/test_balancing.py @@ -278,28 +278,37 @@ def test_scale_up_and_down_is_picked_up_without_errors(api): def test_compose_policy_limits_ports_secrets_images(): config = json.loads(compose("config", "--format", "json").stdout) services = config["services"] - assert set(services) == {"postgres", "valkey", "migrate", "api", "lb"} + # The reconciler (profile "tools") is started on demand and not part of the running stand. + assert set(services) == {"postgres", "valkey", "s3", "migrate", "s3-init", "api", "lb"} for name, svc in services.items(): limits = svc.get("deploy", {}).get("resources", {}).get("limits", {}) assert limits.get("cpus") and limits.get("memory"), f"{name}: CPU/RAM limits" assert "container_name" not in svc, f"{name}: container_name prevents scaling" if name in ("api", "migrate"): assert "build" in svc # built from this repository's source + elif name == "s3-init": + assert svc["image"] == services["api"]["image"] # the same locally built image else: assert "@sha256:" in svc["image"], f"{name}: image must be pinned by digest" assert services["api"]["deploy"]["replicas"] >= 2 and not services["api"].get("ports") published = {name: svc.get("ports", []) for name, svc in services.items() if svc.get("ports")} assert set(published) == {"lb"} and all(p["host_ip"] in ("127.0.0.1", "::1") for p in published["lb"]) secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in services.items()} - assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"], - "migrate": ["database_url"], "api": ["cache_url", "database_url"], "lb": []} + s3_keys = ['s3_admin_access_key_id', 's3_admin_secret_access_key', 's3_api_access_key_id', 's3_api_secret_access_key', 's3_reconciler_access_key_id', 's3_reconciler_secret_access_key'] + assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"], "s3": s3_keys, + "migrate": ["database_url"], "s3-init": s3_keys, "lb": [], + # the API: its own S3 key pair only (no admin, no reconciler keys) + "api": ["cache_url", "database_url", "s3_api_access_key_id", "s3_api_secret_access_key"]} volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"] for name, svc in services.items()} - assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "migrate": [], "api": [], "lb": []} + assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "s3": ["s3data"], "migrate": [], + "s3-init": [], "api": [], "lb": []} networks = {name: sorted(svc.get("networks", {})) for name, svc in services.items()} - assert networks == {"postgres": ["data"], "valkey": ["data"], "migrate": ["data"], "api": ["data", "edge"], + assert networks == {"postgres": ["data"], "valkey": ["data"], "s3": ["storage"], "migrate": ["data"], + "s3-init": ["storage"], "api": ["data", "edge", "storage"], "lb": ["edge"]}, networks # the LB cannot reach the stores at all assert config["networks"]["data"].get("internal") is True + assert config["networks"]["storage"].get("internal") is True def test_running_containers_have_limits_and_no_bypass_ports(): @@ -317,7 +326,7 @@ def test_running_containers_have_limits_and_no_bypass_ports(): def test_migrations_ran_once_by_the_migrate_service(): applied = psql("SELECT version FROM schema_migrations ORDER BY version").split() - assert applied == ["0001_init", "0002_idempotency_keys", "0003_post_message_function"] + assert applied == ["0001_init", "0002_idempotency_keys", "0003_post_message_function", "0004_attachments"] migrate = compose("ps", "-a", "--format", "json", "migrate").stdout rows = [json.loads(line) for line in migrate.splitlines() if line.strip()] assert rows and all(r["State"] == "exited" and r["ExitCode"] == 0 for r in rows) diff --git a/student-tests/test_consistency.py b/student-tests/test_consistency.py new file mode 100644 index 0000000..9e375a9 --- /dev/null +++ b/student-tests/test_consistency.py @@ -0,0 +1,111 @@ +"""Consistency of PostgreSQL and S3 around one upload (lab 4, level 4). + +The two-store write is: intent row 'pending' -> S3 PUT -> row 'ready' -> 201. Failures between +the steps are injected with FAULT_POINTS (compose variable, empty in normal runs; the API +raises 503 at that exact point, as if the process or the network failed there): + +* after_s3_put object written, row still pending, client got 503 -> the janitor removes + the object and the row after PENDING_GRACE_SECONDS (here 3 s); +* after_ready_commit row ready and object written, but the answer is lost (client got 503) + -> nothing may be deleted: the attachment stays intact and downloadable. + +Plus: an upload interrupted by the client and a stalled body leave nothing behind.""" +from __future__ import annotations + +import socket +import time +from pathlib import Path + +import pytest + +import imagegen +from conftest import api_with_env, attachment_row, psql, s3_size + +pytestmark = pytest.mark.stack + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +def rows_of_chat(chat_id: str) -> list[tuple[str, str, str]]: + out = psql(f"SELECT id || ' ' || status || ' ' || object_key FROM attachments WHERE chat_id = '{chat_id}'") + return [tuple(line.split()) for line in out.splitlines() if line] + + +def wait_for(condition, timeout: float, what: str) -> None: + deadline = time.monotonic() + timeout + while not condition(): + assert time.monotonic() < deadline, what + time.sleep(0.5) + + +def raw_upload(token: str, chat_id: str, declared: int, body: bytes, *, wait_answer: float | None) -> bytes: + """Hand-made HTTP request: announce `declared` bytes, send only `body`, then either close the + socket (client gone) or keep it open and read the answer after `wait_answer` seconds.""" + sock = socket.create_connection(("127.0.0.1", 8080), timeout=30) + head = (f"POST /api/v1/chats/{chat_id}/attachments?filename=x.png HTTP/1.1\r\nHost: localhost\r\n" + f"Authorization: Bearer {token}\r\nContent-Type: image/png\r\nContent-Length: {declared}\r\n\r\n") + sock.sendall(head.encode() + body) + if wait_answer is None: + sock.close() + return b"" + time.sleep(wait_answer) + answer = sock.recv(65536) + sock.close() + return answer + + +def test_interrupted_and_stalled_uploads_leave_nothing(api): + alice = api.user("alice") + chat = api.chat(alice) + data = imagegen.png_of_size(2_000_000) + raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=None) # client disconnects + with api_with_env(UPLOAD_READ_TIMEOUT_SECONDS="2"): + answer = raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=5) # client stalls + assert answer.startswith(b"HTTP/1.1 400") and b"invalid_request" in answer, answer[:200] + assert b"x-instance-id" in answer.lower() + time.sleep(1) + assert rows_of_chat(chat["id"]) == [] # nothing was written: the body is checked before any write + + +def test_crash_between_s3_put_and_commit_is_cleaned_by_janitor(api): + alice = api.user("alice") + chat = api.chat(alice) + with api_with_env(FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"): + assert api.upload(alice, chat, SAMPLE, 503).json()["error"]["code"] == "unavailable" + [(attachment_id, status, key)] = rows_of_chat(chat["id"]) + assert status == "pending" and s3_size(key) == len(SAMPLE) # the orphan-to-be exists + api.call("GET", f"/api/v1/attachments/{attachment_id}", 404, token=alice["token"]) # invisible + api.post_message(alice, chat, {"text": "", "attachment_ids": [attachment_id]}, 404) # not attachable + wait_for(lambda: rows_of_chat(chat["id"]) == [], 30, "janitor did not remove the stale intent") + assert s3_size(key) is None # object removed first + # Normal configuration again: an upload in the same chat works and is not touched. + att = api.upload(alice, chat, SAMPLE).json() + assert attachment_row(att["id"])["status"] == "ready" + + +def test_lost_answer_after_commit_never_deletes_the_object(api): + """The reviewer found this in another PR: COMMIT succeeded, the answer was lost, the API + 'compensated' by deleting the object -> ready row without bytes. Here nothing compensates + in the request path, and the janitor only touches 'pending' rows.""" + alice, bob = api.user("alice"), api.user("bob") + chat = api.chat(alice, bob) + with api_with_env(FAULT_POINTS="after_ready_commit", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"): + api.upload(alice, chat, SAMPLE, 503) + [(attachment_id, status, key)] = rows_of_chat(chat["id"]) + assert status == "ready" + time.sleep(6) # several janitor rounds + assert rows_of_chat(chat["id"]) == [(attachment_id, "ready", key)] and s3_size(key) == len(SAMPLE) + got = api.call("GET", f"/api/v1/attachments/{attachment_id}/content", 200, token=bob["token"]) + assert got.content == SAMPLE + + +def test_fresh_pending_upload_is_not_touched_by_janitor(api): + """An intent younger than PENDING_GRACE_SECONDS is an upload in progress: never removed.""" + alice = api.user("alice") + chat = api.chat(alice) + with api_with_env(FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="600", JANITOR_INTERVAL_SECONDS="1"): + api.upload(alice, chat, SAMPLE, 503) + time.sleep(4) + [(_, status, key)] = rows_of_chat(chat["id"]) + assert status == "pending" and s3_size(key) == len(SAMPLE) diff --git a/student-tests/test_db.py b/student-tests/test_db.py index 5fb4a26..544d378 100644 --- a/student-tests/test_db.py +++ b/student-tests/test_db.py @@ -78,7 +78,8 @@ def test_migrations_rerun_is_a_noop(): assert "0001_init: already applied, skipped" in result.stdout assert "0002_idempotency_keys: already applied, skipped" in result.stdout assert "0003_post_message_function: already applied, skipped" in result.stdout - assert psql("SELECT count(*) FROM schema_migrations") == "3" + assert "0004_attachments: already applied, skipped" in result.stdout + assert psql("SELECT count(*) FROM schema_migrations") == "4" assert psql("SELECT count(*) FROM users") == users_before diff --git a/student-tests/test_reconcile.py b/student-tests/test_reconcile.py new file mode 100644 index 0000000..a0a2de8 --- /dev/null +++ b/student-tests/test_reconcile.py @@ -0,0 +1,209 @@ +"""Track 2: the reconciler of S3 and PostgreSQL (python -m messenger.reconcile). + +Faults are injected on BOTH sides, directly into the stores (S3 with the API/reconciler key +pairs from inside the storage network, PostgreSQL with psql): + + S3 side: orphan object (no row), object deleted under a ready row, object overwritten with + other bytes (size / same-size checksum), object of a quarantined row restored; + DB side: stale intent with and without object, quarantine mark on a healthy row; + fresh: orphan and intent younger than --grace (an upload in progress). + +The tests prove: dry-run changes nothing in either store; --apply fixes exactly the old findings; +fresh and used attachments are never deleted (ready rows only ever get a reversible quarantine); +the second run finds nothing to do; the safety valve and store outages stop the run without +damage. A small --grace (8-30 s, --allow-small-grace) keeps the tests fast.""" +from __future__ import annotations + +import concurrent.futures +import hashlib +import time +import uuid +from pathlib import Path + +import pytest + +import imagegen +from conftest import (attachment_row, findings_by_key, psql, reconcile, s3_delete, s3_keys, s3_put, s3_size, + service_down) + +pytestmark = pytest.mark.stack + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() +GRACE = ["--grace", "8", "--allow-small-grace"] +# The main scenario needs "fresh" items to stay younger than --grace for the whole dry-run + snapshot +# (each docker exec costs 1-3 s on Windows), hence a longer grace there. +LONG_GRACE = ["--grace", "30", "--allow-small-grace"] + + +def new_key() -> str: + return "o/" + uuid.uuid4().hex + + +def insert_intent(chat: dict, owner: dict, key: str, data: bytes, age_seconds: int) -> str: + """A 'pending' row as an upload leaves it after the intent COMMIT (crash afterwards).""" + attachment_id = str(uuid.uuid4()) + psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, " + "object_key, status, created_at, updated_at) VALUES ('%s', '%s', '%s', 'x.png', 'image/png', %d, " + "decode('%s', 'hex'), 320, 200, '%s', 'pending', clock_timestamp() - make_interval(secs => %d), " + "clock_timestamp() - make_interval(secs => %d))" + % (attachment_id, chat["id"], owner["id"], len(data), hashlib.sha256(data).hexdigest(), key, age_seconds, + age_seconds)) + return attachment_id + + +def snapshot() -> tuple[str, frozenset]: + """Fingerprint of everything the reconciler could change: every attachment row + the key list.""" + rows = psql("SELECT count(*) || ':' || coalesce(md5(string_agg(id::text || status || object_key || " + "coalesce(lost_reason, '-') || updated_at::text, ',' ORDER BY id)), '') FROM attachments") + return rows, frozenset(s3_keys()) + + +def content_status(api, actor, attachment_id) -> int: + return api.client.get(f"/api/v1/attachments/{attachment_id}/content", + headers={"Authorization": "Bearer " + actor["token"]}).status_code + + +def test_detect_and_safely_fix_faults_on_both_sides(api): + alice, bob = api.user("alice"), api.user("bob") + chat = api.chat(alice, bob) + up = {name: api.upload(alice, chat, imagegen.png(30 + i, 20)).json() for i, name in enumerate("ABCDE")} + data = {name: api.call("GET", f"/api/v1/attachments/{a['id']}/content", 200, token=alice["token"]).content + for name, a in up.items()} + key = {name: attachment_row(a["id"])["object_key"] for name, a in up.items()} + api.post_message(alice, chat, {"text": "used", "attachment_ids": [up["A"]["id"], up["C"]["id"], up["E"]["id"]]}) + + # ---- inject (old) + orphan_old = new_key() + s3_put(orphan_old, SAMPLE) + stale_with, stale_without = new_key(), new_key() + s3_put(stale_with, SAMPLE) + stale_with_id = insert_intent(chat, alice, stale_with, SAMPLE, 600) # < janitor grace 900 s + stale_without_id = insert_intent(chat, alice, stale_without, SAMPLE, 600) + s3_delete(key["C"]) # ready + used, bytes lost + s3_put(key["D"], data["D"][:len(data["D"]) // 2]) # ready, unused, wrong size + psql(f"UPDATE attachments SET lost_at = clock_timestamp(), lost_reason = 'missing_object' " + f"WHERE id = '{up['E']['id']}'") # quarantined, but bytes are fine + time.sleep(32) # everything above is now older than --grace + # ---- inject (fresh: an upload in progress) + fresh_orphan, fresh_intent = new_key(), new_key() + s3_put(fresh_orphan, SAMPLE) + s3_put(fresh_intent, SAMPLE) + fresh_intent_id = insert_intent(chat, alice, fresh_intent, SAMPLE, 0) + + # ---- dry-run: report everything, change nothing + before = snapshot() + dry = reconcile(*LONG_GRACE) + assert snapshot() == before, "dry-run modified a store" + f = findings_by_key(dry) + expected = {orphan_old: "orphan_object", fresh_orphan: "fresh_orphan_object", stale_with: "stale_intent", + stale_without: "stale_intent", fresh_intent: "fresh_intent", key["C"]: "missing_object", + key["D"]: "size_mismatch", key["E"]: "restored_object"} + assert {k: f[k]["kind"] for k in expected} == expected + assert key["A"] not in f and key["B"] not in f + assert f[key["C"]]["used"] is True and dry["mode"] == "dry-run" + assert all(f[k]["result"] in ("planned", "not_actionable") for k in expected) + + # ---- apply + applied = findings_by_key(reconcile("--apply", *LONG_GRACE)) + for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"]): + assert applied[k]["result"] == "applied", (k, applied[k]) + keys = s3_keys() + assert orphan_old not in keys and stale_with not in keys + assert attachment_row(stale_with_id) is None and attachment_row(stale_without_id) is None + # fresh: untouched + assert fresh_orphan in keys and fresh_intent in keys and attachment_row(fresh_intent_id)["status"] == "pending" + # used, lost bytes: quarantined, NOT deleted; the message still references it + row_c = attachment_row(up["C"]["id"]) + assert row_c["status"] == "ready" and row_c["lost_reason"] == "missing_object" + items, _ = api.history(bob, chat, 100) + assert any(up["C"]["id"] in m["attachment_ids"] for m in items) + assert content_status(api, bob, up["C"]["id"]) == 404 + api.post_message(alice, chat, {"text": "", "attachment_ids": [up["C"]["id"]]}, 409) # cannot attach a lost file + assert attachment_row(up["D"]["id"])["lost_reason"] == "size_mismatch" + assert attachment_row(up["E"]["id"])["lost_reason"] is None # quarantine lifted + for name in "ABE": + got = api.call("GET", f"/api/v1/attachments/{up[name]['id']}/content", 200, token=bob["token"]) + assert got.content == data[name] + + # ---- second run: idempotent, nothing left to do for these keys + again = findings_by_key(reconcile("--apply", *LONG_GRACE)) + for k in (orphan_old, stale_with, stale_without, key["C"], key["D"], key["E"], key["A"], key["B"]): + assert k not in again, (k, again.get(k)) + + # ---- the lost object comes back from a backup: the quarantine is lifted, the file is served again + s3_put(key["C"], data["C"]) + restored = findings_by_key(reconcile("--apply", *LONG_GRACE)) + assert restored[key["C"]]["kind"] == "restored_object" and restored[key["C"]]["result"] == "applied" + assert api.call("GET", f"/api/v1/attachments/{up['C']['id']}/content", 200, token=bob["token"]).content == data["C"] + # tidy up the fresh items of this test once they are old enough + time.sleep(31) + reconcile("--apply", *LONG_GRACE) + assert fresh_orphan not in s3_keys() and attachment_row(fresh_intent_id) is None + + +def test_checksum_mode_finds_same_size_corruption(api): + alice = api.user("alice") + chat = api.chat(alice) + att = api.upload(alice, chat, SAMPLE).json() + k = attachment_row(att["id"])["object_key"] + s3_put(k, SAMPLE[:1000] + bytes([SAMPLE[1000] ^ 1]) + SAMPLE[1001:]) # same size, other bytes + time.sleep(9) + assert k not in findings_by_key(reconcile(*GRACE)) # sizes match: listing cannot see it + report = reconcile("--verify-checksums", *GRACE) + assert findings_by_key(report)[k]["kind"] == "checksum_mismatch" + assert report["cost"]["s3_get_bytes"] >= len(SAMPLE) + assert findings_by_key(reconcile("--apply", "--verify-checksums", *GRACE))[k]["result"] == "applied" + assert attachment_row(att["id"])["lost_reason"] == "checksum_mismatch" + + +def test_safety_valve_refuses_mass_deletion(): + """If (almost) every object looked orphaned - e.g. the reconciler pointed at an empty database - + --apply must stop before deleting anything.""" + orphan = new_key() + s3_put(orphan, SAMPLE) + time.sleep(9) + report = reconcile("--apply", "--max-orphan-fraction", "0.000001", *GRACE, expect_code=3) + assert report["error"].startswith("safety_valve") and report["actions"]["applied"] == 0 + assert orphan in s3_keys() + reconcile("--apply", *GRACE) + assert orphan not in s3_keys() + + +@pytest.mark.parametrize("service", ["s3", "postgres"]) +def test_store_outage_stops_the_run_without_damage(api, service): + with service_down(service, "stop"): + report = reconcile(*GRACE, expect_code=1) + assert report["error"].startswith("store_unavailable") + assert reconcile(*GRACE)["error"] is None + + +def test_live_uploads_are_never_hurt_by_concurrent_apply(api): + """Uploads and messages run while --apply runs repeatedly: every upload that got 201 stays + ready and downloadable byte for byte, every message keeps its attachments.""" + alice = api.user("alice") + chat = api.chat(alice) + payloads = [imagegen.png(16 + i, 16) for i in range(40)] + done: list[tuple[dict, bytes]] = [] + + def worker(chunk: list[bytes]) -> None: + for p in chunk: + att = api.upload(alice, chat, p).json() + api.post_message(alice, chat, {"text": "", "attachment_ids": [att["id"]]}) + done.append((att, p)) + + with concurrent.futures.ThreadPoolExecutor(4) as pool: + futures = [pool.submit(worker, payloads[i::4]) for i in range(4)] + runs = 0 + while not all(x.done() for x in futures): + report = reconcile("--apply", *GRACE) + assert report["error"] is None + runs += 1 + for x in futures: + x.result() + assert runs >= 1 and len(done) == len(payloads) + for att, p in done: + row = attachment_row(att["id"]) + assert row["status"] == "ready" and row["lost_reason"] is None and s3_size(row["object_key"]) == len(p) + for att, p in done[::8]: + assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]).content == p diff --git a/student-tests/test_storage_failures.py b/student-tests/test_storage_failures.py new file mode 100644 index 0000000..412b210 --- /dev/null +++ b/student-tests/test_storage_failures.py @@ -0,0 +1,107 @@ +"""S3 outages, private bucket and failover of attachments (lab 4, levels 3 and 4). + +Every injected failure is undone in `finally` (conftest.service_down / replica_stopped), and the +stand is waited for until HAProxy sees all replicas UP again.""" +from __future__ import annotations + +import time +from pathlib import Path + +import pytest + +import imagegen +from conftest import attachment_row, compose, probe_inside, replica_stopped, running_replicas, service_down + +pytestmark = pytest.mark.stack + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +@pytest.fixture() +def pair(api): + alice, bob = api.user("alice"), api.user("bob") + return alice, bob, api.chat(alice, bob) + + +@pytest.mark.parametrize("how", ["stop", "pause"]) +def test_s3_outage_is_503_and_text_keeps_working(api, pair, how): + """S3 down (stopped: connection refused; paused: no answer at all): + * readiness is 503 on every replica (and through the LB), quickly; + * uploads/downloads are 503 `unavailable` within the S3 deadline, never 500; + * the LB's degraded mode keeps chats/text messages working (HAProxy backend api_core); + * after recovery the earlier file is intact.""" + alice, bob, chat = pair + att = api.upload(alice, chat, SAMPLE).json() + with service_down("s3", how): + deadline = time.monotonic() + 15 + while api.client.get("/health/ready").status_code != 503: + assert time.monotonic() < deadline, "readiness did not turn 503" + time.sleep(0.5) + for replica in running_replicas(): + assert probe_inside(replica, "/health/ready") == 503 + assert probe_inside(replica, "/health/ready/core") == 200 + started = time.monotonic() + error = api.upload(alice, chat, SAMPLE, 503).json() + assert error["error"]["code"] == "unavailable" + assert time.monotonic() - started < 8 + started = time.monotonic() + api.call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"]) + assert time.monotonic() - started < 8 + api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]) # metadata: PostgreSQL only + api.send(alice, chat, "текст работает без S3") # degraded mode + got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + assert got.content == SAMPLE + api.upload(alice, chat, SAMPLE) + + +def test_upload_survives_failover_of_the_serving_replica(api, pair): + """Upload -> stop the replica that served it -> another replica downloads the same bytes + (nothing of the file lives in the replica).""" + alice, bob, chat = pair + data = imagegen.png_of_size(500_000) + response = api.upload(alice, chat, data) + att, served_by = response.json(), response.headers["x-instance-id"] + replica = next(r for r in running_replicas() if r["hostname"] == served_by) + with replica_stopped(replica, "kill"): + for _ in range(5): + got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + assert got.headers["x-instance-id"] != served_by and got.content == data + + +def test_bucket_is_private_outside_the_api(api, pair): + """Inside the storage network (where the API is): an UNSIGNED GET of a real object and an + unsigned LIST are refused; the same GET signed with the API key pair works (control). + SeaweedFS admin endpoints (master 9333, filer 8888, volume 8080) are not reachable.""" + alice, _, chat = pair + att = api.upload(alice, chat, SAMPLE).json() + key = attachment_row(att["id"])["object_key"] + probe = ( + "import urllib.request as u, urllib.error as e, socket\n" + "def st(url):\n" + " try: return u.urlopen(url, timeout=3).status\n" + " except e.HTTPError as x: return x.code\n" + f"print(st('http://s3:8333/messenger-attachments/{key}'), st('http://s3:8333/messenger-attachments?list-type=2'))\n" + "for port in (9333, 8888, 8080):\n" + " s = socket.socket(); s.settimeout(2)\n" + " print(port, s.connect_ex(('s3', port)))\n") + out = compose("exec", "-T", "api", "python", "-c", probe).stdout.split("\n") + get_status, list_status = out[0].split() + assert get_status in ("401", "403") and list_status in ("401", "403"), out[0] + for line in out[1:4]: + port, code = line.split() + assert code != "0", f"admin port {port} reachable" + signed = compose("exec", "-T", "api", "python", "-c", + "import asyncio\nfrom messenger.config import Settings\nfrom messenger.storage import Storage\n" + "async def m():\n s=Storage(Settings.from_env()); await s.open()\n" + f" print((await s.head('{key}')).size); await s.close()\nasyncio.run(m())").stdout.strip() + assert signed == str(len(SAMPLE)) + + +def test_s3_restart_keeps_objects(api, pair): + alice, bob, chat = pair + data = imagegen.png_of_size(200_000) + att = api.upload(alice, chat, data).json() + with service_down("s3", "stop"): + pass + assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]).content == data diff --git a/student-tests/unit/test_images.py b/student-tests/unit/test_images.py new file mode 100644 index 0000000..94d57f0 --- /dev/null +++ b/student-tests/unit/test_images.py @@ -0,0 +1,148 @@ +"""Unit tests of the image checks (src/messenger/images.py) and the filename rules. No services. + +The JPEG decoder (djpeg) exists in the API image; locally these tests are skipped without it +(the same cases run against the stand in test_attachments.py).""" +from __future__ import annotations + +import asyncio +import shutil +import sys +import time +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "student-tests")) + +import imagegen # noqa: E402 +from messenger import images, validation # noqa: E402 +from messenger.errors import InvalidRequest # noqa: E402 + +LIMIT = 40_000_000 +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() +has_djpeg = pytest.mark.skipif(shutil.which("djpeg") is None, reason="djpeg (libjpeg-turbo-progs) not installed") + + +def status(data: bytes) -> int: + """201 if accepted, else the HTTP status of the ImageError (413/415).""" + try: + info = images.sniff(data, LIMIT) + if info is None: + return 415 # header never completed + asyncio.run(images.verify(data, info)) + return 201 + except images.ImageError as exc: + return exc.status + + +def test_public_fixture_is_accepted(): + assert images.sniff(SAMPLE, LIMIT) == images.ImageInfo("image/png", 320, 200) + assert status(SAMPLE) == 201 + + +@pytest.mark.parametrize("data", [ + imagegen.png(37, 23, "RGBA"), imagegen.png(33, 17, "P"), imagegen.png(33, 17, "1"), + imagegen.png(33, 17, "L"), imagegen.png(81, 57, "RGB", noise=True), imagegen.png(1, 1), +], ids=["rgba", "palette", "1bit", "gray", "noise", "1x1"]) +def test_valid_png_kinds(data): + assert status(data) == 201 + + +def test_interlaced_png_is_accepted(): + # Pillow cannot write Adam7; build one by hand: 9 x 9 grey, every pass with filter 0. + from messenger.images import _png_segments + raw = b"".join(b"\x00" + bytes(stride - 1) for size, stride in _png_segments(9, 9, 8, 0, 1) + for _ in range(size // stride)) + data = imagegen.PNG_SIGNATURE + imagegen.ihdr(9, 9, 8, 0, 1) + imagegen.chunk(b"IDAT", __import__("zlib").compress(raw)) \ + + imagegen.chunk(b"IEND", b"") + assert status(data) == 201 + + +BROKEN_PNG = { + "first 33 bytes of the fixture": SAMPLE[:33], + "truncated (no IEND)": SAMPLE[:-12], + "truncated in IDAT": SAMPLE[:len(SAMPLE) // 2], + "flipped byte -> CRC error": SAMPLE[:500] + bytes([SAMPLE[500] ^ 0xFF]) + SAMPLE[501:], + "garbage IDAT with valid CRC": imagegen.png_corrupt_idat(), + "fewer rows than IHDR": imagegen.png_short_pixels(), + "bytes after IEND (polyglot)": SAMPLE + b"PK\x03\x04zip", + "text": b"just text, definitely not an image", + "gif": b"GIF89a" + bytes(64), + "two bytes of a signature": b"\x89P", +} + + +@pytest.mark.parametrize("name", list(BROKEN_PNG)) +def test_broken_png_is_415(name): + assert status(BROKEN_PNG[name]) == 415, name + + +def test_pixel_limit_is_checked_from_the_header(): + with pytest.raises(images.ImageError) as exc: + images.sniff(imagegen.png_bomb_header()[:33], LIMIT) # 33 bytes are enough + assert exc.value.status == 413 + exact = imagegen.PNG_SIGNATURE + imagegen.ihdr(8000, 5000) # exactly 40 000 000: allowed + assert images.sniff(exact, LIMIT).width == 8000 + with pytest.raises(images.ImageError) as exc: + images.sniff(imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), LIMIT) + assert exc.value.status == 413 + with pytest.raises(images.ImageError) as exc: + images.sniff(imagegen.jpeg_bomb_header(), LIMIT) + assert exc.value.status == 413 + + +def test_png_verification_is_streaming_and_fast(): + """40 Mpx RGB = 120 MB of raw pixels: inflated in 1 MiB steps and discarded.""" + big = imagegen.png(8000, 5000, compress_level=9) + started = time.perf_counter() + assert status(big) == 201 + assert time.perf_counter() - started < 5 + + +def test_jpeg_header_is_parsed_incrementally(): + data = imagegen.jpeg(320, 200) + assert images.sniff(data[:4], LIMIT) is None # SOF not arrived yet + assert images.sniff(data, LIMIT) == images.ImageInfo("image/jpeg", 320, 200) + + +@has_djpeg +@pytest.mark.parametrize("data", [imagegen.jpeg(), imagegen.jpeg(640, 480, progressive=True)], ids=["base", "prog"]) +def test_valid_jpeg(data): + assert status(data) == 201 + + +@has_djpeg +@pytest.mark.parametrize("name,data", [ + ("no scan data after SOS", imagegen.jpeg_without_scan_data()), + ("truncated", imagegen.jpeg(320, 240, noise=True)[:3000]), +], ids=lambda v: v if isinstance(v, str) else "") +def test_broken_jpeg_is_415(name, data): + assert status(data) == 415, name + + +# ------------------------------------------------------------------ filename +@pytest.mark.parametrize("name", ["photo.png", "фото отпуска.jpg", "a" * 255, "emoji 👨‍👩‍👧.png", " x ", "..."[:2] + "x"]) +def test_good_filenames(name): + assert validation.filename_param([("filename", name)]) == name + + +@pytest.mark.parametrize("name", ["", " ", " ", ".", "..", "a/b.png", "a\\b.png", "a\x00b", "c1\x85.png", + "tab\t.png", "del\x7f.png", "x
y", "photo‮gnp.exe", "a" * 256]) +def test_bad_filenames(name): + with pytest.raises(InvalidRequest): + validation.filename_param([("filename", name)]) + + +def test_filename_must_be_single_and_present(): + with pytest.raises(InvalidRequest): + validation.filename_param([]) + with pytest.raises(InvalidRequest): + validation.filename_param([("filename", "a.png"), ("filename", "b.png")]) + + +def test_query_must_be_valid_utf8(): + assert validation.query_items(b"filename=%D1%84.png") == [("filename", "ф.png")] + with pytest.raises(InvalidRequest): + validation.query_items(b"filename=%FF%FE.png") diff --git a/student-tests/unit/test_unit.py b/student-tests/unit/test_unit.py index 3840880..adf227f 100644 --- a/student-tests/unit/test_unit.py +++ b/student-tests/unit/test_unit.py @@ -114,6 +114,29 @@ def test_argon2id_hash_and_verify(): # ------------------------------------------------------------------ config +@pytest.fixture(autouse=True) +def s3_credentials(monkeypatch): + """Lab 4: Settings require the API's S3 key pair; synthetic values for the unit tests.""" + monkeypatch.setenv("S3_ACCESS_KEY_ID", "AKTEST") + monkeypatch.setenv("S3_SECRET_ACCESS_KEY", "secret-for-unit-tests") + + +def test_s3_settings_are_validated_and_contract_limits_cannot_be_raised(monkeypatch): + monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z") + settings = Settings.from_env() + assert (settings.max_upload_bytes, settings.max_image_pixels) == (10 * 1024 * 1024, 40_000_000) + assert settings.fault_points == frozenset() and settings.upload_concurrency == 4 + for name, value in (("MAX_UPLOAD_BYTES", "10485761"), ("MAX_IMAGE_PIXELS", "40000001"), + ("FAULT_POINTS", "drop_database"), ("S3_ENDPOINT", "ftp://s3"), ("S3_BUCKET", "Bad_Bucket")): + monkeypatch.setenv(name, value) + with pytest.raises(ConfigError): + Settings.from_env() + monkeypatch.delenv(name) + monkeypatch.delenv("S3_SECRET_ACCESS_KEY") + with pytest.raises(ConfigError): + Settings.from_env() + + def test_settings_read_secret_files_and_contract_names(tmp_path, monkeypatch): url_file = tmp_path / "database_url" url_file.write_text("postgresql://u:p@db/x\n", encoding="utf-8")