import asyncio import json import socket import subprocess import time from collections.abc import AsyncIterator, Iterator from datetime import UTC, datetime, timedelta from hashlib import sha256 from http.client import HTTPResponse from typing import Any from urllib.parse import quote, urlsplit from uuid import uuid4 import httpx import pytest from httpx import AsyncClient, Response from pictures import GIF, JPEG, SAMPLE, WEBP, png from stand import BASE_URL, bearer, compose, enroll, lb_up, psql, waited pytestmark = pytest.mark.stack LIMIT = 10_485_760 BUCKET = "messenger-attachments" REQUEST_TIMEOUT = 8 SWEEP_WAIT = 90 CODES = { 400: "invalid_request", 404: "not_found", 413: "payload_too_large", 415: "unsupported_media_type", } HEAD = ( 'curl -s -o /dev/null -w "%{http_code}" -I --aws-sigv4 "aws:amz:${RUSTFS_REGION:-us-east-1}:s3"' ' --user "$(cat /run/secrets/s3_root_user):$(cat /run/secrets/s3_root_password)"' ' "http://127.0.0.1:9000/$BUCKET/$KEY"' ) FAULTS = """ CREATE TABLE lab4_fault_seen (id uuid PRIMARY KEY, object_key text, filename text); CREATE FUNCTION lab4_fault_seen() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN INSERT INTO lab4_fault_seen VALUES (NEW.id, NEW.staging_key, NEW.filename); RETURN NEW; END $$; CREATE FUNCTION lab4_fault_commit() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename LIKE 'fault-fail-%' THEN RAISE EXCEPTION 'injected commit failure' USING ERRCODE = '55P03'; END IF; PERFORM pg_sleep(1.8); RETURN NEW; END $$; CREATE FUNCTION lab4_fault_sleeper() RETURNS text LANGUAGE plpgsql AS $$ DECLARE found text; BEGIN FOR i IN 1..500 LOOP PERFORM pg_stat_clear_snapshot(); SELECT host(client_addr) INTO found FROM pg_stat_activity WHERE wait_event = 'PgSleep' AND query LIKE 'UPDATE attachments%' LIMIT 1; IF found IS NOT NULL THEN RETURN found; END IF; PERFORM pg_sleep(0.02); END LOOP; RETURN NULL; END $$; CREATE TRIGGER lab4_fault_seen AFTER INSERT ON attachments FOR EACH ROW WHEN (NEW.filename LIKE 'fault-%') EXECUTE FUNCTION lab4_fault_seen(); CREATE TRIGGER lab4_fault_commit BEFORE UPDATE ON attachments FOR EACH ROW WHEN (NEW.filename LIKE 'fault-%' AND OLD.status = 'pending' AND NEW.status = 'queued') EXECUTE FUNCTION lab4_fault_commit(); """ CLEANUP = """ DROP TRIGGER IF EXISTS lab4_fault_commit ON attachments; DROP TRIGGER IF EXISTS lab4_fault_seen ON attachments; DROP FUNCTION IF EXISTS lab4_fault_commit(), lab4_fault_seen(), lab4_fault_sleeper(); DROP TABLE IF EXISTS lab4_fault_seen; """ def sql(text: str, **variables: str) -> str: return psql(text, ON_ERROR_STOP="1", **variables) def object_status(key: str) -> int: found = compose( "exec", "-T", "-e", f"KEY={key}", "-e", f"BUCKET={BUCKET}", "s3", "sh", "-c", HEAD ) return int(found) def bucket_keys() -> set[str]: listing = compose( "run", "--rm", "--no-deps", "-T", "s3-cli", "object", "list", "--recursive", "--json", f"root/{BUCKET}/staging/", ) return {item["key"] for item in json.loads(listing)["items"]} def seen(name: str) -> tuple[str, str] | None: row = sql("SELECT id, object_key FROM lab4_fault_seen WHERE filename = :'name'", name=name) if not row: return None attachment_id, key = row.split("|") return attachment_id, key def status_of(attachment_id: str) -> str: return sql("SELECT status FROM attachments WHERE id = :'id'", id=attachment_id) def logs(since: str) -> list[dict[str, Any]]: out = compose("logs", "api", "--since", since, "--no-log-prefix") return [json.loads(line) for line in out.splitlines() if line.startswith("{")] def moment() -> str: return (datetime.now(UTC) - timedelta(seconds=1)).strftime("%Y-%m-%dT%H:%M:%SZ") def kill(container: str) -> None: subprocess.run(["docker", "kill", "-s", "KILL", container], check=True, capture_output=True) def replica_at(address: str) -> str: ids = compose("ps", "-q", "api").split() template = "{{.Id}} {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}" found = subprocess.run( ["docker", "inspect", "-f", template, *ids], check=True, capture_output=True, text=True ).stdout.split("\n") return next(line.split()[0] for line in found if line.endswith(f" {address}")) def raw( path: str, headers: dict[str, str], body: bytes, *, hang_up: bool, piece: int = 0, delay: float = 0.0, ) -> tuple[int, dict[str, Any], float]: parts = urlsplit(BASE_URL) head = f"POST {path} HTTP/1.1\r\nHost: {parts.netloc}\r\nConnection: close\r\n" head += "".join(f"{name}: {value}\r\n" for name, value in headers.items()) + "\r\n" started = time.perf_counter() with socket.create_connection((parts.hostname, parts.port or 80), timeout=30) as sock: sock.sendall(head.encode("latin-1")) for start in range(0, len(body), piece or max(len(body), 1)): sock.sendall(body[start : start + (piece or len(body))]) time.sleep(delay) if hang_up: return 0, {}, 0.0 response = HTTPResponse(sock) response.begin() return response.status, json.loads(response.read()), time.perf_counter() - started READER = """ import hashlib, http.client, sys, time path, token, request_id, piece, delay = sys.argv[1:6] connection = http.client.HTTPConnection("lb", 8080, timeout=30) headers = {"Authorization": "Bearer " + token, "X-Request-Id": request_id} connection.request("GET", path, headers=headers) response, digest, started = connection.getresponse(), hashlib.sha256(), time.monotonic() while chunk := response.read(int(piece)): digest.update(chunk) time.sleep(float(delay)) print(response.status, digest.hexdigest(), round(time.monotonic() - started, 2)) """ def slow_read(path: str, token: str, request_id: str, piece: int, delay: float) -> list[str]: template = "{{range $name, $_ := .NetworkSettings.Networks}}{{$name}} {{end}}" networks = subprocess.run( ["docker", "inspect", "-f", template, compose("ps", "-q", "lb")], check=True, capture_output=True, text=True, ).stdout.split() image = subprocess.run( ["docker", "inspect", "-f", "{{.Config.Image}}", compose("ps", "-q", "api").split()[0]], check=True, capture_output=True, text=True, ).stdout.strip() command = [ "docker", "run", "--rm", "--network", next(name for name in networks if name.endswith("_edge")), "--sysctl", "net.ipv4.tcp_rmem=4096 16384 65536", "--entrypoint", "python", image, "-c", READER, path, token, request_id, str(piece), str(delay), ] return subprocess.run(command, check=True, capture_output=True, text=True).stdout.split() @pytest.fixture async def http() -> AsyncIterator[AsyncClient]: async with AsyncClient(base_url=BASE_URL, timeout=30, trust_env=False) as client: yield client @pytest.fixture(scope="module") def faults() -> Iterator[None]: sql(CLEANUP + FAULTS) try: yield finally: sql(CLEANUP) async def scene(http: AsyncClient) -> dict[str, Any]: (alice, alice_token), (bob, bob_token), (eve, eve_token) = [ await enroll(http, label) for label in ("alice", "bob", "eve") ] chats = [] for owner, token, members in ( (alice, alice_token, [bob]), (alice, alice_token, []), (eve, eve_token, []), ): created = await http.post( "/api/v1/chats", json={"title": "Стенд", "member_ids": [owner["id"], *(m["id"] for m in members)]}, headers=bearer(token), ) assert created.status_code == 201 chats.append(created.json()["id"]) return {"alice": alice_token, "bob": bob_token, "eve": eve_token, "chats": chats} async def upload( http: AsyncClient, token: str, chat: str, body: bytes | AsyncIterator[bytes] = SAMPLE, *, filename: str | None = "sample.png", kind: str | None = "image/png", ) -> Response: query = "" if filename is None else f"?filename={quote(filename, safe='')}" headers = {**bearer(token), **({"Content-Type": kind} if kind else {})} return await http.post( f"/api/v1/chats/{chat}/attachments{query}", content=body, headers=headers ) async def message(http: AsyncClient, token: str, chat: str, body: dict[str, Any]) -> Response: headers = {**bearer(token), "Idempotency-Key": uuid4().hex} return await http.post(f"/api/v1/chats/{chat}/messages", json=body, headers=headers) def expect(label: str, response: Response, status: int) -> None: print(f"{label}: {response.status_code}") assert response.status_code == status, f"{label}: {response.status_code} {response.text}" assert response.headers["X-Instance-Id"] error = response.json()["error"] assert error["code"] == CODES[status] assert error["request_id"] == response.headers["X-Request-Id"] async def test_user_errors_never_become_500(http: AsyncClient) -> None: since = moment() s = await scene(http) alice, eve, (chat, solo, other) = s["alice"], s["eve"], s["chats"] created = 0 attachment = await upload(http, alice, chat) exact = await upload(http, alice, chat, png(16, 16, size=LIMIT)) assert (attachment.status_code, exact.status_code) == (202, 202) assert exact.json()["size_bytes"] == LIMIT created += 2 mine = attachment.json()["id"] foreign = (await upload(http, alice, solo)).json()["id"] created_elsewhere = 1 oversize = png(16, 16, size=LIMIT + 1) async def chunked() -> AsyncIterator[bytes]: for start in range(0, len(oversize), 1 << 20): yield oversize[start : start + (1 << 20)] path = f"/api/v1/chats/{chat}/attachments?filename=big.png" status, body, _ = await asyncio.to_thread( raw, path, {**bearer(alice), "Content-Type": "image/png", "Content-Length": str(LIMIT + 1)}, b"", hang_up=False, ) print(f"10 MiB + 1 с Content-Length, тело не отправлено: {status}") assert (status, body["error"]["code"]) == (413, "payload_too_large") expect("10 MiB + 1 chunked", await upload(http, alice, chat, chunked()), 413) expect( "PNG 100000×100000 (бомба)", await upload(http, alice, chat, png(100_000, 100_000, rows=False)), 413, ) expect("PNG 8001×5000", await upload(http, alice, chat, png(8001, 5000, rows=False)), 413) for label, content, kind in ( ("обрезанный PNG", SAMPLE[: len(SAMPLE) // 2], "image/png"), ("обрезанный JPEG", JPEG[: len(JPEG) // 2], "image/jpeg"), ("JPEG как image/png", JPEG, "image/png"), ("GIF как image/png", GIF, "image/png"), ("WebP как image/png", WEBP, "image/png"), ("текст как image/png", b"plain text, not an image", "image/png"), ("PNG как image/jpeg", SAMPLE, "image/jpeg"), ("пустое тело", b"", "image/png"), ("application/octet-stream", SAMPLE, "application/octet-stream"), ("без Content-Type", SAMPLE, None), ): expect(label, await upload(http, alice, chat, content, kind=kind), 415) for name in ( "../a.png", "a/b.png", "a\\b.png", "a\x00.png", "a\x01.png", "a\x7f.png", "", " ", "x" * 256, None, ): expect(f"filename={name!r}"[:40], await upload(http, alice, chat, filename=name), 400) expect("посторонний: upload в чужой чат", await upload(http, eve, chat), 404) expect( "посторонний: метаданные", await http.get(f"/api/v1/attachments/{mine}", headers=bearer(eve)), 404, ) expect( "посторонний: /content", await http.get(f"/api/v1/attachments/{mine}/content", headers=bearer(eve)), 404, ) expect( "посторонний: ссылка на чужое вложение", await message(http, eve, other, {"text": "x", "attachment_ids": [mine]}), 404, ) expect( "вложение другого чата", await message(http, alice, chat, {"text": "x", "attachment_ids": [foreign]}), 404, ) expect( "неизвестное вложение", await message(http, alice, chat, {"text": "x", "attachment_ids": [str(uuid4())]}), 404, ) for label, ids in ( ("11 вложений", [mine] + [str(uuid4()) for _ in range(10)]), ("повтор вложения", [mine, mine]), ("невалидный UUID", ["not-a-uuid"]), ): expect(label, await message(http, alice, chat, {"text": "x", "attachment_ids": ids}), 400) expect( "пустой текст без вложений", await message(http, alice, chat, {"text": "", "attachment_ids": []}), 400, ) expect( "невалидный id вложения", await http.get("/api/v1/attachments/x/content", headers=bearer(alice)), 400, ) expect( "variant=preview", await http.get( f"/api/v1/attachments/{mine}/content?variant=preview", headers=bearer(alice) ), 400, ) rows = sql("SELECT count(*) FROM attachments WHERE chat_id = :'chat'", chat=chat) assert int(rows) == created assert sql("SELECT count(*) FROM attachments WHERE chat_id = :'chat'", chat=solo) == str( created_elsewhere ) statuses = [entry["status"] for entry in logs(since) if entry.get("event") == "request.handled"] print( f"строк вложений в чате: {rows} (= ответов 202); ответов в журнале реплик: " f"{len(statuses)}, из них 5xx: {sum(code >= 500 for code in statuses)}" ) assert not [status for status in statuses if status >= 500] async def test_commit_failure_is_compensated_at_once(http: AsyncClient, faults: None) -> None: since = moment() s = await scene(http) name = f"fault-fail-{uuid4().hex}.png" response = await upload(http, s["alice"], s["chats"][0], filename=name) print( "отказ COMMIT после записи объекта:", response.status_code, response.json()["error"]["code"], ) assert response.status_code == 503 found = seen(name) assert found is not None attachment_id, key = found print( f"сразу после ответа: строка {status_of(attachment_id) or 'нет'},", f"объект HEAD {object_status(key)}", ) assert status_of(attachment_id) == "" assert object_status(key) == 404 def journaled() -> bool: return any( entry.get("event") == "audit.attachment_discarded" and entry.get("attachment_id") == attachment_id and entry.get("reason") == "compensation" for entry in logs(since) ) await waited(journaled, 10) print("журнал: audit.attachment_discarded reason=compensation") async def crash(http: AsyncClient, token: str, chat: str) -> tuple[str, str]: name = f"fault-crash-{uuid4().hex}.png" sent = asyncio.create_task(upload(http, token, chat, filename=name)) address = await asyncio.to_thread(sql, "SELECT lab4_fault_sleeper()") assert address, "UPDATE queued не дошёл до триггера" victim = replica_at(address) kill(victim) try: outcome = str((await sent).status_code) except httpx.HTTPError as error: outcome = type(error).__name__ compose("up", "-d", "--wait", "--no-deps", "--no-recreate", "api") await waited(lambda: lb_up() == 2, 60) found = seen(name) assert found is not None print(f"реплика {victim[:12]} убита между PUT и COMMIT, клиент получил {outcome}") return found async def test_crash_between_put_and_commit_is_swept_after_grace( http: AsyncClient, faults: None ) -> None: s = await scene(http) alice, chat = s["alice"], s["chats"][0] kept = (await upload(http, alice, chat, filename="kept.png")).json() await waited(lambda: status_of(kept["id"]) == "ready", 60) assert ( await message(http, alice, chat, {"text": "", "attachment_ids": [kept["id"]]}) ).status_code == 201 old, fresh = await crash(http, alice, chat), await crash(http, alice, chat) for attachment_id, key in (old, fresh): state = (status_of(attachment_id), object_status(key)) print(f"остаток после падения: строка {state[0]}, объект HEAD {state[1]}") assert state == ("pending", 200) aged = "UPDATE attachments SET created_at = created_at - interval '2 hours' WHERE id = :'id'" sql(aged, id=old[0]) sql(aged, id=kept["id"]) swept = await waited(lambda: status_of(old[0]) == "", SWEEP_WAIT) print(f"старше grace: строка убрана через {swept:.1f} с, объект HEAD {object_status(old[1])}") assert object_status(old[1]) == 404 print(f"моложе grace: строка {status_of(fresh[0])}, объект HEAD {object_status(fresh[1])}") assert (status_of(fresh[0]), object_status(fresh[1])) == ("pending", 200) sql(aged, id=fresh[0]) swept = await waited(lambda: status_of(fresh[0]) == "", SWEEP_WAIT) print(f"тот же после сдвига: убран через {swept:.1f} с, объект HEAD {object_status(fresh[1])}") assert object_status(fresh[1]) == 404 content = await http.get(f"/api/v1/attachments/{kept['id']}/content", headers=bearer(s["bob"])) assert content.status_code == 200 assert sha256(content.content).hexdigest() == kept["sha256"] print("готовое вложение в сообщении со сдвинутым created_at скачивается, SHA-256 совпадает") async def test_aborted_upload_leaves_nothing(http: AsyncClient, faults: None) -> None: since = moment() s = await scene(http) name, request_id = f"fault-abort-{uuid4().hex}.png", f"abort-{uuid4().hex}" before = bucket_keys() headers = { **bearer(s["alice"]), "Content-Type": "image/png", "Content-Length": str(5 << 20), "X-Request-Id": request_id, } path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename={name}" await asyncio.to_thread(raw, path, headers, SAMPLE[:8] + bytes(256 << 10), hang_up=True) def handled() -> list[int]: return [ e["status"] for e in logs(since) if e.get("request_id") == request_id and e.get("event") == "request.handled" ] await waited(lambda: bool(handled()), 15) print( f"обрыв после 256 KiB из 5 MiB: статус в журнале реплики {handled()},", f"намерение {seen(name)}", ) assert handled() == [400] assert seen(name) is None assert bucket_keys() == before async def test_stalled_upload_is_cut_and_leaves_nothing(http: AsyncClient, faults: None) -> None: s = await scene(http) name = f"fault-stall-{uuid4().hex}.png" headers = {**bearer(s["alice"]), "Content-Type": "image/png", "Content-Length": str(1 << 20)} path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename={name}" status, body, elapsed = await asyncio.to_thread( raw, path, headers, SAMPLE[:1024], hang_up=False ) print(f"тело зависло после 1 KiB: {status} {body['error']['code']} за {elapsed:.2f} с") assert (status, body["error"]["code"]) == (503, "unavailable") assert elapsed <= REQUEST_TIMEOUT + 2 assert seen(name) is None assert (await upload(http, s["alice"], s["chats"][0])).status_code == 202 async def test_token_is_checked_before_the_declared_size(http: AsyncClient) -> None: s = await scene(http) headers = {"Content-Type": "image/png", "Content-Length": str(LIMIT + 1)} path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename=a.png" status, body, _ = await asyncio.to_thread(raw, path, headers, SAMPLE[:1024], hang_up=False) print(f"без токена, Content-Length {LIMIT + 1}: {status} {body['error']['code']}") assert (status, body["error"]["code"]) == (401, "unauthorized") async def test_slow_but_steady_transfers_outlive_the_request_timeout(http: AsyncClient) -> None: since = moment() s = await scene(http) picture = png(16, 16, size=3 << 20) sent, read = f"slow-upload-{uuid4().hex}", f"slow-download-{uuid4().hex}" headers = { **bearer(s["alice"]), "Content-Type": "image/png", "Content-Length": str(len(picture)), "X-Request-Id": sent, } path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename=slow.png" status, body, up = await asyncio.to_thread( raw, path, headers, picture, hang_up=False, piece=64 << 10, delay=0.25 ) assert status == 202, body assert body["sha256"] == sha256(picture).hexdigest() await waited(lambda: status_of(body["id"]) == "ready", 60) content = f"/api/v1/attachments/{body['id']}/content" code, digest, down = await asyncio.to_thread(slow_read, content, s["bob"], read, 64 << 10, 0.33) assert (code, digest) == ("200", body["sha256"]) def handled() -> dict[str, tuple[int, float]]: return { e["request_id"]: (e["status"], e["duration_ms"] / 1000) for e in logs(since) if e.get("event") == "request.handled" and e.get("request_id") in (sent, read) } await waited(lambda: len(handled()) == 2, 15) replica = handled() print(f"upload 3 MiB ≈ 256 KiB/s: клиент {up:.1f} с, реплика {replica[sent]}") print(f"скачивание 3 MiB ≈ 192 KiB/s из сети edge: клиент {down} с, реплика {replica[read]}") assert replica[sent][0] == 202 and replica[sent][1] > REQUEST_TIMEOUT assert replica[read][0] == 200 and replica[read][1] > REQUEST_TIMEOUT