Files

580 lines
22 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import asyncio
import json
import socket
import subprocess
import time
from collections.abc import AsyncIterator, Iterator
from datetime import UTC, datetime, timedelta
from hashlib import sha256
from http.client import HTTPResponse
from typing import Any
from urllib.parse import quote, urlsplit
from uuid import uuid4
import httpx
import pytest
from httpx import AsyncClient, Response
from pictures import BROKEN_PNG, DUPLICATE_IHDR_PNG, GIF, JPEG, SAMPLE, WEBP, png
from stand import BASE_URL, bearer, compose, enroll, lb_up, psql, waited
pytestmark = pytest.mark.stack
LIMIT = 10_485_760
BUCKET = "messenger-attachments"
REQUEST_TIMEOUT = 8
SWEEP_WAIT = 90
CODES = {
400: "invalid_request",
404: "not_found",
413: "payload_too_large",
415: "unsupported_media_type",
}
HEAD = (
'curl -s -o /dev/null -w "%{http_code}" -I --aws-sigv4 "aws:amz:${RUSTFS_REGION:-us-east-1}:s3"'
' --user "$(cat /run/secrets/s3_root_user):$(cat /run/secrets/s3_root_password)"'
' "http://127.0.0.1:9000/$BUCKET/$KEY"'
)
FAULTS = """
CREATE TABLE lab4_fault_seen (id uuid PRIMARY KEY, object_key text, filename text);
CREATE FUNCTION lab4_fault_seen() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
INSERT INTO lab4_fault_seen VALUES (NEW.id, NEW.staging_key, NEW.filename);
RETURN NEW;
END $$;
CREATE FUNCTION lab4_fault_commit() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF NEW.filename LIKE 'fault-fail-%' THEN
RAISE EXCEPTION 'injected commit failure' USING ERRCODE = '55P03';
END IF;
PERFORM pg_sleep(1.8);
RETURN NEW;
END $$;
CREATE FUNCTION lab4_fault_sleeper() RETURNS text LANGUAGE plpgsql AS $$
DECLARE found text;
BEGIN
FOR i IN 1..500 LOOP
PERFORM pg_stat_clear_snapshot();
SELECT host(client_addr) INTO found FROM pg_stat_activity
WHERE wait_event = 'PgSleep' AND query LIKE 'UPDATE attachments%' LIMIT 1;
IF found IS NOT NULL THEN RETURN found; END IF;
PERFORM pg_sleep(0.02);
END LOOP;
RETURN NULL;
END $$;
CREATE TRIGGER lab4_fault_seen AFTER INSERT ON attachments FOR EACH ROW
WHEN (NEW.filename LIKE 'fault-%') EXECUTE FUNCTION lab4_fault_seen();
CREATE TRIGGER lab4_fault_commit BEFORE UPDATE ON attachments FOR EACH ROW
WHEN (NEW.filename LIKE 'fault-%' AND OLD.status = 'pending' AND NEW.status = 'queued')
EXECUTE FUNCTION lab4_fault_commit();
"""
CLEANUP = """
DROP TRIGGER IF EXISTS lab4_fault_commit ON attachments;
DROP TRIGGER IF EXISTS lab4_fault_seen ON attachments;
DROP FUNCTION IF EXISTS lab4_fault_commit(), lab4_fault_seen(), lab4_fault_sleeper();
DROP TABLE IF EXISTS lab4_fault_seen;
"""
def sql(text: str, **variables: str) -> str:
return psql(text, ON_ERROR_STOP="1", **variables)
def object_status(key: str) -> int:
found = compose(
"exec", "-T", "-e", f"KEY={key}", "-e", f"BUCKET={BUCKET}", "s3", "sh", "-c", HEAD
)
return int(found)
def bucket_keys() -> set[str]:
listing = compose(
"run",
"--rm",
"--no-deps",
"-T",
"s3-cli",
"object",
"list",
"--recursive",
"--json",
f"root/{BUCKET}/staging/",
)
return {item["key"] for item in json.loads(listing)["items"]}
def seen(name: str) -> tuple[str, str] | None:
row = sql("SELECT id, object_key FROM lab4_fault_seen WHERE filename = :'name'", name=name)
if not row:
return None
attachment_id, key = row.split("|")
return attachment_id, key
def status_of(attachment_id: str) -> str:
return sql("SELECT status FROM attachments WHERE id = :'id'", id=attachment_id)
def logs(since: str) -> list[dict[str, Any]]:
out = compose("logs", "api", "--since", since, "--no-log-prefix")
return [json.loads(line) for line in out.splitlines() if line.startswith("{")]
def moment() -> str:
return (datetime.now(UTC) - timedelta(seconds=1)).strftime("%Y-%m-%dT%H:%M:%SZ")
def kill(container: str) -> None:
subprocess.run(["docker", "kill", "-s", "KILL", container], check=True, capture_output=True)
def replica_at(address: str) -> str:
ids = compose("ps", "-q", "api").split()
template = "{{.Id}} {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}"
found = subprocess.run(
["docker", "inspect", "-f", template, *ids], check=True, capture_output=True, text=True
).stdout.split("\n")
return next(line.split()[0] for line in found if line.endswith(f" {address}"))
def raw(
path: str,
headers: dict[str, str],
body: bytes,
*,
hang_up: bool,
piece: int = 0,
delay: float = 0.0,
) -> tuple[int, dict[str, Any], float]:
parts = urlsplit(BASE_URL)
head = f"POST {path} HTTP/1.1\r\nHost: {parts.netloc}\r\nConnection: close\r\n"
head += "".join(f"{name}: {value}\r\n" for name, value in headers.items()) + "\r\n"
started = time.perf_counter()
with socket.create_connection((parts.hostname, parts.port or 80), timeout=30) as sock:
sock.sendall(head.encode("latin-1"))
for start in range(0, len(body), piece or max(len(body), 1)):
sock.sendall(body[start : start + (piece or len(body))])
time.sleep(delay)
if hang_up:
return 0, {}, 0.0
response = HTTPResponse(sock)
response.begin()
return response.status, json.loads(response.read()), time.perf_counter() - started
READER = """
import hashlib, http.client, sys, time
path, token, request_id, piece, delay = sys.argv[1:6]
connection = http.client.HTTPConnection("lb", 8080, timeout=30)
headers = {"Authorization": "Bearer " + token, "X-Request-Id": request_id}
connection.request("GET", path, headers=headers)
response, digest, started = connection.getresponse(), hashlib.sha256(), time.monotonic()
while chunk := response.read(int(piece)):
digest.update(chunk)
time.sleep(float(delay))
print(response.status, digest.hexdigest(), round(time.monotonic() - started, 2))
"""
def slow_read(path: str, token: str, request_id: str, piece: int, delay: float) -> list[str]:
template = "{{range $name, $_ := .NetworkSettings.Networks}}{{$name}} {{end}}"
networks = subprocess.run(
["docker", "inspect", "-f", template, compose("ps", "-q", "lb")],
check=True,
capture_output=True,
text=True,
).stdout.split()
image = subprocess.run(
["docker", "inspect", "-f", "{{.Config.Image}}", compose("ps", "-q", "api").split()[0]],
check=True,
capture_output=True,
text=True,
).stdout.strip()
command = [
"docker",
"run",
"--rm",
"--network",
next(name for name in networks if name.endswith("_edge")),
"--sysctl",
"net.ipv4.tcp_rmem=4096 16384 65536",
"--entrypoint",
"python",
image,
"-c",
READER,
path,
token,
request_id,
str(piece),
str(delay),
]
return subprocess.run(command, check=True, capture_output=True, text=True).stdout.split()
@pytest.fixture
async def http() -> AsyncIterator[AsyncClient]:
async with AsyncClient(base_url=BASE_URL, timeout=30, trust_env=False) as client:
yield client
@pytest.fixture(scope="module")
def faults() -> Iterator[None]:
sql(CLEANUP + FAULTS)
try:
yield
finally:
sql(CLEANUP)
async def scene(http: AsyncClient) -> dict[str, Any]:
(alice, alice_token), (bob, bob_token), (eve, eve_token) = [
await enroll(http, label) for label in ("alice", "bob", "eve")
]
chats = []
for owner, token, members in (
(alice, alice_token, [bob]),
(alice, alice_token, []),
(eve, eve_token, []),
):
created = await http.post(
"/api/v1/chats",
json={"title": "Стенд", "member_ids": [owner["id"], *(m["id"] for m in members)]},
headers=bearer(token),
)
assert created.status_code == 201
chats.append(created.json()["id"])
return {"alice": alice_token, "bob": bob_token, "eve": eve_token, "chats": chats}
async def upload(
http: AsyncClient,
token: str,
chat: str,
body: bytes | AsyncIterator[bytes] = SAMPLE,
*,
filename: str | None = "sample.png",
kind: str | None = "image/png",
) -> Response:
query = "" if filename is None else f"?filename={quote(filename, safe='')}"
headers = {**bearer(token), **({"Content-Type": kind} if kind else {})}
return await http.post(
f"/api/v1/chats/{chat}/attachments{query}", content=body, headers=headers
)
async def message(http: AsyncClient, token: str, chat: str, body: dict[str, Any]) -> Response:
headers = {**bearer(token), "Idempotency-Key": uuid4().hex}
return await http.post(f"/api/v1/chats/{chat}/messages", json=body, headers=headers)
def expect(label: str, response: Response, status: int) -> None:
print(f"{label}: {response.status_code}")
assert response.status_code == status, f"{label}: {response.status_code} {response.text}"
assert response.headers["X-Instance-Id"]
error = response.json()["error"]
assert error["code"] == CODES[status]
assert error["request_id"] == response.headers["X-Request-Id"]
async def test_user_errors_never_become_500(http: AsyncClient) -> None:
since = moment()
s = await scene(http)
alice, eve, (chat, solo, other) = s["alice"], s["eve"], s["chats"]
created = 0
attachment = await upload(http, alice, chat)
exact = await upload(http, alice, chat, png(16, 16, size=LIMIT))
assert (attachment.status_code, exact.status_code) == (202, 202)
assert exact.json()["size_bytes"] == LIMIT
created += 2
mine = attachment.json()["id"]
foreign = (await upload(http, alice, solo)).json()["id"]
created_elsewhere = 1
oversize = png(16, 16, size=LIMIT + 1)
async def chunked() -> AsyncIterator[bytes]:
for start in range(0, len(oversize), 1 << 20):
yield oversize[start : start + (1 << 20)]
path = f"/api/v1/chats/{chat}/attachments?filename=big.png"
status, body, _ = await asyncio.to_thread(
raw,
path,
{**bearer(alice), "Content-Type": "image/png", "Content-Length": str(LIMIT + 1)},
b"",
hang_up=False,
)
print(f"10 MiB + 1 с Content-Length, тело не отправлено: {status}")
assert (status, body["error"]["code"]) == (413, "payload_too_large")
expect("10 MiB + 1 chunked", await upload(http, alice, chat, chunked()), 413)
expect(
"PNG 100000×100000 (бомба)",
await upload(http, alice, chat, png(100_000, 100_000, rows=False)),
413,
)
expect("PNG 8001×5000", await upload(http, alice, chat, png(8001, 5000, rows=False)), 413)
for label, content, kind in (
("PNG с корректными CRC и битым IDAT", BROKEN_PNG, "image/png"),
("PNG с повторным IHDR", DUPLICATE_IHDR_PNG, "image/png"),
("обрезанный PNG", SAMPLE[: len(SAMPLE) // 2], "image/png"),
("обрезанный JPEG", JPEG[: len(JPEG) // 2], "image/jpeg"),
("JPEG как image/png", JPEG, "image/png"),
("GIF как image/png", GIF, "image/png"),
("WebP как image/png", WEBP, "image/png"),
("текст как image/png", b"plain text, not an image", "image/png"),
("PNG как image/jpeg", SAMPLE, "image/jpeg"),
("пустое тело", b"", "image/png"),
("application/octet-stream", SAMPLE, "application/octet-stream"),
("без Content-Type", SAMPLE, None),
):
expect(label, await upload(http, alice, chat, content, kind=kind), 415)
for name in (
"../a.png",
"a/b.png",
"a\\b.png",
"a\x00.png",
"a\x01.png",
"a\x7f.png",
"",
" ",
"x" * 256,
None,
):
expect(f"filename={name!r}"[:40], await upload(http, alice, chat, filename=name), 400)
expect("посторонний: upload в чужой чат", await upload(http, eve, chat), 404)
expect(
"посторонний: метаданные",
await http.get(f"/api/v1/attachments/{mine}", headers=bearer(eve)),
404,
)
expect(
"посторонний: /content",
await http.get(f"/api/v1/attachments/{mine}/content", headers=bearer(eve)),
404,
)
expect(
"посторонний: ссылка на чужое вложение",
await message(http, eve, other, {"text": "x", "attachment_ids": [mine]}),
404,
)
expect(
"вложение другого чата",
await message(http, alice, chat, {"text": "x", "attachment_ids": [foreign]}),
404,
)
expect(
"неизвестное вложение",
await message(http, alice, chat, {"text": "x", "attachment_ids": [str(uuid4())]}),
404,
)
for label, ids in (
("11 вложений", [mine] + [str(uuid4()) for _ in range(10)]),
("повтор вложения", [mine, mine]),
("невалидный UUID", ["not-a-uuid"]),
):
expect(label, await message(http, alice, chat, {"text": "x", "attachment_ids": ids}), 400)
expect(
"пустой текст без вложений",
await message(http, alice, chat, {"text": "", "attachment_ids": []}),
400,
)
expect(
"невалидный id вложения",
await http.get("/api/v1/attachments/x/content", headers=bearer(alice)),
400,
)
expect(
"variant=preview",
await http.get(
f"/api/v1/attachments/{mine}/content?variant=preview", headers=bearer(alice)
),
400,
)
rows = sql("SELECT count(*) FROM attachments WHERE chat_id = :'chat'", chat=chat)
assert int(rows) == created
assert sql("SELECT count(*) FROM attachments WHERE chat_id = :'chat'", chat=solo) == str(
created_elsewhere
)
statuses = [entry["status"] for entry in logs(since) if entry.get("event") == "request.handled"]
print(
f"строк вложений в чате: {rows} (= ответов 202); ответов в журнале реплик: "
f"{len(statuses)}, из них 5xx: {sum(code >= 500 for code in statuses)}"
)
assert not [status for status in statuses if status >= 500]
async def test_commit_failure_is_compensated_at_once(http: AsyncClient, faults: None) -> None:
since = moment()
s = await scene(http)
name = f"fault-fail-{uuid4().hex}.png"
response = await upload(http, s["alice"], s["chats"][0], filename=name)
print(
"отказ COMMIT после записи объекта:",
response.status_code,
response.json()["error"]["code"],
)
assert response.status_code == 503
found = seen(name)
assert found is not None
attachment_id, key = found
print(
f"сразу после ответа: строка {status_of(attachment_id) or 'нет'},",
f"объект HEAD {object_status(key)}",
)
assert status_of(attachment_id) == ""
assert object_status(key) == 404
def journaled() -> bool:
return any(
entry.get("event") == "audit.attachment_discarded"
and entry.get("attachment_id") == attachment_id
and entry.get("reason") == "compensation"
for entry in logs(since)
)
await waited(journaled, 10)
print("журнал: audit.attachment_discarded reason=compensation")
async def crash(http: AsyncClient, token: str, chat: str) -> tuple[str, str]:
name = f"fault-crash-{uuid4().hex}.png"
sent = asyncio.create_task(upload(http, token, chat, filename=name))
address = await asyncio.to_thread(sql, "SELECT lab4_fault_sleeper()")
assert address, "UPDATE queued не дошёл до триггера"
victim = replica_at(address)
kill(victim)
try:
outcome = str((await sent).status_code)
except httpx.HTTPError as error:
outcome = type(error).__name__
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", "api")
await waited(lambda: lb_up() == 2, 60)
found = seen(name)
assert found is not None
print(f"реплика {victim[:12]} убита между PUT и COMMIT, клиент получил {outcome}")
return found
async def test_crash_between_put_and_commit_is_swept_after_grace(
http: AsyncClient, faults: None
) -> None:
s = await scene(http)
alice, chat = s["alice"], s["chats"][0]
kept = (await upload(http, alice, chat, filename="kept.png")).json()
await waited(lambda: status_of(kept["id"]) == "ready", 60)
assert (
await message(http, alice, chat, {"text": "", "attachment_ids": [kept["id"]]})
).status_code == 201
old, fresh = await crash(http, alice, chat), await crash(http, alice, chat)
for attachment_id, key in (old, fresh):
state = (status_of(attachment_id), object_status(key))
print(f"остаток после падения: строка {state[0]}, объект HEAD {state[1]}")
assert state == ("pending", 200)
aged = "UPDATE attachments SET created_at = created_at - interval '2 hours' WHERE id = :'id'"
sql(aged, id=old[0])
sql(aged, id=kept["id"])
swept = await waited(lambda: status_of(old[0]) == "", SWEEP_WAIT)
print(f"старше grace: строка убрана через {swept:.1f} с, объект HEAD {object_status(old[1])}")
assert object_status(old[1]) == 404
print(f"моложе grace: строка {status_of(fresh[0])}, объект HEAD {object_status(fresh[1])}")
assert (status_of(fresh[0]), object_status(fresh[1])) == ("pending", 200)
sql(aged, id=fresh[0])
swept = await waited(lambda: status_of(fresh[0]) == "", SWEEP_WAIT)
print(f"тот же после сдвига: убран через {swept:.1f} с, объект HEAD {object_status(fresh[1])}")
assert object_status(fresh[1]) == 404
content = await http.get(f"/api/v1/attachments/{kept['id']}/content", headers=bearer(s["bob"]))
assert content.status_code == 200
assert sha256(content.content).hexdigest() == kept["sha256"]
print("готовое вложение в сообщении со сдвинутым created_at скачивается, SHA-256 совпадает")
async def test_aborted_upload_leaves_nothing(http: AsyncClient, faults: None) -> None:
since = moment()
s = await scene(http)
name, request_id = f"fault-abort-{uuid4().hex}.png", f"abort-{uuid4().hex}"
before = bucket_keys()
headers = {
**bearer(s["alice"]),
"Content-Type": "image/png",
"Content-Length": str(5 << 20),
"X-Request-Id": request_id,
}
path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename={name}"
await asyncio.to_thread(raw, path, headers, SAMPLE[:8] + bytes(256 << 10), hang_up=True)
def handled() -> list[int]:
return [
e["status"]
for e in logs(since)
if e.get("request_id") == request_id and e.get("event") == "request.handled"
]
await waited(lambda: bool(handled()), 15)
print(
f"обрыв после 256 KiB из 5 MiB: статус в журнале реплики {handled()},",
f"намерение {seen(name)}",
)
assert handled() == [400]
assert seen(name) is None
assert bucket_keys() == before
async def test_stalled_upload_is_cut_and_leaves_nothing(http: AsyncClient, faults: None) -> None:
s = await scene(http)
name = f"fault-stall-{uuid4().hex}.png"
headers = {**bearer(s["alice"]), "Content-Type": "image/png", "Content-Length": str(1 << 20)}
path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename={name}"
status, body, elapsed = await asyncio.to_thread(
raw, path, headers, SAMPLE[:1024], hang_up=False
)
print(f"тело зависло после 1 KiB: {status} {body['error']['code']} за {elapsed:.2f} с")
assert (status, body["error"]["code"]) == (503, "unavailable")
assert elapsed <= REQUEST_TIMEOUT + 2
assert seen(name) is None
assert (await upload(http, s["alice"], s["chats"][0])).status_code == 202
async def test_token_is_checked_before_the_declared_size(http: AsyncClient) -> None:
s = await scene(http)
headers = {"Content-Type": "image/png", "Content-Length": str(LIMIT + 1)}
path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename=a.png"
status, body, _ = await asyncio.to_thread(raw, path, headers, SAMPLE[:1024], hang_up=False)
print(f"без токена, Content-Length {LIMIT + 1}: {status} {body['error']['code']}")
assert (status, body["error"]["code"]) == (401, "unauthorized")
async def test_slow_but_steady_transfers_outlive_the_request_timeout(http: AsyncClient) -> None:
since = moment()
s = await scene(http)
picture = png(16, 16, size=3 << 20)
sent, read = f"slow-upload-{uuid4().hex}", f"slow-download-{uuid4().hex}"
headers = {
**bearer(s["alice"]),
"Content-Type": "image/png",
"Content-Length": str(len(picture)),
"X-Request-Id": sent,
}
path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename=slow.png"
status, body, up = await asyncio.to_thread(
raw, path, headers, picture, hang_up=False, piece=64 << 10, delay=0.25
)
assert status == 202, body
assert body["sha256"] == sha256(picture).hexdigest()
await waited(lambda: status_of(body["id"]) == "ready", 60)
content = f"/api/v1/attachments/{body['id']}/content"
code, digest, down = await asyncio.to_thread(slow_read, content, s["bob"], read, 64 << 10, 0.33)
assert (code, digest) == ("200", body["sha256"])
def handled() -> dict[str, tuple[int, float]]:
return {
e["request_id"]: (e["status"], e["duration_ms"] / 1000)
for e in logs(since)
if e.get("event") == "request.handled" and e.get("request_id") in (sent, read)
}
await waited(lambda: len(handled()) == 2, 15)
replica = handled()
print(f"upload 3 MiB ≈ 256 KiB/s: клиент {up:.1f} с, реплика {replica[sent]}")
print(f"скачивание 3 MiB ≈ 192 KiB/s из сети edge: клиент {down} с, реплика {replica[read]}")
assert replica[sent][0] == 202 and replica[sent][1] > REQUEST_TIMEOUT
assert replica[read][0] == 200 and replica[read][1] > REQUEST_TIMEOUT