Source: immutable lab5 snapshot f88e9ba43cdfd7eb6a5a7ae5feb4247b4e820071; forward the allowed diff from 64d5c67. Carry SQLAlchemy, Alembic, reliability fixes, versioned jobs, infrastructure and student tests without changing protected coursework files. Validation: make check, validate, lint; 515 student tests passed with one Linux-only skip; public smoke with telemetry enabled and repeat migration passed. Stop the owned stack and retain volumes. Preserve the existing OTel work in progress; grade implementation remains a separate phase.
578 lines
22 KiB
Python
578 lines
22 KiB
Python
import asyncio
|
||
import json
|
||
import socket
|
||
import subprocess
|
||
import time
|
||
from collections.abc import AsyncIterator, Iterator
|
||
from datetime import UTC, datetime, timedelta
|
||
from hashlib import sha256
|
||
from http.client import HTTPResponse
|
||
from typing import Any
|
||
from urllib.parse import quote, urlsplit
|
||
from uuid import uuid4
|
||
|
||
import httpx
|
||
import pytest
|
||
from httpx import AsyncClient, Response
|
||
|
||
from pictures import GIF, JPEG, SAMPLE, WEBP, png
|
||
from stand import BASE_URL, bearer, compose, enroll, lb_up, psql, waited
|
||
|
||
pytestmark = pytest.mark.stack
|
||
LIMIT = 10_485_760
|
||
BUCKET = "messenger-attachments"
|
||
REQUEST_TIMEOUT = 8
|
||
SWEEP_WAIT = 90
|
||
CODES = {
|
||
400: "invalid_request",
|
||
404: "not_found",
|
||
413: "payload_too_large",
|
||
415: "unsupported_media_type",
|
||
}
|
||
HEAD = (
|
||
'curl -s -o /dev/null -w "%{http_code}" -I --aws-sigv4 "aws:amz:${RUSTFS_REGION:-us-east-1}:s3"'
|
||
' --user "$(cat /run/secrets/s3_root_user):$(cat /run/secrets/s3_root_password)"'
|
||
' "http://127.0.0.1:9000/$BUCKET/$KEY"'
|
||
)
|
||
FAULTS = """
|
||
CREATE TABLE lab4_fault_seen (id uuid PRIMARY KEY, object_key text, filename text);
|
||
CREATE FUNCTION lab4_fault_seen() RETURNS trigger LANGUAGE plpgsql AS $$
|
||
BEGIN
|
||
INSERT INTO lab4_fault_seen VALUES (NEW.id, NEW.staging_key, NEW.filename);
|
||
RETURN NEW;
|
||
END $$;
|
||
CREATE FUNCTION lab4_fault_commit() RETURNS trigger LANGUAGE plpgsql AS $$
|
||
BEGIN
|
||
IF NEW.filename LIKE 'fault-fail-%' THEN
|
||
RAISE EXCEPTION 'injected commit failure' USING ERRCODE = '55P03';
|
||
END IF;
|
||
PERFORM pg_sleep(1.8);
|
||
RETURN NEW;
|
||
END $$;
|
||
CREATE FUNCTION lab4_fault_sleeper() RETURNS text LANGUAGE plpgsql AS $$
|
||
DECLARE found text;
|
||
BEGIN
|
||
FOR i IN 1..500 LOOP
|
||
PERFORM pg_stat_clear_snapshot();
|
||
SELECT host(client_addr) INTO found FROM pg_stat_activity
|
||
WHERE wait_event = 'PgSleep' AND query LIKE 'UPDATE attachments%' LIMIT 1;
|
||
IF found IS NOT NULL THEN RETURN found; END IF;
|
||
PERFORM pg_sleep(0.02);
|
||
END LOOP;
|
||
RETURN NULL;
|
||
END $$;
|
||
CREATE TRIGGER lab4_fault_seen AFTER INSERT ON attachments FOR EACH ROW
|
||
WHEN (NEW.filename LIKE 'fault-%') EXECUTE FUNCTION lab4_fault_seen();
|
||
CREATE TRIGGER lab4_fault_commit BEFORE UPDATE ON attachments FOR EACH ROW
|
||
WHEN (NEW.filename LIKE 'fault-%' AND OLD.status = 'pending' AND NEW.status = 'queued')
|
||
EXECUTE FUNCTION lab4_fault_commit();
|
||
"""
|
||
CLEANUP = """
|
||
DROP TRIGGER IF EXISTS lab4_fault_commit ON attachments;
|
||
DROP TRIGGER IF EXISTS lab4_fault_seen ON attachments;
|
||
DROP FUNCTION IF EXISTS lab4_fault_commit(), lab4_fault_seen(), lab4_fault_sleeper();
|
||
DROP TABLE IF EXISTS lab4_fault_seen;
|
||
"""
|
||
|
||
|
||
def sql(text: str, **variables: str) -> str:
|
||
return psql(text, ON_ERROR_STOP="1", **variables)
|
||
|
||
|
||
def object_status(key: str) -> int:
|
||
found = compose(
|
||
"exec", "-T", "-e", f"KEY={key}", "-e", f"BUCKET={BUCKET}", "s3", "sh", "-c", HEAD
|
||
)
|
||
return int(found)
|
||
|
||
|
||
def bucket_keys() -> set[str]:
|
||
listing = compose(
|
||
"run",
|
||
"--rm",
|
||
"--no-deps",
|
||
"-T",
|
||
"s3-cli",
|
||
"object",
|
||
"list",
|
||
"--recursive",
|
||
"--json",
|
||
f"root/{BUCKET}/staging/",
|
||
)
|
||
return {item["key"] for item in json.loads(listing)["items"]}
|
||
|
||
|
||
def seen(name: str) -> tuple[str, str] | None:
|
||
row = sql("SELECT id, object_key FROM lab4_fault_seen WHERE filename = :'name'", name=name)
|
||
if not row:
|
||
return None
|
||
attachment_id, key = row.split("|")
|
||
return attachment_id, key
|
||
|
||
|
||
def status_of(attachment_id: str) -> str:
|
||
return sql("SELECT status FROM attachments WHERE id = :'id'", id=attachment_id)
|
||
|
||
|
||
def logs(since: str) -> list[dict[str, Any]]:
|
||
out = compose("logs", "api", "--since", since, "--no-log-prefix")
|
||
return [json.loads(line) for line in out.splitlines() if line.startswith("{")]
|
||
|
||
|
||
def moment() -> str:
|
||
return (datetime.now(UTC) - timedelta(seconds=1)).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||
|
||
|
||
def kill(container: str) -> None:
|
||
subprocess.run(["docker", "kill", "-s", "KILL", container], check=True, capture_output=True)
|
||
|
||
|
||
def replica_at(address: str) -> str:
|
||
ids = compose("ps", "-q", "api").split()
|
||
template = "{{.Id}} {{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}"
|
||
found = subprocess.run(
|
||
["docker", "inspect", "-f", template, *ids], check=True, capture_output=True, text=True
|
||
).stdout.split("\n")
|
||
return next(line.split()[0] for line in found if line.endswith(f" {address}"))
|
||
|
||
|
||
def raw(
|
||
path: str,
|
||
headers: dict[str, str],
|
||
body: bytes,
|
||
*,
|
||
hang_up: bool,
|
||
piece: int = 0,
|
||
delay: float = 0.0,
|
||
) -> tuple[int, dict[str, Any], float]:
|
||
parts = urlsplit(BASE_URL)
|
||
head = f"POST {path} HTTP/1.1\r\nHost: {parts.netloc}\r\nConnection: close\r\n"
|
||
head += "".join(f"{name}: {value}\r\n" for name, value in headers.items()) + "\r\n"
|
||
started = time.perf_counter()
|
||
with socket.create_connection((parts.hostname, parts.port or 80), timeout=30) as sock:
|
||
sock.sendall(head.encode("latin-1"))
|
||
for start in range(0, len(body), piece or max(len(body), 1)):
|
||
sock.sendall(body[start : start + (piece or len(body))])
|
||
time.sleep(delay)
|
||
if hang_up:
|
||
return 0, {}, 0.0
|
||
response = HTTPResponse(sock)
|
||
response.begin()
|
||
return response.status, json.loads(response.read()), time.perf_counter() - started
|
||
|
||
|
||
READER = """
|
||
import hashlib, http.client, sys, time
|
||
path, token, request_id, piece, delay = sys.argv[1:6]
|
||
connection = http.client.HTTPConnection("lb", 8080, timeout=30)
|
||
headers = {"Authorization": "Bearer " + token, "X-Request-Id": request_id}
|
||
connection.request("GET", path, headers=headers)
|
||
response, digest, started = connection.getresponse(), hashlib.sha256(), time.monotonic()
|
||
while chunk := response.read(int(piece)):
|
||
digest.update(chunk)
|
||
time.sleep(float(delay))
|
||
print(response.status, digest.hexdigest(), round(time.monotonic() - started, 2))
|
||
"""
|
||
|
||
|
||
def slow_read(path: str, token: str, request_id: str, piece: int, delay: float) -> list[str]:
|
||
template = "{{range $name, $_ := .NetworkSettings.Networks}}{{$name}} {{end}}"
|
||
networks = subprocess.run(
|
||
["docker", "inspect", "-f", template, compose("ps", "-q", "lb")],
|
||
check=True,
|
||
capture_output=True,
|
||
text=True,
|
||
).stdout.split()
|
||
image = subprocess.run(
|
||
["docker", "inspect", "-f", "{{.Config.Image}}", compose("ps", "-q", "api").split()[0]],
|
||
check=True,
|
||
capture_output=True,
|
||
text=True,
|
||
).stdout.strip()
|
||
command = [
|
||
"docker",
|
||
"run",
|
||
"--rm",
|
||
"--network",
|
||
next(name for name in networks if name.endswith("_edge")),
|
||
"--sysctl",
|
||
"net.ipv4.tcp_rmem=4096 16384 65536",
|
||
"--entrypoint",
|
||
"python",
|
||
image,
|
||
"-c",
|
||
READER,
|
||
path,
|
||
token,
|
||
request_id,
|
||
str(piece),
|
||
str(delay),
|
||
]
|
||
return subprocess.run(command, check=True, capture_output=True, text=True).stdout.split()
|
||
|
||
|
||
@pytest.fixture
|
||
async def http() -> AsyncIterator[AsyncClient]:
|
||
async with AsyncClient(base_url=BASE_URL, timeout=30, trust_env=False) as client:
|
||
yield client
|
||
|
||
|
||
@pytest.fixture(scope="module")
|
||
def faults() -> Iterator[None]:
|
||
sql(CLEANUP + FAULTS)
|
||
try:
|
||
yield
|
||
finally:
|
||
sql(CLEANUP)
|
||
|
||
|
||
async def scene(http: AsyncClient) -> dict[str, Any]:
|
||
(alice, alice_token), (bob, bob_token), (eve, eve_token) = [
|
||
await enroll(http, label) for label in ("alice", "bob", "eve")
|
||
]
|
||
chats = []
|
||
for owner, token, members in (
|
||
(alice, alice_token, [bob]),
|
||
(alice, alice_token, []),
|
||
(eve, eve_token, []),
|
||
):
|
||
created = await http.post(
|
||
"/api/v1/chats",
|
||
json={"title": "Стенд", "member_ids": [owner["id"], *(m["id"] for m in members)]},
|
||
headers=bearer(token),
|
||
)
|
||
assert created.status_code == 201
|
||
chats.append(created.json()["id"])
|
||
return {"alice": alice_token, "bob": bob_token, "eve": eve_token, "chats": chats}
|
||
|
||
|
||
async def upload(
|
||
http: AsyncClient,
|
||
token: str,
|
||
chat: str,
|
||
body: bytes | AsyncIterator[bytes] = SAMPLE,
|
||
*,
|
||
filename: str | None = "sample.png",
|
||
kind: str | None = "image/png",
|
||
) -> Response:
|
||
query = "" if filename is None else f"?filename={quote(filename, safe='')}"
|
||
headers = {**bearer(token), **({"Content-Type": kind} if kind else {})}
|
||
return await http.post(
|
||
f"/api/v1/chats/{chat}/attachments{query}", content=body, headers=headers
|
||
)
|
||
|
||
|
||
async def message(http: AsyncClient, token: str, chat: str, body: dict[str, Any]) -> Response:
|
||
headers = {**bearer(token), "Idempotency-Key": uuid4().hex}
|
||
return await http.post(f"/api/v1/chats/{chat}/messages", json=body, headers=headers)
|
||
|
||
|
||
def expect(label: str, response: Response, status: int) -> None:
|
||
print(f"{label}: {response.status_code}")
|
||
assert response.status_code == status, f"{label}: {response.status_code} {response.text}"
|
||
assert response.headers["X-Instance-Id"]
|
||
error = response.json()["error"]
|
||
assert error["code"] == CODES[status]
|
||
assert error["request_id"] == response.headers["X-Request-Id"]
|
||
|
||
|
||
async def test_user_errors_never_become_500(http: AsyncClient) -> None:
|
||
since = moment()
|
||
s = await scene(http)
|
||
alice, eve, (chat, solo, other) = s["alice"], s["eve"], s["chats"]
|
||
created = 0
|
||
attachment = await upload(http, alice, chat)
|
||
exact = await upload(http, alice, chat, png(16, 16, size=LIMIT))
|
||
assert (attachment.status_code, exact.status_code) == (202, 202)
|
||
assert exact.json()["size_bytes"] == LIMIT
|
||
created += 2
|
||
mine = attachment.json()["id"]
|
||
foreign = (await upload(http, alice, solo)).json()["id"]
|
||
created_elsewhere = 1
|
||
oversize = png(16, 16, size=LIMIT + 1)
|
||
|
||
async def chunked() -> AsyncIterator[bytes]:
|
||
for start in range(0, len(oversize), 1 << 20):
|
||
yield oversize[start : start + (1 << 20)]
|
||
|
||
path = f"/api/v1/chats/{chat}/attachments?filename=big.png"
|
||
status, body, _ = await asyncio.to_thread(
|
||
raw,
|
||
path,
|
||
{**bearer(alice), "Content-Type": "image/png", "Content-Length": str(LIMIT + 1)},
|
||
b"",
|
||
hang_up=False,
|
||
)
|
||
print(f"10 MiB + 1 с Content-Length, тело не отправлено: {status}")
|
||
assert (status, body["error"]["code"]) == (413, "payload_too_large")
|
||
expect("10 MiB + 1 chunked", await upload(http, alice, chat, chunked()), 413)
|
||
expect(
|
||
"PNG 100000×100000 (бомба)",
|
||
await upload(http, alice, chat, png(100_000, 100_000, rows=False)),
|
||
413,
|
||
)
|
||
expect("PNG 8001×5000", await upload(http, alice, chat, png(8001, 5000, rows=False)), 413)
|
||
for label, content, kind in (
|
||
("обрезанный PNG", SAMPLE[: len(SAMPLE) // 2], "image/png"),
|
||
("обрезанный JPEG", JPEG[: len(JPEG) // 2], "image/jpeg"),
|
||
("JPEG как image/png", JPEG, "image/png"),
|
||
("GIF как image/png", GIF, "image/png"),
|
||
("WebP как image/png", WEBP, "image/png"),
|
||
("текст как image/png", b"plain text, not an image", "image/png"),
|
||
("PNG как image/jpeg", SAMPLE, "image/jpeg"),
|
||
("пустое тело", b"", "image/png"),
|
||
("application/octet-stream", SAMPLE, "application/octet-stream"),
|
||
("без Content-Type", SAMPLE, None),
|
||
):
|
||
expect(label, await upload(http, alice, chat, content, kind=kind), 415)
|
||
for name in (
|
||
"../a.png",
|
||
"a/b.png",
|
||
"a\\b.png",
|
||
"a\x00.png",
|
||
"a\x01.png",
|
||
"a\x7f.png",
|
||
"",
|
||
" ",
|
||
"x" * 256,
|
||
None,
|
||
):
|
||
expect(f"filename={name!r}"[:40], await upload(http, alice, chat, filename=name), 400)
|
||
expect("посторонний: upload в чужой чат", await upload(http, eve, chat), 404)
|
||
expect(
|
||
"посторонний: метаданные",
|
||
await http.get(f"/api/v1/attachments/{mine}", headers=bearer(eve)),
|
||
404,
|
||
)
|
||
expect(
|
||
"посторонний: /content",
|
||
await http.get(f"/api/v1/attachments/{mine}/content", headers=bearer(eve)),
|
||
404,
|
||
)
|
||
expect(
|
||
"посторонний: ссылка на чужое вложение",
|
||
await message(http, eve, other, {"text": "x", "attachment_ids": [mine]}),
|
||
404,
|
||
)
|
||
expect(
|
||
"вложение другого чата",
|
||
await message(http, alice, chat, {"text": "x", "attachment_ids": [foreign]}),
|
||
404,
|
||
)
|
||
expect(
|
||
"неизвестное вложение",
|
||
await message(http, alice, chat, {"text": "x", "attachment_ids": [str(uuid4())]}),
|
||
404,
|
||
)
|
||
for label, ids in (
|
||
("11 вложений", [mine] + [str(uuid4()) for _ in range(10)]),
|
||
("повтор вложения", [mine, mine]),
|
||
("невалидный UUID", ["not-a-uuid"]),
|
||
):
|
||
expect(label, await message(http, alice, chat, {"text": "x", "attachment_ids": ids}), 400)
|
||
expect(
|
||
"пустой текст без вложений",
|
||
await message(http, alice, chat, {"text": "", "attachment_ids": []}),
|
||
400,
|
||
)
|
||
expect(
|
||
"невалидный id вложения",
|
||
await http.get("/api/v1/attachments/x/content", headers=bearer(alice)),
|
||
400,
|
||
)
|
||
expect(
|
||
"variant=preview",
|
||
await http.get(
|
||
f"/api/v1/attachments/{mine}/content?variant=preview", headers=bearer(alice)
|
||
),
|
||
400,
|
||
)
|
||
rows = sql("SELECT count(*) FROM attachments WHERE chat_id = :'chat'", chat=chat)
|
||
assert int(rows) == created
|
||
assert sql("SELECT count(*) FROM attachments WHERE chat_id = :'chat'", chat=solo) == str(
|
||
created_elsewhere
|
||
)
|
||
statuses = [entry["status"] for entry in logs(since) if entry.get("event") == "request.handled"]
|
||
print(
|
||
f"строк вложений в чате: {rows} (= ответов 202); ответов в журнале реплик: "
|
||
f"{len(statuses)}, из них 5xx: {sum(code >= 500 for code in statuses)}"
|
||
)
|
||
assert not [status for status in statuses if status >= 500]
|
||
|
||
|
||
async def test_commit_failure_is_compensated_at_once(http: AsyncClient, faults: None) -> None:
|
||
since = moment()
|
||
s = await scene(http)
|
||
name = f"fault-fail-{uuid4().hex}.png"
|
||
response = await upload(http, s["alice"], s["chats"][0], filename=name)
|
||
print(
|
||
"отказ COMMIT после записи объекта:",
|
||
response.status_code,
|
||
response.json()["error"]["code"],
|
||
)
|
||
assert response.status_code == 503
|
||
found = seen(name)
|
||
assert found is not None
|
||
attachment_id, key = found
|
||
print(
|
||
f"сразу после ответа: строка {status_of(attachment_id) or 'нет'},",
|
||
f"объект HEAD {object_status(key)}",
|
||
)
|
||
assert status_of(attachment_id) == ""
|
||
assert object_status(key) == 404
|
||
|
||
def journaled() -> bool:
|
||
return any(
|
||
entry.get("event") == "audit.attachment_discarded"
|
||
and entry.get("attachment_id") == attachment_id
|
||
and entry.get("reason") == "compensation"
|
||
for entry in logs(since)
|
||
)
|
||
|
||
await waited(journaled, 10)
|
||
print("журнал: audit.attachment_discarded reason=compensation")
|
||
|
||
|
||
async def crash(http: AsyncClient, token: str, chat: str) -> tuple[str, str]:
|
||
name = f"fault-crash-{uuid4().hex}.png"
|
||
sent = asyncio.create_task(upload(http, token, chat, filename=name))
|
||
address = await asyncio.to_thread(sql, "SELECT lab4_fault_sleeper()")
|
||
assert address, "UPDATE queued не дошёл до триггера"
|
||
victim = replica_at(address)
|
||
kill(victim)
|
||
try:
|
||
outcome = str((await sent).status_code)
|
||
except httpx.HTTPError as error:
|
||
outcome = type(error).__name__
|
||
compose("up", "-d", "--wait", "--no-deps", "--no-recreate", "api")
|
||
await waited(lambda: lb_up() == 2, 60)
|
||
found = seen(name)
|
||
assert found is not None
|
||
print(f"реплика {victim[:12]} убита между PUT и COMMIT, клиент получил {outcome}")
|
||
return found
|
||
|
||
|
||
async def test_crash_between_put_and_commit_is_swept_after_grace(
|
||
http: AsyncClient, faults: None
|
||
) -> None:
|
||
s = await scene(http)
|
||
alice, chat = s["alice"], s["chats"][0]
|
||
kept = (await upload(http, alice, chat, filename="kept.png")).json()
|
||
await waited(lambda: status_of(kept["id"]) == "ready", 60)
|
||
assert (
|
||
await message(http, alice, chat, {"text": "", "attachment_ids": [kept["id"]]})
|
||
).status_code == 201
|
||
old, fresh = await crash(http, alice, chat), await crash(http, alice, chat)
|
||
for attachment_id, key in (old, fresh):
|
||
state = (status_of(attachment_id), object_status(key))
|
||
print(f"остаток после падения: строка {state[0]}, объект HEAD {state[1]}")
|
||
assert state == ("pending", 200)
|
||
aged = "UPDATE attachments SET created_at = created_at - interval '2 hours' WHERE id = :'id'"
|
||
sql(aged, id=old[0])
|
||
sql(aged, id=kept["id"])
|
||
swept = await waited(lambda: status_of(old[0]) == "", SWEEP_WAIT)
|
||
print(f"старше grace: строка убрана через {swept:.1f} с, объект HEAD {object_status(old[1])}")
|
||
assert object_status(old[1]) == 404
|
||
print(f"моложе grace: строка {status_of(fresh[0])}, объект HEAD {object_status(fresh[1])}")
|
||
assert (status_of(fresh[0]), object_status(fresh[1])) == ("pending", 200)
|
||
sql(aged, id=fresh[0])
|
||
swept = await waited(lambda: status_of(fresh[0]) == "", SWEEP_WAIT)
|
||
print(f"тот же после сдвига: убран через {swept:.1f} с, объект HEAD {object_status(fresh[1])}")
|
||
assert object_status(fresh[1]) == 404
|
||
content = await http.get(f"/api/v1/attachments/{kept['id']}/content", headers=bearer(s["bob"]))
|
||
assert content.status_code == 200
|
||
assert sha256(content.content).hexdigest() == kept["sha256"]
|
||
print("готовое вложение в сообщении со сдвинутым created_at скачивается, SHA-256 совпадает")
|
||
|
||
|
||
async def test_aborted_upload_leaves_nothing(http: AsyncClient, faults: None) -> None:
|
||
since = moment()
|
||
s = await scene(http)
|
||
name, request_id = f"fault-abort-{uuid4().hex}.png", f"abort-{uuid4().hex}"
|
||
before = bucket_keys()
|
||
headers = {
|
||
**bearer(s["alice"]),
|
||
"Content-Type": "image/png",
|
||
"Content-Length": str(5 << 20),
|
||
"X-Request-Id": request_id,
|
||
}
|
||
path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename={name}"
|
||
await asyncio.to_thread(raw, path, headers, SAMPLE[:8] + bytes(256 << 10), hang_up=True)
|
||
|
||
def handled() -> list[int]:
|
||
return [
|
||
e["status"]
|
||
for e in logs(since)
|
||
if e.get("request_id") == request_id and e.get("event") == "request.handled"
|
||
]
|
||
|
||
await waited(lambda: bool(handled()), 15)
|
||
print(
|
||
f"обрыв после 256 KiB из 5 MiB: статус в журнале реплики {handled()},",
|
||
f"намерение {seen(name)}",
|
||
)
|
||
assert handled() == [400]
|
||
assert seen(name) is None
|
||
assert bucket_keys() == before
|
||
|
||
|
||
async def test_stalled_upload_is_cut_and_leaves_nothing(http: AsyncClient, faults: None) -> None:
|
||
s = await scene(http)
|
||
name = f"fault-stall-{uuid4().hex}.png"
|
||
headers = {**bearer(s["alice"]), "Content-Type": "image/png", "Content-Length": str(1 << 20)}
|
||
path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename={name}"
|
||
status, body, elapsed = await asyncio.to_thread(
|
||
raw, path, headers, SAMPLE[:1024], hang_up=False
|
||
)
|
||
print(f"тело зависло после 1 KiB: {status} {body['error']['code']} за {elapsed:.2f} с")
|
||
assert (status, body["error"]["code"]) == (503, "unavailable")
|
||
assert elapsed <= REQUEST_TIMEOUT + 2
|
||
assert seen(name) is None
|
||
assert (await upload(http, s["alice"], s["chats"][0])).status_code == 202
|
||
|
||
|
||
async def test_token_is_checked_before_the_declared_size(http: AsyncClient) -> None:
|
||
s = await scene(http)
|
||
headers = {"Content-Type": "image/png", "Content-Length": str(LIMIT + 1)}
|
||
path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename=a.png"
|
||
status, body, _ = await asyncio.to_thread(raw, path, headers, SAMPLE[:1024], hang_up=False)
|
||
print(f"без токена, Content-Length {LIMIT + 1}: {status} {body['error']['code']}")
|
||
assert (status, body["error"]["code"]) == (401, "unauthorized")
|
||
|
||
|
||
async def test_slow_but_steady_transfers_outlive_the_request_timeout(http: AsyncClient) -> None:
|
||
since = moment()
|
||
s = await scene(http)
|
||
picture = png(16, 16, size=3 << 20)
|
||
sent, read = f"slow-upload-{uuid4().hex}", f"slow-download-{uuid4().hex}"
|
||
headers = {
|
||
**bearer(s["alice"]),
|
||
"Content-Type": "image/png",
|
||
"Content-Length": str(len(picture)),
|
||
"X-Request-Id": sent,
|
||
}
|
||
path = f"/api/v1/chats/{s['chats'][0]}/attachments?filename=slow.png"
|
||
status, body, up = await asyncio.to_thread(
|
||
raw, path, headers, picture, hang_up=False, piece=64 << 10, delay=0.25
|
||
)
|
||
assert status == 202, body
|
||
assert body["sha256"] == sha256(picture).hexdigest()
|
||
await waited(lambda: status_of(body["id"]) == "ready", 60)
|
||
content = f"/api/v1/attachments/{body['id']}/content"
|
||
code, digest, down = await asyncio.to_thread(slow_read, content, s["bob"], read, 64 << 10, 0.33)
|
||
assert (code, digest) == ("200", body["sha256"])
|
||
|
||
def handled() -> dict[str, tuple[int, float]]:
|
||
return {
|
||
e["request_id"]: (e["status"], e["duration_ms"] / 1000)
|
||
for e in logs(since)
|
||
if e.get("event") == "request.handled" and e.get("request_id") in (sent, read)
|
||
}
|
||
|
||
await waited(lambda: len(handled()) == 2, 15)
|
||
replica = handled()
|
||
print(f"upload 3 MiB ≈ 256 KiB/s: клиент {up:.1f} с, реплика {replica[sent]}")
|
||
print(f"скачивание 3 MiB ≈ 192 KiB/s из сети edge: клиент {down} с, реплика {replica[read]}")
|
||
assert replica[sent][0] == 202 and replica[sent][1] > REQUEST_TIMEOUT
|
||
assert replica[read][0] == 200 and replica[read][1] > REQUEST_TIMEOUT
|