Files
mtusi2026lab4/student-tests/test_attachments_ops.py
T
Mikhail Verkovykh 728e53cc55 Реализовать lab4 на оценку 4: SeaweedFS, вложения и согласованность БД/S3.
Перенесён стенд lab3 (API, Redis-сессии, идемпотентность, HAProxy, реплики API). Добавлены приватный bucket SeaweedFS с отдельным ключом API, загрузка PNG/JPEG с лимитами и полной проверкой, компенсация и reconciler, плавная остановка API, тесты и evidence.
2026-11-06 19:48:37 +03:00

163 lines
7.0 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Lab4 grade 4 operator checks that need the Docker CLI (RUN_DOCKER_TESTS=1).
Non-destructive: nothing is stopped; checks run inside the API containers.
"""
from __future__ import annotations
import concurrent.futures
import json
import os
import subprocess
import threading
import time
import imagegen
from helpers import call, chat_for, register, upload
from test_attachments import _raw_partial_upload
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
def _enabled(name: str) -> bool:
if os.environ.get("RUN_DOCKER_TESTS") != "1":
print(f"SKIP {name} (set RUN_DOCKER_TESTS=1)")
return False
return True
def _run(args, check=True):
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8")
if check and proc.returncode != 0:
raise AssertionError(f"{' '.join(args[:6])} rc={proc.returncode}: {proc.stdout[-800:]}{proc.stderr[-400:]}")
return proc
def _api_containers():
out = _run(["docker", "compose", "--env-file", ".env", "ps", "-q", "api"]).stdout.split()
assert len(out) >= 2, out
return out
def _exec_json(container: str, *args):
proc = _run(["docker", "exec", container, "python", "-m", *args])
return json.loads(proc.stdout[proc.stdout.index("{"):])
def _bucket_state(container: str) -> dict:
return _exec_json(container, "messenger.reconcile", "--dry-run", "--grace-seconds", "0")
def test_bucket_private_and_api_key_least_privilege():
if not _enabled("test_bucket_private_and_api_key_least_privilege"):
return
report = _exec_json(_api_containers()[0], "messenger.selfcheck", "s3")
failed = [r for r in report["results"] if not r["ok"]]
assert report["passed"] and not failed, failed
def test_s3_db_failure_compensation_and_reconcile():
if not _enabled("test_s3_db_failure_compensation_and_reconcile"):
return
report = _exec_json(_api_containers()[0], "messenger.selfcheck", "consistency")
failed = [r for r in report["results"] if not r["ok"]]
assert report["passed"] and not failed, failed
def test_interrupted_and_stalled_uploads_leave_nothing():
if not _enabled("test_interrupted_and_stalled_uploads_leave_nothing"):
return
containers = _api_containers()
alice, bob = register("cut_a"), register("cut_b")
chat = chat_for(alice, bob)
before = _bucket_state(containers[0])
big = imagegen.large_png(3 * 1024 * 1024)
for _ in range(4):
_raw_partial_upload(alice["token"], chat["id"], len(big), big[: 512 * 1024])
# Stalled client: headers + part of body, then silence past UPLOAD_IDLE_TIMEOUT_SECONDS (10 s).
_raw_partial_upload(alice["token"], chat["id"], len(big), big[: 256 * 1024], hold=12)
time.sleep(1)
after = _bucket_state(containers[0])
assert after["objects_scanned"] == before["objects_scanned"], (before, after)
# Leftovers of earlier S3-outage runs (intents kept for the reconciler) are not ours.
assert after["stale_intents"] == before["stale_intents"], (before, after)
assert after["orphans_found"] == before["orphans_found"], (before, after)
for container in containers:
spool = _exec_json(container, "messenger.selfcheck", "spool")
assert spool["passed"], (container, spool)
att = upload(alice["token"], chat["id"], imagegen.rgb_png(8, 8), "ok.png", "image/png", 201)["json"]
call("GET", "/api/v1/attachments/" + att["id"] + "/content", 200, token=bob["token"])
def test_parallel_large_uploads_keep_s3_within_memory():
# 2 replicas × 8 slots = 16 PUTs in flight; SeaweedFS holds ≈30–40 MiB per 10 MiB PUT.
# With mem_limit 512m it ran at the limit and was OOM-killed (exit 137) under 40 uploads.
if not _enabled("test_parallel_large_uploads_keep_s3_within_memory"):
return
s3 = _run(["docker", "compose", "--env-file", ".env", "ps", "-q", "s3"]).stdout.split()[0]
limit = int(_run(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", s3]).stdout.strip())
alice = register("par_a")
chat = chat_for(alice)
payload = imagegen.large_png(int(9.5 * 1024 * 1024))
peak = [0]
stop = threading.Event()
def sample():
while not stop.is_set():
stat = _run(["docker", "exec", s3, "cat", "/sys/fs/cgroup/memory.stat"], check=False).stdout
for line in stat.splitlines():
if line.startswith("anon "):
peak[0] = max(peak[0], int(line.split()[1]))
time.sleep(0.1)
def send(i):
return call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=par{i}.png", token=alice["token"],
raw=payload, headers={"Content-Type": "image/png"})["status"]
sampler = threading.Thread(target=sample)
sampler.start()
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool:
statuses = list(pool.map(send, range(16)))
finally:
time.sleep(1)
stop.set()
sampler.join()
state = _run(["docker", "inspect", "-f", "{{.State.Running}} {{.State.OOMKilled}}", s3]).stdout.split()
assert state == ["true", "false"], state
# 503 is the documented backpressure (slot wait > 5 s), never an S3 failure.
assert set(statuses) <= {201, 503} and statuses.count(201) >= 10, statuses
assert peak[0] < 0.8 * limit, (peak[0] // 2**20, limit // 2**20)
call("GET", "/health/ready", 200)
def test_s3_can_grow_volumes():
# SeaweedFS grows a collection by 7 volumes at a time; when no slot is left every PUT
# fails ("No writable volumes") while readiness stays 200. -volume.max=16 hit that at ≈4.7 GiB.
if not _enabled("test_s3_can_grow_volumes"):
return
s3 = _run(["docker", "compose", "--env-file", ".env", "ps", "-q", "s3"]).stdout.split()[0]
status = json.loads(_run(["docker", "exec", s3, "wget", "-q", "-O", "-", "http://127.0.0.1:9333/dir/status"]).stdout)
topology = status["Topology"]
assert topology["Free"] >= 8, {k: topology[k] for k in ("Max", "Free")}
def test_no_original_kept_on_api_replicas():
if not _enabled("test_no_original_kept_on_api_replicas"):
return
alice, bob = register("loc_a"), register("loc_b")
chat = chat_for(alice, bob)
for i in range(4):
upload(alice["token"], chat["id"], imagegen.rgb_png(10 + i, 10), f"l{i}.png", "image/png", 201)
for container in _api_containers():
spool = _exec_json(container, "messenger.selfcheck", "spool")
assert spool["passed"], (container, spool)
# Any regular file created after the container started, anywhere on its writable layer
# (/etc/hosts, /etc/resolv.conf, /etc/hostname are bind mounts written by Docker itself).
started = _run(["docker", "inspect", "-f", "{{.State.StartedAt}}", container]).stdout.strip()
found = _run(["docker", "exec", "-u", "0", container, "find", "/", "-xdev", "-type", "f",
"-newermt", started, "-not", "-path", "/proc/*",
"-not", "-path", "/etc/hosts", "-not", "-path", "/etc/resolv.conf",
"-not", "-path", "/etc/hostname"]).stdout.split()
assert not found, (container, found[:5])