Перенесён стенд lab3 (API, Redis-сессии, идемпотентность, HAProxy, реплики API). Добавлены приватный bucket SeaweedFS с отдельным ключом API, загрузка PNG/JPEG с лимитами и полной проверкой, компенсация и reconciler, плавная остановка API, тесты и evidence.
163 lines
7.0 KiB
Python
163 lines
7.0 KiB
Python
"""Lab4 grade 4 operator checks that need the Docker CLI (RUN_DOCKER_TESTS=1).
|
||
|
||
Non-destructive: nothing is stopped; checks run inside the API containers.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import concurrent.futures
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import threading
|
||
import time
|
||
|
||
import imagegen
|
||
from helpers import call, chat_for, register, upload
|
||
from test_attachments import _raw_partial_upload
|
||
|
||
ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||
|
||
|
||
def _enabled(name: str) -> bool:
|
||
if os.environ.get("RUN_DOCKER_TESTS") != "1":
|
||
print(f"SKIP {name} (set RUN_DOCKER_TESTS=1)")
|
||
return False
|
||
return True
|
||
|
||
|
||
def _run(args, check=True):
|
||
proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8")
|
||
if check and proc.returncode != 0:
|
||
raise AssertionError(f"{' '.join(args[:6])} rc={proc.returncode}: {proc.stdout[-800:]}{proc.stderr[-400:]}")
|
||
return proc
|
||
|
||
|
||
def _api_containers():
|
||
out = _run(["docker", "compose", "--env-file", ".env", "ps", "-q", "api"]).stdout.split()
|
||
assert len(out) >= 2, out
|
||
return out
|
||
|
||
|
||
def _exec_json(container: str, *args):
|
||
proc = _run(["docker", "exec", container, "python", "-m", *args])
|
||
return json.loads(proc.stdout[proc.stdout.index("{"):])
|
||
|
||
|
||
def _bucket_state(container: str) -> dict:
|
||
return _exec_json(container, "messenger.reconcile", "--dry-run", "--grace-seconds", "0")
|
||
|
||
|
||
def test_bucket_private_and_api_key_least_privilege():
|
||
if not _enabled("test_bucket_private_and_api_key_least_privilege"):
|
||
return
|
||
report = _exec_json(_api_containers()[0], "messenger.selfcheck", "s3")
|
||
failed = [r for r in report["results"] if not r["ok"]]
|
||
assert report["passed"] and not failed, failed
|
||
|
||
|
||
def test_s3_db_failure_compensation_and_reconcile():
|
||
if not _enabled("test_s3_db_failure_compensation_and_reconcile"):
|
||
return
|
||
report = _exec_json(_api_containers()[0], "messenger.selfcheck", "consistency")
|
||
failed = [r for r in report["results"] if not r["ok"]]
|
||
assert report["passed"] and not failed, failed
|
||
|
||
|
||
def test_interrupted_and_stalled_uploads_leave_nothing():
|
||
if not _enabled("test_interrupted_and_stalled_uploads_leave_nothing"):
|
||
return
|
||
containers = _api_containers()
|
||
alice, bob = register("cut_a"), register("cut_b")
|
||
chat = chat_for(alice, bob)
|
||
before = _bucket_state(containers[0])
|
||
big = imagegen.large_png(3 * 1024 * 1024)
|
||
for _ in range(4):
|
||
_raw_partial_upload(alice["token"], chat["id"], len(big), big[: 512 * 1024])
|
||
# Stalled client: headers + part of body, then silence past UPLOAD_IDLE_TIMEOUT_SECONDS (10 s).
|
||
_raw_partial_upload(alice["token"], chat["id"], len(big), big[: 256 * 1024], hold=12)
|
||
time.sleep(1)
|
||
after = _bucket_state(containers[0])
|
||
assert after["objects_scanned"] == before["objects_scanned"], (before, after)
|
||
# Leftovers of earlier S3-outage runs (intents kept for the reconciler) are not ours.
|
||
assert after["stale_intents"] == before["stale_intents"], (before, after)
|
||
assert after["orphans_found"] == before["orphans_found"], (before, after)
|
||
for container in containers:
|
||
spool = _exec_json(container, "messenger.selfcheck", "spool")
|
||
assert spool["passed"], (container, spool)
|
||
att = upload(alice["token"], chat["id"], imagegen.rgb_png(8, 8), "ok.png", "image/png", 201)["json"]
|
||
call("GET", "/api/v1/attachments/" + att["id"] + "/content", 200, token=bob["token"])
|
||
|
||
|
||
def test_parallel_large_uploads_keep_s3_within_memory():
|
||
# 2 replicas × 8 slots = 16 PUTs in flight; SeaweedFS holds ≈30–40 MiB per 10 MiB PUT.
|
||
# With mem_limit 512m it ran at the limit and was OOM-killed (exit 137) under 40 uploads.
|
||
if not _enabled("test_parallel_large_uploads_keep_s3_within_memory"):
|
||
return
|
||
s3 = _run(["docker", "compose", "--env-file", ".env", "ps", "-q", "s3"]).stdout.split()[0]
|
||
limit = int(_run(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", s3]).stdout.strip())
|
||
alice = register("par_a")
|
||
chat = chat_for(alice)
|
||
payload = imagegen.large_png(int(9.5 * 1024 * 1024))
|
||
peak = [0]
|
||
stop = threading.Event()
|
||
|
||
def sample():
|
||
while not stop.is_set():
|
||
stat = _run(["docker", "exec", s3, "cat", "/sys/fs/cgroup/memory.stat"], check=False).stdout
|
||
for line in stat.splitlines():
|
||
if line.startswith("anon "):
|
||
peak[0] = max(peak[0], int(line.split()[1]))
|
||
time.sleep(0.1)
|
||
|
||
def send(i):
|
||
return call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=par{i}.png", token=alice["token"],
|
||
raw=payload, headers={"Content-Type": "image/png"})["status"]
|
||
|
||
sampler = threading.Thread(target=sample)
|
||
sampler.start()
|
||
try:
|
||
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool:
|
||
statuses = list(pool.map(send, range(16)))
|
||
finally:
|
||
time.sleep(1)
|
||
stop.set()
|
||
sampler.join()
|
||
state = _run(["docker", "inspect", "-f", "{{.State.Running}} {{.State.OOMKilled}}", s3]).stdout.split()
|
||
assert state == ["true", "false"], state
|
||
# 503 is the documented backpressure (slot wait > 5 s), never an S3 failure.
|
||
assert set(statuses) <= {201, 503} and statuses.count(201) >= 10, statuses
|
||
assert peak[0] < 0.8 * limit, (peak[0] // 2**20, limit // 2**20)
|
||
call("GET", "/health/ready", 200)
|
||
|
||
|
||
def test_s3_can_grow_volumes():
|
||
# SeaweedFS grows a collection by 7 volumes at a time; when no slot is left every PUT
|
||
# fails ("No writable volumes") while readiness stays 200. -volume.max=16 hit that at ≈4.7 GiB.
|
||
if not _enabled("test_s3_can_grow_volumes"):
|
||
return
|
||
s3 = _run(["docker", "compose", "--env-file", ".env", "ps", "-q", "s3"]).stdout.split()[0]
|
||
status = json.loads(_run(["docker", "exec", s3, "wget", "-q", "-O", "-", "http://127.0.0.1:9333/dir/status"]).stdout)
|
||
topology = status["Topology"]
|
||
assert topology["Free"] >= 8, {k: topology[k] for k in ("Max", "Free")}
|
||
|
||
|
||
def test_no_original_kept_on_api_replicas():
|
||
if not _enabled("test_no_original_kept_on_api_replicas"):
|
||
return
|
||
alice, bob = register("loc_a"), register("loc_b")
|
||
chat = chat_for(alice, bob)
|
||
for i in range(4):
|
||
upload(alice["token"], chat["id"], imagegen.rgb_png(10 + i, 10), f"l{i}.png", "image/png", 201)
|
||
for container in _api_containers():
|
||
spool = _exec_json(container, "messenger.selfcheck", "spool")
|
||
assert spool["passed"], (container, spool)
|
||
# Any regular file created after the container started, anywhere on its writable layer
|
||
# (/etc/hosts, /etc/resolv.conf, /etc/hostname are bind mounts written by Docker itself).
|
||
started = _run(["docker", "inspect", "-f", "{{.State.StartedAt}}", container]).stdout.strip()
|
||
found = _run(["docker", "exec", "-u", "0", container, "find", "/", "-xdev", "-type", "f",
|
||
"-newermt", started, "-not", "-path", "/proc/*",
|
||
"-not", "-path", "/etc/hosts", "-not", "-path", "/etc/resolv.conf",
|
||
"-not", "-path", "/etc/hostname"]).stdout.split()
|
||
assert not found, (container, found[:5])
|