forked from mtusi2026/mtusi2026lab4
Хранить вложения в приватном S3 и сверять их с PostgreSQL.
This commit is contained in:
@@ -0,0 +1,171 @@
|
||||
"""Ten concurrent uploads at two sizes. Records container RSS, not a universal capacity claim."""
|
||||
|
||||
import json
|
||||
import random
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client # noqa: E402
|
||||
|
||||
|
||||
def chunk(tag: bytes, data: bytes) -> bytes:
|
||||
crc = zlib.crc32(tag + data) & 0xFFFFFFFF
|
||||
return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", crc)
|
||||
|
||||
|
||||
def make_png(width: int, height: int, seed: int) -> bytes:
|
||||
rng = random.Random(seed)
|
||||
raw = bytearray()
|
||||
for _y in range(height):
|
||||
raw.append(0)
|
||||
raw.extend(rng.randbytes(width * 3))
|
||||
compressor = zlib.compressobj(level=0)
|
||||
payload = compressor.compress(bytes(raw)) + compressor.flush()
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", payload) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def parse_bytes(text: str) -> int:
|
||||
"""docker stats MemUsage looks like '12.3MiB / 768MiB'."""
|
||||
value = text.split("/")[0].strip()
|
||||
number = ""
|
||||
unit = ""
|
||||
for char in value:
|
||||
if char.isdigit() or char == ".":
|
||||
number += char
|
||||
elif char != " ":
|
||||
unit += char
|
||||
scale = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3, "kB": 1000, "MB": 1000**2, "GB": 1000**3}
|
||||
return int(float(number) * scale[unit])
|
||||
|
||||
|
||||
def sample(stop: threading.Event, peaks: dict) -> None:
|
||||
while not stop.is_set():
|
||||
try:
|
||||
raw = subprocess.check_output(
|
||||
["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}"],
|
||||
text=True,
|
||||
)
|
||||
except subprocess.CalledProcessError:
|
||||
time.sleep(0.3)
|
||||
continue
|
||||
for line in raw.splitlines():
|
||||
name, usage = line.split("\t", 1)
|
||||
if name not in {"mtusi-api-1", "mtusi-api-2"}:
|
||||
continue
|
||||
peaks[name] = max(peaks.get(name, 0), parse_bytes(usage))
|
||||
time.sleep(0.25)
|
||||
|
||||
|
||||
def vmhwm() -> dict:
|
||||
result = {}
|
||||
for name in ("mtusi-api-1", "mtusi-api-2"):
|
||||
text = subprocess.check_output(["docker", "exec", name, "sh", "-c", "grep VmHWM /proc/1/status"], text=True)
|
||||
result[name] = int(text.split()[1])
|
||||
return result
|
||||
|
||||
|
||||
def wave(client: Client, actor, chat, payload: bytes, label: str) -> dict:
|
||||
stop = threading.Event()
|
||||
peaks = {}
|
||||
thread = threading.Thread(target=sample, args=(stop, peaks))
|
||||
before = vmhwm()
|
||||
thread.start()
|
||||
errors = []
|
||||
|
||||
def once(index):
|
||||
local = Client("http://127.0.0.1:8080", timeout=60)
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename={label}-{index}.png"
|
||||
try:
|
||||
response = local.call(
|
||||
"POST",
|
||||
path,
|
||||
201,
|
||||
actor=actor,
|
||||
raw=payload,
|
||||
headers={"Content-Type": "image/png"},
|
||||
schema="Attachment",
|
||||
)
|
||||
return response["json"]["id"]
|
||||
except Exception as exc:
|
||||
errors.append(str(exc))
|
||||
return None
|
||||
|
||||
started = time.perf_counter()
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
ids = list(pool.map(once, range(10)))
|
||||
elapsed = time.perf_counter() - started
|
||||
time.sleep(0.6)
|
||||
stop.set()
|
||||
thread.join()
|
||||
after = vmhwm()
|
||||
return {
|
||||
"label": label,
|
||||
"file_bytes": len(payload),
|
||||
"concurrency": 10,
|
||||
"successes": sum(1 for item in ids if item),
|
||||
"errors": errors[:5],
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"docker_stats_peak_bytes": peaks,
|
||||
"vmhwm_kib_before": before,
|
||||
"vmhwm_kib_after": after,
|
||||
}
|
||||
|
||||
|
||||
def restart_apis() -> None:
|
||||
subprocess.check_call(["docker", "compose", "--env-file", ".env", "restart", "api-1", "api-2"], cwd=ROOT)
|
||||
deadline = time.time() + 90
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2).read()
|
||||
return
|
||||
except Exception:
|
||||
time.sleep(1)
|
||||
raise SystemExit("API did not become ready after restart")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
client = Client("http://127.0.0.1:8080", timeout=30)
|
||||
alice = client.user("mem_alice")
|
||||
bob = client.user("mem_bob")
|
||||
chat = client.call(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
201,
|
||||
actor=alice,
|
||||
body={"title": "Память", "member_ids": [alice["id"], bob["id"]]},
|
||||
schema="Chat",
|
||||
)["json"]
|
||||
small = make_png(64, 64, 1)
|
||||
large = make_png(1200, 900, 2)
|
||||
restart_apis()
|
||||
first = wave(client, alice, chat, small, "small")
|
||||
restart_apis()
|
||||
second = wave(client, alice, chat, large, "large")
|
||||
report = {
|
||||
"hypothesis": "A single upload is capped at 10 MiB and does not decode bitmaps above 1_000_000 pixels; ten concurrent uploads stay inside the 768 MiB container limit.",
|
||||
"limits": {"container_memory": "768MiB", "max_upload_bytes": 10485760, "full_decode_pixels": 1000000},
|
||||
"waves": [first, second],
|
||||
"note": "docker stats includes page cache. VmHWM is the process high-water RSS in KiB after a fresh API restart, so the two waves are comparable.",
|
||||
}
|
||||
dest = ROOT / "evidence" / "memory.json"
|
||||
dest.parent.mkdir(exist_ok=True)
|
||||
dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if first["successes"] != 10 or second["successes"] != 10:
|
||||
raise SystemExit("not all concurrent uploads succeeded")
|
||||
print("PASS: ten concurrent uploads at two sizes")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user