Добавить приватное S3-хранилище, вложения, presigned upload, reconciler и доказательства.
148 lines
4.7 KiB
Python
148 lines
4.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Lab4 experiment: 10+ concurrent uploads with API container peak memory.
|
|
|
|
The uploads are streamed to an ephemeral file and decoded under a bounded
|
|
semaphore, so this test looks for a flat memory profile rather than a linear
|
|
growth with concurrency.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import io
|
|
import json
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import threading
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
|
|
from PIL import Image
|
|
|
|
from common import HttpClient, assert_status
|
|
|
|
MEM_RE = re.compile(r"([0-9.]+)\s*(B|KiB|MiB|GiB)")
|
|
|
|
|
|
def make_png(width: int, height: int) -> bytes:
|
|
# Random pixels compress poorly, giving a realistic multi-MiB upload.
|
|
image = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3))
|
|
buffer = io.BytesIO()
|
|
image.save(buffer, format="PNG", compress_level=1)
|
|
return buffer.getvalue()
|
|
|
|
|
|
def memory_mib(text: str) -> float | None:
|
|
match = MEM_RE.search(text)
|
|
if not match:
|
|
return None
|
|
value = float(match.group(1))
|
|
unit = match.group(2)
|
|
return value * {"B": 1 / 1024 / 1024, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}[unit]
|
|
|
|
|
|
def sample_api_memory(prefix: str) -> dict[str, float]:
|
|
try:
|
|
result = subprocess.run(
|
|
[
|
|
"docker",
|
|
"stats",
|
|
"--no-stream",
|
|
"--format",
|
|
"{{.Name}} {{.MemUsage}}",
|
|
],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=10,
|
|
)
|
|
except (OSError, subprocess.SubprocessError):
|
|
return {}
|
|
if result.returncode != 0:
|
|
return {}
|
|
usage = {}
|
|
for line in result.stdout.splitlines():
|
|
name, _, mem = line.partition(" ")
|
|
if prefix in name:
|
|
value = memory_mib(mem)
|
|
if value is not None:
|
|
usage[name] = value
|
|
return usage
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--base-url", default="http://localhost:8080")
|
|
parser.add_argument("--workers", type=int, default=10)
|
|
parser.add_argument("--width", type=int, default=1400)
|
|
parser.add_argument("--height", type=int, default=1000)
|
|
parser.add_argument("--container-prefix", default="mtusi-lab4-api")
|
|
parser.add_argument("--output", default="")
|
|
args = parser.parse_args()
|
|
|
|
client = HttpClient(args.base_url)
|
|
alice, alice_reg = client.register()
|
|
alice_token = client.login(alice_reg)["token"]
|
|
status, _h, chat, _r = client.call(
|
|
"POST",
|
|
"/api/v1/chats",
|
|
body={"title": "Upload memory experiment", "member_ids": [alice["id"]]},
|
|
token=alice_token,
|
|
)
|
|
assert_status(status, 201, "create chat")
|
|
|
|
content = make_png(args.width, args.height)
|
|
peak: dict[str, float] = {}
|
|
stop = threading.Event()
|
|
|
|
def sampler() -> None:
|
|
while not stop.is_set():
|
|
for name, value in sample_api_memory(args.container_prefix).items():
|
|
peak[name] = max(peak.get(name, 0.0), value)
|
|
time.sleep(0.2)
|
|
|
|
sampler_thread = threading.Thread(target=sampler, name="memory-sampler", daemon=True)
|
|
sampler_thread.start()
|
|
baseline = sample_api_memory(args.container_prefix)
|
|
|
|
def upload(index: int):
|
|
return client.raw(
|
|
"POST",
|
|
"/api/v1/chats/" + chat["id"] + "/attachments?filename=load-%d.png" % index,
|
|
data=content,
|
|
headers={"Authorization": "Bearer " + alice_token, "Content-Type": "image/png"},
|
|
timeout=120,
|
|
)
|
|
|
|
started = time.monotonic()
|
|
with ThreadPoolExecutor(max_workers=args.workers) as pool:
|
|
results = list(pool.map(upload, range(args.workers)))
|
|
elapsed = time.monotonic() - started
|
|
stop.set()
|
|
sampler_thread.join(timeout=2)
|
|
|
|
statuses = [status for status, _h, _p, _r in results]
|
|
assert_status(min(statuses), 201, "concurrent uploads")
|
|
if max(statuses) != 201:
|
|
raise AssertionError(f"some uploads failed: {statuses}")
|
|
|
|
summary = {
|
|
"workers": args.workers,
|
|
"file_bytes": len(content),
|
|
"elapsed_seconds": round(elapsed, 3),
|
|
"successful_uploads": len(statuses),
|
|
"baseline_mib": {k: round(v, 2) for k, v in baseline.items()},
|
|
"peak_mib": {k: round(v, 2) for k, v in peak.items()},
|
|
"peak_growth_mib": {
|
|
k: round(peak.get(k, 0.0) - baseline.get(k, 0.0), 2) for k in peak
|
|
},
|
|
}
|
|
print(json.dumps(summary, ensure_ascii=False, sort_keys=True))
|
|
if args.output:
|
|
with open(args.output, "w", encoding="utf-8") as handle:
|
|
json.dump(summary, handle, ensure_ascii=False, indent=2, sort_keys=True)
|
|
print("PASS: concurrent uploads completed; inspect peak_mib for the bounded-memory claim")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|