Добавить вложения: S3, метаданные в PostgreSQL и очистка незавершённых загрузок

This commit is contained in:
2026-10-30 23:10:44 +03:00
parent ec540a1a72
commit d812d8f8d8
75 changed files with 5968 additions and 16 deletions
+97
View File
@@ -0,0 +1,97 @@
#!/usr/bin/env python3
"""Пик памяти реплик API при одновременных загрузках файлов около 10 MiB.
Перед замером реплики перезапускаются, чтобы обнулить VmHWM (пик памяти процесса). Уровни идут по
возрастанию, поэтому VmHWM после каждого уровня показывает пик этого уровня. memory.peak контейнера
больше: в него входит кэш страниц временных файлов, который система может освободить.
"""
import argparse
import json
import subprocess
import sys
import time
import urllib.parse
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
from api import call, new_chat, new_user
from samples import noisy_png
LEVELS = (1, 10, 30)
def compose(*args):
command = ["docker", "compose", "--env-file", ".env", *args]
return subprocess.run(command, cwd=ROOT, check=True, capture_output=True, text=True).stdout
def memory():
result = {}
for container in compose("ps", "-q", "api").split():
status = subprocess.check_output(["docker", "exec", container, "cat", "/proc/1/status"], text=True)
fields = dict(line.split(":", 1) for line in status.splitlines() if ":" in line)
peak = subprocess.check_output(["docker", "exec", container, "cat", "/sys/fs/cgroup/memory.peak"], text=True)
result[container[:12]] = {
"process_peak_rss_mib": round(int(fields["VmHWM"].split()[0]) / 1024, 1),
"cgroup_peak_mib": round(int(peak) / 1024 / 1024, 1),
}
return result
def wait_ready():
deadline = time.monotonic() + 60
seen = set()
while time.monotonic() < deadline and len(seen) < 2:
try:
status, _, headers = call("GET", "/health/ready", timeout=2)
if status == 200:
seen.add(headers["X-Instance-Id"])
except OSError:
pass
time.sleep(0.2)
def main():
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--output")
args = parser.parse_args()
data = noisy_png(10 * 1024 * 1024 - 4096)
compose("restart", "api")
wait_ready()
user = new_user("peak")
chat = new_chat(user)
path = "/api/v1/chats/%s/attachments?filename=peak.png" % urllib.parse.quote(chat["id"])
levels = [{"parallel_uploads": 0, "memory": memory()}]
for level in LEVELS:
started = time.monotonic()
def post(_):
sent = time.monotonic()
status = call("POST", path, raw=data, user=user, headers={"Content-Type": "image/png"}, timeout=60)[0]
return status, time.monotonic() - sent
with ThreadPoolExecutor(max_workers=level) as pool:
results = list(pool.map(post, range(level)))
statuses = [status for status, _ in results]
entry = {
"parallel_uploads": level,
"file_bytes": len(data),
"seconds": round(time.monotonic() - started, 2),
"slowest_upload_s": round(max(took for _, took in results), 2),
"statuses": {str(s): statuses.count(s) for s in sorted(set(statuses))},
"memory": memory(),
}
print(json.dumps(entry, ensure_ascii=False))
levels.append(entry)
report = {"limit_per_replica_mib": 256, "replicas": 2, "levels": levels}
if args.output:
Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
if any(set(level.get("statuses", {"201": 1})) != {"201"} for level in levels):
raise SystemExit("FAIL: не все загрузки успешны")
print("PASS: все загрузки успешны")
if __name__ == "__main__":
main()