forked from mtusi2026/mtusi2026lab6
- api и worker отдают traces, metrics и logs по OTLP в Collector (memory_limiter, redaction, batch, ограниченная очередь и retry); хранилища: Prometheus (OTLP receiver), Loki, Tempo; Grafana с provisioning datasources и dashboard «Messenger lab6» - W3C trace context через outbox и RabbitMQ: producer/consumer spans связывают upload, публикации и попытки worker в одном trace - SLO доступности API (SLI на HAProxy) и времени до ready, multiwindow burn-rate alerts, ApiHighErrorRate, JobQueueLagGrowing, DbPoolSaturated, TelemetryExportFailing; unit-тесты promtool, runbooks - кампания трека 2 (k6, 6 отказов по расписанию), разбор и графики; проверки сигналов и истории telemetry, evidence и REPORT.md
135 lines
6.4 KiB
Python
135 lines
6.4 KiB
Python
#!/usr/bin/env python3
|
||
"""Пик памяти API при N одновременных загрузках ~9.9 MiB (уровень 4).
|
||
Гипотеза: загрузка идёт потоком с обратным давлением, поэтому рост памяти реплики ≪ N × размер файла.
|
||
Перед каждым уровнем реплики перезапускаются (новая cgroup), затем прогрев и выборка memory.current каждые 50 мс
|
||
во всех api и s3 контейнерах. Запуск: python3 research/upload_memory.py --base-url http://localhost:8080"""
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
import statistics
|
||
import struct
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import zlib
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT / 'tests'))
|
||
from client import Client, arguments, check, run # noqa: E402
|
||
|
||
MiB = 1024 * 1024
|
||
|
||
|
||
def noise_png(side):
|
||
def chunk(kind, data):
|
||
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
|
||
raw = b''.join(b'\0' + os.urandom(side * 3) for _ in range(side))
|
||
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', side, side, 8, 2, 0, 0, 0))
|
||
+ chunk(b'IDAT', zlib.compress(raw, 1)) + chunk(b'IEND', b''))
|
||
|
||
|
||
def containers(service):
|
||
out = subprocess.run(['docker', 'compose', 'ps', '-q', service], cwd=ROOT, capture_output=True, text=True, check=True).stdout
|
||
return out.split()
|
||
|
||
|
||
def cgroup(cid, name):
|
||
return int(subprocess.run(['docker', 'exec', cid, 'cat', '/sys/fs/cgroup/' + name], capture_output=True, text=True, check=True).stdout)
|
||
|
||
|
||
class Sampler(threading.Thread):
|
||
"""Максимум memory.current контейнера, опрос каждые 50 мс одним долгим docker exec."""
|
||
def __init__(self, cid):
|
||
super().__init__(daemon=True)
|
||
self.max = 0
|
||
self.proc = subprocess.Popen(['docker', 'exec', cid, 'sh', '-c', 'while :; do cat /sys/fs/cgroup/memory.current; sleep 0.05; done'],
|
||
stdout=subprocess.PIPE, text=True)
|
||
|
||
def run(self):
|
||
for line in self.proc.stdout:
|
||
self.max = max(self.max, int(line))
|
||
|
||
def stop(self):
|
||
self.proc.terminate()
|
||
return self.max
|
||
|
||
|
||
def restart_api(c):
|
||
subprocess.run(['docker', 'compose', 'restart', 'api'], cwd=ROOT, check=True, capture_output=True)
|
||
seen, deadline = set(), time.monotonic() + 60
|
||
while len(seen) < 2:
|
||
check(time.monotonic() < deadline, 'replicas not ready')
|
||
try:
|
||
r = c.call('GET', '/health/ready', [200, 503])
|
||
if r['status'] == 200:
|
||
seen.add(r['headers']['X-Instance-Id'])
|
||
except AssertionError:
|
||
pass
|
||
time.sleep(0.2)
|
||
|
||
|
||
def main():
|
||
args = arguments('Concurrent upload memory experiment', lambda p: p.add_argument('--levels', default='1,10,20,40'))
|
||
c = Client(args.base_url, args.ca_file, timeout=120)
|
||
data = noise_png(1860)
|
||
small = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||
results = []
|
||
for n in [int(x) for x in args.levels.split(',')]:
|
||
restart_api(c)
|
||
alice, _, _, chat = c.bootstrap()
|
||
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=load.png'
|
||
for _ in range(4): # прогрев: модули SDK, пул соединений, JIT
|
||
c.call('POST', path, 201, actor=alice, raw=small, headers={'Content-Type': 'image/png'})
|
||
api = containers('api')
|
||
watched = {('api', cid): None for cid in api} | {('s3', cid): None for cid in containers('s3')}
|
||
base = {k: cgroup(k[1], 'memory.current') for k in watched}
|
||
samplers = {k: Sampler(k[1]) for k in watched}
|
||
for s in samplers.values():
|
||
s.start()
|
||
time.sleep(0.5)
|
||
barrier = threading.Barrier(n)
|
||
done = []
|
||
|
||
def upload():
|
||
local = Client(args.base_url, args.ca_file, timeout=120)
|
||
barrier.wait()
|
||
r = local.call('POST', path, [201, 503], actor=alice, raw=data, headers={'Content-Type': 'image/png'})
|
||
done.append((r['status'], r['elapsed'], r['headers'].get('X-Instance-Id')))
|
||
|
||
start = time.monotonic()
|
||
threads = [threading.Thread(target=upload) for _ in range(n)]
|
||
for t in threads:
|
||
t.start()
|
||
for t in threads:
|
||
t.join()
|
||
wall = time.monotonic() - start
|
||
time.sleep(0.5)
|
||
peak = {k: s.stop() for k, s in samplers.items()}
|
||
lat = sorted(e for _, e, _ in done)
|
||
row = {
|
||
'concurrency': n, 'file_mib': round(len(data) / MiB, 2), 'ok': sum(s == 201 for s, _, _ in done), 'errors': sum(s != 201 for s, _, _ in done),
|
||
'wall_s': round(wall, 2), 'throughput_mib_s': round(n * len(data) / MiB / wall, 1),
|
||
'latency_p50_s': round(statistics.median(lat), 2), 'latency_max_s': round(lat[-1], 2),
|
||
'per_replica': sorted(sum(1 for *_, i in done if i == cid[:12]) for cid in api),
|
||
'api_baseline_mib': [round(base[('api', cid)] / MiB, 1) for cid in api],
|
||
'api_peak_mib': [round(peak[('api', cid)] / MiB, 1) for cid in api],
|
||
'api_cgroup_peak_mib': [round(cgroup(cid, 'memory.peak') / MiB, 1) for cid in api],
|
||
's3_baseline_mib': [round(base[k] / MiB, 1) for k in watched if k[0] == 's3'],
|
||
's3_peak_mib': [round(peak[k] / MiB, 1) for k in watched if k[0] == 's3'],
|
||
}
|
||
row['api_max_growth_mib'] = round(max(p - b for p, b in zip(row['api_peak_mib'], row['api_baseline_mib'])), 1)
|
||
results.append(row)
|
||
print(json.dumps(row, ensure_ascii=False))
|
||
print('\n| N | OK/ошибки | время, с | MiB/s | p50 / max, с | загрузок на реплику | API до → пик, MiB | рост max, MiB | S3-шлюз до → пик, MiB |')
|
||
print('| --- | --- | --- | --- | --- | --- | --- | --- | --- |')
|
||
for r in results:
|
||
print('| %d | %d/%d | %.2f | %.1f | %.2f / %.2f | %s | %s → %s | %.1f | %s → %s |' % (
|
||
r['concurrency'], r['ok'], r['errors'], r['wall_s'], r['throughput_mib_s'], r['latency_p50_s'], r['latency_max_s'],
|
||
r['per_replica'], r['api_baseline_mib'], r['api_peak_mib'], r['api_max_growth_mib'], r['s3_baseline_mib'], r['s3_peak_mib']))
|
||
check(all(r['errors'] == 0 for r in results), 'some uploads failed')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
run(main)
|