84 lines
9.1 KiB
Python
84 lines
9.1 KiB
Python
#!/usr/bin/env python3
|
|
from support import *
|
|
import random,threading,concurrent.futures,statistics,datetime,hashlib
|
|
from client import arguments,run
|
|
|
|
def worker_sample():
|
|
command='''printf 'memory ';cat /sys/fs/cgroup/memory.current; printf 'cpu ';awk '/^usage_usec/{print $2}' /sys/fs/cgroup/cpu.stat; printf 'rss '; for p in $(cat /sys/fs/cgroup/cgroup.procs); do if [ "$(cat /proc/$p/comm 2>/dev/null)" = dotnet ]; then awk '/^VmRSS:/{print $2}' /proc/$p/status 2>/dev/null; fi; done'''
|
|
lines=compose('exec','-T','worker','sh','-c',command).splitlines()
|
|
mem=int(lines[0].split()[1]);cpu=int(lines[1].split()[1]);rss=sum(int(x) for x in ('\n'.join(lines[2:]).replace('rss ','').split()))*1024
|
|
return {'time':time.time(),'memory_bytes':mem,'cpu_usec':cpu,'rss_bytes':rss}
|
|
def main():
|
|
args=arguments('Concurrency/prefetch overload research');c=Client(args.base_url,args.ca_file,timeout=100);a,b,e,chat=c.bootstrap()
|
|
rng=random.Random(202605)
|
|
large=Image.frombytes('RGB',(1024,1024),rng.randbytes(1024*1024*3));out=io.BytesIO();large.save(out,format='PNG',compress_level=1)
|
|
images={'small':(ROOT/'tests/fixtures/sample.png').read_bytes(),'large':out.getvalue()}
|
|
report={'hypothesis':'With fixed worker CPU, more simultaneous image processes need not improve throughput and can increase RSS; bounded admission protects the queue when the worker stops.',
|
|
'profile':{'settings':[[1,1],[2,2],[4,4]],'repeats':3,'batch':8,'upload_clients':4,'worker_cpu':1,'worker_memory_limit_mib':1024,'warmup_images':1,'image_sizes_bytes':{k:len(v) for k,v in images.items()}},'runs':[]}
|
|
configurations=[(1,1),(2,2),(4,4)]
|
|
try:
|
|
wait(lambda:sql("SELECT count(*) FROM attachment_jobs WHERE state IN ('queued','processing')")=='0')
|
|
for repeat in range(3):
|
|
for label in (['small','large'] if repeat%2==0 else ['large','small']):
|
|
for concurrency,prefetch in configurations[repeat:]+configurations[:repeat]:
|
|
env={'WORKER_CONCURRENCY':str(concurrency),'WORKER_PREFETCH':str(prefetch)}
|
|
compose('up','-d','--no-deps','--force-recreate','worker',env=env)
|
|
wait(lambda:'Worker ready:' in compose('logs','--tail','5','worker'),40)
|
|
warm=upload(c,a,chat,images[label],name='warm.png')['json'];ready(c,a,warm)
|
|
samples=[];stop=threading.Event()
|
|
def poll():
|
|
while not stop.is_set():
|
|
try:samples.append(worker_sample())
|
|
except Exception:pass
|
|
stop.wait(.2)
|
|
thread=threading.Thread(target=poll);thread.start();started=time.monotonic();requests=[]
|
|
def submit(index):
|
|
begin=time.monotonic();result=upload(c,a,chat,images[label],name=f'{label}-{repeat}-{index}.png')
|
|
requests.append({'upload_seconds':time.monotonic()-begin,'status':result['status']});return result['json']
|
|
try:
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=4) as pool:attachments=list(pool.map(submit,range(8)))
|
|
ids=','.join("'"+item['id']+"'" for item in attachments)
|
|
wait(lambda:sql('SELECT count(*) FROM attachment_jobs WHERE state=\'ready\' AND attachment_id IN ('+ids+')')=='8',180)
|
|
finally:stop.set();thread.join()
|
|
elapsed=time.monotonic()-started
|
|
values=json.loads(sql('''SELECT json_agg(json_build_object('id',j.id,'ready_seconds',extract(epoch from j.completed_at-j.created_at),'queue_lag_seconds',extract(epoch from (SELECT min(e.created_at) FROM attachment_job_events e WHERE e.job_id=j.id AND e.event='processing')-j.created_at),'attempts',j.attempts)) FROM attachment_jobs j WHERE j.attachment_id IN ('''+ids+')'))
|
|
check(all(row['attempts']==1 for row in values),'unplanned retries in comparison run')
|
|
row={'repeat':repeat+1,'image':label,'concurrency':concurrency,'prefetch':prefetch,'successful_jobs':8,'errors':0,'seconds':elapsed,'throughput':8/elapsed,
|
|
'p95_ready_seconds':sorted(v['ready_seconds'] for v in values)[-1], 'p95_queue_lag_seconds':sorted(v['queue_lag_seconds'] for v in values)[-1],
|
|
'peak_rss_bytes':max(v['rss_bytes'] for v in samples),'peak_cgroup_memory_bytes':max(v['memory_bytes'] for v in samples),
|
|
'worker_cpu_seconds':(samples[-1]['cpu_usec']-samples[0]['cpu_usec'])/1e6,'requests':requests,'jobs':values,'samples':samples}
|
|
report['runs'].append(row)
|
|
(ROOT/'evidence/overload-progress.json').write_text(json.dumps(report,indent=2)+'\n')
|
|
print(f"run {len(report['runs'])}/18: {label}, {concurrency}/{prefetch}, {row['throughput']:.2f} ready/s",flush=True)
|
|
compose('stop','worker');compose('up','-d','--no-deps','--force-recreate','--wait','api',env={'PIPELINE_CAPACITY':'4'});wait_api(c)
|
|
def pressure(index):return upload(c,a,chat,name='pressure-'+str(index)+'.png',expected=[202,429])
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:attempts=list(pool.map(pressure,range(8)))
|
|
accepted=[json.loads(r['bytes']) for r in attempts if r['status']==202];rejected=sum(r['status']==429 for r in attempts)
|
|
pending=int(sql("SELECT count(*) FROM attachment_jobs WHERE state IN ('queued','processing')"))
|
|
check(len(accepted)==4 and rejected==4 and pending==4,'admission bound was exceeded')
|
|
before=time.monotonic();compose('up','-d','--no-deps','--force-recreate','worker')
|
|
for att in accepted:ready(c,a,att)
|
|
report['backpressure']={'attempted':8,'accepted':4,'rejected_429':4,'pending_at_limit':pending,'all_accepted_recovered':True,'recovery_seconds':time.monotonic()-before}
|
|
report['environment']=subprocess.check_output(['docker','info','--format','{{.OSType}} {{.Architecture}} {{.NCPU}} CPUs {{.MemTotal}} bytes RAM']).decode().strip()
|
|
h=hashlib.sha256()
|
|
for source in sorted((ROOT/'src/Messenger.Api').rglob('*.cs')):
|
|
if 'obj' in source.parts or 'bin' in source.parts:continue
|
|
h.update(str(source.relative_to(ROOT)).encode());h.update(source.read_bytes())
|
|
report['source_sha256']=h.hexdigest()
|
|
(ROOT/'evidence/overload.json').write_text(json.dumps(report,indent=2)+'\n')
|
|
lines=['# Очередь под нагрузкой','','Три повтора; одинаковые CPU/RAM worker; batch 8, четыре HTTP-клиента.','',
|
|
'| Файл | Concurrency / prefetch | Готовых/с, медиана | p95 до ready, s | p95 ожидания, s | RSS процессов, MiB |','| --- | --- | ---: | ---: | ---: | ---: |']
|
|
for label in images:
|
|
for concurrency,prefetch in configurations:
|
|
group=[r for r in report['runs'] if r['image']==label and r['concurrency']==concurrency]
|
|
med=lambda key:statistics.median(r[key] for r in group)
|
|
lines.append(f"| {label} | {concurrency}/{prefetch} | {med('throughput'):.3f} | {med('p95_ready_seconds'):.3f} | {med('p95_queue_lag_seconds'):.3f} | {med('peak_rss_bytes')/1048576:.1f} |")
|
|
lines+=['','p95: nearest rank; при batch 8 это максимальная задержка. RSS: сумма VmRSS','процессов dotnet внутри worker, включая дочерние декодеры; общие страницы могут','считаться несколько раз. memory.current измерен отдельно. Период выборок 200 ms','плюс docker exec; короткие пики могут быть пропущены. Время включает загрузку','batch и ожидание последнего ready, без регистрации и прогрева. Queue lag: от','создания записи задания до первой processing. Время до ready — по часам PostgreSQL.','',
|
|
'Backpressure: при остановленном worker лимит 4 принял ровно 4 из 8 файлов;','остальные получили 429. После запуска обработались все четыре принятых задания.','',
|
|
'Исходные результаты: evidence/overload.json. Один Docker-host, небольшие batch,','параллельно могут работать другие локальные приложения. Эти значения не являются','пределом производительности сервера. Вывод сравнения относится к этому профилю.']
|
|
lines+=['','','','В этом профиле для small один обработчик оказался быстрее двух и четырёх.','Для large вариант 2/2 дал лучшую медиану throughput и меньшую p95 готовности,','чем 1/1, при большем RSS. Четыре обработчика уменьшили время ожидания начала,','но увеличили полное время обработки и память. Поэтому по умолчанию оставлены','2/1: два параллельных обработчика с небольшим prefetch. Низкий prefetch','ограничивает число взятых сообщений; пара 2/1 отдельно не входит в сравнение,','поэтому не заявляется как измеренный оптимум.']
|
|
(ROOT/'docs/OVERLOAD.md').write_text('\n'.join(lines)+'\n')
|
|
print('PASS: overload research and bounded admission',flush=True)
|
|
finally:compose('up','-d','--no-deps','--force-recreate','--wait','api','worker')
|
|
if __name__=='__main__':run(main)
|