выполнение lab3, track-1
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
"""Create local secrets once. Never print values or overwrite existing secrets."""
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import shutil
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
folder = root / 'secrets'
|
||||
folder.mkdir(exist_ok=True)
|
||||
folder.chmod(0o700)
|
||||
def put(name, value):
|
||||
path = folder / name
|
||||
if not path.exists():
|
||||
with path.open('x', encoding='utf-8', newline='\n') as f:
|
||||
f.write(value + '\n')
|
||||
# Parent directory is private; Compose mounts individual files for non-root services.
|
||||
path.chmod(0o644)
|
||||
return path.read_text(encoding='utf-8').strip()
|
||||
|
||||
pg = put('postgres_password', secrets.token_hex(32))
|
||||
vk = put('valkey_password', secrets.token_hex(32))
|
||||
put('database_url', f'postgres://messenger:{pg}@db:5432/messenger?sslmode=disable')
|
||||
put('cache_url', f'redis://:{vk}@valkey:6379/0')
|
||||
put('cursor_secret', secrets.token_hex(32))
|
||||
put('valkey.conf', '\n'.join(['bind 0.0.0.0', 'protected-mode yes', f'requirepass {vk}',
|
||||
'dir /data', 'appendonly yes', 'appendfsync always', 'maxmemory 128mb',
|
||||
'maxmemory-policy noeviction', 'save 60 1000']))
|
||||
if not (root / '.env').exists():
|
||||
shutil.copyfile(root / '.env.example', root / '.env')
|
||||
for name in ('restart.sh', 'stop-one.sh'):
|
||||
(root / 'scripts' / name).chmod(0o755)
|
||||
print('Local .env and secrets ready; existing values preserved.')
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Run the unchanged public make targets with executable Linux action hooks."""
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('target', choices=['check', 'validate', 'test', 'test-own', 'test-race', 'test-persistence', 'test-failover'])
|
||||
args = p.parse_args()
|
||||
command = ['docker', 'run', '--rm', '--network', 'mtusi-lab3_default',
|
||||
'--mount', 'type=bind,source='+str(root)+',target=/workspace']
|
||||
hooks = {'test-persistence': 'scripts/restart.sh', 'test-failover': 'scripts/stop-one.sh'}
|
||||
if args.target in hooks:
|
||||
command += ['--mount', 'type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock']
|
||||
command += ['mtusi-messenger:lab3-tests', 'make', args.target, 'BASE_URL=http://lb:8080']
|
||||
if args.target in hooks:
|
||||
command += ['ACTION_SCRIPT='+hooks[args.target]]
|
||||
raise SystemExit(subprocess.call(command, cwd=root))
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Cross-platform operator commands for the local lab; never delete volumes."""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def docker(*args, capture=False, env=None):
|
||||
result = subprocess.run(['docker', *args], cwd=ROOT, check=True,
|
||||
text=True, encoding='utf-8', capture_output=capture, env=env)
|
||||
return result.stdout.strip() if capture else None
|
||||
|
||||
|
||||
def replicas():
|
||||
ids = docker('compose', 'ps', '-q', 'api', capture=True).split()
|
||||
if not ids:
|
||||
return []
|
||||
# Only operational fields; never persist Env or secret values.
|
||||
raw = docker('inspect', '--format', '{{json .Config.Hostname}} {{json .Id}}', *ids, capture=True)
|
||||
return [dict(zip(('instance', 'id'), (json.loads(x) for x in line.split()))) for line in raw.splitlines()]
|
||||
|
||||
|
||||
def ready(count=2, base='http://127.0.0.1:8080'):
|
||||
deadline = time.monotonic()+90
|
||||
observed = set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(base+'/health/ready', timeout=2) as response:
|
||||
observed.add(response.headers['X-Instance-Id'])
|
||||
if len(observed) >= count:
|
||||
return
|
||||
except OSError:
|
||||
observed.clear()
|
||||
time.sleep(.2)
|
||||
raise RuntimeError('expected ready replicas did not appear through LB')
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('action', choices=['start','stop','restart','restore','status','stop-one','rolling'])
|
||||
p.add_argument('--instance')
|
||||
args = p.parse_args()
|
||||
if args.action == 'start':
|
||||
subprocess.run([sys.executable, '-X', 'utf8', 'scripts/bootstrap.py'], cwd=ROOT, check=True)
|
||||
docker('compose', 'up', '-d', '--build', '--wait')
|
||||
ready()
|
||||
elif args.action == 'stop':
|
||||
docker('compose', 'down')
|
||||
elif args.action == 'restart':
|
||||
docker('compose', 'restart', 'db', 'valkey', 'api', 'lb')
|
||||
ready()
|
||||
elif args.action == 'restore':
|
||||
docker('compose', 'up', '-d', '--no-deps', '--scale', 'api=2', 'api')
|
||||
ready()
|
||||
elif args.action == 'status':
|
||||
docker('compose', 'ps', '-a')
|
||||
print(json.dumps(replicas(), indent=2))
|
||||
ids = docker('compose','ps','-a','-q',capture=True).split()
|
||||
template = '[{{json .Id}},{{json .Name}},{{json .Config.Hostname}},{{json .State.Status}},{{json .State.StartedAt}},{{.HostConfig.NanoCpus}},{{.HostConfig.Memory}},{{json .NetworkSettings.Ports}},{{json .Config.Image}}]'
|
||||
containers = [dict(zip(('id','name','instance','state','started_at','nano_cpus','memory_bytes','ports','image'),json.loads(line)))
|
||||
for line in docker('inspect','--format',template,*ids,capture=True).splitlines()]
|
||||
query = "SELECT json_build_object('users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats),'messages',(SELECT count(*) FROM messages),'idempotency_keys',(SELECT count(*) FROM message_keys),'database_bytes',pg_database_size(current_database()),'fsync',current_setting('fsync'),'synchronous_commit',current_setting('synchronous_commit'),'full_page_writes',current_setting('full_page_writes'))"
|
||||
counts = json.loads(docker('compose','exec','-T','db','psql','-U','messenger','-d','messenger','-At','-c',query,capture=True))
|
||||
snapshot = {'timestamp_utc':dt.datetime.now(dt.timezone.utc).isoformat(),'containers':containers,'database':counts}
|
||||
folder = ROOT/'evidence'
|
||||
folder.mkdir(exist_ok=True)
|
||||
(folder/'infrastructure.json').write_text(json.dumps(snapshot,indent=2)+'\n',encoding='utf-8')
|
||||
elif args.action == 'stop-one':
|
||||
selected = [r for r in replicas() if r['instance']==args.instance]
|
||||
if len(selected) != 1:
|
||||
p.error('--instance must match exactly one running API hostname (see status)')
|
||||
docker('stop', '--time', '9', selected[0]['id'])
|
||||
else:
|
||||
original = replicas()
|
||||
if len(original) != 2:
|
||||
p.error('rolling replacement requires exactly two running replicas')
|
||||
for old in original:
|
||||
docker('compose', 'up', '-d', '--no-deps', '--scale', 'api=3', 'api')
|
||||
ready(3)
|
||||
docker('stop', '--time', '9', old['id'])
|
||||
docker('rm', old['id'])
|
||||
ready(2)
|
||||
print(json.dumps({'before':original, 'after':replicas()}, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,42 @@
|
||||
param([string]$BaseUrl = 'http://127.0.0.1:8080')
|
||||
# Dot-source this script to keep the synthetic session in the current terminal.
|
||||
$Lab = @{ BaseUrl = $BaseUrl.TrimEnd('/'); Headers = @{} }
|
||||
|
||||
function Invoke-LabApi {
|
||||
param(
|
||||
[string]$Method,
|
||||
[string]$Path,
|
||||
[object]$Body = $null,
|
||||
[hashtable]$Headers = $Lab.Headers
|
||||
)
|
||||
$request = @{ Method=$Method; Uri=($Lab.BaseUrl+$Path); Headers=$Headers; UseBasicParsing=$true; ErrorAction='Stop' }
|
||||
if ($null -ne $Body) {
|
||||
$request.ContentType = 'application/json; charset=utf-8'
|
||||
$request.Body = [Text.Encoding]::UTF8.GetBytes(($Body | ConvertTo-Json -Depth 8 -Compress))
|
||||
}
|
||||
$response = Invoke-WebRequest @request
|
||||
$Lab.LastInstance = $response.Headers['X-Instance-Id'] -join ','
|
||||
Write-Host ('HTTP {0}; X-Instance-Id={1}; X-Request-Id={2}' -f [int]$response.StatusCode, $Lab.LastInstance, ($response.Headers['X-Request-Id'] -join ','))
|
||||
if ($response.Content) { return ($response.Content | ConvertFrom-Json) }
|
||||
}
|
||||
|
||||
$suffix = [guid]::NewGuid().ToString('N').Substring(0,10)
|
||||
$password = 'Synthetic-' + [guid]::NewGuid().ToString('N')
|
||||
$username = 'manual_a_' + $suffix
|
||||
$Lab.Alice = Invoke-LabApi POST '/api/v1/users' @{username=$username; display_name='Alice'; password=$password}
|
||||
$Lab.Bob = Invoke-LabApi POST '/api/v1/users' @{username=('manual_b_'+$suffix); display_name='Bob'; password=$password}
|
||||
$basic = [Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes(($username+':'+$password)))
|
||||
$session = Invoke-LabApi POST '/api/v1/auth/sessions' -Headers @{Authorization=('Basic '+$basic)}
|
||||
$Lab.Headers = @{ Authorization=('Bearer '+$session.token) }
|
||||
$Lab.Chat = Invoke-LabApi POST '/api/v1/chats' @{title='Manual lab3'; member_ids=@($Lab.Alice.id,$Lab.Bob.id)}
|
||||
$Lab.MessagePath = '/api/v1/chats/'+$Lab.Chat.id+'/messages'
|
||||
$Lab.Key = [guid]::NewGuid().ToString('N')
|
||||
$Lab.Headers['Idempotency-Key'] = $Lab.Key
|
||||
$Lab.Body = @{ text='Manual idempotent message' }
|
||||
$Lab.Message = Invoke-LabApi POST $Lab.MessagePath $Lab.Body
|
||||
for ($i=0; $i -lt 4; $i++) {
|
||||
$replay = Invoke-LabApi POST $Lab.MessagePath $Lab.Body
|
||||
if ($replay.id -ne $Lab.Message.id) { throw 'Replay created a duplicate' }
|
||||
}
|
||||
Remove-Variable password,basic,session
|
||||
Write-Host ('Ready: chat={0}; message={1}. Session is kept in $Lab; do not print or save its Authorization header.' -f $Lab.Chat.id,$Lab.Message.id)
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Grade-5 track 1: controlled scaling, saturation sweep and pool tuning."""
|
||||
import argparse
|
||||
import csv
|
||||
import datetime as dt
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import statistics
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
|
||||
from manage import ROOT, docker, ready
|
||||
|
||||
|
||||
def sql(query):
|
||||
return json.loads(docker('compose','exec','-T','db','psql','-U','messenger','-d','messenger','-At','-c',query,capture=True))
|
||||
|
||||
|
||||
def db_stats():
|
||||
return sql("SELECT row_to_json(s) FROM (SELECT xact_commit,xact_rollback,blks_read,blks_hit,tup_inserted,deadlocks FROM pg_stat_database WHERE datname=current_database()) s")
|
||||
|
||||
|
||||
def lb_stats():
|
||||
text = docker('compose','exec','-T','lb','wget','-q','-O','-','http://127.0.0.1:8404/stats;csv',capture=True)
|
||||
rows = list(csv.DictReader(io.StringIO(text.removeprefix('# '))))
|
||||
return [{k:r.get(k) for k in ('svname','status','qcur','qmax','scur','smax','econ','eresp','stot')} for r in rows if r.get('pxname')=='api']
|
||||
|
||||
|
||||
def configure(n, cpu, memory, pool):
|
||||
env = dict(os.environ, DB_CPUS='1.0', API_REPLICAS=str(n), API_CPUS=str(cpu), API_MEMORY=str(memory)+'m',
|
||||
DB_POOL_MAX=str(pool), POOL_METRICS='1', REQUEST_LOG='0')
|
||||
docker('compose','up','-d','--no-deps','--force-recreate','--scale','api='+str(n),'api',env=env)
|
||||
ready(n)
|
||||
return env
|
||||
|
||||
|
||||
def measure(config, duration, warmup):
|
||||
began = dt.datetime.now(dt.timezone.utc).isoformat()
|
||||
before, lb_before = db_stats(), lb_stats()
|
||||
samples = []
|
||||
finished = threading.Event()
|
||||
sample_errors = []
|
||||
|
||||
def monitor():
|
||||
while not finished.is_set():
|
||||
try:
|
||||
ids = docker('ps','-q','--filter','label=com.docker.compose.project=mtusi-lab3',capture=True).split()
|
||||
ids += docker('ps','-q','--filter','label=mtusi.lab3.role=load',capture=True).split()
|
||||
text = docker('stats','--no-stream','--format','{{json .}}',*ids,capture=True)
|
||||
stats = [json.loads(line) for line in text.splitlines() if line.strip()]
|
||||
waits = sql("SELECT coalesce(json_object_agg(kind,n),'{}') FROM (SELECT coalesce(wait_event_type,'CPU/runnable') kind,count(*) n FROM pg_stat_activity WHERE datname=current_database() AND pid<>pg_backend_pid() AND state='active' GROUP BY 1) s")
|
||||
samples.append({'time':dt.datetime.now(dt.timezone.utc).isoformat(),'containers':stats,'db_active_waits':waits})
|
||||
except (subprocess.CalledProcessError, ValueError) as error:
|
||||
sample_errors.append(type(error).__name__)
|
||||
finished.wait(.5)
|
||||
|
||||
thread = threading.Thread(target=monitor)
|
||||
thread.start()
|
||||
try:
|
||||
output = docker('run','--rm','--network','mtusi-lab3_default','--cpus','2','--memory','256m',
|
||||
'--label','mtusi.lab3.role=load','mtusi-messenger:lab3-load',
|
||||
'-duration',str(duration)+'s','-warmup',str(warmup)+'s',
|
||||
'-concurrency',str(config['concurrency']),capture=True)
|
||||
result = json.loads(output)
|
||||
finally:
|
||||
finished.set()
|
||||
thread.join()
|
||||
after = db_stats()
|
||||
logs = docker('compose','logs','--no-log-prefix','--since',began,'api',capture=True)
|
||||
pools = {}
|
||||
for line in logs.splitlines():
|
||||
try:
|
||||
item = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if item.get('msg')=='pool':
|
||||
pools.setdefault(item['instance'],[]).append(item)
|
||||
pool_summary = {}
|
||||
for instance, values in pools.items():
|
||||
pool_summary[instance] = {'max_acquired':max(v['acquired'] for v in values),'max_connections':values[-1]['max'],
|
||||
'empty_acquires_delta':values[-1]['empty_acquires']-values[0]['empty_acquires'],
|
||||
'acquire_seconds_delta':(values[-1]['acquire_ns']-values[0]['acquire_ns'])/1e9}
|
||||
if len(result['instances']) != config['replicas']:
|
||||
raise RuntimeError('load did not reach the configured number of replicas')
|
||||
if not samples or not pool_summary:
|
||||
raise RuntimeError('missing CPU/DB/pool observations')
|
||||
return dict(config, result=result, samples=samples, sample_errors=sample_errors, pool_stats=pool_summary,
|
||||
db_delta={k:after[k]-before[k] for k in before},lb_before=lb_before,lb_after=lb_stats())
|
||||
|
||||
|
||||
def summarize(rows, folder):
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
grouped = {}
|
||||
for row in rows:
|
||||
key = (row['series'],row['replicas'],row['concurrency'],row['pool'])
|
||||
grouped.setdefault(key,[]).append(row)
|
||||
summaries = []
|
||||
for key, values in sorted(grouped.items()):
|
||||
rps = [r['result']['successful_rps'] for r in values]
|
||||
summaries.append({'series':key[0],'replicas':key[1],'concurrency':key[2],'pool':key[3],
|
||||
'rps_median':statistics.median(rps),'rps_min':min(rps),'rps_max':max(rps),
|
||||
'p95_ms_median':statistics.median(r['result']['latency_ms']['p95'] for r in values),
|
||||
'failed_requests':sum(r['result']['failed_requests'] for r in values),
|
||||
'successful_requests':sum(r['result']['successful_requests'] for r in values)})
|
||||
with (folder/'research.csv').open('w',newline='',encoding='utf-8') as output:
|
||||
writer = csv.DictWriter(output,fieldnames=list(summaries[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(summaries)
|
||||
fig, axes = plt.subplots(1,2,figsize=(11,4),layout='constrained')
|
||||
labels = {'fixed':'fixed total: 2 CPU / 1024 MiB','growing':'growing: 1 CPU / 512 MiB per API'}
|
||||
for series in ('fixed','growing'):
|
||||
points = sorted((s for s in summaries if s['series']==series),key=lambda p:p['replicas'])
|
||||
axes[0].errorbar([p['replicas'] for p in points],[p['rps_median'] for p in points],
|
||||
yerr=[[p['rps_median']-p['rps_min'] for p in points],[p['rps_max']-p['rps_median'] for p in points]],marker='o',capsize=4,label=labels[series])
|
||||
axes[1].plot([p['replicas'] for p in points],[p['p95_ms_median'] for p in points],marker='o',label=labels[series])
|
||||
for axis in axes:
|
||||
axis.set_xlabel('API replicas')
|
||||
axis.set_xticks([1,2,4])
|
||||
axis.grid(alpha=.25)
|
||||
axis.legend(fontsize=8)
|
||||
axis.set_ylim(bottom=0)
|
||||
fig.suptitle('3 repetitions; 15 s after 3 s warm-up; 32 closed-loop clients')
|
||||
axes[0].set_ylabel('Successful HTTP confirmations / second')
|
||||
axes[0].set_title('Median and range across repetitions')
|
||||
axes[1].set_ylabel('p95 of successful requests, ms')
|
||||
fig.savefig(folder/'scaling.png',dpi=160)
|
||||
plt.close(fig)
|
||||
sweep = sorted((s for s in summaries if s['series']=='saturation'),key=lambda p:p['concurrency'])
|
||||
fig, axes = plt.subplots(1,3,figsize=(12,4),layout='constrained')
|
||||
clients = [p['concurrency'] for p in sweep]
|
||||
axes[0].errorbar(clients,[p['rps_median'] for p in sweep],
|
||||
yerr=[[p['rps_median']-p['rps_min'] for p in sweep],[p['rps_max']-p['rps_median'] for p in sweep]],marker='o',capsize=4)
|
||||
axes[0].set_ylabel('Successful confirmations / second')
|
||||
axes[1].plot(clients,[p['p95_ms_median'] for p in sweep],marker='o')
|
||||
axes[1].set_ylabel('p95 of successful requests, ms')
|
||||
axes[2].plot(clients,[100*p['failed_requests']/(p['failed_requests']+p['successful_requests']) for p in sweep],marker='o',color='tab:red')
|
||||
axes[2].set_ylabel('Pooled failed attempts, %')
|
||||
for axis in axes:
|
||||
axis.set_xscale('log',base=2)
|
||||
axis.set_xticks(clients,labels=[str(c) for c in clients])
|
||||
axis.set_xlabel('Concurrent clients')
|
||||
axis.set_ylim(bottom=0)
|
||||
axis.grid(alpha=.25)
|
||||
fig.suptitle('Saturation: 4 API replicas; 3 repetitions; 15 s measurements')
|
||||
fig.savefig(folder/'saturation.png',dpi=160)
|
||||
plt.close(fig)
|
||||
tuning = {s['pool']:s for s in summaries if s['series']=='pool-tuning' and s['concurrency']==32}
|
||||
if set(tuning)!={1,8}:
|
||||
raise RuntimeError('controlled pool tuning at concurrency 32 is missing; continue with --resume')
|
||||
improvement = tuning[8]['rps_median']/tuning[1]['rps_median']-1
|
||||
confirmed = improvement>.05 and tuning[8]['failed_requests']<=tuning[1]['failed_requests']
|
||||
lines = ['# Измерения трека 1','',
|
||||
'Fixed: суммарно 2 CPU / 1024 MiB API, суммарный пул 8. Growing: 1 CPU / 512 MiB и пул 8 на каждую API. PostgreSQL 1 CPU / 512 MiB, LB 0.5 CPU / 128 MiB, Valkey 0.5 CPU / 256 MiB, генератор 2 CPU / 256 MiB.','',
|
||||
'| Серия | API | Клиенты | Пул/реплика | RPS median [min;max] | p95 ms | Ошибки |',
|
||||
'|---|---:|---:|---:|---:|---:|---:|']
|
||||
for s in summaries:
|
||||
lines.append('| {series} | {replicas} | {concurrency} | {pool} | {rps_median:.1f} [{rps_min:.1f}; {rps_max:.1f}] | {p95_ms_median:.1f} | {failed_requests} |'.format(**s))
|
||||
lines += ['',f'Контролируемая настройка пула 1 → 8 (2 API, 32 клиента): изменение median RPS {improvement:+.1%}. Критерий подтверждения: >5% без увеличения общего числа ошибок. Подтверждено: {confirmed}.',
|
||||
'', 'Диагностика: research.json содержит CPU/RAM каждого контейнера, активные ожидания PostgreSQL, счётчики ожидания пула и очереди/ошибки HAProxy. Pool-tuning — отдельный намеренно ограниченный сценарий; он не выдаётся за дефект исходного пула 8.',
|
||||
'', 'Нужно интерпретировать насыщение по совместным данным: занятый пул и рост empty_acquires указывают на очередь пула; PostgreSQL около 100% при лимите 1 CPU и отсутствие прироста RPS указывают на CPU БД; очередь/CPU LB — на LB. Один плоский график не доказывает причину.',
|
||||
'', 'Ограничения: один Docker Desktop/хост; closed-loop и coordinated omission; p95 только успехов; контейнерные CPU/RAM и PostgreSQL-счётчики включают прогрев; история растёт; дисковый кэш между сериями не сбрасывается; max RPS промышленной системы не заявляется.','']
|
||||
(folder/'research.md').write_text('\n'.join(lines),encoding='utf-8')
|
||||
return confirmed
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('--duration',type=int,default=15)
|
||||
p.add_argument('--warmup',type=int,default=3)
|
||||
p.add_argument('--repeats',type=int,default=3)
|
||||
p.add_argument('--resume',action='store_true',help='reuse completed runs with identical measurement parameters')
|
||||
p.add_argument('--include-overload',action='store_true',help='also reproduce the rejected 64-client pool experiment')
|
||||
args = p.parse_args()
|
||||
if args.duration<10 or args.warmup<2 or args.repeats<3:
|
||||
p.error('research needs duration>=10, warmup>=2, repeats>=3')
|
||||
import matplotlib # fail before altering running services if dependencies are absent
|
||||
folder = ROOT/'evidence'
|
||||
folder.mkdir(exist_ok=True)
|
||||
metadata = {'timestamp_utc':dt.datetime.now(dt.timezone.utc).isoformat(),'host':platform.platform(),
|
||||
'host_cpu':platform.processor(),'logical_cpus':os.cpu_count(),
|
||||
'docker':json.loads(docker('info','--format','{"cpus":{{.NCPU}},"memory":{{.MemTotal}},"version":"{{.ServerVersion}}","os":"{{.OperatingSystem}}"}',capture=True)),
|
||||
'profile':vars(args),'hypothesis':'Shared DB/LB or pool eventually limits scaling; more API replicas alone cannot remove that limit.'}
|
||||
rows = []
|
||||
saved = folder/'research.json'
|
||||
if args.resume and saved.exists():
|
||||
previous = json.loads(saved.read_text(encoding='utf-8'))
|
||||
old_profile = previous['metadata']['profile']
|
||||
if any(old_profile[k]!=getattr(args,k) for k in ('duration','warmup','repeats')):
|
||||
p.error('saved measurement parameters differ; cannot resume')
|
||||
metadata = previous['metadata']
|
||||
rows = previous['runs']
|
||||
# Upgrade early evidence where the pool telemetry shadowed its setting.
|
||||
for row in rows:
|
||||
if isinstance(row.get('pool'),dict):
|
||||
row['pool_stats'] = row['pool']
|
||||
row['pool'] = next(iter(row['pool_stats'].values()))['max_connections']
|
||||
if row['series']=='pool-tuning' and row['concurrency']==64:
|
||||
row['series']='pool-overload'
|
||||
print('Reusing %d completed measurements' % len(rows),flush=True)
|
||||
try:
|
||||
for repeat in range(args.repeats):
|
||||
configs = []
|
||||
for series in ('fixed','growing'):
|
||||
order = [1,2,4]
|
||||
order = order[repeat%3:]+order[:repeat%3]
|
||||
for n in order:
|
||||
configs.append({'series':series,'replicas':n,'cpu_per_api':2/n if series=='fixed' else 1,
|
||||
'memory_mib_per_api':1024//n if series=='fixed' else 512,
|
||||
'pool':8//n if series=='fixed' else 8,'concurrency':32,'repeat':repeat+1})
|
||||
for pool in ([1,8] if repeat%2==0 else [8,1]):
|
||||
configs.append({'series':'pool-tuning','replicas':2,'cpu_per_api':1,'memory_mib_per_api':512,'pool':pool,'concurrency':32,'repeat':repeat+1})
|
||||
if args.include_overload:
|
||||
for pool in ([1,8] if repeat%2==0 else [8,1]):
|
||||
configs.append({'series':'pool-overload','replicas':2,'cpu_per_api':1,'memory_mib_per_api':512,'pool':pool,'concurrency':64,'repeat':repeat+1})
|
||||
for c in ([1,8,32,64,128] if repeat%2==0 else [128,64,32,8,1]):
|
||||
configs.append({'series':'saturation','replicas':4,'cpu_per_api':1,'memory_mib_per_api':512,'pool':8,'concurrency':c,'repeat':repeat+1})
|
||||
previous = None
|
||||
for config in configs:
|
||||
if any(all(row[k]==config[k] for k in ('series','replicas','concurrency','pool','repeat')) for row in rows):
|
||||
continue
|
||||
state = (config['replicas'],config['cpu_per_api'],config['memory_mib_per_api'],config['pool'])
|
||||
if state!=previous:
|
||||
configure(*state)
|
||||
previous = state
|
||||
print('Measuring '+json.dumps(config),flush=True)
|
||||
rows.append(measure(config,args.duration,args.warmup))
|
||||
(folder/'research.json').write_text(json.dumps({'metadata':metadata,'runs':rows},indent=2)+'\n',encoding='utf-8')
|
||||
confirmed = summarize(rows,folder)
|
||||
if not confirmed:
|
||||
raise RuntimeError('pool tuning hypothesis was not confirmed; see research.md before claiming grade 5')
|
||||
finally:
|
||||
docker('compose','up','-d','--no-deps','--force-recreate','--scale','api=2','api')
|
||||
ready(2)
|
||||
print('PASS: repeated scaling/saturation series and pool improvement measured; see evidence/research.md')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p evidence
|
||||
ids=$(docker compose ps -q db valkey api lb)
|
||||
docker inspect --format '{{.Id}} {{index .Config.Labels "com.docker.compose.service"}} {{.State.StartedAt}} {{.State.Status}}' $ids > evidence/restart-before.txt
|
||||
docker compose restart db valkey api lb
|
||||
docker inspect --format '{{.Id}} {{index .Config.Labels "com.docker.compose.service"}} {{.State.StartedAt}} {{.State.Status}}' $ids > evidence/restart-after.txt
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
for id in $(docker compose ps -q api); do
|
||||
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id")
|
||||
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
|
||||
mkdir -p evidence
|
||||
docker inspect --format '{{.Id}} {{.State.Status}} {{.State.StartedAt}}' "$id" > evidence/stopped-before.txt
|
||||
docker stop --time 9 "$id" > /dev/null
|
||||
docker inspect --format '{{.Id}} {{.State.Status}} {{.State.FinishedAt}} {{.State.ExitCode}}' "$id" > evidence/stopped-after.txt
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
echo 'Target is not a running API container' >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Run the complete lab-3 verification from an ordinary operator terminal."""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = ROOT/'evidence'
|
||||
|
||||
|
||||
def public_integrity():
|
||||
names = subprocess.check_output(['git','ls-files','--','tests','contracts'],cwd=ROOT,text=True).splitlines()
|
||||
hashes = {}
|
||||
for name in names:
|
||||
original = subprocess.check_output(['git','show','HEAD:'+name],cwd=ROOT).replace(b'\r\n',b'\n')
|
||||
current = (ROOT/name).read_bytes().replace(b'\r\n',b'\n')
|
||||
if current != original:
|
||||
raise RuntimeError('protected source changed: '+name)
|
||||
hashes[name] = hashlib.sha256(current).hexdigest()
|
||||
return hashes
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('--no-research',action='store_true',help='run levels 3/4 only; does not certify track 5')
|
||||
p.add_argument('--resume',action='store_true',help='reuse completed work; API/infrastructure sources must be unchanged')
|
||||
p.add_argument('--include-overload',action='store_true',help='also reproduce the additional overload experiment')
|
||||
args = p.parse_args()
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
hashes = public_integrity()
|
||||
results = []
|
||||
previous_checks = {}
|
||||
status = EVIDENCE/'verification.json'
|
||||
if args.resume and status.exists():
|
||||
previous_checks = {item['check']:item for item in json.loads(status.read_text(encoding='utf-8'))['checks'] if item['exit_code']==0}
|
||||
env = dict(os.environ,PYTHONUTF8='1',PYTHONUNBUFFERED='1',GOTELEMETRY='off')
|
||||
|
||||
def run(name,command):
|
||||
if args.resume and name in previous_checks and not name.startswith('final-') and name!='research':
|
||||
results.append(previous_checks[name])
|
||||
print('Reusing passed check: '+name,flush=True)
|
||||
return
|
||||
print('Running '+name+' ...',flush=True)
|
||||
result = subprocess.run(command,cwd=ROOT,env=env,text=True,encoding='utf-8',errors='replace',stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
|
||||
(EVIDENCE/(name+'.txt')).write_text(result.stdout,encoding='utf-8')
|
||||
results.append({'check':name,'exit_code':result.returncode,'command':command})
|
||||
(EVIDENCE/'verification.json').write_text(json.dumps({'timestamp_utc':dt.datetime.now(dt.timezone.utc).isoformat(),'complete':False,'checks':results},indent=2)+'\n',encoding='utf-8')
|
||||
if result.returncode:
|
||||
print(result.stdout[-5000:])
|
||||
raise RuntimeError(name+' failed; see evidence/'+name+'.txt')
|
||||
print('PASS: '+name,flush=True)
|
||||
|
||||
python = sys.executable
|
||||
run('bootstrap',[python,'-X','utf8','scripts/bootstrap.py'])
|
||||
run('docker-info',['docker','info','--format','{{.ServerVersion}} {{.NCPU}} CPU {{.MemTotal}} bytes'])
|
||||
run('compose-check',['docker','compose','config','--quiet'])
|
||||
run('build-start',['docker','compose','up','-d','--build','--wait'])
|
||||
run('build-tests',['docker','build','-f','Dockerfile.tests','-t','mtusi-messenger:lab3-tests','.'])
|
||||
for target in ('check','validate','test','test-own','test-race','test-persistence','test-failover'):
|
||||
run(target,[python,'-X','utf8','scripts/linux_tests.py',target])
|
||||
run('restore',[python,'-X','utf8','scripts/manage.py','restore'])
|
||||
run('failures',[python,'-X','utf8','student-tests/failures.py'])
|
||||
if not args.no_research:
|
||||
run('build-load',['docker','build','--target','load','-t','mtusi-messenger:lab3-load','.'])
|
||||
venv = ROOT/'.venv'
|
||||
vpython = venv/('Scripts/python.exe' if os.name=='nt' else 'bin/python')
|
||||
if not vpython.exists():
|
||||
run('research-env',[python,'-m','venv',str(venv)])
|
||||
run('research-dependencies',[str(vpython),'-m','pip','install','--disable-pip-version-check','-r','requirements-research.txt'])
|
||||
research_command = [str(vpython),'-X','utf8','scripts/research.py']
|
||||
if args.resume:
|
||||
research_command.append('--resume')
|
||||
if args.include_overload:
|
||||
research_command.append('--include-overload')
|
||||
run('research',research_command)
|
||||
run('final-smoke',[python,'-X','utf8','tests/smoke.py','--base-url','http://127.0.0.1:8080'])
|
||||
run('final-own',[python,'-X','utf8','student-tests/acceptance.py','--base-url','http://127.0.0.1:8080'])
|
||||
if public_integrity()!=hashes:
|
||||
raise RuntimeError('public sources changed during verification')
|
||||
(EVIDENCE/'public-integrity.json').write_text(json.dumps(hashes,indent=2)+'\n',encoding='utf-8')
|
||||
run('final-containers',[python,'-X','utf8','scripts/manage.py','status'])
|
||||
complete = not args.no_research
|
||||
(EVIDENCE/'verification.json').write_text(json.dumps({'timestamp_utc':dt.datetime.now(dt.timezone.utc).isoformat(),'complete':complete,'levels_3_4_passed':True,'track_5_measured':complete,'checks':results},indent=2)+'\n',encoding='utf-8')
|
||||
print('Verification finished. '+('Full track-1 evidence is ready.' if complete else 'Levels 3/4 only; run without --no-research for track 5.'))
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user