lab4 done

This commit is contained in:
Khakimov Dmitrij
2026-10-28 22:41:10 +03:00
parent a7bedda183
commit dbc700fafe
75 changed files with 5611 additions and 227 deletions
+7 -2
View File
@@ -4,12 +4,17 @@ cd "$(dirname "$0")/.."
umask 077
mkdir -p secrets
chmod 700 secrets
for name in postgres_password cursor_key; do
for name in postgres_password cursor_key s3_root_secret_key s3_secret_key; do
if [ ! -s "secrets/$name" ]; then
openssl rand -hex 32 > "secrets/$name"
fi
done
for name in s3_root_access_key s3_access_key; do
if [ ! -s "secrets/$name" ]; then
openssl rand -hex 8 > "secrets/$name"
fi
done
# Compose mounts files directly on Linux; the unprivileged API must read them.
# The host directory remains private (0700).
chmod 444 secrets/postgres_password secrets/cursor_key
chmod 444 secrets/postgres_password secrets/cursor_key secrets/s3_root_access_key secrets/s3_root_secret_key secrets/s3_access_key secrets/s3_secret_key
printf '%s\n' 'Local secrets are ready.'
+85
View File
@@ -0,0 +1,85 @@
#!/usr/bin/env python3
"""Ten simultaneous proxy uploads, checksums, latency and cgroup memory samples."""
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
import math
from pathlib import Path
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT/'tests/own'))
from helpers import Client, check, containers, docker, png, save, upload
def memory(ids):
result = {}
for container in ids:
raw = docker('exec',container,'cat','/sys/fs/cgroup/memory.current').stdout
result[container] = int(raw)
return result
def percentile(values, p):
return sorted(values)[max(0, math.ceil(len(values)*p)-1)]
def kernel_peak(ids):
return {container:int(docker('exec',container,'cat','/sys/fs/cgroup/memory.peak').stdout)
for container in ids}
def main():
c = Client(timeout=55)
alice,bob,_,chat = c.bootstrap()
upload(c,alice,chat,png())
ids = containers('api')
results = {'concurrency':10, 'warmup_uploads':1, 'memory_method':'cgroup v2 memory.current sampled every >=100ms; values include file cache', 'runs':[]}
for width,height in ((512,512),(1024,768)):
content = png(width,height,noisy=True)
sha = hashlib.sha256(content).hexdigest()
baseline = memory(ids)
baseline_kernel_peak = kernel_peak(ids)
samples = []
barrier = threading.Barrier(11)
def worker(number):
local = Client(timeout=55)
barrier.wait()
return upload(local,alice,chat,content,filename='concurrent-'+str(number)+'.png')
started = time.monotonic()
with ThreadPoolExecutor(max_workers=10) as pool:
futures = [pool.submit(worker,n) for n in range(10)]
barrier.wait()
while not all(f.done() for f in futures):
samples.append({'seconds':round(time.monotonic()-started,3), 'bytes':memory(ids)})
time.sleep(.1)
responses = [f.result() for f in futures]
elapsed = time.monotonic()-started
samples.append({'seconds':round(elapsed,3), 'bytes':memory(ids)})
for response in responses:
att = response['json']
check(att['sha256']==sha,'concurrent upload checksum mismatch')
downloaded = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
check(hashlib.sha256(downloaded['bytes']).hexdigest()==sha,'concurrent downloaded bytes changed')
latency = [r['elapsed']*1000 for r in responses]
peak = {container:max(sample['bytes'][container] for sample in samples) for container in ids}
run = {'width':width, 'height':height, 'size_bytes':len(content), 'sha256':sha,
'requests':10, 'successful':len(responses), 'errors':0, 'duration_seconds':round(elapsed,3),
'successful_rps':round(len(responses)/elapsed,3),
'latency_ms':{str(p):round(percentile(latency,p),3) for p in (.5,.95,.99)},
'baseline_bytes':baseline, 'sampled_peak_bytes':peak,
'kernel_lifetime_peak_before_bytes':baseline_kernel_peak,
'kernel_lifetime_peak_after_bytes':kernel_peak(ids),
'sampled_peak_sum_bytes':max(sum(sample['bytes'].values()) for sample in samples),
'samples':samples}
check(all(value<1536*1024*1024 for value in peak.values()),'API exceeded memory limit')
results['runs'].append(run)
print(json.dumps({k:v for k,v in run.items() if k!='samples'}),flush=True)
save('lab4-upload-memory.json',results)
print('PASS: ten simultaneous uploads at two sizes; all originals verified; memory samples saved.')
if __name__=='__main__':
main()
+48 -7
View File
@@ -14,7 +14,11 @@ STACK = os.environ.get('STACK_NAME', 'mtusi-lab4')
API = os.environ.get('API_IMAGE', 'mtusi-messenger:lab4')
MIGRATIONS = os.environ.get('MIGRATIONS_IMAGE', 'mtusi-migrations:lab4')
TESTS = os.environ.get('TESTS_IMAGE', 'mtusi-messenger-tests:lab4')
S3 = os.environ.get('S3_IMAGE', 'mtusi-minio:RELEASE.2025-09-07T16-13-09Z')
MC = os.environ.get('S3_CLIENT_IMAGE', 'mtusi-mc:RELEASE.2025-08-13T08-35-41Z')
PORT = os.environ.get('PORT', '8080')
SECRETS = ('postgres_password', 'cursor_key', 's3_root_access_key',
's3_root_secret_key', 's3_access_key', 's3_secret_key')
def docker(*args, capture=False, **kwargs):
@@ -65,7 +69,7 @@ def wait_api():
def job(kind, image, command, integration=False):
name = STACK+'_'+kind
# Fixed service name is a deployment lock: concurrent runners fail to create it.
args = ['service', 'create', '--detach', '--name', name, '--mode', 'replicated-job',
args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job',
'--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data',
'--constraint', 'node.labels.mtusi_lab4_storage==true',
'--limit-cpu', '1', '--limit-memory', '1g' if integration else '128m',
@@ -73,15 +77,23 @@ def job(kind, image, command, integration=False):
'--env', 'POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password']
if integration:
for value in ['INTEGRATION_TESTS=1', 'POSTGRES_HOST=postgres', 'POSTGRES_USER=messenger',
'POSTGRES_DB=messenger', 'REDIS_ADDR=redis:6379', 'CURSOR_KEY_FILE=/run/secrets/cursor_key']:
'POSTGRES_DB=messenger', 'REDIS_ADDR=redis:6379', 'CURSOR_KEY_FILE=/run/secrets/cursor_key',
'S3_ENDPOINT=minio:9000', 'S3_BUCKET=attachments',
'S3_ACCESS_KEY_FILE=/run/secrets/s3_access_key', 'S3_SECRET_KEY_FILE=/run/secrets/s3_secret_key']:
args += ['--env', value]
args += ['--secret', 'source='+STACK+'_cursor_key,target=cursor_key']
for secret in ('s3_access_key', 's3_secret_key'):
args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret]
else:
args += ['--env', 'GOOSE_DRIVER=postgres', '--env',
'GOOSE_DBSTRING=host=postgres user=messenger dbname=messenger sslmode=disable connect_timeout=5']
docker(*args, image, *command)
wait_job(name, kind, 300 if integration else 90)
def wait_job(name, kind, timeout=90):
try:
deadline = time.monotonic()+(300 if integration else 90)
deadline = time.monotonic()+timeout
while time.monotonic() < deadline:
ids = docker('service', 'ps', '-q', name, capture=True).split()
states = [json.loads(docker('inspect', i, capture=True))[0]['Status']['State'] for i in ids]
@@ -97,7 +109,31 @@ def job(kind, image, command, integration=False):
docker('service', 'rm', name)
def bootstrap_s3():
name = STACK+'_s3-bootstrap'
args = ['service', 'create', '--detach', '--no-resolve-image', '--name', name, '--mode', 'replicated-job',
'--replicas', '1', '--restart-condition', 'none', '--network', STACK+'_data',
'--constraint', 'node.labels.mtusi_lab4_storage==true',
'--limit-cpu', '0.5', '--limit-memory', '512m', '--entrypoint', '/bin/sh',
'--config', 'source='+STACK+'_s3_bootstrap,target=/bootstrap.sh',
'--config', 'source='+STACK+'_s3_policy,target=/policy.json']
for secret in SECRETS[2:]:
args += ['--secret', 'source='+STACK+'_'+secret+',target='+secret]
docker(*args, MC, '/bootstrap.sh')
wait_job(name, 'S3 bootstrap')
def reconcile():
active = tasks('api')
if not active:
raise RuntimeError('no API task available for reconciliation')
container = active[0]['Status']['ContainerStatus']['ContainerID']
docker('exec', container, '/usr/local/bin/reconcile', *sys.argv[2:])
def build():
docker('build', '-f', 'Dockerfile.storage', '--target', 'minio', '-t', S3, '.')
docker('build', '-f', 'Dockerfile.storage', '--target', 'mc', '-t', MC, '.')
docker('build', '-t', API, '.')
docker('build', '--target', 'migrations', '-t', MIGRATIONS, '.')
@@ -113,7 +149,7 @@ def up():
raise RuntimeError('automatic lifecycle supports one Swarm node; see REPORT.md for image distribution')
docker('node', 'update', '--label-add', 'mtusi_lab4_storage=true', swarm['NodeID'])
subprocess.run(['./scripts/bootstrap.sh'], check=True)
for secret in ('postgres_password', 'cursor_key'):
for secret in SECRETS:
known = docker('secret', 'ls', '--format', '{{.Name}}', capture=True).split()
if STACK+'_'+secret not in known:
docker('secret', 'create', STACK+'_'+secret, 'secrets/'+secret)
@@ -131,6 +167,8 @@ def up():
docker('stack', 'deploy', '--resolve-image', 'never', '-c', 'stack.yaml', STACK, env=env)
wait_service('postgres')
wait_service('redis')
wait_service('minio')
bootstrap_s3()
if fresh:
job('migrations', MIGRATIONS, ['up'])
docker('service', 'scale', '--detach', STACK+'_api='+os.environ.get('API_REPLICAS', '2'))
@@ -143,7 +181,7 @@ def up():
def restart():
# Force replaces every task while retaining the same Swarm secrets and volumes.
services = ('postgres', 'redis', 'api', 'lb')
services = ('postgres', 'redis', 'minio', 'api', 'lb')
previous = {name: {t['ID'] for t in tasks(name)} for name in services}
for service in services:
docker('service', 'update', '--detach', '--force', STACK+'_'+service)
@@ -155,6 +193,7 @@ def restart():
wait_service('lb')
wait_service('postgres')
wait_service('redis')
wait_service('minio')
wait_api()
@@ -171,7 +210,7 @@ def stop_one():
def update():
previous = {t['ID'] for t in tasks('api')}
docker('service', 'update', '--detach', '--force', '--image', API, STACK+'_api')
docker('service', 'update', '--detach', '--force', '--no-resolve-image', '--image', API, STACK+'_api')
deadline = time.monotonic()+180
while time.monotonic() < deadline:
state = inspect('service', STACK+'_api').get('UpdateStatus', {}).get('State')
@@ -187,7 +226,7 @@ def update():
def status():
result = {'stack': STACK, 'docker': docker('version','--format','{{.Server.Version}}',capture=True).strip(), 'services': {}}
for name in ('lb','api','postgres','redis'):
for name in ('lb','api','postgres','redis','minio'):
service = inspect('service', STACK+'_'+name)
spec = service['Spec']
template = spec['TaskTemplate']
@@ -207,6 +246,8 @@ def main():
if action == 'status': status()
elif action == 'up': up()
elif action == 'build': build()
elif action == 'bootstrap-s3': bootstrap_s3()
elif action == 'reconcile': reconcile()
elif action == 'wait': wait_api()
elif action == 'restart': restart()
elif action == 'stop-one': stop_one()