forked from mtusi2026/mtusi2026lab5
lab5
This commit is contained in:
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
endpoint="${S3_ENDPOINT:-http://s3:9000}"
|
||||
bucket="messenger-attachments"
|
||||
export MC_CONFIG_DIR=/tmp/mc-config
|
||||
attempt=0
|
||||
until mc alias set local "$endpoint" "$(cat /run/secrets/s3_root_user)" \
|
||||
"$(cat /run/secrets/s3_root_password)" >/dev/null 2>&1; do
|
||||
attempt=$((attempt + 1))
|
||||
test "$attempt" -lt 60 || exit 1
|
||||
sleep 1
|
||||
done
|
||||
mc mb --ignore-existing "local/$bucket" >/dev/null
|
||||
mc anonymous set none "local/$bucket" >/dev/null
|
||||
mc admin policy create local messenger-api /policies/s3-api-policy.json >/dev/null
|
||||
mc admin user add local "$(cat /run/secrets/s3_access_key_id)" \
|
||||
"$(cat /run/secrets/s3_secret_access_key)" >/dev/null
|
||||
mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null
|
||||
echo 'Private bucket and limited API credentials ready'
|
||||
@@ -0,0 +1,163 @@
|
||||
"""Трек 2: повторяемые реальные crashes, дубли, reorder, версии и восстановление."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import statistics
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT))
|
||||
from scripts.stack import command,containers,wait,save
|
||||
|
||||
|
||||
class Runner:
|
||||
def __init__(self):
|
||||
self.env=dict(os.environ,ALLOW_FAULT_INJECTION='1',WORKER_CONCURRENCY='1')
|
||||
self.base=os.getenv('BASE_URL','http://localhost:8080')
|
||||
|
||||
def call(self,mode,ctx=None,**options):
|
||||
arguments=['docker','compose','run','--rm','--no-deps','-T','-e','BASE_URL=http://lb:8080',
|
||||
'dispatcher','python','student-tests/pipeline_scenario.py',mode]
|
||||
for key,value in options.items():arguments.extend(['--'+key.replace('_','-'),str(value)])
|
||||
# Контекст с Bearer остаётся в памяти клиента и stdin, не в argv/evidence.
|
||||
return json.loads(command(arguments,env=self.env,input=json.dumps(ctx or {}),timeout=150))
|
||||
|
||||
def service(self,action,name):
|
||||
if action=='stop':command(['docker','compose','stop',name],env=self.env)
|
||||
elif action=='restart':
|
||||
command(['docker','compose','stop',name],env=self.env)
|
||||
command(['docker','compose','up','-d','--no-deps','--wait',name],env=self.env)
|
||||
else:
|
||||
args=['docker','compose','up','-d','--no-deps']
|
||||
if action=='recreate':args.append('--force-recreate')
|
||||
command([*args,name],env=self.env)
|
||||
if name in {'api','s3'} and action!='stop':wait(2,self.base)
|
||||
|
||||
def infrastructure(self):return containers()
|
||||
|
||||
def finish(self):
|
||||
normal=dict(os.environ,ALLOW_FAULT_INJECTION='0')
|
||||
command(['docker','compose','up','-d','--no-deps','--force-recreate','worker','dispatcher'],env=normal)
|
||||
command(['docker','compose','up','-d','--no-deps','reconciler'],env=normal)
|
||||
wait(2,self.base)
|
||||
|
||||
|
||||
def same_effect(before,after):
|
||||
assert before['attachment']==after['attachment'],'Duplicate altered public metadata'
|
||||
assert before['current_keys']==after['current_keys'],'Duplicate multiplied variants'
|
||||
assert before['variants_sha256']==after['variants_sha256'],'Duplicate changed bytes'
|
||||
assert before['original_sha256']==after['original_sha256']
|
||||
|
||||
|
||||
def run_suite(runner,repeats,on_result):
|
||||
results=[]
|
||||
def ready(ctx):
|
||||
runner.call('wait',ctx)
|
||||
return runner.call('snapshot',ctx)
|
||||
def record(name,repeat,started,**values):
|
||||
row=dict(case=name,repetition=repeat,status='PASS',elapsed_seconds=round(time.monotonic()-started,3),**values)
|
||||
results.append(row);on_result(results)
|
||||
print(f'PASS: {name}, repeat {repeat}/{repeats}',flush=True)
|
||||
runner.service('stop','reconciler')
|
||||
for repeat in range(1,repeats+1):
|
||||
started=time.monotonic();runner.service('stop','worker');runner.service('start','dispatcher')
|
||||
ctx=runner.call('create');queued=runner.call('queued',ctx)
|
||||
published=runner.call('published',ctx)
|
||||
runner.service('stop','dispatcher');runner.service('recreate','api');runner.service('restart','broker')
|
||||
restored=runner.call('published',ctx);assert restored['stream_ids']==published['stream_ids']
|
||||
runner.service('start','dispatcher');runner.service('recreate','worker')
|
||||
record('accepted_restart_broker_api',repeat,started,queued=queued,broker_preserved=restored,result=ready(ctx),infrastructure=runner.infrastructure())
|
||||
for phase in ['before_publish','after_publish']:
|
||||
started=time.monotonic();runner.service('stop','worker');runner.service('stop','dispatcher')
|
||||
ctx=runner.call('create');runner.call('arm',ctx,phase=phase)
|
||||
runner.service('recreate','dispatcher');fired=runner.call('fault-fired',ctx,phase=phase)
|
||||
runner.service('recreate','worker');result=ready(ctx)
|
||||
record(phase,repeat,started,fault=fired,result=result,infrastructure=runner.infrastructure())
|
||||
for phase in ['before_effect','after_original','after_effects','after_commit']:
|
||||
started=time.monotonic();runner.service('stop','worker')
|
||||
ctx=runner.call('create');runner.call('arm',ctx,phase=phase)
|
||||
runner.service('recreate','worker');fired=runner.call('fault-fired',ctx,phase=phase)
|
||||
result=ready(ctx)
|
||||
if phase=='after_commit':assert result['attempts']==1,'Committed result was processed again'
|
||||
else:assert result['attempts']>=2,'Abandoned lease was not recovered'
|
||||
record(phase,repeat,started,fault=fired,result=result,infrastructure=runner.infrastructure())
|
||||
started=time.monotonic();ctx=runner.call('create');before=ready(ctx)
|
||||
copies=runner.call('duplicate',dict(ctx,copies=16))
|
||||
acked=runner.call('drained',dict(ctx,stream_ids=copies['stream_ids']))
|
||||
after=runner.call('snapshot',ctx);same_effect(before,after)
|
||||
rejected=runner.call('invalid-job',ctx)
|
||||
assert len([r for r in after['events'] if r['event']=='duplicate_ready'])>=16
|
||||
record('duplicates_and_invalid_carrier',repeat,started,duplicates=acked,invalid_carrier=rejected,result=after)
|
||||
# Новая job приходит первой, старая после неё: контролируемый reorder.
|
||||
started=time.monotonic();runner.service('stop','worker');runner.service('stop','dispatcher')
|
||||
old=runner.call('create');new=runner.call('version',old)
|
||||
first=runner.call('duplicate',new);second=runner.call('duplicate',old)
|
||||
runner.service('recreate','worker')
|
||||
runner.call('drained',dict(new,stream_ids=first['stream_ids']+second['stream_ids']))
|
||||
result=ready(new);old_audit=runner.call('audit',old)
|
||||
assert any(r['event']=='stale_ignored' for r in old_audit['events'])
|
||||
assert result['pipeline_version']==2
|
||||
record('reorder_new_before_old',repeat,started,result=result,old_job=old_audit)
|
||||
runner.service('start','dispatcher')
|
||||
# Теперь старая версия уже имела ready; её поздний дубль не меняет v2.
|
||||
started=time.monotonic();old=runner.call('create');v1=ready(old)
|
||||
new=runner.call('version',old);before=ready(new)
|
||||
assert before['variants_sha256']!=v1['variants_sha256'],'Pipeline version did not change actual output'
|
||||
copies=runner.call('duplicate',dict(old,copies=3));runner.call('drained',dict(old,stream_ids=copies['stream_ids']))
|
||||
after=runner.call('snapshot',new);same_effect(before,after)
|
||||
record('stale_after_new_ready',repeat,started,result=after,old_job=runner.call('audit',old))
|
||||
started=time.monotonic();runner.service('stop','worker');ctx=runner.call('create')
|
||||
runner.call('missing-stage',ctx);runner.service('recreate','worker');runner.call('wait',ctx,state='failed')
|
||||
failed=runner.call('failed',ctx);assert failed['error_code']=='staging_missing'
|
||||
runner.call('restore-stage',ctx);runner.call('replay',ctx);result=ready(ctx)
|
||||
noop=runner.call('replay',ctx);assert noop['status']=='noop'
|
||||
record('permanent_failure_replay',repeat,started,failed=failed,ready_replay=noop,result=result)
|
||||
# Физический stop S3, а не подмена ответа или бесконечная retry-петля.
|
||||
started=time.monotonic();runner.service('stop','worker');ctx=runner.call('create')
|
||||
runner.service('stop','s3')
|
||||
try:
|
||||
time.sleep(4);unready=runner.call('unready',ctx)
|
||||
runner.service('recreate','worker');runner.call('wait',ctx,state='failed',timeout=120)
|
||||
finally:runner.service('start','s3')
|
||||
failed=runner.call('failed',ctx);assert failed['error_code']=='retry_exhausted'
|
||||
assert failed['attempts']==5
|
||||
runner.call('replay',ctx);result=ready(ctx)
|
||||
record('bounded_retry_and_replay',repeat,started,unready=unready,failed=failed,result=result)
|
||||
started=time.monotonic();runner.service('stop','worker');ctx=runner.call('create')
|
||||
cleaned=runner.call('cleanup',ctx);runner.service('recreate','worker');result=ready(ctx)
|
||||
privacy=runner.call('privacy',ctx)
|
||||
record('staging_retention_and_privacy',repeat,started,cleanup=cleaned,privacy=privacy,result=result)
|
||||
return results
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser();p.add_argument('--repeats',type=int,default=3);args=p.parse_args()
|
||||
if not 1<=args.repeats<=5:p.error('repeats must be 1..5')
|
||||
runner=Runner()
|
||||
save('pipeline_environment.json',dict(docker=command(['docker','version','--format','{{.Server.Version}}']),
|
||||
compose=command(['docker','compose','version','--short']),containers=containers(),client_cpus=os.cpu_count(),
|
||||
pipeline=runner.call('config')))
|
||||
def persist(rows):save('pipeline_raw.json',dict(status='RUNNING',track=2,repeats=args.repeats,cases=rows))
|
||||
persist([]);save('pipeline_summary.json',dict(status='RUNNING',expected_checks=13*args.repeats))
|
||||
try:
|
||||
try:rows=run_suite(runner,args.repeats,persist)
|
||||
finally:runner.finish()
|
||||
except Exception as exc:
|
||||
save('pipeline_summary.json',dict(status='FAILED',reason=type(exc).__name__))
|
||||
raise
|
||||
save('pipeline_raw.json',dict(status='PASS',track=2,repeats=args.repeats,cases=rows))
|
||||
summary=[]
|
||||
for case in dict.fromkeys(r['case'] for r in rows):
|
||||
subset=[r for r in rows if r['case']==case]
|
||||
summary.append(dict(case=case,repetitions=len(subset),status='PASS',
|
||||
median_seconds=statistics.median(r['elapsed_seconds'] for r in subset)))
|
||||
save('pipeline_summary.json',dict(status='PASS',track=2,cases=summary,checks=len(rows),
|
||||
invariants=['accepted work preserved','one ready commit per job','exactly three named variants',
|
||||
'original checksum preserved','stale version cannot replace current metadata/bytes',
|
||||
'bounded retries and explicit replay','staging cleanup protects active jobs']))
|
||||
print('Experiment completed; results saved in evidence/',flush=True)
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
import urllib.request
|
||||
|
||||
|
||||
host = "127.0.0.1"
|
||||
port = os.getenv("APP_PORT", "8080")
|
||||
with urllib.request.urlopen(f"http://{host}:{port}/health/ready", timeout=2) as response:
|
||||
payload = json.load(response)
|
||||
if response.status != 200 or payload != {"status": "ok"}:
|
||||
raise SystemExit(1)
|
||||
Executable
+10
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop worker dispatcher reconciler lb api cache db s3 broker
|
||||
docker compose up -d --wait db cache s3 broker
|
||||
docker compose run --rm --no-deps migrate
|
||||
docker compose run --rm --no-deps s3-bootstrap
|
||||
docker compose up -d --no-deps --wait api
|
||||
docker compose up -d --no-deps lb dispatcher worker reconciler
|
||||
python3 scripts/stack.py wait --count 2
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose start api
|
||||
python3 scripts/stack.py wait --count 2
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p .secrets
|
||||
chmod 700 .secrets
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
password_file = Path('.secrets/db_password')
|
||||
if not password_file.exists():
|
||||
password_file.write_text(secrets.token_hex(24))
|
||||
password = password_file.read_text().strip()
|
||||
Path('.secrets/database_url').write_text(f'postgresql://messenger:{password}@db:5432/messenger')
|
||||
for name in ['s3_root_user', 's3_root_password', 's3_access_key_id', 's3_secret_access_key']:
|
||||
file = Path('.secrets') / name
|
||||
if not file.exists():
|
||||
file.write_text(secrets.token_hex(16 if name.endswith(('user', 'id')) else 32))
|
||||
broker_password=Path('.secrets/broker_password')
|
||||
if not broker_password.exists(): broker_password.write_text(secrets.token_hex(24))
|
||||
Path('.secrets/broker_url').write_text('redis://:'+broker_password.read_text().strip()+'@broker:6379/0')
|
||||
for file in Path('.secrets').iterdir():
|
||||
file.chmod(0o644 if file.name in {'database_url', 's3_access_key_id', 's3_secret_access_key', 'broker_url'} else 0o600)
|
||||
if not Path('.env').exists():
|
||||
Path('.env').write_text('APP_PORT=8080\nSESSION_TTL_SECONDS=3600\nDB_POOL_MAX=8\nAPI_CPU=1.0\nAPI_MEMORY=512m\nWORKER_CONCURRENCY=1\nJOB_MAX_ATTEMPTS=5\nJOB_TIMEOUT_SECONDS=30\nJOB_LEASE_SECONDS=15\nSTAGING_RETENTION_SECONDS=86400\nMAX_PENDING_JOBS=1000\nALLOW_FAULT_INJECTION=0\n')
|
||||
print('Локальные настройки подготовлены; секреты не выводятся.')
|
||||
PY
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Небольшие операции со стендом: состояние, ожидание, масштабирование и замена."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def command(args, env=None, timeout=120, input=None):
|
||||
result = subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True, timeout=timeout, input=input)
|
||||
if result.returncode:
|
||||
raise RuntimeError(result.stderr[-2000:])
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def containers(service=None):
|
||||
args = ['docker', 'compose', 'ps', '-a', '-q']
|
||||
if service:
|
||||
args.append(service)
|
||||
ids = command(args).split()
|
||||
if not ids:
|
||||
return []
|
||||
# Сохраняем только нужные поля, без environment с возможными секретами.
|
||||
rows = json.loads(command(['docker', 'inspect', *ids]))
|
||||
return [dict(id=r['Id'], name=r['Name'].lstrip('/'), hostname=r['Config']['Hostname'],
|
||||
service=r['Config']['Labels']['com.docker.compose.service'],
|
||||
running=r['State']['Running'], started_at=r['State']['StartedAt'], restart_count=r.get('RestartCount',0),
|
||||
cpu_limit=r['HostConfig']['NanoCpus'] / 1e9,
|
||||
memory_limit_bytes=r['HostConfig']['Memory'],
|
||||
published_ports=r['NetworkSettings']['Ports']) for r in rows]
|
||||
|
||||
|
||||
def save(name, data):
|
||||
path = ROOT / 'evidence' / name
|
||||
path.parent.mkdir(exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
|
||||
|
||||
|
||||
def wait(count, base=None):
|
||||
base = base or os.getenv('BASE_URL', 'http://localhost:8080')
|
||||
deadline = time.monotonic() + 75
|
||||
while time.monotonic() < deadline:
|
||||
observed = set()
|
||||
for _ in range(max(12, count * 8)):
|
||||
try:
|
||||
with urllib.request.urlopen(base + '/health/ready', timeout=3) as response:
|
||||
observed.add(response.headers['X-Instance-Id'])
|
||||
except (OSError, ValueError):
|
||||
break
|
||||
if len(observed) == count and None not in observed:
|
||||
return sorted(observed)
|
||||
time.sleep(.5)
|
||||
raise RuntimeError(f'Не дождались {count} готовых API через LB')
|
||||
|
||||
|
||||
def stop(instance):
|
||||
hostname = instance.split(':')[0]
|
||||
found = [r for r in containers('api') if r['hostname'] == hostname and r['running']]
|
||||
if len(found) != 1:
|
||||
raise RuntimeError('Instance alias не соответствует одной работающей API-реплике')
|
||||
before = containers()
|
||||
command(['docker', 'stop', '--time', '3', found[0]['id']])
|
||||
result = dict(target=instance, stopped_container=found[0]['id'], before=before, after=containers())
|
||||
save('last_stop.json', result)
|
||||
return result
|
||||
|
||||
|
||||
def scale(count, cpu, memory, pool):
|
||||
env = dict(os.environ, API_CPU=str(cpu), API_MEMORY=memory, DB_POOL_MAX=str(pool))
|
||||
command(['docker', 'compose', 'up', '-d', '--no-deps', '--force-recreate',
|
||||
'--scale', f'api={count}', 'api'], env)
|
||||
instances = wait(count)
|
||||
rows = containers('api')
|
||||
if len(rows) != count or not all(r['running'] for r in rows):
|
||||
raise RuntimeError('Фактическое число контейнеров отличается от запрошенного')
|
||||
for row in rows:
|
||||
if abs(row['cpu_limit'] - cpu) > .001:
|
||||
raise RuntimeError('Docker не применил CPU limit')
|
||||
if row['memory_limit_bytes'] != int(memory.rstrip('m')) * 1024 * 1024:
|
||||
raise RuntimeError('Docker не применил RAM limit')
|
||||
return dict(instances=instances, containers=rows)
|
||||
|
||||
|
||||
def rolling():
|
||||
old = [r['id'] for r in containers('api') if r['running']]
|
||||
if len(old) != 2:
|
||||
raise RuntimeError('Rolling проверяется на двух репликах обычного стенда')
|
||||
before = containers()
|
||||
command(['docker', 'compose', 'build', 'api'])
|
||||
steps = []
|
||||
for container in old:
|
||||
# Не пересоздаём одновременно старые контейнеры при изменении image.
|
||||
command(['docker', 'compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=3', 'api'])
|
||||
wait(3)
|
||||
command(['docker', 'stop', '--time', '10', container])
|
||||
command(['docker', 'rm', container])
|
||||
wait(2)
|
||||
steps.append(containers('api'))
|
||||
save('rolling_containers.json', dict(before=before, steps=steps, after=containers()))
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('action', choices=['wait', 'snapshot', 'stop', 'rolling'])
|
||||
p.add_argument('--count', type=int, default=2)
|
||||
p.add_argument('--instance')
|
||||
p.add_argument('--output', default='containers.json')
|
||||
args = p.parse_args()
|
||||
if args.action == 'wait':
|
||||
print('API ready:', ', '.join(wait(args.count)))
|
||||
elif args.action == 'snapshot':
|
||||
save(args.output, containers())
|
||||
print('Container identities and limits saved')
|
||||
elif args.action == 'stop':
|
||||
if not args.instance:
|
||||
p.error('--instance required')
|
||||
stop(args.instance)
|
||||
print('Selected API stopped')
|
||||
else:
|
||||
rolling()
|
||||
print('API replaced one at a time')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose up -d --build
|
||||
python3 scripts/stack.py wait --count 2
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
python3 scripts/stack.py stop --instance "$TARGET_INSTANCE_ID"
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
restore() {
|
||||
docker compose up -d --no-deps --force-recreate --scale api=2 api
|
||||
python3 scripts/stack.py wait --count 2
|
||||
}
|
||||
trap restore EXIT
|
||||
SESSION_TTL_SECONDS=2 docker compose up -d --no-deps --force-recreate --scale api=2 api
|
||||
python3 scripts/stack.py wait --count 2
|
||||
python3 student-tests/checks.py --expiry
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Ожидание готовности API без сторонних библиотек."""
|
||||
import os
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
url = os.getenv('BASE_URL', 'http://localhost:8080') + '/health/ready'
|
||||
for attempt in range(90):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=2) as response:
|
||||
if response.status == 200:
|
||||
print('API ready')
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise SystemExit('API не стал готов за 90 попыток')
|
||||
Reference in New Issue
Block a user