This commit is contained in:
2026-10-26 18:05:06 +03:00
parent 276dbdad8a
commit 78591e033c
96 changed files with 17965 additions and 21 deletions
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
set -eu
endpoint="${S3_ENDPOINT:-http://s3:9000}"
bucket="messenger-attachments"
export MC_CONFIG_DIR=/tmp/mc-config
attempt=0
until mc alias set local "$endpoint" "$(cat /run/secrets/s3_root_user)" \
"$(cat /run/secrets/s3_root_password)" >/dev/null 2>&1; do
attempt=$((attempt + 1))
test "$attempt" -lt 60 || exit 1
sleep 1
done
mc mb --ignore-existing "local/$bucket" >/dev/null
mc anonymous set none "local/$bucket" >/dev/null
mc admin policy create local messenger-api /policies/s3-api-policy.json >/dev/null
mc admin user add local "$(cat /run/secrets/s3_access_key_id)" \
"$(cat /run/secrets/s3_secret_access_key)" >/dev/null
mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null
echo 'Private bucket and limited API credentials ready'
+163
View File
@@ -0,0 +1,163 @@
"""Трек 2: повторяемые реальные crashes, дубли, reorder, версии и восстановление."""
import argparse
import json
import os
from pathlib import Path
import statistics
import sys
import time
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT))
from scripts.stack import command,containers,wait,save
class Runner:
def __init__(self):
self.env=dict(os.environ,ALLOW_FAULT_INJECTION='1',WORKER_CONCURRENCY='1')
self.base=os.getenv('BASE_URL','http://localhost:8080')
def call(self,mode,ctx=None,**options):
arguments=['docker','compose','run','--rm','--no-deps','-T','-e','BASE_URL=http://lb:8080',
'dispatcher','python','student-tests/pipeline_scenario.py',mode]
for key,value in options.items():arguments.extend(['--'+key.replace('_','-'),str(value)])
# Контекст с Bearer остаётся в памяти клиента и stdin, не в argv/evidence.
return json.loads(command(arguments,env=self.env,input=json.dumps(ctx or {}),timeout=150))
def service(self,action,name):
if action=='stop':command(['docker','compose','stop',name],env=self.env)
elif action=='restart':
command(['docker','compose','stop',name],env=self.env)
command(['docker','compose','up','-d','--no-deps','--wait',name],env=self.env)
else:
args=['docker','compose','up','-d','--no-deps']
if action=='recreate':args.append('--force-recreate')
command([*args,name],env=self.env)
if name in {'api','s3'} and action!='stop':wait(2,self.base)
def infrastructure(self):return containers()
def finish(self):
normal=dict(os.environ,ALLOW_FAULT_INJECTION='0')
command(['docker','compose','up','-d','--no-deps','--force-recreate','worker','dispatcher'],env=normal)
command(['docker','compose','up','-d','--no-deps','reconciler'],env=normal)
wait(2,self.base)
def same_effect(before,after):
assert before['attachment']==after['attachment'],'Duplicate altered public metadata'
assert before['current_keys']==after['current_keys'],'Duplicate multiplied variants'
assert before['variants_sha256']==after['variants_sha256'],'Duplicate changed bytes'
assert before['original_sha256']==after['original_sha256']
def run_suite(runner,repeats,on_result):
results=[]
def ready(ctx):
runner.call('wait',ctx)
return runner.call('snapshot',ctx)
def record(name,repeat,started,**values):
row=dict(case=name,repetition=repeat,status='PASS',elapsed_seconds=round(time.monotonic()-started,3),**values)
results.append(row);on_result(results)
print(f'PASS: {name}, repeat {repeat}/{repeats}',flush=True)
runner.service('stop','reconciler')
for repeat in range(1,repeats+1):
started=time.monotonic();runner.service('stop','worker');runner.service('start','dispatcher')
ctx=runner.call('create');queued=runner.call('queued',ctx)
published=runner.call('published',ctx)
runner.service('stop','dispatcher');runner.service('recreate','api');runner.service('restart','broker')
restored=runner.call('published',ctx);assert restored['stream_ids']==published['stream_ids']
runner.service('start','dispatcher');runner.service('recreate','worker')
record('accepted_restart_broker_api',repeat,started,queued=queued,broker_preserved=restored,result=ready(ctx),infrastructure=runner.infrastructure())
for phase in ['before_publish','after_publish']:
started=time.monotonic();runner.service('stop','worker');runner.service('stop','dispatcher')
ctx=runner.call('create');runner.call('arm',ctx,phase=phase)
runner.service('recreate','dispatcher');fired=runner.call('fault-fired',ctx,phase=phase)
runner.service('recreate','worker');result=ready(ctx)
record(phase,repeat,started,fault=fired,result=result,infrastructure=runner.infrastructure())
for phase in ['before_effect','after_original','after_effects','after_commit']:
started=time.monotonic();runner.service('stop','worker')
ctx=runner.call('create');runner.call('arm',ctx,phase=phase)
runner.service('recreate','worker');fired=runner.call('fault-fired',ctx,phase=phase)
result=ready(ctx)
if phase=='after_commit':assert result['attempts']==1,'Committed result was processed again'
else:assert result['attempts']>=2,'Abandoned lease was not recovered'
record(phase,repeat,started,fault=fired,result=result,infrastructure=runner.infrastructure())
started=time.monotonic();ctx=runner.call('create');before=ready(ctx)
copies=runner.call('duplicate',dict(ctx,copies=16))
acked=runner.call('drained',dict(ctx,stream_ids=copies['stream_ids']))
after=runner.call('snapshot',ctx);same_effect(before,after)
rejected=runner.call('invalid-job',ctx)
assert len([r for r in after['events'] if r['event']=='duplicate_ready'])>=16
record('duplicates_and_invalid_carrier',repeat,started,duplicates=acked,invalid_carrier=rejected,result=after)
# Новая job приходит первой, старая после неё: контролируемый reorder.
started=time.monotonic();runner.service('stop','worker');runner.service('stop','dispatcher')
old=runner.call('create');new=runner.call('version',old)
first=runner.call('duplicate',new);second=runner.call('duplicate',old)
runner.service('recreate','worker')
runner.call('drained',dict(new,stream_ids=first['stream_ids']+second['stream_ids']))
result=ready(new);old_audit=runner.call('audit',old)
assert any(r['event']=='stale_ignored' for r in old_audit['events'])
assert result['pipeline_version']==2
record('reorder_new_before_old',repeat,started,result=result,old_job=old_audit)
runner.service('start','dispatcher')
# Теперь старая версия уже имела ready; её поздний дубль не меняет v2.
started=time.monotonic();old=runner.call('create');v1=ready(old)
new=runner.call('version',old);before=ready(new)
assert before['variants_sha256']!=v1['variants_sha256'],'Pipeline version did not change actual output'
copies=runner.call('duplicate',dict(old,copies=3));runner.call('drained',dict(old,stream_ids=copies['stream_ids']))
after=runner.call('snapshot',new);same_effect(before,after)
record('stale_after_new_ready',repeat,started,result=after,old_job=runner.call('audit',old))
started=time.monotonic();runner.service('stop','worker');ctx=runner.call('create')
runner.call('missing-stage',ctx);runner.service('recreate','worker');runner.call('wait',ctx,state='failed')
failed=runner.call('failed',ctx);assert failed['error_code']=='staging_missing'
runner.call('restore-stage',ctx);runner.call('replay',ctx);result=ready(ctx)
noop=runner.call('replay',ctx);assert noop['status']=='noop'
record('permanent_failure_replay',repeat,started,failed=failed,ready_replay=noop,result=result)
# Физический stop S3, а не подмена ответа или бесконечная retry-петля.
started=time.monotonic();runner.service('stop','worker');ctx=runner.call('create')
runner.service('stop','s3')
try:
time.sleep(4);unready=runner.call('unready',ctx)
runner.service('recreate','worker');runner.call('wait',ctx,state='failed',timeout=120)
finally:runner.service('start','s3')
failed=runner.call('failed',ctx);assert failed['error_code']=='retry_exhausted'
assert failed['attempts']==5
runner.call('replay',ctx);result=ready(ctx)
record('bounded_retry_and_replay',repeat,started,unready=unready,failed=failed,result=result)
started=time.monotonic();runner.service('stop','worker');ctx=runner.call('create')
cleaned=runner.call('cleanup',ctx);runner.service('recreate','worker');result=ready(ctx)
privacy=runner.call('privacy',ctx)
record('staging_retention_and_privacy',repeat,started,cleanup=cleaned,privacy=privacy,result=result)
return results
def main():
p=argparse.ArgumentParser();p.add_argument('--repeats',type=int,default=3);args=p.parse_args()
if not 1<=args.repeats<=5:p.error('repeats must be 1..5')
runner=Runner()
save('pipeline_environment.json',dict(docker=command(['docker','version','--format','{{.Server.Version}}']),
compose=command(['docker','compose','version','--short']),containers=containers(),client_cpus=os.cpu_count(),
pipeline=runner.call('config')))
def persist(rows):save('pipeline_raw.json',dict(status='RUNNING',track=2,repeats=args.repeats,cases=rows))
persist([]);save('pipeline_summary.json',dict(status='RUNNING',expected_checks=13*args.repeats))
try:
try:rows=run_suite(runner,args.repeats,persist)
finally:runner.finish()
except Exception as exc:
save('pipeline_summary.json',dict(status='FAILED',reason=type(exc).__name__))
raise
save('pipeline_raw.json',dict(status='PASS',track=2,repeats=args.repeats,cases=rows))
summary=[]
for case in dict.fromkeys(r['case'] for r in rows):
subset=[r for r in rows if r['case']==case]
summary.append(dict(case=case,repetitions=len(subset),status='PASS',
median_seconds=statistics.median(r['elapsed_seconds'] for r in subset)))
save('pipeline_summary.json',dict(status='PASS',track=2,cases=summary,checks=len(rows),
invariants=['accepted work preserved','one ready commit per job','exactly three named variants',
'original checksum preserved','stale version cannot replace current metadata/bytes',
'bounded retries and explicit replay','staging cleanup protects active jobs']))
print('Experiment completed; results saved in evidence/',flush=True)
if __name__=='__main__':main()
+12
View File
@@ -0,0 +1,12 @@
#!/usr/bin/env python3
import json
import os
import urllib.request
host = "127.0.0.1"
port = os.getenv("APP_PORT", "8080")
with urllib.request.urlopen(f"http://{host}:{port}/health/ready", timeout=2) as response:
payload = json.load(response)
if response.status != 200 or payload != {"status": "ok"}:
raise SystemExit(1)
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
docker compose stop worker dispatcher reconciler lb api cache db s3 broker
docker compose up -d --wait db cache s3 broker
docker compose run --rm --no-deps migrate
docker compose run --rm --no-deps s3-bootstrap
docker compose up -d --no-deps --wait api
docker compose up -d --no-deps lb dispatcher worker reconciler
python3 scripts/stack.py wait --count 2
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
docker compose start api
python3 scripts/stack.py wait --count 2
+26
View File
@@ -0,0 +1,26 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
mkdir -p .secrets
chmod 700 .secrets
python3 - <<'PY'
from pathlib import Path
import secrets
password_file = Path('.secrets/db_password')
if not password_file.exists():
password_file.write_text(secrets.token_hex(24))
password = password_file.read_text().strip()
Path('.secrets/database_url').write_text(f'postgresql://messenger:{password}@db:5432/messenger')
for name in ['s3_root_user', 's3_root_password', 's3_access_key_id', 's3_secret_access_key']:
file = Path('.secrets') / name
if not file.exists():
file.write_text(secrets.token_hex(16 if name.endswith(('user', 'id')) else 32))
broker_password=Path('.secrets/broker_password')
if not broker_password.exists(): broker_password.write_text(secrets.token_hex(24))
Path('.secrets/broker_url').write_text('redis://:'+broker_password.read_text().strip()+'@broker:6379/0')
for file in Path('.secrets').iterdir():
file.chmod(0o644 if file.name in {'database_url', 's3_access_key_id', 's3_secret_access_key', 'broker_url'} else 0o600)
if not Path('.env').exists():
Path('.env').write_text('APP_PORT=8080\nSESSION_TTL_SECONDS=3600\nDB_POOL_MAX=8\nAPI_CPU=1.0\nAPI_MEMORY=512m\nWORKER_CONCURRENCY=1\nJOB_MAX_ATTEMPTS=5\nJOB_TIMEOUT_SECONDS=30\nJOB_LEASE_SECONDS=15\nSTAGING_RETENTION_SECONDS=86400\nMAX_PENDING_JOBS=1000\nALLOW_FAULT_INJECTION=0\n')
print('Локальные настройки подготовлены; секреты не выводятся.')
PY
+129
View File
@@ -0,0 +1,129 @@
"""Небольшие операции со стендом: состояние, ожидание, масштабирование и замена."""
import argparse
import json
import os
from pathlib import Path
import subprocess
import time
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
def command(args, env=None, timeout=120, input=None):
result = subprocess.run(args, cwd=ROOT, env=env, text=True, capture_output=True, timeout=timeout, input=input)
if result.returncode:
raise RuntimeError(result.stderr[-2000:])
return result.stdout.strip()
def containers(service=None):
args = ['docker', 'compose', 'ps', '-a', '-q']
if service:
args.append(service)
ids = command(args).split()
if not ids:
return []
# Сохраняем только нужные поля, без environment с возможными секретами.
rows = json.loads(command(['docker', 'inspect', *ids]))
return [dict(id=r['Id'], name=r['Name'].lstrip('/'), hostname=r['Config']['Hostname'],
service=r['Config']['Labels']['com.docker.compose.service'],
running=r['State']['Running'], started_at=r['State']['StartedAt'], restart_count=r.get('RestartCount',0),
cpu_limit=r['HostConfig']['NanoCpus'] / 1e9,
memory_limit_bytes=r['HostConfig']['Memory'],
published_ports=r['NetworkSettings']['Ports']) for r in rows]
def save(name, data):
path = ROOT / 'evidence' / name
path.parent.mkdir(exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n')
def wait(count, base=None):
base = base or os.getenv('BASE_URL', 'http://localhost:8080')
deadline = time.monotonic() + 75
while time.monotonic() < deadline:
observed = set()
for _ in range(max(12, count * 8)):
try:
with urllib.request.urlopen(base + '/health/ready', timeout=3) as response:
observed.add(response.headers['X-Instance-Id'])
except (OSError, ValueError):
break
if len(observed) == count and None not in observed:
return sorted(observed)
time.sleep(.5)
raise RuntimeError(f'Не дождались {count} готовых API через LB')
def stop(instance):
hostname = instance.split(':')[0]
found = [r for r in containers('api') if r['hostname'] == hostname and r['running']]
if len(found) != 1:
raise RuntimeError('Instance alias не соответствует одной работающей API-реплике')
before = containers()
command(['docker', 'stop', '--time', '3', found[0]['id']])
result = dict(target=instance, stopped_container=found[0]['id'], before=before, after=containers())
save('last_stop.json', result)
return result
def scale(count, cpu, memory, pool):
env = dict(os.environ, API_CPU=str(cpu), API_MEMORY=memory, DB_POOL_MAX=str(pool))
command(['docker', 'compose', 'up', '-d', '--no-deps', '--force-recreate',
'--scale', f'api={count}', 'api'], env)
instances = wait(count)
rows = containers('api')
if len(rows) != count or not all(r['running'] for r in rows):
raise RuntimeError('Фактическое число контейнеров отличается от запрошенного')
for row in rows:
if abs(row['cpu_limit'] - cpu) > .001:
raise RuntimeError('Docker не применил CPU limit')
if row['memory_limit_bytes'] != int(memory.rstrip('m')) * 1024 * 1024:
raise RuntimeError('Docker не применил RAM limit')
return dict(instances=instances, containers=rows)
def rolling():
old = [r['id'] for r in containers('api') if r['running']]
if len(old) != 2:
raise RuntimeError('Rolling проверяется на двух репликах обычного стенда')
before = containers()
command(['docker', 'compose', 'build', 'api'])
steps = []
for container in old:
# Не пересоздаём одновременно старые контейнеры при изменении image.
command(['docker', 'compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=3', 'api'])
wait(3)
command(['docker', 'stop', '--time', '10', container])
command(['docker', 'rm', container])
wait(2)
steps.append(containers('api'))
save('rolling_containers.json', dict(before=before, steps=steps, after=containers()))
def main():
p = argparse.ArgumentParser()
p.add_argument('action', choices=['wait', 'snapshot', 'stop', 'rolling'])
p.add_argument('--count', type=int, default=2)
p.add_argument('--instance')
p.add_argument('--output', default='containers.json')
args = p.parse_args()
if args.action == 'wait':
print('API ready:', ', '.join(wait(args.count)))
elif args.action == 'snapshot':
save(args.output, containers())
print('Container identities and limits saved')
elif args.action == 'stop':
if not args.instance:
p.error('--instance required')
stop(args.instance)
print('Selected API stopped')
else:
rolling()
print('API replaced one at a time')
if __name__ == '__main__':
main()
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
docker compose up -d --build
python3 scripts/stack.py wait --count 2
+5
View File
@@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
python3 scripts/stack.py stop --instance "$TARGET_INSTANCE_ID"
+11
View File
@@ -0,0 +1,11 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
restore() {
docker compose up -d --no-deps --force-recreate --scale api=2 api
python3 scripts/stack.py wait --count 2
}
trap restore EXIT
SESSION_TTL_SECONDS=2 docker compose up -d --no-deps --force-recreate --scale api=2 api
python3 scripts/stack.py wait --count 2
python3 student-tests/checks.py --expiry
+17
View File
@@ -0,0 +1,17 @@
"""Ожидание готовности API без сторонних библиотек."""
import os
import time
import urllib.request
url = os.getenv('BASE_URL', 'http://localhost:8080') + '/health/ready'
for attempt in range(90):
try:
with urllib.request.urlopen(url, timeout=2) as response:
if response.status == 200:
print('API ready')
break
except OSError:
pass
time.sleep(1)
else:
raise SystemExit('API не стал готов за 90 попыток')