forked from mtusi2026/mtusi2026lab4
Добавить сценарии рестарта, остановки реплики и локальных проверок
restart.sh (полный цикл стенда, без volumes/secrets), stop-one.sh (одна реплика по TARGET_INSTANCE_ID, данные и секреты сохраняются), check_local.py и check_idempotency.py — воспроизводимые локальные проверки.
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
"""Concurrent HTTP contract check through LB, using one session on two replicas."""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from uuid import uuid4
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
args = parser.parse_args()
|
||||
c = Client(args.base_url)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
key = uuid4().hex
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
||||
def send(_):
|
||||
return Client(args.base_url).message(alice, chat, 'Одно сообщение 👋', key=key)
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
responses = list(pool.map(send, range(24)))
|
||||
first = responses[0]['json']
|
||||
check(all(r['json'] == first and r['status'] == 201 for r in responses), 'Concurrent replay differs')
|
||||
instances = {r['headers']['X-Instance-Id'] for r in responses}
|
||||
check(len(instances) >= 2, 'Concurrent requests did not reach both replicas')
|
||||
page = c.call('GET', path, actor=bob, schema='MessagePage')['json']
|
||||
check(page['items'] == [first], 'Concurrent repeat created duplicate messages')
|
||||
c.call('POST', path, 409, actor=alice, body={'text': 'Different'}, headers={'Idempotency-Key': key})
|
||||
c.call('POST', path, 400, actor=alice, body={'text': 'Missing key'})
|
||||
c.call('POST', path, 400, actor=alice, body={'text': 'Bad key'}, headers={'Idempotency-Key': 'bad key'})
|
||||
c.call('POST', path, 404, actor=eve, body={'text': 'Одно сообщение 👋'}, headers={'Idempotency-Key': key})
|
||||
other = c.message(bob, chat, 'Same key, different user', key=key)['json']
|
||||
check(other['id'] != first['id'], 'User scope is not independent')
|
||||
second_chat = c.call('POST', '/api/v1/chats', 201, actor=alice,
|
||||
body={'title': 'Second', 'member_ids': [alice['id']]}, schema='Chat')['json']
|
||||
check(c.message(alice, second_chat, 'Other chat', key=key)['json']['id'] != first['id'], 'Chat scope is not independent')
|
||||
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
|
||||
c.call('POST', path, 401, actor=alice, body={'text': 'Одно сообщение 👋'}, headers={'Idempotency-Key': key})
|
||||
print('PASS: 24 concurrent requests / 8 clients / %d replicas -> one message; conflict, scopes, access and logout verified.' % len(instances))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,201 @@
|
||||
"""Real local PostgreSQL + 2 API processes + HAProxy. Optional, not a Docker substitute.
|
||||
python scripts/check_local.py --pg-bin /path/to/bin --haproxy /path/to/haproxy
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from urllib.request import urlopen
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def pg(state, *args):
|
||||
subprocess.run([str(Path(state['bin']) / 'pg_ctl'), '-D', state['data'], *args],
|
||||
check=True, stdout=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def wait_url(port):
|
||||
for _ in range(150):
|
||||
try:
|
||||
with urlopen('http://127.0.0.1:%s/health/ready' % port, timeout=1) as response:
|
||||
if response.status == 200:
|
||||
return
|
||||
except OSError:
|
||||
time.sleep(.1)
|
||||
raise RuntimeError('Service did not become ready on port %s' % port)
|
||||
|
||||
|
||||
def launch(state, name):
|
||||
temp = Path(state['data']).parent
|
||||
if name == 'lb':
|
||||
command = [state['haproxy'], '-db', '-f', str(temp / 'haproxy.cfg')]
|
||||
env = dict(os.environ)
|
||||
else:
|
||||
command = [sys.executable, '-m', 'messenger']
|
||||
env = dict(os.environ, APP_HOST='127.0.0.1', APP_PORT=str(state['ports'][name]))
|
||||
with (temp / (name + '.log')).open('a') as log:
|
||||
process = subprocess.Popen(command, env=env, cwd=ROOT, stdout=log, stderr=log, start_new_session=True)
|
||||
state['pids'][name] = process.pid
|
||||
return process
|
||||
|
||||
|
||||
def stop(pid):
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
return
|
||||
for _ in range(70):
|
||||
try:
|
||||
# Zombie processes no longer own sockets and await their parent.
|
||||
status = Path('/proc/%s/stat' % pid)
|
||||
if not status.exists() or status.read_text().split()[2] == 'Z':
|
||||
return
|
||||
except FileNotFoundError:
|
||||
return
|
||||
time.sleep(.1)
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
|
||||
|
||||
def hook(path, action):
|
||||
state = json.loads(path.read_text())
|
||||
if action == 'stop':
|
||||
target = os.environ['TARGET_INSTANCE_ID']
|
||||
for name, port in state['ports'].items():
|
||||
try:
|
||||
with urlopen('http://127.0.0.1:%s/health/live' % port, timeout=1) as r:
|
||||
matches = r.headers['X-Instance-Id'] == target
|
||||
except OSError:
|
||||
continue
|
||||
if matches:
|
||||
pid = state['pids'][name]
|
||||
start = time.monotonic()
|
||||
stop(pid)
|
||||
state['stopped'] = name
|
||||
path.write_text(json.dumps(state))
|
||||
with (ROOT / 'evidence/local-stops.txt').open('a') as f:
|
||||
f.write('%s PID=%s instance=%s stopped in %.3fs; remains stopped\n' % (name, pid, target, time.monotonic()-start))
|
||||
return
|
||||
raise RuntimeError('Target instance not found')
|
||||
import psycopg
|
||||
with psycopg.connect(os.environ['DATABASE_URL']) as conn:
|
||||
before = conn.execute('SELECT pg_postmaster_start_time()').fetchone()[0]
|
||||
old = dict(state['pids'])
|
||||
for pid in old.values():
|
||||
stop(pid)
|
||||
pg(state, '-m', 'fast', '-w', 'stop')
|
||||
pg(state, '-l', str(path.parent / 'postgres.log'), '-w', 'start')
|
||||
for name in ('api1', 'api2', 'lb'):
|
||||
launch(state, name)
|
||||
path.write_text(json.dumps(state))
|
||||
wait_url(18080)
|
||||
with psycopg.connect(os.environ['DATABASE_URL']) as conn:
|
||||
after = conn.execute('SELECT pg_postmaster_start_time()').fetchone()[0]
|
||||
assert before != after and all(old[k] != state['pids'][k] for k in old)
|
||||
(ROOT / 'evidence/local-restart.txt').write_text('Actual process PIDs before: %s\nAfter: %s\nPostgreSQL start: %s -> %s\nSame data directory, no reseed or migrations in hook.\n' %
|
||||
(old, state['pids'], before.isoformat(), after.isoformat()))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--pg-bin')
|
||||
parser.add_argument('--haproxy')
|
||||
parser.add_argument('--hook', choices=['stop', 'restart'])
|
||||
parser.add_argument('--state', type=Path)
|
||||
args = parser.parse_args()
|
||||
if args.hook:
|
||||
hook(args.state, args.hook)
|
||||
return
|
||||
if not args.pg_bin or not args.haproxy:
|
||||
parser.error('--pg-bin and --haproxy required')
|
||||
binary = Path(args.pg_bin).resolve()
|
||||
with tempfile.TemporaryDirectory(prefix='messenger-lab3-') as tmp:
|
||||
temp = Path(tmp)
|
||||
path = temp / 'state.json'
|
||||
state = {'bin': str(binary), 'haproxy': str(Path(args.haproxy).resolve()),
|
||||
'data': str(temp / 'data'), 'ports': {'api1': 18081, 'api2': 18082}, 'pids': {}}
|
||||
os.environ.update(DATABASE_URL='postgresql://127.0.0.1:15432/postgres',
|
||||
SESSION_STORE='postgres', SESSION_TTL_SECONDS='3600', IDEMPOTENCY_TTL_SECONDS='86400')
|
||||
os.environ.pop('DATABASE_URL_FILE', None)
|
||||
subprocess.run([str(binary / 'initdb'), '-D', state['data'], '-A', 'trust', '--no-locale', '-E', 'UTF8'],
|
||||
check=True, stdout=subprocess.DEVNULL)
|
||||
with (Path(state['data']) / 'postgresql.conf').open('a') as f:
|
||||
f.write("\nport=15432\nlisten_addresses='127.0.0.1'\nunix_socket_directories=''\ntimezone='UTC'\n")
|
||||
config = (ROOT / 'infra/haproxy.cfg').read_text().replace('bind :8080', 'bind 127.0.0.1:18080')
|
||||
for name, port in state['ports'].items():
|
||||
config = config.replace(name + ':8080', '127.0.0.1:' + str(port))
|
||||
# Local fixed loopback addresses have no Docker DNS; all LB behavior is unchanged.
|
||||
config = config.replace(' resolvers docker resolve-prefer ipv4 init-addr libc,none', '')
|
||||
(temp / 'haproxy.cfg').write_text(config)
|
||||
processes = []
|
||||
def run(command):
|
||||
print('$ ' + ' '.join(command), flush=True)
|
||||
subprocess.run(command, cwd=ROOT, check=True)
|
||||
try:
|
||||
pg(state, '-l', str(temp / 'postgres.log'), '-w', 'start')
|
||||
run([sys.executable, '-m', 'messenger.migrate'])
|
||||
run([sys.executable, '-m', 'messenger.migrate'])
|
||||
run([state['haproxy'], '-c', '-f', str(temp / 'haproxy.cfg')])
|
||||
for name in ('api1', 'api2', 'lb'):
|
||||
processes.append(launch(state, name))
|
||||
path.write_text(json.dumps(state))
|
||||
wait_url(18081); wait_url(18082); wait_url(18080)
|
||||
print('Distinct running processes:', state['pids'], flush=True)
|
||||
(ROOT / 'evidence/local-stops.txt').write_text('Actual local API stops (not Docker):\n')
|
||||
run(['make', 'check', 'validate', 'student-test', 'PYTHON=' + sys.executable])
|
||||
run(['make', 'test', 'test-idempotency', 'BASE_URL=http://127.0.0.1:18080', 'PYTHON=' + sys.executable])
|
||||
for action in ('restart', 'stop'):
|
||||
script = temp / (action + '.sh')
|
||||
script.write_text('#!/bin/sh\nexec ' + shlex.join([sys.executable, str(Path(__file__).resolve()),
|
||||
'--hook', action, '--state', str(path)]) + '\n')
|
||||
script.chmod(0o700)
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client
|
||||
from uuid import uuid4
|
||||
client = Client('http://127.0.0.1:18080')
|
||||
actor, _, _, durable_chat = client.bootstrap()
|
||||
durable_key = uuid4().hex
|
||||
original = client.message(actor, durable_chat, 'Replay across restart', key=durable_key)['json']
|
||||
run(['make', 'test-persistence' , 'ACTION_SCRIPT=' + str(temp / 'restart.sh'),
|
||||
'BASE_URL=http://127.0.0.1:18080', 'PYTHON=' + sys.executable])
|
||||
assert client.message(actor, durable_chat, 'Replay across restart', key=durable_key)['json'] == original
|
||||
print('PASS: same key + old session after full restart returns original Message', flush=True)
|
||||
run(['make', 'test-idempotency', 'BASE_URL=http://127.0.0.1:18080', 'PYTHON=' + sys.executable])
|
||||
for attempt in range(2):
|
||||
if attempt:
|
||||
# Shift the round-robin position so the second public scenario
|
||||
# selects the other process, without changing the public test.
|
||||
with urlopen('http://127.0.0.1:18080/health/live'):
|
||||
pass
|
||||
run(['make', 'test-failover', 'ACTION_SCRIPT=' + str(temp / 'stop.sh'),
|
||||
'BASE_URL=http://127.0.0.1:18080', 'PYTHON=' + sys.executable])
|
||||
assert client.message(actor, durable_chat, 'Replay across restart', key=durable_key)['json'] == original
|
||||
print('PASS: original idempotent result survives selected replica stop', flush=True)
|
||||
state = json.loads(path.read_text())
|
||||
processes.append(launch(state, state.pop('stopped')))
|
||||
path.write_text(json.dumps(state))
|
||||
wait_url(18081); wait_url(18082)
|
||||
time.sleep(1.2)
|
||||
print((ROOT / 'evidence/local-restart.txt').read_text(), flush=True)
|
||||
print((ROOT / 'evidence/local-stops.txt').read_text(), flush=True)
|
||||
run([state['haproxy'], '-v'])
|
||||
run([str(binary / 'postgres'), '--version'])
|
||||
finally:
|
||||
if path.exists():
|
||||
state = json.loads(path.read_text())
|
||||
for pid in state['pids'].values():
|
||||
stop(pid)
|
||||
for process in processes:
|
||||
process.wait(timeout=10)
|
||||
subprocess.run([str(binary / 'pg_ctl'), '-D', state['data'], '-m', 'fast', '-w', 'stop'],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+26
@@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p evidence
|
||||
snapshot() {
|
||||
for service in db s3 api1 api2 lb; do
|
||||
id=$(docker compose ps -aq "$service")
|
||||
docker inspect --format '{{.Name}} {{.State.StartedAt}} {{.State.Status}}' "$id"
|
||||
done
|
||||
}
|
||||
snapshot > evidence/restart-before.txt
|
||||
docker compose stop -t 7 lb api1 api2
|
||||
docker compose restart -t 10 db s3
|
||||
i=0
|
||||
until docker compose exec -T db pg_isready -U messenger -d messenger >/dev/null 2>&1; do
|
||||
i=$((i + 1)); [ "$i" -lt 30 ] || exit 1
|
||||
sleep 1
|
||||
done
|
||||
i=0
|
||||
until docker compose exec -T s3 sh -c 'curl -fsS http://127.0.0.1:9000/minio/health/live >/dev/null' 2>/dev/null; do
|
||||
i=$((i+1)); [ "$i" -lt 30 ] || exit 1
|
||||
sleep 1
|
||||
done
|
||||
# Existing schema is retained; the restart hook does not migrate/reseed.
|
||||
docker compose start api1 api2 lb
|
||||
snapshot > evidence/restart-after.txt
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?Set TARGET_INSTANCE_ID to the observed API instance}"
|
||||
mkdir -p evidence
|
||||
for service in api1 api2; do
|
||||
instance=$(docker compose exec -T "$service" python -c 'import urllib.request; r=urllib.request.urlopen("http://127.0.0.1:8080/health/live",timeout=2); print(r.headers["X-Instance-Id"])' 2>/dev/null) || continue
|
||||
if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then
|
||||
id=$(docker compose ps -q "$service")
|
||||
docker inspect --format '{{.Name}} {{.State.StartedAt}} {{.State.Status}}' "$id" > "evidence/stop-$service.txt"
|
||||
# SIGTERM, then bounded fallback. No automatic restart policy is enabled.
|
||||
docker compose stop -t 7 "$service"
|
||||
docker inspect --format '{{.Name}} {{.State.FinishedAt}} {{.State.Status}}' "$id" >> "evidence/stop-$service.txt"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
printf '%s\n' 'No running replica matches TARGET_INSTANCE_ID' >&2
|
||||
exit 1
|
||||
Reference in New Issue
Block a user