restart.sh (полный цикл стенда, без volumes/secrets), stop-one.sh (одна реплика по TARGET_INSTANCE_ID, данные и секреты сохраняются), check_local.py и check_idempotency.py — воспроизводимые локальные проверки.
202 lines
9.4 KiB
Python
202 lines
9.4 KiB
Python
"""Real local PostgreSQL + 2 API processes + HAProxy. Optional, not a Docker substitute.
|
|
python scripts/check_local.py --pg-bin /path/to/bin --haproxy /path/to/haproxy
|
|
"""
|
|
import argparse
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import shlex
|
|
import signal
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from urllib.request import urlopen
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def pg(state, *args):
|
|
subprocess.run([str(Path(state['bin']) / 'pg_ctl'), '-D', state['data'], *args],
|
|
check=True, stdout=subprocess.DEVNULL)
|
|
|
|
|
|
def wait_url(port):
|
|
for _ in range(150):
|
|
try:
|
|
with urlopen('http://127.0.0.1:%s/health/ready' % port, timeout=1) as response:
|
|
if response.status == 200:
|
|
return
|
|
except OSError:
|
|
time.sleep(.1)
|
|
raise RuntimeError('Service did not become ready on port %s' % port)
|
|
|
|
|
|
def launch(state, name):
|
|
temp = Path(state['data']).parent
|
|
if name == 'lb':
|
|
command = [state['haproxy'], '-db', '-f', str(temp / 'haproxy.cfg')]
|
|
env = dict(os.environ)
|
|
else:
|
|
command = [sys.executable, '-m', 'messenger']
|
|
env = dict(os.environ, APP_HOST='127.0.0.1', APP_PORT=str(state['ports'][name]))
|
|
with (temp / (name + '.log')).open('a') as log:
|
|
process = subprocess.Popen(command, env=env, cwd=ROOT, stdout=log, stderr=log, start_new_session=True)
|
|
state['pids'][name] = process.pid
|
|
return process
|
|
|
|
|
|
def stop(pid):
|
|
try:
|
|
os.kill(pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
return
|
|
for _ in range(70):
|
|
try:
|
|
# Zombie processes no longer own sockets and await their parent.
|
|
status = Path('/proc/%s/stat' % pid)
|
|
if not status.exists() or status.read_text().split()[2] == 'Z':
|
|
return
|
|
except FileNotFoundError:
|
|
return
|
|
time.sleep(.1)
|
|
os.kill(pid, signal.SIGKILL)
|
|
|
|
|
|
def hook(path, action):
|
|
state = json.loads(path.read_text())
|
|
if action == 'stop':
|
|
target = os.environ['TARGET_INSTANCE_ID']
|
|
for name, port in state['ports'].items():
|
|
try:
|
|
with urlopen('http://127.0.0.1:%s/health/live' % port, timeout=1) as r:
|
|
matches = r.headers['X-Instance-Id'] == target
|
|
except OSError:
|
|
continue
|
|
if matches:
|
|
pid = state['pids'][name]
|
|
start = time.monotonic()
|
|
stop(pid)
|
|
state['stopped'] = name
|
|
path.write_text(json.dumps(state))
|
|
with (ROOT / 'evidence/local-stops.txt').open('a') as f:
|
|
f.write('%s PID=%s instance=%s stopped in %.3fs; remains stopped\n' % (name, pid, target, time.monotonic()-start))
|
|
return
|
|
raise RuntimeError('Target instance not found')
|
|
import psycopg
|
|
with psycopg.connect(os.environ['DATABASE_URL']) as conn:
|
|
before = conn.execute('SELECT pg_postmaster_start_time()').fetchone()[0]
|
|
old = dict(state['pids'])
|
|
for pid in old.values():
|
|
stop(pid)
|
|
pg(state, '-m', 'fast', '-w', 'stop')
|
|
pg(state, '-l', str(path.parent / 'postgres.log'), '-w', 'start')
|
|
for name in ('api1', 'api2', 'lb'):
|
|
launch(state, name)
|
|
path.write_text(json.dumps(state))
|
|
wait_url(18080)
|
|
with psycopg.connect(os.environ['DATABASE_URL']) as conn:
|
|
after = conn.execute('SELECT pg_postmaster_start_time()').fetchone()[0]
|
|
assert before != after and all(old[k] != state['pids'][k] for k in old)
|
|
(ROOT / 'evidence/local-restart.txt').write_text('Actual process PIDs before: %s\nAfter: %s\nPostgreSQL start: %s -> %s\nSame data directory, no reseed or migrations in hook.\n' %
|
|
(old, state['pids'], before.isoformat(), after.isoformat()))
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--pg-bin')
|
|
parser.add_argument('--haproxy')
|
|
parser.add_argument('--hook', choices=['stop', 'restart'])
|
|
parser.add_argument('--state', type=Path)
|
|
args = parser.parse_args()
|
|
if args.hook:
|
|
hook(args.state, args.hook)
|
|
return
|
|
if not args.pg_bin or not args.haproxy:
|
|
parser.error('--pg-bin and --haproxy required')
|
|
binary = Path(args.pg_bin).resolve()
|
|
with tempfile.TemporaryDirectory(prefix='messenger-lab3-') as tmp:
|
|
temp = Path(tmp)
|
|
path = temp / 'state.json'
|
|
state = {'bin': str(binary), 'haproxy': str(Path(args.haproxy).resolve()),
|
|
'data': str(temp / 'data'), 'ports': {'api1': 18081, 'api2': 18082}, 'pids': {}}
|
|
os.environ.update(DATABASE_URL='postgresql://127.0.0.1:15432/postgres',
|
|
SESSION_STORE='postgres', SESSION_TTL_SECONDS='3600', IDEMPOTENCY_TTL_SECONDS='86400')
|
|
os.environ.pop('DATABASE_URL_FILE', None)
|
|
subprocess.run([str(binary / 'initdb'), '-D', state['data'], '-A', 'trust', '--no-locale', '-E', 'UTF8'],
|
|
check=True, stdout=subprocess.DEVNULL)
|
|
with (Path(state['data']) / 'postgresql.conf').open('a') as f:
|
|
f.write("\nport=15432\nlisten_addresses='127.0.0.1'\nunix_socket_directories=''\ntimezone='UTC'\n")
|
|
config = (ROOT / 'infra/haproxy.cfg').read_text().replace('bind :8080', 'bind 127.0.0.1:18080')
|
|
for name, port in state['ports'].items():
|
|
config = config.replace(name + ':8080', '127.0.0.1:' + str(port))
|
|
# Local fixed loopback addresses have no Docker DNS; all LB behavior is unchanged.
|
|
config = config.replace(' resolvers docker resolve-prefer ipv4 init-addr libc,none', '')
|
|
(temp / 'haproxy.cfg').write_text(config)
|
|
processes = []
|
|
def run(command):
|
|
print('$ ' + ' '.join(command), flush=True)
|
|
subprocess.run(command, cwd=ROOT, check=True)
|
|
try:
|
|
pg(state, '-l', str(temp / 'postgres.log'), '-w', 'start')
|
|
run([sys.executable, '-m', 'messenger.migrate'])
|
|
run([sys.executable, '-m', 'messenger.migrate'])
|
|
run([state['haproxy'], '-c', '-f', str(temp / 'haproxy.cfg')])
|
|
for name in ('api1', 'api2', 'lb'):
|
|
processes.append(launch(state, name))
|
|
path.write_text(json.dumps(state))
|
|
wait_url(18081); wait_url(18082); wait_url(18080)
|
|
print('Distinct running processes:', state['pids'], flush=True)
|
|
(ROOT / 'evidence/local-stops.txt').write_text('Actual local API stops (not Docker):\n')
|
|
run(['make', 'check', 'validate', 'student-test', 'PYTHON=' + sys.executable])
|
|
run(['make', 'test', 'test-idempotency', 'BASE_URL=http://127.0.0.1:18080', 'PYTHON=' + sys.executable])
|
|
for action in ('restart', 'stop'):
|
|
script = temp / (action + '.sh')
|
|
script.write_text('#!/bin/sh\nexec ' + shlex.join([sys.executable, str(Path(__file__).resolve()),
|
|
'--hook', action, '--state', str(path)]) + '\n')
|
|
script.chmod(0o700)
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client
|
|
from uuid import uuid4
|
|
client = Client('http://127.0.0.1:18080')
|
|
actor, _, _, durable_chat = client.bootstrap()
|
|
durable_key = uuid4().hex
|
|
original = client.message(actor, durable_chat, 'Replay across restart', key=durable_key)['json']
|
|
run(['make', 'test-persistence' , 'ACTION_SCRIPT=' + str(temp / 'restart.sh'),
|
|
'BASE_URL=http://127.0.0.1:18080', 'PYTHON=' + sys.executable])
|
|
assert client.message(actor, durable_chat, 'Replay across restart', key=durable_key)['json'] == original
|
|
print('PASS: same key + old session after full restart returns original Message', flush=True)
|
|
run(['make', 'test-idempotency', 'BASE_URL=http://127.0.0.1:18080', 'PYTHON=' + sys.executable])
|
|
for attempt in range(2):
|
|
if attempt:
|
|
# Shift the round-robin position so the second public scenario
|
|
# selects the other process, without changing the public test.
|
|
with urlopen('http://127.0.0.1:18080/health/live'):
|
|
pass
|
|
run(['make', 'test-failover', 'ACTION_SCRIPT=' + str(temp / 'stop.sh'),
|
|
'BASE_URL=http://127.0.0.1:18080', 'PYTHON=' + sys.executable])
|
|
assert client.message(actor, durable_chat, 'Replay across restart', key=durable_key)['json'] == original
|
|
print('PASS: original idempotent result survives selected replica stop', flush=True)
|
|
state = json.loads(path.read_text())
|
|
processes.append(launch(state, state.pop('stopped')))
|
|
path.write_text(json.dumps(state))
|
|
wait_url(18081); wait_url(18082)
|
|
time.sleep(1.2)
|
|
print((ROOT / 'evidence/local-restart.txt').read_text(), flush=True)
|
|
print((ROOT / 'evidence/local-stops.txt').read_text(), flush=True)
|
|
run([state['haproxy'], '-v'])
|
|
run([str(binary / 'postgres'), '--version'])
|
|
finally:
|
|
if path.exists():
|
|
state = json.loads(path.read_text())
|
|
for pid in state['pids'].values():
|
|
stop(pid)
|
|
for process in processes:
|
|
process.wait(timeout=10)
|
|
subprocess.run([str(binary / 'pg_ctl'), '-D', state['data'], '-m', 'fast', '-w', 'stop'],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|