From 64503fd0ddfac1384def56422d6fe2e8f8a72b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B8=D1=80=D0=B5=D0=BD=D0=BA=D0=BE=20=D0=98=D0=BB?= =?UTF-8?q?=D1=8C=D1=8F?= Date: Tue, 17 Nov 2026 03:37:38 +0000 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D1=81=D1=86=D0=B5=D0=BD=D0=B0=D1=80=D0=B8=D0=B8=20?= =?UTF-8?q?=D1=80=D0=B5=D1=81=D1=82=D0=B0=D1=80=D1=82=D0=B0,=20=D0=BE?= =?UTF-8?q?=D1=81=D1=82=D0=B0=D0=BD=D0=BE=D0=B2=D0=BA=D0=B8=20=D1=80=D0=B5?= =?UTF-8?q?=D0=BF=D0=BB=D0=B8=D0=BA=D0=B8=20=D0=B8=20=D0=BB=D0=BE=D0=BA?= =?UTF-8?q?=D0=B0=D0=BB=D1=8C=D0=BD=D1=8B=D1=85=20=D0=BF=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B5=D1=80=D0=BE=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit restart.sh (полный цикл стенда, без volumes/secrets), stop-one.sh (одна реплика по TARGET_INSTANCE_ID, данные и секреты сохраняются), check_local.py и check_idempotency.py — воспроизводимые локальные проверки. --- scripts/check_idempotency.py | 44 ++++++++ scripts/check_local.py | 201 +++++++++++++++++++++++++++++++++++ scripts/restart.sh | 26 +++++ scripts/stop-one.sh | 18 ++++ 4 files changed, 289 insertions(+) create mode 100644 scripts/check_idempotency.py create mode 100644 scripts/check_local.py create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh diff --git a/scripts/check_idempotency.py b/scripts/check_idempotency.py new file mode 100644 index 00000000..5353bf93 --- /dev/null +++ b/scripts/check_idempotency.py @@ -0,0 +1,44 @@ +"""Concurrent HTTP contract check through LB, using one session on two replicas.""" +import argparse +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +import sys +from uuid import uuid4 +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client, check + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--base-url', default='http://localhost:8080') + args = parser.parse_args() + c = Client(args.base_url) + alice, bob, eve, chat = c.bootstrap() + key = uuid4().hex + path = '/api/v1/chats/' + chat['id'] + '/messages' + def send(_): + return Client(args.base_url).message(alice, chat, 'Одно сообщение 👋', key=key) + with ThreadPoolExecutor(max_workers=8) as pool: + responses = list(pool.map(send, range(24))) + first = responses[0]['json'] + check(all(r['json'] == first and r['status'] == 201 for r in responses), 'Concurrent replay differs') + instances = {r['headers']['X-Instance-Id'] for r in responses} + check(len(instances) >= 2, 'Concurrent requests did not reach both replicas') + page = c.call('GET', path, actor=bob, schema='MessagePage')['json'] + check(page['items'] == [first], 'Concurrent repeat created duplicate messages') + c.call('POST', path, 409, actor=alice, body={'text': 'Different'}, headers={'Idempotency-Key': key}) + c.call('POST', path, 400, actor=alice, body={'text': 'Missing key'}) + c.call('POST', path, 400, actor=alice, body={'text': 'Bad key'}, headers={'Idempotency-Key': 'bad key'}) + c.call('POST', path, 404, actor=eve, body={'text': 'Одно сообщение 👋'}, headers={'Idempotency-Key': key}) + other = c.message(bob, chat, 'Same key, different user', key=key)['json'] + check(other['id'] != first['id'], 'User scope is not independent') + second_chat = c.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'Second', 'member_ids': [alice['id']]}, schema='Chat')['json'] + check(c.message(alice, second_chat, 'Other chat', key=key)['json']['id'] != first['id'], 'Chat scope is not independent') + c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice) + c.call('POST', path, 401, actor=alice, body={'text': 'Одно сообщение 👋'}, headers={'Idempotency-Key': key}) + print('PASS: 24 concurrent requests / 8 clients / %d replicas -> one message; conflict, scopes, access and logout verified.' % len(instances)) + + +if __name__ == '__main__': + main() diff --git a/scripts/check_local.py b/scripts/check_local.py new file mode 100644 index 00000000..844c1fc2 --- /dev/null +++ b/scripts/check_local.py @@ -0,0 +1,201 @@ +"""Real local PostgreSQL + 2 API processes + HAProxy. Optional, not a Docker substitute. +python scripts/check_local.py --pg-bin /path/to/bin --haproxy /path/to/haproxy +""" +import argparse +import json +import os +from pathlib import Path +import shlex +import signal +import subprocess +import sys +import tempfile +import time +from urllib.request import urlopen + +ROOT = Path(__file__).resolve().parents[1] + + +def pg(state, *args): + subprocess.run([str(Path(state['bin']) / 'pg_ctl'), '-D', state['data'], *args], + check=True, stdout=subprocess.DEVNULL) + + +def wait_url(port): + for _ in range(150): + try: + with urlopen('http://127.0.0.1:%s/health/ready' % port, timeout=1) as response: + if response.status == 200: + return + except OSError: + time.sleep(.1) + raise RuntimeError('Service did not become ready on port %s' % port) + + +def launch(state, name): + temp = Path(state['data']).parent + if name == 'lb': + command = [state['haproxy'], '-db', '-f', str(temp / 'haproxy.cfg')] + env = dict(os.environ) + else: + command = [sys.executable, '-m', 'messenger'] + env = dict(os.environ, APP_HOST='127.0.0.1', APP_PORT=str(state['ports'][name])) + with (temp / (name + '.log')).open('a') as log: + process = subprocess.Popen(command, env=env, cwd=ROOT, stdout=log, stderr=log, start_new_session=True) + state['pids'][name] = process.pid + return process + + +def stop(pid): + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + return + for _ in range(70): + try: + # Zombie processes no longer own sockets and await their parent. + status = Path('/proc/%s/stat' % pid) + if not status.exists() or status.read_text().split()[2] == 'Z': + return + except FileNotFoundError: + return + time.sleep(.1) + os.kill(pid, signal.SIGKILL) + + +def hook(path, action): + state = json.loads(path.read_text()) + if action == 'stop': + target = os.environ['TARGET_INSTANCE_ID'] + for name, port in state['ports'].items(): + try: + with urlopen('http://127.0.0.1:%s/health/live' % port, timeout=1) as r: + matches = r.headers['X-Instance-Id'] == target + except OSError: + continue + if matches: + pid = state['pids'][name] + start = time.monotonic() + stop(pid) + state['stopped'] = name + path.write_text(json.dumps(state)) + with (ROOT / 'evidence/local-stops.txt').open('a') as f: + f.write('%s PID=%s instance=%s stopped in %.3fs; remains stopped\n' % (name, pid, target, time.monotonic()-start)) + return + raise RuntimeError('Target instance not found') + import psycopg + with psycopg.connect(os.environ['DATABASE_URL']) as conn: + before = conn.execute('SELECT pg_postmaster_start_time()').fetchone()[0] + old = dict(state['pids']) + for pid in old.values(): + stop(pid) + pg(state, '-m', 'fast', '-w', 'stop') + pg(state, '-l', str(path.parent / 'postgres.log'), '-w', 'start') + for name in ('api1', 'api2', 'lb'): + launch(state, name) + path.write_text(json.dumps(state)) + wait_url(18080) + with psycopg.connect(os.environ['DATABASE_URL']) as conn: + after = conn.execute('SELECT pg_postmaster_start_time()').fetchone()[0] + assert before != after and all(old[k] != state['pids'][k] for k in old) + (ROOT / 'evidence/local-restart.txt').write_text('Actual process PIDs before: %s\nAfter: %s\nPostgreSQL start: %s -> %s\nSame data directory, no reseed or migrations in hook.\n' % + (old, state['pids'], before.isoformat(), after.isoformat())) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--pg-bin') + parser.add_argument('--haproxy') + parser.add_argument('--hook', choices=['stop', 'restart']) + parser.add_argument('--state', type=Path) + args = parser.parse_args() + if args.hook: + hook(args.state, args.hook) + return + if not args.pg_bin or not args.haproxy: + parser.error('--pg-bin and --haproxy required') + binary = Path(args.pg_bin).resolve() + with tempfile.TemporaryDirectory(prefix='messenger-lab3-') as tmp: + temp = Path(tmp) + path = temp / 'state.json' + state = {'bin': str(binary), 'haproxy': str(Path(args.haproxy).resolve()), + 'data': str(temp / 'data'), 'ports': {'api1': 18081, 'api2': 18082}, 'pids': {}} + os.environ.update(DATABASE_URL='postgresql://127.0.0.1:15432/postgres', + SESSION_STORE='postgres', SESSION_TTL_SECONDS='3600', IDEMPOTENCY_TTL_SECONDS='86400') + os.environ.pop('DATABASE_URL_FILE', None) + subprocess.run([str(binary / 'initdb'), '-D', state['data'], '-A', 'trust', '--no-locale', '-E', 'UTF8'], + check=True, stdout=subprocess.DEVNULL) + with (Path(state['data']) / 'postgresql.conf').open('a') as f: + f.write("\nport=15432\nlisten_addresses='127.0.0.1'\nunix_socket_directories=''\ntimezone='UTC'\n") + config = (ROOT / 'infra/haproxy.cfg').read_text().replace('bind :8080', 'bind 127.0.0.1:18080') + for name, port in state['ports'].items(): + config = config.replace(name + ':8080', '127.0.0.1:' + str(port)) + # Local fixed loopback addresses have no Docker DNS; all LB behavior is unchanged. + config = config.replace(' resolvers docker resolve-prefer ipv4 init-addr libc,none', '') + (temp / 'haproxy.cfg').write_text(config) + processes = [] + def run(command): + print('$ ' + ' '.join(command), flush=True) + subprocess.run(command, cwd=ROOT, check=True) + try: + pg(state, '-l', str(temp / 'postgres.log'), '-w', 'start') + run([sys.executable, '-m', 'messenger.migrate']) + run([sys.executable, '-m', 'messenger.migrate']) + run([state['haproxy'], '-c', '-f', str(temp / 'haproxy.cfg')]) + for name in ('api1', 'api2', 'lb'): + processes.append(launch(state, name)) + path.write_text(json.dumps(state)) + wait_url(18081); wait_url(18082); wait_url(18080) + print('Distinct running processes:', state['pids'], flush=True) + (ROOT / 'evidence/local-stops.txt').write_text('Actual local API stops (not Docker):\n') + run(['make', 'check', 'validate', 'student-test', 'PYTHON=' + sys.executable]) + run(['make', 'test', 'test-idempotency', 'BASE_URL=http://127.0.0.1:18080', 'PYTHON=' + sys.executable]) + for action in ('restart', 'stop'): + script = temp / (action + '.sh') + script.write_text('#!/bin/sh\nexec ' + shlex.join([sys.executable, str(Path(__file__).resolve()), + '--hook', action, '--state', str(path)]) + '\n') + script.chmod(0o700) + sys.path.insert(0, str(ROOT / 'tests')) + from client import Client + from uuid import uuid4 + client = Client('http://127.0.0.1:18080') + actor, _, _, durable_chat = client.bootstrap() + durable_key = uuid4().hex + original = client.message(actor, durable_chat, 'Replay across restart', key=durable_key)['json'] + run(['make', 'test-persistence' , 'ACTION_SCRIPT=' + str(temp / 'restart.sh'), + 'BASE_URL=http://127.0.0.1:18080', 'PYTHON=' + sys.executable]) + assert client.message(actor, durable_chat, 'Replay across restart', key=durable_key)['json'] == original + print('PASS: same key + old session after full restart returns original Message', flush=True) + run(['make', 'test-idempotency', 'BASE_URL=http://127.0.0.1:18080', 'PYTHON=' + sys.executable]) + for attempt in range(2): + if attempt: + # Shift the round-robin position so the second public scenario + # selects the other process, without changing the public test. + with urlopen('http://127.0.0.1:18080/health/live'): + pass + run(['make', 'test-failover', 'ACTION_SCRIPT=' + str(temp / 'stop.sh'), + 'BASE_URL=http://127.0.0.1:18080', 'PYTHON=' + sys.executable]) + assert client.message(actor, durable_chat, 'Replay across restart', key=durable_key)['json'] == original + print('PASS: original idempotent result survives selected replica stop', flush=True) + state = json.loads(path.read_text()) + processes.append(launch(state, state.pop('stopped'))) + path.write_text(json.dumps(state)) + wait_url(18081); wait_url(18082) + time.sleep(1.2) + print((ROOT / 'evidence/local-restart.txt').read_text(), flush=True) + print((ROOT / 'evidence/local-stops.txt').read_text(), flush=True) + run([state['haproxy'], '-v']) + run([str(binary / 'postgres'), '--version']) + finally: + if path.exists(): + state = json.loads(path.read_text()) + for pid in state['pids'].values(): + stop(pid) + for process in processes: + process.wait(timeout=10) + subprocess.run([str(binary / 'pg_ctl'), '-D', state['data'], '-m', 'fast', '-w', 'stop'], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..fc2246fe --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +mkdir -p evidence +snapshot() { + for service in db s3 api1 api2 lb; do + id=$(docker compose ps -aq "$service") + docker inspect --format '{{.Name}} {{.State.StartedAt}} {{.State.Status}}' "$id" + done +} +snapshot > evidence/restart-before.txt +docker compose stop -t 7 lb api1 api2 +docker compose restart -t 10 db s3 +i=0 +until docker compose exec -T db pg_isready -U messenger -d messenger >/dev/null 2>&1; do + i=$((i + 1)); [ "$i" -lt 30 ] || exit 1 + sleep 1 +done +i=0 +until docker compose exec -T s3 sh -c 'curl -fsS http://127.0.0.1:9000/minio/health/live >/dev/null' 2>/dev/null; do + i=$((i+1)); [ "$i" -lt 30 ] || exit 1 + sleep 1 +done +# Existing schema is retained; the restart hook does not migrate/reseed. +docker compose start api1 api2 lb +snapshot > evidence/restart-after.txt diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..358d6d3d --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?Set TARGET_INSTANCE_ID to the observed API instance}" +mkdir -p evidence +for service in api1 api2; do + instance=$(docker compose exec -T "$service" python -c 'import urllib.request; r=urllib.request.urlopen("http://127.0.0.1:8080/health/live",timeout=2); print(r.headers["X-Instance-Id"])' 2>/dev/null) || continue + if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then + id=$(docker compose ps -q "$service") + docker inspect --format '{{.Name}} {{.State.StartedAt}} {{.State.Status}}' "$id" > "evidence/stop-$service.txt" + # SIGTERM, then bounded fallback. No automatic restart policy is enabled. + docker compose stop -t 7 "$service" + docker inspect --format '{{.Name}} {{.State.FinishedAt}} {{.State.Status}}' "$id" >> "evidence/stop-$service.txt" + exit 0 + fi +done +printf '%s\n' 'No running replica matches TARGET_INSTANCE_ID' >&2 +exit 1