Files
mtusi2026lab3/student-tests/experiment_grade4.py
T
vika 59f8c55f93 Поставить несколько реплик API за общий балансировщик.
Сессия и ключ идемпотентности общие для реплик, миграция выполняется один раз, остановка одной реплики не теряет подтверждённые сообщения.
2026-10-30 01:15:28 +03:00

255 lines
9.7 KiB
Python

"""Grade 4 evidence: parallel idempotency, each replica stopped, rolling replace.
Leaves two API replicas running. Does not print tokens.
"""
import json
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
sys.path.insert(0, str(Path(__file__).resolve().parent))
from client import Client, Failure
import stand
ROOT = stand.ROOT
OUT = ROOT / 'evidence' / 'grade4.json'
def history_texts(client, actor, chat_id):
page = client.call('GET', '/api/v1/chats/%s/messages?limit=100' % chat_id, actor=actor, schema='MessagePage')['json']
return [item['text'] for item in page['items']]
def parallel_same_key(client, alice, chat):
key = uuid.uuid4().hex
text = 'параллельный ключ ' + key[:8]
barrier = threading.Barrier(16)
found = []
def worker():
local = Client(stand.BASE, timeout=5)
barrier.wait()
try:
response = local.message(alice, chat, text, key=key)
found.append(('ok', response['json']['id'], response['headers']['X-Instance-Id']))
except Failure as exc:
found.append(('err', str(exc), ''))
threads = [threading.Thread(target=worker) for _ in range(16)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
ids = {item[1] for item in found if item[0] == 'ok'}
instances = {item[2] for item in found if item[0] == 'ok'}
texts = history_texts(client, alice, chat['id'])
return {
'attempts': len(found),
'successes': sum(1 for item in found if item[0] == 'ok'),
'errors': sum(1 for item in found if item[0] != 'ok'),
'distinct_message_ids': len(ids),
'history_copies': texts.count(text),
'instances': sorted(instances),
}
def parallel_conflict(client, alice, chat):
key = uuid.uuid4().hex
barrier = threading.Barrier(8)
found = []
def worker(index):
local = Client(stand.BASE, timeout=5)
barrier.wait()
try:
response = local.message(alice, chat, 'вариант-%s-%s' % (key[:6], index), key=key)
found.append(response['status'])
except Failure as exc:
text = str(exc)
found.append(409 if '409' in text else text)
threads = [threading.Thread(target=worker, args=(index,)) for index in range(8)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
created = sum(1 for item in found if item == 201)
conflicts = sum(1 for item in found if item == 409)
return {'created': created, 'conflicts': conflicts, 'other': [item for item in found if item not in (201, 409)]}
def run_load(alice, chat, seconds, errors, successes, stop_flag):
def worker(worker_id):
local = Client(stand.BASE, timeout=2)
index = 0
while not stop_flag['flag']:
try:
response = local.message(alice, chat, 'нагрузка-%s-%s' % (worker_id, index))
successes.append((time.monotonic(), response['headers']['X-Instance-Id']))
except Failure:
errors.append(time.monotonic())
index += 1
threads = [threading.Thread(target=worker, args=(i,)) for i in range(4)]
for thread in threads:
thread.start()
return threads
def stop_each(alice, chat):
rounds = []
containers = stand.api_containers()
if len(containers) < 2:
raise SystemExit('need 2 running replicas before sequential stop')
for target in containers:
errors, successes = [], []
stop_flag = {'flag': False}
threads = run_load(alice, chat, 0, errors, successes, stop_flag)
time.sleep(2)
before = len(successes)
started = time.monotonic()
subprocess.run(['docker', 'stop', '-t', '6', target['id']], check=True, capture_output=True)
stopped_at = time.monotonic()
deadline = time.monotonic() + 10
recovered = None
while time.monotonic() < deadline:
recent = [item for item in successes if item[0] >= started and item[1] != target['hostname']]
if recent:
recovered = recent[0][0] - started
break
time.sleep(0.1)
time.sleep(2)
stop_flag['flag'] = True
for thread in threads:
thread.join()
after = [item for item in successes if item[0] >= started]
err_after = [item for item in errors if item >= started]
subprocess.run(['docker', 'start', target['id']], check=True, capture_output=True)
back = time.monotonic()
seen_again = False
while time.monotonic() - back < 30:
running = {row['hostname'] for row in stand.api_containers()}
if target['hostname'] in running:
ids = stand.observe_instances(alice, 2, attempts=15)
seen_again = target['hostname'] in ids
if seen_again:
break
time.sleep(0.4)
rounds.append({
'stopped_instance': target['hostname'],
'container': target['name'],
'successful_writes_before_stop': before,
'successful_writes_after_stop_call': len(after),
'failed_attempts_after_stop_call': len(err_after),
'docker_stop_seconds': round(stopped_at - started, 3),
'first_other_instance_seconds': None if recovered is None else round(recovered, 3),
'target_seen_again_after_start': seen_again,
})
stand.wait_ready()
restored = stand.observe_instances(alice, 2, attempts=50)
if len(restored) < 2:
raise SystemExit('replica %s did not return to the load balancer' % target['hostname'])
return rounds
def rolling(alice):
errors = []
successes = []
stop_flag = {'flag': False}
def worker():
local = Client(stand.BASE, timeout=2)
while not stop_flag['flag']:
try:
response = local.call('GET', '/api/v1/users/me', actor=alice, schema='User')
successes.append((time.monotonic(), response['headers']['X-Instance-Id']))
except Failure:
errors.append(time.monotonic())
threads = [threading.Thread(target=worker) for _ in range(4)]
for thread in threads:
thread.start()
time.sleep(1)
started = time.monotonic()
script = ROOT / 'scripts' / 'rolling-replace.sh'
# Windows executes the PowerShell equivalent; the shell script is the Linux procedure.
if os_name_is_windows():
completed = subprocess.run(
['powershell', '-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', str(ROOT / 'scripts' / 'rolling-replace.ps1')],
cwd=ROOT,
text=True,
capture_output=True,
)
else:
completed = subprocess.run(['sh', str(script)], cwd=ROOT, text=True, capture_output=True)
finished = time.monotonic()
time.sleep(2)
stop_flag['flag'] = True
for thread in threads:
thread.join()
during = [item for item in successes if item[0] >= started]
err = [item for item in errors if item >= started]
instances = stand.observe_instances(alice, 2, attempts=40)
return {
'script_exit': completed.returncode,
'elapsed_seconds': round(finished - started, 3),
'successful_reads': len(during),
'failed_reads': len(err),
'instances_after': sorted(instances),
'stderr_tail': (completed.stderr or '')[-400:],
}
def os_name_is_windows():
return sys.platform.startswith('win')
def main():
stand.wait_ready()
client = Client(stand.BASE, timeout=10)
alice, _bob, _eve, chat = client.bootstrap()
seen = stand.observe_instances(alice, 2)
if len(seen) < 2:
raise SystemExit('observed %s instances, need 2' % len(seen))
report = {
'instances_before': sorted(seen),
'parallel_same_key': parallel_same_key(client, alice, chat),
'parallel_different_body': parallel_conflict(client, alice, chat),
'stop_each_replica': stop_each(alice, chat),
'rolling_replace': rolling(alice),
}
same = report['parallel_same_key']
conflict = report['parallel_different_body']
if same['distinct_message_ids'] != 1 or same['history_copies'] != 1:
raise SystemExit('same key created more than one message')
if conflict['created'] != 1 or conflict['other']:
raise SystemExit('different bodies with one key were not one 201 and the rest 409')
for item in report['stop_each_replica']:
if item['first_other_instance_seconds'] is None or item['first_other_instance_seconds'] > 10:
raise SystemExit('replica did not fail over within 10s: %s' % item['stopped_instance'])
if report['rolling_replace']['script_exit'] != 0 or len(report['rolling_replace']['instances_after']) < 2:
raise SystemExit('rolling replace did not leave two instances')
OUT.parent.mkdir(parents=True, exist_ok=True)
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
print(json.dumps({
'same_key_ids': same['distinct_message_ids'],
'same_key_instances': same['instances'],
'conflict_created': conflict['created'],
'conflict_409': conflict['conflicts'],
'failovers': [
{'instance': item['stopped_instance'], 'recovery_s': item['first_other_instance_seconds'], 'errors': item['failed_attempts_after_stop_call']}
for item in report['stop_each_replica']
],
'rolling_errors': report['rolling_replace']['failed_reads'],
'rolling_ok': report['rolling_replace']['successful_reads'],
}, ensure_ascii=False, indent=2))
if __name__ == '__main__':
main()