forked from mtusi2026/mtusi2026lab3
Сессия и ключ идемпотентности общие для реплик, миграция выполняется один раз, остановка одной реплики не теряет подтверждённые сообщения.
255 lines
9.7 KiB
Python
255 lines
9.7 KiB
Python
"""Grade 4 evidence: parallel idempotency, each replica stopped, rolling replace.
|
|
|
|
Leaves two API replicas running. Does not print tokens.
|
|
"""
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import uuid
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
|
|
from client import Client, Failure
|
|
import stand
|
|
|
|
ROOT = stand.ROOT
|
|
OUT = ROOT / 'evidence' / 'grade4.json'
|
|
|
|
|
|
def history_texts(client, actor, chat_id):
|
|
page = client.call('GET', '/api/v1/chats/%s/messages?limit=100' % chat_id, actor=actor, schema='MessagePage')['json']
|
|
return [item['text'] for item in page['items']]
|
|
|
|
|
|
def parallel_same_key(client, alice, chat):
|
|
key = uuid.uuid4().hex
|
|
text = 'параллельный ключ ' + key[:8]
|
|
barrier = threading.Barrier(16)
|
|
found = []
|
|
|
|
def worker():
|
|
local = Client(stand.BASE, timeout=5)
|
|
barrier.wait()
|
|
try:
|
|
response = local.message(alice, chat, text, key=key)
|
|
found.append(('ok', response['json']['id'], response['headers']['X-Instance-Id']))
|
|
except Failure as exc:
|
|
found.append(('err', str(exc), ''))
|
|
|
|
threads = [threading.Thread(target=worker) for _ in range(16)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join()
|
|
ids = {item[1] for item in found if item[0] == 'ok'}
|
|
instances = {item[2] for item in found if item[0] == 'ok'}
|
|
texts = history_texts(client, alice, chat['id'])
|
|
return {
|
|
'attempts': len(found),
|
|
'successes': sum(1 for item in found if item[0] == 'ok'),
|
|
'errors': sum(1 for item in found if item[0] != 'ok'),
|
|
'distinct_message_ids': len(ids),
|
|
'history_copies': texts.count(text),
|
|
'instances': sorted(instances),
|
|
}
|
|
|
|
|
|
def parallel_conflict(client, alice, chat):
|
|
key = uuid.uuid4().hex
|
|
barrier = threading.Barrier(8)
|
|
found = []
|
|
|
|
def worker(index):
|
|
local = Client(stand.BASE, timeout=5)
|
|
barrier.wait()
|
|
try:
|
|
response = local.message(alice, chat, 'вариант-%s-%s' % (key[:6], index), key=key)
|
|
found.append(response['status'])
|
|
except Failure as exc:
|
|
text = str(exc)
|
|
found.append(409 if '409' in text else text)
|
|
|
|
threads = [threading.Thread(target=worker, args=(index,)) for index in range(8)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join()
|
|
created = sum(1 for item in found if item == 201)
|
|
conflicts = sum(1 for item in found if item == 409)
|
|
return {'created': created, 'conflicts': conflicts, 'other': [item for item in found if item not in (201, 409)]}
|
|
|
|
|
|
def run_load(alice, chat, seconds, errors, successes, stop_flag):
|
|
def worker(worker_id):
|
|
local = Client(stand.BASE, timeout=2)
|
|
index = 0
|
|
while not stop_flag['flag']:
|
|
try:
|
|
response = local.message(alice, chat, 'нагрузка-%s-%s' % (worker_id, index))
|
|
successes.append((time.monotonic(), response['headers']['X-Instance-Id']))
|
|
except Failure:
|
|
errors.append(time.monotonic())
|
|
index += 1
|
|
|
|
threads = [threading.Thread(target=worker, args=(i,)) for i in range(4)]
|
|
for thread in threads:
|
|
thread.start()
|
|
return threads
|
|
|
|
|
|
def stop_each(alice, chat):
|
|
rounds = []
|
|
containers = stand.api_containers()
|
|
if len(containers) < 2:
|
|
raise SystemExit('need 2 running replicas before sequential stop')
|
|
for target in containers:
|
|
errors, successes = [], []
|
|
stop_flag = {'flag': False}
|
|
threads = run_load(alice, chat, 0, errors, successes, stop_flag)
|
|
time.sleep(2)
|
|
before = len(successes)
|
|
started = time.monotonic()
|
|
subprocess.run(['docker', 'stop', '-t', '6', target['id']], check=True, capture_output=True)
|
|
stopped_at = time.monotonic()
|
|
deadline = time.monotonic() + 10
|
|
recovered = None
|
|
while time.monotonic() < deadline:
|
|
recent = [item for item in successes if item[0] >= started and item[1] != target['hostname']]
|
|
if recent:
|
|
recovered = recent[0][0] - started
|
|
break
|
|
time.sleep(0.1)
|
|
time.sleep(2)
|
|
stop_flag['flag'] = True
|
|
for thread in threads:
|
|
thread.join()
|
|
after = [item for item in successes if item[0] >= started]
|
|
err_after = [item for item in errors if item >= started]
|
|
subprocess.run(['docker', 'start', target['id']], check=True, capture_output=True)
|
|
back = time.monotonic()
|
|
seen_again = False
|
|
while time.monotonic() - back < 30:
|
|
running = {row['hostname'] for row in stand.api_containers()}
|
|
if target['hostname'] in running:
|
|
ids = stand.observe_instances(alice, 2, attempts=15)
|
|
seen_again = target['hostname'] in ids
|
|
if seen_again:
|
|
break
|
|
time.sleep(0.4)
|
|
rounds.append({
|
|
'stopped_instance': target['hostname'],
|
|
'container': target['name'],
|
|
'successful_writes_before_stop': before,
|
|
'successful_writes_after_stop_call': len(after),
|
|
'failed_attempts_after_stop_call': len(err_after),
|
|
'docker_stop_seconds': round(stopped_at - started, 3),
|
|
'first_other_instance_seconds': None if recovered is None else round(recovered, 3),
|
|
'target_seen_again_after_start': seen_again,
|
|
})
|
|
stand.wait_ready()
|
|
restored = stand.observe_instances(alice, 2, attempts=50)
|
|
if len(restored) < 2:
|
|
raise SystemExit('replica %s did not return to the load balancer' % target['hostname'])
|
|
return rounds
|
|
|
|
|
|
def rolling(alice):
|
|
errors = []
|
|
successes = []
|
|
stop_flag = {'flag': False}
|
|
|
|
def worker():
|
|
local = Client(stand.BASE, timeout=2)
|
|
while not stop_flag['flag']:
|
|
try:
|
|
response = local.call('GET', '/api/v1/users/me', actor=alice, schema='User')
|
|
successes.append((time.monotonic(), response['headers']['X-Instance-Id']))
|
|
except Failure:
|
|
errors.append(time.monotonic())
|
|
|
|
threads = [threading.Thread(target=worker) for _ in range(4)]
|
|
for thread in threads:
|
|
thread.start()
|
|
time.sleep(1)
|
|
started = time.monotonic()
|
|
script = ROOT / 'scripts' / 'rolling-replace.sh'
|
|
# Windows executes the PowerShell equivalent; the shell script is the Linux procedure.
|
|
if os_name_is_windows():
|
|
completed = subprocess.run(
|
|
['powershell', '-NoProfile', '-ExecutionPolicy', 'Bypass', '-File', str(ROOT / 'scripts' / 'rolling-replace.ps1')],
|
|
cwd=ROOT,
|
|
text=True,
|
|
capture_output=True,
|
|
)
|
|
else:
|
|
completed = subprocess.run(['sh', str(script)], cwd=ROOT, text=True, capture_output=True)
|
|
finished = time.monotonic()
|
|
time.sleep(2)
|
|
stop_flag['flag'] = True
|
|
for thread in threads:
|
|
thread.join()
|
|
during = [item for item in successes if item[0] >= started]
|
|
err = [item for item in errors if item >= started]
|
|
instances = stand.observe_instances(alice, 2, attempts=40)
|
|
return {
|
|
'script_exit': completed.returncode,
|
|
'elapsed_seconds': round(finished - started, 3),
|
|
'successful_reads': len(during),
|
|
'failed_reads': len(err),
|
|
'instances_after': sorted(instances),
|
|
'stderr_tail': (completed.stderr or '')[-400:],
|
|
}
|
|
|
|
|
|
def os_name_is_windows():
|
|
return sys.platform.startswith('win')
|
|
|
|
|
|
def main():
|
|
stand.wait_ready()
|
|
client = Client(stand.BASE, timeout=10)
|
|
alice, _bob, _eve, chat = client.bootstrap()
|
|
seen = stand.observe_instances(alice, 2)
|
|
if len(seen) < 2:
|
|
raise SystemExit('observed %s instances, need 2' % len(seen))
|
|
report = {
|
|
'instances_before': sorted(seen),
|
|
'parallel_same_key': parallel_same_key(client, alice, chat),
|
|
'parallel_different_body': parallel_conflict(client, alice, chat),
|
|
'stop_each_replica': stop_each(alice, chat),
|
|
'rolling_replace': rolling(alice),
|
|
}
|
|
same = report['parallel_same_key']
|
|
conflict = report['parallel_different_body']
|
|
if same['distinct_message_ids'] != 1 or same['history_copies'] != 1:
|
|
raise SystemExit('same key created more than one message')
|
|
if conflict['created'] != 1 or conflict['other']:
|
|
raise SystemExit('different bodies with one key were not one 201 and the rest 409')
|
|
for item in report['stop_each_replica']:
|
|
if item['first_other_instance_seconds'] is None or item['first_other_instance_seconds'] > 10:
|
|
raise SystemExit('replica did not fail over within 10s: %s' % item['stopped_instance'])
|
|
if report['rolling_replace']['script_exit'] != 0 or len(report['rolling_replace']['instances_after']) < 2:
|
|
raise SystemExit('rolling replace did not leave two instances')
|
|
OUT.parent.mkdir(parents=True, exist_ok=True)
|
|
OUT.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
|
print(json.dumps({
|
|
'same_key_ids': same['distinct_message_ids'],
|
|
'same_key_instances': same['instances'],
|
|
'conflict_created': conflict['created'],
|
|
'conflict_409': conflict['conflicts'],
|
|
'failovers': [
|
|
{'instance': item['stopped_instance'], 'recovery_s': item['first_other_instance_seconds'], 'errors': item['failed_attempts_after_stop_call']}
|
|
for item in report['stop_each_replica']
|
|
],
|
|
'rolling_errors': report['rolling_replace']['failed_reads'],
|
|
'rolling_ok': report['rolling_replace']['successful_reads'],
|
|
}, ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|