forked from mtusi2026/mtusi2026lab3
ЛР 3 — БВТ2304 — Черников
HAProxy перед двумя репликами API, X-Instance-Id, общая идемпотентность отправки сообщения в PostgreSQL, двухшаговая остановка реплики по SIGTERM, лимиты CPU/RAM, хуки stop-one и restart. Проверки отказов, эксперимент и поочерёдное обновление в scripts/lab3_checks.py, отчёт в REPORT.md, результаты в evidence/, команды для показа в DEMO.md. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Executable
+139
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
"""EXPLAIN (ANALYZE, BUFFERS) of the history page query on 10 000+ messages.
|
||||
|
||||
python scripts/explain_history.py --output evidence/explain.txt
|
||||
|
||||
The stand must be running. The script registers two synthetic users through the API,
|
||||
creates a target chat and a noise chat, bulk-inserts messages with SQL, runs ANALYZE and
|
||||
prints the plans of the same query the API runs (src/messenger/db.py, list_messages):
|
||||
first page, a page in the middle, the last page, the generic plan of the parameterized
|
||||
query, and for comparison the middle page without the index (DROP INDEX inside a
|
||||
transaction that is rolled back, so the database is left unchanged).
|
||||
Standard library only.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = 'http://127.0.0.1:8080'
|
||||
PSQL = ['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
|
||||
'-X', '-v', 'ON_ERROR_STOP=1']
|
||||
PAGE = 51 # limit 50 + 1 row to learn whether the next page exists, as the API does
|
||||
|
||||
lines = []
|
||||
|
||||
|
||||
def out(text=''):
|
||||
lines.append(text)
|
||||
print(text, flush=True)
|
||||
|
||||
|
||||
def psql(sql, echo=True):
|
||||
"""Run SQL in one psql session; with echo the transcript shows every statement before its result."""
|
||||
args = PSQL + (['--echo-all'] if echo else ['-At']) + ['-f', '-']
|
||||
done = subprocess.run(args, cwd=str(ROOT), input=sql.encode('utf-8'), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
text = done.stdout.decode('utf-8', 'replace').strip()
|
||||
if done.returncode != 0:
|
||||
out(text)
|
||||
raise SystemExit('psql failed')
|
||||
return text
|
||||
|
||||
|
||||
def api(method, path, body=None, headers=None):
|
||||
headers = dict(headers or {})
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode('utf-8')
|
||||
headers['Content-Type'] = 'application/json'
|
||||
request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method)
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
return json.loads(response.read())
|
||||
|
||||
|
||||
def user(label):
|
||||
username = label + '_' + uuid.uuid4().hex[:10]
|
||||
password = 'Explain-' + uuid.uuid4().hex
|
||||
created = api('POST', '/api/v1/users', {'username': username, 'display_name': label, 'password': password})
|
||||
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
session = api('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
|
||||
return created['id'], session['token']
|
||||
|
||||
|
||||
def page_sql(chat_id, after=None):
|
||||
"""The same SQL text as Repository.list_messages, with literal values."""
|
||||
sql = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' " % chat_id
|
||||
if after:
|
||||
sql += "AND (created_at, id) > ('%s', '%s') " % after
|
||||
return sql + 'ORDER BY created_at, id LIMIT %d' % PAGE
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='EXPLAIN of the history query; inserts synthetic messages')
|
||||
parser.add_argument('--messages', type=int, default=10000, help='messages in the target chat (>= 10000)')
|
||||
parser.add_argument('--noise', type=int, default=20000, help='messages in another chat')
|
||||
parser.add_argument('--output', default=None)
|
||||
args = parser.parse_args()
|
||||
sys.stdout.reconfigure(errors='replace')
|
||||
|
||||
out('# %s UTC, %s' % (time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()), psql('SELECT version()', echo=False)))
|
||||
alice, alice_token = user('explain_a')
|
||||
bob, _ = user('explain_b')
|
||||
target = api('POST', '/api/v1/chats', {'title': 'EXPLAIN target', 'member_ids': [alice, bob]},
|
||||
{'Authorization': 'Bearer ' + alice_token})['id']
|
||||
noise = api('POST', '/api/v1/chats', {'title': 'EXPLAIN noise', 'member_ids': [alice]},
|
||||
{'Authorization': 'Bearer ' + alice_token})['id']
|
||||
out('# target chat %s: %d messages; noise chat %s: %d messages' % (target, args.messages, noise, args.noise))
|
||||
|
||||
started = time.monotonic()
|
||||
psql('''
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), '%(target)s', CASE WHEN i %% 2 = 0 THEN '%(alice)s'::uuid ELSE '%(bob)s'::uuid END,
|
||||
'seed message ' || i, clock_timestamp() + i * interval '1 millisecond'
|
||||
FROM generate_series(1, %(messages)d) AS i;
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), '%(noise)s', '%(alice)s', 'noise message ' || i, clock_timestamp() + i * interval '1 millisecond'
|
||||
FROM generate_series(1, %(noise_count)d) AS i;
|
||||
ANALYZE messages;
|
||||
''' % {'target': target, 'noise': noise, 'alice': alice, 'bob': bob, 'messages': args.messages, 'noise_count': args.noise},
|
||||
echo=False)
|
||||
out('# inserted and analyzed in %.1f s' % (time.monotonic() - started))
|
||||
out(psql("SELECT 'messages total: ' || count(*) || ', in target chat: ' || count(*) FILTER (WHERE chat_id = '%s') FROM messages" % target, echo=False))
|
||||
|
||||
def key_at(position):
|
||||
row = psql("SELECT created_at || '|' || id FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET %d LIMIT 1"
|
||||
% (target, position - 1), echo=False)
|
||||
return tuple(row.split('|'))
|
||||
|
||||
middle, last = key_at(args.messages // 2), key_at(args.messages - 20)
|
||||
explain = 'EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) '
|
||||
out()
|
||||
out('== 1. First page, middle page and last page: index range scan, no sort ==')
|
||||
out(psql('\n'.join(explain + page_sql(target, after) + ';' for after in (None, middle, last))))
|
||||
out()
|
||||
out('== 2. Generic plan of the parameterized query (how a prepared statement with $1..$4 runs) ==')
|
||||
out(psql('''SET plan_cache_mode = force_generic_plan;
|
||||
PREPARE history_page(uuid, timestamptz, uuid, int) AS
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT $4;
|
||||
%sEXECUTE history_page('%s', '%s', '%s', %d);''' % (explain, target, middle[0], middle[1], PAGE)))
|
||||
out()
|
||||
out('== 3. For comparison: the same middle page without the index (rolled back) ==')
|
||||
out(psql('''BEGIN;
|
||||
DROP INDEX messages_chat_history_idx;
|
||||
%s%s;
|
||||
ROLLBACK;''' % (explain, page_sql(target, middle))))
|
||||
out(psql("SELECT 'index still exists: ' || (to_regclass('messages_chat_history_idx') IS NOT NULL)", echo=False))
|
||||
if args.output:
|
||||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.output).write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local secret files for the compose stand.
|
||||
|
||||
python scripts/init_secrets.py
|
||||
|
||||
Passwords are random and synthetic; they are created only if missing and never printed.
|
||||
The URL/config files are always rebuilt from the password files, so the pair stays consistent.
|
||||
secrets/ is listed in .gitignore and .dockerignore.
|
||||
|
||||
PostgreSQL applies its password only when the data volume is created. If secrets/ is
|
||||
deleted while the pgdata volume is kept, restore the old files or recreate the volume
|
||||
(docker compose down -v deletes ALL lab data).
|
||||
"""
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DIR = ROOT / 'secrets'
|
||||
|
||||
|
||||
def write(name, content):
|
||||
path = DIR / name
|
||||
path.write_text(content, encoding='utf-8', newline='\n')
|
||||
# Readable for the non-root users inside containers (files are bind-mounted as is).
|
||||
path.chmod(0o644)
|
||||
|
||||
|
||||
def password(name):
|
||||
path = DIR / name
|
||||
if path.is_file() and path.read_text(encoding='utf-8').strip():
|
||||
print('keep secrets/' + name)
|
||||
return path.read_text(encoding='utf-8').strip()
|
||||
value = secrets.token_hex(24) # hex: safe inside URLs without escaping
|
||||
write(name, value + '\n')
|
||||
print('created secrets/' + name)
|
||||
return value
|
||||
|
||||
|
||||
def main():
|
||||
DIR.mkdir(exist_ok=True)
|
||||
postgres = password('postgres_password')
|
||||
valkey = password('valkey_password')
|
||||
write('database_url', 'postgresql://messenger:%s@postgres:5432/messenger\n' % postgres)
|
||||
write('valkey_auth', 'requirepass %s\n' % valkey)
|
||||
write('cache_url', 'valkey://:%s@valkey:6379/0\n' % valkey)
|
||||
print('written secrets/database_url, secrets/valkey_auth, secrets/cache_url')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+800
@@ -0,0 +1,800 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Infrastructure checks for lab 3 that need control over the compose stand.
|
||||
|
||||
python -X utf8 scripts/lab3_checks.py --output evidence/lab3-checks.txt
|
||||
python -X utf8 scripts/lab3_checks.py --only failover experiment rolling
|
||||
|
||||
The stand must already be running (docker compose up -d --build --wait). Every command is
|
||||
printed together with its output. Sections stop, kill and start containers, but never delete
|
||||
volumes, never flush Valkey and never print secrets or session tokens. All API traffic goes
|
||||
through the load balancer on 127.0.0.1:8080, as a client would send it.
|
||||
Standard library only; works on Windows, Linux and macOS (-X utf8 is needed on Windows only
|
||||
because the public tests read the contract in the default encoding).
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import concurrent.futures
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = 'http://127.0.0.1:8080'
|
||||
STATS = 'http://127.0.0.1:8404/;csv'
|
||||
PROJECT = os.environ.get('COMPOSE_PROJECT_NAME', 'mtusi-lab3')
|
||||
VALKEY_CLI = 'valkey-cli --no-auth-warning -a "$(sed -n "s/^requirepass //p" /run/secrets/valkey_auth)" '
|
||||
PSQL = ['exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-v', 'ON_ERROR_STOP=1', '-At', '-c']
|
||||
PY = [sys.executable, '-X', 'utf8']
|
||||
|
||||
lines = []
|
||||
results = []
|
||||
|
||||
|
||||
class Stop(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def out(text=''):
|
||||
lines.append(text)
|
||||
print(text, flush=True)
|
||||
|
||||
|
||||
def run(command, check=True, quiet=False, env=None):
|
||||
"""Run a command and echo it with its output; return (output, exit code, seconds)."""
|
||||
out('$ ' + ' '.join(command))
|
||||
started = time.monotonic()
|
||||
done = subprocess.run(command, cwd=str(ROOT), env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
elapsed = time.monotonic() - started
|
||||
text = done.stdout.decode('utf-8', 'replace').strip()
|
||||
if text and not quiet:
|
||||
out(text)
|
||||
if check and done.returncode != 0:
|
||||
if quiet and text:
|
||||
out(text)
|
||||
raise Stop('command failed with exit code %d' % done.returncode)
|
||||
return text, done.returncode, elapsed
|
||||
|
||||
|
||||
def silent(command):
|
||||
"""Run a command without echo; return its stdout."""
|
||||
return subprocess.run(command, cwd=str(ROOT), stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True).stdout.strip()
|
||||
|
||||
|
||||
def compose(*args, **kwargs):
|
||||
return run(['docker', 'compose'] + list(args), **kwargs)
|
||||
|
||||
|
||||
def psql(sql, quiet=False):
|
||||
if quiet:
|
||||
return silent(['docker', 'compose'] + PSQL + [sql])
|
||||
return compose(*(PSQL + [sql]))[0]
|
||||
|
||||
|
||||
def valkey_cli(command):
|
||||
return compose('exec', '-T', 'valkey', 'sh', '-c', VALKEY_CLI + command)[0]
|
||||
|
||||
|
||||
def verdict(name, ok, detail=''):
|
||||
results.append((name, ok))
|
||||
out('=> %s: %s%s' % ('PASS' if ok else 'FAIL', name, ' (' + detail + ')' if detail else ''))
|
||||
out()
|
||||
|
||||
|
||||
def http(method, path, token=None, body=None, headers=None, timeout=10):
|
||||
"""(status, parsed JSON or None, seconds, X-Instance-Id); status None when the connection fails."""
|
||||
all_headers = dict(headers or {})
|
||||
if token:
|
||||
all_headers['Authorization'] = 'Bearer ' + token
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode('utf-8')
|
||||
all_headers['Content-Type'] = 'application/json'
|
||||
request = urllib.request.Request(BASE + path, data=data, headers=all_headers, method=method)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as response:
|
||||
raw, status, instance = response.read(), response.status, response.headers.get('X-Instance-Id')
|
||||
except urllib.error.HTTPError as error:
|
||||
with error:
|
||||
raw, status, instance = error.read(), error.code, error.headers.get('X-Instance-Id')
|
||||
except OSError:
|
||||
return None, None, time.monotonic() - started, None
|
||||
try:
|
||||
payload = json.loads(raw) if raw else None
|
||||
except ValueError:
|
||||
payload = None
|
||||
return status, payload, time.monotonic() - started, instance
|
||||
|
||||
|
||||
def basic(username, password):
|
||||
return {'Authorization': 'Basic ' + base64.b64encode((username + ':' + password).encode()).decode()}
|
||||
|
||||
|
||||
def new_user(label='check'):
|
||||
"""Register a synthetic user and log in: (user, password, token). The token is never printed."""
|
||||
username = label + '_' + uuid.uuid4().hex[:12]
|
||||
password = 'Check-' + uuid.uuid4().hex
|
||||
status, user, _, _ = http('POST', '/api/v1/users', body={'username': username, 'display_name': 'Check', 'password': password})
|
||||
if status != 201:
|
||||
raise Stop('registration failed with %s' % status)
|
||||
status, session, _, _ = http('POST', '/api/v1/auth/sessions', headers=basic(username, password))
|
||||
if status != 201:
|
||||
raise Stop('login failed with %s' % status)
|
||||
return user, password, session['token']
|
||||
|
||||
|
||||
def post_message(token, chat_id, text, key):
|
||||
return http('POST', '/api/v1/chats/%s/messages' % chat_id, token, {'text': text}, {'Idempotency-Key': key}, timeout=5)
|
||||
|
||||
|
||||
def history(token, chat_id):
|
||||
items, cursor = [], None
|
||||
while True:
|
||||
path = '/api/v1/chats/%s/messages?limit=100' % chat_id + ('&cursor=' + cursor if cursor else '')
|
||||
status, page, _, _ = http('GET', path, token)
|
||||
if status != 200:
|
||||
raise Stop('history read failed with %s' % status)
|
||||
items += page['items']
|
||||
cursor = page['next_cursor']
|
||||
if not cursor:
|
||||
return items
|
||||
|
||||
|
||||
def wait_ready(seconds=60):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
if http('GET', '/health/ready', timeout=3)[0] == 200:
|
||||
return True
|
||||
time.sleep(0.5)
|
||||
return False
|
||||
|
||||
|
||||
def api_replicas(all_states=False):
|
||||
"""{short container id: container name} of the api service of this project."""
|
||||
command = ['docker', 'ps', '--filter', 'label=com.docker.compose.project=' + PROJECT,
|
||||
'--filter', 'label=com.docker.compose.service=api', '--format', '{{.ID}} {{.Names}}']
|
||||
if all_states:
|
||||
command.insert(2, '-a')
|
||||
return dict(line.split(' ', 1) for line in silent(command).splitlines() if line)
|
||||
|
||||
|
||||
def container(service):
|
||||
return silent(['docker', 'compose', 'ps', '-a', '-q', service])
|
||||
|
||||
|
||||
def inspect(target, template):
|
||||
return silent(['docker', 'inspect', '-f', template, target])
|
||||
|
||||
|
||||
def wait_replicas(count, seconds=90):
|
||||
"""Wait until `count` api replicas are healthy and all of them answer through the load balancer."""
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
replicas = api_replicas()
|
||||
healthy = [r for r in replicas if inspect(r, '{{.State.Health.Status}}') == 'healthy']
|
||||
if len(healthy) == count:
|
||||
seen = {http('GET', '/health/live', timeout=3)[3] for _ in range(4 * count)}
|
||||
if set(healthy) <= seen:
|
||||
return True
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
|
||||
def show(label, result):
|
||||
status, payload, seconds, instance = result
|
||||
code = payload['error']['code'] if isinstance(payload, dict) and 'error' in payload else ''
|
||||
out('%-58s -> %s %s in %.2f s, X-Instance-Id %s' % (label, status, code, seconds, instance or '-'))
|
||||
return status
|
||||
|
||||
|
||||
class StatsPoller(threading.Thread):
|
||||
"""Polls the HAProxy stats CSV every 100 ms and records when each server slot changes state.
|
||||
|
||||
A slot (api1..api8) is matched to a container once, by its IP address while it is UP.
|
||||
A stopped replica first turns DOWN (failed checks); a killed one usually goes to
|
||||
MAINT (resolution) first, because Docker DNS drops its address and the slot loses it.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(daemon=True)
|
||||
self.stop_event = threading.Event()
|
||||
self.changes = [] # (monotonic time, slot, new status)
|
||||
|
||||
@staticmethod
|
||||
def snapshot():
|
||||
"""{slot: (status, ip)} for the api backend."""
|
||||
try:
|
||||
with urllib.request.urlopen(STATS, timeout=1) as response:
|
||||
text = response.read().decode('utf-8').lstrip('# ')
|
||||
except OSError:
|
||||
return {}
|
||||
return {row['svname']: (row['status'].split(' ')[0], row.get('addr', '').split(':')[0])
|
||||
for row in csv.DictReader(io.StringIO(text))
|
||||
if row.get('pxname') == 'api' and row.get('svname') not in ('FRONTEND', 'BACKEND')}
|
||||
|
||||
def slot_of(self, address):
|
||||
for slot, (status, ip) in self.snapshot().items():
|
||||
if ip == address:
|
||||
return slot
|
||||
return None
|
||||
|
||||
def run(self):
|
||||
last = {}
|
||||
while not self.stop_event.is_set():
|
||||
now = time.monotonic()
|
||||
for slot, (status, _) in self.snapshot().items():
|
||||
if last.get(slot) != status:
|
||||
self.changes.append((now, slot, status))
|
||||
last[slot] = status
|
||||
time.sleep(0.1)
|
||||
|
||||
def first_change(self, slot, statuses, after):
|
||||
for moment, changed, status in self.changes:
|
||||
if changed == slot and status in statuses and moment >= after:
|
||||
return moment
|
||||
return None
|
||||
|
||||
|
||||
class Load:
|
||||
"""Closed-loop load through the load balancer: each worker posts a message and reads /users/me.
|
||||
|
||||
A failed POST is retried with the SAME Idempotency-Key and body, exactly like a careful client,
|
||||
so the final history must contain every acknowledged message once and no duplicates.
|
||||
"""
|
||||
|
||||
def __init__(self, workers=6):
|
||||
self.user, _, self.token = new_user(label='load')
|
||||
self.chat = http('POST', '/api/v1/chats', self.token, {'title': 'Load', 'member_ids': [self.user['id']]})[1]
|
||||
self.workers = workers
|
||||
self.events = [] # (monotonic time, ok, instance, kind)
|
||||
self.acknowledged = []
|
||||
self.lock = threading.Lock()
|
||||
self.stop_event = threading.Event()
|
||||
self.threads = []
|
||||
|
||||
def worker(self, index):
|
||||
while not self.stop_event.is_set():
|
||||
key = uuid.uuid4().hex
|
||||
text = 'load %d %s' % (index, key)
|
||||
while not self.stop_event.is_set():
|
||||
status, payload, _, instance = post_message(self.token, self.chat['id'], text, key)
|
||||
with self.lock:
|
||||
self.events.append((time.monotonic(), status == 201, instance, 'write'))
|
||||
if status == 201:
|
||||
self.acknowledged.append(payload['id'])
|
||||
if status == 201:
|
||||
break
|
||||
time.sleep(0.05)
|
||||
status, _, _, instance = http('GET', '/api/v1/users/me', self.token, timeout=5)
|
||||
with self.lock:
|
||||
self.events.append((time.monotonic(), status == 200, instance, 'read'))
|
||||
|
||||
def start(self):
|
||||
for index in range(self.workers):
|
||||
thread = threading.Thread(target=self.worker, args=(index,), daemon=True)
|
||||
thread.start()
|
||||
self.threads.append(thread)
|
||||
|
||||
def stop(self):
|
||||
self.stop_event.set()
|
||||
for thread in self.threads:
|
||||
thread.join(timeout=10)
|
||||
|
||||
def window(self, start, end):
|
||||
with self.lock:
|
||||
selected = [e for e in self.events if start <= e[0] < end]
|
||||
ok = sum(1 for e in selected if e[1])
|
||||
return {'requests': len(selected), 'ok': ok, 'errors': len(selected) - ok,
|
||||
'ok_rps': round(ok / (end - start), 1) if end > start else 0.0}
|
||||
|
||||
def max_gap(self, start, end):
|
||||
"""Longest interval without a successful request."""
|
||||
with self.lock:
|
||||
moments = sorted(e[0] for e in self.events if e[1] and start <= e[0] < end)
|
||||
points = [start] + moments + [end]
|
||||
return max(b - a for a, b in zip(points, points[1:]))
|
||||
|
||||
def last_served_by(self, instance, after):
|
||||
with self.lock:
|
||||
moments = [e[0] for e in self.events if e[2] == instance and e[0] >= after]
|
||||
return max(moments) if moments else None
|
||||
|
||||
def first_served_by(self, instance, after):
|
||||
with self.lock:
|
||||
moments = [e[0] for e in self.events if e[2] == instance and e[0] >= after]
|
||||
return min(moments) if moments else None
|
||||
|
||||
def verify(self):
|
||||
"""(acknowledged, in history, duplicates by text) for the load chat."""
|
||||
items = history(self.token, self.chat['id'])
|
||||
texts = [m['text'] for m in items]
|
||||
ids = {m['id'] for m in items}
|
||||
return len(set(self.acknowledged)), len(set(self.acknowledged) & ids), len(texts) - len(set(texts))
|
||||
|
||||
|
||||
# --- sections ---------------------------------------------------------------
|
||||
|
||||
def section_stack():
|
||||
out('== Stack: replicas, pinned images, health, limits, published ports, secrets ==')
|
||||
compose('ps', '-a', '--format', 'table {{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}')
|
||||
config = json.loads(compose('config', '--format', 'json', quiet=True)[0])
|
||||
services = config['services']
|
||||
images = {name: services[name].get('image', '') for name in ('lb', 'postgres', 'valkey')}
|
||||
verdict('third-party images are pinned by tag and digest', all('@sha256:' in image for image in images.values()),
|
||||
', '.join(images.values()))
|
||||
replicas = api_replicas()
|
||||
names = sorted(replicas.values())
|
||||
out('api replicas: ' + ', '.join('%s (%s)' % (name, cid) for cid, name in sorted(replicas.items(), key=lambda x: x[1])))
|
||||
no_container_name = not any('container_name' in service for service in services.values())
|
||||
verdict('at least two api replicas of one compose service, no container_name', len(replicas) >= 2 and no_container_name,
|
||||
', '.join(names))
|
||||
health = {name: inspect(cid, '{{.State.Health.Status}}') for cid, name in replicas.items()}
|
||||
for service in ('postgres', 'valkey'):
|
||||
health[service] = inspect(container(service), '{{.State.Health.Status}}')
|
||||
out('health: ' + ', '.join('%s=%s' % item for item in sorted(health.items())))
|
||||
verdict('all api replicas, postgres and valkey are healthy', set(health.values()) == {'healthy'})
|
||||
users = {name: silent(['docker', 'exec', cid, 'id', '-u']) for cid, name in replicas.items()}
|
||||
verdict('API processes do not run as root', set(users.values()) == {'10001'}, ', '.join('%s uid=%s' % i for i in sorted(users.items())))
|
||||
out('%-26s %8s %10s %s' % ('container', 'cpus', 'memory', 'published ports'))
|
||||
limited, bypass = True, []
|
||||
targets = [(name, cid) for cid, name in replicas.items()] + [(s, container(s)) for s in ('lb', 'postgres', 'valkey')]
|
||||
for name, cid in sorted(targets):
|
||||
cpus = int(inspect(cid, '{{.HostConfig.NanoCpus}}') or 0) / 1e9
|
||||
memory = int(inspect(cid, '{{.HostConfig.Memory}}') or 0) // (1024 * 1024)
|
||||
ports = inspect(cid, '{{range $p, $b := .HostConfig.PortBindings}}{{range $b}}{{.HostIp}}:{{.HostPort}}->{{$p}} {{end}}{{end}}')
|
||||
out('%-26s %8.2f %8d MB %s' % (name, cpus, memory, ports or '-'))
|
||||
limited = limited and cpus > 0 and memory > 0
|
||||
if ports and name != 'lb':
|
||||
bypass.append(name)
|
||||
if ports and any(not part.startswith('127.0.0.1:') for part in ports.split()):
|
||||
bypass.append(name + ' (not 127.0.0.1)')
|
||||
verdict('every container has CPU and memory limits', limited)
|
||||
verdict('only the load balancer publishes ports, and only on 127.0.0.1', not bypass, ', '.join(bypass))
|
||||
secrets = {name: sorted(s['source'] for s in service.get('secrets', [])) for name, service in services.items()}
|
||||
out('secrets per service: ' + '; '.join('%s=%s' % (k, ','.join(v) or '-') for k, v in sorted(secrets.items())))
|
||||
verdict('secrets are given only where needed', secrets.get('lb') == [] and secrets['api'] == ['cache_url', 'database_url']
|
||||
and secrets['postgres'] == ['postgres_password'] and secrets['valkey'] == ['valkey_auth'])
|
||||
aof = ' '.join(valkey_cli('CONFIG GET ' + name) for name in ('appendonly', 'appendfsync', 'maxmemory-policy'))
|
||||
verdict('Valkey persistence is AOF with fsync always, no eviction', all(w in aof.split() for w in ('yes', 'always', 'noeviction')))
|
||||
|
||||
|
||||
def section_migrations():
|
||||
out('== Migrations: run once by the migrate service, idempotent re-run ==')
|
||||
psql("SELECT version || ' ' || applied_at FROM schema_migrations ORDER BY version")
|
||||
psql("SELECT format('%s %s %s', conrelid::regclass, contype, conname) FROM pg_constraint "
|
||||
"WHERE conrelid::regclass::text = 'idempotency_keys' ORDER BY 1")
|
||||
before = psql("SELECT (SELECT count(*) FROM schema_migrations) || ' ' || (SELECT count(*) FROM users)")
|
||||
output = compose('run', '--rm', '--no-deps', 'migrate')[0]
|
||||
after = psql("SELECT (SELECT count(*) FROM schema_migrations) || ' ' || (SELECT count(*) FROM users)")
|
||||
out('schema_migrations and users rows before/after: %s / %s' % (before, after))
|
||||
verdict('re-running migrations applies nothing and duplicates nothing', 'applied_now=0' in output and before == after)
|
||||
|
||||
|
||||
def section_balancing():
|
||||
out('== Balancing: one session, many replicas ==')
|
||||
_, _, token = new_user(label='balance')
|
||||
replicas = api_replicas()
|
||||
counts = {}
|
||||
for _ in range(40):
|
||||
status, _, _, instance = http('GET', '/api/v1/users/me', token)
|
||||
counts[(instance, status)] = counts.get((instance, status), 0) + 1
|
||||
for (instance, status), count in sorted(counts.items(), key=str):
|
||||
out('X-Instance-Id %s (%s): %d responses with status %s' % (instance, replicas.get(instance, 'unknown container'), count, status))
|
||||
instances = {instance for instance, _ in counts}
|
||||
verdict('40 requests with one token are served by every replica without a new login',
|
||||
instances == set(replicas) and {status for _, status in counts} == {200})
|
||||
|
||||
|
||||
def section_idempotency():
|
||||
out('== Idempotency: 20 parallel POSTs with one key through the load balancer ==')
|
||||
user, _, token = new_user(label='idem')
|
||||
chat = http('POST', '/api/v1/chats', token, {'title': 'Idempotency', 'member_ids': [user['id']]})[1]
|
||||
key, text = uuid.uuid4().hex, 'idempotent ' + uuid.uuid4().hex
|
||||
barrier = threading.Barrier(20)
|
||||
|
||||
def post(_):
|
||||
barrier.wait()
|
||||
return post_message(token, chat['id'], text, key)
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as pool:
|
||||
responses = list(pool.map(post, range(20)))
|
||||
statuses = sorted({r[0] for r in responses}, key=str)
|
||||
ids = {r[1]['id'] for r in responses if r[0] == 201}
|
||||
instances = {r[3] for r in responses}
|
||||
rows = psql("SELECT count(*) FROM messages WHERE chat_id = '%s'" % chat['id'])
|
||||
keys = psql("SELECT count(*) FROM idempotency_keys WHERE key = '%s'" % key)
|
||||
out('statuses: %s; distinct message ids: %d; replicas that answered: %s' % (statuses, len(ids), ', '.join(sorted(instances))))
|
||||
out('messages in the chat: %s; idempotency_keys rows for the key: %s' % (rows, keys))
|
||||
conflict = show('same key, another body', post_message(token, chat['id'], text + ' changed', key))
|
||||
verdict('one message for 20 concurrent requests with one key on several replicas',
|
||||
statuses == [201] and len(ids) == 1 and len(instances) >= 2 and rows == '1' and keys == '1')
|
||||
verdict('the same key with another body is rejected with 409', conflict == 409)
|
||||
|
||||
|
||||
def section_rollback():
|
||||
out('== Rollback: chat creation fails on the second INSERT ==')
|
||||
user, _, token = new_user(label='rollback')
|
||||
title = 'rollback-probe-' + uuid.uuid4().hex[:10]
|
||||
status = show('POST /api/v1/chats with one unknown member',
|
||||
http('POST', '/api/v1/chats', token, {'title': title, 'member_ids': [user['id'], str(uuid.uuid4())]}))
|
||||
rows = psql("SELECT count(*) FROM chats WHERE title = '%s'" % title)
|
||||
orphans = psql('SELECT count(*) FROM chats c WHERE NOT EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)')
|
||||
out('rows in chats with this title: %s; chats without members: %s' % (rows, orphans))
|
||||
verdict('composite chat creation is rolled back as a whole', status == 400 and rows == '0' and orphans == '0')
|
||||
|
||||
|
||||
def section_race():
|
||||
out('== Registration race: 20 parallel requests with one username ==')
|
||||
username = 'race_' + uuid.uuid4().hex[:12]
|
||||
barrier = threading.Barrier(20)
|
||||
|
||||
def register(index):
|
||||
barrier.wait()
|
||||
body = {'username': username, 'display_name': 'Race %d' % index, 'password': 'Race-password-%02d' % index}
|
||||
return http('POST', '/api/v1/users', body=body)
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=20) as pool:
|
||||
responses = list(pool.map(register, range(20)))
|
||||
statuses = sorted((r[0] for r in responses), key=str)
|
||||
out('statuses: ' + ' '.join(str(s) for s in statuses) + '; replicas: ' + ', '.join(sorted({r[3] for r in responses})))
|
||||
rows = psql("SELECT count(*) FROM users WHERE username = '%s'" % username)
|
||||
out('rows in users with this username: ' + rows)
|
||||
verdict('exactly one registration wins, the other 19 get 409', statuses == [201] + [409] * 19 and rows == '1')
|
||||
|
||||
|
||||
def section_pool():
|
||||
replicas = len(api_replicas())
|
||||
out('== Connection pools: 60 parallel clients, at most 10 connections per replica (%d replicas) ==' % replicas)
|
||||
users = [new_user(label='pool') for _ in range(4)]
|
||||
chats = [http('POST', '/api/v1/chats', token, {'title': 'Pool', 'member_ids': [user['id']]})[1] for user, _, token in users]
|
||||
count_sql = "SELECT count(*) FROM pg_stat_activity WHERE application_name = 'messenger-api'"
|
||||
stop = threading.Event()
|
||||
samples = []
|
||||
|
||||
def sample():
|
||||
while not stop.is_set():
|
||||
value = psql(count_sql, quiet=True)
|
||||
if value.isdigit():
|
||||
samples.append(int(value))
|
||||
time.sleep(0.2)
|
||||
|
||||
def client(index):
|
||||
_, _, token = users[index % 4]
|
||||
path = '/api/v1/chats/%s/messages?limit=20' % chats[index % 4]['id']
|
||||
statuses = []
|
||||
deadline = time.monotonic() + 5
|
||||
while time.monotonic() < deadline:
|
||||
statuses.append(http('GET', path, token)[0])
|
||||
return statuses
|
||||
|
||||
sampler = threading.Thread(target=sample)
|
||||
sampler.start()
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=60) as pool:
|
||||
statuses = [s for batch in pool.map(client, range(60)) for s in batch]
|
||||
finally:
|
||||
stop.set()
|
||||
sampler.join()
|
||||
counts = {status: statuses.count(status) for status in sorted(set(statuses), key=str)}
|
||||
out('requests: %d, statuses: %s' % (len(statuses), counts))
|
||||
out('API connections in pg_stat_activity, %d samples: max %s' % (len(samples), max(samples) if samples else '-'))
|
||||
verdict('pools never open more than 10 connections per replica and all requests succeed',
|
||||
bool(samples) and max(samples) <= 10 * replicas and set(counts) == {200})
|
||||
|
||||
|
||||
def section_timeouts():
|
||||
out('== Timeouts: chat row locked for 6 s by another session, API answers 503 after lock_timeout=3s ==')
|
||||
user, _, token = new_user(label='lock')
|
||||
chat = http('POST', '/api/v1/chats', token, {'title': 'Lock', 'member_ids': [user['id']]})[1]
|
||||
lock = "BEGIN; SELECT 1 FROM chats WHERE id = '%s' FOR UPDATE; SELECT pg_sleep(6); ROLLBACK;" % chat['id']
|
||||
out('$ docker compose ' + ' '.join(PSQL) + ' "' + lock + '" (in the background)')
|
||||
locker = subprocess.Popen(['docker', 'compose'] + PSQL + [lock], cwd=str(ROOT), stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
time.sleep(1.5)
|
||||
result = post_message(token, chat['id'], 'while locked', uuid.uuid4().hex)
|
||||
show('POST message while the chat row is locked', result)
|
||||
locker.wait(timeout=30)
|
||||
after = show('POST message after the lock is released', post_message(token, chat['id'], 'after lock', uuid.uuid4().hex))
|
||||
verdict('waiting for a lock is bounded: 503 after about 3 s, then writes work again',
|
||||
result[0] == 503 and 2.5 <= result[2] <= 4.5 and after == 201, '%.2f s' % result[2])
|
||||
|
||||
|
||||
def section_logs():
|
||||
out('== Logs: useful fields, no message texts, passwords or tokens (all api replicas and lb) ==')
|
||||
user, password, token = new_user(label='logs')
|
||||
chat = http('POST', '/api/v1/chats', token, {'title': 'Logs', 'member_ids': [user['id']]})[1]
|
||||
marker = 'log-marker-' + uuid.uuid4().hex
|
||||
post_message(token, chat['id'], marker, uuid.uuid4().hex)
|
||||
logs = compose('logs', '--no-log-prefix', 'api', 'lb', quiet=True)[0]
|
||||
compose('logs', '--tail', '2', 'api')
|
||||
compose('logs', '--no-log-prefix', '--tail', '2', 'lb')
|
||||
leaked = [name for name, value in (('message text', marker), ('password', password), ('token', token)) if value in logs]
|
||||
leaked += [p.name for p in (ROOT / 'secrets').glob('*_password') if p.read_text().strip() in logs]
|
||||
out('searched %d log lines; found: %s' % (logs.count('\n') + 1, ', '.join(leaked) or 'nothing'))
|
||||
verdict('logs contain request_id/status/duration and no secrets', 'request_id=' in logs and not leaked)
|
||||
|
||||
|
||||
def section_expiry():
|
||||
out('== Session expiry: api replicas recreated with SESSION_TTL_SECONDS=2, then restored ==')
|
||||
count = len(api_replicas())
|
||||
short = dict(os.environ, SESSION_TTL_SECONDS='2')
|
||||
compose('up', '-d', '--no-deps', '--wait', 'api', env=short, quiet=True)
|
||||
try:
|
||||
if not wait_replicas(count):
|
||||
raise Stop('replicas are not ready with the short TTL')
|
||||
_, _, token = new_user(label='expiry')
|
||||
key = 'session:' + hashlib.sha256(token.encode()).hexdigest()
|
||||
first = show('GET /users/me right after login', http('GET', '/api/v1/users/me', token))
|
||||
pttl = valkey_cli('PTTL ' + key)
|
||||
out('PTTL of the session key in Valkey (ms): ' + pttl)
|
||||
time.sleep(3)
|
||||
expired = show('GET /users/me 3 s later', http('GET', '/api/v1/users/me', token))
|
||||
gone = valkey_cli('EXISTS ' + key)
|
||||
out('session key still exists in Valkey: ' + gone)
|
||||
verdict('session expires after SESSION_TTL_SECONDS and the key disappears from Valkey',
|
||||
first == 200 and 0 < int(pttl) <= 2000 and expired == 401 and gone == '0')
|
||||
finally:
|
||||
compose('up', '-d', '--no-deps', '--wait', 'api', quiet=True)
|
||||
wait_replicas(count)
|
||||
|
||||
|
||||
def outage(service, limit):
|
||||
"""Stop one shared store and look at the answers through the load balancer."""
|
||||
user, password, token = new_user(label=service)
|
||||
compose('stop', service)
|
||||
try:
|
||||
me = show('GET /users/me with a valid token (first seconds)', http('GET', '/api/v1/users/me', token))
|
||||
time.sleep(3) # the load balancer has seen failing readiness checks by now
|
||||
me_later = show('GET /users/me with a valid token (after 3 s)', http('GET', '/api/v1/users/me', token))
|
||||
anonymous = show('GET /users/me without a token', http('GET', '/api/v1/users/me'))
|
||||
ready = show('GET /health/ready', http('GET', '/health/ready'))
|
||||
slowest = max(http('GET', '/api/v1/users/me', token)[2] for _ in range(3))
|
||||
out('slowest of 3 more authenticated requests: %.2f s' % slowest)
|
||||
finally:
|
||||
compose('start', service)
|
||||
started = time.monotonic()
|
||||
recovered = False
|
||||
while time.monotonic() - started < 60 and not recovered:
|
||||
recovered = http('GET', '/api/v1/users/me', token, timeout=5)[0] == 200
|
||||
time.sleep(0.5)
|
||||
out('same token after %s is back: %s' % (service, '200 after %.1f s' % (time.monotonic() - started) if recovered else 'not accepted'))
|
||||
verdict('without %s: bounded 503, nothing is served as anonymous' % service,
|
||||
me == 503 and me_later == 503 and anonymous in (401, 503) and ready == 503 and slowest < limit)
|
||||
verdict('the stand recovers by itself when %s is back, the session is still valid' % service, recovered)
|
||||
|
||||
|
||||
def section_valkey_down():
|
||||
out('== Valkey is down ==')
|
||||
outage('valkey', 3)
|
||||
|
||||
|
||||
def section_postgres_down():
|
||||
out('== PostgreSQL is down ==')
|
||||
outage('postgres', 5)
|
||||
|
||||
|
||||
def section_failover():
|
||||
out('== Public failover test: tests/resilience.py --mode failover, repeated until every replica was the target ==')
|
||||
hook = 'scripts/stop-one.cmd' if os.name == 'nt' else 'scripts/stop-one.sh'
|
||||
replicas = api_replicas()
|
||||
targeted = set()
|
||||
for attempt in range(1, 7):
|
||||
before = set(api_replicas())
|
||||
_, code, seconds = run(PY + ['tests/resilience.py', '--mode', 'failover', '--action-script', hook, '--base-url', BASE], check=False)
|
||||
stopped = before - set(api_replicas())
|
||||
for cid in stopped:
|
||||
out('attempt %d: stopped container %s (%s), State: %s, ExitCode %s' % (
|
||||
attempt, cid, replicas.get(cid), inspect(cid, '{{.State.Status}}'), inspect(cid, '{{.State.ExitCode}}')))
|
||||
verdict('public failover test passes (attempt %d)' % attempt, code == 0 and len(stopped) == 1, '%.1f s' % seconds)
|
||||
targeted |= stopped
|
||||
compose('up', '-d', '--wait', quiet=True)
|
||||
if not wait_replicas(len(replicas)):
|
||||
raise Stop('replicas did not come back')
|
||||
if targeted >= set(replicas):
|
||||
break
|
||||
verdict('every replica was stopped by the hook at least once', targeted >= set(replicas),
|
||||
', '.join(sorted(replicas[c] for c in targeted if c in replicas)))
|
||||
|
||||
|
||||
def section_experiment():
|
||||
out('== Experiment: stop, then kill, each replica in turn under load; restore before the next one ==')
|
||||
replicas = api_replicas()
|
||||
count = len(replicas)
|
||||
load = Load()
|
||||
poller = StatsPoller()
|
||||
poller.start()
|
||||
load.start()
|
||||
rows = []
|
||||
try:
|
||||
time.sleep(5)
|
||||
for mode in ('stop', 'kill'):
|
||||
for cid, name in sorted(replicas.items(), key=lambda item: item[1]):
|
||||
address = inspect(cid, '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}')
|
||||
slot = poller.slot_of(address)
|
||||
base_start = time.monotonic() - 5
|
||||
baseline = load.window(base_start, time.monotonic())
|
||||
t0 = time.monotonic()
|
||||
_, _, command_seconds = run(['docker', mode, cid], quiet=True)
|
||||
time.sleep(8)
|
||||
t1 = time.monotonic()
|
||||
during = load.window(t0, t1)
|
||||
gap = load.max_gap(t0, t1)
|
||||
detected = poller.first_change(slot, ('DOWN', 'MAINT', 'NOLB'), t0)
|
||||
last_served = load.last_served_by(cid, t0)
|
||||
run(['docker', 'start', cid], quiet=True)
|
||||
t2 = time.monotonic()
|
||||
back = None
|
||||
while time.monotonic() - t2 < 60 and back is None:
|
||||
time.sleep(0.2)
|
||||
back = load.first_served_by(cid, t2)
|
||||
time.sleep(3)
|
||||
after = load.window(back or t2, time.monotonic())
|
||||
row = {
|
||||
'replica': name, 'lb_slot': slot, 'action': 'docker ' + mode, 'command_s': round(command_seconds, 2),
|
||||
'baseline_ok_rps': baseline['ok_rps'], 'during_ok_rps': during['ok_rps'],
|
||||
'errors_during': during['errors'], 'requests_during': during['requests'],
|
||||
'max_gap_s': round(gap, 2),
|
||||
'lb_marked_down_s': round(detected - t0, 2) if detected else None,
|
||||
'last_served_s': round(last_served - t0, 2) if last_served else None,
|
||||
'back_in_rotation_s': round(back - t2, 2) if back else None,
|
||||
'after_ok_rps': after['ok_rps'], 'errors_after': after['errors'],
|
||||
}
|
||||
rows.append(row)
|
||||
out(json.dumps(row, ensure_ascii=False))
|
||||
if not wait_replicas(count):
|
||||
raise Stop('replicas did not come back after %s' % name)
|
||||
time.sleep(2)
|
||||
finally:
|
||||
load.stop()
|
||||
poller.stop_event.set()
|
||||
acknowledged, present, duplicates = load.verify()
|
||||
out()
|
||||
out('%-16s %-12s %6s %8s %8s %7s %8s %8s %8s %8s' % (
|
||||
'replica', 'action', 'cmd,s', 'rps0', 'rps', 'errors', 'gap,s', 'down,s', 'back,s', 'rps1'))
|
||||
for r in rows:
|
||||
out('%-16s %-12s %6s %8s %8s %7s %8s %8s %8s %8s' % (
|
||||
r['replica'], r['action'], r['command_s'], r['baseline_ok_rps'], r['during_ok_rps'], r['errors_during'],
|
||||
r['max_gap_s'], r['lb_marked_down_s'], r['back_in_rotation_s'], r['after_ok_rps']))
|
||||
out('acknowledged writes: %d, found in history: %d, duplicates: %d' % (acknowledged, present, duplicates))
|
||||
stops = [r for r in rows if r['action'] == 'docker stop']
|
||||
kills = [r for r in rows if r['action'] == 'docker kill']
|
||||
verdict('graceful stop of each replica: no client errors, LB removes it within 3 s',
|
||||
len(stops) == count and all(r['errors_during'] == 0 and r['lb_marked_down_s'] is not None
|
||||
and r['lb_marked_down_s'] <= 3 for r in stops))
|
||||
verdict('kill of each replica: LB removes it within 3 s, writes go on within 10 s',
|
||||
len(kills) == count and all(r['lb_marked_down_s'] is not None and r['lb_marked_down_s'] <= 3
|
||||
and r['max_gap_s'] <= 10 for r in kills))
|
||||
verdict('every replica comes back into rotation after docker start', all(r['back_in_rotation_s'] is not None for r in rows))
|
||||
verdict('no acknowledged write is lost or duplicated (retries reuse the key)', acknowledged == present and duplicates == 0)
|
||||
|
||||
|
||||
def section_rolling():
|
||||
out('== Rolling update under load: one replica at a time, new image, migrations once ==')
|
||||
replicas = api_replicas()
|
||||
count = len(replicas)
|
||||
old_image = inspect('mtusi-messenger:lab3', '{{.Id}}')
|
||||
version = 'rolling-%d' % int(time.time())
|
||||
env = dict(os.environ, APP_VERSION=version)
|
||||
compose('build', 'api', env=env, quiet=True)
|
||||
new_image = inspect('mtusi-messenger:lab3', '{{.Id}}')
|
||||
out('image before %s, after %s, APP_VERSION=%s' % (old_image[7:19], new_image[7:19], version))
|
||||
migrate = compose('run', '--rm', '--no-deps', 'migrate', env=env)[0]
|
||||
load = Load()
|
||||
load.start()
|
||||
started = time.monotonic()
|
||||
steps = []
|
||||
try:
|
||||
time.sleep(3)
|
||||
for old in sorted(replicas, key=lambda cid: replicas[cid]):
|
||||
known = set(api_replicas(all_states=True))
|
||||
compose('up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % (count + 1), 'api', env=env, quiet=True)
|
||||
new = [cid for cid in api_replicas() if cid not in known]
|
||||
if len(new) != 1 or not wait_replicas(count + 1):
|
||||
raise Stop('new replica did not become ready')
|
||||
run(['docker', 'stop', old], quiet=True)
|
||||
run(['docker', 'rm', old], quiet=True)
|
||||
steps.append('%s (%s) -> %s' % (replicas[old], old, new[0]))
|
||||
out('replaced ' + steps[-1])
|
||||
time.sleep(3)
|
||||
finally:
|
||||
load.stop()
|
||||
stats = load.window(started, time.monotonic())
|
||||
acknowledged, present, duplicates = load.verify()
|
||||
final = api_replicas()
|
||||
images = {cid: inspect(cid, '{{.Image}}') for cid in final}
|
||||
versions = {cid: silent(['docker', 'exec', cid, 'printenv', 'APP_VERSION']) for cid in final}
|
||||
out('during the update: %s' % json.dumps(stats))
|
||||
out('final replicas: ' + ', '.join('%s %s version=%s' % (final[c], c, versions[c]) for c in sorted(final)))
|
||||
out('acknowledged writes: %d, found in history: %d, duplicates: %d' % (acknowledged, present, duplicates))
|
||||
verdict('migrations are applied once and the change is backward compatible', 'applied_now=0' in migrate)
|
||||
verdict('every replica was replaced one by one by the new image under load',
|
||||
len(steps) == count and len(final) == count and set(images.values()) == {new_image}
|
||||
and set(versions.values()) == {version})
|
||||
verdict('no client errors and no lost or duplicated writes during the rolling update',
|
||||
stats['errors'] == 0 and acknowledged == present and duplicates == 0,
|
||||
'%d requests, %.1f ok RPS' % (stats['requests'], stats['ok_rps']))
|
||||
|
||||
|
||||
def section_persistence():
|
||||
out('== Full restart: public tests/resilience.py --mode persistence with the restart hook ==')
|
||||
hook = 'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh'
|
||||
services = {name: cid for cid, name in api_replicas().items()}
|
||||
services.update({s: container(s) for s in ('postgres', 'valkey', 'lb')})
|
||||
before = {name: inspect(cid, '{{.State.StartedAt}}') for name, cid in services.items()}
|
||||
volumes = ['%s_pgdata' % PROJECT, '%s_valkeydata' % PROJECT]
|
||||
volumes_before = run(['docker', 'volume', 'inspect', '-f', '{{.Name}} {{.CreatedAt}}'] + volumes)[0]
|
||||
_, code, seconds = run(PY + ['tests/resilience.py', '--mode', 'persistence', '--action-script', hook, '--base-url', BASE], check=False)
|
||||
after = {name: inspect(cid, '{{.State.StartedAt}}') for name, cid in services.items()}
|
||||
volumes_after = run(['docker', 'volume', 'inspect', '-f', '{{.Name}} {{.CreatedAt}}'] + volumes)[0]
|
||||
for name in sorted(before):
|
||||
out('%-18s StartedAt %s -> %s' % (name, before[name][11:23], after[name][11:23]))
|
||||
restarted = all(before[name] != after[name] for name in before)
|
||||
verdict('persistence test passes after restarting every replica, postgres, valkey and lb with the same volumes',
|
||||
code == 0 and restarted and volumes_before == volumes_after, 'hook %s, %.1f s' % (hook, seconds))
|
||||
|
||||
|
||||
SECTIONS = {
|
||||
'stack': section_stack,
|
||||
'migrations': section_migrations,
|
||||
'balancing': section_balancing,
|
||||
'idempotency': section_idempotency,
|
||||
'rollback': section_rollback,
|
||||
'race': section_race,
|
||||
'pool': section_pool,
|
||||
'timeouts': section_timeouts,
|
||||
'logs': section_logs,
|
||||
'expiry': section_expiry,
|
||||
'valkey-down': section_valkey_down,
|
||||
'postgres-down': section_postgres_down,
|
||||
'failover': section_failover,
|
||||
'experiment': section_experiment,
|
||||
'rolling': section_rolling,
|
||||
'persistence': section_persistence,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Lab 3 infrastructure checks; creates synthetic users and messages')
|
||||
parser.add_argument('--only', nargs='+', choices=sorted(SECTIONS), help='run only these sections')
|
||||
parser.add_argument('--output', default=None, help='also write the transcript to this file')
|
||||
args = parser.parse_args()
|
||||
sys.stdout.reconfigure(errors='replace')
|
||||
out('# %s UTC, python %s, %s' % (time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()), sys.version.split()[0], sys.platform))
|
||||
out('# ' + silent(['docker', 'version', '--format', 'docker {{.Server.Version}}']))
|
||||
if not wait_ready(seconds=30):
|
||||
out('API is not ready on %s: start the stand with docker compose up -d --build --wait' % BASE)
|
||||
return 2
|
||||
for name in args.only or SECTIONS:
|
||||
try:
|
||||
SECTIONS[name]()
|
||||
except (Stop, KeyError, TypeError, ValueError) as error:
|
||||
verdict('section %s completed' % name, False, str(error) or type(error).__name__)
|
||||
if not wait_ready(seconds=60):
|
||||
verdict('stand is ready after section %s' % name, False)
|
||||
break
|
||||
out('== Summary ==')
|
||||
for name, ok in results:
|
||||
out('%s %s' % ('PASS' if ok else 'FAIL', name))
|
||||
failed = [name for name, ok in results if not ok]
|
||||
out('RESULT: %s' % ('FAIL' if failed else 'PASS'))
|
||||
if args.output:
|
||||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.output).write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
sys.exit(main())
|
||||
except FileNotFoundError:
|
||||
print('docker CLI was not found: install Docker and make sure it is on PATH')
|
||||
sys.exit(2)
|
||||
@@ -0,0 +1,10 @@
|
||||
@echo off
|
||||
rem Windows twin of scripts/restart.sh: tests/resilience.py starts the hook directly,
|
||||
rem and Windows cannot execute a .sh file. Same actions, volumes are kept.
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
docker compose stop api || exit /b 1
|
||||
docker compose restart postgres valkey || exit /b 1
|
||||
docker compose start api || exit /b 1
|
||||
docker compose restart lb || exit /b 1
|
||||
exit /b 0
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
# Hook for: make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
# Restarts the whole stand: all API replicas, PostgreSQL, Valkey and the load balancer,
|
||||
# keeping the named volumes: no "down -v", no FLUSHALL, no re-seed.
|
||||
# The test itself waits for API readiness through the load balancer afterwards.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop api
|
||||
docker compose restart postgres valkey
|
||||
docker compose start api
|
||||
docker compose restart lb
|
||||
@@ -0,0 +1,19 @@
|
||||
@echo off
|
||||
rem Windows twin of scripts/stop-one.sh: tests/resilience.py starts the hook directly,
|
||||
rem and Windows cannot execute a .sh file. Stops ONE api replica by X-Instance-Id (container ID).
|
||||
setlocal
|
||||
if "%TARGET_INSTANCE_ID%"=="" (
|
||||
echo TARGET_INSTANCE_ID is not set 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
cd /d "%~dp0.."
|
||||
set PROJECT=%COMPOSE_PROJECT_NAME%
|
||||
if "%PROJECT%"=="" set PROJECT=mtusi-lab3
|
||||
set ID=
|
||||
for /f %%i in ('docker ps -q --filter "id=%TARGET_INSTANCE_ID%" --filter "label=com.docker.compose.project=%PROJECT%" --filter "label=com.docker.compose.service=api"') do set ID=%%i
|
||||
if "%ID%"=="" (
|
||||
echo TARGET_INSTANCE_ID is not a running api replica of %PROJECT% 1>&2
|
||||
exit /b 1
|
||||
)
|
||||
docker stop %ID% >nul || exit /b 1
|
||||
exit /b 0
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
# Hook for: make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
# Stops exactly ONE API replica and leaves it stopped. TARGET_INSTANCE_ID is the X-Instance-Id
|
||||
# of the replica, which is its container hostname = short container ID. The hook checks that
|
||||
# the ID belongs to a running "api" container of this compose project before stopping it.
|
||||
# docker stop sends SIGTERM: the replica reports not ready, HAProxy removes it, then it exits.
|
||||
# Restore afterwards: docker compose up -d (or docker start <id>)
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
cd "$(dirname "$0")/.."
|
||||
project="${COMPOSE_PROJECT_NAME:-mtusi-lab3}"
|
||||
id=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \
|
||||
--filter "label=com.docker.compose.project=$project" \
|
||||
--filter "label=com.docker.compose.service=api")
|
||||
if [ -z "$id" ]; then
|
||||
echo "TARGET_INSTANCE_ID is not a running api replica of $project" >&2
|
||||
exit 1
|
||||
fi
|
||||
docker stop "$id" >/dev/null
|
||||
Reference in New Issue
Block a user