forked from mtusi2026/mtusi2026lab5
ЛР 5 — БВТ2304 — Черников — на 4
This commit is contained in:
@@ -0,0 +1,208 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Regression: pause PostgreSQL, direct API 503, discard connection and recover.
|
||||
|
||||
No published replica ports: direct HTTP is sent from inside each API container.
|
||||
PostgreSQL is always unpaused in finally. Uses existing local synthetic stand.
|
||||
"""
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
from pathlib import Path
|
||||
import queue
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check
|
||||
|
||||
LOG = []
|
||||
|
||||
|
||||
def record(value):
|
||||
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
|
||||
LOG.append(text)
|
||||
print(text, flush=True)
|
||||
|
||||
|
||||
def compose(*args, timeout=40):
|
||||
proc = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True,
|
||||
text=True, encoding='utf-8', errors='replace', timeout=timeout)
|
||||
if proc.returncode:
|
||||
raise RuntimeError('docker compose ' + ' '.join(args[:3]) + ': ' + proc.stderr[-1200:])
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
PROBE = r'''
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import http.client, json, sys, time
|
||||
|
||||
expected, count = map(int, sys.argv[1:])
|
||||
def request(path, status):
|
||||
c = http.client.HTTPConnection('127.0.0.1', 8080, timeout=8)
|
||||
start = time.monotonic()
|
||||
try:
|
||||
c.request('GET', path)
|
||||
response = c.getresponse()
|
||||
body = json.loads(response.read())
|
||||
assert response.status == status, (response.status, body)
|
||||
assert response.getheader('X-Request-Id')
|
||||
assert response.getheader('X-Instance-Id')
|
||||
if status == 503:
|
||||
assert body['error']['code'] == 'unavailable'
|
||||
assert body['error']['request_id'] == response.getheader('X-Request-Id')
|
||||
return round(time.monotonic() - start, 3)
|
||||
finally:
|
||||
c.close()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=count) as executor:
|
||||
futures = [executor.submit(request, '/health/ready', expected) for _ in range(count)]
|
||||
live = request('/health/live', 200)
|
||||
elapsed = [future.result() for future in futures]
|
||||
print(json.dumps(dict(status=expected, requests=count, seconds=elapsed, live_seconds=live)))
|
||||
'''
|
||||
|
||||
|
||||
WORKER = r'''
|
||||
import json, sys, time
|
||||
from messenger.config import read_secret
|
||||
from messenger.db import create_pool, Repository
|
||||
from messenger.errors import Unavailable
|
||||
|
||||
class SyntheticRollback(Exception): pass
|
||||
pool = create_pool(read_secret('DATABASE_URL'), 1)
|
||||
pool.open(wait=True, timeout=10)
|
||||
repo = Repository(pool)
|
||||
phase = sys.argv[1]
|
||||
try:
|
||||
try:
|
||||
with repo._transaction() as conn:
|
||||
conn.execute('SELECT 1')
|
||||
print('ARMED', flush=True)
|
||||
assert sys.stdin.readline().strip() == 'go'
|
||||
started = time.monotonic()
|
||||
if phase == 'query':
|
||||
conn.execute('SELECT 1')
|
||||
elif phase == 'rollback':
|
||||
raise SyntheticRollback()
|
||||
# commit: leave the block while PostgreSQL is paused.
|
||||
except (Unavailable, SyntheticRollback) as error:
|
||||
elapsed = time.monotonic() - started
|
||||
assert conn.closed, 'timed-out connection was returned open'
|
||||
assert elapsed < 8, elapsed
|
||||
expected = SyntheticRollback if phase == 'rollback' else Unavailable
|
||||
assert isinstance(error, expected), type(error)
|
||||
print('TIMED_OUT', flush=True)
|
||||
else:
|
||||
raise AssertionError('operation unexpectedly succeeded while PostgreSQL paused')
|
||||
assert sys.stdin.readline().strip() == 'resume'
|
||||
recovery = time.monotonic()
|
||||
while True:
|
||||
try:
|
||||
repo.ping()
|
||||
break
|
||||
except Unavailable:
|
||||
if time.monotonic() - recovery >= 15:
|
||||
raise
|
||||
time.sleep(0.1)
|
||||
assert pool.get_stats().get('requests_waiting', 0) == 0
|
||||
print(json.dumps(dict(phase=phase, timeout_seconds=round(elapsed, 3),
|
||||
connection_closed=conn.closed, pool_max_size=1,
|
||||
recovered_seconds=round(time.monotonic()-recovery, 3))), flush=True)
|
||||
finally:
|
||||
pool.close(timeout=1)
|
||||
'''
|
||||
|
||||
|
||||
def line(proc, timeout=15):
|
||||
result = queue.Queue()
|
||||
threading.Thread(target=lambda: result.put(proc.stdout.readline()), daemon=True).start()
|
||||
try:
|
||||
return result.get(timeout=timeout).strip()
|
||||
except queue.Empty:
|
||||
raise AssertionError('database test worker did not finish in time') from None
|
||||
|
||||
|
||||
def ready(c):
|
||||
start = time.monotonic()
|
||||
while time.monotonic() - start < 30:
|
||||
try:
|
||||
c.call('GET', '/health/ready', schema='Health')
|
||||
return round(time.monotonic() - start, 3)
|
||||
except Failure:
|
||||
time.sleep(0.2)
|
||||
raise AssertionError('API did not recover after PostgreSQL unpause')
|
||||
|
||||
|
||||
def readiness(c, ids):
|
||||
a, b, eve, chat = c.bootstrap()
|
||||
for index in range(len(ids)):
|
||||
compose('exec', '--index', str(index + 1), '-T', 'api', 'python', '-c', PROBE, '200', '1')
|
||||
for cycle in range(2):
|
||||
try:
|
||||
compose('pause', 'postgres')
|
||||
with ThreadPoolExecutor(max_workers=len(ids)) as executor:
|
||||
futures = [executor.submit(compose, 'exec', '--index', str(index + 1), '-T', 'api',
|
||||
'python', '-c', PROBE, '503', '12') for index in range(len(ids))]
|
||||
for index, future in enumerate(futures):
|
||||
record(dict(cycle=cycle + 1, api=index + 1, **json.loads(future.result())))
|
||||
finally:
|
||||
compose('unpause', 'postgres')
|
||||
elapsed = ready(c)
|
||||
# Same token and a real DB write/read after recovery, no API restart.
|
||||
message = c.message(a, chat, 'PostgreSQL pause regression ' + str(cycle))['json']
|
||||
page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=b, schema='MessagePage')['json']
|
||||
check(message in page['items'], 'confirmed write not visible after unpause')
|
||||
record(dict(cycle=cycle + 1, recovered_seconds=elapsed, existing_session_and_write='PASS'))
|
||||
|
||||
|
||||
def operation(phase):
|
||||
proc = subprocess.Popen(['docker', 'compose', 'exec', '-T', 'api', 'python', '-u', '-c', WORKER, phase],
|
||||
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
text=True, encoding='utf-8', errors='replace')
|
||||
try:
|
||||
check(line(proc) == 'ARMED', 'worker could not open a transaction')
|
||||
try:
|
||||
compose('pause', 'postgres')
|
||||
proc.stdin.write('go\n')
|
||||
proc.stdin.flush()
|
||||
check(line(proc) == 'TIMED_OUT', 'operation did not time out and close connection: ' + phase)
|
||||
finally:
|
||||
compose('unpause', 'postgres')
|
||||
out, err = proc.communicate('resume\n', timeout=25)
|
||||
check(proc.returncode == 0, 'worker failed: ' + err[-1200:])
|
||||
record(json.loads(out))
|
||||
finally:
|
||||
if proc.poll() is None:
|
||||
proc.kill()
|
||||
proc.communicate(timeout=5)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
|
||||
parser.add_argument('--output', type=Path)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
c = Client(args.base_url, timeout=8)
|
||||
ready(c)
|
||||
ids = compose('ps', '-q', 'api').splitlines()
|
||||
check(len(ids) >= 2, 'start at least two API replicas')
|
||||
readiness(c, ids)
|
||||
for phase in ('query', 'commit', 'rollback'):
|
||||
operation(phase)
|
||||
ready(c)
|
||||
record('PASS: paused PostgreSQL returns bounded direct 503; query/commit/rollback discard connections; same pools recover.')
|
||||
return 0
|
||||
except Exception as error:
|
||||
record('FAIL: ' + str(error))
|
||||
return 1
|
||||
finally:
|
||||
if args.output:
|
||||
args.output.write_text('\n'.join(LOG) + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
"""EXPLAIN (ANALYZE, BUFFERS) of the history page query on 10 000+ messages.
|
||||
|
||||
python scripts/explain_history.py --output evidence/explain.txt
|
||||
|
||||
The stand must be running. The script registers two synthetic users through the API,
|
||||
creates a target chat and a noise chat, bulk-inserts messages with SQL, runs ANALYZE and
|
||||
prints the plans of the same query the API runs (src/messenger/db.py, list_messages):
|
||||
first page, a page in the middle, the last page, the generic plan of the parameterized
|
||||
query, and for comparison the middle page without the index (DROP INDEX inside a
|
||||
transaction that is rolled back, so the database is left unchanged).
|
||||
Standard library only.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE = 'http://127.0.0.1:8080'
|
||||
PSQL = ['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
|
||||
'-X', '-v', 'ON_ERROR_STOP=1']
|
||||
PAGE = 51 # limit 50 + 1 row to learn whether the next page exists, as the API does
|
||||
|
||||
lines = []
|
||||
|
||||
|
||||
def out(text=''):
|
||||
lines.append(text)
|
||||
print(text, flush=True)
|
||||
|
||||
|
||||
def psql(sql, echo=True):
|
||||
"""Run SQL in one psql session; with echo the transcript shows every statement before its result."""
|
||||
args = PSQL + (['--echo-all'] if echo else ['-At']) + ['-f', '-']
|
||||
done = subprocess.run(args, cwd=str(ROOT), input=sql.encode('utf-8'), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
text = done.stdout.decode('utf-8', 'replace').strip()
|
||||
if done.returncode != 0:
|
||||
out(text)
|
||||
raise SystemExit('psql failed')
|
||||
return text
|
||||
|
||||
|
||||
def api(method, path, body=None, headers=None):
|
||||
headers = dict(headers or {})
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode('utf-8')
|
||||
headers['Content-Type'] = 'application/json'
|
||||
request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method)
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
return json.loads(response.read())
|
||||
|
||||
|
||||
def user(label):
|
||||
username = label + '_' + uuid.uuid4().hex[:10]
|
||||
password = 'Explain-' + uuid.uuid4().hex
|
||||
created = api('POST', '/api/v1/users', {'username': username, 'display_name': label, 'password': password})
|
||||
credential = base64.b64encode((username + ':' + password).encode()).decode()
|
||||
session = api('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
|
||||
return created['id'], session['token']
|
||||
|
||||
|
||||
def page_sql(chat_id, after=None):
|
||||
"""The same SQL text as Repository.list_messages, with literal values."""
|
||||
sql = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' " % chat_id
|
||||
if after:
|
||||
sql += "AND (created_at, id) > ('%s', '%s') " % after
|
||||
return sql + 'ORDER BY created_at, id LIMIT %d' % PAGE
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='EXPLAIN of the history query; inserts synthetic messages')
|
||||
parser.add_argument('--messages', type=int, default=10000, help='messages in the target chat (>= 10000)')
|
||||
parser.add_argument('--noise', type=int, default=20000, help='messages in another chat')
|
||||
parser.add_argument('--output', default=None)
|
||||
args = parser.parse_args()
|
||||
sys.stdout.reconfigure(errors='replace')
|
||||
|
||||
out('# %s UTC, %s' % (time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()), psql('SELECT version()', echo=False)))
|
||||
alice, alice_token = user('explain_a')
|
||||
bob, _ = user('explain_b')
|
||||
target = api('POST', '/api/v1/chats', {'title': 'EXPLAIN target', 'member_ids': [alice, bob]},
|
||||
{'Authorization': 'Bearer ' + alice_token})['id']
|
||||
noise = api('POST', '/api/v1/chats', {'title': 'EXPLAIN noise', 'member_ids': [alice]},
|
||||
{'Authorization': 'Bearer ' + alice_token})['id']
|
||||
out('# target chat %s: %d messages; noise chat %s: %d messages' % (target, args.messages, noise, args.noise))
|
||||
|
||||
started = time.monotonic()
|
||||
psql('''
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), '%(target)s', CASE WHEN i %% 2 = 0 THEN '%(alice)s'::uuid ELSE '%(bob)s'::uuid END,
|
||||
'seed message ' || i, clock_timestamp() + i * interval '1 millisecond'
|
||||
FROM generate_series(1, %(messages)d) AS i;
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), '%(noise)s', '%(alice)s', 'noise message ' || i, clock_timestamp() + i * interval '1 millisecond'
|
||||
FROM generate_series(1, %(noise_count)d) AS i;
|
||||
ANALYZE messages;
|
||||
''' % {'target': target, 'noise': noise, 'alice': alice, 'bob': bob, 'messages': args.messages, 'noise_count': args.noise},
|
||||
echo=False)
|
||||
out('# inserted and analyzed in %.1f s' % (time.monotonic() - started))
|
||||
out(psql("SELECT 'messages total: ' || count(*) || ', in target chat: ' || count(*) FILTER (WHERE chat_id = '%s') FROM messages" % target, echo=False))
|
||||
|
||||
def key_at(position):
|
||||
row = psql("SELECT created_at || '|' || id FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET %d LIMIT 1"
|
||||
% (target, position - 1), echo=False)
|
||||
return tuple(row.split('|'))
|
||||
|
||||
middle, last = key_at(args.messages // 2), key_at(args.messages - 20)
|
||||
explain = 'EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) '
|
||||
out()
|
||||
out('== 1. First page, middle page and last page: index range scan, no sort ==')
|
||||
out(psql('\n'.join(explain + page_sql(target, after) + ';' for after in (None, middle, last))))
|
||||
out()
|
||||
out('== 2. Generic plan of the parameterized query (how a prepared statement with $1..$4 runs) ==')
|
||||
out(psql('''SET plan_cache_mode = force_generic_plan;
|
||||
PREPARE history_page(uuid, timestamptz, uuid, int) AS
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT $4;
|
||||
%sEXECUTE history_page('%s', '%s', '%s', %d);''' % (explain, target, middle[0], middle[1], PAGE)))
|
||||
out()
|
||||
out('== 3. For comparison: the same middle page without the index (rolled back) ==')
|
||||
out(psql('''BEGIN;
|
||||
DROP INDEX messages_chat_history_idx;
|
||||
%s%s;
|
||||
ROLLBACK;''' % (explain, page_sql(target, middle))))
|
||||
out(psql("SELECT 'index still exists: ' || (to_regclass('messages_chat_history_idx') IS NOT NULL)", echo=False))
|
||||
if args.output:
|
||||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.output).write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local secret files for the compose stand.
|
||||
|
||||
python scripts/init_secrets.py
|
||||
|
||||
Passwords are random and synthetic; they are created only if missing and never printed.
|
||||
The URL/config files are always rebuilt from the password files, so the pair stays consistent.
|
||||
secrets/ is listed in .gitignore and .dockerignore.
|
||||
|
||||
PostgreSQL applies its password only when the data volume is created. If secrets/ is
|
||||
deleted while the pgdata volume is kept, restore the old files or recreate the volume
|
||||
(docker compose down -v deletes ALL lab data).
|
||||
"""
|
||||
from pathlib import Path
|
||||
import json
|
||||
import secrets
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DIR = ROOT / 'secrets'
|
||||
|
||||
|
||||
def write(name, content):
|
||||
path = DIR / name
|
||||
path.write_text(content, encoding='utf-8', newline='\n')
|
||||
# Readable for the non-root users inside containers (files are bind-mounted as is).
|
||||
path.chmod(0o644)
|
||||
|
||||
|
||||
def password(name):
|
||||
path = DIR / name
|
||||
if path.is_file() and path.read_text(encoding='utf-8').strip():
|
||||
print('keep secrets/' + name)
|
||||
return path.read_text(encoding='utf-8').strip()
|
||||
value = secrets.token_hex(24) # hex: safe inside URLs without escaping
|
||||
write(name, value + '\n')
|
||||
print('created secrets/' + name)
|
||||
return value
|
||||
|
||||
|
||||
def main():
|
||||
DIR.mkdir(exist_ok=True)
|
||||
postgres = password('postgres_password')
|
||||
valkey = password('valkey_password')
|
||||
broker = password('broker_password')
|
||||
write('database_url', 'postgresql://messenger:%s@postgres:5432/messenger\n' % postgres)
|
||||
write('valkey_auth', 'requirepass %s\n' % valkey)
|
||||
write('cache_url', 'valkey://:%s@valkey:6379/0\n' % valkey)
|
||||
write('broker_auth', 'requirepass %s\n' % broker)
|
||||
write('broker_url', 'valkey://:%s@broker:6379/0\n' % broker)
|
||||
admin_key = password('s3_admin_access_key_id')
|
||||
admin_secret = password('s3_admin_secret_access_key')
|
||||
api_key = password('s3_access_key_id')
|
||||
api_secret = password('s3_secret_access_key')
|
||||
worker_key = password('s3_worker_access_key_id')
|
||||
worker_secret = password('s3_worker_secret_access_key')
|
||||
identities = [
|
||||
dict(name='bootstrap', credentials=[dict(accessKey=admin_key, secretKey=admin_secret)],
|
||||
actions=['Admin', 'Read', 'Write', 'List', 'Tagging']),
|
||||
dict(name='messenger-api', credentials=[dict(accessKey=api_key, secretKey=api_secret)],
|
||||
actions=['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments']),
|
||||
dict(name='messenger-worker', credentials=[dict(accessKey=worker_key, secretKey=worker_secret)],
|
||||
actions=['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'])]
|
||||
# Нет anonymous identity: ни объект, ни список bucket не доступны анонимно.
|
||||
write('s3_config', json.dumps(dict(identities=identities)) + '\n')
|
||||
print('written secrets/database_url, secrets/valkey_auth, secrets/cache_url')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,487 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Repeatable grade 3/4 failure checks for the local synthetic lab5 Compose stand.
|
||||
|
||||
Run separately from other tests: scenarios deliberately restart/stop services.
|
||||
Every scenario restores the original services in finally. No volumes, secrets or
|
||||
application records are removed. Tokens live only in process memory.
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check
|
||||
|
||||
|
||||
def command(*args):
|
||||
result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True,
|
||||
encoding='utf-8', errors='replace')
|
||||
if result.returncode:
|
||||
# Never print environment, response bodies, credentials or arbitrary Docker logs.
|
||||
raise Failure('command failed: ' + ' '.join(args[:4]) + ' (exit %d)' % result.returncode)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return command('docker', 'compose', *args)
|
||||
|
||||
|
||||
def sql(query):
|
||||
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', query)
|
||||
|
||||
|
||||
def rows(query):
|
||||
value = sql("SELECT COALESCE(json_agg(t), '[]'::json) FROM (%s) t" % query)
|
||||
return json.loads(value)
|
||||
|
||||
|
||||
def identifier(value):
|
||||
return str(uuid.UUID(value))
|
||||
|
||||
|
||||
def job(attachment_id):
|
||||
data = rows("SELECT * FROM attachment_jobs WHERE attachment_id='%s'" % identifier(attachment_id))
|
||||
check(len(data) == 1, 'attachment must have exactly one logical job')
|
||||
return data[0]
|
||||
|
||||
|
||||
def audit(job_id):
|
||||
return rows("SELECT action,actor,reason,details,created_at FROM job_audit WHERE job_id='%s' ORDER BY id"
|
||||
% identifier(job_id))
|
||||
|
||||
|
||||
def cli(*args):
|
||||
# The explicit operator service has broker/worker permissions. The API does
|
||||
# not receive broker credentials merely to make an evidence script easier.
|
||||
output = compose('--profile', 'tools', 'run', '--rm', '--no-deps', 'jobs', *args)
|
||||
return json.loads(output.splitlines()[-1]) if output else None
|
||||
|
||||
|
||||
def inject(job_id, action, count=1):
|
||||
return cli('inject', identifier(job_id), '--action', action, '--count', str(count),
|
||||
'--actor', 'lab5-evidence', '--reason', 'repeatable synthetic grade 4 failure check')
|
||||
|
||||
|
||||
def wait_until(predicate, description, seconds=180):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
value = predicate()
|
||||
if value:
|
||||
return value
|
||||
time.sleep(0.4)
|
||||
raise Failure('deadline exceeded: ' + description)
|
||||
|
||||
|
||||
def ready(client):
|
||||
def probe():
|
||||
try:
|
||||
client.call('GET', '/health/ready', schema='Health')
|
||||
return True
|
||||
except Failure:
|
||||
return False
|
||||
wait_until(probe, 'API ready', 90)
|
||||
|
||||
|
||||
def restore(client, *services):
|
||||
# start preserves stopped container configuration and all named volumes.
|
||||
compose('start', *services)
|
||||
ready(client)
|
||||
|
||||
|
||||
def upload(client, actor, chat, label):
|
||||
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
result = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + label + '.png',
|
||||
202, actor=actor, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
||||
attachment = result['json']
|
||||
check(attachment['status'] == 'queued', '202 must carry a queued snapshot')
|
||||
check(attachment['variants'] == [] and attachment['error_code'] is None, 'queued result fields')
|
||||
check(attachment['sha256'] == hashlib.sha256(content).hexdigest(), 'accepted original checksum')
|
||||
return attachment, content
|
||||
|
||||
|
||||
def metadata(client, actor, attachment):
|
||||
return client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=actor, schema='Attachment')['json']
|
||||
|
||||
|
||||
def wait_state(client, actor, attachment, status):
|
||||
def probe():
|
||||
current = metadata(client, actor, attachment)
|
||||
if current['status'] == status:
|
||||
return current
|
||||
check(current['status'] != 'failed' or status == 'failed', 'unexpected terminal failed state')
|
||||
return wait_until(probe, 'attachment ' + status)
|
||||
|
||||
|
||||
def verified_ready(client, actor, attachment, content):
|
||||
result = wait_state(client, actor, attachment, 'ready')
|
||||
check(result['id'] == attachment['id'], 'logical attachment identity changed')
|
||||
check(result['error_code'] is None, 'ready error_code must be null')
|
||||
check({item['name'] for item in result['variants']} == {'thumbnail', 'crop', 'optimized'}, 'grade 4 variants')
|
||||
check(len(result['variants']) == 3, 'duplicate result variants')
|
||||
for name in ['original', 'thumbnail', 'crop', 'optimized']:
|
||||
data = client.call('GET', '/api/v1/attachments/' + result['id'] + '/content?variant=' + name, actor=actor)
|
||||
record = result if name == 'original' else next(v for v in result['variants'] if v['name'] == name)
|
||||
check(len(data['bytes']) == record['size_bytes'], 'published object size mismatch')
|
||||
check(hashlib.sha256(data['bytes']).hexdigest() == record['sha256'], 'published object checksum mismatch')
|
||||
if name == 'original':
|
||||
check(data['bytes'] == content, 'original bytes changed')
|
||||
count = int(sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % result['id']))
|
||||
check(count == 3, 'exactly three persistent variant records required')
|
||||
return result
|
||||
|
||||
|
||||
def object_state(attachment_id):
|
||||
# Runs with the API's restricted bucket credentials; only selected facts return.
|
||||
aid = identifier(attachment_id)
|
||||
code = """import json
|
||||
from messenger.config import read_secret
|
||||
from messenger.db import create_pool
|
||||
from messenger.objects import ObjectStore
|
||||
from botocore.exceptions import ClientError
|
||||
p=create_pool(read_secret('DATABASE_URL'),1);p.open(wait=True)
|
||||
with p.connection() as c:
|
||||
r=c.execute('SELECT object_key,staging_key,staging_deleted_at,status,pipeline_version FROM attachments WHERE id=%%s',(%r,)).fetchone()
|
||||
s=ObjectStore()
|
||||
def exists(key):
|
||||
try: s.client.head_object(Bucket=s.bucket,Key=key);return True
|
||||
except ClientError as e:
|
||||
if e.response['ResponseMetadata']['HTTPStatusCode']==404:return False
|
||||
raise
|
||||
prefix='variants/'+%r.replace('-','')+'/'
|
||||
objects=s.client.list_objects_v2(Bucket=s.bucket,Prefix=prefix).get('Contents',[])
|
||||
print(json.dumps(dict(final=exists(r['object_key']),staging=exists(r['staging_key']),status=r['status'],staging_deleted=r['staging_deleted_at'] is not None,variant_objects=len(objects))))
|
||||
p.close()
|
||||
""" % (aid, aid)
|
||||
return json.loads(compose('exec', '-T', 'api', 'python', '-c', code).splitlines()[-1])
|
||||
|
||||
|
||||
def effect_snapshot(attachment_id):
|
||||
aid = identifier(attachment_id)
|
||||
return rows("SELECT a.status,a.object_key,a.sha256,a.updated_at,j.id AS job_id,j.attempt,j.updated_at AS job_updated_at,"
|
||||
"(SELECT json_agg(v ORDER BY v.name) FROM attachment_variants v WHERE v.attachment_id=a.id) AS variants "
|
||||
"FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE a.id='%s'" % aid)[0]
|
||||
|
||||
|
||||
def fault_consumed(job_id, action):
|
||||
return bool(rows("SELECT 1 AS consumed FROM job_faults WHERE job_id='%s' AND action='%s' AND remaining=0"
|
||||
% (identifier(job_id), action)))
|
||||
|
||||
|
||||
def stale_lease_fenced(attachment_id, old_job):
|
||||
"""A dead invocation cannot commit success or failure over the recovered result."""
|
||||
old_claim = dict(job_id=old_job['id'], lease_token=old_job['lease_token'],
|
||||
attempt=old_job['attempt'], pipeline_version=old_job['pipeline_version'])
|
||||
code = """import json
|
||||
from messenger.config import read_secret
|
||||
from messenger.db import create_pool
|
||||
from messenger.jobs import Jobs
|
||||
p=create_pool(read_secret('DATABASE_URL'),1);p.open(wait=True);j=Jobs(p)
|
||||
claim=json.loads(%r)
|
||||
with p.connection() as c:
|
||||
a=c.execute('SELECT object_key FROM attachments WHERE id=%%s',(%r,)).fetchone()
|
||||
variants=c.execute('SELECT * FROM attachment_variants WHERE attachment_id=%%s ORDER BY name',(%r,)).fetchall()
|
||||
assert j.finish(claim,a['object_key'],variants) is False,'old lease committed success'
|
||||
assert j.fail(claim,'stale_worker',False,5,0) is False,'old lease committed failure'
|
||||
p.close();print(json.dumps(dict(old_finish=False,old_fail=False)))
|
||||
""" % (json.dumps(old_claim), identifier(attachment_id), identifier(attachment_id))
|
||||
return json.loads(compose('exec', '-T', 'dispatcher', 'python', '-c', code).splitlines()[-1])
|
||||
|
||||
|
||||
def stack(client):
|
||||
config = json.loads(compose('config', '--format', 'json'))
|
||||
services = config['services']
|
||||
for name in ('api', 'worker', 'dispatcher', 'postgres', 'valkey', 'broker', 's3'):
|
||||
check(name in services, 'missing independent service ' + name)
|
||||
check(not services[name].get('ports'), name + ': internal ports must not be published')
|
||||
check(services['api']['deploy']['replicas'] >= 2, 'two API replicas required')
|
||||
check(any(v.get('type') == 'volume' for v in services['broker'].get('volumes', [])), 'broker needs persistent volume')
|
||||
check(services['worker'].get('deploy', {}).get('resources', {}).get('limits', {}).get('memory'),
|
||||
'worker memory limit required')
|
||||
print('PASS stack: separate durable broker/worker/dispatcher, private internal ports, bounded worker memory', flush=True)
|
||||
|
||||
|
||||
def durability(client):
|
||||
actor, member, _, chat = client.bootstrap()
|
||||
try:
|
||||
compose('stop', 'worker')
|
||||
attachment, content = upload(client, actor, chat, 'worker-down')
|
||||
logical_job = job(attachment['id'])['id']
|
||||
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=member)
|
||||
message = client.message(member, chat, '', attachments=[attachment['id']])['json']
|
||||
check(message['attachment_ids'] == [attachment['id']], 'queued attachment cannot be linked')
|
||||
check(object_state(attachment['id'])['staging'], '202 lost durable staging')
|
||||
wait_until(lambda: bool(rows("SELECT 1 FROM attachment_outbox WHERE job_id='%s' AND published_at IS NOT NULL"
|
||||
% logical_job)), 'broker publication before restart')
|
||||
compose('restart', 'api', 'broker')
|
||||
ready(client)
|
||||
check(metadata(client, member, attachment)['status'] == 'queued', 'worker-down status must remain queued')
|
||||
history = client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=member, schema='MessagePage')['json']
|
||||
check(any(item['id'] == message['id'] for item in history['items']), 'linked message lost across restart')
|
||||
compose('start', 'worker')
|
||||
verified_ready(client, member, attachment, content)
|
||||
check(job(attachment['id'])['id'] == logical_job, 'restart created a replacement logical job')
|
||||
print('PASS durability: worker down -> 202+linked message+409 -> API/broker restart with volumes -> same job ready; job_id=' + logical_job, flush=True)
|
||||
finally:
|
||||
restore(client, 'api', 'broker', 'worker')
|
||||
|
||||
|
||||
def outbox(client):
|
||||
actor, member, _, chat = client.bootstrap()
|
||||
try:
|
||||
compose('stop', 'dispatcher', 'worker')
|
||||
attachment, content = upload(client, actor, chat, 'commit-publish-gap')
|
||||
logical_job = job(attachment['id'])['id']
|
||||
pending = rows("SELECT attempt,published_at FROM attachment_outbox WHERE job_id='%s'" % logical_job)
|
||||
check(len(pending) == 1 and pending[0]['published_at'] is None, 'COMMIT/publish gap was not isolated')
|
||||
# API is killed after its successful COMMIT while publisher is absent.
|
||||
compose('kill', '-s', 'SIGKILL', 'api')
|
||||
compose('start', 'api')
|
||||
ready(client)
|
||||
check(job(attachment['id'])['id'] == logical_job, 'COMMIT intent lost after API crash')
|
||||
compose('start', 'dispatcher', 'worker')
|
||||
verified_ready(client, member, attachment, content)
|
||||
check(job(attachment['id'])['id'] == logical_job, 'outbox recovery recreated the job')
|
||||
print('PASS outbox: COMMIT persisted while dispatcher stopped -> kill API before publish -> restart -> same job ready; job_id=' + logical_job, flush=True)
|
||||
finally:
|
||||
restore(client, 'api', 'dispatcher', 'worker')
|
||||
|
||||
|
||||
def broker_outage(client):
|
||||
actor, member, _, chat = client.bootstrap()
|
||||
try:
|
||||
compose('stop', 'broker', 'worker')
|
||||
ready(client)
|
||||
attachment, content = upload(client, actor, chat, 'broker-down')
|
||||
logical_job = job(attachment['id'])['id']
|
||||
client.message(member, chat, 'broker outage does not stop messages', attachments=[attachment['id']])
|
||||
check(bool(rows("SELECT 1 FROM attachment_outbox WHERE job_id='%s' AND published_at IS NULL" % logical_job)),
|
||||
'broker-down upload lacks durable outbox intent')
|
||||
compose('start', 'broker', 'worker')
|
||||
verified_ready(client, member, attachment, content)
|
||||
print('PASS broker outage: API remains ready, upload 202 and messages commit, durable outbox later reaches ready; job_id=' + logical_job, flush=True)
|
||||
finally:
|
||||
restore(client, 'broker', 'dispatcher', 'worker')
|
||||
|
||||
|
||||
def wait_delivery_ack(message_id):
|
||||
code = """import json
|
||||
from messenger.broker import Broker
|
||||
b=Broker();target=tuple(map(int,%r.split('-')))
|
||||
groups=b.client.xinfo_groups(b.stream)
|
||||
group=next((g for g in groups if g['name']==b.group),None)
|
||||
delivered=bool(group and tuple(map(int,group['last-delivered-id'].split('-')))>=target)
|
||||
pending=b.client.xpending_range(b.stream,b.group,%r,%r,1)
|
||||
print(json.dumps(dict(done=delivered and not pending)));b.close()
|
||||
""" % (message_id, message_id, message_id)
|
||||
wait_until(lambda: json.loads(compose('exec', '-T', 'dispatcher', 'python', '-c', code))['done'], 'redelivery ack')
|
||||
|
||||
|
||||
def redelivery(client):
|
||||
actor, member, _, chat = client.bootstrap()
|
||||
attachment, content = upload(client, actor, chat, 'redelivery')
|
||||
completed = verified_ready(client, member, attachment, content)
|
||||
before = effect_snapshot(attachment['id'])
|
||||
published = cli('redeliver', before['job_id'])
|
||||
message_id = published['message_id']
|
||||
wait_delivery_ack(message_id)
|
||||
after = effect_snapshot(attachment['id'])
|
||||
check(before == after, 'ready duplicate mutated persistent result, job attempt or timestamps')
|
||||
check(metadata(client, member, attachment) == completed, 'ready duplicate changed public metadata')
|
||||
print('PASS redelivery: same ready job acked without mutation, same 3 variants/checksums; job_id=' + before['job_id'], flush=True)
|
||||
|
||||
|
||||
def crash(client):
|
||||
actor, member, _, chat = client.bootstrap()
|
||||
try:
|
||||
compose('stop', 'worker')
|
||||
attachment, content = upload(client, actor, chat, 'crash-after-objects')
|
||||
logical_job = job(attachment['id'])['id']
|
||||
inject(logical_job, 'crash_after_objects')
|
||||
compose('start', 'worker')
|
||||
wait_until(lambda: fault_consumed(logical_job, 'crash_after_objects'), 'crash fault consumed')
|
||||
at_crash = object_state(attachment['id'])
|
||||
check(at_crash['final'] and at_crash['variant_objects'] == 3,
|
||||
'injected crash occurred before all four final objects existed')
|
||||
check(at_crash['status'] == 'processing' and at_crash['staging'],
|
||||
'crash window must precede ready commit and retain staging')
|
||||
check(not rows("SELECT 1 FROM attachment_variants WHERE attachment_id='%s'" % attachment['id']),
|
||||
'crash window must precede publication of variant metadata')
|
||||
# The durable hook is consumed before exit, so container restart cannot loop it.
|
||||
compose('start', 'worker')
|
||||
verified_ready(client, member, attachment, content)
|
||||
result = job(attachment['id'])
|
||||
check(result['id'] == logical_job and result['attempt'] >= 2, 'crashed job did not recover its expired lease')
|
||||
check(object_state(attachment['id'])['variant_objects'] == 3, 'redelivery duplicated S3 variants')
|
||||
print('PASS crash after 4 S3 writes before DB result/ack: same job recovered; no duplicate variants; job_id=%s attempts=%s window=%s'
|
||||
% (logical_job, result['attempt'], json.dumps(at_crash, sort_keys=True)), flush=True)
|
||||
finally:
|
||||
restore(client, 'worker')
|
||||
|
||||
|
||||
def lease(client):
|
||||
actor, member, _, chat = client.bootstrap()
|
||||
try:
|
||||
compose('stop', 'worker')
|
||||
attachment, content = upload(client, actor, chat, 'expired-processing')
|
||||
logical_job = job(attachment['id'])['id']
|
||||
inject(logical_job, 'stall_processing')
|
||||
compose('start', 'worker')
|
||||
wait_until(lambda: fault_consumed(logical_job, 'stall_processing'), 'processing stall hook')
|
||||
held = job(attachment['id'])
|
||||
check(held['status'] == 'processing' and held['lease_until'], 'stall did not hold a processing lease')
|
||||
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=member)
|
||||
compose('kill', '-s', 'SIGKILL', 'worker')
|
||||
compose('start', 'worker')
|
||||
verified_ready(client, member, attachment, content)
|
||||
recovered = job(attachment['id'])
|
||||
check(recovered['id'] == logical_job and recovered['attempt'] > held['attempt'], 'expired processing was not reclaimed')
|
||||
snapshot = effect_snapshot(attachment['id'])
|
||||
stale_lease_fenced(attachment['id'], held)
|
||||
check(effect_snapshot(attachment['id']) == snapshot, 'stale lease changed the recovered ready result')
|
||||
print('PASS expired lease: killed stalled processing worker -> automatic recovery, same job; job_id=%s old_lease_until=%s attempts=%d'
|
||||
% (logical_job, held['lease_until'], recovered['attempt']), flush=True)
|
||||
print('PASS fencing: old lease cannot finish or fail after recovered ready commit', flush=True)
|
||||
finally:
|
||||
restore(client, 'worker')
|
||||
|
||||
|
||||
def failed_contract(client, actor, chat, attachment):
|
||||
failed = wait_state(client, actor, attachment, 'failed')
|
||||
check(bool(re.fullmatch(r'[a-z][a-z0-9_]{0,63}', failed['error_code'] or '')), 'safe terminal error code required')
|
||||
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=actor)
|
||||
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=actor,
|
||||
body={'text': '', 'attachment_ids': [attachment['id']]}, headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
return failed
|
||||
|
||||
|
||||
def retries(client):
|
||||
actor, member, _, chat = client.bootstrap()
|
||||
config = json.loads(compose('config', '--format', 'json'))['services']['worker'].get('environment', {})
|
||||
max_attempts = int(config.get('JOB_MAX_ATTEMPTS', 5))
|
||||
retry_base = float(config.get('JOB_RETRY_BASE_SECONDS', 1))
|
||||
retry_max = float(config.get('JOB_RETRY_MAX_SECONDS', 30))
|
||||
check(max_attempts >= 3, 'retry demonstration requires a budget of at least three attempts')
|
||||
try:
|
||||
compose('stop', 'worker')
|
||||
attachment, content = upload(client, actor, chat, 'transient-retry')
|
||||
logical_job = job(attachment['id'])['id']
|
||||
inject(logical_job, 'transient', 2)
|
||||
compose('start', 'worker')
|
||||
verified_ready(client, member, attachment, content)
|
||||
retried = job(attachment['id'])
|
||||
check(retried['attempt'] >= 2, 'transient failure was not retried')
|
||||
events = audit(logical_job)
|
||||
retry_events = [event for event in events if event['action'] == 'retry' and event['reason'] == 'injected_transient']
|
||||
check(len(retry_events) == 2, 'both retry delays must be observable in durable audit')
|
||||
randomized = False
|
||||
for event in retry_events:
|
||||
details = event['details']
|
||||
cap = min(retry_max, retry_base * 2.0 ** (details['run_attempts'] - 1))
|
||||
check(cap / 2 <= details['delay_seconds'] <= cap, 'retry is outside its equal-jitter exponential interval')
|
||||
randomized |= details['delay_seconds'] not in (cap / 2, cap)
|
||||
created = dt.datetime.fromisoformat(event['created_at'].replace('Z', '+00:00'))
|
||||
available = dt.datetime.fromisoformat(details['available_at'].replace('Z', '+00:00'))
|
||||
check(abs((available - created).total_seconds() - details['delay_seconds']) < 0.25,
|
||||
'retry due time does not persist the chosen jitter delay')
|
||||
check(randomized, 'retry evidence contains no randomized delay')
|
||||
print('PASS transient errors retry with durable backoff/jitter; job_id=%s audit=%s'
|
||||
% (logical_job, json.dumps(retry_events, sort_keys=True)), flush=True)
|
||||
|
||||
compose('stop', 'worker')
|
||||
attachment, content = upload(client, actor, chat, 'retry-exhausted')
|
||||
logical_job = job(attachment['id'])['id']
|
||||
inject(logical_job, 'transient', max_attempts)
|
||||
compose('start', 'worker')
|
||||
failed = failed_contract(client, member, chat, attachment)
|
||||
check(failed['error_code'] == 'retry_exhausted', 'exhausted job needs a distinct terminal error code')
|
||||
exhausted = job(attachment['id'])
|
||||
check(exhausted['run_attempts'] == max_attempts, 'retry budget not enforced')
|
||||
check(fault_consumed(logical_job, 'transient'), 'retry fault count does not match exhausted attempts')
|
||||
failed_jobs = cli('failed', '--limit', '1000')
|
||||
check(any(item['job_id'] == logical_job for item in failed_jobs), 'failed job missing from operator registry')
|
||||
check(object_state(attachment['id'])['staging'], 'failed job lost original needed for replay')
|
||||
cli('cleanup')
|
||||
check(object_state(attachment['id'])['staging'], 'cleanup removed failed input required for replay')
|
||||
replay = cli('replay', attachment['id'], '--actor', 'lab5-evidence', '--reason', 'operator resolved synthetic failure')
|
||||
check(replay['job_id'] == logical_job, 'operator replay created a different logical job')
|
||||
verified_ready(client, member, attachment, content)
|
||||
before = effect_snapshot(attachment['id'])
|
||||
noop = cli('replay', attachment['id'], '--actor', 'lab5-evidence', '--reason', 'verify successful replay is a no-op')
|
||||
check(effect_snapshot(attachment['id']) == before, 'replay re-executed a successful job')
|
||||
actions = {event['action'] for event in audit(logical_job) if event['actor'] == 'lab5-evidence'}
|
||||
check({'replay', 'replay_noop'} <= actions, 'explicit replay and successful no-op need operator audit records')
|
||||
print('PASS retry exhaustion -> failed+409+retained staging -> explicit audited replay -> ready -> ready replay no-op; job_id=%s max_attempts=%d error_code=%s replay=%s'
|
||||
% (logical_job, max_attempts, failed['error_code'], json.dumps(noop, sort_keys=True)), flush=True)
|
||||
finally:
|
||||
restore(client, 'worker')
|
||||
|
||||
|
||||
def permanent(client):
|
||||
actor, member, _, chat = client.bootstrap()
|
||||
try:
|
||||
compose('stop', 'worker')
|
||||
attachment, _ = upload(client, actor, chat, 'permanent-failure')
|
||||
logical_job = job(attachment['id'])['id']
|
||||
inject(logical_job, 'permanent')
|
||||
compose('start', 'worker')
|
||||
failed = failed_contract(client, member, chat, attachment)
|
||||
check(failed['error_code'] == 'invalid_image', 'permanent errors need a distinct safe error code')
|
||||
check(job(attachment['id'])['run_attempts'] == 1, 'permanent errors must not consume all retries')
|
||||
print('PASS permanent failure: failed after one attempt; content/message rejected with409; job_id=%s error_code=%s'
|
||||
% (logical_job, failed['error_code']), flush=True)
|
||||
finally:
|
||||
restore(client, 'worker')
|
||||
|
||||
|
||||
def staging(client):
|
||||
actor, member, _, chat = client.bootstrap()
|
||||
try:
|
||||
compose('stop', 'worker')
|
||||
attachment, content = upload(client, actor, chat, 'staging-safety')
|
||||
before = object_state(attachment['id'])
|
||||
check(before['staging'] and not before['final'], 'staging/final publication boundary is not visible')
|
||||
cli('cleanup')
|
||||
check(object_state(attachment['id']) == before, 'cleanup removed current queued staging')
|
||||
compose('start', 'worker')
|
||||
verified_ready(client, member, attachment, content)
|
||||
cli('cleanup')
|
||||
after = wait_until(lambda: (lambda s: s if not s['staging'] and s['staging_deleted'] else None)
|
||||
(object_state(attachment['id'])), 'post-commit staging cleanup')
|
||||
check(after['final'] and after['status'] == 'ready', 'cleanup removed committed original')
|
||||
print('PASS staging cleanup: queued protected; staging deleted only after ready commit; final original retained; attachment_id=' + attachment['id'], flush=True)
|
||||
finally:
|
||||
restore(client, 'worker')
|
||||
|
||||
|
||||
SCENARIOS = ['stack', 'durability', 'outbox', 'broker_outage', 'redelivery', 'crash', 'lease', 'retries', 'permanent', 'staging']
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://127.0.0.1:8080'))
|
||||
parser.add_argument('--only', nargs='+', choices=SCENARIOS)
|
||||
args = parser.parse_args()
|
||||
client = Client(args.base_url, timeout=15)
|
||||
ready(client)
|
||||
for name in args.only or SCENARIOS:
|
||||
ready(client)
|
||||
print('START ' + name + ' ' + dt.datetime.now(dt.timezone.utc).isoformat(), flush=True)
|
||||
globals()[name](client)
|
||||
print('PASS selected lab5 grade 3/4 checks; no grade 5 track claimed', flush=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
main()
|
||||
except (Failure, ValueError, OSError) as exc:
|
||||
print('FAIL: ' + str(exc), flush=True)
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,10 @@
|
||||
@echo off
|
||||
rem Windows twin of scripts/restart.sh: tests/resilience.py starts the hook directly,
|
||||
rem and Windows cannot execute a .sh file. Same actions, volumes are kept.
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
docker compose stop api worker dispatcher || exit /b 1
|
||||
docker compose restart postgres valkey s3 broker || exit /b 1
|
||||
docker compose start api dispatcher worker || exit /b 1
|
||||
docker compose restart lb || exit /b 1
|
||||
exit /b 0
|
||||
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
# Hook for: make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
# Restarts the whole stand: all API replicas, PostgreSQL, Valkey and the load balancer,
|
||||
# keeping the named volumes: no "down -v", no FLUSHALL, no re-seed.
|
||||
# The test itself waits for API readiness through the load balancer afterwards.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop api worker dispatcher
|
||||
docker compose restart postgres valkey s3 broker
|
||||
docker compose start api dispatcher worker
|
||||
docker compose restart lb
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sanitized runtime snapshot: real processes, start times, limits and volumes; no environment/secrets."""
|
||||
import datetime as dt
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def command(*args):
|
||||
return subprocess.check_output(list(args), cwd=ROOT, text=True, encoding='utf-8')
|
||||
|
||||
|
||||
def main():
|
||||
ids = command('docker', 'compose', 'ps', '-a', '-q').split()
|
||||
containers = json.loads(command('docker', 'inspect', *ids))
|
||||
result = {'captured_at': dt.datetime.now(dt.timezone.utc).isoformat(), 'containers': []}
|
||||
for item in containers:
|
||||
# Only these selected fields are persisted; Config.Env is deliberately excluded.
|
||||
result['containers'].append(dict(
|
||||
name=item['Name'].lstrip('/'), image=item['Config']['Image'],
|
||||
status=item['State']['Status'], started_at=item['State']['StartedAt'],
|
||||
health=item['State'].get('Health', {}).get('Status'),
|
||||
memory_limit_bytes=item['HostConfig']['Memory'], cpu_limit_nanos=item['HostConfig']['NanoCpus'],
|
||||
volumes=[dict(name=m.get('Name'), destination=m['Destination'])
|
||||
for m in item['Mounts'] if m['Type'] == 'volume'],
|
||||
published_ports=item['NetworkSettings']['Ports']))
|
||||
text = json.dumps(result, indent=2)
|
||||
(ROOT / 'evidence/stack.json').write_text(text + '\n', encoding='utf-8')
|
||||
print('Saved evidence/stack.json (process start times, versions, limits, volumes and ports)')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,19 @@
|
||||
@echo off
|
||||
rem Windows twin of scripts/stop-one.sh: tests/resilience.py starts the hook directly,
|
||||
rem and Windows cannot execute a .sh file. Stops ONE api replica by X-Instance-Id (container ID).
|
||||
setlocal
|
||||
if "%TARGET_INSTANCE_ID%"=="" (
|
||||
echo TARGET_INSTANCE_ID is not set 1>&2
|
||||
exit /b 2
|
||||
)
|
||||
cd /d "%~dp0.."
|
||||
set PROJECT=%COMPOSE_PROJECT_NAME%
|
||||
if "%PROJECT%"=="" set PROJECT=mtusi-lab5
|
||||
set ID=
|
||||
for /f %%i in ('docker ps -q --filter "id=%TARGET_INSTANCE_ID%" --filter "label=com.docker.compose.project=%PROJECT%" --filter "label=com.docker.compose.service=api"') do set ID=%%i
|
||||
if "%ID%"=="" (
|
||||
echo TARGET_INSTANCE_ID is not a running api replica of %PROJECT% 1>&2
|
||||
exit /b 1
|
||||
)
|
||||
docker stop %ID% >nul || exit /b 1
|
||||
exit /b 0
|
||||
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
# Hook for: make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
# Stops exactly ONE API replica and leaves it stopped. TARGET_INSTANCE_ID is the X-Instance-Id
|
||||
# of the replica, which is its container hostname = short container ID. The hook checks that
|
||||
# the ID belongs to a running "api" container of this compose project before stopping it.
|
||||
# docker stop sends SIGTERM: the replica reports not ready, HAProxy removes it, then it exits.
|
||||
# Restore afterwards: docker compose up -d (or docker start <id>)
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
cd "$(dirname "$0")/.."
|
||||
project="${COMPOSE_PROJECT_NAME:-mtusi-lab5}"
|
||||
id=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \
|
||||
--filter "label=com.docker.compose.project=$project" \
|
||||
--filter "label=com.docker.compose.service=api")
|
||||
if [ -z "$id" ]; then
|
||||
echo "TARGET_INSTANCE_ID is not a running api replica of $project" >&2
|
||||
exit 1
|
||||
fi
|
||||
docker stop "$id" >/dev/null
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run reproducible acceptance checks against an already running local lab5 stand.
|
||||
|
||||
The default checks do not stop services. --faults explicitly includes the grade 4
|
||||
fault suite; each fault restores services without deleting volumes or secrets.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure
|
||||
|
||||
|
||||
def wait_ready(url):
|
||||
client = Client(url)
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
client.call('GET', '/health/ready', schema='Health')
|
||||
return
|
||||
except Failure:
|
||||
time.sleep(0.5)
|
||||
raise Failure('API did not become ready before verification block')
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://127.0.0.1:8080'))
|
||||
parser.add_argument('--faults', action='store_true', help='Also run disruptive grade 4 fault checks')
|
||||
parser.add_argument('--only', nargs='+', choices=['offline', 'unit', 'smoke', 'tests', 'lab5-checks',
|
||||
'persistence', 'failover', 'postgres-pause', 'cleanup', 'stack'])
|
||||
parser.add_argument('--output-prefix', default='', help='Keep a separate set of UTF-8 evidence logs')
|
||||
options = parser.parse_args()
|
||||
if Path(options.output_prefix).name != options.output_prefix or any(c in options.output_prefix for c in '/\\'):
|
||||
parser.error('output prefix must be a filename prefix')
|
||||
checks = [
|
||||
('offline', ['scripts/check_contract.py']),
|
||||
('unit', ['docker', 'compose', 'run', '--rm', '--no-deps', '--entrypoint', 'python',
|
||||
'-v', './unit-tests:/app/unit-tests:ro', 'worker', '-m', 'unittest', 'discover', '-s', 'unit-tests', '-v']),
|
||||
('smoke', ['tests/smoke.py', '--base-url', options.base_url]),
|
||||
('tests', ['-m', 'unittest', 'discover', '-s', 'student-tests', '-v']),
|
||||
('cleanup', ['docker', 'compose', '--profile', 'tools', 'run', '--rm', '--no-deps', 'cleanup']),
|
||||
('stack', ['scripts/stack_evidence.py']),
|
||||
]
|
||||
if options.faults or (options.only and 'lab5-checks' in options.only):
|
||||
checks.insert(4, ('lab5-checks', ['scripts/lab5_checks.py', '--base-url', options.base_url]))
|
||||
for name, mode, hook in (
|
||||
('persistence', 'persistence', 'restart'), ('failover', 'failover', 'stop-one')
|
||||
):
|
||||
if options.faults or (options.only and name in options.only):
|
||||
script = 'scripts/%s.%s' % (hook, 'cmd' if os.name == 'nt' else 'sh')
|
||||
checks.insert(-1, (name, ['tests/resilience.py', '--mode', mode, '--action-script', script,
|
||||
'--base-url', options.base_url]))
|
||||
if options.faults or (options.only and 'postgres-pause' in options.only):
|
||||
checks.insert(-1, ('postgres-pause', ['scripts/check_postgres_pause.py', '--base-url', options.base_url]))
|
||||
(ROOT / 'evidence').mkdir(exist_ok=True)
|
||||
for name, args in checks:
|
||||
if options.only and name not in options.only:
|
||||
continue
|
||||
if name != 'offline':
|
||||
wait_ready(options.base_url)
|
||||
invocation = args if name in ('unit', 'cleanup') else [sys.executable, '-X', 'utf8', *args]
|
||||
result = subprocess.run(invocation, cwd=ROOT,
|
||||
env=dict(os.environ, BASE_URL=options.base_url), capture_output=True)
|
||||
if name == 'failover':
|
||||
restored = subprocess.run(['docker', 'compose', 'up', '-d', '--wait'], cwd=ROOT,
|
||||
capture_output=True)
|
||||
if restored.returncode:
|
||||
print('Failed to restore API replica after failover', flush=True)
|
||||
return restored.returncode
|
||||
content = result.stdout.decode('utf-8', errors='replace') + result.stderr.decode('utf-8', errors='replace')
|
||||
(ROOT / 'evidence' / (options.output_prefix + name + '.txt')).write_text(content, encoding='utf-8')
|
||||
print(name + ': ' + ('PASS' if result.returncode == 0 else 'FAIL'), flush=True)
|
||||
if result.returncode:
|
||||
print(content[-6000:], flush=True)
|
||||
return result.returncode
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user