ЛР 5 — БВТ2304 — Черников — на 4

This commit is contained in:
2026-11-11 23:56:28 +03:00
parent 276dbdad8a
commit 0b399073d6
77 changed files with 6838 additions and 27 deletions
+208
View File
@@ -0,0 +1,208 @@
#!/usr/bin/env python3
"""Regression: pause PostgreSQL, direct API 503, discard connection and recover.
No published replica ports: direct HTTP is sent from inside each API container.
PostgreSQL is always unpaused in finally. Uses existing local synthetic stand.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import queue
import subprocess
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check
LOG = []
def record(value):
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False)
LOG.append(text)
print(text, flush=True)
def compose(*args, timeout=40):
proc = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True,
text=True, encoding='utf-8', errors='replace', timeout=timeout)
if proc.returncode:
raise RuntimeError('docker compose ' + ' '.join(args[:3]) + ': ' + proc.stderr[-1200:])
return proc.stdout.strip()
PROBE = r'''
from concurrent.futures import ThreadPoolExecutor
import http.client, json, sys, time
expected, count = map(int, sys.argv[1:])
def request(path, status):
c = http.client.HTTPConnection('127.0.0.1', 8080, timeout=8)
start = time.monotonic()
try:
c.request('GET', path)
response = c.getresponse()
body = json.loads(response.read())
assert response.status == status, (response.status, body)
assert response.getheader('X-Request-Id')
assert response.getheader('X-Instance-Id')
if status == 503:
assert body['error']['code'] == 'unavailable'
assert body['error']['request_id'] == response.getheader('X-Request-Id')
return round(time.monotonic() - start, 3)
finally:
c.close()
with ThreadPoolExecutor(max_workers=count) as executor:
futures = [executor.submit(request, '/health/ready', expected) for _ in range(count)]
live = request('/health/live', 200)
elapsed = [future.result() for future in futures]
print(json.dumps(dict(status=expected, requests=count, seconds=elapsed, live_seconds=live)))
'''
WORKER = r'''
import json, sys, time
from messenger.config import read_secret
from messenger.db import create_pool, Repository
from messenger.errors import Unavailable
class SyntheticRollback(Exception): pass
pool = create_pool(read_secret('DATABASE_URL'), 1)
pool.open(wait=True, timeout=10)
repo = Repository(pool)
phase = sys.argv[1]
try:
try:
with repo._transaction() as conn:
conn.execute('SELECT 1')
print('ARMED', flush=True)
assert sys.stdin.readline().strip() == 'go'
started = time.monotonic()
if phase == 'query':
conn.execute('SELECT 1')
elif phase == 'rollback':
raise SyntheticRollback()
# commit: leave the block while PostgreSQL is paused.
except (Unavailable, SyntheticRollback) as error:
elapsed = time.monotonic() - started
assert conn.closed, 'timed-out connection was returned open'
assert elapsed < 8, elapsed
expected = SyntheticRollback if phase == 'rollback' else Unavailable
assert isinstance(error, expected), type(error)
print('TIMED_OUT', flush=True)
else:
raise AssertionError('operation unexpectedly succeeded while PostgreSQL paused')
assert sys.stdin.readline().strip() == 'resume'
recovery = time.monotonic()
while True:
try:
repo.ping()
break
except Unavailable:
if time.monotonic() - recovery >= 15:
raise
time.sleep(0.1)
assert pool.get_stats().get('requests_waiting', 0) == 0
print(json.dumps(dict(phase=phase, timeout_seconds=round(elapsed, 3),
connection_closed=conn.closed, pool_max_size=1,
recovered_seconds=round(time.monotonic()-recovery, 3))), flush=True)
finally:
pool.close(timeout=1)
'''
def line(proc, timeout=15):
result = queue.Queue()
threading.Thread(target=lambda: result.put(proc.stdout.readline()), daemon=True).start()
try:
return result.get(timeout=timeout).strip()
except queue.Empty:
raise AssertionError('database test worker did not finish in time') from None
def ready(c):
start = time.monotonic()
while time.monotonic() - start < 30:
try:
c.call('GET', '/health/ready', schema='Health')
return round(time.monotonic() - start, 3)
except Failure:
time.sleep(0.2)
raise AssertionError('API did not recover after PostgreSQL unpause')
def readiness(c, ids):
a, b, eve, chat = c.bootstrap()
for index in range(len(ids)):
compose('exec', '--index', str(index + 1), '-T', 'api', 'python', '-c', PROBE, '200', '1')
for cycle in range(2):
try:
compose('pause', 'postgres')
with ThreadPoolExecutor(max_workers=len(ids)) as executor:
futures = [executor.submit(compose, 'exec', '--index', str(index + 1), '-T', 'api',
'python', '-c', PROBE, '503', '12') for index in range(len(ids))]
for index, future in enumerate(futures):
record(dict(cycle=cycle + 1, api=index + 1, **json.loads(future.result())))
finally:
compose('unpause', 'postgres')
elapsed = ready(c)
# Same token and a real DB write/read after recovery, no API restart.
message = c.message(a, chat, 'PostgreSQL pause regression ' + str(cycle))['json']
page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=b, schema='MessagePage')['json']
check(message in page['items'], 'confirmed write not visible after unpause')
record(dict(cycle=cycle + 1, recovered_seconds=elapsed, existing_session_and_write='PASS'))
def operation(phase):
proc = subprocess.Popen(['docker', 'compose', 'exec', '-T', 'api', 'python', '-u', '-c', WORKER, phase],
cwd=ROOT, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
text=True, encoding='utf-8', errors='replace')
try:
check(line(proc) == 'ARMED', 'worker could not open a transaction')
try:
compose('pause', 'postgres')
proc.stdin.write('go\n')
proc.stdin.flush()
check(line(proc) == 'TIMED_OUT', 'operation did not time out and close connection: ' + phase)
finally:
compose('unpause', 'postgres')
out, err = proc.communicate('resume\n', timeout=25)
check(proc.returncode == 0, 'worker failed: ' + err[-1200:])
record(json.loads(out))
finally:
if proc.poll() is None:
proc.kill()
proc.communicate(timeout=5)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default='http://127.0.0.1:8080')
parser.add_argument('--output', type=Path)
args = parser.parse_args()
try:
c = Client(args.base_url, timeout=8)
ready(c)
ids = compose('ps', '-q', 'api').splitlines()
check(len(ids) >= 2, 'start at least two API replicas')
readiness(c, ids)
for phase in ('query', 'commit', 'rollback'):
operation(phase)
ready(c)
record('PASS: paused PostgreSQL returns bounded direct 503; query/commit/rollback discard connections; same pools recover.')
return 0
except Exception as error:
record('FAIL: ' + str(error))
return 1
finally:
if args.output:
args.output.write_text('\n'.join(LOG) + '\n', encoding='utf-8')
if __name__ == '__main__':
sys.exit(main())
+139
View File
@@ -0,0 +1,139 @@
#!/usr/bin/env python3
"""EXPLAIN (ANALYZE, BUFFERS) of the history page query on 10 000+ messages.
python scripts/explain_history.py --output evidence/explain.txt
The stand must be running. The script registers two synthetic users through the API,
creates a target chat and a noise chat, bulk-inserts messages with SQL, runs ANALYZE and
prints the plans of the same query the API runs (src/messenger/db.py, list_messages):
first page, a page in the middle, the last page, the generic plan of the parameterized
query, and for comparison the middle page without the index (DROP INDEX inside a
transaction that is rolled back, so the database is left unchanged).
Standard library only.
"""
import argparse
import base64
import json
from pathlib import Path
import subprocess
import sys
import time
import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
BASE = 'http://127.0.0.1:8080'
PSQL = ['docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger',
'-X', '-v', 'ON_ERROR_STOP=1']
PAGE = 51 # limit 50 + 1 row to learn whether the next page exists, as the API does
lines = []
def out(text=''):
lines.append(text)
print(text, flush=True)
def psql(sql, echo=True):
"""Run SQL in one psql session; with echo the transcript shows every statement before its result."""
args = PSQL + (['--echo-all'] if echo else ['-At']) + ['-f', '-']
done = subprocess.run(args, cwd=str(ROOT), input=sql.encode('utf-8'), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
text = done.stdout.decode('utf-8', 'replace').strip()
if done.returncode != 0:
out(text)
raise SystemExit('psql failed')
return text
def api(method, path, body=None, headers=None):
headers = dict(headers or {})
data = None
if body is not None:
data = json.dumps(body).encode('utf-8')
headers['Content-Type'] = 'application/json'
request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method)
with urllib.request.urlopen(request, timeout=10) as response:
return json.loads(response.read())
def user(label):
username = label + '_' + uuid.uuid4().hex[:10]
password = 'Explain-' + uuid.uuid4().hex
created = api('POST', '/api/v1/users', {'username': username, 'display_name': label, 'password': password})
credential = base64.b64encode((username + ':' + password).encode()).decode()
session = api('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credential})
return created['id'], session['token']
def page_sql(chat_id, after=None):
"""The same SQL text as Repository.list_messages, with literal values."""
sql = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = '%s' " % chat_id
if after:
sql += "AND (created_at, id) > ('%s', '%s') " % after
return sql + 'ORDER BY created_at, id LIMIT %d' % PAGE
def main():
parser = argparse.ArgumentParser(description='EXPLAIN of the history query; inserts synthetic messages')
parser.add_argument('--messages', type=int, default=10000, help='messages in the target chat (>= 10000)')
parser.add_argument('--noise', type=int, default=20000, help='messages in another chat')
parser.add_argument('--output', default=None)
args = parser.parse_args()
sys.stdout.reconfigure(errors='replace')
out('# %s UTC, %s' % (time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()), psql('SELECT version()', echo=False)))
alice, alice_token = user('explain_a')
bob, _ = user('explain_b')
target = api('POST', '/api/v1/chats', {'title': 'EXPLAIN target', 'member_ids': [alice, bob]},
{'Authorization': 'Bearer ' + alice_token})['id']
noise = api('POST', '/api/v1/chats', {'title': 'EXPLAIN noise', 'member_ids': [alice]},
{'Authorization': 'Bearer ' + alice_token})['id']
out('# target chat %s: %d messages; noise chat %s: %d messages' % (target, args.messages, noise, args.noise))
started = time.monotonic()
psql('''
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), '%(target)s', CASE WHEN i %% 2 = 0 THEN '%(alice)s'::uuid ELSE '%(bob)s'::uuid END,
'seed message ' || i, clock_timestamp() + i * interval '1 millisecond'
FROM generate_series(1, %(messages)d) AS i;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), '%(noise)s', '%(alice)s', 'noise message ' || i, clock_timestamp() + i * interval '1 millisecond'
FROM generate_series(1, %(noise_count)d) AS i;
ANALYZE messages;
''' % {'target': target, 'noise': noise, 'alice': alice, 'bob': bob, 'messages': args.messages, 'noise_count': args.noise},
echo=False)
out('# inserted and analyzed in %.1f s' % (time.monotonic() - started))
out(psql("SELECT 'messages total: ' || count(*) || ', in target chat: ' || count(*) FILTER (WHERE chat_id = '%s') FROM messages" % target, echo=False))
def key_at(position):
row = psql("SELECT created_at || '|' || id FROM messages WHERE chat_id = '%s' ORDER BY created_at, id OFFSET %d LIMIT 1"
% (target, position - 1), echo=False)
return tuple(row.split('|'))
middle, last = key_at(args.messages // 2), key_at(args.messages - 20)
explain = 'EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) '
out()
out('== 1. First page, middle page and last page: index range scan, no sort ==')
out(psql('\n'.join(explain + page_sql(target, after) + ';' for after in (None, middle, last))))
out()
out('== 2. Generic plan of the parameterized query (how a prepared statement with $1..$4 runs) ==')
out(psql('''SET plan_cache_mode = force_generic_plan;
PREPARE history_page(uuid, timestamptz, uuid, int) AS
SELECT id, chat_id, sender_id, text, created_at FROM messages
WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT $4;
%sEXECUTE history_page('%s', '%s', '%s', %d);''' % (explain, target, middle[0], middle[1], PAGE)))
out()
out('== 3. For comparison: the same middle page without the index (rolled back) ==')
out(psql('''BEGIN;
DROP INDEX messages_chat_history_idx;
%s%s;
ROLLBACK;''' % (explain, page_sql(target, middle))))
out(psql("SELECT 'index still exists: ' || (to_regclass('messages_chat_history_idx') IS NOT NULL)", echo=False))
if args.output:
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
Path(args.output).write_text('\n'.join(lines) + '\n', encoding='utf-8')
if __name__ == '__main__':
main()
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env python3
"""Create local secret files for the compose stand.
python scripts/init_secrets.py
Passwords are random and synthetic; they are created only if missing and never printed.
The URL/config files are always rebuilt from the password files, so the pair stays consistent.
secrets/ is listed in .gitignore and .dockerignore.
PostgreSQL applies its password only when the data volume is created. If secrets/ is
deleted while the pgdata volume is kept, restore the old files or recreate the volume
(docker compose down -v deletes ALL lab data).
"""
from pathlib import Path
import json
import secrets
ROOT = Path(__file__).resolve().parents[1]
DIR = ROOT / 'secrets'
def write(name, content):
path = DIR / name
path.write_text(content, encoding='utf-8', newline='\n')
# Readable for the non-root users inside containers (files are bind-mounted as is).
path.chmod(0o644)
def password(name):
path = DIR / name
if path.is_file() and path.read_text(encoding='utf-8').strip():
print('keep secrets/' + name)
return path.read_text(encoding='utf-8').strip()
value = secrets.token_hex(24) # hex: safe inside URLs without escaping
write(name, value + '\n')
print('created secrets/' + name)
return value
def main():
DIR.mkdir(exist_ok=True)
postgres = password('postgres_password')
valkey = password('valkey_password')
broker = password('broker_password')
write('database_url', 'postgresql://messenger:%s@postgres:5432/messenger\n' % postgres)
write('valkey_auth', 'requirepass %s\n' % valkey)
write('cache_url', 'valkey://:%s@valkey:6379/0\n' % valkey)
write('broker_auth', 'requirepass %s\n' % broker)
write('broker_url', 'valkey://:%s@broker:6379/0\n' % broker)
admin_key = password('s3_admin_access_key_id')
admin_secret = password('s3_admin_secret_access_key')
api_key = password('s3_access_key_id')
api_secret = password('s3_secret_access_key')
worker_key = password('s3_worker_access_key_id')
worker_secret = password('s3_worker_secret_access_key')
identities = [
dict(name='bootstrap', credentials=[dict(accessKey=admin_key, secretKey=admin_secret)],
actions=['Admin', 'Read', 'Write', 'List', 'Tagging']),
dict(name='messenger-api', credentials=[dict(accessKey=api_key, secretKey=api_secret)],
actions=['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments']),
dict(name='messenger-worker', credentials=[dict(accessKey=worker_key, secretKey=worker_secret)],
actions=['Read:messenger-attachments', 'Write:messenger-attachments', 'List:messenger-attachments'])]
# Нет anonymous identity: ни объект, ни список bucket не доступны анонимно.
write('s3_config', json.dumps(dict(identities=identities)) + '\n')
print('written secrets/database_url, secrets/valkey_auth, secrets/cache_url')
if __name__ == '__main__':
main()
+487
View File
@@ -0,0 +1,487 @@
#!/usr/bin/env python3
"""Repeatable grade 3/4 failure checks for the local synthetic lab5 Compose stand.
Run separately from other tests: scenarios deliberately restart/stop services.
Every scenario restores the original services in finally. No volumes, secrets or
application records are removed. Tokens live only in process memory.
"""
import argparse
import datetime as dt
import hashlib
import json
import os
from pathlib import Path
import re
import subprocess
import sys
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check
def command(*args):
result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True,
encoding='utf-8', errors='replace')
if result.returncode:
# Never print environment, response bodies, credentials or arbitrary Docker logs.
raise Failure('command failed: ' + ' '.join(args[:4]) + ' (exit %d)' % result.returncode)
return result.stdout.strip()
def compose(*args):
return command('docker', 'compose', *args)
def sql(query):
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', query)
def rows(query):
value = sql("SELECT COALESCE(json_agg(t), '[]'::json) FROM (%s) t" % query)
return json.loads(value)
def identifier(value):
return str(uuid.UUID(value))
def job(attachment_id):
data = rows("SELECT * FROM attachment_jobs WHERE attachment_id='%s'" % identifier(attachment_id))
check(len(data) == 1, 'attachment must have exactly one logical job')
return data[0]
def audit(job_id):
return rows("SELECT action,actor,reason,details,created_at FROM job_audit WHERE job_id='%s' ORDER BY id"
% identifier(job_id))
def cli(*args):
# The explicit operator service has broker/worker permissions. The API does
# not receive broker credentials merely to make an evidence script easier.
output = compose('--profile', 'tools', 'run', '--rm', '--no-deps', 'jobs', *args)
return json.loads(output.splitlines()[-1]) if output else None
def inject(job_id, action, count=1):
return cli('inject', identifier(job_id), '--action', action, '--count', str(count),
'--actor', 'lab5-evidence', '--reason', 'repeatable synthetic grade 4 failure check')
def wait_until(predicate, description, seconds=180):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
value = predicate()
if value:
return value
time.sleep(0.4)
raise Failure('deadline exceeded: ' + description)
def ready(client):
def probe():
try:
client.call('GET', '/health/ready', schema='Health')
return True
except Failure:
return False
wait_until(probe, 'API ready', 90)
def restore(client, *services):
# start preserves stopped container configuration and all named volumes.
compose('start', *services)
ready(client)
def upload(client, actor, chat, label):
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
result = client.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + label + '.png',
202, actor=actor, raw=content, headers={'Content-Type': 'image/png'}, schema='Attachment')
attachment = result['json']
check(attachment['status'] == 'queued', '202 must carry a queued snapshot')
check(attachment['variants'] == [] and attachment['error_code'] is None, 'queued result fields')
check(attachment['sha256'] == hashlib.sha256(content).hexdigest(), 'accepted original checksum')
return attachment, content
def metadata(client, actor, attachment):
return client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=actor, schema='Attachment')['json']
def wait_state(client, actor, attachment, status):
def probe():
current = metadata(client, actor, attachment)
if current['status'] == status:
return current
check(current['status'] != 'failed' or status == 'failed', 'unexpected terminal failed state')
return wait_until(probe, 'attachment ' + status)
def verified_ready(client, actor, attachment, content):
result = wait_state(client, actor, attachment, 'ready')
check(result['id'] == attachment['id'], 'logical attachment identity changed')
check(result['error_code'] is None, 'ready error_code must be null')
check({item['name'] for item in result['variants']} == {'thumbnail', 'crop', 'optimized'}, 'grade 4 variants')
check(len(result['variants']) == 3, 'duplicate result variants')
for name in ['original', 'thumbnail', 'crop', 'optimized']:
data = client.call('GET', '/api/v1/attachments/' + result['id'] + '/content?variant=' + name, actor=actor)
record = result if name == 'original' else next(v for v in result['variants'] if v['name'] == name)
check(len(data['bytes']) == record['size_bytes'], 'published object size mismatch')
check(hashlib.sha256(data['bytes']).hexdigest() == record['sha256'], 'published object checksum mismatch')
if name == 'original':
check(data['bytes'] == content, 'original bytes changed')
count = int(sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % result['id']))
check(count == 3, 'exactly three persistent variant records required')
return result
def object_state(attachment_id):
# Runs with the API's restricted bucket credentials; only selected facts return.
aid = identifier(attachment_id)
code = """import json
from messenger.config import read_secret
from messenger.db import create_pool
from messenger.objects import ObjectStore
from botocore.exceptions import ClientError
p=create_pool(read_secret('DATABASE_URL'),1);p.open(wait=True)
with p.connection() as c:
r=c.execute('SELECT object_key,staging_key,staging_deleted_at,status,pipeline_version FROM attachments WHERE id=%%s',(%r,)).fetchone()
s=ObjectStore()
def exists(key):
try: s.client.head_object(Bucket=s.bucket,Key=key);return True
except ClientError as e:
if e.response['ResponseMetadata']['HTTPStatusCode']==404:return False
raise
prefix='variants/'+%r.replace('-','')+'/'
objects=s.client.list_objects_v2(Bucket=s.bucket,Prefix=prefix).get('Contents',[])
print(json.dumps(dict(final=exists(r['object_key']),staging=exists(r['staging_key']),status=r['status'],staging_deleted=r['staging_deleted_at'] is not None,variant_objects=len(objects))))
p.close()
""" % (aid, aid)
return json.loads(compose('exec', '-T', 'api', 'python', '-c', code).splitlines()[-1])
def effect_snapshot(attachment_id):
aid = identifier(attachment_id)
return rows("SELECT a.status,a.object_key,a.sha256,a.updated_at,j.id AS job_id,j.attempt,j.updated_at AS job_updated_at,"
"(SELECT json_agg(v ORDER BY v.name) FROM attachment_variants v WHERE v.attachment_id=a.id) AS variants "
"FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE a.id='%s'" % aid)[0]
def fault_consumed(job_id, action):
return bool(rows("SELECT 1 AS consumed FROM job_faults WHERE job_id='%s' AND action='%s' AND remaining=0"
% (identifier(job_id), action)))
def stale_lease_fenced(attachment_id, old_job):
"""A dead invocation cannot commit success or failure over the recovered result."""
old_claim = dict(job_id=old_job['id'], lease_token=old_job['lease_token'],
attempt=old_job['attempt'], pipeline_version=old_job['pipeline_version'])
code = """import json
from messenger.config import read_secret
from messenger.db import create_pool
from messenger.jobs import Jobs
p=create_pool(read_secret('DATABASE_URL'),1);p.open(wait=True);j=Jobs(p)
claim=json.loads(%r)
with p.connection() as c:
a=c.execute('SELECT object_key FROM attachments WHERE id=%%s',(%r,)).fetchone()
variants=c.execute('SELECT * FROM attachment_variants WHERE attachment_id=%%s ORDER BY name',(%r,)).fetchall()
assert j.finish(claim,a['object_key'],variants) is False,'old lease committed success'
assert j.fail(claim,'stale_worker',False,5,0) is False,'old lease committed failure'
p.close();print(json.dumps(dict(old_finish=False,old_fail=False)))
""" % (json.dumps(old_claim), identifier(attachment_id), identifier(attachment_id))
return json.loads(compose('exec', '-T', 'dispatcher', 'python', '-c', code).splitlines()[-1])
def stack(client):
config = json.loads(compose('config', '--format', 'json'))
services = config['services']
for name in ('api', 'worker', 'dispatcher', 'postgres', 'valkey', 'broker', 's3'):
check(name in services, 'missing independent service ' + name)
check(not services[name].get('ports'), name + ': internal ports must not be published')
check(services['api']['deploy']['replicas'] >= 2, 'two API replicas required')
check(any(v.get('type') == 'volume' for v in services['broker'].get('volumes', [])), 'broker needs persistent volume')
check(services['worker'].get('deploy', {}).get('resources', {}).get('limits', {}).get('memory'),
'worker memory limit required')
print('PASS stack: separate durable broker/worker/dispatcher, private internal ports, bounded worker memory', flush=True)
def durability(client):
actor, member, _, chat = client.bootstrap()
try:
compose('stop', 'worker')
attachment, content = upload(client, actor, chat, 'worker-down')
logical_job = job(attachment['id'])['id']
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=member)
message = client.message(member, chat, '', attachments=[attachment['id']])['json']
check(message['attachment_ids'] == [attachment['id']], 'queued attachment cannot be linked')
check(object_state(attachment['id'])['staging'], '202 lost durable staging')
wait_until(lambda: bool(rows("SELECT 1 FROM attachment_outbox WHERE job_id='%s' AND published_at IS NOT NULL"
% logical_job)), 'broker publication before restart')
compose('restart', 'api', 'broker')
ready(client)
check(metadata(client, member, attachment)['status'] == 'queued', 'worker-down status must remain queued')
history = client.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=member, schema='MessagePage')['json']
check(any(item['id'] == message['id'] for item in history['items']), 'linked message lost across restart')
compose('start', 'worker')
verified_ready(client, member, attachment, content)
check(job(attachment['id'])['id'] == logical_job, 'restart created a replacement logical job')
print('PASS durability: worker down -> 202+linked message+409 -> API/broker restart with volumes -> same job ready; job_id=' + logical_job, flush=True)
finally:
restore(client, 'api', 'broker', 'worker')
def outbox(client):
actor, member, _, chat = client.bootstrap()
try:
compose('stop', 'dispatcher', 'worker')
attachment, content = upload(client, actor, chat, 'commit-publish-gap')
logical_job = job(attachment['id'])['id']
pending = rows("SELECT attempt,published_at FROM attachment_outbox WHERE job_id='%s'" % logical_job)
check(len(pending) == 1 and pending[0]['published_at'] is None, 'COMMIT/publish gap was not isolated')
# API is killed after its successful COMMIT while publisher is absent.
compose('kill', '-s', 'SIGKILL', 'api')
compose('start', 'api')
ready(client)
check(job(attachment['id'])['id'] == logical_job, 'COMMIT intent lost after API crash')
compose('start', 'dispatcher', 'worker')
verified_ready(client, member, attachment, content)
check(job(attachment['id'])['id'] == logical_job, 'outbox recovery recreated the job')
print('PASS outbox: COMMIT persisted while dispatcher stopped -> kill API before publish -> restart -> same job ready; job_id=' + logical_job, flush=True)
finally:
restore(client, 'api', 'dispatcher', 'worker')
def broker_outage(client):
actor, member, _, chat = client.bootstrap()
try:
compose('stop', 'broker', 'worker')
ready(client)
attachment, content = upload(client, actor, chat, 'broker-down')
logical_job = job(attachment['id'])['id']
client.message(member, chat, 'broker outage does not stop messages', attachments=[attachment['id']])
check(bool(rows("SELECT 1 FROM attachment_outbox WHERE job_id='%s' AND published_at IS NULL" % logical_job)),
'broker-down upload lacks durable outbox intent')
compose('start', 'broker', 'worker')
verified_ready(client, member, attachment, content)
print('PASS broker outage: API remains ready, upload 202 and messages commit, durable outbox later reaches ready; job_id=' + logical_job, flush=True)
finally:
restore(client, 'broker', 'dispatcher', 'worker')
def wait_delivery_ack(message_id):
code = """import json
from messenger.broker import Broker
b=Broker();target=tuple(map(int,%r.split('-')))
groups=b.client.xinfo_groups(b.stream)
group=next((g for g in groups if g['name']==b.group),None)
delivered=bool(group and tuple(map(int,group['last-delivered-id'].split('-')))>=target)
pending=b.client.xpending_range(b.stream,b.group,%r,%r,1)
print(json.dumps(dict(done=delivered and not pending)));b.close()
""" % (message_id, message_id, message_id)
wait_until(lambda: json.loads(compose('exec', '-T', 'dispatcher', 'python', '-c', code))['done'], 'redelivery ack')
def redelivery(client):
actor, member, _, chat = client.bootstrap()
attachment, content = upload(client, actor, chat, 'redelivery')
completed = verified_ready(client, member, attachment, content)
before = effect_snapshot(attachment['id'])
published = cli('redeliver', before['job_id'])
message_id = published['message_id']
wait_delivery_ack(message_id)
after = effect_snapshot(attachment['id'])
check(before == after, 'ready duplicate mutated persistent result, job attempt or timestamps')
check(metadata(client, member, attachment) == completed, 'ready duplicate changed public metadata')
print('PASS redelivery: same ready job acked without mutation, same 3 variants/checksums; job_id=' + before['job_id'], flush=True)
def crash(client):
actor, member, _, chat = client.bootstrap()
try:
compose('stop', 'worker')
attachment, content = upload(client, actor, chat, 'crash-after-objects')
logical_job = job(attachment['id'])['id']
inject(logical_job, 'crash_after_objects')
compose('start', 'worker')
wait_until(lambda: fault_consumed(logical_job, 'crash_after_objects'), 'crash fault consumed')
at_crash = object_state(attachment['id'])
check(at_crash['final'] and at_crash['variant_objects'] == 3,
'injected crash occurred before all four final objects existed')
check(at_crash['status'] == 'processing' and at_crash['staging'],
'crash window must precede ready commit and retain staging')
check(not rows("SELECT 1 FROM attachment_variants WHERE attachment_id='%s'" % attachment['id']),
'crash window must precede publication of variant metadata')
# The durable hook is consumed before exit, so container restart cannot loop it.
compose('start', 'worker')
verified_ready(client, member, attachment, content)
result = job(attachment['id'])
check(result['id'] == logical_job and result['attempt'] >= 2, 'crashed job did not recover its expired lease')
check(object_state(attachment['id'])['variant_objects'] == 3, 'redelivery duplicated S3 variants')
print('PASS crash after 4 S3 writes before DB result/ack: same job recovered; no duplicate variants; job_id=%s attempts=%s window=%s'
% (logical_job, result['attempt'], json.dumps(at_crash, sort_keys=True)), flush=True)
finally:
restore(client, 'worker')
def lease(client):
actor, member, _, chat = client.bootstrap()
try:
compose('stop', 'worker')
attachment, content = upload(client, actor, chat, 'expired-processing')
logical_job = job(attachment['id'])['id']
inject(logical_job, 'stall_processing')
compose('start', 'worker')
wait_until(lambda: fault_consumed(logical_job, 'stall_processing'), 'processing stall hook')
held = job(attachment['id'])
check(held['status'] == 'processing' and held['lease_until'], 'stall did not hold a processing lease')
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=member)
compose('kill', '-s', 'SIGKILL', 'worker')
compose('start', 'worker')
verified_ready(client, member, attachment, content)
recovered = job(attachment['id'])
check(recovered['id'] == logical_job and recovered['attempt'] > held['attempt'], 'expired processing was not reclaimed')
snapshot = effect_snapshot(attachment['id'])
stale_lease_fenced(attachment['id'], held)
check(effect_snapshot(attachment['id']) == snapshot, 'stale lease changed the recovered ready result')
print('PASS expired lease: killed stalled processing worker -> automatic recovery, same job; job_id=%s old_lease_until=%s attempts=%d'
% (logical_job, held['lease_until'], recovered['attempt']), flush=True)
print('PASS fencing: old lease cannot finish or fail after recovered ready commit', flush=True)
finally:
restore(client, 'worker')
def failed_contract(client, actor, chat, attachment):
failed = wait_state(client, actor, attachment, 'failed')
check(bool(re.fullmatch(r'[a-z][a-z0-9_]{0,63}', failed['error_code'] or '')), 'safe terminal error code required')
client.call('GET', '/api/v1/attachments/' + attachment['id'] + '/content', 409, actor=actor)
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=actor,
body={'text': '', 'attachment_ids': [attachment['id']]}, headers={'Idempotency-Key': uuid.uuid4().hex})
return failed
def retries(client):
actor, member, _, chat = client.bootstrap()
config = json.loads(compose('config', '--format', 'json'))['services']['worker'].get('environment', {})
max_attempts = int(config.get('JOB_MAX_ATTEMPTS', 5))
retry_base = float(config.get('JOB_RETRY_BASE_SECONDS', 1))
retry_max = float(config.get('JOB_RETRY_MAX_SECONDS', 30))
check(max_attempts >= 3, 'retry demonstration requires a budget of at least three attempts')
try:
compose('stop', 'worker')
attachment, content = upload(client, actor, chat, 'transient-retry')
logical_job = job(attachment['id'])['id']
inject(logical_job, 'transient', 2)
compose('start', 'worker')
verified_ready(client, member, attachment, content)
retried = job(attachment['id'])
check(retried['attempt'] >= 2, 'transient failure was not retried')
events = audit(logical_job)
retry_events = [event for event in events if event['action'] == 'retry' and event['reason'] == 'injected_transient']
check(len(retry_events) == 2, 'both retry delays must be observable in durable audit')
randomized = False
for event in retry_events:
details = event['details']
cap = min(retry_max, retry_base * 2.0 ** (details['run_attempts'] - 1))
check(cap / 2 <= details['delay_seconds'] <= cap, 'retry is outside its equal-jitter exponential interval')
randomized |= details['delay_seconds'] not in (cap / 2, cap)
created = dt.datetime.fromisoformat(event['created_at'].replace('Z', '+00:00'))
available = dt.datetime.fromisoformat(details['available_at'].replace('Z', '+00:00'))
check(abs((available - created).total_seconds() - details['delay_seconds']) < 0.25,
'retry due time does not persist the chosen jitter delay')
check(randomized, 'retry evidence contains no randomized delay')
print('PASS transient errors retry with durable backoff/jitter; job_id=%s audit=%s'
% (logical_job, json.dumps(retry_events, sort_keys=True)), flush=True)
compose('stop', 'worker')
attachment, content = upload(client, actor, chat, 'retry-exhausted')
logical_job = job(attachment['id'])['id']
inject(logical_job, 'transient', max_attempts)
compose('start', 'worker')
failed = failed_contract(client, member, chat, attachment)
check(failed['error_code'] == 'retry_exhausted', 'exhausted job needs a distinct terminal error code')
exhausted = job(attachment['id'])
check(exhausted['run_attempts'] == max_attempts, 'retry budget not enforced')
check(fault_consumed(logical_job, 'transient'), 'retry fault count does not match exhausted attempts')
failed_jobs = cli('failed', '--limit', '1000')
check(any(item['job_id'] == logical_job for item in failed_jobs), 'failed job missing from operator registry')
check(object_state(attachment['id'])['staging'], 'failed job lost original needed for replay')
cli('cleanup')
check(object_state(attachment['id'])['staging'], 'cleanup removed failed input required for replay')
replay = cli('replay', attachment['id'], '--actor', 'lab5-evidence', '--reason', 'operator resolved synthetic failure')
check(replay['job_id'] == logical_job, 'operator replay created a different logical job')
verified_ready(client, member, attachment, content)
before = effect_snapshot(attachment['id'])
noop = cli('replay', attachment['id'], '--actor', 'lab5-evidence', '--reason', 'verify successful replay is a no-op')
check(effect_snapshot(attachment['id']) == before, 'replay re-executed a successful job')
actions = {event['action'] for event in audit(logical_job) if event['actor'] == 'lab5-evidence'}
check({'replay', 'replay_noop'} <= actions, 'explicit replay and successful no-op need operator audit records')
print('PASS retry exhaustion -> failed+409+retained staging -> explicit audited replay -> ready -> ready replay no-op; job_id=%s max_attempts=%d error_code=%s replay=%s'
% (logical_job, max_attempts, failed['error_code'], json.dumps(noop, sort_keys=True)), flush=True)
finally:
restore(client, 'worker')
def permanent(client):
actor, member, _, chat = client.bootstrap()
try:
compose('stop', 'worker')
attachment, _ = upload(client, actor, chat, 'permanent-failure')
logical_job = job(attachment['id'])['id']
inject(logical_job, 'permanent')
compose('start', 'worker')
failed = failed_contract(client, member, chat, attachment)
check(failed['error_code'] == 'invalid_image', 'permanent errors need a distinct safe error code')
check(job(attachment['id'])['run_attempts'] == 1, 'permanent errors must not consume all retries')
print('PASS permanent failure: failed after one attempt; content/message rejected with409; job_id=%s error_code=%s'
% (logical_job, failed['error_code']), flush=True)
finally:
restore(client, 'worker')
def staging(client):
actor, member, _, chat = client.bootstrap()
try:
compose('stop', 'worker')
attachment, content = upload(client, actor, chat, 'staging-safety')
before = object_state(attachment['id'])
check(before['staging'] and not before['final'], 'staging/final publication boundary is not visible')
cli('cleanup')
check(object_state(attachment['id']) == before, 'cleanup removed current queued staging')
compose('start', 'worker')
verified_ready(client, member, attachment, content)
cli('cleanup')
after = wait_until(lambda: (lambda s: s if not s['staging'] and s['staging_deleted'] else None)
(object_state(attachment['id'])), 'post-commit staging cleanup')
check(after['final'] and after['status'] == 'ready', 'cleanup removed committed original')
print('PASS staging cleanup: queued protected; staging deleted only after ready commit; final original retained; attachment_id=' + attachment['id'], flush=True)
finally:
restore(client, 'worker')
SCENARIOS = ['stack', 'durability', 'outbox', 'broker_outage', 'redelivery', 'crash', 'lease', 'retries', 'permanent', 'staging']
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://127.0.0.1:8080'))
parser.add_argument('--only', nargs='+', choices=SCENARIOS)
args = parser.parse_args()
client = Client(args.base_url, timeout=15)
ready(client)
for name in args.only or SCENARIOS:
ready(client)
print('START ' + name + ' ' + dt.datetime.now(dt.timezone.utc).isoformat(), flush=True)
globals()[name](client)
print('PASS selected lab5 grade 3/4 checks; no grade 5 track claimed', flush=True)
if __name__ == '__main__':
try:
main()
except (Failure, ValueError, OSError) as exc:
print('FAIL: ' + str(exc), flush=True)
sys.exit(1)
+10
View File
@@ -0,0 +1,10 @@
@echo off
rem Windows twin of scripts/restart.sh: tests/resilience.py starts the hook directly,
rem and Windows cannot execute a .sh file. Same actions, volumes are kept.
setlocal
cd /d "%~dp0.."
docker compose stop api worker dispatcher || exit /b 1
docker compose restart postgres valkey s3 broker || exit /b 1
docker compose start api dispatcher worker || exit /b 1
docker compose restart lb || exit /b 1
exit /b 0
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
# Hook for: make test-persistence ACTION_SCRIPT=scripts/restart.sh
# Restarts the whole stand: all API replicas, PostgreSQL, Valkey and the load balancer,
# keeping the named volumes: no "down -v", no FLUSHALL, no re-seed.
# The test itself waits for API readiness through the load balancer afterwards.
set -eu
cd "$(dirname "$0")/.."
docker compose stop api worker dispatcher
docker compose restart postgres valkey s3 broker
docker compose start api dispatcher worker
docker compose restart lb
+35
View File
@@ -0,0 +1,35 @@
#!/usr/bin/env python3
"""Sanitized runtime snapshot: real processes, start times, limits and volumes; no environment/secrets."""
import datetime as dt
import json
from pathlib import Path
import subprocess
ROOT = Path(__file__).resolve().parents[1]
def command(*args):
return subprocess.check_output(list(args), cwd=ROOT, text=True, encoding='utf-8')
def main():
ids = command('docker', 'compose', 'ps', '-a', '-q').split()
containers = json.loads(command('docker', 'inspect', *ids))
result = {'captured_at': dt.datetime.now(dt.timezone.utc).isoformat(), 'containers': []}
for item in containers:
# Only these selected fields are persisted; Config.Env is deliberately excluded.
result['containers'].append(dict(
name=item['Name'].lstrip('/'), image=item['Config']['Image'],
status=item['State']['Status'], started_at=item['State']['StartedAt'],
health=item['State'].get('Health', {}).get('Status'),
memory_limit_bytes=item['HostConfig']['Memory'], cpu_limit_nanos=item['HostConfig']['NanoCpus'],
volumes=[dict(name=m.get('Name'), destination=m['Destination'])
for m in item['Mounts'] if m['Type'] == 'volume'],
published_ports=item['NetworkSettings']['Ports']))
text = json.dumps(result, indent=2)
(ROOT / 'evidence/stack.json').write_text(text + '\n', encoding='utf-8')
print('Saved evidence/stack.json (process start times, versions, limits, volumes and ports)')
if __name__ == '__main__':
main()
+19
View File
@@ -0,0 +1,19 @@
@echo off
rem Windows twin of scripts/stop-one.sh: tests/resilience.py starts the hook directly,
rem and Windows cannot execute a .sh file. Stops ONE api replica by X-Instance-Id (container ID).
setlocal
if "%TARGET_INSTANCE_ID%"=="" (
echo TARGET_INSTANCE_ID is not set 1>&2
exit /b 2
)
cd /d "%~dp0.."
set PROJECT=%COMPOSE_PROJECT_NAME%
if "%PROJECT%"=="" set PROJECT=mtusi-lab5
set ID=
for /f %%i in ('docker ps -q --filter "id=%TARGET_INSTANCE_ID%" --filter "label=com.docker.compose.project=%PROJECT%" --filter "label=com.docker.compose.service=api"') do set ID=%%i
if "%ID%"=="" (
echo TARGET_INSTANCE_ID is not a running api replica of %PROJECT% 1>&2
exit /b 1
)
docker stop %ID% >nul || exit /b 1
exit /b 0
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
# Hook for: make test-failover ACTION_SCRIPT=scripts/stop-one.sh
# Stops exactly ONE API replica and leaves it stopped. TARGET_INSTANCE_ID is the X-Instance-Id
# of the replica, which is its container hostname = short container ID. The hook checks that
# the ID belongs to a running "api" container of this compose project before stopping it.
# docker stop sends SIGTERM: the replica reports not ready, HAProxy removes it, then it exits.
# Restore afterwards: docker compose up -d (or docker start <id>)
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
cd "$(dirname "$0")/.."
project="${COMPOSE_PROJECT_NAME:-mtusi-lab5}"
id=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \
--filter "label=com.docker.compose.project=$project" \
--filter "label=com.docker.compose.service=api")
if [ -z "$id" ]; then
echo "TARGET_INSTANCE_ID is not a running api replica of $project" >&2
exit 1
fi
docker stop "$id" >/dev/null
+86
View File
@@ -0,0 +1,86 @@
#!/usr/bin/env python3
"""Run reproducible acceptance checks against an already running local lab5 stand.
The default checks do not stop services. --faults explicitly includes the grade 4
fault suite; each fault restores services without deleting volumes or secrets.
"""
import argparse
import os
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure
def wait_ready(url):
client = Client(url)
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
try:
client.call('GET', '/health/ready', schema='Health')
return
except Failure:
time.sleep(0.5)
raise Failure('API did not become ready before verification block')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://127.0.0.1:8080'))
parser.add_argument('--faults', action='store_true', help='Also run disruptive grade 4 fault checks')
parser.add_argument('--only', nargs='+', choices=['offline', 'unit', 'smoke', 'tests', 'lab5-checks',
'persistence', 'failover', 'postgres-pause', 'cleanup', 'stack'])
parser.add_argument('--output-prefix', default='', help='Keep a separate set of UTF-8 evidence logs')
options = parser.parse_args()
if Path(options.output_prefix).name != options.output_prefix or any(c in options.output_prefix for c in '/\\'):
parser.error('output prefix must be a filename prefix')
checks = [
('offline', ['scripts/check_contract.py']),
('unit', ['docker', 'compose', 'run', '--rm', '--no-deps', '--entrypoint', 'python',
'-v', './unit-tests:/app/unit-tests:ro', 'worker', '-m', 'unittest', 'discover', '-s', 'unit-tests', '-v']),
('smoke', ['tests/smoke.py', '--base-url', options.base_url]),
('tests', ['-m', 'unittest', 'discover', '-s', 'student-tests', '-v']),
('cleanup', ['docker', 'compose', '--profile', 'tools', 'run', '--rm', '--no-deps', 'cleanup']),
('stack', ['scripts/stack_evidence.py']),
]
if options.faults or (options.only and 'lab5-checks' in options.only):
checks.insert(4, ('lab5-checks', ['scripts/lab5_checks.py', '--base-url', options.base_url]))
for name, mode, hook in (
('persistence', 'persistence', 'restart'), ('failover', 'failover', 'stop-one')
):
if options.faults or (options.only and name in options.only):
script = 'scripts/%s.%s' % (hook, 'cmd' if os.name == 'nt' else 'sh')
checks.insert(-1, (name, ['tests/resilience.py', '--mode', mode, '--action-script', script,
'--base-url', options.base_url]))
if options.faults or (options.only and 'postgres-pause' in options.only):
checks.insert(-1, ('postgres-pause', ['scripts/check_postgres_pause.py', '--base-url', options.base_url]))
(ROOT / 'evidence').mkdir(exist_ok=True)
for name, args in checks:
if options.only and name not in options.only:
continue
if name != 'offline':
wait_ready(options.base_url)
invocation = args if name in ('unit', 'cleanup') else [sys.executable, '-X', 'utf8', *args]
result = subprocess.run(invocation, cwd=ROOT,
env=dict(os.environ, BASE_URL=options.base_url), capture_output=True)
if name == 'failover':
restored = subprocess.run(['docker', 'compose', 'up', '-d', '--wait'], cwd=ROOT,
capture_output=True)
if restored.returncode:
print('Failed to restore API replica after failover', flush=True)
return restored.returncode
content = result.stdout.decode('utf-8', errors='replace') + result.stderr.decode('utf-8', errors='replace')
(ROOT / 'evidence' / (options.output_prefix + name + '.txt')).write_text(content, encoding='utf-8')
print(name + ': ' + ('PASS' if result.returncode == 0 else 'FAIL'), flush=True)
if result.returncode:
print(content[-6000:], flush=True)
return result.returncode
return 0
if __name__ == '__main__':
sys.exit(main())