Files
mtusi2026lab4/scripts/lab4_checks.py
T

212 lines
9.9 KiB
Python

#!/usr/bin/env python3
"""Повторяемые интеграционные проверки lab4. Запуск на локальном синтетическом стенде.
Ничего не удаляет из volumes. Отказы восстанавливаются в finally; токены не печатаются.
"""
import argparse
from concurrent.futures import ThreadPoolExecutor
import hashlib
import json
import os
from pathlib import Path
import random
import socket
import struct
import subprocess
import sys
import threading
import time
from urllib.parse import urlsplit
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check
def command(*args):
result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding='utf-8', errors='replace')
if result.returncode:
raise RuntimeError('Command failed: ' + ' '.join(args[:4]) + '\n' + result.stderr[-1500:])
return result.stdout.strip()
def compose(*args):
return command('docker', 'compose', *args)
def sql(query):
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', query)
def ready(c):
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
try:
c.call('GET', '/health/ready', schema='Health')
return
except Failure:
time.sleep(0.5)
raise AssertionError('API did not become ready')
def stack(c):
config = json.loads(compose('config', '--format', 'json'))
for name in ('api', 'postgres', 'valkey', 's3'):
check(not config['services'][name].get('ports'), name + ': internal ports must not be published')
check(config['services']['api']['deploy']['replicas'] >= 2, 'two replicas required')
compose('run', '--rm', '--no-deps', 'bucket')
print('PASS stack: internal ports, 2 replicas, repeated bucket bootstrap')
def private(c):
a, b, eve, chat = c.bootstrap()
att, data = c.attachment(a, chat)
# Ключ доступен только оператору через БД, через публичный API его нет.
key = sql("SELECT object_key FROM attachments WHERE id='%s'" % att['id'])
code = """import urllib.request, urllib.error
from messenger.objects import ObjectStore
from botocore.exceptions import ClientError
s=ObjectStore()
try:
urllib.request.urlopen('http://s3:8333/'+s.bucket+'/'+%r)
raise AssertionError('anonymous GET allowed')
except urllib.error.HTTPError as e:
assert e.code==403, e.code
try:
s.client.create_bucket(Bucket='lab4-forbidden-bucket')
raise AssertionError('API has admin rights')
except ClientError as e:
assert e.response['ResponseMetadata']['HTTPStatusCode']==403
print('PASS private: anonymous object GET 403; API cannot create buckets')
""" % key
print(compose('exec', '-T', 'api', 'python', '-c', code))
def failover(c):
a, b, eve, chat = c.bootstrap()
data = (ROOT / 'tests/fixtures/sample.png').read_bytes()
result = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=failover.png', 201,
actor=a, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
target = result['headers']['X-Instance-Id']
ids = compose('ps', '-q', 'api').splitlines()
check(any(i.startswith(target) for i in ids), 'upload instance must be a replica of this stack')
try:
command('docker', 'stop', target)
ready(c)
download = c.call('GET', '/api/v1/attachments/' + result['json']['id'] + '/content', actor=b)
check(download['headers']['X-Instance-Id'] != target, 'download used stopped replica')
check(download['bytes'] == data, 'failover changed bytes')
print('PASS upload replica stopped -> another replica download; SHA256=' + hashlib.sha256(data).hexdigest())
finally:
command('docker', 'start', target)
compose('up', '-d', '--wait')
def outage(c):
a, b, eve, chat = c.bootstrap()
att, content = c.attachment(a, chat)
start = time.monotonic()
try:
compose('stop', 's3')
# После остановки dependency LB тоже исключает реплики. Проверяем обычным HTTP:
# созданный LB ответ вправе не иметь X-Instance-Id.
import urllib.request
import urllib.error
req = urllib.request.Request(c.base + '/api/v1/attachments/' + att['id'] + '/content',
headers={'Authorization': 'Bearer ' + b['token']})
try:
urllib.request.urlopen(req, timeout=6)
raise AssertionError('S3 down but content succeeded')
except urllib.error.HTTPError as e:
check(e.code == 503, 'S3 down must return 503')
payload = json.loads(e.read())
check(payload['error']['code'] == 'unavailable', 'invalid outage error')
print('PASS S3 unavailable -> 503; elapsed_s=' + str(round(time.monotonic()-start, 3)))
finally:
compose('start', 's3')
ready(c)
check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=b)['bytes'] == content,
'S3 outage lost object')
print('PASS same token and original after S3 recovery')
def png_noise():
# 3 MiB несжимаемых пикселей, реальный PNG. Случайность фиксирована.
rng = random.Random(42)
raw = b''.join(b'\0' + rng.randbytes(1024 * 3) for _ in range(1024))
def chunk(kind, data):
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 1024, 1024, 8, 2, 0, 0, 0))
+ chunk(b'IDAT', zlib.compress(raw)) + chunk(b'IEND', b''))
def memory(c):
a, b, eve, chat = c.bootstrap()
data = png_noise()
ids = compose('ps', '-q', 'api').splitlines()
# Kernel memory.peak учитывает дочерний decoder и tmpfs; sampling не пропустит пик.
before = [int(command('docker', 'exec', i, 'cat', '/sys/fs/cgroup/memory.current')) for i in ids]
barrier = threading.Barrier(10)
def upload(_):
barrier.wait()
start = time.monotonic()
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=memory.png', 201,
actor=a, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
return round(time.monotonic() - start, 3), r['headers']['X-Instance-Id'], r['json']
start = time.monotonic()
with ThreadPoolExecutor(max_workers=10) as pool:
results = list(pool.map(upload, range(10)))
elapsed = time.monotonic() - start
peak = [int(command('docker', 'exec', i, 'cat', '/sys/fs/cgroup/memory.peak')) for i in ids]
for _, _, att in results:
got = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=b)
check(hashlib.sha256(got['bytes']).hexdigest() == att['sha256'], 'concurrent checksum mismatch')
print(json.dumps(dict(concurrent=10, successful=10, bytes_per_file=len(data), dimensions=[1024, 1024],
seconds=round(elapsed, 3), successful_rps=round(10/elapsed, 3),
latency_seconds=sorted(r[0] for r in results),
replicas=len(set(r[1] for r in results)), memory_before_bytes=before,
memory_peak_bytes=peak, peak_scope='container lifetime; cgroup includes decoder and tmpfs')))
def compensation(c):
a, b, eve, chat = c.bootstrap()
before = int(sql('SELECT count(*) FROM attachments'))
sql("""CREATE OR REPLACE FUNCTION lab4_force_failure() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN IF NEW.filename='lab4-force-rollback.png' THEN RAISE EXCEPTION 'synthetic lab4 failure'; END IF;
RETURN NEW; END $$;
CREATE TRIGGER lab4_force_failure BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION lab4_force_failure();""")
try:
c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=lab4-force-rollback.png', 500,
actor=a, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(), headers={'Content-Type': 'image/png'})
check(int(sql('SELECT count(*) FROM attachments')) == before, 'failed transaction left metadata')
finally:
sql('DROP TRIGGER IF EXISTS lab4_force_failure ON attachments; DROP FUNCTION IF EXISTS lab4_force_failure()')
print('PASS injected DB failure after S3 PUT: metadata rolled back, no 201; fresh orphan protected by grace')
print(compose('--profile', 'tools', 'run', '--rm', '--no-deps', 'cleanup'))
# Обрыв клиента до завершения тела: INSERT/PUT ещё не выполнялись.
address = urlsplit(c.base)
with socket.create_connection((address.hostname, address.port or 80), timeout=5) as sock:
body = (ROOT / 'tests/fixtures/sample.png').read_bytes()
headers = ('POST /api/v1/chats/%s/attachments?filename=interrupted.png HTTP/1.1\r\n'
'Host: local\r\nAuthorization: Bearer %s\r\nContent-Type: image/png\r\n'
'Content-Length: %d\r\n\r\n') % (chat['id'], a['token'], len(body))
sock.sendall(headers.encode() + body[:len(body)//2])
time.sleep(1)
check(int(sql('SELECT count(*) FROM attachments')) == before, 'interrupted upload committed metadata')
print('PASS interrupted upload: no attachment committed')
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://127.0.0.1:8080'))
parser.add_argument('--only', nargs='+', choices=['stack', 'private', 'failover', 'outage', 'memory', 'compensation'])
args = parser.parse_args()
c = Client(args.base_url, timeout=30)
for name in args.only or ['stack', 'private', 'failover', 'outage', 'memory', 'compensation']:
globals()[name](c)
if __name__ == '__main__':
main()