forked from mtusi2026/mtusi2026lab4
212 lines
9.9 KiB
Python
212 lines
9.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Повторяемые интеграционные проверки lab4. Запуск на локальном синтетическом стенде.
|
|
|
|
Ничего не удаляет из volumes. Отказы восстанавливаются в finally; токены не печатаются.
|
|
"""
|
|
import argparse
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
import hashlib
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import socket
|
|
import struct
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
from urllib.parse import urlsplit
|
|
import zlib
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client, Failure, check
|
|
|
|
|
|
def command(*args):
|
|
result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding='utf-8', errors='replace')
|
|
if result.returncode:
|
|
raise RuntimeError('Command failed: ' + ' '.join(args[:4]) + '\n' + result.stderr[-1500:])
|
|
return result.stdout.strip()
|
|
|
|
|
|
def compose(*args):
|
|
return command('docker', 'compose', *args)
|
|
|
|
|
|
def sql(query):
|
|
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', query)
|
|
|
|
|
|
def ready(c):
|
|
deadline = time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
c.call('GET', '/health/ready', schema='Health')
|
|
return
|
|
except Failure:
|
|
time.sleep(0.5)
|
|
raise AssertionError('API did not become ready')
|
|
|
|
|
|
def stack(c):
|
|
config = json.loads(compose('config', '--format', 'json'))
|
|
for name in ('api', 'postgres', 'valkey', 's3'):
|
|
check(not config['services'][name].get('ports'), name + ': internal ports must not be published')
|
|
check(config['services']['api']['deploy']['replicas'] >= 2, 'two replicas required')
|
|
compose('run', '--rm', '--no-deps', 'bucket')
|
|
print('PASS stack: internal ports, 2 replicas, repeated bucket bootstrap')
|
|
|
|
|
|
def private(c):
|
|
a, b, eve, chat = c.bootstrap()
|
|
att, data = c.attachment(a, chat)
|
|
# Ключ доступен только оператору через БД, через публичный API его нет.
|
|
key = sql("SELECT object_key FROM attachments WHERE id='%s'" % att['id'])
|
|
code = """import urllib.request, urllib.error
|
|
from messenger.objects import ObjectStore
|
|
from botocore.exceptions import ClientError
|
|
s=ObjectStore()
|
|
try:
|
|
urllib.request.urlopen('http://s3:8333/'+s.bucket+'/'+%r)
|
|
raise AssertionError('anonymous GET allowed')
|
|
except urllib.error.HTTPError as e:
|
|
assert e.code==403, e.code
|
|
try:
|
|
s.client.create_bucket(Bucket='lab4-forbidden-bucket')
|
|
raise AssertionError('API has admin rights')
|
|
except ClientError as e:
|
|
assert e.response['ResponseMetadata']['HTTPStatusCode']==403
|
|
print('PASS private: anonymous object GET 403; API cannot create buckets')
|
|
""" % key
|
|
print(compose('exec', '-T', 'api', 'python', '-c', code))
|
|
|
|
|
|
def failover(c):
|
|
a, b, eve, chat = c.bootstrap()
|
|
data = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
result = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=failover.png', 201,
|
|
actor=a, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
target = result['headers']['X-Instance-Id']
|
|
ids = compose('ps', '-q', 'api').splitlines()
|
|
check(any(i.startswith(target) for i in ids), 'upload instance must be a replica of this stack')
|
|
try:
|
|
command('docker', 'stop', target)
|
|
ready(c)
|
|
download = c.call('GET', '/api/v1/attachments/' + result['json']['id'] + '/content', actor=b)
|
|
check(download['headers']['X-Instance-Id'] != target, 'download used stopped replica')
|
|
check(download['bytes'] == data, 'failover changed bytes')
|
|
print('PASS upload replica stopped -> another replica download; SHA256=' + hashlib.sha256(data).hexdigest())
|
|
finally:
|
|
command('docker', 'start', target)
|
|
compose('up', '-d', '--wait')
|
|
|
|
|
|
def outage(c):
|
|
a, b, eve, chat = c.bootstrap()
|
|
att, content = c.attachment(a, chat)
|
|
start = time.monotonic()
|
|
try:
|
|
compose('stop', 's3')
|
|
# После остановки dependency LB тоже исключает реплики. Проверяем обычным HTTP:
|
|
# созданный LB ответ вправе не иметь X-Instance-Id.
|
|
import urllib.request
|
|
import urllib.error
|
|
req = urllib.request.Request(c.base + '/api/v1/attachments/' + att['id'] + '/content',
|
|
headers={'Authorization': 'Bearer ' + b['token']})
|
|
try:
|
|
urllib.request.urlopen(req, timeout=6)
|
|
raise AssertionError('S3 down but content succeeded')
|
|
except urllib.error.HTTPError as e:
|
|
check(e.code == 503, 'S3 down must return 503')
|
|
payload = json.loads(e.read())
|
|
check(payload['error']['code'] == 'unavailable', 'invalid outage error')
|
|
print('PASS S3 unavailable -> 503; elapsed_s=' + str(round(time.monotonic()-start, 3)))
|
|
finally:
|
|
compose('start', 's3')
|
|
ready(c)
|
|
check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=b)['bytes'] == content,
|
|
'S3 outage lost object')
|
|
print('PASS same token and original after S3 recovery')
|
|
|
|
|
|
def png_noise():
|
|
# 3 MiB несжимаемых пикселей, реальный PNG. Случайность фиксирована.
|
|
rng = random.Random(42)
|
|
raw = b''.join(b'\0' + rng.randbytes(1024 * 3) for _ in range(1024))
|
|
def chunk(kind, data):
|
|
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data))
|
|
return (b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 1024, 1024, 8, 2, 0, 0, 0))
|
|
+ chunk(b'IDAT', zlib.compress(raw)) + chunk(b'IEND', b''))
|
|
|
|
|
|
def memory(c):
|
|
a, b, eve, chat = c.bootstrap()
|
|
data = png_noise()
|
|
ids = compose('ps', '-q', 'api').splitlines()
|
|
# Kernel memory.peak учитывает дочерний decoder и tmpfs; sampling не пропустит пик.
|
|
before = [int(command('docker', 'exec', i, 'cat', '/sys/fs/cgroup/memory.current')) for i in ids]
|
|
barrier = threading.Barrier(10)
|
|
def upload(_):
|
|
barrier.wait()
|
|
start = time.monotonic()
|
|
r = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=memory.png', 201,
|
|
actor=a, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment')
|
|
return round(time.monotonic() - start, 3), r['headers']['X-Instance-Id'], r['json']
|
|
start = time.monotonic()
|
|
with ThreadPoolExecutor(max_workers=10) as pool:
|
|
results = list(pool.map(upload, range(10)))
|
|
elapsed = time.monotonic() - start
|
|
peak = [int(command('docker', 'exec', i, 'cat', '/sys/fs/cgroup/memory.peak')) for i in ids]
|
|
for _, _, att in results:
|
|
got = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=b)
|
|
check(hashlib.sha256(got['bytes']).hexdigest() == att['sha256'], 'concurrent checksum mismatch')
|
|
print(json.dumps(dict(concurrent=10, successful=10, bytes_per_file=len(data), dimensions=[1024, 1024],
|
|
seconds=round(elapsed, 3), successful_rps=round(10/elapsed, 3),
|
|
latency_seconds=sorted(r[0] for r in results),
|
|
replicas=len(set(r[1] for r in results)), memory_before_bytes=before,
|
|
memory_peak_bytes=peak, peak_scope='container lifetime; cgroup includes decoder and tmpfs')))
|
|
|
|
|
|
def compensation(c):
|
|
a, b, eve, chat = c.bootstrap()
|
|
before = int(sql('SELECT count(*) FROM attachments'))
|
|
sql("""CREATE OR REPLACE FUNCTION lab4_force_failure() RETURNS trigger LANGUAGE plpgsql AS $$
|
|
BEGIN IF NEW.filename='lab4-force-rollback.png' THEN RAISE EXCEPTION 'synthetic lab4 failure'; END IF;
|
|
RETURN NEW; END $$;
|
|
CREATE TRIGGER lab4_force_failure BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION lab4_force_failure();""")
|
|
try:
|
|
c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=lab4-force-rollback.png', 500,
|
|
actor=a, raw=(ROOT / 'tests/fixtures/sample.png').read_bytes(), headers={'Content-Type': 'image/png'})
|
|
check(int(sql('SELECT count(*) FROM attachments')) == before, 'failed transaction left metadata')
|
|
finally:
|
|
sql('DROP TRIGGER IF EXISTS lab4_force_failure ON attachments; DROP FUNCTION IF EXISTS lab4_force_failure()')
|
|
print('PASS injected DB failure after S3 PUT: metadata rolled back, no 201; fresh orphan protected by grace')
|
|
print(compose('--profile', 'tools', 'run', '--rm', '--no-deps', 'cleanup'))
|
|
# Обрыв клиента до завершения тела: INSERT/PUT ещё не выполнялись.
|
|
address = urlsplit(c.base)
|
|
with socket.create_connection((address.hostname, address.port or 80), timeout=5) as sock:
|
|
body = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
headers = ('POST /api/v1/chats/%s/attachments?filename=interrupted.png HTTP/1.1\r\n'
|
|
'Host: local\r\nAuthorization: Bearer %s\r\nContent-Type: image/png\r\n'
|
|
'Content-Length: %d\r\n\r\n') % (chat['id'], a['token'], len(body))
|
|
sock.sendall(headers.encode() + body[:len(body)//2])
|
|
time.sleep(1)
|
|
check(int(sql('SELECT count(*) FROM attachments')) == before, 'interrupted upload committed metadata')
|
|
print('PASS interrupted upload: no attachment committed')
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://127.0.0.1:8080'))
|
|
parser.add_argument('--only', nargs='+', choices=['stack', 'private', 'failover', 'outage', 'memory', 'compensation'])
|
|
args = parser.parse_args()
|
|
c = Client(args.base_url, timeout=30)
|
|
for name in args.only or ['stack', 'private', 'failover', 'outage', 'memory', 'compensation']:
|
|
globals()[name](c)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|