Files
mtusi2026lab4/scripts/attachment-scenario.py
T

191 lines
8.9 KiB
Python
Executable File

#!/usr/bin/env python3
import hashlib
import http.client
import json
import os
from pathlib import Path
import socket
import subprocess
import sys
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client
HOST = os.environ.get('LB_HOST', 'lb')
PORT = int(os.environ.get('LB_PORT', '8080'))
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
SAMPLE_SHA = hashlib.sha256(SAMPLE).hexdigest()
BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments')
FAULT_NAME = 'lab4_fault'
def sh(*args, check=True):
return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip()
def sql(statement):
return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', statement)
def request(method, path, token=None, body=None, content_type=None):
headers = {}
if token:
headers['Authorization'] = 'Bearer ' + token
if content_type:
headers['Content-Type'] = content_type
conn = http.client.HTTPConnection(HOST, PORT, timeout=60)
conn.request(method, path, body=body, headers=headers)
response = conn.getresponse()
data = response.read()
conn.close()
return response, data
def upload(token, chat):
return request('POST', '/api/v1/chats/' + chat + '/attachments?filename=sample.png', token, SAMPLE, 'image/png')
def download(token, attachment):
return request('GET', '/api/v1/attachments/' + attachment + '/content', token)
def reconcile(*flags):
output = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', 'reconciler', 'node', 'dist/reconcile.js', *flags)
line = output.splitlines()[-1]
print('$ reconcile', ' '.join(flags))
print(line)
return json.loads(line)
def replicas():
return [cid[:12] for cid in sh('docker', 'compose', 'ps', '-q', 'api').split()]
def wait_ready():
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
try:
response, _ = request('GET', '/health/ready')
if response.status == 200:
return
except OSError:
pass
time.sleep(0.3)
raise SystemExit('not ready')
def header(text):
print('\n## ' + text, flush=True)
def defence(client):
header('1. Сценарий защиты: загрузка, сообщение, второй участник, посторонний, S3 мимо API')
alice, bob, eve, chat = client.bootstrap()
attachment, _ = client.attachment(alice, chat)
print('POST /attachments -> 201 status=%s variants=%s error_code=%s sha256=%s' % (attachment['status'], attachment['variants'], attachment['error_code'], attachment['sha256']))
message = client.message(alice, chat, '', attachments=[attachment['id']])['json']
print('POST /messages text="" -> 201 attachment_ids совпадают:', message['attachment_ids'] == [attachment['id']])
response, data = download(bob['token'], attachment['id'])
print('Bob GET /content -> %d %s sha256 совпадает: %s' % (response.status, response.getheader('Content-Type'), hashlib.sha256(data).hexdigest() == SAMPLE_SHA))
for suffix in ('', '/content'):
response, _ = request('GET', '/api/v1/attachments/' + attachment['id'] + suffix, eve['token'])
print('Eve GET /attachments/{id}%s -> %d' % (suffix, response.status))
key = sql("SELECT object_key FROM attachments WHERE id = '%s'" % attachment['id'])
print('object_key в БД: attachments/<uuid>, filename в ключ не входит:', 'sample' not in key)
try:
code = urllib.request.urlopen('http://s3:9000/%s/%s' % (BUCKET, key), timeout=5).status
except urllib.error.HTTPError as error:
code = error.code
print('Анонимный GET объекта напрямую в S3 ->', code)
return alice, chat['id']
def failover(alice, chat):
header('2. Остановка реплики, обслужившей загрузку; скачивание через другую')
token = alice['token']
tested = set()
while len(tested) < 2:
response, data = upload(token, chat)
target = response.getheader('X-Instance-Id')
if target in tested:
continue
tested.add(target)
attachment = json.loads(data)
print('upload -> %d, обслужила реплика %s' % (response.status, target))
result = subprocess.run(['sh', 'scripts/stop-one.sh'], env={**os.environ, 'TARGET_INSTANCE_ID': target}, capture_output=True, text=True)
print('stop-one.sh:', result.stdout.strip())
others = [cid for cid in replicas() if cid != target]
started = time.monotonic()
served = []
for _ in range(6):
response, data = download(token, attachment['id'])
served.append(response.getheader('X-Instance-Id'))
ok = response.status == 200 and hashlib.sha256(data).hexdigest() == SAMPLE_SHA
print('download -> %d реплика=%s sha256 совпадает: %s' % (response.status, served[-1], ok))
if not ok:
raise SystemExit('download failed')
print('остановленная реплика среди обслуживших:', target in served, '; %.2fs' % (time.monotonic() - started))
print('Файл оригинала на диске обслужившей реплики (поиск по точному размеру %d байт):' % len(SAMPLE))
for other in others:
found = sh('docker', 'exec', other, 'sh', '-c', 'find /tmp /app /home -xdev -type f -size %dc 2>/dev/null | grep -v node_modules || true' % len(SAMPLE))
print(' %s -> %s' % (other, found or 'нет'))
sh('docker', 'start', target)
wait_ready()
time.sleep(3)
print('реплика %s возвращена' % target)
def interrupted(alice, chat):
header('3. Прерванная загрузка через LB: ни строки, ни объекта, ни временного файла')
before_rows = sql('SELECT count(*) FROM attachments')
before = reconcile('--grace-seconds=3600')['scanned']
connection = socket.create_connection((HOST, PORT))
connection.sendall(('POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: lb\r\nAuthorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n' % (chat, alice['token'], len(SAMPLE))).encode())
connection.sendall(SAMPLE[:20000])
time.sleep(0.5)
connection.close()
time.sleep(1)
after = reconcile('--grace-seconds=3600')['scanned']
print('строк attachments: %s -> %s; объектов: %d -> %d' % (before_rows, sql('SELECT count(*) FROM attachments'), before, after))
for cid in replicas():
print('временных каталогов в %s: %s' % (cid, sh('docker', 'exec', cid, 'sh', '-c', 'ls /tmp | grep -c messenger-upload || true') or '0'))
def inject(alice, chat, title, body):
header(title)
token = alice['token']
before = reconcile('--grace-seconds=3600')['scanned']
rows = sql('SELECT count(*) FROM attachments')
sql("CREATE FUNCTION %s() RETURNS trigger AS $$ BEGIN %s RETURN NEW; END $$ LANGUAGE plpgsql; CREATE TRIGGER %s BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION %s();" % (FAULT_NAME, body, FAULT_NAME, FAULT_NAME))
try:
response, data = upload(token, chat)
print('upload при отказе INSERT -> %d %s' % (response.status, json.loads(data)['error']['code']))
finally:
sql('DROP TRIGGER %s ON attachments; DROP FUNCTION %s();' % (FAULT_NAME, FAULT_NAME))
print('строк attachments: %s -> %s' % (rows, sql('SELECT count(*) FROM attachments')))
fresh = reconcile('--apply', '--grace-seconds=3600')
print('объектов было %d, стало %d; свежий объект защищён grace: удалено %d' % (before, fresh['scanned'], fresh['deleted']))
time.sleep(2)
cleaned = reconcile('--apply', '--grace-seconds=1')
print('после grace: сирот %d, удалено %d, привязанные объекты не тронуты: %d' % (cleaned['orphans'], cleaned['deleted'], cleaned['referenced']))
response, data = upload(token, chat)
print('следующая загрузка без отказа ->', response.status)
wait_ready()
def main():
client = Client('http://%s:%d' % (HOST, PORT))
alice, chat = defence(client)
failover(alice, chat)
interrupted(alice, chat)
inject(alice, chat, '4. Реальный отказ PostgreSQL между PUT и INSERT: триггер отклоняет вставку', "RAISE EXCEPTION 'injected fault';")
inject(alice, chat, '5. Потеря соединения с PostgreSQL между PUT и INSERT: исход неясен, объект остаётся до сверки', 'PERFORM pg_terminate_backend(pg_backend_pid());')
if __name__ == '__main__':
main()