191 lines
8.9 KiB
Python
Executable File
191 lines
8.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import hashlib
|
|
import http.client
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(ROOT / 'tests'))
|
|
from client import Client
|
|
|
|
HOST = os.environ.get('LB_HOST', 'lb')
|
|
PORT = int(os.environ.get('LB_PORT', '8080'))
|
|
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
|
SAMPLE_SHA = hashlib.sha256(SAMPLE).hexdigest()
|
|
BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments')
|
|
FAULT_NAME = 'lab4_fault'
|
|
|
|
|
|
def sh(*args, check=True):
|
|
return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip()
|
|
|
|
|
|
def sql(statement):
|
|
return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', statement)
|
|
|
|
|
|
def request(method, path, token=None, body=None, content_type=None):
|
|
headers = {}
|
|
if token:
|
|
headers['Authorization'] = 'Bearer ' + token
|
|
if content_type:
|
|
headers['Content-Type'] = content_type
|
|
conn = http.client.HTTPConnection(HOST, PORT, timeout=60)
|
|
conn.request(method, path, body=body, headers=headers)
|
|
response = conn.getresponse()
|
|
data = response.read()
|
|
conn.close()
|
|
return response, data
|
|
|
|
|
|
def upload(token, chat):
|
|
return request('POST', '/api/v1/chats/' + chat + '/attachments?filename=sample.png', token, SAMPLE, 'image/png')
|
|
|
|
|
|
def download(token, attachment):
|
|
return request('GET', '/api/v1/attachments/' + attachment + '/content', token)
|
|
|
|
|
|
def reconcile(*flags):
|
|
output = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', 'reconciler', 'node', 'dist/reconcile.js', *flags)
|
|
line = output.splitlines()[-1]
|
|
print('$ reconcile', ' '.join(flags))
|
|
print(line)
|
|
return json.loads(line)
|
|
|
|
|
|
def replicas():
|
|
return [cid[:12] for cid in sh('docker', 'compose', 'ps', '-q', 'api').split()]
|
|
|
|
|
|
def wait_ready():
|
|
deadline = time.monotonic() + 90
|
|
while time.monotonic() < deadline:
|
|
try:
|
|
response, _ = request('GET', '/health/ready')
|
|
if response.status == 200:
|
|
return
|
|
except OSError:
|
|
pass
|
|
time.sleep(0.3)
|
|
raise SystemExit('not ready')
|
|
|
|
|
|
def header(text):
|
|
print('\n## ' + text, flush=True)
|
|
|
|
|
|
def defence(client):
|
|
header('1. Сценарий защиты: загрузка, сообщение, второй участник, посторонний, S3 мимо API')
|
|
alice, bob, eve, chat = client.bootstrap()
|
|
attachment, _ = client.attachment(alice, chat)
|
|
print('POST /attachments -> 201 status=%s variants=%s error_code=%s sha256=%s' % (attachment['status'], attachment['variants'], attachment['error_code'], attachment['sha256']))
|
|
message = client.message(alice, chat, '', attachments=[attachment['id']])['json']
|
|
print('POST /messages text="" -> 201 attachment_ids совпадают:', message['attachment_ids'] == [attachment['id']])
|
|
response, data = download(bob['token'], attachment['id'])
|
|
print('Bob GET /content -> %d %s sha256 совпадает: %s' % (response.status, response.getheader('Content-Type'), hashlib.sha256(data).hexdigest() == SAMPLE_SHA))
|
|
for suffix in ('', '/content'):
|
|
response, _ = request('GET', '/api/v1/attachments/' + attachment['id'] + suffix, eve['token'])
|
|
print('Eve GET /attachments/{id}%s -> %d' % (suffix, response.status))
|
|
key = sql("SELECT object_key FROM attachments WHERE id = '%s'" % attachment['id'])
|
|
print('object_key в БД: attachments/<uuid>, filename в ключ не входит:', 'sample' not in key)
|
|
try:
|
|
code = urllib.request.urlopen('http://s3:9000/%s/%s' % (BUCKET, key), timeout=5).status
|
|
except urllib.error.HTTPError as error:
|
|
code = error.code
|
|
print('Анонимный GET объекта напрямую в S3 ->', code)
|
|
return alice, chat['id']
|
|
|
|
|
|
def failover(alice, chat):
|
|
header('2. Остановка реплики, обслужившей загрузку; скачивание через другую')
|
|
token = alice['token']
|
|
tested = set()
|
|
while len(tested) < 2:
|
|
response, data = upload(token, chat)
|
|
target = response.getheader('X-Instance-Id')
|
|
if target in tested:
|
|
continue
|
|
tested.add(target)
|
|
attachment = json.loads(data)
|
|
print('upload -> %d, обслужила реплика %s' % (response.status, target))
|
|
result = subprocess.run(['sh', 'scripts/stop-one.sh'], env={**os.environ, 'TARGET_INSTANCE_ID': target}, capture_output=True, text=True)
|
|
print('stop-one.sh:', result.stdout.strip())
|
|
others = [cid for cid in replicas() if cid != target]
|
|
started = time.monotonic()
|
|
served = []
|
|
for _ in range(6):
|
|
response, data = download(token, attachment['id'])
|
|
served.append(response.getheader('X-Instance-Id'))
|
|
ok = response.status == 200 and hashlib.sha256(data).hexdigest() == SAMPLE_SHA
|
|
print('download -> %d реплика=%s sha256 совпадает: %s' % (response.status, served[-1], ok))
|
|
if not ok:
|
|
raise SystemExit('download failed')
|
|
print('остановленная реплика среди обслуживших:', target in served, '; %.2fs' % (time.monotonic() - started))
|
|
print('Файл оригинала на диске обслужившей реплики (поиск по точному размеру %d байт):' % len(SAMPLE))
|
|
for other in others:
|
|
found = sh('docker', 'exec', other, 'sh', '-c', 'find /tmp /app /home -xdev -type f -size %dc 2>/dev/null | grep -v node_modules || true' % len(SAMPLE))
|
|
print(' %s -> %s' % (other, found or 'нет'))
|
|
sh('docker', 'start', target)
|
|
wait_ready()
|
|
time.sleep(3)
|
|
print('реплика %s возвращена' % target)
|
|
|
|
|
|
def interrupted(alice, chat):
|
|
header('3. Прерванная загрузка через LB: ни строки, ни объекта, ни временного файла')
|
|
before_rows = sql('SELECT count(*) FROM attachments')
|
|
before = reconcile('--grace-seconds=3600')['scanned']
|
|
connection = socket.create_connection((HOST, PORT))
|
|
connection.sendall(('POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: lb\r\nAuthorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n' % (chat, alice['token'], len(SAMPLE))).encode())
|
|
connection.sendall(SAMPLE[:20000])
|
|
time.sleep(0.5)
|
|
connection.close()
|
|
time.sleep(1)
|
|
after = reconcile('--grace-seconds=3600')['scanned']
|
|
print('строк attachments: %s -> %s; объектов: %d -> %d' % (before_rows, sql('SELECT count(*) FROM attachments'), before, after))
|
|
for cid in replicas():
|
|
print('временных каталогов в %s: %s' % (cid, sh('docker', 'exec', cid, 'sh', '-c', 'ls /tmp | grep -c messenger-upload || true') or '0'))
|
|
|
|
|
|
def inject(alice, chat, title, body):
|
|
header(title)
|
|
token = alice['token']
|
|
before = reconcile('--grace-seconds=3600')['scanned']
|
|
rows = sql('SELECT count(*) FROM attachments')
|
|
sql("CREATE FUNCTION %s() RETURNS trigger AS $$ BEGIN %s RETURN NEW; END $$ LANGUAGE plpgsql; CREATE TRIGGER %s BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION %s();" % (FAULT_NAME, body, FAULT_NAME, FAULT_NAME))
|
|
try:
|
|
response, data = upload(token, chat)
|
|
print('upload при отказе INSERT -> %d %s' % (response.status, json.loads(data)['error']['code']))
|
|
finally:
|
|
sql('DROP TRIGGER %s ON attachments; DROP FUNCTION %s();' % (FAULT_NAME, FAULT_NAME))
|
|
print('строк attachments: %s -> %s' % (rows, sql('SELECT count(*) FROM attachments')))
|
|
fresh = reconcile('--apply', '--grace-seconds=3600')
|
|
print('объектов было %d, стало %d; свежий объект защищён grace: удалено %d' % (before, fresh['scanned'], fresh['deleted']))
|
|
time.sleep(2)
|
|
cleaned = reconcile('--apply', '--grace-seconds=1')
|
|
print('после grace: сирот %d, удалено %d, привязанные объекты не тронуты: %d' % (cleaned['orphans'], cleaned['deleted'], cleaned['referenced']))
|
|
response, data = upload(token, chat)
|
|
print('следующая загрузка без отказа ->', response.status)
|
|
wait_ready()
|
|
|
|
|
|
def main():
|
|
client = Client('http://%s:%d' % (HOST, PORT))
|
|
alice, chat = defence(client)
|
|
failover(alice, chat)
|
|
interrupted(alice, chat)
|
|
inject(alice, chat, '4. Реальный отказ PostgreSQL между PUT и INSERT: триггер отклоняет вставку', "RAISE EXCEPTION 'injected fault';")
|
|
inject(alice, chat, '5. Потеря соединения с PostgreSQL между PUT и INSERT: исход неясен, объект остаётся до сверки', 'PERFORM pg_terminate_backend(pg_backend_pid());')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|