#!/usr/bin/env python3 import hashlib import http.client import json import os from pathlib import Path import socket import subprocess import sys import time import urllib.error import urllib.request ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client HOST = os.environ.get('LB_HOST', 'lb') PORT = int(os.environ.get('LB_PORT', '8080')) SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() SAMPLE_SHA = hashlib.sha256(SAMPLE).hexdigest() BUCKET = os.environ.get('S3_BUCKET', 'messenger-attachments') FAULT_NAME = 'lab4_fault' def sh(*args, check=True): return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip() def sql(statement): return sh('docker', 'compose', 'exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-tAc', statement) def request(method, path, token=None, body=None, content_type=None): headers = {} if token: headers['Authorization'] = 'Bearer ' + token if content_type: headers['Content-Type'] = content_type conn = http.client.HTTPConnection(HOST, PORT, timeout=60) conn.request(method, path, body=body, headers=headers) response = conn.getresponse() data = response.read() conn.close() return response, data def upload(token, chat): return request('POST', '/api/v1/chats/' + chat + '/attachments?filename=sample.png', token, SAMPLE, 'image/png') def download(token, attachment): return request('GET', '/api/v1/attachments/' + attachment + '/content', token) def reconcile(*flags): output = sh('docker', 'compose', 'run', '--rm', '--no-deps', '-T', 'reconciler', 'node', 'dist/reconcile.js', *flags) line = output.splitlines()[-1] print('$ reconcile', ' '.join(flags)) print(line) return json.loads(line) def replicas(): return [cid[:12] for cid in sh('docker', 'compose', 'ps', '-q', 'api').split()] def wait_ready(): deadline = time.monotonic() + 90 while time.monotonic() < deadline: try: response, _ = request('GET', '/health/ready') if response.status == 200: return except OSError: pass time.sleep(0.3) raise SystemExit('not ready') def header(text): print('\n## ' + text, flush=True) def defence(client): header('1. Сценарий защиты: загрузка, сообщение, второй участник, посторонний, S3 мимо API') alice, bob, eve, chat = client.bootstrap() attachment, _ = client.attachment(alice, chat) print('POST /attachments -> 201 status=%s variants=%s error_code=%s sha256=%s' % (attachment['status'], attachment['variants'], attachment['error_code'], attachment['sha256'])) message = client.message(alice, chat, '', attachments=[attachment['id']])['json'] print('POST /messages text="" -> 201 attachment_ids совпадают:', message['attachment_ids'] == [attachment['id']]) response, data = download(bob['token'], attachment['id']) print('Bob GET /content -> %d %s sha256 совпадает: %s' % (response.status, response.getheader('Content-Type'), hashlib.sha256(data).hexdigest() == SAMPLE_SHA)) for suffix in ('', '/content'): response, _ = request('GET', '/api/v1/attachments/' + attachment['id'] + suffix, eve['token']) print('Eve GET /attachments/{id}%s -> %d' % (suffix, response.status)) key = sql("SELECT object_key FROM attachments WHERE id = '%s'" % attachment['id']) print('object_key в БД: attachments/, filename в ключ не входит:', 'sample' not in key) try: code = urllib.request.urlopen('http://s3:9000/%s/%s' % (BUCKET, key), timeout=5).status except urllib.error.HTTPError as error: code = error.code print('Анонимный GET объекта напрямую в S3 ->', code) return alice, chat['id'] def failover(alice, chat): header('2. Остановка реплики, обслужившей загрузку; скачивание через другую') token = alice['token'] tested = set() while len(tested) < 2: response, data = upload(token, chat) target = response.getheader('X-Instance-Id') if target in tested: continue tested.add(target) attachment = json.loads(data) print('upload -> %d, обслужила реплика %s' % (response.status, target)) result = subprocess.run(['sh', 'scripts/stop-one.sh'], env={**os.environ, 'TARGET_INSTANCE_ID': target}, capture_output=True, text=True) print('stop-one.sh:', result.stdout.strip()) others = [cid for cid in replicas() if cid != target] started = time.monotonic() served = [] for _ in range(6): response, data = download(token, attachment['id']) served.append(response.getheader('X-Instance-Id')) ok = response.status == 200 and hashlib.sha256(data).hexdigest() == SAMPLE_SHA print('download -> %d реплика=%s sha256 совпадает: %s' % (response.status, served[-1], ok)) if not ok: raise SystemExit('download failed') print('остановленная реплика среди обслуживших:', target in served, '; %.2fs' % (time.monotonic() - started)) print('Файл оригинала на диске обслужившей реплики (поиск по точному размеру %d байт):' % len(SAMPLE)) for other in others: found = sh('docker', 'exec', other, 'sh', '-c', 'find /tmp /app /home -xdev -type f -size %dc 2>/dev/null | grep -v node_modules || true' % len(SAMPLE)) print(' %s -> %s' % (other, found or 'нет')) sh('docker', 'start', target) wait_ready() time.sleep(3) print('реплика %s возвращена' % target) def interrupted(alice, chat): header('3. Прерванная загрузка через LB: ни строки, ни объекта, ни временного файла') before_rows = sql('SELECT count(*) FROM attachments') before = reconcile('--grace-seconds=3600')['scanned'] connection = socket.create_connection((HOST, PORT)) connection.sendall(('POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: lb\r\nAuthorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n' % (chat, alice['token'], len(SAMPLE))).encode()) connection.sendall(SAMPLE[:20000]) time.sleep(0.5) connection.close() time.sleep(1) after = reconcile('--grace-seconds=3600')['scanned'] print('строк attachments: %s -> %s; объектов: %d -> %d' % (before_rows, sql('SELECT count(*) FROM attachments'), before, after)) for cid in replicas(): print('временных каталогов в %s: %s' % (cid, sh('docker', 'exec', cid, 'sh', '-c', 'ls /tmp | grep -c messenger-upload || true') or '0')) def inject(alice, chat, title, body): header(title) token = alice['token'] before = reconcile('--grace-seconds=3600')['scanned'] rows = sql('SELECT count(*) FROM attachments') sql("CREATE FUNCTION %s() RETURNS trigger AS $$ BEGIN %s RETURN NEW; END $$ LANGUAGE plpgsql; CREATE TRIGGER %s BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION %s();" % (FAULT_NAME, body, FAULT_NAME, FAULT_NAME)) try: response, data = upload(token, chat) print('upload при отказе INSERT -> %d %s' % (response.status, json.loads(data)['error']['code'])) finally: sql('DROP TRIGGER %s ON attachments; DROP FUNCTION %s();' % (FAULT_NAME, FAULT_NAME)) print('строк attachments: %s -> %s' % (rows, sql('SELECT count(*) FROM attachments'))) fresh = reconcile('--apply', '--grace-seconds=3600') print('объектов было %d, стало %d; свежий объект защищён grace: удалено %d' % (before, fresh['scanned'], fresh['deleted'])) time.sleep(2) cleaned = reconcile('--apply', '--grace-seconds=1') print('после grace: сирот %d, удалено %d, привязанные объекты не тронуты: %d' % (cleaned['orphans'], cleaned['deleted'], cleaned['referenced'])) response, data = upload(token, chat) print('следующая загрузка без отказа ->', response.status) wait_ready() def main(): client = Client('http://%s:%d' % (HOST, PORT)) alice, chat = defence(client) failover(alice, chat) interrupted(alice, chat) inject(alice, chat, '4. Реальный отказ PostgreSQL между PUT и INSERT: триггер отклоняет вставку', "RAISE EXCEPTION 'injected fault';") inject(alice, chat, '5. Потеря соединения с PostgreSQL между PUT и INSERT: исход неясен, объект остаётся до сверки', 'PERFORM pg_terminate_backend(pg_backend_pid());') if __name__ == '__main__': main()