"""Integration checks against the local Compose stack. --lifecycle restarts its services.""" import argparse import base64 import json from pathlib import Path import struct import subprocess import sys import time import uuid from urllib.parse import quote import zlib ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / 'tests')) from client import Client, Failure, check, run def compose(*args): result = subprocess.run(['docker', 'compose', *args], cwd=ROOT, capture_output=True, text=True) check(result.returncode == 0, 'Compose action failed: ' + ' '.join(args[:2])) return result.stdout.strip() def wait_ready(client): deadline = time.monotonic() + 90 observed = set() stable_since = None while time.monotonic() < deadline: try: response = client.call('GET', '/health/ready', schema='Health') upstreams = compose('exec', '-T', 'nginx', 'cat', '/etc/nginx/ready-upstreams.conf') # A single 200 can come from an old Nginx worker while a removal # reload is still pending. Wait for both re-added replicas to settle. check(sum(':8080' in line for line in upstreams.splitlines()) == 2, 'Both upstreams have not recovered') observed.add(response['headers']['X-Instance-Id']) if stable_since is None: stable_since = time.monotonic() if len(observed) == 2 and time.monotonic() - stable_since >= 4: return except Failure: observed.clear() stable_since = None time.sleep(1) raise Failure('API did not recover') def direct_call(method, path, expected, actor=None, raw=None, content_type=None): """Probe the API process when the LB has excluded every unready replica.""" headers = {'Authorization': 'Bearer ' + actor['token']} if actor else {} if content_type: headers['Content-Type'] = content_type request = {'method': method, 'path': path, 'headers': headers, 'body': base64.b64encode(raw).decode() if raw is not None else None} script = """const q=JSON.parse(require('node:fs').readFileSync(0,'utf8')); fetch('http://127.0.0.1:8080'+q.path,{method:q.method,headers:q.headers, body:q.body===null?undefined:Buffer.from(q.body,'base64')}).then(async r=>{ console.log(JSON.stringify({status:r.status,body:await r.json(), requestId:r.headers.get('x-request-id'),instanceId:r.headers.get('x-instance-id')})); }).catch(()=>process.exit(1));""" result = subprocess.run(['docker', 'compose', 'exec', '-T', '--index', '1', 'api', 'node', '-e', script], cwd=ROOT, input=json.dumps(request), capture_output=True, text=True) check(result.returncode == 0, 'direct API probe failed') response = json.loads(result.stdout) check(response['status'] == expected, 'direct API returned wrong status') check(response['requestId'] and response['instanceId'], 'direct API lacks response identifiers') if expected >= 400: check(response['body']['error']['request_id'] == response['requestId'], 'error request ID differs') def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('--base-url', default='http://localhost:8080') parser.add_argument('--lifecycle', action='store_true') args = parser.parse_args() c = Client(args.base_url) alice, bob, eve, chat = c.bootstrap() path = '/api/v1/chats/' + chat['id'] png = (ROOT / 'tests/fixtures/sample.png').read_bytes() def upload(data=png, name='фото.png', mime='image/png', expected=201, actor=alice): return c.call('POST', path + '/attachments?filename=' + quote(name, safe=''), expected, actor=actor, raw=data, headers={'Content-Type': mime}, schema='Attachment' if expected == 201 else None) att = upload()['json'] item = '/api/v1/attachments/' + att['id'] check(c.call('GET', item, actor=bob, schema='Attachment')['json'] == att, 'metadata differs') check(c.call('GET', item + '/content', actor=bob)['bytes'] == png, 'PNG changed') c.call('GET', item + '/content?variant=thumbnail', 404, actor=bob) c.call('GET', item, 401) c.call('GET', item, 404, actor=eve) c.call('GET', item + '/content', 404, actor=eve) upload(actor=eve, expected=404) upload(name='../photo.png', expected=400) upload(name='photo\\bad.png', expected=400) upload(name='bad\x00.png', expected=400) upload(name=' ', expected=400) upload(data=b'not an image', expected=415) upload(data=png[:50], expected=415) upload(mime='image/jpeg', expected=415) upload(data=b'x' * (10485760 + 1), expected=413) def chunk(kind, value): return struct.pack('>I', len(value)) + kind + value + struct.pack('>I', zlib.crc32(kind + value)) huge = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 10000, 5000, 8, 2, 0, 0, 0)) huge += chunk(b'IDAT', zlib.compress(b'\x00\x00\x00\x00')) + chunk(b'IEND', b'') upload(data=huge, expected=413) jpeg = subprocess.run(['node', '-e', "require('sharp')('tests/fixtures/sample.png').jpeg().toBuffer().then(b=>process.stdout.write(b))"], cwd=ROOT, capture_output=True, check=True).stdout jpg = upload(data=jpeg, name='photo.jpg', mime='image/jpeg')['json'] check(c.call('GET', '/api/v1/attachments/' + jpg['id'] + '/content', actor=bob)['bytes'] == jpeg, 'JPEG changed') key = uuid.uuid4().hex message = c.message(alice, chat, '', key, [att['id']])['json'] check(c.message(alice, chat, '', key, [att['id']])['json'] == message, 'replay differs') c.call('POST', path + '/messages', 409, actor=alice, body={'text': '', 'attachment_ids': [jpg['id']]}, headers={'Idempotency-Key': key}) for ids, status in (([att['id'], att['id']], 400), ([str(uuid.uuid4())], 404), (None, 400)): c.call('POST', path + '/messages', status, actor=alice, body={'text': '', 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex}) other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'Other chat', 'member_ids': [alice['id']]}, schema='Chat')['json'] c.call('POST', '/api/v1/chats/' + other['id'] + '/messages', 404, actor=alice, body={'text': '', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': uuid.uuid4().hex}) if args.lifecycle: anonymous_url = 'http://s3:9000/messenger-attachments/originals/' + chat['id'] + '/' + att['id'] status = compose('exec', '-T', 'api', 'node', '-e', 'fetch(process.argv[1]).then(r=>console.log(r.status))', anonymous_url) check(status == '403', 'S3 object is publicly readable') try: compose('stop', 's3') direct_call('GET', '/health/ready', 503) direct_call('GET', item, 200, actor=bob) direct_call('GET', item + '/content', 503, actor=bob) direct_call('POST', path + '/attachments?filename=outage.png', 503, actor=alice, raw=png, content_type='image/png') deadline = time.monotonic() + 15 while True: upstreams = compose('exec', '-T', 'nginx', 'cat', '/etc/nginx/ready-upstreams.conf') if ':8080' not in upstreams: break check(time.monotonic() < deadline, 'Nginx still routes to unready replicas') time.sleep(0.5) finally: compose('start', 's3') wait_ready(c) check(c.call('GET', item + '/content', actor=bob)['bytes'] == png, 'original lost after storage recovery') compose('stop') compose('up', '-d', '--wait', '--wait-timeout', '180') wait_ready(c) check(c.call('GET', item + '/content', actor=bob)['bytes'] == png, 'original lost after stack restart') check(c.message(alice, chat, '', key, [att['id']])['json'] == message, 'message lost after restart') print('PASS: attachments, validation, ACL, idempotency' + (', private S3, outage and persistence' if args.lifecycle else '')) if __name__ == '__main__': run(main)