From 29f45ca0eb0ca060f77a8ada1e20aeca4d7e0785 Mon Sep 17 00:00:00 2001 From: kudesn1k Date: Mon, 16 Nov 2026 12:37:21 +0300 Subject: [PATCH] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D1=8C=D1=82?= =?UTF-8?q?=D0=B5=20=D0=B2=D0=BE=D1=81=D0=BF=D1=80=D0=BE=D0=B8=D0=B7=D0=B2?= =?UTF-8?q?=D0=BE=D0=B4=D0=B8=D0=BC=D1=8B=D0=B5=20=D1=8D=D0=BA=D1=81=D0=BF?= =?UTF-8?q?=D0=B5=D1=80=D0=B8=D0=BC=D0=B5=D0=BD=D1=82=D1=8B=20=D0=BF=D0=BE?= =?UTF-8?q?=20=D0=BF=D0=B0=D0=BC=D1=8F=D1=82=D0=B8=20=D0=B8=20=D1=81=D0=BE?= =?UTF-8?q?=D0=B3=D0=BB=D0=B0=D1=81=D0=BE=D0=B2=D0=B0=D0=BD=D0=BD=D0=BE?= =?UTF-8?q?=D1=81=D1=82=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 14 + scripts/lab4/decoder-memory.py | 209 ++++++++++++ scripts/lab4/failover.py | 95 ++++++ scripts/lab4/generate-decoder-fixtures.py | 45 +++ scripts/lab4/test_decoder_memory.py | 55 +++ scripts/lab4/test_upload_memory.py | 139 ++++++++ scripts/lab4/upload-memory.py | 376 +++++++++++++++++++++ tools/reconcilebench/main.go | 394 ++++++++++++++++++++++ tools/reconcilebench/main_test.go | 46 +++ tools/reconcilebench/options.go | 43 +++ 10 files changed, 1416 insertions(+) create mode 100644 scripts/lab4/decoder-memory.py create mode 100644 scripts/lab4/failover.py create mode 100644 scripts/lab4/generate-decoder-fixtures.py create mode 100644 scripts/lab4/test_decoder_memory.py create mode 100644 scripts/lab4/test_upload_memory.py create mode 100644 scripts/lab4/upload-memory.py create mode 100644 tools/reconcilebench/main.go create mode 100644 tools/reconcilebench/main_test.go create mode 100644 tools/reconcilebench/options.go diff --git a/Makefile b/Makefile index 21b7346d..96ccf732 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,7 @@ secrets: up: docker compose up -d --build --wait + sh scripts/wait-ready.sh # Stops and removes the containers; volumes and therefore data are kept. down: @@ -63,6 +64,19 @@ destroy: logs: docker compose logs +# Safe default: inspection only; APPLY=1 explicitly enables repairs. +APPLY ?= 0 +DEEP ?= 0 +.PHONY: reconcile upload-memory reconcile-bench +reconcile: + docker compose --profile recon run --rm reconcile $(if $(filter 1,$(APPLY)),--apply,) $(if $(filter 1,$(DEEP)),--deep,) + +upload-memory: + $(PYTHON) -X utf8 scripts/lab4/upload-memory.py --base-url "$(BASE_URL)" --output "$(EVIDENCE_DIR)/upload-memory.json" + +reconcile-bench: + go run ./tools/reconcilebench --output "$(EVIDENCE_DIR)/track2/results.json" + # Which container answers under which X-Instance-Id. instances: sh scripts/instances.sh diff --git a/scripts/lab4/decoder-memory.py b/scripts/lab4/decoder-memory.py new file mode 100644 index 00000000..230db079 --- /dev/null +++ b/scripts/lab4/decoder-memory.py @@ -0,0 +1,209 @@ +#!/usr/bin/env python3 +"""File-driven decoder stress evidence. Requires rebuilt APIs with >=2 GiB limits. + +Run with python -X utf8 -B. Default is one upload; no container lifecycle actions. +""" +import argparse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +import datetime as dt +import hashlib +import importlib.util +import json +from pathlib import Path +import re +import struct +import subprocess +import sys +import threading +import time +from urllib.parse import urlsplit +import zlib + +sys.dont_write_bytecode = True +_spec = importlib.util.spec_from_file_location('upload_memory', Path(__file__).with_name('upload-memory.py')) +memory = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(memory) + + +def write_rgba64_png(path, width=40_000_000, height=1, chunk_bytes=65536): + """Stream zero scanlines, including filter bytes; never allocate a full row.""" + if min(width, height, chunk_bytes) <= 0 or width * height > 40_000_000: + raise ValueError('invalid stress dimensions or chunk size') + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + encoder = zlib.compressobj(1) + remaining = height * (width * 8 + 1) + zeros = bytes(min(chunk_bytes, remaining)) + with path.open('wb') as out: + out.write(b'\x89PNG\r\n\x1a\n') + out.write(memory.png_chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 16, 6, 0, 0, 0))) + while remaining: + count = min(len(zeros), remaining) + compressed = encoder.compress(zeros[:count]) + if compressed: + out.write(memory.png_chunk(b'IDAT', compressed)) + remaining -= count + out.write(memory.png_chunk(b'IDAT', encoder.flush())) + out.write(memory.png_chunk(b'IEND', b'')) + + +def fixture_info(content, mime): + if mime == 'image/png' and content.startswith(b'\x89PNG\r\n\x1a\n') and len(content) >= 33: + width, height, depth, color = struct.unpack_from('>IIBB', content, 16) + return {'width': width, 'height': height, 'bit_depth': depth, 'color_type': color} + if mime == 'image/jpeg' and content.startswith(b'\xff\xd8'): + pos = 2 + while pos + 4 <= len(content): + if content[pos] != 255: + break + while pos < len(content) and content[pos] == 255: + pos += 1 + marker = content[pos] + pos += 1 + length = struct.unpack_from('>H', content, pos)[0] + frame = content[pos + 2:pos + length] + pos += length + if marker in (0xc0, 0xc1, 0xc2): + height, width = struct.unpack_from('>HH', frame, 1) + count = frame[5] + return {'width': width, 'height': height, 'bit_depth': frame[0], 'components': count, + 'progressive': marker == 0xc2, 'sampling': [frame[7 + 3 * i] for i in range(count)]} + if length < 2 or marker == 0xda: + break + raise ValueError('fixture MIME/header mismatch or missing image header') + + +def read_fixture(path, mime): + if not 0 < path.stat().st_size <= 10 * 1024 * 1024: + raise ValueError('fixture must be between 1 byte and 10 MiB') + content = path.read_bytes() + info = fixture_info(content, mime) + if not 0 < info['width'] * info['height'] <= 40_000_000: + raise ValueError('fixture exceeds 40M pixels') + return content, info + + +def api_limits(ids): + template = '{"id":{{json .Id}},"limit_bytes":{{json .HostConfig.Memory}}}' + result = subprocess.run(['docker', 'inspect', '--format', template, *ids], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, text=True, check=True, timeout=20) + return {row['id']: row['limit_bytes'] for row in (json.loads(line) for line in result.stdout.splitlines())} + + +def experiment(args, content, report): + sys.path.insert(0, str(memory.ROOT / 'tests')) + import client as module + ids = memory.api_containers(args) + report['api_limits'] = api_limits(ids) + if any(report['api_limits'].get(cid, 0) < 2 * 1024**3 for cid in ids): + raise RuntimeError('stress uploads require every API container limit to be >=2 GiB') + report['state_before'] = memory.container_states(ids) + bootstrap = module.Client(args.base_url, timeout=args.timeout) + actor, reader, _, chat = bootstrap.bootstrap() + clients = [module.Client(args.base_url, timeout=args.timeout) for _ in range(args.concurrency)] + with memory.DockerStats(ids) as stats: + stats.mark(args.label, 'baseline') + stats.wait_for_baseline(args.label) + time.sleep(args.baseline_seconds) + barrier = threading.Barrier(args.concurrency + 1) + with ThreadPoolExecutor(max_workers=args.concurrency) as executor: + futures = [executor.submit(memory.upload, module, client, actor, chat, content, report['sha256'], + barrier, index, args.label, mime=args.mime) for index, client in enumerate(clients)] + stats.mark(args.label, 'load') + started = time.monotonic() + barrier.wait(timeout=30) + results = [future.result() for future in futures] + report['duration_s'] = time.monotonic() - started + time.sleep(args.tail_seconds) + stats.mark(args.label, 'idle') + report['memory_samples'] = stats.snapshot(args.label) + report['sampling_error'] = stats.error + report['state_after'] = memory.container_states(ids) + stable = all(report['state_after'][cid]['running'] and not report['state_after'][cid]['oom_killed'] + and report['state_after'][cid]['restart_count'] == report['state_before'][cid]['restart_count'] for cid in ids) + report['restart_or_oom_detected'] = not stable + try: + report['memory'] = memory.summarize_memory(report['memory_samples'], ids) + except RuntimeError as exc: + report['memory_error'] = str(exc) + report['observed_max_inflight'] = memory.max_inflight([(r['_started'], r['_finished']) for r in results]) + accepted = [r for r in results if r['ok']] + report['accepted'] = len(accepted) + report['status_counts'] = dict(Counter(str(r['status']) if r['status'] else 'transport_error' for r in results)) + report['latency'] = memory.latency_summary([r['latency_ms'] for r in results]) + report['attempted_bytes'], report['successful_bytes'] = len(content) * len(results), len(content) * len(accepted) + for result in results: + result['start_offset_ms'] = round((result.pop('_started') - started) * 1000, 3) + result.pop('_finished') + report['results'] = results + report['download'] = {'ok': False} + if accepted: + try: + response = bootstrap.call('GET', '/api/v1/attachments/' + accepted[0]['attachment_id'] + '/content', actor=reader) + body = response['bytes'] + report['download'] = {'ok': body == content and response['headers'].get_content_type() == args.mime, + 'status': response['status'], 'bytes': len(body), 'sha256': hashlib.sha256(body).hexdigest()} + except Exception as exc: + report['download']['error'] = str(exc) if isinstance(exc, module.Failure) else type(exc).__name__ + report['ok'] = (len(accepted) == args.concurrency and report['download']['ok'] and stable + and not report.get('memory_error') and not report['sampling_error'] + and report['observed_max_inflight'] == args.concurrency) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--file', type=Path) + parser.add_argument('--mime', choices=('image/png', 'image/jpeg')) + parser.add_argument('--label', default='decoder') + parser.add_argument('--concurrency', type=int, default=1, help='1-10, default single-upload acceptance') + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--compose-file', type=Path, default=memory.ROOT / 'compose.yaml') + parser.add_argument('--project-name') + parser.add_argument('--timeout', type=float, default=45) + parser.add_argument('--baseline-seconds', type=float, default=3) + parser.add_argument('--tail-seconds', type=float, default=3) + parser.add_argument('--output', type=Path) + parser.add_argument('--check-file', action='store_true', help='inspect bytes/header without HTTP or Docker') + parser.add_argument('--generate-wide', type=Path, help='generate 40Mx1 RGBA64 PNG offline using bounded zlib input') + args = parser.parse_args() + if args.generate_wide: + write_rgba64_png(args.generate_wide) + content, info = read_fixture(args.generate_wide, 'image/png') + print(json.dumps({**info, 'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()})) + return 0 + if not args.file or not args.mime: + parser.error('--file and --mime are required') + if not 1 <= args.concurrency <= 10 or args.timeout <= 0 or args.baseline_seconds < 1 or args.tail_seconds < 2: + parser.error('require concurrency1-10, timeout>0, baseline>=1s, tail>=2s') + if not re.fullmatch(r'[A-Za-z0-9_-]{1,40}', args.label): + parser.error('label must be 1-40 safe ASCII characters') + url = urlsplit(args.base_url) + if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password or url.query or url.fragment: + parser.error('base URL must be HTTP(S) without credentials/query/fragment') + content, info = read_fixture(args.file, args.mime) + report = {'schema_version': 1, 'name': args.label, 'mime': args.mime, 'fixture': info, + 'payload_bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest(), + 'requests': args.concurrency, 'ok': False, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(), + 'measurement': {'source': 'docker stats JSON stream, formatted working-set values', + 'limitation': 'sampled/rounded peaks can miss instantaneous maxima', + 'baseline_seconds': args.baseline_seconds, 'tail_seconds': args.tail_seconds}} + if args.check_file: + print(json.dumps(report, indent=2)) + return 0 + if not args.output: + parser.error('--output is required for a live run') + try: + experiment(args, content, report) + except Exception as exc: + report['error'] = type(exc).__name__ + if isinstance(exc, RuntimeError) or type(exc).__name__ == 'Failure': + report['failure'] = str(exc) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + print(json.dumps({key: value for key, value in report.items() if key not in ('memory_samples', 'results')})) + return 0 if report['ok'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/scripts/lab4/failover.py b/scripts/lab4/failover.py new file mode 100644 index 00000000..a16b81f2 --- /dev/null +++ b/scripts/lab4/failover.py @@ -0,0 +1,95 @@ +"""Stop each actual uploader replica and verify its bytes through the survivor.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + + +def instances(): + text = subprocess.check_output(['sh', 'scripts/api-instances.sh'], cwd=ROOT, + encoding='utf-8') + result = [] + for line in text.splitlines(): + container, replica, state, alias, image = line.split() + result.append({'container': container, 'replica': replica, + 'state': state, 'alias': alias, 'image': image}) + return result + + +def restore(c): + subprocess.run(['docker', 'compose', 'start', 'api'], cwd=ROOT, + check=True, stdout=subprocess.DEVNULL) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + observed = {c.call('GET', '/health/live')['headers']['X-Instance-Id'] + for _ in range(6)} + if len(observed) >= 2: + return + time.sleep(0.3) + raise AssertionError('two replicas did not return after restore') + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--base-url', default='http://localhost:8080') + p.add_argument('--output', type=Path, default=ROOT / 'evidence/failover-uploaders.json') + args = p.parse_args() + c = Client(args.base_url, timeout=10) + actor, reader, _, chat = c.bootstrap() + data = (ROOT / 'tests/fixtures/sample.png').read_bytes() + wanted = instances() + check(len(wanted) == 2 and all(x['state'] == 'running' for x in wanted), + 'this experiment requires two running API replicas') + report = {'scenario': 'stop actual upload-serving replica; download via survivor', + 'cases': [], 'ok': False} + try: + for target in wanted: + upload = None + for _ in range(10): + attempt = c.call('POST', '/api/v1/chats/' + chat['id'] + + '/attachments?filename=sample.png', 201, + actor=actor, raw=data, + headers={'Content-Type': 'image/png'}, schema='Attachment') + if attempt['headers']['X-Instance-Id'] == target['alias']: + upload = attempt + break + check(upload is not None, 'target replica did not serve an upload') + before = instances() + env = dict(os.environ, TARGET_INSTANCE_ID=target['alias']) + started = time.monotonic() + subprocess.run(['sh', 'scripts/stop-one.sh'], cwd=ROOT, env=env, + check=True, stdout=subprocess.DEVNULL, timeout=15) + stop_seconds = time.monotonic() - started + downloaded = c.call('GET', '/api/v1/attachments/' + + upload['json']['id'] + '/content', actor=reader) + survivor = downloaded['headers']['X-Instance-Id'] + check(survivor != target['alias'], 'stopped uploader still served download') + check(downloaded['bytes'] == data, 'bytes changed after uploader stop') + after = instances() + check(any(x['container'] == target['container'] and x['state'] != 'running' + for x in after), 'uploader was not left stopped') + report['cases'].append({'uploader': target, 'download_instance': survivor, + 'stop_seconds': round(stop_seconds, 3), + 'sha256': hashlib.sha256(downloaded['bytes']).hexdigest(), + 'same_bytes': True, 'before': before, 'after': after}) + restore(c) + report['ok'] = True + finally: + # This script only restores the API containers that it deliberately stopped. + restore(c) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + print(json.dumps({'ok': report['ok'], 'cases': len(report['cases']), + 'output': str(args.output)})) + + +if __name__ == '__main__': + main() diff --git a/scripts/lab4/generate-decoder-fixtures.py b/scripts/lab4/generate-decoder-fixtures.py new file mode 100644 index 00000000..dfc752ba --- /dev/null +++ b/scripts/lab4/generate-decoder-fixtures.py @@ -0,0 +1,45 @@ +"""Generate optional 40 MP regression images; requires Pillow==11.1.0.""" +import argparse +import hashlib +import importlib.util +import json +from pathlib import Path +import sys + +sys.dont_write_bytecode = True + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output-dir', type=Path, default=Path('bin/decoder-fixtures')) + args = parser.parse_args() + try: + from PIL import Image + except ImportError: + parser.error('install optional fixture dependency: pip install Pillow==11.1.0') + spec = importlib.util.spec_from_file_location('decoder_memory', + Path(__file__).with_name('decoder-memory.py')) + decoder = importlib.util.module_from_spec(spec) + spec.loader.exec_module(decoder) + args.output_dir.mkdir(parents=True, exist_ok=True) + manifest = {'pillow_version': Image.__version__, 'fixtures': []} + for mode, color in [('RGB', (25, 90, 160)), ('CMYK', (20, 40, 60, 10))]: + path = args.output_dir / ('progressive-' + mode.lower() + '-444-40mp.jpg') + with Image.new(mode, (8000, 5000), color) as image: + image.save(path, format='JPEG', progressive=True, subsampling=0, quality=85) + content, info = decoder.read_fixture(path, 'image/jpeg') + if not info['progressive'] or any(value != 0x11 for value in info['sampling']): + raise ValueError('encoder did not produce progressive 4:4:4') + manifest['fixtures'].append({'name': path.name, 'mime': 'image/jpeg', **info, + 'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()}) + wide = args.output_dir / 'wide-rgba64-40mp.png' + decoder.write_rgba64_png(wide) + content, info = decoder.read_fixture(wide, 'image/png') + manifest['fixtures'].append({'name': wide.name, 'mime': 'image/png', **info, + 'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()}) + (args.output_dir / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n', encoding='utf-8') + print(json.dumps(manifest, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/lab4/test_decoder_memory.py b/scripts/lab4/test_decoder_memory.py new file mode 100644 index 00000000..45e198ef --- /dev/null +++ b/scripts/lab4/test_decoder_memory.py @@ -0,0 +1,55 @@ +"""Offline checks for bounded stress fixtures and format identification.""" +import importlib.util +from pathlib import Path +import struct +import tempfile +import unittest +import zlib + + +class DecoderMemoryTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + path = Path(__file__).with_name('decoder-memory.py') + cls.module = None + if path.exists(): + spec = importlib.util.spec_from_file_location('decoder_memory', path) + cls.module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cls.module) + + def setUp(self): + self.assertIsNotNone(self.module, 'decoder-memory.py is not implemented') + + def test_streamed_rgba64_png_has_valid_crc_and_zero_pixels(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'wide.png' + self.module.write_rgba64_png(path, width=19, height=3, chunk_bytes=7) + payload = path.read_bytes() + self.assertEqual(payload[:8], b'\x89PNG\r\n\x1a\n') + pos, compressed = 8, bytearray() + while pos < len(payload): + size = struct.unpack_from('>I', payload, pos)[0] + kind, body = payload[pos + 4:pos + 8], payload[pos + 8:pos + 8 + size] + self.assertEqual(struct.unpack_from('>I', payload, pos + 8 + size)[0], zlib.crc32(kind + body) & 0xffffffff) + if kind == b'IDAT': + compressed.extend(body) + pos += size + 12 + self.assertEqual(zlib.decompress(compressed), bytes(3 * (19 * 8 + 1))) + self.assertEqual(self.module.fixture_info(payload, 'image/png'), + {'width': 19, 'height': 3, 'bit_depth': 16, 'color_type': 6}) + + def test_progressive_jpeg_444_components_are_identified(self): + sof = b'\x08' + struct.pack('>HHB', 5000, 8000, 3) + bytes([1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0]) + payload = b'\xff\xd8\xff\xc2' + struct.pack('>H', len(sof) + 2) + sof + info = self.module.fixture_info(payload, 'image/jpeg') + self.assertEqual((info['width'], info['height'], info['components']), (8000, 5000, 3)) + self.assertTrue(info['progressive']) + self.assertEqual(info['sampling'], [0x11, 0x11, 0x11]) + + def test_fixture_byte_cap_is_checked_before_reading(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'oversized.png' + with path.open('wb') as out: + out.truncate(10 * 1024 * 1024 + 1) + with self.assertRaises(ValueError): + self.module.read_fixture(path, 'image/png') diff --git a/scripts/lab4/test_upload_memory.py b/scripts/lab4/test_upload_memory.py new file mode 100644 index 00000000..63c75620 --- /dev/null +++ b/scripts/lab4/test_upload_memory.py @@ -0,0 +1,139 @@ +"""Offline checks for reproducible PNGs and Docker memory evidence.""" +import importlib.util +import json +from pathlib import Path +import struct +import unittest +from types import SimpleNamespace +import io +import zlib + + +class UploadMemoryTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + spec = importlib.util.spec_from_file_location('upload_memory', Path(__file__).with_name('upload-memory.py')) + cls.module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cls.module) + + def inspect_png(self, payload, width, height): + self.assertEqual(payload[:8], b'\x89PNG\r\n\x1a\n') + pos, chunks, compressed = 8, [], bytearray() + while pos < len(payload): + size = struct.unpack_from('>I', payload, pos)[0] + kind = payload[pos + 4:pos + 8] + data = payload[pos + 8:pos + 8 + size] + crc = struct.unpack_from('>I', payload, pos + 8 + size)[0] + self.assertEqual(crc, zlib.crc32(kind + data) & 0xffffffff) + if kind == b'IHDR': + self.assertEqual(struct.unpack('>IIBBBBB', data), (width, height, 8, 2, 0, 0, 0)) + if kind == b'IDAT': + compressed.extend(data) + chunks.append(kind) + pos += size + 12 + self.assertEqual(pos, len(payload)) + self.assertEqual(chunks[0], b'IHDR') + self.assertEqual(chunks[-1], b'IEND') + self.assertIn(b'npAD', chunks) + rows = zlib.decompress(compressed) + self.assertEqual(len(rows), height * (width * 3 + 1)) + self.assertTrue(all(rows[row * (width * 3 + 1)] == 0 for row in range(height))) + + def test_seeded_png_has_correct_crc_pixels_and_exact_size(self): + first = self.module.make_png(32, 24, 4096, 123) + self.assertEqual(len(first), 4096) + self.assertEqual(first, self.module.make_png(32, 24, 4096, 123)) + self.assertNotEqual(first, self.module.make_png(32, 24, 4096, 124)) + self.inspect_png(first, 32, 24) + + def test_both_experiment_payloads_decode_and_reach_exact_limits(self): + for name, width, height, size, seed in self.module.CASES: + with self.subTest(name=name): + payload = self.module.make_png(width, height, size, seed) + self.assertEqual(len(payload), size) + self.assertLessEqual(len(payload), 10 * 1024 * 1024) + self.assertLessEqual(width * height, 40_000_000) + self.inspect_png(payload, width, height) + + def test_impossible_png_size_is_rejected(self): + with self.assertRaises(ValueError): + self.module.make_png(32, 32, 64, 1) + + def test_docker_memory_uses_binary_and_decimal_units(self): + for raw, expected in [('1.5MiB / 512MiB', 1572864), ('2MB / 512MB', 2000000), ('4096B / 1GiB', 4096)]: + self.assertEqual(self.module.memory_bytes(raw), expected) + with self.assertRaises(ValueError): + self.module.memory_bytes('not available') + + def test_memory_summary_separates_baseline_and_load_for_each_replica(self): + samples = [ + {'container_id': 'a', 'phase': 'baseline', 'used_bytes': 10}, + {'container_id': 'b', 'phase': 'baseline', 'used_bytes': 20}, + {'container_id': 'a', 'phase': 'baseline', 'used_bytes': 12}, + {'container_id': 'a', 'phase': 'load', 'used_bytes': 35}, + {'container_id': 'a', 'phase': 'load', 'used_bytes': 30}, + {'container_id': 'b', 'phase': 'load', 'used_bytes': 50}, + ] + result = self.module.summarize_memory(samples, ['a', 'b']) + self.assertEqual(result['a']['baseline_bytes'], 12) + self.assertEqual(result['a']['sampled_peak_bytes'], 35) + self.assertEqual(result['a']['delta_bytes'], 23) + self.assertEqual(result['b']['sampled_peak_bytes'], 50) + with self.assertRaises(RuntimeError): + self.module.summarize_memory(samples, ['a', 'b', 'missing']) + + def test_latency_percentiles_and_overlap_are_explicit(self): + self.assertEqual(self.module.latency_summary(list(range(1, 11))), + {'p50_ms': 5.5, 'p95_ms': 10, 'max_ms': 10}) + self.assertEqual(self.module.max_inflight([(0, 4), (1, 3), (2, 5)]), 3) + self.assertEqual(self.module.max_inflight([(0, 1), (1, 2)]), 1) + + def test_docker_stream_ignores_ansi_only_lines_between_samples(self): + cid = 'a' * 64 + row = json.dumps({'ID': cid, 'Name': 'lab4-api-1', 'MemUsage': '12MiB / 512MiB'}) + stats = self.module.DockerStats([cid]) + stats.process = SimpleNamespace(stdout=iter(['\x1b[H' + row + '\n', ' \x1b[K\n', row + '\n'])) + stats.mark('small', 'baseline') + stats._read() + self.assertIsNone(stats.error) + self.assertEqual(len(stats.samples), 2) + + def test_rgba64_profile_has_valid_header_crc_and_zero_scanlines(self): + payload = self.module.make_rgba64_png(8, 5) + pos, compressed = 8, bytearray() + while pos < len(payload): + size = struct.unpack_from('>I', payload, pos)[0] + kind, data = payload[pos + 4:pos + 8], payload[pos + 8:pos + 8 + size] + self.assertEqual(struct.unpack_from('>I', payload, pos + 8 + size)[0], zlib.crc32(kind + data) & 0xffffffff) + if kind == b'IHDR': + self.assertEqual(struct.unpack('>IIBBBBB', data), (8, 5, 16, 6, 0, 0, 0)) + if kind == b'IDAT': + compressed.extend(data) + pos += size + 12 + self.assertEqual(zlib.decompress(compressed), bytes(5 * (1 + 8 * 8))) + self.assertEqual(pos, len(payload)) + + def test_max_pixels_profile_fits_byte_limit_without_full_raw_buffer(self): + payload = self.module.make_rgba64_png(8000, 5000) + self.assertLess(len(payload), 10 * 1024 * 1024) + self.assertEqual(struct.unpack_from('>IIBB', payload, 16), (8000, 5000, 16, 6)) + + def test_upload_uses_requested_jpeg_mime(self): + captured = [] + class Opener: + def open(self, request, timeout): + captured.append(request) + response = io.BytesIO(b'') + response.code, response.headers = 415, {} + return response + client = SimpleNamespace(base='http://localhost', opener=Opener(), timeout=1) + import threading + result = self.module.upload(None, client, {'token': 'synthetic'}, {'id': 'chat'}, b'jpeg', 'hash', + threading.Barrier(1), 0, 'decoder', mime='image/jpeg') + self.assertEqual(captured[0].get_header('Content-type'), 'image/jpeg') + self.assertTrue(captured[0].full_url.endswith('decoder-0.jpg')) + self.assertEqual(result['status'], 415) + + +if __name__ == '__main__': + unittest.main() diff --git a/scripts/lab4/upload-memory.py b/scripts/lab4/upload-memory.py new file mode 100644 index 00000000..81515831 --- /dev/null +++ b/scripts/lab4/upload-memory.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 +"""Concurrent upload experiment; stdlib only, no stand lifecycle operations.""" + +import argparse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +import datetime as dt +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import random +import re +import statistics +import struct +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +from urllib.parse import urlsplit +import zlib + + +ROOT = Path(__file__).resolve().parents[2] +sys.dont_write_bytecode = True + +CASES = ( + ('small', 512, 512, 1024 * 1024, 41), + ('limit', 1600, 1600, 10 * 1024 * 1024, 42), +) +MAXPIXEL_CASE = ('maxpixels', 8000, 5000, None, None) + + +def png_chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff) + + +def make_png(width, height, size, seed): + rng = random.Random(seed) + rows = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height)) + header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0) + prefix = b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', header) + png_chunk(b'IDAT', zlib.compress(rows, 1)) + ending = png_chunk(b'IEND', b'') + gap = size - len(prefix) - len(ending) + if gap < 0 or 0 < gap < 12: + raise ValueError('target cannot contain this PNG') + # Ancillary, private, reserved bit valid: unknown npAD is safely ignored. + padding = png_chunk(b'npAD', bytes(gap - 12)) if gap else b'' + return prefix + padding + ending + + +def make_rgba64_png(width, height): + encoder = zlib.compressobj(1) + row = bytes(1 + width * 8) + pieces = [encoder.compress(row) for _ in range(height)] + pieces.append(encoder.flush()) + header = struct.pack('>IIBBBBB', width, height, 16, 6, 0, 0, 0) + return (b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', header) + + png_chunk(b'IDAT', b''.join(pieces)) + png_chunk(b'IEND', b'')) + + +def payload_for(case): + name, width, height, size, seed = case + return make_rgba64_png(width, height) if name == 'maxpixels' else make_png(width, height, size, seed) + + +def memory_bytes(value): + match = re.fullmatch(r'([0-9]+(?:\.[0-9]+)?)\s*(B|kB|KB|KiB|MB|MiB|GB|GiB|TB|TiB)', value.split('/')[0].strip()) + if not match: + raise ValueError('unrecognized docker memory value') + units = {'B': 1, 'kB': 1000, 'KB': 1000, 'KiB': 1024, 'MB': 1000**2, 'MiB': 1024**2, + 'GB': 1000**3, 'GiB': 1024**3, 'TB': 1000**4, 'TiB': 1024**4} + return round(float(match[1]) * units[match[2]]) + + +def summarize_memory(samples, container_ids): + result = {} + for container_id in container_ids: + baseline = [s for s in samples if s['container_id'] == container_id and s['phase'] == 'baseline'] + load = [s for s in samples if s['container_id'] == container_id and s['phase'] == 'load'] + if not baseline or not load: + raise RuntimeError('missing baseline or load sample for an API replica') + before, peak = baseline[-1]['used_bytes'], max(s['used_bytes'] for s in load) + result[container_id] = {'baseline_bytes': before, 'sampled_peak_bytes': peak, 'delta_bytes': peak - before, + 'baseline_samples': len(baseline), 'load_samples': len(load)} + return result + + +def latency_summary(values): + ordered = sorted(values) + return {'p50_ms': statistics.median(ordered), 'p95_ms': ordered[math.ceil(len(ordered) * .95) - 1], + 'max_ms': ordered[-1]} + + +def max_inflight(intervals): + active, peak = 0, 0 + events = [(start, 1) for start, _ in intervals] + [(end, -1) for _, end in intervals] + for _, delta in sorted(events): + active += delta + peak = max(peak, active) + return peak + + +class DockerStats: + def __init__(self, container_ids): + self.ids = container_ids + self.samples = [] + self.lock = threading.Condition() + self.case, self.phase = '', 'idle' + self.error = None + self.started = time.monotonic() + + def __enter__(self): + self.process = subprocess.Popen(['docker', 'stats', '--no-trunc', '--format', '{{json .}}', *self.ids], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + text=True, encoding='utf-8', errors='replace', bufsize=1) + self.thread = threading.Thread(target=self._read, daemon=True) + self.thread.start() + return self + + def _read(self): + for line in self.process.stdout: + try: + cleaned = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line).strip() + if not cleaned: + continue + raw = json.loads(cleaned) + reported_id = raw.get('ID') or raw.get('Container', '') + container_id = next(cid for cid in self.ids if cid.startswith(reported_id) or reported_id.startswith(cid)) + used = memory_bytes(raw['MemUsage']) + limit = memory_bytes(raw['MemUsage'].split('/')[1]) + if used == 0: + continue + with self.lock: + self.samples.append({'container_id': container_id, 'name': raw['Name'], 'used_bytes': used, + 'limit_bytes': limit, 'elapsed_s': round(time.monotonic() - self.started, 6), + 'case': self.case, 'phase': self.phase}) + self.lock.notify_all() + except (ValueError, KeyError, IndexError, StopIteration) as exc: + with self.lock: + self.error = 'invalid_docker_stats_sample:' + type(exc).__name__ + ':' + repr(line[:120]) + self.lock.notify_all() + with self.lock: + self.lock.notify_all() + + def mark(self, case, phase): + with self.lock: + self.case, self.phase = case, phase + + def snapshot(self, case): + with self.lock: + return [dict(s) for s in self.samples if s['case'] == case] + + def wait_for_baseline(self, case, timeout=20): + deadline = time.monotonic() + timeout + with self.lock: + while True: + seen = {s['container_id'] for s in self.samples if s['case'] == case and s['phase'] == 'baseline'} + if seen == set(self.ids): + return + if self.error or self.process.poll() is not None or time.monotonic() >= deadline: + raise RuntimeError('docker stats baseline unavailable') + self.lock.wait(min(.1, max(0, deadline - time.monotonic()))) + + def __exit__(self, *_): + if self.process.poll() is None: + self.process.terminate() + try: + self.process.wait(timeout=5) + except subprocess.TimeoutExpired: + self.process.kill() + self.process.wait(timeout=5) + self.thread.join(timeout=5) + self.process.stdout.close() + + +def api_containers(args): + command = ['docker', 'compose', '--project-directory', str(ROOT), '-f', str(args.compose_file)] + if args.project_name: + command.extend(['--project-name', args.project_name]) + result = subprocess.run(command + ['ps', '--status', 'running', '-q', 'api'], cwd=ROOT, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, check=True, timeout=20) + ids = result.stdout.split() + if not ids or any(not re.fullmatch(r'[a-f0-9]{12,64}', cid) for cid in ids): + raise RuntimeError('compose has no running API containers') + return ids + + +def container_states(container_ids): + template = ('{"id":{{json .Id}},"oom_killed":{{json .State.OOMKilled}},' + '"running":{{json .State.Running}},"restart_count":{{json .RestartCount}},' + '"exit_code":{{json .State.ExitCode}}}') + result = subprocess.run(['docker', 'inspect', '--format', template, *container_ids], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, check=True, timeout=20) + return {state['id']: state for state in (json.loads(line) for line in result.stdout.splitlines())} + + +def upload(client_module, client, actor, chat, content, checksum, barrier, index, case, mime='image/png'): + result = {'index': index, 'status': None, 'ok': False, 'request_bytes': len(content)} + barrier.wait(timeout=30) + started = time.monotonic() + try: + suffix = '.jpg' if mime == 'image/jpeg' else '.png' + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + case + '-' + str(index) + suffix + req = urllib.request.Request(client.base + path, data=content, method='POST', headers={ + 'Authorization': 'Bearer ' + actor['token'], 'Content-Type': mime, + 'Content-Length': str(len(content)), 'Accept': 'application/json'}) + try: + response = client.opener.open(req, timeout=client.timeout) + except urllib.error.HTTPError as exc: + response = exc + with response: + result['status'] = response.code + result['instance_id'] = response.headers.get('X-Instance-Id') + body = response.read(65537) + if result['status'] != 201: + result['error'] = 'http_error' + elif len(body) > 65536: + result['error'] = 'oversized_metadata' + else: + attachment = json.loads(body) + client_module.shape(attachment, 'Attachment') + valid = (attachment['sha256'] == checksum and attachment['size_bytes'] == len(content) + and attachment['status'] == 'ready' and attachment['owner_id'] == actor['id'] + and attachment['chat_id'] == chat['id']) + result['ok'] = valid + result['attachment_id'] = attachment['id'] + if not valid: + result['error'] = 'attachment_metadata_mismatch' + except Exception as exc: + # Exception text and response bodies are deliberately excluded from evidence. + result['error'] = type(exc).__name__ + finished = time.monotonic() + result.update(_started=started, _finished=finished, latency_ms=round((finished - started) * 1000, 3)) + return result + + +def run_case(module, bootstrap_client, actor, reader, chat, stats, args, case): + name, width, height, size, seed = case + content = payload_for(case) + size = len(content) + checksum = hashlib.sha256(content).hexdigest() + clients = [module.Client(args.base_url, timeout=args.timeout) for _ in range(args.concurrency)] + state_before = container_states(stats.ids) + stats.mark(name, 'baseline') + stats.wait_for_baseline(name) + time.sleep(args.baseline_seconds) + barrier = threading.Barrier(args.concurrency + 1) + with ThreadPoolExecutor(max_workers=args.concurrency) as executor: + futures = [executor.submit(upload, module, client, actor, chat, content, checksum, barrier, index, name) + for index, client in enumerate(clients)] + stats.mark(name, 'load') + started = time.monotonic() + barrier.wait(timeout=30) + results = [future.result() for future in futures] + duration = time.monotonic() - started + # Retain a tail because docker stats reports its own sampled windows. + time.sleep(args.tail_seconds) + stats.mark(name, 'idle') + samples = stats.snapshot(name) + memory_error = None + try: + memory = summarize_memory(samples, stats.ids) + except RuntimeError as exc: + memory, memory_error = {}, str(exc) + intervals = [(r['_started'], r['_finished']) for r in results] + peak_requests = max_inflight(intervals) + successes = [r for r in results if r['ok']] + download = {'ok': False} + if successes: + try: + response = bootstrap_client.call('GET', '/api/v1/attachments/' + successes[0]['attachment_id'] + '/content', actor=reader) + downloaded = response['bytes'] + download = {'ok': downloaded == content and response['headers'].get_content_type() == 'image/png', + 'bytes': len(downloaded), 'sha256': hashlib.sha256(downloaded).hexdigest(), + 'status': response['status']} + except Exception as exc: + download['error'] = type(exc).__name__ + distinct = len({r['attachment_id'] for r in successes}) + state_after = container_states(stats.ids) + stable = all(state_after[cid]['running'] and not state_after[cid]['oom_killed'] + and state_after[cid]['restart_count'] == state_before[cid]['restart_count'] for cid in stats.ids) + for result in results: + result['start_offset_ms'] = round((result.pop('_started') - started) * 1000, 3) + result.pop('_finished') + return {'name': name, 'width': width, 'height': height, 'seed': seed, 'payload_bytes': size, 'sha256': checksum, + 'bit_depth': 16 if name == 'maxpixels' else 8, 'color_type': 6 if name == 'maxpixels' else 2, + 'nominal_decoded_bytes': width * height * (8 if name == 'maxpixels' else 4), + 'requests': args.concurrency, 'attempted_bytes': size * args.concurrency, + 'successful_bytes': size * len(successes), 'accepted': len(successes), + 'status_counts': dict(Counter(str(r['status']) if r['status'] else 'transport_error' for r in results)), + 'errors': len(results) - len(successes), 'duration_s': round(duration, 6), + 'observed_max_inflight': peak_requests, 'distinct_attachment_ids': distinct, + 'latency': latency_summary([r['latency_ms'] for r in results]), 'memory': memory, + 'memory_samples': samples, 'memory_error': memory_error, 'results': results, 'download': download, + 'state_before': state_before, 'state_after': state_after, 'restart_or_oom_detected': not stable, + 'ok': (len(successes) == args.concurrency and distinct == args.concurrency and peak_requests >= 10 + and download['ok'] and not memory_error and stable)} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, epilog='On Windows, run python -X utf8 -B to match tests/client.py contract-file encoding.') + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--compose-file', type=Path, default=ROOT / 'compose.yaml') + parser.add_argument('--project-name', default=None) + parser.add_argument('--output', type=Path, help='write sanitized JSON evidence here') + parser.add_argument('--profile', choices=('standard', 'maxpixels'), default='standard', + help='standard: 1/10 MiB; maxpixels: 8000x5000 RGBA64 (320 MB decoded)') + parser.add_argument('--concurrency', type=int, default=10, help='simultaneous uploads per case, 10-100 (default 10)') + parser.add_argument('--timeout', type=float, default=45) + parser.add_argument('--baseline-seconds', type=float, default=3) + parser.add_argument('--tail-seconds', type=float, default=3) + parser.add_argument('--check-generator', action='store_true', help='check payload construction only; no HTTP or Docker') + args = parser.parse_args() + selected_cases = CASES if args.profile == 'standard' else (MAXPIXEL_CASE,) + if args.check_generator: + print(json.dumps([{'case': name, 'width': width, 'height': height, 'bytes': len(payload), + 'sha256': hashlib.sha256(payload).hexdigest()} + for name, width, height, size, seed in selected_cases + for payload in [payload_for((name, width, height, size, seed))]], indent=2)) + return 0 + if not args.output: + parser.error('--output is required for live execution') + if not 10 <= args.concurrency <= 100 or args.timeout <= 0 or args.baseline_seconds < 1 or args.tail_seconds < 2: + parser.error('require concurrency 10-100, timeout > 0, baseline >= 1 s and tail >= 2 s') + target = urlsplit(args.base_url) + if target.scheme not in ('http', 'https') or not target.hostname or target.username or target.password or target.query or target.fragment: + parser.error('--base-url must be HTTP(S) without credentials, query or fragment') + report = {'schema_version': 1, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(), 'ok': False, + 'driver': {'os': platform.system(), 'architecture': platform.machine(), + 'logical_cpus': os.cpu_count(), 'python': platform.python_version()}, + 'measurement': {'source': 'docker stats JSON stream', 'cadence': 'Docker-managed, typically about 1 second', + 'limitation': 'sampled peaks may miss sub-second maxima; includes post-upload sampling tail', + 'memory_semantics': 'Docker CLI memory usage (working set, cache excluded where supported)', + 'baseline_seconds': args.baseline_seconds, 'tail_seconds': args.tail_seconds, + 'same_bytes_per_case': True, 'p95_method': 'nearest rank', + 'profile': args.profile, + 'png_compression': ('zlib level 1; random RGB; npAD padding to exact byte limit' if args.profile == 'standard' + else 'streamed zero RGBA64 scanlines; no full decoded buffer in driver')}, 'cases': []} + try: + sys.path.insert(0, str(ROOT / 'tests')) + import client as client_module + ids = api_containers(args) + report['api_container_ids'] = ids + bootstrap_client = client_module.Client(args.base_url, timeout=args.timeout) + report['stage'] = 'bootstrap' + actor, reader, _, chat = bootstrap_client.bootstrap() + with DockerStats(ids) as stats: + for case in selected_cases: + report['stage'] = case[0] + report['cases'].append(run_case(client_module, bootstrap_client, actor, reader, chat, stats, args, case)) + if stats.error: + report['sampling_error'] = stats.error + report['ok'] = all(case['ok'] for case in report['cases']) and len(report['cases']) == len(selected_cases) and not stats.error + report['stage'] = 'complete' + except Exception as exc: + report['error'] = type(exc).__name__ + if isinstance(exc, RuntimeError) or ('client_module' in locals() and isinstance(exc, client_module.Failure)): + # Public helper diagnostics contain only paths/schema names, never values or credentials. + report['failure'] = str(exc) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + print(json.dumps({'ok': report['ok'], 'output': str(args.output), + 'cases': [{'case': c['name'], 'accepted': c['accepted'], 'requests': c['requests'], + 'observed_max_inflight': c['observed_max_inflight'], 'latency': c['latency'], + 'memory': c['memory'], 'download_ok': c['download']['ok']} for c in report['cases']], + 'error': report.get('error')})) + return 0 if report['ok'] else 1 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/tools/reconcilebench/main.go b/tools/reconcilebench/main.go new file mode 100644 index 00000000..b1b3ab47 --- /dev/null +++ b/tools/reconcilebench/main.go @@ -0,0 +1,394 @@ +// Command reconcilebench measures reconciliation against disposable real +// PostgreSQL and S3 services, independently of the compose stand. +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "os/signal" + "path/filepath" + "runtime" + "strings" + "syscall" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "messenger/internal/attachment" + "messenger/internal/testenv" +) + +func main() { + o, err := parseOptions(os.Args[1:], os.Stderr) + if errors.Is(err, flag.ErrHelp) { + return + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + ctx, cancel := context.WithTimeout(ctx, 30*time.Minute) + var records bytes.Buffer + var resultOut io.Writer = os.Stdout + if o.output != "" { + resultOut = io.MultiWriter(os.Stdout, &records) + } + err = exercise(ctx, o, resultOut, os.Stderr) + if o.output != "" { + if saveErr := writeResults(o.output, records.Bytes()); saveErr != nil { + err = errors.Join(err, fmt.Errorf("save results: %w", saveErr)) + } + } + cancel() + stop() + if err != nil { + fmt.Fprintln(os.Stderr, "reconcilebench:", err) + os.Exit(1) + } +} + +func writeResults(path string, records []byte) error { + decoder := json.NewDecoder(bytes.NewReader(records)) + rows := make([]json.RawMessage, 0) + for { + var row json.RawMessage + err := decoder.Decode(&row) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return err + } + rows = append(rows, row) + } + raw, err := json.MarshalIndent(rows, "", " ") + if err != nil { + return err + } + if err = os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return err + } + return os.WriteFile(path, append(raw, '\n'), 0644) +} + +type measurement struct { + Phase string `json:"phase"` + Mode string `json:"mode"` + TargetFiles int `json:"target_files"` + GoHeapPeakBytes uint64 `json:"go_heap_peak_bytes"` + Stats attachment.Stats `json:"stats"` + Findings []attachment.Finding `json:"findings,omitempty"` +} + +// Heap samples cover this Go process during Run. They exclude service processes +// and are deliberately not described as process RSS or total deployment memory. +func measure(ctx context.Context, r *attachment.Reconciler, phase, mode string, n int, apply, deep bool, out io.Writer) (attachment.Stats, error) { + done := make(chan struct{}) + peakCh := make(chan uint64, 1) + go func() { + var mem runtime.MemStats + runtime.ReadMemStats(&mem) + peak := mem.HeapAlloc + ticker := time.NewTicker(10 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ticker.C: + runtime.ReadMemStats(&mem) + if mem.HeapAlloc > peak { + peak = mem.HeapAlloc + } + case <-done: + runtime.ReadMemStats(&mem) + if mem.HeapAlloc > peak { + peak = mem.HeapAlloc + } + peakCh <- peak + return + } + } + }() + m := measurement{Phase: phase, Mode: mode, TargetFiles: n} + stats, err := r.Run(ctx, apply, deep, func(f attachment.Finding) { m.Findings = append(m.Findings, f) }) + close(done) + m.GoHeapPeakBytes = <-peakCh + m.Stats = stats + if writeErr := json.NewEncoder(out).Encode(m); writeErr != nil { + return stats, writeErr + } + return stats, err +} + +func exercise(ctx context.Context, o options, out, progress io.Writer) error { + db, err := testenv.StartPostgres(ctx) + if err != nil { + return err + } + defer db.Close() + s3, err := testenv.StartS3(ctx) + if err != nil { + return err + } + defer s3.Close() + if err = json.NewEncoder(out).Encode(map[string]any{ + "phase": "environment", "postgres_image": testenv.PostgresImage, "s3_image": testenv.S3Image, + "grace_ms": o.grace.Milliseconds(), "wait_ms": o.wait.Milliseconds(), + "memory_metric": "sampled Go process HeapAlloc bytes during Run; not RSS; excludes PostgreSQL and S3", + }); err != nil { + return err + } + f, err := seedFaults(ctx, db.Pool, s3.Store) + if err != nil { + return err + } + fmt.Fprintf(progress, "fault objects seeded; waiting %s for real S3 timestamps to age\n", o.wait) + if err = waitForAge(ctx, o.wait, progress); err != nil { + return err + } + if err = f.seedFresh(ctx); err != nil { + return err + } + r, err := attachment.NewReconciler(db.Pool, s3.Store, o.grace) + if err != nil { + return err + } + for _, run := range []struct { + mode string + apply bool + repairs, findings int64 + }{{"dry", false, 0, 6}, {"apply", true, 5, 6}, {"repeat", true, 0, 1}} { + stats, err := measure(ctx, r, "fault", run.mode, 7, run.apply, true, out) + if err != nil { + return err + } + if stats.Repaired != run.repairs || stats.Findings != run.findings { + return fmt.Errorf("fault %s: wanted %d repairs/%d findings, got %+v", run.mode, run.repairs, run.findings, stats) + } + if err = f.assertState(ctx, run.apply); err != nil { + return fmt.Errorf("fault %s: %w", run.mode, err) + } + } + fmt.Fprintln(progress, "fault dry/apply/repeat passed; referenced messages, corrupt used blobs and fresh objects preserved") + // These resources were created above solely for this run; no compose + // endpoint or pre-existing database/bucket is ever accepted by this tool. + if _, err = db.Pool.Exec(ctx, `TRUNCATE message_attachments,attachments,files`); err != nil { + return err + } + if err = emptyBucket(ctx, s3.Store); err != nil { + return err + } + seeded := 0 + for _, n := range o.sizes { + for i := seeded; i < n; i++ { + if err = seedScaleFile(ctx, db.Pool, s3.Store, f.user, f.chat, i); err != nil { + return err + } + if (i+1)%500 == 0 { + fmt.Fprintf(progress, "seeded %d referenced real objects\n", i+1) + } + } + seeded = n + stats, err := measure(ctx, r, "cost", "dry", n, false, false, out) + if err != nil { + return err + } + if stats.ScannedFiles != int64(n) || stats.ScannedObjects != int64(n) || stats.Findings != 0 || stats.Repaired != 0 { + return fmt.Errorf("cost %d: inconsistent fixture %+v", n, stats) + } + } + return json.NewEncoder(out).Encode(map[string]any{"phase": "complete", "fault_checks": "passed", "sizes": o.sizes}) +} + +func waitForAge(ctx context.Context, d time.Duration, progress io.Writer) error { + start := time.Now() + timer := time.NewTimer(d) + defer timer.Stop() + ticker := time.NewTicker(30 * time.Second) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + case <-ticker.C: + fmt.Fprintf(progress, "aging real S3 objects: %s / %s\n", time.Since(start).Round(time.Second), d) + } + } +} + +type benchFile struct { + hash []byte + before, after string +} +type benchObject struct { + key string + after bool +} +type faultFixture struct { + db *pgxpool.Pool + blobs attachment.Blobs + user, chat uuid.UUID + files []benchFile + objects []benchObject +} + +func seedFaults(ctx context.Context, db *pgxpool.Pool, blobs attachment.Blobs) (*faultFixture, error) { + f := &faultFixture{db: db, blobs: blobs, user: uuid.New(), chat: uuid.New()} + if _, err := db.Exec(ctx, `INSERT INTO users(id,username,display_name,password_hash,created_at) VALUES($1,$2,'bench','unused',now())`, f.user, "bench-"+f.user.String()); err != nil { + return nil, err + } + if _, err := db.Exec(ctx, `INSERT INTO chats(id,title,created_at) VALUES($1,'reconcilebench',now())`, f.chat); err != nil { + return nil, err + } + if err := f.put(ctx, "staging/old", []byte("stale staging"), false); err != nil { + return nil, err + } + orphan := sha256.Sum256([]byte("old orphan")) + if err := f.put(ctx, attachment.BlobKey(orphan[:]), []byte("old orphan"), false); err != nil { + return nil, err + } + for _, c := range []struct { + label, status, after string + used, present, corrupt bool + }{ + {"pending", "pending", "", false, true, false}, + {"unused", "ready", "", false, true, false}, + {"used", "ready", "ready", true, true, false}, + {"corrupt", "ready", "ready", true, true, true}, + {"missing", "ready", "failed", true, false, false}, + {"already_failed", "failed", "failed", true, false, false}, + } { + if err := f.file(ctx, c.label, c.status, c.after, c.used, c.present, c.corrupt, false); err != nil { + return nil, err + } + } + return f, nil +} + +func (f *faultFixture) put(ctx context.Context, key string, body []byte, keep bool) error { + if err := f.blobs.Put(ctx, key, bytes.NewReader(body), int64(len(body)), "image/png"); err != nil { + return err + } + f.objects = append(f.objects, benchObject{key: key, after: keep}) + return nil +} + +func (f *faultFixture) file(ctx context.Context, label, status, after string, used, present, corrupt, fresh bool) error { + body := []byte("fixture/" + label) + hash := sha256.Sum256(body) + created := time.Now().Add(-2 * time.Hour) + if fresh { + created = time.Now() + } + if _, err := f.db.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status,created_at) VALUES($1,$2,'image/png',$3,$4)`, hash[:], len(body), status, created); err != nil { + return err + } + f.files = append(f.files, benchFile{hash: hash[:], before: status, after: after}) + if used { + id, message := uuid.New(), uuid.New() + if _, err := f.db.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,'fixture.png',$4,now())`, id, f.chat, f.user, hash[:]); err != nil { + return err + } + if _, err := f.db.Exec(ctx, `INSERT INTO messages(chat_id,id,sender_id,text,created_at) VALUES($1,$2,$3,'preserve',now())`, f.chat, message, f.user); err != nil { + return err + } + if _, err := f.db.Exec(ctx, `INSERT INTO message_attachments(chat_id,message_id,attachment_id,ordinal) VALUES($1,$2,$3,0)`, f.chat, message, id); err != nil { + return err + } + } + if present { + if corrupt { + body[0] ^= 1 + } + return f.put(ctx, attachment.BlobKey(hash[:]), body, after != "") + } + return nil +} + +func (f *faultFixture) seedFresh(ctx context.Context) error { + if err := f.put(ctx, "staging/fresh", []byte("fresh staging"), true); err != nil { + return err + } + orphan := sha256.Sum256([]byte("fresh orphan")) + if err := f.put(ctx, attachment.BlobKey(orphan[:]), []byte("fresh orphan"), true); err != nil { + return err + } + return f.file(ctx, "fresh pending", "pending", "pending", false, true, false, true) +} + +func (f *faultFixture) assertState(ctx context.Context, applied bool) error { + for _, file := range f.files { + var status string + err := f.db.QueryRow(ctx, `SELECT status FROM files WHERE hash=$1`, file.hash).Scan(&status) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return err + } + want := file.before + if applied { + want = file.after + } + if status != want { + return fmt.Errorf("file %s: status=%q, want %q", attachment.BlobKey(file.hash), status, want) + } + } + for _, obj := range f.objects { + _, err := f.blobs.Stat(ctx, obj.key) + want := !applied || obj.after + if err != nil && !errors.Is(err, attachment.ErrObjectMissing) { + return err + } + if (err == nil) != want { + return fmt.Errorf("object %s: present=%v want %v", obj.key, err == nil, want) + } + } + var attachments, links int + if err := f.db.QueryRow(ctx, `SELECT (SELECT count(*) FROM attachments),(SELECT count(*) FROM message_attachments)`).Scan(&attachments, &links); err != nil { + return err + } + if attachments != 4 || links != 4 { + return fmt.Errorf("referenced attachments/messages lost: %d/%d", attachments, links) + } + return nil +} + +func emptyBucket(ctx context.Context, blobs attachment.Blobs) error { + for { + page, err := blobs.List(ctx, "", "", 500) + if err != nil { + return err + } + if len(page) == 0 { + return nil + } + for _, obj := range page { + if err := blobs.Delete(ctx, obj.Key); err != nil { + return err + } + } + } +} + +func seedScaleFile(ctx context.Context, db *pgxpool.Pool, blobs attachment.Blobs, user, chat uuid.UUID, i int) error { + body := fmt.Sprintf("scale/%09d", i) + hash := sha256.Sum256([]byte(body)) + if err := blobs.Put(ctx, attachment.BlobKey(hash[:]), strings.NewReader(body), int64(len(body)), "image/png"); err != nil { + return err + } + _, err := db.Exec(ctx, `WITH f AS ( + INSERT INTO files(hash,size_bytes,mime_type,status,created_at) + VALUES($1,$2,'image/png','ready',now()-interval '2 hour') RETURNING hash) + INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) + SELECT $3,$4,$5,'scale.png',hash,now() FROM f`, hash[:], len(body), uuid.New(), chat, user) + return err +} diff --git a/tools/reconcilebench/main_test.go b/tools/reconcilebench/main_test.go new file mode 100644 index 00000000..e77ecf01 --- /dev/null +++ b/tools/reconcilebench/main_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "encoding/json" + "io" + "os" + "path/filepath" + "reflect" + "testing" + "time" +) + +func TestOptionsProtectRealAgingAndValidateSizes(t *testing.T) { + got, err := parseOptions(nil, io.Discard) + if err != nil || got.grace != 2*time.Minute || got.wait != 125*time.Second || !reflect.DeepEqual(got.sizes, []int{100, 1000, 5000}) { + t.Fatalf("defaults: %+v %v", got, err) + } + for _, args := range [][]string{{"--grace=1m"}, {"--wait=2m"}, {"--sizes=0"}, {"--sizes=10,10"}, {"--sizes=10,5"}, {"--sizes=oops"}, {"extra"}} { + if _, err := parseOptions(args, io.Discard); err == nil { + t.Errorf("accepted unsafe options %v", args) + } + } + got, err = parseOptions([]string{"--sizes=2,7", "--grace=3m", "--wait=3m2s"}, io.Discard) + if err != nil || !reflect.DeepEqual(got.sizes, []int{2, 7}) { + t.Fatalf("explicit options: %+v %v", got, err) + } +} + +func TestOutputOptionWritesStructuredJSON(t *testing.T) { + path := filepath.Join(t.TempDir(), "track2", "results.json") + o, err := parseOptions([]string{"--output", path}, io.Discard) + if err != nil || o.output != path { + t.Fatalf("output flag: %+v %v", o, err) + } + if err = writeResults(path, []byte("{\"phase\":\"fault\"}\n{\"phase\":\"complete\"}\n")); err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + var rows []map[string]any + if err = json.Unmarshal(raw, &rows); err != nil || len(rows) != 2 || rows[1]["phase"] != "complete" { + t.Fatalf("results JSON: %s %v", raw, err) + } +} diff --git a/tools/reconcilebench/options.go b/tools/reconcilebench/options.go new file mode 100644 index 00000000..5e6344e8 --- /dev/null +++ b/tools/reconcilebench/options.go @@ -0,0 +1,43 @@ +package main + +import ( + "errors" + "flag" + "fmt" + "io" + "strconv" + "strings" + "time" +) + +type options struct { + grace, wait time.Duration + sizes []int + output string +} + +func parseOptions(args []string, out io.Writer) (options, error) { + var o options + f := flag.NewFlagSet("reconcilebench", flag.ContinueOnError) + f.SetOutput(out) + f.DurationVar(&o.grace, "grace", 2*time.Minute, "reconciler grace (minimum 2m)") + f.DurationVar(&o.wait, "wait", 125*time.Second, "actual S3 aging wait (must exceed grace by at least 1s)") + f.StringVar(&o.output, "output", "", "also save measurement records as a JSON array at this path") + sizes := f.String("sizes", "100,1000,5000", "strictly increasing referenced object counts") + if err := f.Parse(args); err != nil { + return o, err + } + if f.NArg() != 0 || o.grace < 2*time.Minute || o.wait-o.grace < time.Second { + return o, errors.New("no positional arguments; grace >= 2m and wait >= grace+1s required") + } + last := 0 + for _, raw := range strings.Split(*sizes, ",") { + n, err := strconv.Atoi(strings.TrimSpace(raw)) + if err != nil || n <= last { + return o, fmt.Errorf("sizes must be strictly increasing positive integers: %q", raw) + } + o.sizes = append(o.sizes, n) + last = n + } + return o, nil +}