Добавьте воспроизводимые эксперименты по памяти и согласованности

This commit is contained in:
2026-11-16 12:37:21 +03:00
parent d761861def
commit 29f45ca0eb
10 changed files with 1416 additions and 0 deletions
+14
View File
@@ -51,6 +51,7 @@ secrets:
up:
docker compose up -d --build --wait
sh scripts/wait-ready.sh
# Stops and removes the containers; volumes and therefore data are kept.
down:
@@ -63,6 +64,19 @@ destroy:
logs:
docker compose logs
# Safe default: inspection only; APPLY=1 explicitly enables repairs.
APPLY ?= 0
DEEP ?= 0
.PHONY: reconcile upload-memory reconcile-bench
reconcile:
docker compose --profile recon run --rm reconcile $(if $(filter 1,$(APPLY)),--apply,) $(if $(filter 1,$(DEEP)),--deep,)
upload-memory:
$(PYTHON) -X utf8 scripts/lab4/upload-memory.py --base-url "$(BASE_URL)" --output "$(EVIDENCE_DIR)/upload-memory.json"
reconcile-bench:
go run ./tools/reconcilebench --output "$(EVIDENCE_DIR)/track2/results.json"
# Which container answers under which X-Instance-Id.
instances:
sh scripts/instances.sh
+209
View File
@@ -0,0 +1,209 @@
#!/usr/bin/env python3
"""File-driven decoder stress evidence. Requires rebuilt APIs with >=2 GiB limits.
Run with python -X utf8 -B. Default is one upload; no container lifecycle actions.
"""
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import datetime as dt
import hashlib
import importlib.util
import json
from pathlib import Path
import re
import struct
import subprocess
import sys
import threading
import time
from urllib.parse import urlsplit
import zlib
sys.dont_write_bytecode = True
_spec = importlib.util.spec_from_file_location('upload_memory', Path(__file__).with_name('upload-memory.py'))
memory = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(memory)
def write_rgba64_png(path, width=40_000_000, height=1, chunk_bytes=65536):
"""Stream zero scanlines, including filter bytes; never allocate a full row."""
if min(width, height, chunk_bytes) <= 0 or width * height > 40_000_000:
raise ValueError('invalid stress dimensions or chunk size')
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
encoder = zlib.compressobj(1)
remaining = height * (width * 8 + 1)
zeros = bytes(min(chunk_bytes, remaining))
with path.open('wb') as out:
out.write(b'\x89PNG\r\n\x1a\n')
out.write(memory.png_chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 16, 6, 0, 0, 0)))
while remaining:
count = min(len(zeros), remaining)
compressed = encoder.compress(zeros[:count])
if compressed:
out.write(memory.png_chunk(b'IDAT', compressed))
remaining -= count
out.write(memory.png_chunk(b'IDAT', encoder.flush()))
out.write(memory.png_chunk(b'IEND', b''))
def fixture_info(content, mime):
if mime == 'image/png' and content.startswith(b'\x89PNG\r\n\x1a\n') and len(content) >= 33:
width, height, depth, color = struct.unpack_from('>IIBB', content, 16)
return {'width': width, 'height': height, 'bit_depth': depth, 'color_type': color}
if mime == 'image/jpeg' and content.startswith(b'\xff\xd8'):
pos = 2
while pos + 4 <= len(content):
if content[pos] != 255:
break
while pos < len(content) and content[pos] == 255:
pos += 1
marker = content[pos]
pos += 1
length = struct.unpack_from('>H', content, pos)[0]
frame = content[pos + 2:pos + length]
pos += length
if marker in (0xc0, 0xc1, 0xc2):
height, width = struct.unpack_from('>HH', frame, 1)
count = frame[5]
return {'width': width, 'height': height, 'bit_depth': frame[0], 'components': count,
'progressive': marker == 0xc2, 'sampling': [frame[7 + 3 * i] for i in range(count)]}
if length < 2 or marker == 0xda:
break
raise ValueError('fixture MIME/header mismatch or missing image header')
def read_fixture(path, mime):
if not 0 < path.stat().st_size <= 10 * 1024 * 1024:
raise ValueError('fixture must be between 1 byte and 10 MiB')
content = path.read_bytes()
info = fixture_info(content, mime)
if not 0 < info['width'] * info['height'] <= 40_000_000:
raise ValueError('fixture exceeds 40M pixels')
return content, info
def api_limits(ids):
template = '{"id":{{json .Id}},"limit_bytes":{{json .HostConfig.Memory}}}'
result = subprocess.run(['docker', 'inspect', '--format', template, *ids], stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL, text=True, check=True, timeout=20)
return {row['id']: row['limit_bytes'] for row in (json.loads(line) for line in result.stdout.splitlines())}
def experiment(args, content, report):
sys.path.insert(0, str(memory.ROOT / 'tests'))
import client as module
ids = memory.api_containers(args)
report['api_limits'] = api_limits(ids)
if any(report['api_limits'].get(cid, 0) < 2 * 1024**3 for cid in ids):
raise RuntimeError('stress uploads require every API container limit to be >=2 GiB')
report['state_before'] = memory.container_states(ids)
bootstrap = module.Client(args.base_url, timeout=args.timeout)
actor, reader, _, chat = bootstrap.bootstrap()
clients = [module.Client(args.base_url, timeout=args.timeout) for _ in range(args.concurrency)]
with memory.DockerStats(ids) as stats:
stats.mark(args.label, 'baseline')
stats.wait_for_baseline(args.label)
time.sleep(args.baseline_seconds)
barrier = threading.Barrier(args.concurrency + 1)
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
futures = [executor.submit(memory.upload, module, client, actor, chat, content, report['sha256'],
barrier, index, args.label, mime=args.mime) for index, client in enumerate(clients)]
stats.mark(args.label, 'load')
started = time.monotonic()
barrier.wait(timeout=30)
results = [future.result() for future in futures]
report['duration_s'] = time.monotonic() - started
time.sleep(args.tail_seconds)
stats.mark(args.label, 'idle')
report['memory_samples'] = stats.snapshot(args.label)
report['sampling_error'] = stats.error
report['state_after'] = memory.container_states(ids)
stable = all(report['state_after'][cid]['running'] and not report['state_after'][cid]['oom_killed']
and report['state_after'][cid]['restart_count'] == report['state_before'][cid]['restart_count'] for cid in ids)
report['restart_or_oom_detected'] = not stable
try:
report['memory'] = memory.summarize_memory(report['memory_samples'], ids)
except RuntimeError as exc:
report['memory_error'] = str(exc)
report['observed_max_inflight'] = memory.max_inflight([(r['_started'], r['_finished']) for r in results])
accepted = [r for r in results if r['ok']]
report['accepted'] = len(accepted)
report['status_counts'] = dict(Counter(str(r['status']) if r['status'] else 'transport_error' for r in results))
report['latency'] = memory.latency_summary([r['latency_ms'] for r in results])
report['attempted_bytes'], report['successful_bytes'] = len(content) * len(results), len(content) * len(accepted)
for result in results:
result['start_offset_ms'] = round((result.pop('_started') - started) * 1000, 3)
result.pop('_finished')
report['results'] = results
report['download'] = {'ok': False}
if accepted:
try:
response = bootstrap.call('GET', '/api/v1/attachments/' + accepted[0]['attachment_id'] + '/content', actor=reader)
body = response['bytes']
report['download'] = {'ok': body == content and response['headers'].get_content_type() == args.mime,
'status': response['status'], 'bytes': len(body), 'sha256': hashlib.sha256(body).hexdigest()}
except Exception as exc:
report['download']['error'] = str(exc) if isinstance(exc, module.Failure) else type(exc).__name__
report['ok'] = (len(accepted) == args.concurrency and report['download']['ok'] and stable
and not report.get('memory_error') and not report['sampling_error']
and report['observed_max_inflight'] == args.concurrency)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--file', type=Path)
parser.add_argument('--mime', choices=('image/png', 'image/jpeg'))
parser.add_argument('--label', default='decoder')
parser.add_argument('--concurrency', type=int, default=1, help='1-10, default single-upload acceptance')
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--compose-file', type=Path, default=memory.ROOT / 'compose.yaml')
parser.add_argument('--project-name')
parser.add_argument('--timeout', type=float, default=45)
parser.add_argument('--baseline-seconds', type=float, default=3)
parser.add_argument('--tail-seconds', type=float, default=3)
parser.add_argument('--output', type=Path)
parser.add_argument('--check-file', action='store_true', help='inspect bytes/header without HTTP or Docker')
parser.add_argument('--generate-wide', type=Path, help='generate 40Mx1 RGBA64 PNG offline using bounded zlib input')
args = parser.parse_args()
if args.generate_wide:
write_rgba64_png(args.generate_wide)
content, info = read_fixture(args.generate_wide, 'image/png')
print(json.dumps({**info, 'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()}))
return 0
if not args.file or not args.mime:
parser.error('--file and --mime are required')
if not 1 <= args.concurrency <= 10 or args.timeout <= 0 or args.baseline_seconds < 1 or args.tail_seconds < 2:
parser.error('require concurrency1-10, timeout>0, baseline>=1s, tail>=2s')
if not re.fullmatch(r'[A-Za-z0-9_-]{1,40}', args.label):
parser.error('label must be 1-40 safe ASCII characters')
url = urlsplit(args.base_url)
if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password or url.query or url.fragment:
parser.error('base URL must be HTTP(S) without credentials/query/fragment')
content, info = read_fixture(args.file, args.mime)
report = {'schema_version': 1, 'name': args.label, 'mime': args.mime, 'fixture': info,
'payload_bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest(),
'requests': args.concurrency, 'ok': False, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(),
'measurement': {'source': 'docker stats JSON stream, formatted working-set values',
'limitation': 'sampled/rounded peaks can miss instantaneous maxima',
'baseline_seconds': args.baseline_seconds, 'tail_seconds': args.tail_seconds}}
if args.check_file:
print(json.dumps(report, indent=2))
return 0
if not args.output:
parser.error('--output is required for a live run')
try:
experiment(args, content, report)
except Exception as exc:
report['error'] = type(exc).__name__
if isinstance(exc, RuntimeError) or type(exc).__name__ == 'Failure':
report['failure'] = str(exc)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
print(json.dumps({key: value for key, value in report.items() if key not in ('memory_samples', 'results')}))
return 0 if report['ok'] else 1
if __name__ == '__main__':
raise SystemExit(main())
+95
View File
@@ -0,0 +1,95 @@
"""Stop each actual uploader replica and verify its bytes through the survivor."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
def instances():
text = subprocess.check_output(['sh', 'scripts/api-instances.sh'], cwd=ROOT,
encoding='utf-8')
result = []
for line in text.splitlines():
container, replica, state, alias, image = line.split()
result.append({'container': container, 'replica': replica,
'state': state, 'alias': alias, 'image': image})
return result
def restore(c):
subprocess.run(['docker', 'compose', 'start', 'api'], cwd=ROOT,
check=True, stdout=subprocess.DEVNULL)
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
observed = {c.call('GET', '/health/live')['headers']['X-Instance-Id']
for _ in range(6)}
if len(observed) >= 2:
return
time.sleep(0.3)
raise AssertionError('two replicas did not return after restore')
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--base-url', default='http://localhost:8080')
p.add_argument('--output', type=Path, default=ROOT / 'evidence/failover-uploaders.json')
args = p.parse_args()
c = Client(args.base_url, timeout=10)
actor, reader, _, chat = c.bootstrap()
data = (ROOT / 'tests/fixtures/sample.png').read_bytes()
wanted = instances()
check(len(wanted) == 2 and all(x['state'] == 'running' for x in wanted),
'this experiment requires two running API replicas')
report = {'scenario': 'stop actual upload-serving replica; download via survivor',
'cases': [], 'ok': False}
try:
for target in wanted:
upload = None
for _ in range(10):
attempt = c.call('POST', '/api/v1/chats/' + chat['id'] +
'/attachments?filename=sample.png', 201,
actor=actor, raw=data,
headers={'Content-Type': 'image/png'}, schema='Attachment')
if attempt['headers']['X-Instance-Id'] == target['alias']:
upload = attempt
break
check(upload is not None, 'target replica did not serve an upload')
before = instances()
env = dict(os.environ, TARGET_INSTANCE_ID=target['alias'])
started = time.monotonic()
subprocess.run(['sh', 'scripts/stop-one.sh'], cwd=ROOT, env=env,
check=True, stdout=subprocess.DEVNULL, timeout=15)
stop_seconds = time.monotonic() - started
downloaded = c.call('GET', '/api/v1/attachments/' +
upload['json']['id'] + '/content', actor=reader)
survivor = downloaded['headers']['X-Instance-Id']
check(survivor != target['alias'], 'stopped uploader still served download')
check(downloaded['bytes'] == data, 'bytes changed after uploader stop')
after = instances()
check(any(x['container'] == target['container'] and x['state'] != 'running'
for x in after), 'uploader was not left stopped')
report['cases'].append({'uploader': target, 'download_instance': survivor,
'stop_seconds': round(stop_seconds, 3),
'sha256': hashlib.sha256(downloaded['bytes']).hexdigest(),
'same_bytes': True, 'before': before, 'after': after})
restore(c)
report['ok'] = True
finally:
# This script only restores the API containers that it deliberately stopped.
restore(c)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
print(json.dumps({'ok': report['ok'], 'cases': len(report['cases']),
'output': str(args.output)}))
if __name__ == '__main__':
main()
+45
View File
@@ -0,0 +1,45 @@
"""Generate optional 40 MP regression images; requires Pillow==11.1.0."""
import argparse
import hashlib
import importlib.util
import json
from pathlib import Path
import sys
sys.dont_write_bytecode = True
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output-dir', type=Path, default=Path('bin/decoder-fixtures'))
args = parser.parse_args()
try:
from PIL import Image
except ImportError:
parser.error('install optional fixture dependency: pip install Pillow==11.1.0')
spec = importlib.util.spec_from_file_location('decoder_memory',
Path(__file__).with_name('decoder-memory.py'))
decoder = importlib.util.module_from_spec(spec)
spec.loader.exec_module(decoder)
args.output_dir.mkdir(parents=True, exist_ok=True)
manifest = {'pillow_version': Image.__version__, 'fixtures': []}
for mode, color in [('RGB', (25, 90, 160)), ('CMYK', (20, 40, 60, 10))]:
path = args.output_dir / ('progressive-' + mode.lower() + '-444-40mp.jpg')
with Image.new(mode, (8000, 5000), color) as image:
image.save(path, format='JPEG', progressive=True, subsampling=0, quality=85)
content, info = decoder.read_fixture(path, 'image/jpeg')
if not info['progressive'] or any(value != 0x11 for value in info['sampling']):
raise ValueError('encoder did not produce progressive 4:4:4')
manifest['fixtures'].append({'name': path.name, 'mime': 'image/jpeg', **info,
'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()})
wide = args.output_dir / 'wide-rgba64-40mp.png'
decoder.write_rgba64_png(wide)
content, info = decoder.read_fixture(wide, 'image/png')
manifest['fixtures'].append({'name': wide.name, 'mime': 'image/png', **info,
'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()})
(args.output_dir / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n', encoding='utf-8')
print(json.dumps(manifest, indent=2))
if __name__ == '__main__':
main()
+55
View File
@@ -0,0 +1,55 @@
"""Offline checks for bounded stress fixtures and format identification."""
import importlib.util
from pathlib import Path
import struct
import tempfile
import unittest
import zlib
class DecoderMemoryTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
path = Path(__file__).with_name('decoder-memory.py')
cls.module = None
if path.exists():
spec = importlib.util.spec_from_file_location('decoder_memory', path)
cls.module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(cls.module)
def setUp(self):
self.assertIsNotNone(self.module, 'decoder-memory.py is not implemented')
def test_streamed_rgba64_png_has_valid_crc_and_zero_pixels(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / 'wide.png'
self.module.write_rgba64_png(path, width=19, height=3, chunk_bytes=7)
payload = path.read_bytes()
self.assertEqual(payload[:8], b'\x89PNG\r\n\x1a\n')
pos, compressed = 8, bytearray()
while pos < len(payload):
size = struct.unpack_from('>I', payload, pos)[0]
kind, body = payload[pos + 4:pos + 8], payload[pos + 8:pos + 8 + size]
self.assertEqual(struct.unpack_from('>I', payload, pos + 8 + size)[0], zlib.crc32(kind + body) & 0xffffffff)
if kind == b'IDAT':
compressed.extend(body)
pos += size + 12
self.assertEqual(zlib.decompress(compressed), bytes(3 * (19 * 8 + 1)))
self.assertEqual(self.module.fixture_info(payload, 'image/png'),
{'width': 19, 'height': 3, 'bit_depth': 16, 'color_type': 6})
def test_progressive_jpeg_444_components_are_identified(self):
sof = b'\x08' + struct.pack('>HHB', 5000, 8000, 3) + bytes([1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0])
payload = b'\xff\xd8\xff\xc2' + struct.pack('>H', len(sof) + 2) + sof
info = self.module.fixture_info(payload, 'image/jpeg')
self.assertEqual((info['width'], info['height'], info['components']), (8000, 5000, 3))
self.assertTrue(info['progressive'])
self.assertEqual(info['sampling'], [0x11, 0x11, 0x11])
def test_fixture_byte_cap_is_checked_before_reading(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / 'oversized.png'
with path.open('wb') as out:
out.truncate(10 * 1024 * 1024 + 1)
with self.assertRaises(ValueError):
self.module.read_fixture(path, 'image/png')
+139
View File
@@ -0,0 +1,139 @@
"""Offline checks for reproducible PNGs and Docker memory evidence."""
import importlib.util
import json
from pathlib import Path
import struct
import unittest
from types import SimpleNamespace
import io
import zlib
class UploadMemoryTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
spec = importlib.util.spec_from_file_location('upload_memory', Path(__file__).with_name('upload-memory.py'))
cls.module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(cls.module)
def inspect_png(self, payload, width, height):
self.assertEqual(payload[:8], b'\x89PNG\r\n\x1a\n')
pos, chunks, compressed = 8, [], bytearray()
while pos < len(payload):
size = struct.unpack_from('>I', payload, pos)[0]
kind = payload[pos + 4:pos + 8]
data = payload[pos + 8:pos + 8 + size]
crc = struct.unpack_from('>I', payload, pos + 8 + size)[0]
self.assertEqual(crc, zlib.crc32(kind + data) & 0xffffffff)
if kind == b'IHDR':
self.assertEqual(struct.unpack('>IIBBBBB', data), (width, height, 8, 2, 0, 0, 0))
if kind == b'IDAT':
compressed.extend(data)
chunks.append(kind)
pos += size + 12
self.assertEqual(pos, len(payload))
self.assertEqual(chunks[0], b'IHDR')
self.assertEqual(chunks[-1], b'IEND')
self.assertIn(b'npAD', chunks)
rows = zlib.decompress(compressed)
self.assertEqual(len(rows), height * (width * 3 + 1))
self.assertTrue(all(rows[row * (width * 3 + 1)] == 0 for row in range(height)))
def test_seeded_png_has_correct_crc_pixels_and_exact_size(self):
first = self.module.make_png(32, 24, 4096, 123)
self.assertEqual(len(first), 4096)
self.assertEqual(first, self.module.make_png(32, 24, 4096, 123))
self.assertNotEqual(first, self.module.make_png(32, 24, 4096, 124))
self.inspect_png(first, 32, 24)
def test_both_experiment_payloads_decode_and_reach_exact_limits(self):
for name, width, height, size, seed in self.module.CASES:
with self.subTest(name=name):
payload = self.module.make_png(width, height, size, seed)
self.assertEqual(len(payload), size)
self.assertLessEqual(len(payload), 10 * 1024 * 1024)
self.assertLessEqual(width * height, 40_000_000)
self.inspect_png(payload, width, height)
def test_impossible_png_size_is_rejected(self):
with self.assertRaises(ValueError):
self.module.make_png(32, 32, 64, 1)
def test_docker_memory_uses_binary_and_decimal_units(self):
for raw, expected in [('1.5MiB / 512MiB', 1572864), ('2MB / 512MB', 2000000), ('4096B / 1GiB', 4096)]:
self.assertEqual(self.module.memory_bytes(raw), expected)
with self.assertRaises(ValueError):
self.module.memory_bytes('not available')
def test_memory_summary_separates_baseline_and_load_for_each_replica(self):
samples = [
{'container_id': 'a', 'phase': 'baseline', 'used_bytes': 10},
{'container_id': 'b', 'phase': 'baseline', 'used_bytes': 20},
{'container_id': 'a', 'phase': 'baseline', 'used_bytes': 12},
{'container_id': 'a', 'phase': 'load', 'used_bytes': 35},
{'container_id': 'a', 'phase': 'load', 'used_bytes': 30},
{'container_id': 'b', 'phase': 'load', 'used_bytes': 50},
]
result = self.module.summarize_memory(samples, ['a', 'b'])
self.assertEqual(result['a']['baseline_bytes'], 12)
self.assertEqual(result['a']['sampled_peak_bytes'], 35)
self.assertEqual(result['a']['delta_bytes'], 23)
self.assertEqual(result['b']['sampled_peak_bytes'], 50)
with self.assertRaises(RuntimeError):
self.module.summarize_memory(samples, ['a', 'b', 'missing'])
def test_latency_percentiles_and_overlap_are_explicit(self):
self.assertEqual(self.module.latency_summary(list(range(1, 11))),
{'p50_ms': 5.5, 'p95_ms': 10, 'max_ms': 10})
self.assertEqual(self.module.max_inflight([(0, 4), (1, 3), (2, 5)]), 3)
self.assertEqual(self.module.max_inflight([(0, 1), (1, 2)]), 1)
def test_docker_stream_ignores_ansi_only_lines_between_samples(self):
cid = 'a' * 64
row = json.dumps({'ID': cid, 'Name': 'lab4-api-1', 'MemUsage': '12MiB / 512MiB'})
stats = self.module.DockerStats([cid])
stats.process = SimpleNamespace(stdout=iter(['\x1b[H' + row + '\n', ' \x1b[K\n', row + '\n']))
stats.mark('small', 'baseline')
stats._read()
self.assertIsNone(stats.error)
self.assertEqual(len(stats.samples), 2)
def test_rgba64_profile_has_valid_header_crc_and_zero_scanlines(self):
payload = self.module.make_rgba64_png(8, 5)
pos, compressed = 8, bytearray()
while pos < len(payload):
size = struct.unpack_from('>I', payload, pos)[0]
kind, data = payload[pos + 4:pos + 8], payload[pos + 8:pos + 8 + size]
self.assertEqual(struct.unpack_from('>I', payload, pos + 8 + size)[0], zlib.crc32(kind + data) & 0xffffffff)
if kind == b'IHDR':
self.assertEqual(struct.unpack('>IIBBBBB', data), (8, 5, 16, 6, 0, 0, 0))
if kind == b'IDAT':
compressed.extend(data)
pos += size + 12
self.assertEqual(zlib.decompress(compressed), bytes(5 * (1 + 8 * 8)))
self.assertEqual(pos, len(payload))
def test_max_pixels_profile_fits_byte_limit_without_full_raw_buffer(self):
payload = self.module.make_rgba64_png(8000, 5000)
self.assertLess(len(payload), 10 * 1024 * 1024)
self.assertEqual(struct.unpack_from('>IIBB', payload, 16), (8000, 5000, 16, 6))
def test_upload_uses_requested_jpeg_mime(self):
captured = []
class Opener:
def open(self, request, timeout):
captured.append(request)
response = io.BytesIO(b'')
response.code, response.headers = 415, {}
return response
client = SimpleNamespace(base='http://localhost', opener=Opener(), timeout=1)
import threading
result = self.module.upload(None, client, {'token': 'synthetic'}, {'id': 'chat'}, b'jpeg', 'hash',
threading.Barrier(1), 0, 'decoder', mime='image/jpeg')
self.assertEqual(captured[0].get_header('Content-type'), 'image/jpeg')
self.assertTrue(captured[0].full_url.endswith('decoder-0.jpg'))
self.assertEqual(result['status'], 415)
if __name__ == '__main__':
unittest.main()
+376
View File
@@ -0,0 +1,376 @@
#!/usr/bin/env python3
"""Concurrent upload experiment; stdlib only, no stand lifecycle operations."""
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import datetime as dt
import hashlib
import json
import math
import os
from pathlib import Path
import platform
import random
import re
import statistics
import struct
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
from urllib.parse import urlsplit
import zlib
ROOT = Path(__file__).resolve().parents[2]
sys.dont_write_bytecode = True
CASES = (
('small', 512, 512, 1024 * 1024, 41),
('limit', 1600, 1600, 10 * 1024 * 1024, 42),
)
MAXPIXEL_CASE = ('maxpixels', 8000, 5000, None, None)
def png_chunk(kind, data):
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff)
def make_png(width, height, size, seed):
rng = random.Random(seed)
rows = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height))
header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
prefix = b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', header) + png_chunk(b'IDAT', zlib.compress(rows, 1))
ending = png_chunk(b'IEND', b'')
gap = size - len(prefix) - len(ending)
if gap < 0 or 0 < gap < 12:
raise ValueError('target cannot contain this PNG')
# Ancillary, private, reserved bit valid: unknown npAD is safely ignored.
padding = png_chunk(b'npAD', bytes(gap - 12)) if gap else b''
return prefix + padding + ending
def make_rgba64_png(width, height):
encoder = zlib.compressobj(1)
row = bytes(1 + width * 8)
pieces = [encoder.compress(row) for _ in range(height)]
pieces.append(encoder.flush())
header = struct.pack('>IIBBBBB', width, height, 16, 6, 0, 0, 0)
return (b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', header)
+ png_chunk(b'IDAT', b''.join(pieces)) + png_chunk(b'IEND', b''))
def payload_for(case):
name, width, height, size, seed = case
return make_rgba64_png(width, height) if name == 'maxpixels' else make_png(width, height, size, seed)
def memory_bytes(value):
match = re.fullmatch(r'([0-9]+(?:\.[0-9]+)?)\s*(B|kB|KB|KiB|MB|MiB|GB|GiB|TB|TiB)', value.split('/')[0].strip())
if not match:
raise ValueError('unrecognized docker memory value')
units = {'B': 1, 'kB': 1000, 'KB': 1000, 'KiB': 1024, 'MB': 1000**2, 'MiB': 1024**2,
'GB': 1000**3, 'GiB': 1024**3, 'TB': 1000**4, 'TiB': 1024**4}
return round(float(match[1]) * units[match[2]])
def summarize_memory(samples, container_ids):
result = {}
for container_id in container_ids:
baseline = [s for s in samples if s['container_id'] == container_id and s['phase'] == 'baseline']
load = [s for s in samples if s['container_id'] == container_id and s['phase'] == 'load']
if not baseline or not load:
raise RuntimeError('missing baseline or load sample for an API replica')
before, peak = baseline[-1]['used_bytes'], max(s['used_bytes'] for s in load)
result[container_id] = {'baseline_bytes': before, 'sampled_peak_bytes': peak, 'delta_bytes': peak - before,
'baseline_samples': len(baseline), 'load_samples': len(load)}
return result
def latency_summary(values):
ordered = sorted(values)
return {'p50_ms': statistics.median(ordered), 'p95_ms': ordered[math.ceil(len(ordered) * .95) - 1],
'max_ms': ordered[-1]}
def max_inflight(intervals):
active, peak = 0, 0
events = [(start, 1) for start, _ in intervals] + [(end, -1) for _, end in intervals]
for _, delta in sorted(events):
active += delta
peak = max(peak, active)
return peak
class DockerStats:
def __init__(self, container_ids):
self.ids = container_ids
self.samples = []
self.lock = threading.Condition()
self.case, self.phase = '', 'idle'
self.error = None
self.started = time.monotonic()
def __enter__(self):
self.process = subprocess.Popen(['docker', 'stats', '--no-trunc', '--format', '{{json .}}', *self.ids],
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
text=True, encoding='utf-8', errors='replace', bufsize=1)
self.thread = threading.Thread(target=self._read, daemon=True)
self.thread.start()
return self
def _read(self):
for line in self.process.stdout:
try:
cleaned = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line).strip()
if not cleaned:
continue
raw = json.loads(cleaned)
reported_id = raw.get('ID') or raw.get('Container', '')
container_id = next(cid for cid in self.ids if cid.startswith(reported_id) or reported_id.startswith(cid))
used = memory_bytes(raw['MemUsage'])
limit = memory_bytes(raw['MemUsage'].split('/')[1])
if used == 0:
continue
with self.lock:
self.samples.append({'container_id': container_id, 'name': raw['Name'], 'used_bytes': used,
'limit_bytes': limit, 'elapsed_s': round(time.monotonic() - self.started, 6),
'case': self.case, 'phase': self.phase})
self.lock.notify_all()
except (ValueError, KeyError, IndexError, StopIteration) as exc:
with self.lock:
self.error = 'invalid_docker_stats_sample:' + type(exc).__name__ + ':' + repr(line[:120])
self.lock.notify_all()
with self.lock:
self.lock.notify_all()
def mark(self, case, phase):
with self.lock:
self.case, self.phase = case, phase
def snapshot(self, case):
with self.lock:
return [dict(s) for s in self.samples if s['case'] == case]
def wait_for_baseline(self, case, timeout=20):
deadline = time.monotonic() + timeout
with self.lock:
while True:
seen = {s['container_id'] for s in self.samples if s['case'] == case and s['phase'] == 'baseline'}
if seen == set(self.ids):
return
if self.error or self.process.poll() is not None or time.monotonic() >= deadline:
raise RuntimeError('docker stats baseline unavailable')
self.lock.wait(min(.1, max(0, deadline - time.monotonic())))
def __exit__(self, *_):
if self.process.poll() is None:
self.process.terminate()
try:
self.process.wait(timeout=5)
except subprocess.TimeoutExpired:
self.process.kill()
self.process.wait(timeout=5)
self.thread.join(timeout=5)
self.process.stdout.close()
def api_containers(args):
command = ['docker', 'compose', '--project-directory', str(ROOT), '-f', str(args.compose_file)]
if args.project_name:
command.extend(['--project-name', args.project_name])
result = subprocess.run(command + ['ps', '--status', 'running', '-q', 'api'], cwd=ROOT,
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, check=True, timeout=20)
ids = result.stdout.split()
if not ids or any(not re.fullmatch(r'[a-f0-9]{12,64}', cid) for cid in ids):
raise RuntimeError('compose has no running API containers')
return ids
def container_states(container_ids):
template = ('{"id":{{json .Id}},"oom_killed":{{json .State.OOMKilled}},'
'"running":{{json .State.Running}},"restart_count":{{json .RestartCount}},'
'"exit_code":{{json .State.ExitCode}}}')
result = subprocess.run(['docker', 'inspect', '--format', template, *container_ids],
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, check=True, timeout=20)
return {state['id']: state for state in (json.loads(line) for line in result.stdout.splitlines())}
def upload(client_module, client, actor, chat, content, checksum, barrier, index, case, mime='image/png'):
result = {'index': index, 'status': None, 'ok': False, 'request_bytes': len(content)}
barrier.wait(timeout=30)
started = time.monotonic()
try:
suffix = '.jpg' if mime == 'image/jpeg' else '.png'
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + case + '-' + str(index) + suffix
req = urllib.request.Request(client.base + path, data=content, method='POST', headers={
'Authorization': 'Bearer ' + actor['token'], 'Content-Type': mime,
'Content-Length': str(len(content)), 'Accept': 'application/json'})
try:
response = client.opener.open(req, timeout=client.timeout)
except urllib.error.HTTPError as exc:
response = exc
with response:
result['status'] = response.code
result['instance_id'] = response.headers.get('X-Instance-Id')
body = response.read(65537)
if result['status'] != 201:
result['error'] = 'http_error'
elif len(body) > 65536:
result['error'] = 'oversized_metadata'
else:
attachment = json.loads(body)
client_module.shape(attachment, 'Attachment')
valid = (attachment['sha256'] == checksum and attachment['size_bytes'] == len(content)
and attachment['status'] == 'ready' and attachment['owner_id'] == actor['id']
and attachment['chat_id'] == chat['id'])
result['ok'] = valid
result['attachment_id'] = attachment['id']
if not valid:
result['error'] = 'attachment_metadata_mismatch'
except Exception as exc:
# Exception text and response bodies are deliberately excluded from evidence.
result['error'] = type(exc).__name__
finished = time.monotonic()
result.update(_started=started, _finished=finished, latency_ms=round((finished - started) * 1000, 3))
return result
def run_case(module, bootstrap_client, actor, reader, chat, stats, args, case):
name, width, height, size, seed = case
content = payload_for(case)
size = len(content)
checksum = hashlib.sha256(content).hexdigest()
clients = [module.Client(args.base_url, timeout=args.timeout) for _ in range(args.concurrency)]
state_before = container_states(stats.ids)
stats.mark(name, 'baseline')
stats.wait_for_baseline(name)
time.sleep(args.baseline_seconds)
barrier = threading.Barrier(args.concurrency + 1)
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
futures = [executor.submit(upload, module, client, actor, chat, content, checksum, barrier, index, name)
for index, client in enumerate(clients)]
stats.mark(name, 'load')
started = time.monotonic()
barrier.wait(timeout=30)
results = [future.result() for future in futures]
duration = time.monotonic() - started
# Retain a tail because docker stats reports its own sampled windows.
time.sleep(args.tail_seconds)
stats.mark(name, 'idle')
samples = stats.snapshot(name)
memory_error = None
try:
memory = summarize_memory(samples, stats.ids)
except RuntimeError as exc:
memory, memory_error = {}, str(exc)
intervals = [(r['_started'], r['_finished']) for r in results]
peak_requests = max_inflight(intervals)
successes = [r for r in results if r['ok']]
download = {'ok': False}
if successes:
try:
response = bootstrap_client.call('GET', '/api/v1/attachments/' + successes[0]['attachment_id'] + '/content', actor=reader)
downloaded = response['bytes']
download = {'ok': downloaded == content and response['headers'].get_content_type() == 'image/png',
'bytes': len(downloaded), 'sha256': hashlib.sha256(downloaded).hexdigest(),
'status': response['status']}
except Exception as exc:
download['error'] = type(exc).__name__
distinct = len({r['attachment_id'] for r in successes})
state_after = container_states(stats.ids)
stable = all(state_after[cid]['running'] and not state_after[cid]['oom_killed']
and state_after[cid]['restart_count'] == state_before[cid]['restart_count'] for cid in stats.ids)
for result in results:
result['start_offset_ms'] = round((result.pop('_started') - started) * 1000, 3)
result.pop('_finished')
return {'name': name, 'width': width, 'height': height, 'seed': seed, 'payload_bytes': size, 'sha256': checksum,
'bit_depth': 16 if name == 'maxpixels' else 8, 'color_type': 6 if name == 'maxpixels' else 2,
'nominal_decoded_bytes': width * height * (8 if name == 'maxpixels' else 4),
'requests': args.concurrency, 'attempted_bytes': size * args.concurrency,
'successful_bytes': size * len(successes), 'accepted': len(successes),
'status_counts': dict(Counter(str(r['status']) if r['status'] else 'transport_error' for r in results)),
'errors': len(results) - len(successes), 'duration_s': round(duration, 6),
'observed_max_inflight': peak_requests, 'distinct_attachment_ids': distinct,
'latency': latency_summary([r['latency_ms'] for r in results]), 'memory': memory,
'memory_samples': samples, 'memory_error': memory_error, 'results': results, 'download': download,
'state_before': state_before, 'state_after': state_after, 'restart_or_oom_detected': not stable,
'ok': (len(successes) == args.concurrency and distinct == args.concurrency and peak_requests >= 10
and download['ok'] and not memory_error and stable)}
def main():
parser = argparse.ArgumentParser(description=__doc__, epilog='On Windows, run python -X utf8 -B to match tests/client.py contract-file encoding.')
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--compose-file', type=Path, default=ROOT / 'compose.yaml')
parser.add_argument('--project-name', default=None)
parser.add_argument('--output', type=Path, help='write sanitized JSON evidence here')
parser.add_argument('--profile', choices=('standard', 'maxpixels'), default='standard',
help='standard: 1/10 MiB; maxpixels: 8000x5000 RGBA64 (320 MB decoded)')
parser.add_argument('--concurrency', type=int, default=10, help='simultaneous uploads per case, 10-100 (default 10)')
parser.add_argument('--timeout', type=float, default=45)
parser.add_argument('--baseline-seconds', type=float, default=3)
parser.add_argument('--tail-seconds', type=float, default=3)
parser.add_argument('--check-generator', action='store_true', help='check payload construction only; no HTTP or Docker')
args = parser.parse_args()
selected_cases = CASES if args.profile == 'standard' else (MAXPIXEL_CASE,)
if args.check_generator:
print(json.dumps([{'case': name, 'width': width, 'height': height, 'bytes': len(payload),
'sha256': hashlib.sha256(payload).hexdigest()}
for name, width, height, size, seed in selected_cases
for payload in [payload_for((name, width, height, size, seed))]], indent=2))
return 0
if not args.output:
parser.error('--output is required for live execution')
if not 10 <= args.concurrency <= 100 or args.timeout <= 0 or args.baseline_seconds < 1 or args.tail_seconds < 2:
parser.error('require concurrency 10-100, timeout > 0, baseline >= 1 s and tail >= 2 s')
target = urlsplit(args.base_url)
if target.scheme not in ('http', 'https') or not target.hostname or target.username or target.password or target.query or target.fragment:
parser.error('--base-url must be HTTP(S) without credentials, query or fragment')
report = {'schema_version': 1, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(), 'ok': False,
'driver': {'os': platform.system(), 'architecture': platform.machine(),
'logical_cpus': os.cpu_count(), 'python': platform.python_version()},
'measurement': {'source': 'docker stats JSON stream', 'cadence': 'Docker-managed, typically about 1 second',
'limitation': 'sampled peaks may miss sub-second maxima; includes post-upload sampling tail',
'memory_semantics': 'Docker CLI memory usage (working set, cache excluded where supported)',
'baseline_seconds': args.baseline_seconds, 'tail_seconds': args.tail_seconds,
'same_bytes_per_case': True, 'p95_method': 'nearest rank',
'profile': args.profile,
'png_compression': ('zlib level 1; random RGB; npAD padding to exact byte limit' if args.profile == 'standard'
else 'streamed zero RGBA64 scanlines; no full decoded buffer in driver')}, 'cases': []}
try:
sys.path.insert(0, str(ROOT / 'tests'))
import client as client_module
ids = api_containers(args)
report['api_container_ids'] = ids
bootstrap_client = client_module.Client(args.base_url, timeout=args.timeout)
report['stage'] = 'bootstrap'
actor, reader, _, chat = bootstrap_client.bootstrap()
with DockerStats(ids) as stats:
for case in selected_cases:
report['stage'] = case[0]
report['cases'].append(run_case(client_module, bootstrap_client, actor, reader, chat, stats, args, case))
if stats.error:
report['sampling_error'] = stats.error
report['ok'] = all(case['ok'] for case in report['cases']) and len(report['cases']) == len(selected_cases) and not stats.error
report['stage'] = 'complete'
except Exception as exc:
report['error'] = type(exc).__name__
if isinstance(exc, RuntimeError) or ('client_module' in locals() and isinstance(exc, client_module.Failure)):
# Public helper diagnostics contain only paths/schema names, never values or credentials.
report['failure'] = str(exc)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
print(json.dumps({'ok': report['ok'], 'output': str(args.output),
'cases': [{'case': c['name'], 'accepted': c['accepted'], 'requests': c['requests'],
'observed_max_inflight': c['observed_max_inflight'], 'latency': c['latency'],
'memory': c['memory'], 'download_ok': c['download']['ok']} for c in report['cases']],
'error': report.get('error')}))
return 0 if report['ok'] else 1
if __name__ == '__main__':
raise SystemExit(main())
+394
View File
@@ -0,0 +1,394 @@
// Command reconcilebench measures reconciliation against disposable real
// PostgreSQL and S3 services, independently of the compose stand.
package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"os"
"os/signal"
"path/filepath"
"runtime"
"strings"
"syscall"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"messenger/internal/attachment"
"messenger/internal/testenv"
)
func main() {
o, err := parseOptions(os.Args[1:], os.Stderr)
if errors.Is(err, flag.ErrHelp) {
return
}
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(2)
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
ctx, cancel := context.WithTimeout(ctx, 30*time.Minute)
var records bytes.Buffer
var resultOut io.Writer = os.Stdout
if o.output != "" {
resultOut = io.MultiWriter(os.Stdout, &records)
}
err = exercise(ctx, o, resultOut, os.Stderr)
if o.output != "" {
if saveErr := writeResults(o.output, records.Bytes()); saveErr != nil {
err = errors.Join(err, fmt.Errorf("save results: %w", saveErr))
}
}
cancel()
stop()
if err != nil {
fmt.Fprintln(os.Stderr, "reconcilebench:", err)
os.Exit(1)
}
}
func writeResults(path string, records []byte) error {
decoder := json.NewDecoder(bytes.NewReader(records))
rows := make([]json.RawMessage, 0)
for {
var row json.RawMessage
err := decoder.Decode(&row)
if errors.Is(err, io.EOF) {
break
}
if err != nil {
return err
}
rows = append(rows, row)
}
raw, err := json.MarshalIndent(rows, "", " ")
if err != nil {
return err
}
if err = os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return err
}
return os.WriteFile(path, append(raw, '\n'), 0644)
}
type measurement struct {
Phase string `json:"phase"`
Mode string `json:"mode"`
TargetFiles int `json:"target_files"`
GoHeapPeakBytes uint64 `json:"go_heap_peak_bytes"`
Stats attachment.Stats `json:"stats"`
Findings []attachment.Finding `json:"findings,omitempty"`
}
// Heap samples cover this Go process during Run. They exclude service processes
// and are deliberately not described as process RSS or total deployment memory.
func measure(ctx context.Context, r *attachment.Reconciler, phase, mode string, n int, apply, deep bool, out io.Writer) (attachment.Stats, error) {
done := make(chan struct{})
peakCh := make(chan uint64, 1)
go func() {
var mem runtime.MemStats
runtime.ReadMemStats(&mem)
peak := mem.HeapAlloc
ticker := time.NewTicker(10 * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-ticker.C:
runtime.ReadMemStats(&mem)
if mem.HeapAlloc > peak {
peak = mem.HeapAlloc
}
case <-done:
runtime.ReadMemStats(&mem)
if mem.HeapAlloc > peak {
peak = mem.HeapAlloc
}
peakCh <- peak
return
}
}
}()
m := measurement{Phase: phase, Mode: mode, TargetFiles: n}
stats, err := r.Run(ctx, apply, deep, func(f attachment.Finding) { m.Findings = append(m.Findings, f) })
close(done)
m.GoHeapPeakBytes = <-peakCh
m.Stats = stats
if writeErr := json.NewEncoder(out).Encode(m); writeErr != nil {
return stats, writeErr
}
return stats, err
}
func exercise(ctx context.Context, o options, out, progress io.Writer) error {
db, err := testenv.StartPostgres(ctx)
if err != nil {
return err
}
defer db.Close()
s3, err := testenv.StartS3(ctx)
if err != nil {
return err
}
defer s3.Close()
if err = json.NewEncoder(out).Encode(map[string]any{
"phase": "environment", "postgres_image": testenv.PostgresImage, "s3_image": testenv.S3Image,
"grace_ms": o.grace.Milliseconds(), "wait_ms": o.wait.Milliseconds(),
"memory_metric": "sampled Go process HeapAlloc bytes during Run; not RSS; excludes PostgreSQL and S3",
}); err != nil {
return err
}
f, err := seedFaults(ctx, db.Pool, s3.Store)
if err != nil {
return err
}
fmt.Fprintf(progress, "fault objects seeded; waiting %s for real S3 timestamps to age\n", o.wait)
if err = waitForAge(ctx, o.wait, progress); err != nil {
return err
}
if err = f.seedFresh(ctx); err != nil {
return err
}
r, err := attachment.NewReconciler(db.Pool, s3.Store, o.grace)
if err != nil {
return err
}
for _, run := range []struct {
mode string
apply bool
repairs, findings int64
}{{"dry", false, 0, 6}, {"apply", true, 5, 6}, {"repeat", true, 0, 1}} {
stats, err := measure(ctx, r, "fault", run.mode, 7, run.apply, true, out)
if err != nil {
return err
}
if stats.Repaired != run.repairs || stats.Findings != run.findings {
return fmt.Errorf("fault %s: wanted %d repairs/%d findings, got %+v", run.mode, run.repairs, run.findings, stats)
}
if err = f.assertState(ctx, run.apply); err != nil {
return fmt.Errorf("fault %s: %w", run.mode, err)
}
}
fmt.Fprintln(progress, "fault dry/apply/repeat passed; referenced messages, corrupt used blobs and fresh objects preserved")
// These resources were created above solely for this run; no compose
// endpoint or pre-existing database/bucket is ever accepted by this tool.
if _, err = db.Pool.Exec(ctx, `TRUNCATE message_attachments,attachments,files`); err != nil {
return err
}
if err = emptyBucket(ctx, s3.Store); err != nil {
return err
}
seeded := 0
for _, n := range o.sizes {
for i := seeded; i < n; i++ {
if err = seedScaleFile(ctx, db.Pool, s3.Store, f.user, f.chat, i); err != nil {
return err
}
if (i+1)%500 == 0 {
fmt.Fprintf(progress, "seeded %d referenced real objects\n", i+1)
}
}
seeded = n
stats, err := measure(ctx, r, "cost", "dry", n, false, false, out)
if err != nil {
return err
}
if stats.ScannedFiles != int64(n) || stats.ScannedObjects != int64(n) || stats.Findings != 0 || stats.Repaired != 0 {
return fmt.Errorf("cost %d: inconsistent fixture %+v", n, stats)
}
}
return json.NewEncoder(out).Encode(map[string]any{"phase": "complete", "fault_checks": "passed", "sizes": o.sizes})
}
func waitForAge(ctx context.Context, d time.Duration, progress io.Writer) error {
start := time.Now()
timer := time.NewTimer(d)
defer timer.Stop()
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
case <-ticker.C:
fmt.Fprintf(progress, "aging real S3 objects: %s / %s\n", time.Since(start).Round(time.Second), d)
}
}
}
type benchFile struct {
hash []byte
before, after string
}
type benchObject struct {
key string
after bool
}
type faultFixture struct {
db *pgxpool.Pool
blobs attachment.Blobs
user, chat uuid.UUID
files []benchFile
objects []benchObject
}
func seedFaults(ctx context.Context, db *pgxpool.Pool, blobs attachment.Blobs) (*faultFixture, error) {
f := &faultFixture{db: db, blobs: blobs, user: uuid.New(), chat: uuid.New()}
if _, err := db.Exec(ctx, `INSERT INTO users(id,username,display_name,password_hash,created_at) VALUES($1,$2,'bench','unused',now())`, f.user, "bench-"+f.user.String()); err != nil {
return nil, err
}
if _, err := db.Exec(ctx, `INSERT INTO chats(id,title,created_at) VALUES($1,'reconcilebench',now())`, f.chat); err != nil {
return nil, err
}
if err := f.put(ctx, "staging/old", []byte("stale staging"), false); err != nil {
return nil, err
}
orphan := sha256.Sum256([]byte("old orphan"))
if err := f.put(ctx, attachment.BlobKey(orphan[:]), []byte("old orphan"), false); err != nil {
return nil, err
}
for _, c := range []struct {
label, status, after string
used, present, corrupt bool
}{
{"pending", "pending", "", false, true, false},
{"unused", "ready", "", false, true, false},
{"used", "ready", "ready", true, true, false},
{"corrupt", "ready", "ready", true, true, true},
{"missing", "ready", "failed", true, false, false},
{"already_failed", "failed", "failed", true, false, false},
} {
if err := f.file(ctx, c.label, c.status, c.after, c.used, c.present, c.corrupt, false); err != nil {
return nil, err
}
}
return f, nil
}
func (f *faultFixture) put(ctx context.Context, key string, body []byte, keep bool) error {
if err := f.blobs.Put(ctx, key, bytes.NewReader(body), int64(len(body)), "image/png"); err != nil {
return err
}
f.objects = append(f.objects, benchObject{key: key, after: keep})
return nil
}
func (f *faultFixture) file(ctx context.Context, label, status, after string, used, present, corrupt, fresh bool) error {
body := []byte("fixture/" + label)
hash := sha256.Sum256(body)
created := time.Now().Add(-2 * time.Hour)
if fresh {
created = time.Now()
}
if _, err := f.db.Exec(ctx, `INSERT INTO files(hash,size_bytes,mime_type,status,created_at) VALUES($1,$2,'image/png',$3,$4)`, hash[:], len(body), status, created); err != nil {
return err
}
f.files = append(f.files, benchFile{hash: hash[:], before: status, after: after})
if used {
id, message := uuid.New(), uuid.New()
if _, err := f.db.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at) VALUES($1,$2,$3,'fixture.png',$4,now())`, id, f.chat, f.user, hash[:]); err != nil {
return err
}
if _, err := f.db.Exec(ctx, `INSERT INTO messages(chat_id,id,sender_id,text,created_at) VALUES($1,$2,$3,'preserve',now())`, f.chat, message, f.user); err != nil {
return err
}
if _, err := f.db.Exec(ctx, `INSERT INTO message_attachments(chat_id,message_id,attachment_id,ordinal) VALUES($1,$2,$3,0)`, f.chat, message, id); err != nil {
return err
}
}
if present {
if corrupt {
body[0] ^= 1
}
return f.put(ctx, attachment.BlobKey(hash[:]), body, after != "")
}
return nil
}
func (f *faultFixture) seedFresh(ctx context.Context) error {
if err := f.put(ctx, "staging/fresh", []byte("fresh staging"), true); err != nil {
return err
}
orphan := sha256.Sum256([]byte("fresh orphan"))
if err := f.put(ctx, attachment.BlobKey(orphan[:]), []byte("fresh orphan"), true); err != nil {
return err
}
return f.file(ctx, "fresh pending", "pending", "pending", false, true, false, true)
}
func (f *faultFixture) assertState(ctx context.Context, applied bool) error {
for _, file := range f.files {
var status string
err := f.db.QueryRow(ctx, `SELECT status FROM files WHERE hash=$1`, file.hash).Scan(&status)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return err
}
want := file.before
if applied {
want = file.after
}
if status != want {
return fmt.Errorf("file %s: status=%q, want %q", attachment.BlobKey(file.hash), status, want)
}
}
for _, obj := range f.objects {
_, err := f.blobs.Stat(ctx, obj.key)
want := !applied || obj.after
if err != nil && !errors.Is(err, attachment.ErrObjectMissing) {
return err
}
if (err == nil) != want {
return fmt.Errorf("object %s: present=%v want %v", obj.key, err == nil, want)
}
}
var attachments, links int
if err := f.db.QueryRow(ctx, `SELECT (SELECT count(*) FROM attachments),(SELECT count(*) FROM message_attachments)`).Scan(&attachments, &links); err != nil {
return err
}
if attachments != 4 || links != 4 {
return fmt.Errorf("referenced attachments/messages lost: %d/%d", attachments, links)
}
return nil
}
func emptyBucket(ctx context.Context, blobs attachment.Blobs) error {
for {
page, err := blobs.List(ctx, "", "", 500)
if err != nil {
return err
}
if len(page) == 0 {
return nil
}
for _, obj := range page {
if err := blobs.Delete(ctx, obj.Key); err != nil {
return err
}
}
}
}
func seedScaleFile(ctx context.Context, db *pgxpool.Pool, blobs attachment.Blobs, user, chat uuid.UUID, i int) error {
body := fmt.Sprintf("scale/%09d", i)
hash := sha256.Sum256([]byte(body))
if err := blobs.Put(ctx, attachment.BlobKey(hash[:]), strings.NewReader(body), int64(len(body)), "image/png"); err != nil {
return err
}
_, err := db.Exec(ctx, `WITH f AS (
INSERT INTO files(hash,size_bytes,mime_type,status,created_at)
VALUES($1,$2,'image/png','ready',now()-interval '2 hour') RETURNING hash)
INSERT INTO attachments(id,chat_id,owner_id,filename,file_hash,created_at)
SELECT $3,$4,$5,'scale.png',hash,now() FROM f`, hash[:], len(body), uuid.New(), chat, user)
return err
}
+46
View File
@@ -0,0 +1,46 @@
package main
import (
"encoding/json"
"io"
"os"
"path/filepath"
"reflect"
"testing"
"time"
)
func TestOptionsProtectRealAgingAndValidateSizes(t *testing.T) {
got, err := parseOptions(nil, io.Discard)
if err != nil || got.grace != 2*time.Minute || got.wait != 125*time.Second || !reflect.DeepEqual(got.sizes, []int{100, 1000, 5000}) {
t.Fatalf("defaults: %+v %v", got, err)
}
for _, args := range [][]string{{"--grace=1m"}, {"--wait=2m"}, {"--sizes=0"}, {"--sizes=10,10"}, {"--sizes=10,5"}, {"--sizes=oops"}, {"extra"}} {
if _, err := parseOptions(args, io.Discard); err == nil {
t.Errorf("accepted unsafe options %v", args)
}
}
got, err = parseOptions([]string{"--sizes=2,7", "--grace=3m", "--wait=3m2s"}, io.Discard)
if err != nil || !reflect.DeepEqual(got.sizes, []int{2, 7}) {
t.Fatalf("explicit options: %+v %v", got, err)
}
}
func TestOutputOptionWritesStructuredJSON(t *testing.T) {
path := filepath.Join(t.TempDir(), "track2", "results.json")
o, err := parseOptions([]string{"--output", path}, io.Discard)
if err != nil || o.output != path {
t.Fatalf("output flag: %+v %v", o, err)
}
if err = writeResults(path, []byte("{\"phase\":\"fault\"}\n{\"phase\":\"complete\"}\n")); err != nil {
t.Fatal(err)
}
raw, err := os.ReadFile(path)
if err != nil {
t.Fatal(err)
}
var rows []map[string]any
if err = json.Unmarshal(raw, &rows); err != nil || len(rows) != 2 || rows[1]["phase"] != "complete" {
t.Fatalf("results JSON: %s %v", raw, err)
}
}
+43
View File
@@ -0,0 +1,43 @@
package main
import (
"errors"
"flag"
"fmt"
"io"
"strconv"
"strings"
"time"
)
type options struct {
grace, wait time.Duration
sizes []int
output string
}
func parseOptions(args []string, out io.Writer) (options, error) {
var o options
f := flag.NewFlagSet("reconcilebench", flag.ContinueOnError)
f.SetOutput(out)
f.DurationVar(&o.grace, "grace", 2*time.Minute, "reconciler grace (minimum 2m)")
f.DurationVar(&o.wait, "wait", 125*time.Second, "actual S3 aging wait (must exceed grace by at least 1s)")
f.StringVar(&o.output, "output", "", "also save measurement records as a JSON array at this path")
sizes := f.String("sizes", "100,1000,5000", "strictly increasing referenced object counts")
if err := f.Parse(args); err != nil {
return o, err
}
if f.NArg() != 0 || o.grace < 2*time.Minute || o.wait-o.grace < time.Second {
return o, errors.New("no positional arguments; grace >= 2m and wait >= grace+1s required")
}
last := 0
for _, raw := range strings.Split(*sizes, ",") {
n, err := strconv.Atoi(strings.TrimSpace(raw))
if err != nil || n <= last {
return o, fmt.Errorf("sizes must be strictly increasing positive integers: %q", raw)
}
o.sizes = append(o.sizes, n)
last = n
}
return o, nil
}