Добавьте воспроизводимые эксперименты по памяти и согласованности

This commit is contained in:
2026-11-16 12:37:21 +03:00
parent d761861def
commit 29f45ca0eb
10 changed files with 1416 additions and 0 deletions
+209
View File
@@ -0,0 +1,209 @@
#!/usr/bin/env python3
"""File-driven decoder stress evidence. Requires rebuilt APIs with >=2 GiB limits.
Run with python -X utf8 -B. Default is one upload; no container lifecycle actions.
"""
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import datetime as dt
import hashlib
import importlib.util
import json
from pathlib import Path
import re
import struct
import subprocess
import sys
import threading
import time
from urllib.parse import urlsplit
import zlib
sys.dont_write_bytecode = True
_spec = importlib.util.spec_from_file_location('upload_memory', Path(__file__).with_name('upload-memory.py'))
memory = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(memory)
def write_rgba64_png(path, width=40_000_000, height=1, chunk_bytes=65536):
"""Stream zero scanlines, including filter bytes; never allocate a full row."""
if min(width, height, chunk_bytes) <= 0 or width * height > 40_000_000:
raise ValueError('invalid stress dimensions or chunk size')
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
encoder = zlib.compressobj(1)
remaining = height * (width * 8 + 1)
zeros = bytes(min(chunk_bytes, remaining))
with path.open('wb') as out:
out.write(b'\x89PNG\r\n\x1a\n')
out.write(memory.png_chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 16, 6, 0, 0, 0)))
while remaining:
count = min(len(zeros), remaining)
compressed = encoder.compress(zeros[:count])
if compressed:
out.write(memory.png_chunk(b'IDAT', compressed))
remaining -= count
out.write(memory.png_chunk(b'IDAT', encoder.flush()))
out.write(memory.png_chunk(b'IEND', b''))
def fixture_info(content, mime):
if mime == 'image/png' and content.startswith(b'\x89PNG\r\n\x1a\n') and len(content) >= 33:
width, height, depth, color = struct.unpack_from('>IIBB', content, 16)
return {'width': width, 'height': height, 'bit_depth': depth, 'color_type': color}
if mime == 'image/jpeg' and content.startswith(b'\xff\xd8'):
pos = 2
while pos + 4 <= len(content):
if content[pos] != 255:
break
while pos < len(content) and content[pos] == 255:
pos += 1
marker = content[pos]
pos += 1
length = struct.unpack_from('>H', content, pos)[0]
frame = content[pos + 2:pos + length]
pos += length
if marker in (0xc0, 0xc1, 0xc2):
height, width = struct.unpack_from('>HH', frame, 1)
count = frame[5]
return {'width': width, 'height': height, 'bit_depth': frame[0], 'components': count,
'progressive': marker == 0xc2, 'sampling': [frame[7 + 3 * i] for i in range(count)]}
if length < 2 or marker == 0xda:
break
raise ValueError('fixture MIME/header mismatch or missing image header')
def read_fixture(path, mime):
if not 0 < path.stat().st_size <= 10 * 1024 * 1024:
raise ValueError('fixture must be between 1 byte and 10 MiB')
content = path.read_bytes()
info = fixture_info(content, mime)
if not 0 < info['width'] * info['height'] <= 40_000_000:
raise ValueError('fixture exceeds 40M pixels')
return content, info
def api_limits(ids):
template = '{"id":{{json .Id}},"limit_bytes":{{json .HostConfig.Memory}}}'
result = subprocess.run(['docker', 'inspect', '--format', template, *ids], stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL, text=True, check=True, timeout=20)
return {row['id']: row['limit_bytes'] for row in (json.loads(line) for line in result.stdout.splitlines())}
def experiment(args, content, report):
sys.path.insert(0, str(memory.ROOT / 'tests'))
import client as module
ids = memory.api_containers(args)
report['api_limits'] = api_limits(ids)
if any(report['api_limits'].get(cid, 0) < 2 * 1024**3 for cid in ids):
raise RuntimeError('stress uploads require every API container limit to be >=2 GiB')
report['state_before'] = memory.container_states(ids)
bootstrap = module.Client(args.base_url, timeout=args.timeout)
actor, reader, _, chat = bootstrap.bootstrap()
clients = [module.Client(args.base_url, timeout=args.timeout) for _ in range(args.concurrency)]
with memory.DockerStats(ids) as stats:
stats.mark(args.label, 'baseline')
stats.wait_for_baseline(args.label)
time.sleep(args.baseline_seconds)
barrier = threading.Barrier(args.concurrency + 1)
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
futures = [executor.submit(memory.upload, module, client, actor, chat, content, report['sha256'],
barrier, index, args.label, mime=args.mime) for index, client in enumerate(clients)]
stats.mark(args.label, 'load')
started = time.monotonic()
barrier.wait(timeout=30)
results = [future.result() for future in futures]
report['duration_s'] = time.monotonic() - started
time.sleep(args.tail_seconds)
stats.mark(args.label, 'idle')
report['memory_samples'] = stats.snapshot(args.label)
report['sampling_error'] = stats.error
report['state_after'] = memory.container_states(ids)
stable = all(report['state_after'][cid]['running'] and not report['state_after'][cid]['oom_killed']
and report['state_after'][cid]['restart_count'] == report['state_before'][cid]['restart_count'] for cid in ids)
report['restart_or_oom_detected'] = not stable
try:
report['memory'] = memory.summarize_memory(report['memory_samples'], ids)
except RuntimeError as exc:
report['memory_error'] = str(exc)
report['observed_max_inflight'] = memory.max_inflight([(r['_started'], r['_finished']) for r in results])
accepted = [r for r in results if r['ok']]
report['accepted'] = len(accepted)
report['status_counts'] = dict(Counter(str(r['status']) if r['status'] else 'transport_error' for r in results))
report['latency'] = memory.latency_summary([r['latency_ms'] for r in results])
report['attempted_bytes'], report['successful_bytes'] = len(content) * len(results), len(content) * len(accepted)
for result in results:
result['start_offset_ms'] = round((result.pop('_started') - started) * 1000, 3)
result.pop('_finished')
report['results'] = results
report['download'] = {'ok': False}
if accepted:
try:
response = bootstrap.call('GET', '/api/v1/attachments/' + accepted[0]['attachment_id'] + '/content', actor=reader)
body = response['bytes']
report['download'] = {'ok': body == content and response['headers'].get_content_type() == args.mime,
'status': response['status'], 'bytes': len(body), 'sha256': hashlib.sha256(body).hexdigest()}
except Exception as exc:
report['download']['error'] = str(exc) if isinstance(exc, module.Failure) else type(exc).__name__
report['ok'] = (len(accepted) == args.concurrency and report['download']['ok'] and stable
and not report.get('memory_error') and not report['sampling_error']
and report['observed_max_inflight'] == args.concurrency)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--file', type=Path)
parser.add_argument('--mime', choices=('image/png', 'image/jpeg'))
parser.add_argument('--label', default='decoder')
parser.add_argument('--concurrency', type=int, default=1, help='1-10, default single-upload acceptance')
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--compose-file', type=Path, default=memory.ROOT / 'compose.yaml')
parser.add_argument('--project-name')
parser.add_argument('--timeout', type=float, default=45)
parser.add_argument('--baseline-seconds', type=float, default=3)
parser.add_argument('--tail-seconds', type=float, default=3)
parser.add_argument('--output', type=Path)
parser.add_argument('--check-file', action='store_true', help='inspect bytes/header without HTTP or Docker')
parser.add_argument('--generate-wide', type=Path, help='generate 40Mx1 RGBA64 PNG offline using bounded zlib input')
args = parser.parse_args()
if args.generate_wide:
write_rgba64_png(args.generate_wide)
content, info = read_fixture(args.generate_wide, 'image/png')
print(json.dumps({**info, 'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()}))
return 0
if not args.file or not args.mime:
parser.error('--file and --mime are required')
if not 1 <= args.concurrency <= 10 or args.timeout <= 0 or args.baseline_seconds < 1 or args.tail_seconds < 2:
parser.error('require concurrency1-10, timeout>0, baseline>=1s, tail>=2s')
if not re.fullmatch(r'[A-Za-z0-9_-]{1,40}', args.label):
parser.error('label must be 1-40 safe ASCII characters')
url = urlsplit(args.base_url)
if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password or url.query or url.fragment:
parser.error('base URL must be HTTP(S) without credentials/query/fragment')
content, info = read_fixture(args.file, args.mime)
report = {'schema_version': 1, 'name': args.label, 'mime': args.mime, 'fixture': info,
'payload_bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest(),
'requests': args.concurrency, 'ok': False, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(),
'measurement': {'source': 'docker stats JSON stream, formatted working-set values',
'limitation': 'sampled/rounded peaks can miss instantaneous maxima',
'baseline_seconds': args.baseline_seconds, 'tail_seconds': args.tail_seconds}}
if args.check_file:
print(json.dumps(report, indent=2))
return 0
if not args.output:
parser.error('--output is required for a live run')
try:
experiment(args, content, report)
except Exception as exc:
report['error'] = type(exc).__name__
if isinstance(exc, RuntimeError) or type(exc).__name__ == 'Failure':
report['failure'] = str(exc)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
print(json.dumps({key: value for key, value in report.items() if key not in ('memory_samples', 'results')}))
return 0 if report['ok'] else 1
if __name__ == '__main__':
raise SystemExit(main())
+95
View File
@@ -0,0 +1,95 @@
"""Stop each actual uploader replica and verify its bytes through the survivor."""
import argparse
import hashlib
import json
import os
from pathlib import Path
import subprocess
import sys
import time
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, check
def instances():
text = subprocess.check_output(['sh', 'scripts/api-instances.sh'], cwd=ROOT,
encoding='utf-8')
result = []
for line in text.splitlines():
container, replica, state, alias, image = line.split()
result.append({'container': container, 'replica': replica,
'state': state, 'alias': alias, 'image': image})
return result
def restore(c):
subprocess.run(['docker', 'compose', 'start', 'api'], cwd=ROOT,
check=True, stdout=subprocess.DEVNULL)
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
observed = {c.call('GET', '/health/live')['headers']['X-Instance-Id']
for _ in range(6)}
if len(observed) >= 2:
return
time.sleep(0.3)
raise AssertionError('two replicas did not return after restore')
def main():
p = argparse.ArgumentParser(description=__doc__)
p.add_argument('--base-url', default='http://localhost:8080')
p.add_argument('--output', type=Path, default=ROOT / 'evidence/failover-uploaders.json')
args = p.parse_args()
c = Client(args.base_url, timeout=10)
actor, reader, _, chat = c.bootstrap()
data = (ROOT / 'tests/fixtures/sample.png').read_bytes()
wanted = instances()
check(len(wanted) == 2 and all(x['state'] == 'running' for x in wanted),
'this experiment requires two running API replicas')
report = {'scenario': 'stop actual upload-serving replica; download via survivor',
'cases': [], 'ok': False}
try:
for target in wanted:
upload = None
for _ in range(10):
attempt = c.call('POST', '/api/v1/chats/' + chat['id'] +
'/attachments?filename=sample.png', 201,
actor=actor, raw=data,
headers={'Content-Type': 'image/png'}, schema='Attachment')
if attempt['headers']['X-Instance-Id'] == target['alias']:
upload = attempt
break
check(upload is not None, 'target replica did not serve an upload')
before = instances()
env = dict(os.environ, TARGET_INSTANCE_ID=target['alias'])
started = time.monotonic()
subprocess.run(['sh', 'scripts/stop-one.sh'], cwd=ROOT, env=env,
check=True, stdout=subprocess.DEVNULL, timeout=15)
stop_seconds = time.monotonic() - started
downloaded = c.call('GET', '/api/v1/attachments/' +
upload['json']['id'] + '/content', actor=reader)
survivor = downloaded['headers']['X-Instance-Id']
check(survivor != target['alias'], 'stopped uploader still served download')
check(downloaded['bytes'] == data, 'bytes changed after uploader stop')
after = instances()
check(any(x['container'] == target['container'] and x['state'] != 'running'
for x in after), 'uploader was not left stopped')
report['cases'].append({'uploader': target, 'download_instance': survivor,
'stop_seconds': round(stop_seconds, 3),
'sha256': hashlib.sha256(downloaded['bytes']).hexdigest(),
'same_bytes': True, 'before': before, 'after': after})
restore(c)
report['ok'] = True
finally:
# This script only restores the API containers that it deliberately stopped.
restore(c)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
print(json.dumps({'ok': report['ok'], 'cases': len(report['cases']),
'output': str(args.output)}))
if __name__ == '__main__':
main()
+45
View File
@@ -0,0 +1,45 @@
"""Generate optional 40 MP regression images; requires Pillow==11.1.0."""
import argparse
import hashlib
import importlib.util
import json
from pathlib import Path
import sys
sys.dont_write_bytecode = True
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output-dir', type=Path, default=Path('bin/decoder-fixtures'))
args = parser.parse_args()
try:
from PIL import Image
except ImportError:
parser.error('install optional fixture dependency: pip install Pillow==11.1.0')
spec = importlib.util.spec_from_file_location('decoder_memory',
Path(__file__).with_name('decoder-memory.py'))
decoder = importlib.util.module_from_spec(spec)
spec.loader.exec_module(decoder)
args.output_dir.mkdir(parents=True, exist_ok=True)
manifest = {'pillow_version': Image.__version__, 'fixtures': []}
for mode, color in [('RGB', (25, 90, 160)), ('CMYK', (20, 40, 60, 10))]:
path = args.output_dir / ('progressive-' + mode.lower() + '-444-40mp.jpg')
with Image.new(mode, (8000, 5000), color) as image:
image.save(path, format='JPEG', progressive=True, subsampling=0, quality=85)
content, info = decoder.read_fixture(path, 'image/jpeg')
if not info['progressive'] or any(value != 0x11 for value in info['sampling']):
raise ValueError('encoder did not produce progressive 4:4:4')
manifest['fixtures'].append({'name': path.name, 'mime': 'image/jpeg', **info,
'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()})
wide = args.output_dir / 'wide-rgba64-40mp.png'
decoder.write_rgba64_png(wide)
content, info = decoder.read_fixture(wide, 'image/png')
manifest['fixtures'].append({'name': wide.name, 'mime': 'image/png', **info,
'bytes': len(content), 'sha256': hashlib.sha256(content).hexdigest()})
(args.output_dir / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n', encoding='utf-8')
print(json.dumps(manifest, indent=2))
if __name__ == '__main__':
main()
+55
View File
@@ -0,0 +1,55 @@
"""Offline checks for bounded stress fixtures and format identification."""
import importlib.util
from pathlib import Path
import struct
import tempfile
import unittest
import zlib
class DecoderMemoryTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
path = Path(__file__).with_name('decoder-memory.py')
cls.module = None
if path.exists():
spec = importlib.util.spec_from_file_location('decoder_memory', path)
cls.module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(cls.module)
def setUp(self):
self.assertIsNotNone(self.module, 'decoder-memory.py is not implemented')
def test_streamed_rgba64_png_has_valid_crc_and_zero_pixels(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / 'wide.png'
self.module.write_rgba64_png(path, width=19, height=3, chunk_bytes=7)
payload = path.read_bytes()
self.assertEqual(payload[:8], b'\x89PNG\r\n\x1a\n')
pos, compressed = 8, bytearray()
while pos < len(payload):
size = struct.unpack_from('>I', payload, pos)[0]
kind, body = payload[pos + 4:pos + 8], payload[pos + 8:pos + 8 + size]
self.assertEqual(struct.unpack_from('>I', payload, pos + 8 + size)[0], zlib.crc32(kind + body) & 0xffffffff)
if kind == b'IDAT':
compressed.extend(body)
pos += size + 12
self.assertEqual(zlib.decompress(compressed), bytes(3 * (19 * 8 + 1)))
self.assertEqual(self.module.fixture_info(payload, 'image/png'),
{'width': 19, 'height': 3, 'bit_depth': 16, 'color_type': 6})
def test_progressive_jpeg_444_components_are_identified(self):
sof = b'\x08' + struct.pack('>HHB', 5000, 8000, 3) + bytes([1, 0x11, 0, 2, 0x11, 0, 3, 0x11, 0])
payload = b'\xff\xd8\xff\xc2' + struct.pack('>H', len(sof) + 2) + sof
info = self.module.fixture_info(payload, 'image/jpeg')
self.assertEqual((info['width'], info['height'], info['components']), (8000, 5000, 3))
self.assertTrue(info['progressive'])
self.assertEqual(info['sampling'], [0x11, 0x11, 0x11])
def test_fixture_byte_cap_is_checked_before_reading(self):
with tempfile.TemporaryDirectory() as directory:
path = Path(directory) / 'oversized.png'
with path.open('wb') as out:
out.truncate(10 * 1024 * 1024 + 1)
with self.assertRaises(ValueError):
self.module.read_fixture(path, 'image/png')
+139
View File
@@ -0,0 +1,139 @@
"""Offline checks for reproducible PNGs and Docker memory evidence."""
import importlib.util
import json
from pathlib import Path
import struct
import unittest
from types import SimpleNamespace
import io
import zlib
class UploadMemoryTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
spec = importlib.util.spec_from_file_location('upload_memory', Path(__file__).with_name('upload-memory.py'))
cls.module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(cls.module)
def inspect_png(self, payload, width, height):
self.assertEqual(payload[:8], b'\x89PNG\r\n\x1a\n')
pos, chunks, compressed = 8, [], bytearray()
while pos < len(payload):
size = struct.unpack_from('>I', payload, pos)[0]
kind = payload[pos + 4:pos + 8]
data = payload[pos + 8:pos + 8 + size]
crc = struct.unpack_from('>I', payload, pos + 8 + size)[0]
self.assertEqual(crc, zlib.crc32(kind + data) & 0xffffffff)
if kind == b'IHDR':
self.assertEqual(struct.unpack('>IIBBBBB', data), (width, height, 8, 2, 0, 0, 0))
if kind == b'IDAT':
compressed.extend(data)
chunks.append(kind)
pos += size + 12
self.assertEqual(pos, len(payload))
self.assertEqual(chunks[0], b'IHDR')
self.assertEqual(chunks[-1], b'IEND')
self.assertIn(b'npAD', chunks)
rows = zlib.decompress(compressed)
self.assertEqual(len(rows), height * (width * 3 + 1))
self.assertTrue(all(rows[row * (width * 3 + 1)] == 0 for row in range(height)))
def test_seeded_png_has_correct_crc_pixels_and_exact_size(self):
first = self.module.make_png(32, 24, 4096, 123)
self.assertEqual(len(first), 4096)
self.assertEqual(first, self.module.make_png(32, 24, 4096, 123))
self.assertNotEqual(first, self.module.make_png(32, 24, 4096, 124))
self.inspect_png(first, 32, 24)
def test_both_experiment_payloads_decode_and_reach_exact_limits(self):
for name, width, height, size, seed in self.module.CASES:
with self.subTest(name=name):
payload = self.module.make_png(width, height, size, seed)
self.assertEqual(len(payload), size)
self.assertLessEqual(len(payload), 10 * 1024 * 1024)
self.assertLessEqual(width * height, 40_000_000)
self.inspect_png(payload, width, height)
def test_impossible_png_size_is_rejected(self):
with self.assertRaises(ValueError):
self.module.make_png(32, 32, 64, 1)
def test_docker_memory_uses_binary_and_decimal_units(self):
for raw, expected in [('1.5MiB / 512MiB', 1572864), ('2MB / 512MB', 2000000), ('4096B / 1GiB', 4096)]:
self.assertEqual(self.module.memory_bytes(raw), expected)
with self.assertRaises(ValueError):
self.module.memory_bytes('not available')
def test_memory_summary_separates_baseline_and_load_for_each_replica(self):
samples = [
{'container_id': 'a', 'phase': 'baseline', 'used_bytes': 10},
{'container_id': 'b', 'phase': 'baseline', 'used_bytes': 20},
{'container_id': 'a', 'phase': 'baseline', 'used_bytes': 12},
{'container_id': 'a', 'phase': 'load', 'used_bytes': 35},
{'container_id': 'a', 'phase': 'load', 'used_bytes': 30},
{'container_id': 'b', 'phase': 'load', 'used_bytes': 50},
]
result = self.module.summarize_memory(samples, ['a', 'b'])
self.assertEqual(result['a']['baseline_bytes'], 12)
self.assertEqual(result['a']['sampled_peak_bytes'], 35)
self.assertEqual(result['a']['delta_bytes'], 23)
self.assertEqual(result['b']['sampled_peak_bytes'], 50)
with self.assertRaises(RuntimeError):
self.module.summarize_memory(samples, ['a', 'b', 'missing'])
def test_latency_percentiles_and_overlap_are_explicit(self):
self.assertEqual(self.module.latency_summary(list(range(1, 11))),
{'p50_ms': 5.5, 'p95_ms': 10, 'max_ms': 10})
self.assertEqual(self.module.max_inflight([(0, 4), (1, 3), (2, 5)]), 3)
self.assertEqual(self.module.max_inflight([(0, 1), (1, 2)]), 1)
def test_docker_stream_ignores_ansi_only_lines_between_samples(self):
cid = 'a' * 64
row = json.dumps({'ID': cid, 'Name': 'lab4-api-1', 'MemUsage': '12MiB / 512MiB'})
stats = self.module.DockerStats([cid])
stats.process = SimpleNamespace(stdout=iter(['\x1b[H' + row + '\n', ' \x1b[K\n', row + '\n']))
stats.mark('small', 'baseline')
stats._read()
self.assertIsNone(stats.error)
self.assertEqual(len(stats.samples), 2)
def test_rgba64_profile_has_valid_header_crc_and_zero_scanlines(self):
payload = self.module.make_rgba64_png(8, 5)
pos, compressed = 8, bytearray()
while pos < len(payload):
size = struct.unpack_from('>I', payload, pos)[0]
kind, data = payload[pos + 4:pos + 8], payload[pos + 8:pos + 8 + size]
self.assertEqual(struct.unpack_from('>I', payload, pos + 8 + size)[0], zlib.crc32(kind + data) & 0xffffffff)
if kind == b'IHDR':
self.assertEqual(struct.unpack('>IIBBBBB', data), (8, 5, 16, 6, 0, 0, 0))
if kind == b'IDAT':
compressed.extend(data)
pos += size + 12
self.assertEqual(zlib.decompress(compressed), bytes(5 * (1 + 8 * 8)))
self.assertEqual(pos, len(payload))
def test_max_pixels_profile_fits_byte_limit_without_full_raw_buffer(self):
payload = self.module.make_rgba64_png(8000, 5000)
self.assertLess(len(payload), 10 * 1024 * 1024)
self.assertEqual(struct.unpack_from('>IIBB', payload, 16), (8000, 5000, 16, 6))
def test_upload_uses_requested_jpeg_mime(self):
captured = []
class Opener:
def open(self, request, timeout):
captured.append(request)
response = io.BytesIO(b'')
response.code, response.headers = 415, {}
return response
client = SimpleNamespace(base='http://localhost', opener=Opener(), timeout=1)
import threading
result = self.module.upload(None, client, {'token': 'synthetic'}, {'id': 'chat'}, b'jpeg', 'hash',
threading.Barrier(1), 0, 'decoder', mime='image/jpeg')
self.assertEqual(captured[0].get_header('Content-type'), 'image/jpeg')
self.assertTrue(captured[0].full_url.endswith('decoder-0.jpg'))
self.assertEqual(result['status'], 415)
if __name__ == '__main__':
unittest.main()
+376
View File
@@ -0,0 +1,376 @@
#!/usr/bin/env python3
"""Concurrent upload experiment; stdlib only, no stand lifecycle operations."""
import argparse
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
import datetime as dt
import hashlib
import json
import math
import os
from pathlib import Path
import platform
import random
import re
import statistics
import struct
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
from urllib.parse import urlsplit
import zlib
ROOT = Path(__file__).resolve().parents[2]
sys.dont_write_bytecode = True
CASES = (
('small', 512, 512, 1024 * 1024, 41),
('limit', 1600, 1600, 10 * 1024 * 1024, 42),
)
MAXPIXEL_CASE = ('maxpixels', 8000, 5000, None, None)
def png_chunk(kind, data):
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff)
def make_png(width, height, size, seed):
rng = random.Random(seed)
rows = b''.join(b'\0' + rng.randbytes(width * 3) for _ in range(height))
header = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
prefix = b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', header) + png_chunk(b'IDAT', zlib.compress(rows, 1))
ending = png_chunk(b'IEND', b'')
gap = size - len(prefix) - len(ending)
if gap < 0 or 0 < gap < 12:
raise ValueError('target cannot contain this PNG')
# Ancillary, private, reserved bit valid: unknown npAD is safely ignored.
padding = png_chunk(b'npAD', bytes(gap - 12)) if gap else b''
return prefix + padding + ending
def make_rgba64_png(width, height):
encoder = zlib.compressobj(1)
row = bytes(1 + width * 8)
pieces = [encoder.compress(row) for _ in range(height)]
pieces.append(encoder.flush())
header = struct.pack('>IIBBBBB', width, height, 16, 6, 0, 0, 0)
return (b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', header)
+ png_chunk(b'IDAT', b''.join(pieces)) + png_chunk(b'IEND', b''))
def payload_for(case):
name, width, height, size, seed = case
return make_rgba64_png(width, height) if name == 'maxpixels' else make_png(width, height, size, seed)
def memory_bytes(value):
match = re.fullmatch(r'([0-9]+(?:\.[0-9]+)?)\s*(B|kB|KB|KiB|MB|MiB|GB|GiB|TB|TiB)', value.split('/')[0].strip())
if not match:
raise ValueError('unrecognized docker memory value')
units = {'B': 1, 'kB': 1000, 'KB': 1000, 'KiB': 1024, 'MB': 1000**2, 'MiB': 1024**2,
'GB': 1000**3, 'GiB': 1024**3, 'TB': 1000**4, 'TiB': 1024**4}
return round(float(match[1]) * units[match[2]])
def summarize_memory(samples, container_ids):
result = {}
for container_id in container_ids:
baseline = [s for s in samples if s['container_id'] == container_id and s['phase'] == 'baseline']
load = [s for s in samples if s['container_id'] == container_id and s['phase'] == 'load']
if not baseline or not load:
raise RuntimeError('missing baseline or load sample for an API replica')
before, peak = baseline[-1]['used_bytes'], max(s['used_bytes'] for s in load)
result[container_id] = {'baseline_bytes': before, 'sampled_peak_bytes': peak, 'delta_bytes': peak - before,
'baseline_samples': len(baseline), 'load_samples': len(load)}
return result
def latency_summary(values):
ordered = sorted(values)
return {'p50_ms': statistics.median(ordered), 'p95_ms': ordered[math.ceil(len(ordered) * .95) - 1],
'max_ms': ordered[-1]}
def max_inflight(intervals):
active, peak = 0, 0
events = [(start, 1) for start, _ in intervals] + [(end, -1) for _, end in intervals]
for _, delta in sorted(events):
active += delta
peak = max(peak, active)
return peak
class DockerStats:
def __init__(self, container_ids):
self.ids = container_ids
self.samples = []
self.lock = threading.Condition()
self.case, self.phase = '', 'idle'
self.error = None
self.started = time.monotonic()
def __enter__(self):
self.process = subprocess.Popen(['docker', 'stats', '--no-trunc', '--format', '{{json .}}', *self.ids],
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
text=True, encoding='utf-8', errors='replace', bufsize=1)
self.thread = threading.Thread(target=self._read, daemon=True)
self.thread.start()
return self
def _read(self):
for line in self.process.stdout:
try:
cleaned = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line).strip()
if not cleaned:
continue
raw = json.loads(cleaned)
reported_id = raw.get('ID') or raw.get('Container', '')
container_id = next(cid for cid in self.ids if cid.startswith(reported_id) or reported_id.startswith(cid))
used = memory_bytes(raw['MemUsage'])
limit = memory_bytes(raw['MemUsage'].split('/')[1])
if used == 0:
continue
with self.lock:
self.samples.append({'container_id': container_id, 'name': raw['Name'], 'used_bytes': used,
'limit_bytes': limit, 'elapsed_s': round(time.monotonic() - self.started, 6),
'case': self.case, 'phase': self.phase})
self.lock.notify_all()
except (ValueError, KeyError, IndexError, StopIteration) as exc:
with self.lock:
self.error = 'invalid_docker_stats_sample:' + type(exc).__name__ + ':' + repr(line[:120])
self.lock.notify_all()
with self.lock:
self.lock.notify_all()
def mark(self, case, phase):
with self.lock:
self.case, self.phase = case, phase
def snapshot(self, case):
with self.lock:
return [dict(s) for s in self.samples if s['case'] == case]
def wait_for_baseline(self, case, timeout=20):
deadline = time.monotonic() + timeout
with self.lock:
while True:
seen = {s['container_id'] for s in self.samples if s['case'] == case and s['phase'] == 'baseline'}
if seen == set(self.ids):
return
if self.error or self.process.poll() is not None or time.monotonic() >= deadline:
raise RuntimeError('docker stats baseline unavailable')
self.lock.wait(min(.1, max(0, deadline - time.monotonic())))
def __exit__(self, *_):
if self.process.poll() is None:
self.process.terminate()
try:
self.process.wait(timeout=5)
except subprocess.TimeoutExpired:
self.process.kill()
self.process.wait(timeout=5)
self.thread.join(timeout=5)
self.process.stdout.close()
def api_containers(args):
command = ['docker', 'compose', '--project-directory', str(ROOT), '-f', str(args.compose_file)]
if args.project_name:
command.extend(['--project-name', args.project_name])
result = subprocess.run(command + ['ps', '--status', 'running', '-q', 'api'], cwd=ROOT,
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, check=True, timeout=20)
ids = result.stdout.split()
if not ids or any(not re.fullmatch(r'[a-f0-9]{12,64}', cid) for cid in ids):
raise RuntimeError('compose has no running API containers')
return ids
def container_states(container_ids):
template = ('{"id":{{json .Id}},"oom_killed":{{json .State.OOMKilled}},'
'"running":{{json .State.Running}},"restart_count":{{json .RestartCount}},'
'"exit_code":{{json .State.ExitCode}}}')
result = subprocess.run(['docker', 'inspect', '--format', template, *container_ids],
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, check=True, timeout=20)
return {state['id']: state for state in (json.loads(line) for line in result.stdout.splitlines())}
def upload(client_module, client, actor, chat, content, checksum, barrier, index, case, mime='image/png'):
result = {'index': index, 'status': None, 'ok': False, 'request_bytes': len(content)}
barrier.wait(timeout=30)
started = time.monotonic()
try:
suffix = '.jpg' if mime == 'image/jpeg' else '.png'
path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + case + '-' + str(index) + suffix
req = urllib.request.Request(client.base + path, data=content, method='POST', headers={
'Authorization': 'Bearer ' + actor['token'], 'Content-Type': mime,
'Content-Length': str(len(content)), 'Accept': 'application/json'})
try:
response = client.opener.open(req, timeout=client.timeout)
except urllib.error.HTTPError as exc:
response = exc
with response:
result['status'] = response.code
result['instance_id'] = response.headers.get('X-Instance-Id')
body = response.read(65537)
if result['status'] != 201:
result['error'] = 'http_error'
elif len(body) > 65536:
result['error'] = 'oversized_metadata'
else:
attachment = json.loads(body)
client_module.shape(attachment, 'Attachment')
valid = (attachment['sha256'] == checksum and attachment['size_bytes'] == len(content)
and attachment['status'] == 'ready' and attachment['owner_id'] == actor['id']
and attachment['chat_id'] == chat['id'])
result['ok'] = valid
result['attachment_id'] = attachment['id']
if not valid:
result['error'] = 'attachment_metadata_mismatch'
except Exception as exc:
# Exception text and response bodies are deliberately excluded from evidence.
result['error'] = type(exc).__name__
finished = time.monotonic()
result.update(_started=started, _finished=finished, latency_ms=round((finished - started) * 1000, 3))
return result
def run_case(module, bootstrap_client, actor, reader, chat, stats, args, case):
name, width, height, size, seed = case
content = payload_for(case)
size = len(content)
checksum = hashlib.sha256(content).hexdigest()
clients = [module.Client(args.base_url, timeout=args.timeout) for _ in range(args.concurrency)]
state_before = container_states(stats.ids)
stats.mark(name, 'baseline')
stats.wait_for_baseline(name)
time.sleep(args.baseline_seconds)
barrier = threading.Barrier(args.concurrency + 1)
with ThreadPoolExecutor(max_workers=args.concurrency) as executor:
futures = [executor.submit(upload, module, client, actor, chat, content, checksum, barrier, index, name)
for index, client in enumerate(clients)]
stats.mark(name, 'load')
started = time.monotonic()
barrier.wait(timeout=30)
results = [future.result() for future in futures]
duration = time.monotonic() - started
# Retain a tail because docker stats reports its own sampled windows.
time.sleep(args.tail_seconds)
stats.mark(name, 'idle')
samples = stats.snapshot(name)
memory_error = None
try:
memory = summarize_memory(samples, stats.ids)
except RuntimeError as exc:
memory, memory_error = {}, str(exc)
intervals = [(r['_started'], r['_finished']) for r in results]
peak_requests = max_inflight(intervals)
successes = [r for r in results if r['ok']]
download = {'ok': False}
if successes:
try:
response = bootstrap_client.call('GET', '/api/v1/attachments/' + successes[0]['attachment_id'] + '/content', actor=reader)
downloaded = response['bytes']
download = {'ok': downloaded == content and response['headers'].get_content_type() == 'image/png',
'bytes': len(downloaded), 'sha256': hashlib.sha256(downloaded).hexdigest(),
'status': response['status']}
except Exception as exc:
download['error'] = type(exc).__name__
distinct = len({r['attachment_id'] for r in successes})
state_after = container_states(stats.ids)
stable = all(state_after[cid]['running'] and not state_after[cid]['oom_killed']
and state_after[cid]['restart_count'] == state_before[cid]['restart_count'] for cid in stats.ids)
for result in results:
result['start_offset_ms'] = round((result.pop('_started') - started) * 1000, 3)
result.pop('_finished')
return {'name': name, 'width': width, 'height': height, 'seed': seed, 'payload_bytes': size, 'sha256': checksum,
'bit_depth': 16 if name == 'maxpixels' else 8, 'color_type': 6 if name == 'maxpixels' else 2,
'nominal_decoded_bytes': width * height * (8 if name == 'maxpixels' else 4),
'requests': args.concurrency, 'attempted_bytes': size * args.concurrency,
'successful_bytes': size * len(successes), 'accepted': len(successes),
'status_counts': dict(Counter(str(r['status']) if r['status'] else 'transport_error' for r in results)),
'errors': len(results) - len(successes), 'duration_s': round(duration, 6),
'observed_max_inflight': peak_requests, 'distinct_attachment_ids': distinct,
'latency': latency_summary([r['latency_ms'] for r in results]), 'memory': memory,
'memory_samples': samples, 'memory_error': memory_error, 'results': results, 'download': download,
'state_before': state_before, 'state_after': state_after, 'restart_or_oom_detected': not stable,
'ok': (len(successes) == args.concurrency and distinct == args.concurrency and peak_requests >= 10
and download['ok'] and not memory_error and stable)}
def main():
parser = argparse.ArgumentParser(description=__doc__, epilog='On Windows, run python -X utf8 -B to match tests/client.py contract-file encoding.')
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--compose-file', type=Path, default=ROOT / 'compose.yaml')
parser.add_argument('--project-name', default=None)
parser.add_argument('--output', type=Path, help='write sanitized JSON evidence here')
parser.add_argument('--profile', choices=('standard', 'maxpixels'), default='standard',
help='standard: 1/10 MiB; maxpixels: 8000x5000 RGBA64 (320 MB decoded)')
parser.add_argument('--concurrency', type=int, default=10, help='simultaneous uploads per case, 10-100 (default 10)')
parser.add_argument('--timeout', type=float, default=45)
parser.add_argument('--baseline-seconds', type=float, default=3)
parser.add_argument('--tail-seconds', type=float, default=3)
parser.add_argument('--check-generator', action='store_true', help='check payload construction only; no HTTP or Docker')
args = parser.parse_args()
selected_cases = CASES if args.profile == 'standard' else (MAXPIXEL_CASE,)
if args.check_generator:
print(json.dumps([{'case': name, 'width': width, 'height': height, 'bytes': len(payload),
'sha256': hashlib.sha256(payload).hexdigest()}
for name, width, height, size, seed in selected_cases
for payload in [payload_for((name, width, height, size, seed))]], indent=2))
return 0
if not args.output:
parser.error('--output is required for live execution')
if not 10 <= args.concurrency <= 100 or args.timeout <= 0 or args.baseline_seconds < 1 or args.tail_seconds < 2:
parser.error('require concurrency 10-100, timeout > 0, baseline >= 1 s and tail >= 2 s')
target = urlsplit(args.base_url)
if target.scheme not in ('http', 'https') or not target.hostname or target.username or target.password or target.query or target.fragment:
parser.error('--base-url must be HTTP(S) without credentials, query or fragment')
report = {'schema_version': 1, 'started_at': dt.datetime.now(dt.timezone.utc).isoformat(), 'ok': False,
'driver': {'os': platform.system(), 'architecture': platform.machine(),
'logical_cpus': os.cpu_count(), 'python': platform.python_version()},
'measurement': {'source': 'docker stats JSON stream', 'cadence': 'Docker-managed, typically about 1 second',
'limitation': 'sampled peaks may miss sub-second maxima; includes post-upload sampling tail',
'memory_semantics': 'Docker CLI memory usage (working set, cache excluded where supported)',
'baseline_seconds': args.baseline_seconds, 'tail_seconds': args.tail_seconds,
'same_bytes_per_case': True, 'p95_method': 'nearest rank',
'profile': args.profile,
'png_compression': ('zlib level 1; random RGB; npAD padding to exact byte limit' if args.profile == 'standard'
else 'streamed zero RGBA64 scanlines; no full decoded buffer in driver')}, 'cases': []}
try:
sys.path.insert(0, str(ROOT / 'tests'))
import client as client_module
ids = api_containers(args)
report['api_container_ids'] = ids
bootstrap_client = client_module.Client(args.base_url, timeout=args.timeout)
report['stage'] = 'bootstrap'
actor, reader, _, chat = bootstrap_client.bootstrap()
with DockerStats(ids) as stats:
for case in selected_cases:
report['stage'] = case[0]
report['cases'].append(run_case(client_module, bootstrap_client, actor, reader, chat, stats, args, case))
if stats.error:
report['sampling_error'] = stats.error
report['ok'] = all(case['ok'] for case in report['cases']) and len(report['cases']) == len(selected_cases) and not stats.error
report['stage'] = 'complete'
except Exception as exc:
report['error'] = type(exc).__name__
if isinstance(exc, RuntimeError) or ('client_module' in locals() and isinstance(exc, client_module.Failure)):
# Public helper diagnostics contain only paths/schema names, never values or credentials.
report['failure'] = str(exc)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
print(json.dumps({'ok': report['ok'], 'output': str(args.output),
'cases': [{'case': c['name'], 'accepted': c['accepted'], 'requests': c['requests'],
'observed_max_inflight': c['observed_max_inflight'], 'latency': c['latency'],
'memory': c['memory'], 'download_ok': c['download']['ok']} for c in report['cases']],
'error': report.get('error')}))
return 0 if report['ok'] else 1
if __name__ == '__main__':
raise SystemExit(main())