Files
2026-11-06 19:22:27 +03:00

241 lines
11 KiB
Python

"""Executed via Docker exec stdin. Prints only sanitized measurements as JSON."""
import argparse
import base64
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
import hashlib
import io
import json
import math
import os
from pathlib import Path
import random
import sys
import time
import urllib.error
import urllib.request
from uuid import uuid4
sys.path.insert(0, '/app')
from PIL import Image
from sqlalchemy import select
from sqlalchemy.orm import sessionmaker
from src.database import create_db_engine
from src.models import Attachment, AttachmentJob
from src.object_storage import BUCKET, s3_client
BASE = 'http://nginx:8080'
def call(method, path, body=None, token=None, raw=None, headers=None):
headers = dict(headers or {})
if token:
headers['Authorization'] = 'Bearer ' + token
if body is not None:
raw = json.dumps(body).encode()
headers['Content-Type'] = 'application/json'
request = urllib.request.Request(BASE + path, data=raw, headers=headers, method=method)
try:
response = urllib.request.urlopen(request, timeout=60)
except urllib.error.HTTPError as error:
response = error
with response:
data = response.read()
payload = json.loads(data) if response.headers.get_content_type() == 'application/json' else data
return response.status, payload, response.headers
def bootstrap():
username, password = 'q_' + uuid4().hex[:20], uuid4().hex
status, user, _ = call('POST', '/api/v1/users', body={
'username': username, 'password': password, 'display_name': 'Synthetic queue experiment'})
assert status == 201, ('user', status)
basic = base64.b64encode(f'{username}:{password}'.encode()).decode()
status, session, _ = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + basic})
assert status == 201, ('session', status)
token = session['token']
status, chat, _ = call('POST', '/api/v1/chats', token=token,
body={'title': 'Synthetic queue experiment', 'member_ids': [user['id']]})
assert status == 201, ('chat', status)
return user['id'], chat['id'], token
def fixture(width, height):
pixels = random.Random(202605).randbytes(width * height * 3)
with Image.frombytes('RGB', (width, height), pixels) as picture, io.BytesIO() as output:
picture.save(output, format='JPEG', quality=85)
return output.getvalue()
def percentile(values):
return sorted(values)[max(0, math.ceil(len(values) * 0.95) - 1)]
def rss_mib():
# Sum process RSS, not cgroup memory.current (which also counts page cache).
# Shared pages may be counted twice. Exclude this measurement process.
total = 0
for path in Path('/proc').glob('[0-9]*/status'):
if path.parent.name == str(os.getpid()):
continue
try:
for line in path.read_text().splitlines():
if line.startswith('VmRSS:'):
total += int(line.split()[1])
except (FileNotFoundError, ProcessLookupError):
pass
return total / 1024
def rows_for(sessions, owner_id):
with sessions() as session:
return list(session.execute(select(Attachment, AttachmentJob).join(
AttachmentJob, AttachmentJob.attachment_id == Attachment.id)
.where(Attachment.owner_id == owner_id)))
def summary(rows):
states = {key: 0 for key in ('queued', 'processing', 'ready', 'failed')}
ages = []
for attachment, _ in rows:
states[attachment.status] += 1
if attachment.status == 'queued':
ages.append((datetime.now(timezone.utc) - attachment.created_at).total_seconds())
return {**states, 'oldest_queued_s': max(ages, default=0)}
def staging_keys():
return {obj['Key'] for page in s3_client().get_paginator('list_objects_v2').paginate(
Bucket=BUCKET, Prefix='staging/') for obj in page.get('Contents', [])}
def upload_batch(args, sessions):
owner, chat, token = bootstrap()
data = fixture(args.width, args.height)
before_staging = staging_keys() if args.mode == 'overload' else set()
samples = []
started = time.monotonic()
def upload(_):
return call('POST', f'/api/v1/chats/{chat}/attachments?filename=load.jpg', token=token,
raw=data, headers={'Content-Type': 'image/jpeg'})
with ThreadPoolExecutor(max_workers=args.clients) as pool:
futures = [pool.submit(upload, i) for i in range(args.count)]
while True:
rows = rows_for(sessions, owner)
snapshot = summary(rows)
samples.append({'elapsed_s': round(time.monotonic() - started, 3),
'rss_mib': round(rss_mib(), 3), **snapshot})
if all(f.done() for f in futures):
responses = [f.result() for f in futures]
accepted = [payload for status, payload, _ in responses if status == 202]
if args.mode == 'overload':
rows = rows_for(sessions, owner)
snapshot = summary(rows)
if args.mode == 'overload' or snapshot['ready'] + snapshot['failed'] == len(accepted):
break
if time.monotonic() - started > 300:
raise AssertionError('Batch did not drain in 300 seconds')
time.sleep(0.1)
elapsed = time.monotonic() - started
codes = {str(code): sum(status == code for status, _, _ in responses)
for code in sorted({status for status, _, _ in responses})}
replicas = len({headers['X-Instance-Id'] for _, _, headers in responses})
result = {'width': args.width, 'height': args.height, 'bytes': len(data),
'sha256': hashlib.sha256(data).hexdigest(), 'requests': args.count,
'clients': args.clients, 'http_statuses': codes, 'api_replicas_seen': replicas,
'elapsed_s': elapsed, 'owner_id': owner, 'attachment_ids': [a['id'] for a in accepted]}
assert len(rows) == len(accepted), 'Rejected upload created metadata'
assert replicas == 2, 'Experiment must exercise both API replicas'
if args.mode == 'overload':
assert len(accepted) == args.limit, (codes, args.limit)
assert codes.get('429') == args.count - args.limit, codes
assert all(a.status == 'queued' for a, _ in rows), 'Worker was not stopped'
assert staging_keys() - before_staging == {'staging/' + a['id'] for a in accepted}, 'Rejected upload left S3 object'
for status, payload, headers in responses:
if status == 429:
assert payload['error']['code'] == 'rate_limited'
assert int(headers['Retry-After']) > 0
assert call('GET', '/health/ready')[0] == 200
assert call('GET', f"/api/v1/attachments/{accepted[0]['id']}/content", token=token)[0] == 409
assert call('POST', f'/api/v1/chats/{chat}/messages', token=token,
body={'text': 'Still responsive under overload'},
headers={'Idempotency-Key': uuid4().hex})[0] == 201
result.update({'limit': args.limit, 'accepted': len(accepted), 'rejected': codes['429'],
'health_and_messages_available': True, 'rejected_leave_no_objects': True})
else:
assert codes == {'202': args.count}, codes
assert all(a.status == 'ready' and j.attempt == 1 for a, j in rows), 'Failed/retried job in baseline'
waits = [(j.first_started_at - j.created_at).total_seconds() for _, j in rows]
latencies = [(a.updated_at - a.created_at).total_seconds() for a, _ in rows]
result.update({'throughput_per_s': len(rows) / elapsed,
'ready_p95_s': percentile(latencies), 'queue_wait_p95_s': percentile(waits),
'queue_lag_max_jobs': max(s['queued'] for s in samples),
'queue_oldest_max_s': max(s['oldest_queued_s'] for s in samples),
'processing_max': max(s['processing'] for s in samples),
'rss_peak_mib': max(s['rss_mib'] for s in samples),
'samples': samples})
return result
def recover(args, sessions):
expected = json.loads(args.expected)
started = time.monotonic()
while time.monotonic() - started < 180:
rows = rows_for(sessions, expected['owner_id'])
assert not any(a.status == 'failed' for a, _ in rows), 'Accepted job failed'
if len(rows) == expected['accepted'] and all(a.status == 'ready' for a, _ in rows):
break
time.sleep(0.2)
else:
raise AssertionError('Accepted work did not recover')
assert {a.id for a, _ in rows} == set(expected['attachment_ids']), 'Lost/replaced accepted IDs'
for attachment, _ in rows:
keys = [attachment.object_key] + [v['object_key'] for v in attachment.variants]
assert len(attachment.variants) == 3
for key in keys:
s3_client().head_object(Bucket=BUCKET, Key=key)
with s3_client().get_object(Bucket=BUCKET, Key=attachment.object_key)['Body'] as body:
assert hashlib.sha256(body.read()).hexdigest() == expected['sha256']
elapsed = time.monotonic() - started
# A new accepted upload proves capacity is released after draining.
# One request need not hit both replicas, so perform this explicitly.
owner, chat, token = bootstrap()
status, _, _ = call('POST', f'/api/v1/chats/{chat}/attachments?filename=after.jpg', token=token,
raw=fixture(320, 200), headers={'Content-Type': 'image/jpeg'})
assert status == 202, 'Admission did not recover'
deadline = time.monotonic() + 60
while time.monotonic() < deadline:
fresh = rows_for(sessions, owner)
if fresh and fresh[0][0].status == 'ready':
break
time.sleep(0.2)
else:
raise AssertionError('New upload did not complete after overload')
return {'accepted_ids_preserved': True, 'all_ready': len(rows), 'original_checksums_match': True,
'all_variants_exist': True, 'drain_observed_s': elapsed, 'new_upload_status': status}
def main():
parser = argparse.ArgumentParser()
parser.add_argument('mode', choices=['measure', 'overload', 'recover'])
parser.add_argument('--width', type=int, default=320)
parser.add_argument('--height', type=int, default=200)
parser.add_argument('--count', type=int, default=18)
parser.add_argument('--clients', type=int, default=6)
parser.add_argument('--limit', type=int, default=12)
parser.add_argument('--expected')
args = parser.parse_args()
engine = create_db_engine()
try:
sessions = sessionmaker(engine, expire_on_commit=False)
result = recover(args, sessions) if args.mode == 'recover' else upload_batch(args, sessions)
print(json.dumps(result))
finally:
engine.dispose()
if __name__ == '__main__':
main()