forked from mtusi2026/mtusi2026lab5
241 lines
11 KiB
Python
241 lines
11 KiB
Python
"""Executed via Docker exec stdin. Prints only sanitized measurements as JSON."""
|
|
import argparse
|
|
import base64
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
import random
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from uuid import uuid4
|
|
|
|
sys.path.insert(0, '/app')
|
|
from PIL import Image
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import sessionmaker
|
|
from src.database import create_db_engine
|
|
from src.models import Attachment, AttachmentJob
|
|
from src.object_storage import BUCKET, s3_client
|
|
|
|
BASE = 'http://nginx:8080'
|
|
|
|
|
|
def call(method, path, body=None, token=None, raw=None, headers=None):
|
|
headers = dict(headers or {})
|
|
if token:
|
|
headers['Authorization'] = 'Bearer ' + token
|
|
if body is not None:
|
|
raw = json.dumps(body).encode()
|
|
headers['Content-Type'] = 'application/json'
|
|
request = urllib.request.Request(BASE + path, data=raw, headers=headers, method=method)
|
|
try:
|
|
response = urllib.request.urlopen(request, timeout=60)
|
|
except urllib.error.HTTPError as error:
|
|
response = error
|
|
with response:
|
|
data = response.read()
|
|
payload = json.loads(data) if response.headers.get_content_type() == 'application/json' else data
|
|
return response.status, payload, response.headers
|
|
|
|
|
|
def bootstrap():
|
|
username, password = 'q_' + uuid4().hex[:20], uuid4().hex
|
|
status, user, _ = call('POST', '/api/v1/users', body={
|
|
'username': username, 'password': password, 'display_name': 'Synthetic queue experiment'})
|
|
assert status == 201, ('user', status)
|
|
basic = base64.b64encode(f'{username}:{password}'.encode()).decode()
|
|
status, session, _ = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + basic})
|
|
assert status == 201, ('session', status)
|
|
token = session['token']
|
|
status, chat, _ = call('POST', '/api/v1/chats', token=token,
|
|
body={'title': 'Synthetic queue experiment', 'member_ids': [user['id']]})
|
|
assert status == 201, ('chat', status)
|
|
return user['id'], chat['id'], token
|
|
|
|
|
|
def fixture(width, height):
|
|
pixels = random.Random(202605).randbytes(width * height * 3)
|
|
with Image.frombytes('RGB', (width, height), pixels) as picture, io.BytesIO() as output:
|
|
picture.save(output, format='JPEG', quality=85)
|
|
return output.getvalue()
|
|
|
|
|
|
def percentile(values):
|
|
return sorted(values)[max(0, math.ceil(len(values) * 0.95) - 1)]
|
|
|
|
|
|
def rss_mib():
|
|
# Sum process RSS, not cgroup memory.current (which also counts page cache).
|
|
# Shared pages may be counted twice. Exclude this measurement process.
|
|
total = 0
|
|
for path in Path('/proc').glob('[0-9]*/status'):
|
|
if path.parent.name == str(os.getpid()):
|
|
continue
|
|
try:
|
|
for line in path.read_text().splitlines():
|
|
if line.startswith('VmRSS:'):
|
|
total += int(line.split()[1])
|
|
except (FileNotFoundError, ProcessLookupError):
|
|
pass
|
|
return total / 1024
|
|
|
|
|
|
def rows_for(sessions, owner_id):
|
|
with sessions() as session:
|
|
return list(session.execute(select(Attachment, AttachmentJob).join(
|
|
AttachmentJob, AttachmentJob.attachment_id == Attachment.id)
|
|
.where(Attachment.owner_id == owner_id)))
|
|
|
|
|
|
def summary(rows):
|
|
states = {key: 0 for key in ('queued', 'processing', 'ready', 'failed')}
|
|
ages = []
|
|
for attachment, _ in rows:
|
|
states[attachment.status] += 1
|
|
if attachment.status == 'queued':
|
|
ages.append((datetime.now(timezone.utc) - attachment.created_at).total_seconds())
|
|
return {**states, 'oldest_queued_s': max(ages, default=0)}
|
|
|
|
|
|
def staging_keys():
|
|
return {obj['Key'] for page in s3_client().get_paginator('list_objects_v2').paginate(
|
|
Bucket=BUCKET, Prefix='staging/') for obj in page.get('Contents', [])}
|
|
|
|
|
|
def upload_batch(args, sessions):
|
|
owner, chat, token = bootstrap()
|
|
data = fixture(args.width, args.height)
|
|
before_staging = staging_keys() if args.mode == 'overload' else set()
|
|
samples = []
|
|
started = time.monotonic()
|
|
|
|
def upload(_):
|
|
return call('POST', f'/api/v1/chats/{chat}/attachments?filename=load.jpg', token=token,
|
|
raw=data, headers={'Content-Type': 'image/jpeg'})
|
|
|
|
with ThreadPoolExecutor(max_workers=args.clients) as pool:
|
|
futures = [pool.submit(upload, i) for i in range(args.count)]
|
|
while True:
|
|
rows = rows_for(sessions, owner)
|
|
snapshot = summary(rows)
|
|
samples.append({'elapsed_s': round(time.monotonic() - started, 3),
|
|
'rss_mib': round(rss_mib(), 3), **snapshot})
|
|
if all(f.done() for f in futures):
|
|
responses = [f.result() for f in futures]
|
|
accepted = [payload for status, payload, _ in responses if status == 202]
|
|
if args.mode == 'overload':
|
|
rows = rows_for(sessions, owner)
|
|
snapshot = summary(rows)
|
|
if args.mode == 'overload' or snapshot['ready'] + snapshot['failed'] == len(accepted):
|
|
break
|
|
if time.monotonic() - started > 300:
|
|
raise AssertionError('Batch did not drain in 300 seconds')
|
|
time.sleep(0.1)
|
|
elapsed = time.monotonic() - started
|
|
codes = {str(code): sum(status == code for status, _, _ in responses)
|
|
for code in sorted({status for status, _, _ in responses})}
|
|
replicas = len({headers['X-Instance-Id'] for _, _, headers in responses})
|
|
result = {'width': args.width, 'height': args.height, 'bytes': len(data),
|
|
'sha256': hashlib.sha256(data).hexdigest(), 'requests': args.count,
|
|
'clients': args.clients, 'http_statuses': codes, 'api_replicas_seen': replicas,
|
|
'elapsed_s': elapsed, 'owner_id': owner, 'attachment_ids': [a['id'] for a in accepted]}
|
|
assert len(rows) == len(accepted), 'Rejected upload created metadata'
|
|
assert replicas == 2, 'Experiment must exercise both API replicas'
|
|
if args.mode == 'overload':
|
|
assert len(accepted) == args.limit, (codes, args.limit)
|
|
assert codes.get('429') == args.count - args.limit, codes
|
|
assert all(a.status == 'queued' for a, _ in rows), 'Worker was not stopped'
|
|
assert staging_keys() - before_staging == {'staging/' + a['id'] for a in accepted}, 'Rejected upload left S3 object'
|
|
for status, payload, headers in responses:
|
|
if status == 429:
|
|
assert payload['error']['code'] == 'rate_limited'
|
|
assert int(headers['Retry-After']) > 0
|
|
assert call('GET', '/health/ready')[0] == 200
|
|
assert call('GET', f"/api/v1/attachments/{accepted[0]['id']}/content", token=token)[0] == 409
|
|
assert call('POST', f'/api/v1/chats/{chat}/messages', token=token,
|
|
body={'text': 'Still responsive under overload'},
|
|
headers={'Idempotency-Key': uuid4().hex})[0] == 201
|
|
result.update({'limit': args.limit, 'accepted': len(accepted), 'rejected': codes['429'],
|
|
'health_and_messages_available': True, 'rejected_leave_no_objects': True})
|
|
else:
|
|
assert codes == {'202': args.count}, codes
|
|
assert all(a.status == 'ready' and j.attempt == 1 for a, j in rows), 'Failed/retried job in baseline'
|
|
waits = [(j.first_started_at - j.created_at).total_seconds() for _, j in rows]
|
|
latencies = [(a.updated_at - a.created_at).total_seconds() for a, _ in rows]
|
|
result.update({'throughput_per_s': len(rows) / elapsed,
|
|
'ready_p95_s': percentile(latencies), 'queue_wait_p95_s': percentile(waits),
|
|
'queue_lag_max_jobs': max(s['queued'] for s in samples),
|
|
'queue_oldest_max_s': max(s['oldest_queued_s'] for s in samples),
|
|
'processing_max': max(s['processing'] for s in samples),
|
|
'rss_peak_mib': max(s['rss_mib'] for s in samples),
|
|
'samples': samples})
|
|
return result
|
|
|
|
|
|
def recover(args, sessions):
|
|
expected = json.loads(args.expected)
|
|
started = time.monotonic()
|
|
while time.monotonic() - started < 180:
|
|
rows = rows_for(sessions, expected['owner_id'])
|
|
assert not any(a.status == 'failed' for a, _ in rows), 'Accepted job failed'
|
|
if len(rows) == expected['accepted'] and all(a.status == 'ready' for a, _ in rows):
|
|
break
|
|
time.sleep(0.2)
|
|
else:
|
|
raise AssertionError('Accepted work did not recover')
|
|
assert {a.id for a, _ in rows} == set(expected['attachment_ids']), 'Lost/replaced accepted IDs'
|
|
for attachment, _ in rows:
|
|
keys = [attachment.object_key] + [v['object_key'] for v in attachment.variants]
|
|
assert len(attachment.variants) == 3
|
|
for key in keys:
|
|
s3_client().head_object(Bucket=BUCKET, Key=key)
|
|
with s3_client().get_object(Bucket=BUCKET, Key=attachment.object_key)['Body'] as body:
|
|
assert hashlib.sha256(body.read()).hexdigest() == expected['sha256']
|
|
elapsed = time.monotonic() - started
|
|
# A new accepted upload proves capacity is released after draining.
|
|
# One request need not hit both replicas, so perform this explicitly.
|
|
owner, chat, token = bootstrap()
|
|
status, _, _ = call('POST', f'/api/v1/chats/{chat}/attachments?filename=after.jpg', token=token,
|
|
raw=fixture(320, 200), headers={'Content-Type': 'image/jpeg'})
|
|
assert status == 202, 'Admission did not recover'
|
|
deadline = time.monotonic() + 60
|
|
while time.monotonic() < deadline:
|
|
fresh = rows_for(sessions, owner)
|
|
if fresh and fresh[0][0].status == 'ready':
|
|
break
|
|
time.sleep(0.2)
|
|
else:
|
|
raise AssertionError('New upload did not complete after overload')
|
|
return {'accepted_ids_preserved': True, 'all_ready': len(rows), 'original_checksums_match': True,
|
|
'all_variants_exist': True, 'drain_observed_s': elapsed, 'new_upload_status': status}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('mode', choices=['measure', 'overload', 'recover'])
|
|
parser.add_argument('--width', type=int, default=320)
|
|
parser.add_argument('--height', type=int, default=200)
|
|
parser.add_argument('--count', type=int, default=18)
|
|
parser.add_argument('--clients', type=int, default=6)
|
|
parser.add_argument('--limit', type=int, default=12)
|
|
parser.add_argument('--expected')
|
|
args = parser.parse_args()
|
|
engine = create_db_engine()
|
|
try:
|
|
sessions = sessionmaker(engine, expire_on_commit=False)
|
|
result = recover(args, sessions) if args.mode == 'recover' else upload_batch(args, sessions)
|
|
print(json.dumps(result))
|
|
finally:
|
|
engine.dispose()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|