forked from mtusi2026/mtusi2026lab6
lab6: Collector, Prometheus/Loki/Tempo/Grafana, alerts, incidents, track 1 (cost of observability)
This commit is contained in:
@@ -0,0 +1,140 @@
|
||||
"""Fixed-rate open-loop workload for lab6 (stdlib only).
|
||||
|
||||
Slot i is planned at t0 + i/rate on a monotonic clock; the generator never
|
||||
waits for earlier requests and never catches up with a burst. When no
|
||||
in-flight slot is available the slot is recorded as client_dropped and stays
|
||||
in the denominator."""
|
||||
import math
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import measurement # noqa: E402
|
||||
import obs # noqa: E402
|
||||
from obs import Failure, stand # noqa: E402
|
||||
|
||||
REQUEST_TIMEOUT = 30
|
||||
MESSAGE_TEXT = 'lab6-track1'
|
||||
|
||||
|
||||
def slot_kind(i, upload_every):
|
||||
if upload_every == 0:
|
||||
return 'message'
|
||||
if upload_every == 1:
|
||||
return 'upload'
|
||||
return 'upload' if (i + 1) % upload_every == 0 else 'message'
|
||||
|
||||
|
||||
class _Local(threading.local):
|
||||
client = None
|
||||
|
||||
|
||||
def run_window(base_url, ca_file, actor, chat, rate, duration, phase_id,
|
||||
upload_every=20, max_inflight=64, stop_event=None, settle=True,
|
||||
obs_client=None):
|
||||
"""Run one window and return {'slots', 'load', 'uploads', 'drain', ...}.
|
||||
|
||||
stop_event stops only new slots; started requests complete. settle=False
|
||||
returns right after HTTP (no job/Collector drain)."""
|
||||
local = _Local()
|
||||
image = stand.sample_bytes()
|
||||
lock = threading.Lock()
|
||||
inflight = [0]
|
||||
planned = []
|
||||
|
||||
def client():
|
||||
if local.client is None:
|
||||
local.client = stand.new_client(base_url, ca_file)
|
||||
local.client.timeout = REQUEST_TIMEOUT
|
||||
return local.client
|
||||
|
||||
def execute(record, kind, t0_mono):
|
||||
record['start'] = time.monotonic() - t0_mono
|
||||
record['start_lag_ms'] = (record['start'] - record['scheduled']) * 1000.0
|
||||
started = time.monotonic()
|
||||
try:
|
||||
c = client()
|
||||
if kind == 'message':
|
||||
result = c.call(
|
||||
'POST', '/api/v1/chats/' + chat['id'] + '/messages',
|
||||
range(200, 600), actor=actor, body={'text': MESSAGE_TEXT},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex,
|
||||
'X-Request-Id': uuid.uuid4().hex},
|
||||
schema='Message')
|
||||
else:
|
||||
result = c.call(
|
||||
'POST', '/api/v1/chats/' + chat['id'] +
|
||||
'/attachments?filename=sample.png', range(200, 600),
|
||||
actor=actor, raw=image,
|
||||
headers={'Content-Type': 'image/png',
|
||||
'X-Request-Id': uuid.uuid4().hex},
|
||||
schema='Attachment')
|
||||
end = time.monotonic()
|
||||
record['status'] = result['status']
|
||||
record['request_id'] = result['headers'].get('X-Request-Id')
|
||||
record['instance_id'] = result['headers'].get('X-Instance-Id')
|
||||
if kind == 'upload' and result['status'] == 202 and result['json']:
|
||||
record['attachment_id'] = result['json']['id']
|
||||
record['outcome'] = 'http'
|
||||
except Failure as error:
|
||||
end = time.monotonic()
|
||||
text = str(error)
|
||||
record['outcome'] = ('timeout' if 'TimeoutError' in text else
|
||||
'transport_error' if 'connection/TLS' in text
|
||||
else 'protocol_error')
|
||||
record['error'] = text[:160]
|
||||
finally:
|
||||
with lock:
|
||||
inflight[0] -= 1
|
||||
record['end'] = end - t0_mono
|
||||
record['service_ms'] = (end - started) * 1000.0
|
||||
record['scheduled_ms'] = (record['end'] - record['scheduled']) * 1000.0
|
||||
|
||||
pool = ThreadPoolExecutor(max_workers=max_inflight)
|
||||
futures = []
|
||||
t0 = time.monotonic()
|
||||
started_utc = obs.utc_now()
|
||||
total = int(math.floor(rate * duration))
|
||||
try:
|
||||
for i in range(total):
|
||||
due = t0 + i / rate
|
||||
while True:
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
break
|
||||
remaining = due - time.monotonic()
|
||||
if remaining <= 0:
|
||||
break
|
||||
time.sleep(min(remaining, 0.05))
|
||||
if stop_event is not None and stop_event.is_set():
|
||||
break
|
||||
kind = slot_kind(i, upload_every)
|
||||
record = {'i': i, 'phase': phase_id, 'name': '%s-%d' % (phase_id, i),
|
||||
'kind': kind, 'scheduled': i / rate, 'outcome': None,
|
||||
'start_lag_ms': None}
|
||||
planned.append(record)
|
||||
with lock:
|
||||
free = inflight[0] < max_inflight
|
||||
if free:
|
||||
inflight[0] += 1
|
||||
if not free:
|
||||
record['outcome'] = 'client_dropped'
|
||||
continue
|
||||
futures.append(pool.submit(execute, record, kind, t0))
|
||||
for future in futures:
|
||||
future.result()
|
||||
finally:
|
||||
pool.shutdown(wait=True)
|
||||
dispatch_end = time.monotonic() - t0
|
||||
load = measurement.summarize_load(planned, rate)
|
||||
uploads = [s['attachment_id'] for s in planned if s.get('attachment_id')]
|
||||
result = {'phase': phase_id, 'slots': planned, 'load': load,
|
||||
'uploads': uploads, 'started': started_utc,
|
||||
'dispatch_seconds': dispatch_end, 'finished_dispatch': obs.utc_now(),
|
||||
'stopped_early': bool(stop_event is not None and stop_event.is_set())}
|
||||
if settle:
|
||||
result['drain'] = obs.wait_drained(uploads, obs=obs_client)
|
||||
return result
|
||||
Reference in New Issue
Block a user