Files
mtusi2026lab6/scripts/incident_worker.py

96 lines
3.7 KiB
Python

import argparse
import base64
import json
import secrets
import subprocess
import time
import urllib.error
import urllib.request
from pathlib import Path
BASE = 'http://localhost:8080'
def call(method, url, body=None, headers=None):
request = urllib.request.Request(url, data=body, headers=headers or {}, method=method)
try:
with urllib.request.urlopen(request, timeout=15) as response:
raw = response.read()
return response.status, json.loads(raw) if raw else {}
except urllib.error.HTTPError as error:
raw = error.read()
return error.code, json.loads(raw) if raw else {}
def json_request(method, path, data, auth=None):
headers = {'Content-Type':'application/json'}
if auth:
headers['Authorization'] = 'Bearer '+auth
return call(method, BASE+path, json.dumps(data).encode(), headers)
def docker(*args):
subprocess.run(['docker', 'compose', *args], check=True)
def alerts():
try:
code, result = call('GET', 'http://localhost:9090/api/v1/alerts')
if code == 200:
return {a['labels'].get('alertname'):a['state'] for a in result['data']['alerts']}
except Exception:
pass
return {}
def main():
cli = argparse.ArgumentParser()
cli.add_argument('--hold', type=int, default=50)
opts = cli.parse_args()
username = 'obs'+secrets.token_hex(6)
password = secrets.token_urlsafe(24)
status, user = json_request('POST','/api/v1/users',{'username':username,'password':password,'display_name':'Lab6 Test'})
assert status == 201, user
basic = base64.b64encode((username+':'+password).encode()).decode()
status, auth = call('POST',BASE+'/api/v1/auth/sessions',b'',{'Authorization':'Basic '+basic})
assert status == 201, auth
token = auth['token']
status, chat = json_request('POST','/api/v1/chats',{'title':'Lab6 Incident','member_ids':[user['id']]},token)
assert status == 201, chat
docker('stop','worker')
ready = False
try:
sample = Path('tests/fixtures/sample.png').read_bytes()
status, attachment = call('POST', BASE+f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png",
sample, {'Content-Type':'image/png','Authorization':'Bearer '+token})
assert status == 202 and attachment['status'] == 'queued', (status,attachment)
attachment_id = attachment['id']
status, _ = call('GET',BASE+f'/api/v1/attachments/{attachment_id}/content',None,{'Authorization':'Bearer '+token})
assert status == 409, status
print('202 queued; content 409; worker stopped')
time.sleep(opts.hold)
before = alerts().get('MessengerQueueLagGrowing','not_observed')
print('QueueLag alert:',before)
finally:
docker('start','worker')
deadline = time.monotonic()+100
while time.monotonic()<deadline:
status, attachment = call('GET',BASE+f'/api/v1/attachments/{attachment_id}',None,{'Authorization':'Bearer '+token})
if status == 200 and attachment['status'] == 'ready':
ready = True
break
time.sleep(2)
assert ready, 'Attachment did not become ready after restoring worker'
names = sorted(v['name'] for v in attachment['variants'])
assert names == ['crop','optimized','thumbnail'], names
result = {'scenario':'stop worker -> 202 queued -> restart -> ready',
'attachment_id':attachment_id,'status':'ready','variants':names,
'queue_alert_before_worker_start':before,'passed':True}
Path('evidence').mkdir(exist_ok=True)
Path('evidence/worker-incident.json').write_text(json.dumps(result,indent=2),encoding='utf-8')
print('PASS worker recovery:',json.dumps(result,ensure_ascii=False))
if __name__=='__main__':
main()