Implement lab6 observability with OpenTelemetry and Grafana

This commit is contained in:
ferraronp
2026-11-17 19:14:43 +03:00
parent 470d9d6f1e
commit e636d4cc76
68 changed files with 5863 additions and 47 deletions
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env python3
"""Additional static checks (not the teacher's make check and not a live deploy)."""
import ast,json,re
from pathlib import Path
import yaml
ROOT=Path(__file__).resolve().parents[1]
def main():
count=0
for path in (ROOT/'app').glob('*.py'):
tree=ast.parse(path.read_text(encoding='utf-8'))
for n in ast.walk(tree):
if isinstance(n,ast.Call) and isinstance(n.func,ast.Attribute) and n.func.attr=='execute' and len(n.args)>1 and isinstance(n.args[0],ast.Constant) and isinstance(n.args[0].value,str) and isinstance(n.args[1],(ast.Tuple,ast.List)):
assert n.args[0].value.count('%s')==len(n.args[1].elts),f'SQL binding mismatch: {path.name}:{n.lineno}'
count+=1
compose=yaml.safe_load((ROOT/'compose.yaml').read_text())
for name,svc in compose['services'].items():
if 'image' in svc:assert ':' in svc['image'] and not svc['image'].endswith(':latest'),name
for port in svc.get('ports',[]):assert port.startswith('127.0.0.1:'),name
for mount in svc.get('volumes',[]):
src=mount.split(':')[0]
assert (ROOT/src).exists() if src.startswith('./') else src in compose['volumes'],f'missing mount {src}'
for network in svc.get('networks',[]):assert network in compose['networks']
for dep in svc.get('depends_on',{}):assert dep in compose['services']
for p in (ROOT/'infra/otel').rglob('*'):
if p.suffix in ('.yml','.yaml'):yaml.safe_load(p.read_text())
services=compose['services']; assert all('telemetry' in services[s]['networks'] for s in ('api1','api2','dispatcher','worker','otel-collector','prometheus','loki','tempo','grafana'))
for v in ('prometheus_data','loki_data','tempo_data','grafana_data'):assert v in compose['volumes']
collector=yaml.safe_load((ROOT/'infra/otel/collector.yaml').read_text())
for signal in ('metrics','logs','traces'):
pipe=collector['service']['pipelines'][signal]
assert pipe['receivers']==['otlp'] and pipe['processors']==['memory_limiter','batch']
assert all(e in collector['exporters'] for e in pipe['exporters'])
for exp in ('otlp/tempo','otlphttp/loki'):
conf=collector['exporters'][exp];assert conf['sending_queue']['queue_size']==64 and conf['retry_on_failure']['max_elapsed_time']=='10s'
dashboard=json.loads((ROOT/'infra/otel/grafana/dashboards/messenger.json').read_text())
expressions='\n'.join(t['expr'] for p in dashboard['panels'] for t in p['targets'])
for metric in ('lab_http_requests_total','lab_http_duration_seconds_bucket','lab_jobs_total','lab_job_duration_seconds_bucket','lab_ready_duration_seconds_bucket','lab_queue_depth','lab_queue_oldest_age_seconds','lab_broker_ready','lab_db_pool_used','lab_db_pool_waiting','lab_worker_busy','lab_worker_capacity','lab_container_cpu_cores','lab_container_memory_bytes'):
assert metric in expressions,metric
assert all(f'histogram_quantile({q}' in expressions for q in (.5,.95,.99))
rules=yaml.safe_load((ROOT/'infra/otel/prometheus/alerts.yml').read_text())
assert {x['alert'] for g in rules['groups'] for x in g['rules']}=={'LabAPIHighErrorRate','LabQueueLag'}
assert all(x.get('for') and x['annotations'].get('runbook') for g in rules['groups'] for x in g['rules'])
print(f'PASS: {count} literal SQL bindings; pinned Compose images, mounts and networks; 3 OTLP pipelines, dashboards and alert rules (static only)')
if __name__=='__main__':main()
+11
View File
@@ -0,0 +1,11 @@
@echo off
setlocal
cd /d "%~dp0.."
docker compose --env-file .env stop --timeout 5 lb api1 api2 worker dispatcher broker s3 redis postgres grafana otel-collector prometheus loki tempo
if errorlevel 1 exit /b 1
docker compose --env-file .env up -d --wait postgres redis broker s3 s3-bootstrap prometheus loki tempo otel-collector grafana
if errorlevel 1 exit /b 1
docker compose --env-file .env up -d --no-deps api1 api2 lb dispatcher worker
if errorlevel 1 exit /b 1
python scripts\wait_ready.py
exit /b %ERRORLEVEL%
+21
View File
@@ -0,0 +1,21 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# Preserve named volumes. Restart all state stores and both API processes.
docker compose --env-file .env stop --timeout 5 lb api1 api2 worker dispatcher broker s3 redis postgres grafana otel-collector prometheus loki tempo
docker compose --env-file .env up -d --wait postgres redis broker s3 s3-bootstrap prometheus loki tempo otel-collector grafana
docker compose --env-file .env up -d --no-deps api1 api2 lb dispatcher worker
python - <<'PY'
import time, urllib.request
for i in range(120):
try:
with urllib.request.urlopen('http://localhost:8080/health/ready', timeout=2) as response:
if response.status == 200:
print('[restart] all services ready')
break
except Exception:
pass
time.sleep(.5)
else:
raise SystemExit('[restart] health timeout')
PY
+10
View File
@@ -0,0 +1,10 @@
@echo off
setlocal
cd /d "%~dp0.."
if "%TARGET_INSTANCE_ID%"=="api1" goto stop
if "%TARGET_INSTANCE_ID%"=="api2" goto stop
echo Unexpected TARGET_INSTANCE_ID 1>&2
exit /b 2
:stop
docker compose --env-file .env stop --timeout 1 %TARGET_INSTANCE_ID%
exit /b %ERRORLEVEL%
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
# The public failover test sets TARGET_INSTANCE_ID from the actual response header.
set -eu
cd "$(dirname "$0")/.."
case "${TARGET_INSTANCE_ID:?missing TARGET_INSTANCE_ID}" in
api1|api2) docker compose --env-file .env stop --timeout 1 "$TARGET_INSTANCE_ID" ;;
*) echo 'Unexpected instance alias' >&2; exit 2 ;;
esac
+14
View File
@@ -0,0 +1,14 @@
"""Operator-facing readiness wait after container replacement."""
import time
import urllib.request
for _ in range(120):
try:
with urllib.request.urlopen("http://localhost:8080/health/ready", timeout=2) as response:
if response.status == 200:
print("[restart] all services ready")
break
except Exception:
pass
time.sleep(.5)
else:
raise SystemExit("[restart] health timeout")