Implement lab6 OpenTelemetry observability stack

This commit is contained in:
2026-11-21 19:29:20 +03:00
parent 470d9d6f1e
commit ad8db1f32b
54 changed files with 5116 additions and 19 deletions
+24
View File
@@ -0,0 +1,24 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
echo "Stopping API and image processing..."
docker compose stop api1 api2 worker dispatcher
echo "Restarting persistent storage and broker..."
docker compose restart db minio rabbitmq
echo "Waiting for PostgreSQL and RabbitMQ..."
docker compose up -d --wait db rabbitmq
echo "Checking S3 bucket..."
docker compose run --rm --no-deps s3-init
echo "Starting dispatcher and worker..."
docker compose up -d dispatcher worker
echo "Starting API replicas..."
docker compose up -d --wait api1 api2
echo "Restart complete. Persistent volumes preserved."
+33
View File
@@ -0,0 +1,33 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
if [[ -z "${TARGET_INSTANCE_ID:-}" ]]; then
echo "TARGET_INSTANCE_ID is not set" >&2
exit 1
fi
for service in api1 api2; do
instance_id=$(
docker compose exec -T "$service" python -c '
import urllib.request
response = urllib.request.urlopen(
"http://127.0.0.1:8080/health/live",
timeout=2
)
print(response.headers.get("X-Instance-Id", ""))
' 2>/dev/null
) || continue
if [[ "$instance_id" == "$TARGET_INSTANCE_ID" ]]; then
echo "Stopping $service"
docker compose stop -t 2 "$service"
exit 0
fi
done
echo "Target API instance not found" >&2
exit 1
+175
View File
@@ -0,0 +1,175 @@
import base64
import json
import subprocess
import time
import urllib.error
import urllib.request
import uuid
BASE_URL = "http://127.0.0.1:8080"
def docker(*args):
subprocess.run(
["docker", "compose", *args],
check=True,
)
def request(method, path, body=None, headers=None):
request_headers = dict(headers or {})
if isinstance(body, dict):
body = json.dumps(body).encode("utf-8")
request_headers["Content-Type"] = "application/json"
req = urllib.request.Request(
BASE_URL + path,
data=body,
headers=request_headers,
method=method,
)
started = time.monotonic()
try:
with urllib.request.urlopen(req, timeout=5) as response:
response.read()
return response.status, time.monotonic() - started
except urllib.error.HTTPError as exc:
exc.read()
return exc.code, time.monotonic() - started
except (urllib.error.URLError, TimeoutError) as exc:
return type(exc).__name__, time.monotonic() - started
def login_header(username, password):
encoded = base64.b64encode(
f"{username}:{password}".encode("utf-8")
).decode("ascii")
return {
"Authorization": f"Basic {encoded}"
}
def wait_for_recovery(headers):
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
status, duration = request(
"POST",
"/api/v1/auth/sessions",
body=b"",
headers=headers,
)
if status == 201:
print(
f"Recovery: HTTP 201, {duration:.3f}s",
flush=True,
)
return
time.sleep(2)
raise RuntimeError(
"Authentication did not recover after DB restart"
)
def main():
suffix = uuid.uuid4().hex[:12]
username = "lab6_db_" + suffix
password = "IncidentTest_" + suffix
status, duration = request(
"POST",
"/api/v1/users",
body={
"username": username,
"display_name": "DB incident test",
"password": password,
},
)
if status != 201:
raise RuntimeError(
f"Cannot create test user: HTTP {status}"
)
headers = login_header(username, password)
status, _ = request(
"POST",
"/api/v1/auth/sessions",
body=b"",
headers=headers,
)
if status != 201:
raise RuntimeError(
f"Baseline login failed: HTTP {status}"
)
print("Baseline: authentication HTTP 201", flush=True)
observed_failures = 0
print("INCIDENT START: stopping PostgreSQL", flush=True)
try:
docker("stop", "db")
for attempt in range(1, 6):
status, duration = request(
"POST",
"/api/v1/auth/sessions",
body=b"",
headers=headers,
)
print(
f"Probe {attempt}: status={status}, "
f"duration={duration:.3f}s",
flush=True,
)
if status != 201:
observed_failures += 1
time.sleep(2)
finally:
print(
"INCIDENT END: restoring PostgreSQL",
flush=True,
)
docker("up", "-d", "--wait", "db")
wait_for_recovery(headers)
print(
f"Failed requests during incident: "
f"{observed_failures}/5",
flush=True,
)
if observed_failures == 0:
raise RuntimeError(
"The database incident did not affect requests"
)
print(
"PASS: database failure observed and recovered",
flush=True,
)
if __name__ == "__main__":
main()
+77
View File
@@ -0,0 +1,77 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
sql() {
docker compose exec -T db \
psql -U messenger -d messenger -At -c "$1"
}
echo "Finding completed image job..."
JOB_ID=$(sql "
SELECT j.id
FROM attachment_jobs j
JOIN attachment_variants v
ON v.attachment_id = j.attachment_id
WHERE j.status = 'ready'
AND v.variant_name = 'thumbnail'
ORDER BY j.created_at DESC
LIMIT 1;
")
if [[ -z "$JOB_ID" ]]; then
echo "FAIL: no completed image job found"
exit 1
fi
echo "Job: $JOB_ID"
STATE_QUERY="
SELECT
j.status || '|' ||
j.attempts || '|' ||
v.sha256 || '|' ||
(
SELECT COUNT(*)
FROM attachment_variants
WHERE attachment_id = j.attachment_id
)
FROM attachment_jobs j
JOIN attachment_variants v
ON v.attachment_id = j.attachment_id
WHERE j.id = '$JOB_ID'
AND v.variant_name = 'thumbnail';
"
BEFORE=$(sql "$STATE_QUERY")
echo "Before: $BEFORE"
echo "Publishing the same job twice..."
docker compose run --rm --no-deps api1 \
python -c '
import sys
from src.queue import publish_job
job_id = sys.argv[1]
publish_job(job_id)
publish_job(job_id)
print("Two duplicate messages published")
' "$JOB_ID"
sleep 5
AFTER=$(sql "$STATE_QUERY")
echo "After: $AFTER"
if [[ "$BEFORE" != "$AFTER" ]]; then
echo "FAIL: completed job changed after redelivery"
exit 1
fi
echo "PASS: duplicate delivery did not change completed job"
echo "PASS: thumbnail checksum and variant count preserved"
+154
View File
@@ -0,0 +1,154 @@
import json
import re
import time
import urllib.parse
import urllib.request
LOKI = "http://loki:3100"
TEMPO = "http://tempo:3200"
TRACE_PATTERN = re.compile(r"^[0-9a-f]{32}$")
SPAN_PATTERN = re.compile(r"^[0-9a-f]{16}$")
def get_json(url):
with urllib.request.urlopen(url, timeout=15) as response:
return json.load(response)
def find_log(service, event):
params = urllib.parse.urlencode({
"query": f'{{service_name="{service}"}}',
"start": str(time.time_ns() - 6 * 3600 * 10**9),
"end": str(time.time_ns()),
"limit": 500,
"direction": "BACKWARD",
})
payload = get_json(
f"{LOKI}/loki/api/v1/query_range?{params}"
)
streams = payload.get("data", {}).get("result", [])
for stream in streams:
labels = stream.get("stream", {})
for entry in stream.get("values", []):
if len(entry) < 2:
continue
message = entry[1]
metadata = entry[2] if len(entry) >= 3 else {}
if event not in message and (
metadata.get("event") != event
):
continue
trace_id = (
metadata.get("trace_id")
or labels.get("trace_id")
)
span_id = (
metadata.get("span_id")
or labels.get("span_id")
)
if not trace_id or not TRACE_PATTERN.fullmatch(
trace_id
):
continue
if not span_id or not SPAN_PATTERN.fullmatch(
span_id
):
continue
return {
"service": service,
"event": event,
"message": message,
"trace_id": trace_id,
"span_id": span_id,
}
return None
def verify_trace(log, expected_kind):
trace_id = log["trace_id"]
payload = get_json(
f"{TEMPO}/api/traces/{trace_id}"
)
batches = payload.get("batches", [])
spans = []
for batch in batches:
for scope in batch.get("scopeSpans", []):
spans.extend(scope.get("spans", []))
if not spans:
raise AssertionError(
f"No spans found for {trace_id}"
)
kinds = [str(span.get("kind", "")) for span in spans]
if not any(expected_kind in kind for kind in kinds):
raise AssertionError(
f"Expected span kind {expected_kind}; got {kinds}"
)
print(f"Service: {log['service']}")
print(f"Event: {log['event']}")
print(f"Trace ID: {trace_id}")
print(f"Span ID: {log['span_id']}")
print(f"Tempo spans: {len(spans)}")
for span in spans:
print(
" Span:",
span.get("name"),
"kind:",
span.get("kind"),
)
print("PASS: Loki log linked to Tempo trace")
print()
def main():
cases = [
(
"messenger-api",
"HTTP request completed",
"SERVER",
),
(
"messenger-worker",
"Image job handled",
"CONSUMER",
),
]
for service, event, kind in cases:
log = find_log(service, event)
if log is None:
raise AssertionError(
f"No correlated log found for {service}"
)
verify_trace(log, kind)
print("PASS: API and worker trace correlation")
if __name__ == "__main__":
main()
+256
View File
@@ -0,0 +1,256 @@
import os
import base64
import hashlib
import json
import subprocess
import time
import urllib.error
import urllib.parse
import urllib.request
import uuid
import zlib
import struct
BASE = "http://127.0.0.1:8080"
def docker(*args):
subprocess.run(
["docker", "compose", *args],
check=True,
)
def api(method, path, token=None, body=None, headers=None):
request_headers = dict(headers or {})
if token:
request_headers["Authorization"] = f"Bearer {token}"
if isinstance(body, dict):
body = json.dumps(body).encode("utf-8")
request_headers["Content-Type"] = "application/json"
request = urllib.request.Request(
BASE + path,
data=body,
headers=request_headers,
method=method,
)
with urllib.request.urlopen(request, timeout=10) as response:
data = response.read()
return (
response.status,
json.loads(data) if data and
"application/json" in response.headers.get(
"Content-Type", ""
) else data,
)
def png_chunk(name, data):
value = name + data
return (
struct.pack(">I", len(data))
+ value
+ struct.pack(">I", zlib.crc32(value) & 0xFFFFFFFF)
)
def make_png():
width, height = 320, 200
scanline = b"\x00" + bytes([220, 80, 40]) * width
return (
b"\x89PNG\r\n\x1a\n"
+ png_chunk(
b"IHDR",
struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0),
)
+ png_chunk(
b"IDAT", zlib.compress(scanline * height)
)
+ png_chunk(b"IEND", b"")
)
def wait_ready():
for _ in range(90):
try:
status, data = api("GET", "/health/ready")
if status == 200 and data.get("status") == "ok":
return
except Exception:
pass
time.sleep(1)
raise RuntimeError("API did not become ready")
def main():
suffix = uuid.uuid4().hex[:12]
username = "lab5_" + suffix
password = "Lab5TestPassword_" + suffix
print("Stopping worker...", flush=True)
docker("stop", "worker")
wait_ready()
status, user = api(
"POST",
"/api/v1/users",
body={
"username": username,
"display_name": "Lab5 recovery test",
"password": password,
},
)
assert status == 201
credentials = base64.b64encode(
f"{username}:{password}".encode()
).decode()
status, session = api(
"POST",
"/api/v1/auth/sessions",
headers={"Authorization": f"Basic {credentials}"},
body=b"",
)
assert status == 201
token = session["token"]
status, chat = api(
"POST",
"/api/v1/chats",
token,
{
"title": "Worker recovery",
"member_ids": [user["id"]],
},
)
assert status == 201
original = make_png()
path = (
f"/api/v1/chats/{chat['id']}/attachments"
"?filename=recovery.png"
)
status, attachment = api(
"POST",
path,
token,
original,
{"Content-Type": "image/png"},
)
assert status == 202, f"Expected 202, got {status}"
assert attachment["status"] == "queued"
attachment_id = attachment["id"]
print("Upload: 202 queued", flush=True)
status, message = api(
"POST",
f"/api/v1/chats/{chat['id']}/messages",
token,
{"text": "", "attachment_ids": [attachment_id]},
{"Idempotency-Key": "recovery-" + suffix},
)
assert status == 201
assert attachment_id in message["attachment_ids"]
print("Message created while worker is stopped", flush=True)
status, metadata = api(
"GET", f"/api/v1/attachments/{attachment_id}", token
)
assert status == 200 and metadata["status"] == "queued"
print("Restarting RabbitMQ and API...", flush=True)
docker("restart", "rabbitmq")
docker("up", "-d", "--wait", "rabbitmq")
docker("restart", "api1", "api2")
wait_ready()
status, metadata = api(
"GET", f"/api/v1/attachments/{attachment_id}", token
)
assert metadata["status"] == "queued"
print("Queued attachment survived restart", flush=True)
pause_seconds = int(
os.getenv("LAB6_WORKER_PAUSE_SECONDS", "60")
)
print(
f"INCIDENT: worker stopped for {pause_seconds} seconds",
flush=True,
)
time.sleep(pause_seconds)
print("Starting worker...", flush=True)
docker("up", "-d", "worker")
deadline = time.monotonic() + 90
while time.monotonic() < deadline:
status, metadata = api(
"GET", f"/api/v1/attachments/{attachment_id}", token
)
if metadata["status"] == "ready":
break
if metadata["status"] == "failed":
raise RuntimeError(
f"Processing failed: {metadata.get('error_code')}"
)
time.sleep(1)
else:
raise RuntimeError("Attachment did not become ready")
status, downloaded = api(
"GET",
f"/api/v1/attachments/{attachment_id}/content",
token,
)
assert status == 200
assert downloaded == original
assert hashlib.sha256(downloaded).hexdigest() == (
hashlib.sha256(original).hexdigest()
)
status, thumbnail = api(
"GET",
f"/api/v1/attachments/{attachment_id}/content"
"?variant=thumbnail",
token,
)
assert status == 200
assert thumbnail.startswith(b"\xff\xd8")
assert thumbnail.endswith(b"\xff\xd9")
variants = metadata["variants"]
assert any(
v["name"] == "thumbnail"
and v["width"] == 256
and v["height"] == 160
for v in variants
)
print("Attachment: ready", flush=True)
print("Original: SHA-256 verified", flush=True)
print("Thumbnail: JPEG 256x160 verified", flush=True)
print("PASS: durable worker recovery", flush=True)
if __name__ == "__main__":
try:
main()
finally:
# Restore the worker even when the test fails.
docker("up", "-d", "worker")