forked from mtusi2026/mtusi2026lab6
Implement lab6 OpenTelemetry observability stack
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
echo "Stopping API and image processing..."
|
||||
docker compose stop api1 api2 worker dispatcher
|
||||
|
||||
echo "Restarting persistent storage and broker..."
|
||||
docker compose restart db minio rabbitmq
|
||||
|
||||
echo "Waiting for PostgreSQL and RabbitMQ..."
|
||||
docker compose up -d --wait db rabbitmq
|
||||
|
||||
echo "Checking S3 bucket..."
|
||||
docker compose run --rm --no-deps s3-init
|
||||
|
||||
echo "Starting dispatcher and worker..."
|
||||
docker compose up -d dispatcher worker
|
||||
|
||||
echo "Starting API replicas..."
|
||||
docker compose up -d --wait api1 api2
|
||||
|
||||
echo "Restart complete. Persistent volumes preserved."
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if [[ -z "${TARGET_INSTANCE_ID:-}" ]]; then
|
||||
echo "TARGET_INSTANCE_ID is not set" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for service in api1 api2; do
|
||||
instance_id=$(
|
||||
docker compose exec -T "$service" python -c '
|
||||
import urllib.request
|
||||
|
||||
response = urllib.request.urlopen(
|
||||
"http://127.0.0.1:8080/health/live",
|
||||
timeout=2
|
||||
)
|
||||
|
||||
print(response.headers.get("X-Instance-Id", ""))
|
||||
' 2>/dev/null
|
||||
) || continue
|
||||
|
||||
if [[ "$instance_id" == "$TARGET_INSTANCE_ID" ]]; then
|
||||
echo "Stopping $service"
|
||||
docker compose stop -t 2 "$service"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Target API instance not found" >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,175 @@
|
||||
|
||||
import base64
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
|
||||
BASE_URL = "http://127.0.0.1:8080"
|
||||
|
||||
|
||||
def docker(*args):
|
||||
subprocess.run(
|
||||
["docker", "compose", *args],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def request(method, path, body=None, headers=None):
|
||||
request_headers = dict(headers or {})
|
||||
|
||||
if isinstance(body, dict):
|
||||
body = json.dumps(body).encode("utf-8")
|
||||
request_headers["Content-Type"] = "application/json"
|
||||
|
||||
req = urllib.request.Request(
|
||||
BASE_URL + path,
|
||||
data=body,
|
||||
headers=request_headers,
|
||||
method=method,
|
||||
)
|
||||
|
||||
started = time.monotonic()
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=5) as response:
|
||||
response.read()
|
||||
return response.status, time.monotonic() - started
|
||||
|
||||
except urllib.error.HTTPError as exc:
|
||||
exc.read()
|
||||
return exc.code, time.monotonic() - started
|
||||
|
||||
except (urllib.error.URLError, TimeoutError) as exc:
|
||||
return type(exc).__name__, time.monotonic() - started
|
||||
|
||||
|
||||
def login_header(username, password):
|
||||
encoded = base64.b64encode(
|
||||
f"{username}:{password}".encode("utf-8")
|
||||
).decode("ascii")
|
||||
|
||||
return {
|
||||
"Authorization": f"Basic {encoded}"
|
||||
}
|
||||
|
||||
|
||||
def wait_for_recovery(headers):
|
||||
deadline = time.monotonic() + 90
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
status, duration = request(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
body=b"",
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
if status == 201:
|
||||
print(
|
||||
f"Recovery: HTTP 201, {duration:.3f}s",
|
||||
flush=True,
|
||||
)
|
||||
return
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
raise RuntimeError(
|
||||
"Authentication did not recover after DB restart"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
suffix = uuid.uuid4().hex[:12]
|
||||
|
||||
username = "lab6_db_" + suffix
|
||||
password = "IncidentTest_" + suffix
|
||||
|
||||
status, duration = request(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
body={
|
||||
"username": username,
|
||||
"display_name": "DB incident test",
|
||||
"password": password,
|
||||
},
|
||||
)
|
||||
|
||||
if status != 201:
|
||||
raise RuntimeError(
|
||||
f"Cannot create test user: HTTP {status}"
|
||||
)
|
||||
|
||||
headers = login_header(username, password)
|
||||
|
||||
status, _ = request(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
body=b"",
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
if status != 201:
|
||||
raise RuntimeError(
|
||||
f"Baseline login failed: HTTP {status}"
|
||||
)
|
||||
|
||||
print("Baseline: authentication HTTP 201", flush=True)
|
||||
|
||||
observed_failures = 0
|
||||
|
||||
print("INCIDENT START: stopping PostgreSQL", flush=True)
|
||||
|
||||
try:
|
||||
docker("stop", "db")
|
||||
|
||||
for attempt in range(1, 6):
|
||||
status, duration = request(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
body=b"",
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
print(
|
||||
f"Probe {attempt}: status={status}, "
|
||||
f"duration={duration:.3f}s",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if status != 201:
|
||||
observed_failures += 1
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
finally:
|
||||
print(
|
||||
"INCIDENT END: restoring PostgreSQL",
|
||||
flush=True,
|
||||
)
|
||||
docker("up", "-d", "--wait", "db")
|
||||
|
||||
wait_for_recovery(headers)
|
||||
|
||||
print(
|
||||
f"Failed requests during incident: "
|
||||
f"{observed_failures}/5",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if observed_failures == 0:
|
||||
raise RuntimeError(
|
||||
"The database incident did not affect requests"
|
||||
)
|
||||
|
||||
print(
|
||||
"PASS: database failure observed and recovered",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,77 @@
|
||||
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
sql() {
|
||||
docker compose exec -T db \
|
||||
psql -U messenger -d messenger -At -c "$1"
|
||||
}
|
||||
|
||||
echo "Finding completed image job..."
|
||||
|
||||
JOB_ID=$(sql "
|
||||
SELECT j.id
|
||||
FROM attachment_jobs j
|
||||
JOIN attachment_variants v
|
||||
ON v.attachment_id = j.attachment_id
|
||||
WHERE j.status = 'ready'
|
||||
AND v.variant_name = 'thumbnail'
|
||||
ORDER BY j.created_at DESC
|
||||
LIMIT 1;
|
||||
")
|
||||
|
||||
if [[ -z "$JOB_ID" ]]; then
|
||||
echo "FAIL: no completed image job found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Job: $JOB_ID"
|
||||
|
||||
STATE_QUERY="
|
||||
SELECT
|
||||
j.status || '|' ||
|
||||
j.attempts || '|' ||
|
||||
v.sha256 || '|' ||
|
||||
(
|
||||
SELECT COUNT(*)
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id = j.attachment_id
|
||||
)
|
||||
FROM attachment_jobs j
|
||||
JOIN attachment_variants v
|
||||
ON v.attachment_id = j.attachment_id
|
||||
WHERE j.id = '$JOB_ID'
|
||||
AND v.variant_name = 'thumbnail';
|
||||
"
|
||||
|
||||
BEFORE=$(sql "$STATE_QUERY")
|
||||
|
||||
echo "Before: $BEFORE"
|
||||
echo "Publishing the same job twice..."
|
||||
|
||||
docker compose run --rm --no-deps api1 \
|
||||
python -c '
|
||||
import sys
|
||||
from src.queue import publish_job
|
||||
|
||||
job_id = sys.argv[1]
|
||||
publish_job(job_id)
|
||||
publish_job(job_id)
|
||||
print("Two duplicate messages published")
|
||||
' "$JOB_ID"
|
||||
|
||||
sleep 5
|
||||
|
||||
AFTER=$(sql "$STATE_QUERY")
|
||||
|
||||
echo "After: $AFTER"
|
||||
|
||||
if [[ "$BEFORE" != "$AFTER" ]]; then
|
||||
echo "FAIL: completed job changed after redelivery"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "PASS: duplicate delivery did not change completed job"
|
||||
echo "PASS: thumbnail checksum and variant count preserved"
|
||||
@@ -0,0 +1,154 @@
|
||||
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
|
||||
LOKI = "http://loki:3100"
|
||||
TEMPO = "http://tempo:3200"
|
||||
|
||||
TRACE_PATTERN = re.compile(r"^[0-9a-f]{32}$")
|
||||
SPAN_PATTERN = re.compile(r"^[0-9a-f]{16}$")
|
||||
|
||||
|
||||
def get_json(url):
|
||||
with urllib.request.urlopen(url, timeout=15) as response:
|
||||
return json.load(response)
|
||||
|
||||
|
||||
def find_log(service, event):
|
||||
params = urllib.parse.urlencode({
|
||||
"query": f'{{service_name="{service}"}}',
|
||||
"start": str(time.time_ns() - 6 * 3600 * 10**9),
|
||||
"end": str(time.time_ns()),
|
||||
"limit": 500,
|
||||
"direction": "BACKWARD",
|
||||
})
|
||||
|
||||
payload = get_json(
|
||||
f"{LOKI}/loki/api/v1/query_range?{params}"
|
||||
)
|
||||
|
||||
streams = payload.get("data", {}).get("result", [])
|
||||
|
||||
for stream in streams:
|
||||
labels = stream.get("stream", {})
|
||||
|
||||
for entry in stream.get("values", []):
|
||||
if len(entry) < 2:
|
||||
continue
|
||||
|
||||
message = entry[1]
|
||||
metadata = entry[2] if len(entry) >= 3 else {}
|
||||
|
||||
if event not in message and (
|
||||
metadata.get("event") != event
|
||||
):
|
||||
continue
|
||||
|
||||
trace_id = (
|
||||
metadata.get("trace_id")
|
||||
or labels.get("trace_id")
|
||||
)
|
||||
|
||||
span_id = (
|
||||
metadata.get("span_id")
|
||||
or labels.get("span_id")
|
||||
)
|
||||
|
||||
if not trace_id or not TRACE_PATTERN.fullmatch(
|
||||
trace_id
|
||||
):
|
||||
continue
|
||||
|
||||
if not span_id or not SPAN_PATTERN.fullmatch(
|
||||
span_id
|
||||
):
|
||||
continue
|
||||
|
||||
return {
|
||||
"service": service,
|
||||
"event": event,
|
||||
"message": message,
|
||||
"trace_id": trace_id,
|
||||
"span_id": span_id,
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def verify_trace(log, expected_kind):
|
||||
trace_id = log["trace_id"]
|
||||
|
||||
payload = get_json(
|
||||
f"{TEMPO}/api/traces/{trace_id}"
|
||||
)
|
||||
|
||||
batches = payload.get("batches", [])
|
||||
spans = []
|
||||
|
||||
for batch in batches:
|
||||
for scope in batch.get("scopeSpans", []):
|
||||
spans.extend(scope.get("spans", []))
|
||||
|
||||
if not spans:
|
||||
raise AssertionError(
|
||||
f"No spans found for {trace_id}"
|
||||
)
|
||||
|
||||
kinds = [str(span.get("kind", "")) for span in spans]
|
||||
|
||||
if not any(expected_kind in kind for kind in kinds):
|
||||
raise AssertionError(
|
||||
f"Expected span kind {expected_kind}; got {kinds}"
|
||||
)
|
||||
|
||||
print(f"Service: {log['service']}")
|
||||
print(f"Event: {log['event']}")
|
||||
print(f"Trace ID: {trace_id}")
|
||||
print(f"Span ID: {log['span_id']}")
|
||||
print(f"Tempo spans: {len(spans)}")
|
||||
|
||||
for span in spans:
|
||||
print(
|
||||
" Span:",
|
||||
span.get("name"),
|
||||
"kind:",
|
||||
span.get("kind"),
|
||||
)
|
||||
|
||||
print("PASS: Loki log linked to Tempo trace")
|
||||
print()
|
||||
|
||||
|
||||
def main():
|
||||
cases = [
|
||||
(
|
||||
"messenger-api",
|
||||
"HTTP request completed",
|
||||
"SERVER",
|
||||
),
|
||||
(
|
||||
"messenger-worker",
|
||||
"Image job handled",
|
||||
"CONSUMER",
|
||||
),
|
||||
]
|
||||
|
||||
for service, event, kind in cases:
|
||||
log = find_log(service, event)
|
||||
|
||||
if log is None:
|
||||
raise AssertionError(
|
||||
f"No correlated log found for {service}"
|
||||
)
|
||||
|
||||
verify_trace(log, kind)
|
||||
|
||||
print("PASS: API and worker trace correlation")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,256 @@
|
||||
import os
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
import zlib
|
||||
import struct
|
||||
|
||||
|
||||
BASE = "http://127.0.0.1:8080"
|
||||
|
||||
|
||||
def docker(*args):
|
||||
subprocess.run(
|
||||
["docker", "compose", *args],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
def api(method, path, token=None, body=None, headers=None):
|
||||
request_headers = dict(headers or {})
|
||||
|
||||
if token:
|
||||
request_headers["Authorization"] = f"Bearer {token}"
|
||||
|
||||
if isinstance(body, dict):
|
||||
body = json.dumps(body).encode("utf-8")
|
||||
request_headers["Content-Type"] = "application/json"
|
||||
|
||||
request = urllib.request.Request(
|
||||
BASE + path,
|
||||
data=body,
|
||||
headers=request_headers,
|
||||
method=method,
|
||||
)
|
||||
|
||||
with urllib.request.urlopen(request, timeout=10) as response:
|
||||
data = response.read()
|
||||
return (
|
||||
response.status,
|
||||
json.loads(data) if data and
|
||||
"application/json" in response.headers.get(
|
||||
"Content-Type", ""
|
||||
) else data,
|
||||
)
|
||||
|
||||
|
||||
def png_chunk(name, data):
|
||||
value = name + data
|
||||
return (
|
||||
struct.pack(">I", len(data))
|
||||
+ value
|
||||
+ struct.pack(">I", zlib.crc32(value) & 0xFFFFFFFF)
|
||||
)
|
||||
|
||||
|
||||
def make_png():
|
||||
width, height = 320, 200
|
||||
scanline = b"\x00" + bytes([220, 80, 40]) * width
|
||||
|
||||
return (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
+ png_chunk(
|
||||
b"IHDR",
|
||||
struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0),
|
||||
)
|
||||
+ png_chunk(
|
||||
b"IDAT", zlib.compress(scanline * height)
|
||||
)
|
||||
+ png_chunk(b"IEND", b"")
|
||||
)
|
||||
|
||||
|
||||
def wait_ready():
|
||||
for _ in range(90):
|
||||
try:
|
||||
status, data = api("GET", "/health/ready")
|
||||
if status == 200 and data.get("status") == "ok":
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1)
|
||||
|
||||
raise RuntimeError("API did not become ready")
|
||||
|
||||
|
||||
def main():
|
||||
suffix = uuid.uuid4().hex[:12]
|
||||
username = "lab5_" + suffix
|
||||
password = "Lab5TestPassword_" + suffix
|
||||
|
||||
print("Stopping worker...", flush=True)
|
||||
docker("stop", "worker")
|
||||
wait_ready()
|
||||
|
||||
status, user = api(
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
body={
|
||||
"username": username,
|
||||
"display_name": "Lab5 recovery test",
|
||||
"password": password,
|
||||
},
|
||||
)
|
||||
assert status == 201
|
||||
|
||||
credentials = base64.b64encode(
|
||||
f"{username}:{password}".encode()
|
||||
).decode()
|
||||
|
||||
status, session = api(
|
||||
"POST",
|
||||
"/api/v1/auth/sessions",
|
||||
headers={"Authorization": f"Basic {credentials}"},
|
||||
body=b"",
|
||||
)
|
||||
assert status == 201
|
||||
token = session["token"]
|
||||
|
||||
status, chat = api(
|
||||
"POST",
|
||||
"/api/v1/chats",
|
||||
token,
|
||||
{
|
||||
"title": "Worker recovery",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert status == 201
|
||||
|
||||
original = make_png()
|
||||
path = (
|
||||
f"/api/v1/chats/{chat['id']}/attachments"
|
||||
"?filename=recovery.png"
|
||||
)
|
||||
|
||||
status, attachment = api(
|
||||
"POST",
|
||||
path,
|
||||
token,
|
||||
original,
|
||||
{"Content-Type": "image/png"},
|
||||
)
|
||||
|
||||
assert status == 202, f"Expected 202, got {status}"
|
||||
assert attachment["status"] == "queued"
|
||||
attachment_id = attachment["id"]
|
||||
print("Upload: 202 queued", flush=True)
|
||||
|
||||
status, message = api(
|
||||
"POST",
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
token,
|
||||
{"text": "", "attachment_ids": [attachment_id]},
|
||||
{"Idempotency-Key": "recovery-" + suffix},
|
||||
)
|
||||
assert status == 201
|
||||
assert attachment_id in message["attachment_ids"]
|
||||
print("Message created while worker is stopped", flush=True)
|
||||
|
||||
status, metadata = api(
|
||||
"GET", f"/api/v1/attachments/{attachment_id}", token
|
||||
)
|
||||
assert status == 200 and metadata["status"] == "queued"
|
||||
|
||||
print("Restarting RabbitMQ and API...", flush=True)
|
||||
docker("restart", "rabbitmq")
|
||||
docker("up", "-d", "--wait", "rabbitmq")
|
||||
docker("restart", "api1", "api2")
|
||||
wait_ready()
|
||||
|
||||
status, metadata = api(
|
||||
"GET", f"/api/v1/attachments/{attachment_id}", token
|
||||
)
|
||||
assert metadata["status"] == "queued"
|
||||
print("Queued attachment survived restart", flush=True)
|
||||
|
||||
pause_seconds = int(
|
||||
os.getenv("LAB6_WORKER_PAUSE_SECONDS", "60")
|
||||
)
|
||||
|
||||
print(
|
||||
f"INCIDENT: worker stopped for {pause_seconds} seconds",
|
||||
flush=True,
|
||||
)
|
||||
time.sleep(pause_seconds)
|
||||
|
||||
print("Starting worker...", flush=True)
|
||||
docker("up", "-d", "worker")
|
||||
|
||||
deadline = time.monotonic() + 90
|
||||
|
||||
while time.monotonic() < deadline:
|
||||
status, metadata = api(
|
||||
"GET", f"/api/v1/attachments/{attachment_id}", token
|
||||
)
|
||||
|
||||
if metadata["status"] == "ready":
|
||||
break
|
||||
|
||||
if metadata["status"] == "failed":
|
||||
raise RuntimeError(
|
||||
f"Processing failed: {metadata.get('error_code')}"
|
||||
)
|
||||
|
||||
time.sleep(1)
|
||||
else:
|
||||
raise RuntimeError("Attachment did not become ready")
|
||||
|
||||
status, downloaded = api(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment_id}/content",
|
||||
token,
|
||||
)
|
||||
assert status == 200
|
||||
assert downloaded == original
|
||||
assert hashlib.sha256(downloaded).hexdigest() == (
|
||||
hashlib.sha256(original).hexdigest()
|
||||
)
|
||||
|
||||
status, thumbnail = api(
|
||||
"GET",
|
||||
f"/api/v1/attachments/{attachment_id}/content"
|
||||
"?variant=thumbnail",
|
||||
token,
|
||||
)
|
||||
|
||||
assert status == 200
|
||||
assert thumbnail.startswith(b"\xff\xd8")
|
||||
assert thumbnail.endswith(b"\xff\xd9")
|
||||
|
||||
variants = metadata["variants"]
|
||||
assert any(
|
||||
v["name"] == "thumbnail"
|
||||
and v["width"] == 256
|
||||
and v["height"] == 160
|
||||
for v in variants
|
||||
)
|
||||
|
||||
print("Attachment: ready", flush=True)
|
||||
print("Original: SHA-256 verified", flush=True)
|
||||
print("Thumbnail: JPEG 256x160 verified", flush=True)
|
||||
print("PASS: durable worker recovery", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
finally:
|
||||
# Restore the worker even when the test fails.
|
||||
docker("up", "-d", "worker")
|
||||
Reference in New Issue
Block a user