forked from mtusi2026/mtusi2026lab6
lab6: базовый коммит из lab5 (0790599)
Перенесено дерево lab5 целиком: NestJS-приложение (API + worker,
RabbitMQ, outbox, S3), Dockerfile, compose.yaml, infra/{haproxy,minio,
rabbitmq}, стендовые скрипты, scripts/lab4, scripts/lab5,
evidence/lab4, evidence/lab5 и contracts/extensions.openapi.json.
Задание lab6 оставлено своим: README, REPORT, COURSE, lab.json,
contracts/{openapi,infrastructure,jobs.schema,attachment-states,
examples,observability}, tests, compose.example.yaml, infra/otel,
.github/pull_request_template.md.
Адаптации при переносе:
- docker-проект mtusi-lab5 -> mtusi-lab6 и тег образа
mtusi-messenger:lab6 (compose.yaml, scripts/stop-one.sh, .env.example,
infra/README.md), чтобы тома и контейнеры не пересекались с lab5;
- Makefile: цели lab5 сохранены целиком (skeleton lab6 не отличается
от skeleton lab5);
- .env.example: переменные приложения и стенда lab5 + блок lab6
(OTEL_*) из skeleton.
This commit is contained in:
Executable
+22
@@ -0,0 +1,22 @@
|
||||
#!/bin/sh
|
||||
# Black-box набор приложения (app/test/*.e2e-spec.ts) против поднятого стенда.
|
||||
# Требует loopback-портов PostgreSQL, Redis и MinIO из compose.yaml.
|
||||
set -eu
|
||||
|
||||
env_file="${COMPOSE_ENV_FILE:-.env}"
|
||||
compose_dir="$(pwd)"
|
||||
# shellcheck disable=SC1090
|
||||
set -a
|
||||
. "./$env_file"
|
||||
set +a
|
||||
|
||||
export API_BASE_URL="${API_BASE_URL:-http://127.0.0.1:${APP_PUBLIC_PORT:-8080}}"
|
||||
export DATABASE_URL="postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in $env_file}@127.0.0.1:${POSTGRES_PUBLISHED_PORT:-5432}/${POSTGRES_DB:-messenger}"
|
||||
export REDIS_URL="redis://127.0.0.1:${REDIS_PUBLISHED_PORT:-6379}"
|
||||
export E2E_S3_PUBLIC_ENDPOINT="${S3_PUBLIC_ENDPOINT:-http://localhost:9000}"
|
||||
export E2E_S3_BUCKET="${S3_BUCKET:-messenger-attachments}"
|
||||
export E2E_COMPOSE_DIR="$compose_dir"
|
||||
export E2E_COMPOSE_ENV_FILE="$env_file"
|
||||
|
||||
cd "$(dirname "$0")/../app"
|
||||
exec npm run test:e2e -- "$@"
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/bin/sh
|
||||
# Идемпотентно создаёт .env и secrets/* для локального стенда. Значения не печатает.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
if [ ! -f .env ]; then
|
||||
(
|
||||
umask 077
|
||||
pg=$(openssl rand -hex 24)
|
||||
mt=$(openssl rand -hex 16)
|
||||
sed \
|
||||
-e "s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=${pg}|" \
|
||||
-e "s|^DATABASE_URL=.*|DATABASE_URL=postgresql://messenger:${pg}@postgres:5432/messenger|" \
|
||||
-e "s|^METRICS_TOKEN=.*|METRICS_TOKEN=${mt}|" \
|
||||
.env.example > .env
|
||||
)
|
||||
echo "created .env"
|
||||
else
|
||||
echo "kept .env"
|
||||
fi
|
||||
|
||||
mkdir -p secrets
|
||||
chmod 700 secrets
|
||||
|
||||
write_secret() {
|
||||
name=$1
|
||||
value=$2
|
||||
if [ -s "secrets/$name" ]; then
|
||||
echo "kept secrets/$name"
|
||||
return
|
||||
fi
|
||||
(umask 133; printf '%s' "$value" > "secrets/$name")
|
||||
chmod 0644 "secrets/$name"
|
||||
echo "created secrets/$name"
|
||||
}
|
||||
|
||||
write_secret minio_root_user "root-$(openssl rand -hex 6)"
|
||||
write_secret minio_root_password "$(openssl rand -hex 24)"
|
||||
write_secret s3_api_access_key_id "api-$(openssl rand -hex 8)"
|
||||
write_secret s3_api_secret_access_key "$(openssl rand -hex 24)"
|
||||
write_secret s3_worker_access_key_id "worker-$(openssl rand -hex 8)"
|
||||
write_secret s3_worker_secret_access_key "$(openssl rand -hex 24)"
|
||||
|
||||
if [ -s secrets/broker_worker_password ] && [ -s secrets/broker_worker_url ]; then
|
||||
echo "kept secrets/broker_worker_password secrets/broker_worker_url"
|
||||
else
|
||||
wp=$(openssl rand -hex 24)
|
||||
(umask 133; printf '%s' "$wp" > secrets/broker_worker_password)
|
||||
chmod 0644 secrets/broker_worker_password
|
||||
(umask 133; printf '%s' "amqp://messenger-worker:${wp}@broker:5672/messenger" > secrets/broker_worker_url)
|
||||
chmod 0644 secrets/broker_worker_url
|
||||
echo "created secrets/broker_worker_password secrets/broker_worker_url"
|
||||
fi
|
||||
|
||||
# Broker password lives in one place and is rendered into both files, so the
|
||||
# URL used by the app and the RabbitMQ config cannot drift apart.
|
||||
if [ -s secrets/broker_url ] && [ -s secrets/rabbitmq.conf ]; then
|
||||
echo "kept secrets/broker_url secrets/rabbitmq.conf"
|
||||
else
|
||||
bp=$(openssl rand -hex 24)
|
||||
(umask 133; printf '%s' "amqp://messenger:${bp}@broker:5672/messenger" > secrets/broker_url)
|
||||
chmod 0644 secrets/broker_url
|
||||
(
|
||||
umask 133
|
||||
{
|
||||
printf 'default_vhost = messenger\n'
|
||||
printf 'default_user = messenger\n'
|
||||
printf 'default_pass = %s\n' "$bp"
|
||||
printf 'default_permissions.configure = ^(attachments\\.jobs|attachments\\.process|attachments\\.dlx|attachments\\.process\\.dlq)$\n'
|
||||
printf 'default_permissions.read = ^attachments\\.(jobs|process|dlx|process\\.dlq)$\n'
|
||||
printf 'default_permissions.write = ^attachments\\.(jobs|process|dlx|process\\.dlq)$\n'
|
||||
} > secrets/rabbitmq.conf
|
||||
)
|
||||
chmod 0644 secrets/rabbitmq.conf
|
||||
echo "created secrets/broker_url secrets/rabbitmq.conf"
|
||||
fi
|
||||
Executable
+340
@@ -0,0 +1,340 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Upload benchmark: API traffic and peak RSS per upload for proxy vs direct mode.
|
||||
|
||||
attachments_bench.py run --mode proxy --size-bytes 1048576 --concurrency 10 \
|
||||
--uploads-per-worker 5 --repeat 3 --output evidence/lab4/bench/proxy-1048576-c10.json
|
||||
attachments_bench.py summarize evidence/lab4/bench/*.json > evidence/lab4/summary.md
|
||||
|
||||
Run from the repository root with the stand up. Tokens, signed URLs and secrets
|
||||
are never written to the output.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
from pngkit import noise_png # noqa: E402
|
||||
|
||||
|
||||
def sh(*args, check=True):
|
||||
return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def read_env(path=".env"):
|
||||
values = {}
|
||||
with open(path) as handle:
|
||||
for line in handle:
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
key, _, value = line.partition("=")
|
||||
values[key] = value
|
||||
return values
|
||||
|
||||
|
||||
def request(host, port, method, path, headers=None, body=None):
|
||||
connection = http.client.HTTPConnection(host, port, timeout=120)
|
||||
try:
|
||||
connection.request(method, path, body=body, headers=headers or {})
|
||||
response = connection.getresponse()
|
||||
payload = response.read()
|
||||
return response.status, payload
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def api_json(env, method, path, token=None, body=None, basic=None):
|
||||
headers = {}
|
||||
data = None
|
||||
if body is not None:
|
||||
headers["content-type"] = "application/json"
|
||||
data = json.dumps(body).encode()
|
||||
if token:
|
||||
headers["authorization"] = f"Bearer {token}"
|
||||
if basic:
|
||||
headers["authorization"] = "Basic " + base64.b64encode(basic.encode()).decode()
|
||||
port = int(env.get("APP_PUBLIC_PORT", "8080"))
|
||||
status, payload = request("127.0.0.1", port, method, path, headers, data)
|
||||
return status, json.loads(payload) if payload else None
|
||||
|
||||
|
||||
def containers():
|
||||
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "api").split()
|
||||
return sorted(ids)
|
||||
|
||||
|
||||
NODE_METRICS = (
|
||||
"fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics'+process.argv[1],"
|
||||
"{method:process.argv[1]?'POST':'GET',headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}})"
|
||||
".then(r=>r.text()).then(console.log)"
|
||||
)
|
||||
|
||||
|
||||
def metrics(container):
|
||||
return json.loads(sh("docker", "exec", container, "node", "-e", NODE_METRICS, ""))
|
||||
|
||||
|
||||
def reset_metrics(container):
|
||||
sh("docker", "exec", container, "node", "-e", NODE_METRICS, "/reset")
|
||||
|
||||
|
||||
def net_bytes(container):
|
||||
for line in sh("docker", "exec", container, "cat", "/proc/net/dev").splitlines():
|
||||
if line.strip().startswith("eth0:"):
|
||||
fields = line.split(":", 1)[1].split()
|
||||
return int(fields[0]), int(fields[8])
|
||||
raise RuntimeError("eth0 not found")
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
ordered = sorted(values)
|
||||
if not ordered:
|
||||
return 0.0
|
||||
return ordered[min(len(ordered) - 1, max(0, int(round(fraction * len(ordered) + 0.5)) - 1))]
|
||||
|
||||
|
||||
def one_upload(env, mode, token, chat_id, data, seed):
|
||||
"""Returns (ok, elapsed_ms, outcome); outcome names the failing step and status, e.g. "complete:429"."""
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
filename = f"bench-{seed}.png"
|
||||
port = int(env.get("APP_PUBLIC_PORT", "8080"))
|
||||
started = time.perf_counter()
|
||||
outcome, attachment = "ok", None
|
||||
try:
|
||||
if mode == "proxy":
|
||||
status, body = request(
|
||||
"127.0.0.1", port, "POST",
|
||||
f"/api/v1/chats/{chat_id}/attachments?filename={urllib.parse.quote(filename)}",
|
||||
{"authorization": f"Bearer {token}", "content-type": "image/png"}, data,
|
||||
)
|
||||
if status == 201:
|
||||
attachment = json.loads(body)
|
||||
else:
|
||||
outcome = f"upload:{status}"
|
||||
else:
|
||||
headers = {"authorization": f"Bearer {token}", "content-type": "application/json"}
|
||||
status, body = request(
|
||||
"127.0.0.1", port, "POST", f"/api/v1/chats/{chat_id}/attachments/direct-uploads", headers,
|
||||
json.dumps({"filename": filename, "content_type": "image/png",
|
||||
"size_bytes": len(data), "sha256": digest}).encode(),
|
||||
)
|
||||
if status != 201:
|
||||
outcome = f"init:{status}"
|
||||
else:
|
||||
ticket = json.loads(body)
|
||||
target = urllib.parse.urlsplit(ticket["url"])
|
||||
put_status, _ = request(
|
||||
target.hostname, target.port, "PUT", target.path + "?" + target.query,
|
||||
{**ticket["headers"]}, data,
|
||||
)
|
||||
if put_status != 200:
|
||||
outcome = f"put:{put_status}"
|
||||
else:
|
||||
status, body = request(
|
||||
"127.0.0.1", port, "POST", f"/api/v1/attachments/{ticket['attachment_id']}/complete",
|
||||
{"authorization": f"Bearer {token}"},
|
||||
)
|
||||
if status == 200:
|
||||
attachment = json.loads(body)
|
||||
else:
|
||||
outcome = f"complete:{status}"
|
||||
except (OSError, http.client.HTTPException, ValueError) as error:
|
||||
outcome = f"error:{type(error).__name__}"
|
||||
elapsed_ms = (time.perf_counter() - started) * 1000
|
||||
if outcome == "ok" and (attachment is None or attachment["sha256"] != digest):
|
||||
outcome = "checksum_mismatch"
|
||||
return outcome == "ok", elapsed_ms, outcome
|
||||
|
||||
|
||||
def payload(args, seed):
|
||||
if args.image:
|
||||
with open(args.image, "rb") as handle:
|
||||
return handle.read()
|
||||
return noise_png(args.size_bytes, seed)
|
||||
|
||||
|
||||
def run_once(env, args, actors, ids):
|
||||
for container in ids:
|
||||
reset_metrics(container)
|
||||
time.sleep(0.2)
|
||||
baseline = {c: metrics(c) for c in ids}
|
||||
net_before = {c: net_bytes(c) for c in ids}
|
||||
latencies, failures = [], []
|
||||
outcomes = {}
|
||||
lock = threading.Lock()
|
||||
|
||||
def worker(index):
|
||||
for step in range(args.uploads_per_worker):
|
||||
seed = index * 1000 + step
|
||||
data = payload(args, seed)
|
||||
token, chat_id = actors[index % len(actors)]
|
||||
ok, elapsed, outcome = one_upload(env, args.mode, token, chat_id, data, seed)
|
||||
with lock:
|
||||
(latencies if ok else failures).append(elapsed)
|
||||
outcomes[outcome] = outcomes.get(outcome, 0) + 1
|
||||
|
||||
started = time.perf_counter()
|
||||
threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
duration = time.perf_counter() - started
|
||||
time.sleep(0.2)
|
||||
after = {c: metrics(c) for c in ids}
|
||||
net_after = {c: net_bytes(c) for c in ids}
|
||||
|
||||
replicas = []
|
||||
for c in ids:
|
||||
base_rss = baseline[c]["memory_bytes"]["rss"]
|
||||
peak = after[c]["memory_peak_bytes"]
|
||||
replicas.append({
|
||||
"instance_id": after[c]["instance_id"],
|
||||
"delta_rx_bytes": net_after[c][0] - net_before[c][0],
|
||||
"delta_tx_bytes": net_after[c][1] - net_before[c][1],
|
||||
"baseline_rss_bytes": base_rss,
|
||||
"peak_memory_bytes": peak,
|
||||
"delta_rss_bytes": peak["rss"] - base_rss,
|
||||
"counters": {k: v for k, v in after[c]["counters"].items() if k.startswith("attachments.")},
|
||||
})
|
||||
total = args.concurrency * args.uploads_per_worker
|
||||
return {
|
||||
"uploads": total,
|
||||
"succeeded": len(latencies),
|
||||
"failed": len(failures),
|
||||
"outcomes": outcomes,
|
||||
"duration_s": round(duration, 3),
|
||||
"latency_ms": {
|
||||
"p50": round(percentile(latencies, 0.50), 1),
|
||||
"p95": round(percentile(latencies, 0.95), 1),
|
||||
"p99": round(percentile(latencies, 0.99), 1),
|
||||
},
|
||||
"replicas": replicas,
|
||||
"total_delta_rx_bytes": sum(r["delta_rx_bytes"] for r in replicas),
|
||||
"total_delta_tx_bytes": sum(r["delta_tx_bytes"] for r in replicas),
|
||||
"rx_bytes_per_upload": round(sum(r["delta_rx_bytes"] for r in replicas) / total),
|
||||
"tx_bytes_per_upload": round(sum(r["delta_tx_bytes"] for r in replicas) / total),
|
||||
"max_delta_rss_bytes": max(r["delta_rss_bytes"] for r in replicas),
|
||||
}
|
||||
|
||||
|
||||
def command_run(args):
|
||||
if args.image:
|
||||
args.size_bytes = os.path.getsize(args.image)
|
||||
if args.size_bytes <= 0:
|
||||
sys.exit("--size-bytes or --image is required")
|
||||
env = read_env()
|
||||
ids = containers()
|
||||
if not ids:
|
||||
sys.exit("no api containers found")
|
||||
# One user per worker: the API caps concurrent uploads per user
|
||||
# (UPLOAD_MAX_IN_FLIGHT_PER_USER), and real traffic comes from many users.
|
||||
actors = []
|
||||
for _ in range(args.concurrency):
|
||||
name = f"bench_{uuid.uuid4().hex[:10]}"
|
||||
password = "bench:" + uuid.uuid4().hex
|
||||
status, user = api_json(env, "POST", "/api/v1/users",
|
||||
body={"username": name, "display_name": "bench", "password": password})
|
||||
assert status == 201, status
|
||||
status, session = api_json(env, "POST", "/api/v1/auth/sessions", basic=f"{name}:{password}")
|
||||
assert status == 201, status
|
||||
token = session["token"]
|
||||
status, chat = api_json(env, "POST", "/api/v1/chats", token, {"title": "bench", "member_ids": [user["id"]]})
|
||||
assert status == 201, status
|
||||
actors.append((token, chat["id"]))
|
||||
|
||||
# Warm-up: two uploads per replica, not measured.
|
||||
for seed in range(len(ids) * 2):
|
||||
token, chat_id = actors[seed % len(actors)]
|
||||
one_upload(env, args.mode, token, chat_id, payload(args, 900000 + seed), seed)
|
||||
|
||||
limits = []
|
||||
for c in ids:
|
||||
limits.append(sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", c))
|
||||
repeats = [run_once(env, args, actors, ids) for _ in range(args.repeat)]
|
||||
mem_total = next(int(l.split()[1]) * 1024 for l in open("/proc/meminfo") if l.startswith("MemTotal"))
|
||||
result = {
|
||||
"date": datetime.now(timezone.utc).isoformat(timespec="seconds"),
|
||||
"git": sh("git", "rev-parse", "--short", "HEAD", check=False),
|
||||
"params": {
|
||||
"mode": args.mode, "size_bytes": args.size_bytes, "concurrency": args.concurrency,
|
||||
"uploads_per_worker": args.uploads_per_worker, "repeat": args.repeat,
|
||||
"users": args.concurrency, "image": os.path.basename(args.image) if args.image else None,
|
||||
},
|
||||
"host": {"nproc": os.cpu_count(), "mem_total_bytes": mem_total},
|
||||
"api_container_limits_nanocpus_memory": limits,
|
||||
"repeats": repeats,
|
||||
}
|
||||
os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True)
|
||||
with open(args.output, "w") as handle:
|
||||
json.dump(result, handle, indent=2)
|
||||
handle.write("\n")
|
||||
for index, item in enumerate(repeats):
|
||||
print(f"repeat {index + 1}: ok={item['succeeded']}/{item['uploads']} "
|
||||
f"rx/upload={item['rx_bytes_per_upload']} tx/upload={item['tx_bytes_per_upload']} "
|
||||
f"maxΔrss={item['max_delta_rss_bytes']} p50={item['latency_ms']['p50']}ms "
|
||||
f"outcomes={item['outcomes']}")
|
||||
|
||||
|
||||
def mib(value):
|
||||
return f"{value / 1048576:.2f}"
|
||||
|
||||
|
||||
def command_summarize(args):
|
||||
rows = []
|
||||
for path in args.paths:
|
||||
with open(path) as handle:
|
||||
data = json.load(handle)
|
||||
p = data["params"]
|
||||
reps = data["repeats"]
|
||||
med = lambda key: statistics.median(r[key] for r in reps) # noqa: E731
|
||||
label = p["mode"] + (f" ({p['image']})" if p.get("image") else "")
|
||||
rows.append((label, p["size_bytes"], p["concurrency"], reps, med, data))
|
||||
rows.sort(key=lambda r: (r[0].count("("), r[1], r[0], r[2]))
|
||||
print("| mode | size, MiB | C | ok | failures | API rx/upload, MiB | API tx/upload, MiB | max Δrss, MiB | peak RSS, MiB | p50, ms | p95, ms |")
|
||||
print("|---|---|---|---|---|---|---|---|---|---|---|")
|
||||
for mode, size, conc, reps, med, _data in rows:
|
||||
peak = max(x["peak_memory_bytes"]["rss"] for r in reps for x in r["replicas"])
|
||||
failed = {}
|
||||
for rep in reps:
|
||||
for name, count in rep.get("outcomes", {}).items():
|
||||
if name != "ok":
|
||||
failed[name] = failed.get(name, 0) + count
|
||||
failures = ", ".join(f"{name}×{count}" for name, count in sorted(failed.items())) or "—"
|
||||
ok = sum(r["succeeded"] for r in reps)
|
||||
total = sum(r["uploads"] for r in reps)
|
||||
p50 = statistics.median(r["latency_ms"]["p50"] for r in reps)
|
||||
p95 = statistics.median(r["latency_ms"]["p95"] for r in reps)
|
||||
print(f"| {mode} | {mib(size)} | {conc} | {ok}/{total} | {failures} | {mib(med('rx_bytes_per_upload'))} | "
|
||||
f"{mib(med('tx_bytes_per_upload'))} | {mib(med('max_delta_rss_bytes'))} | {mib(peak)} | {p50:.0f} | {p95:.0f} |")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
run = sub.add_parser("run")
|
||||
run.add_argument("--mode", choices=["proxy", "direct"], required=True)
|
||||
run.add_argument("--size-bytes", type=int, default=0)
|
||||
run.add_argument("--image", help="upload this file instead of generated noise PNGs")
|
||||
run.add_argument("--concurrency", type=int, default=10)
|
||||
run.add_argument("--uploads-per-worker", type=int, default=5)
|
||||
run.add_argument("--repeat", type=int, default=3)
|
||||
run.add_argument("--output", required=True)
|
||||
summarize = sub.add_parser("summarize")
|
||||
summarize.add_argument("paths", nargs="+")
|
||||
args = parser.parse_args()
|
||||
{"run": command_run, "summarize": command_summarize}[args.command](args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+54
@@ -0,0 +1,54 @@
|
||||
#!/bin/sh
|
||||
# Проверка приватности S3 вне API: анонимный доступ, права отдельных credentials
|
||||
# API, закрытая консоль. Секреты не печатаются.
|
||||
set -u
|
||||
cd "$(dirname "$0")/../.."
|
||||
env_file="${COMPOSE_ENV_FILE:-.env}"
|
||||
# shellcheck disable=SC1090
|
||||
set -a; . "./$env_file"; set +a
|
||||
endpoint="${S3_PUBLIC_ENDPOINT:-http://localhost:9000}"
|
||||
bucket="${S3_BUCKET:-messenger-attachments}"
|
||||
compose="docker compose --env-file $env_file"
|
||||
fail=0
|
||||
|
||||
report() { # status name
|
||||
if [ "$1" = ok ]; then echo "PASS $2"; else echo "FAIL $2"; fail=1; fi
|
||||
}
|
||||
|
||||
key=$($compose exec -T postgres psql -U "${POSTGRES_USER:-messenger}" -d "${POSTGRES_DB:-messenger}" -At \
|
||||
-c "SELECT object_key FROM attachments WHERE status = 'ready' ORDER BY created_at DESC LIMIT 1")
|
||||
if [ -z "$key" ]; then
|
||||
echo "FAIL no ready attachment to check; upload one first (make test)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
code=$(curl -s -o /dev/null -w '%{http_code}' "$endpoint/$bucket/$key")
|
||||
[ "$code" = 403 ] && report ok "anonymous GET of a ready object -> 403" || report fail "anonymous GET of a ready object -> $code (want 403)"
|
||||
|
||||
code=$(curl -s -o /dev/null -w '%{http_code}' "$endpoint/$bucket/")
|
||||
[ "$code" = 403 ] && report ok "anonymous list of the bucket -> 403" || report fail "anonymous list of the bucket -> $code (want 403)"
|
||||
|
||||
code=$(curl -s -o /dev/null -w '%{http_code}' -X PUT --data-binary x "$endpoint/$bucket/attachments/anonymous-write")
|
||||
[ "$code" = 403 ] && report ok "anonymous PUT -> 403" || report fail "anonymous PUT -> $code (want 403)"
|
||||
|
||||
# The API credentials only, inside a one-off container (secrets come from files).
|
||||
api_mc() {
|
||||
$compose run --rm --no-deps -T --entrypoint /bin/sh s3-init -c '
|
||||
IFS= read -r k < /run/secrets/s3_api_access_key_id
|
||||
IFS= read -r s < /run/secrets/s3_api_secret_access_key
|
||||
export MC_CONFIG_DIR=/tmp/.mc MC_HOST_api="http://$k:$s@s3:9000"
|
||||
exec mc "$@"' sh "$@" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
api_mc mb api/forbidden-bucket && report fail "API credentials can create buckets" || report ok "API credentials cannot create buckets"
|
||||
api_mc admin info api && report fail "API credentials can use admin API" || report ok "API credentials cannot use admin API"
|
||||
api_mc stat "api/$bucket/$key" && report ok "API credentials can read attachments/*" || report fail "API credentials cannot read attachments/*"
|
||||
api_mc stat "api/$bucket/outside-prefix" ; rc=$?
|
||||
api_mc cp /etc/hostname "api/$bucket/outside-prefix" && report fail "API credentials can write outside attachments/" || report ok "API credentials cannot write outside attachments/"
|
||||
: "$rc"
|
||||
|
||||
port=$($compose port s3 9001 2>/dev/null || true)
|
||||
[ -z "$port" ] && report ok "console port 9001 is not published" || report fail "console port 9001 is published ($port)"
|
||||
if curl -s -o /dev/null --max-time 3 http://localhost:9001; then report fail "console reachable on localhost:9001"; else report ok "console not reachable on localhost:9001"; fi
|
||||
|
||||
exit "$fail"
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/sh
|
||||
# Сверка bucket и БД (только чтение): orphans — объект без строки, missing —
|
||||
# ready-строка без объекта. Код выхода 1, если что-то найдено.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/../.."
|
||||
env_file="${COMPOSE_ENV_FILE:-.env}"
|
||||
# shellcheck disable=SC1090
|
||||
set -a; . "./$env_file"; set +a
|
||||
bucket="${S3_BUCKET:-messenger-attachments}"
|
||||
compose="docker compose --env-file $env_file"
|
||||
|
||||
objects=$(mktemp)
|
||||
rows=$(mktemp)
|
||||
trap 'rm -f "$objects" "$rows"' EXIT
|
||||
|
||||
$compose run --rm --no-deps -T --entrypoint /bin/sh s3-init -c '
|
||||
IFS= read -r u < /run/secrets/minio_root_user
|
||||
IFS= read -r p < /run/secrets/minio_root_password
|
||||
export MC_CONFIG_DIR=/tmp/.mc MC_HOST_stand="http://$u:$p@s3:9000"
|
||||
exec mc ls --recursive "stand/$S3_BUCKET"' 2>/dev/null \
|
||||
| sed -n 's/.* \(attachments\/.*\)$/\1/p' | sort > "$objects"
|
||||
$compose exec -T postgres psql -U "${POSTGRES_USER:-messenger}" -d "${POSTGRES_DB:-messenger}" -At \
|
||||
-c "SELECT object_key, status FROM attachments" | sort > "$rows"
|
||||
|
||||
echo "bucket: $bucket"
|
||||
echo "objects: $(wc -l < "$objects" | tr -d ' ')"
|
||||
echo "rows by status:"
|
||||
cut -d'|' -f2 "$rows" | sort | uniq -c | sed 's/^ */ /'
|
||||
orphans=$(cut -d'|' -f1 "$rows" | sort | comm -13 - "$objects")
|
||||
missing=$(grep '|ready$' "$rows" | cut -d'|' -f1 | sort | comm -23 - "$objects")
|
||||
echo "orphans: $(printf '%s' "$orphans" | grep -c . || true)"
|
||||
[ -z "$orphans" ] || printf '%s\n' "$orphans" | sed 's/^/ orphan /'
|
||||
echo "missing: $(printf '%s' "$missing" | grep -c . || true)"
|
||||
[ -z "$missing" ] || printf '%s\n' "$missing" | sed 's/^/ missing /'
|
||||
[ -z "$orphans$missing" ]
|
||||
@@ -0,0 +1,33 @@
|
||||
// Peak RSS of one full decode check with the same sharp pipeline as
|
||||
// app/src/attachments/image/image-decoder.ts, on worst-case inputs near
|
||||
// MAX_IMAGE_PIXELS. Runs where the API runs (same libvips, VIPS_DISC_THRESHOLD):
|
||||
// docker compose exec -T api node - < scripts/lab4/decode_memory.mjs # reads files from IMAGES (paths in the container)
|
||||
// or on the host from app/: IMAGES="a.png b.jpg" node ../scripts/lab4/decode_memory.mjs
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { createRequire } from 'node:module';
|
||||
|
||||
const sharp = createRequire(`${process.cwd()}/`)('sharp');
|
||||
sharp.cache(false);
|
||||
sharp.concurrency(1);
|
||||
for (const file of (process.env.IMAGES ?? '').split(/\s+/u).filter(Boolean)) {
|
||||
const data = readFileSync(file);
|
||||
const meta = await sharp(data).metadata();
|
||||
const base = process.memoryUsage().rss;
|
||||
let peak = base;
|
||||
const timer = setInterval(() => {
|
||||
peak = Math.max(peak, process.memoryUsage().rss);
|
||||
}, 5);
|
||||
const started = Date.now();
|
||||
let verdict = 'ok';
|
||||
try {
|
||||
await sharp(data, { failOn: 'warning', sequentialRead: true, limitInputPixels: 40_000_000 })
|
||||
.resize(64, 64, { fit: 'inside' })
|
||||
.toBuffer();
|
||||
} catch (error) {
|
||||
verdict = `rejected: ${String(error.message).split('\n')[0]}`;
|
||||
}
|
||||
clearInterval(timer);
|
||||
console.log(
|
||||
`${meta.format} ${meta.width}x${meta.height} ${meta.channels}ch ${meta.depth}${meta.isProgressive ? ' progressive' : ''}${meta.isPalette ? ' palette' : ''} bytes=${data.length} decode_ms=${Date.now() - started} peak_delta_rss_mib=${((peak - base) / 1048576).toFixed(1)} ${verdict}`,
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Synthetic PNG helpers (stdlib only); same semantics as app/test/support/png.ts."""
|
||||
import random
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
NOISE_WIDTH = 1024
|
||||
NOISE_ROW = 1 + NOISE_WIDTH * 3
|
||||
|
||||
|
||||
def chunk(kind: bytes, data: bytes) -> bytes:
|
||||
crc = zlib.crc32(kind + data) & 0xFFFFFFFF
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", crc)
|
||||
|
||||
|
||||
def build_png(width: int, height: int, idat: bytes) -> bytes:
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return SIGNATURE + chunk(b"IHDR", ihdr) + chunk(b"IDAT", idat) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def pad_png(data: bytes, size: int) -> bytes:
|
||||
"""Insert a private ancillary chunk before IEND to reach exactly `size` bytes."""
|
||||
extra = size - len(data)
|
||||
if extra == 0:
|
||||
return data
|
||||
if extra < 12:
|
||||
raise ValueError("padding needs at least 12 bytes")
|
||||
return data[:-12] + chunk(b"prVt", bytes(extra - 12)) + data[-12:]
|
||||
|
||||
|
||||
def _noise_of_height(height: int, rng: random.Random) -> bytes:
|
||||
raw = bytearray(rng.randbytes(NOISE_ROW * height))
|
||||
for row in range(height):
|
||||
raw[row * NOISE_ROW] = 0 # filter type "none"
|
||||
compressor = zlib.compressobj(0)
|
||||
return build_png(NOISE_WIDTH, height, compressor.compress(bytes(raw)) + compressor.flush())
|
||||
|
||||
|
||||
def noise_png(size: int, seed: int = 0) -> bytes:
|
||||
"""Incompressible, structurally valid PNG of exactly `size` bytes."""
|
||||
rng = random.Random(seed)
|
||||
height = max(1, size // NOISE_ROW)
|
||||
while True:
|
||||
png = _noise_of_height(height, rng)
|
||||
if len(png) > size or 0 < size - len(png) < 12:
|
||||
height -= 1
|
||||
if height < 1:
|
||||
raise ValueError("size is too small")
|
||||
continue
|
||||
return pad_png(png, size)
|
||||
|
||||
|
||||
ADAM7 = [(0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)]
|
||||
|
||||
|
||||
def worst_decode_png(size: int = 10_485_760, width: int = 8000, height: int = 5000) -> bytes:
|
||||
"""The heaviest valid upload to decode: 40 MP RGBA 16-bit, Adam7-interlaced
|
||||
(the decoder must hold the whole image), padded to exactly `size` bytes."""
|
||||
compressor = zlib.compressobj(1)
|
||||
parts = []
|
||||
for x0, y0, dx, dy in ADAM7:
|
||||
pass_width = (width - x0 + dx - 1) // dx if width > x0 else 0
|
||||
pass_height = (height - y0 + dy - 1) // dy if height > y0 else 0
|
||||
if pass_width == 0 or pass_height == 0:
|
||||
continue
|
||||
row = bytes(1 + pass_width * 8)
|
||||
for _ in range(pass_height):
|
||||
parts.append(compressor.compress(row))
|
||||
parts.append(compressor.flush())
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 16, 6, 0, 0, 1)
|
||||
png = SIGNATURE + chunk(b"IHDR", ihdr) + chunk(b"IDAT", b"".join(parts)) + chunk(b"IEND", b"")
|
||||
return pad_png(png, size)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
|
||||
sys.stdout.buffer.write(worst_decode_png())
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/bin/sh
|
||||
# Experiment matrix: sizes x {proxy,direct} at C=10, plus proxy 9.5 MiB at C=20.
|
||||
# Needs a running stand (make stand-up) and `make init-env` credentials.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/../.."
|
||||
out=evidence/lab4/bench
|
||||
mkdir -p "$out"
|
||||
bench() {
|
||||
mode=$1 size=$2 conc=$3
|
||||
python3 scripts/lab4/attachments_bench.py run --mode "$mode" --size-bytes "$size" \
|
||||
--concurrency "$conc" --uploads-per-worker 5 --repeat 3 \
|
||||
--output "$out/$mode-$size-c$conc.json"
|
||||
}
|
||||
for size in 131072 1048576 9961472; do
|
||||
for mode in proxy direct; do
|
||||
echo "== $mode size=$size c=10"
|
||||
bench "$mode" "$size" 10
|
||||
done
|
||||
done
|
||||
echo "== proxy size=9961472 c=20"
|
||||
bench proxy 9961472 20
|
||||
# Worst case for memory: every request carries the heaviest valid image
|
||||
# (40 MP RGBA16 Adam7 PNG padded to 10 MiB, ~340 MiB per decode) and C=32
|
||||
# saturates UPLOAD_MAX_IN_FLIGHT (16) on both replicas.
|
||||
tmp=$(mktemp -d)
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
worst="$tmp/worst-40mp-rgba16-adam7.png"
|
||||
python3 scripts/lab4/pngkit.py > "$worst"
|
||||
for mode in proxy direct; do
|
||||
echo "== $mode worst-case image c=32"
|
||||
python3 scripts/lab4/attachments_bench.py run --mode "$mode" --image "$worst" \
|
||||
--concurrency 32 --uploads-per-worker 2 --repeat 3 \
|
||||
--output "$out/$mode-worst-c32.json"
|
||||
done
|
||||
python3 scripts/lab4/attachments_bench.py summarize "$out"/*.json > evidence/lab4/summary.md
|
||||
cat evidence/lab4/summary.md
|
||||
Executable
+749
@@ -0,0 +1,749 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Automated fault injection for lab5 levels 4 and 5 (track 2)."""
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'lab4'))
|
||||
|
||||
from stand import ( # noqa: E402
|
||||
ROOT,
|
||||
Failure,
|
||||
check,
|
||||
clear_fault,
|
||||
clear_faults,
|
||||
cli,
|
||||
compose,
|
||||
docker,
|
||||
new_client,
|
||||
rabbit_queues,
|
||||
sample_bytes,
|
||||
set_fault,
|
||||
sha256,
|
||||
upload,
|
||||
variant_sha,
|
||||
wait_for,
|
||||
wait_ready,
|
||||
worker_env,
|
||||
worker_start_count,
|
||||
)
|
||||
from pngkit import worst_decode_png # noqa: E402
|
||||
|
||||
HINT = ('FAULT_INJECTION_ENABLED=1 JOB_TIMEOUT_SECONDS=10 '
|
||||
'JOB_LEASE_SECONDS=8 JOB_RETRY_BASE_MS=500 JOB_RETRY_MAX_MS=4000 '
|
||||
'make stand-up')
|
||||
|
||||
LEVEL4 = [
|
||||
'crash_after_commit_before_publish',
|
||||
'crash_after_side_effect',
|
||||
'crash_after_commit_before_ack',
|
||||
'retry_and_replay',
|
||||
'hung_worker',
|
||||
'poison',
|
||||
'big_image',
|
||||
]
|
||||
TRACK2 = [
|
||||
'duplicate',
|
||||
'reorder',
|
||||
'crash_before_side_effect',
|
||||
'crash_after_side_effect',
|
||||
'crash_after_commit_before_ack',
|
||||
'relay_crash_after_publish',
|
||||
'stale_version',
|
||||
]
|
||||
|
||||
|
||||
class Tracker(threading.Thread):
|
||||
"""Polls metadata while a scenario runs to prove readiness never regresses."""
|
||||
|
||||
def __init__(self, base_url, actor, ca_file=None):
|
||||
super().__init__(daemon=True)
|
||||
self.client = new_client(base_url, ca_file)
|
||||
self.actor = actor
|
||||
self.ids = []
|
||||
self.records = {}
|
||||
self.stop_event = threading.Event()
|
||||
|
||||
def track(self, attachment_id):
|
||||
if attachment_id not in self.records:
|
||||
self.ids.append(attachment_id)
|
||||
self.records[attachment_id] = []
|
||||
|
||||
def run(self):
|
||||
while not self.stop_event.is_set():
|
||||
for attachment_id in list(self.ids):
|
||||
try:
|
||||
meta = self.client.call(
|
||||
'GET', '/api/v1/attachments/' + attachment_id,
|
||||
actor=self.actor, schema='Attachment',
|
||||
)['json']
|
||||
self.records[attachment_id].append(
|
||||
(time.monotonic(), meta['status'],
|
||||
meta.get('ready_pipeline_version'))
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
self.stop_event.wait(0.2)
|
||||
|
||||
def stop(self):
|
||||
self.stop_event.set()
|
||||
self.join(timeout=5)
|
||||
|
||||
|
||||
def build_message(status, job, attempt=None):
|
||||
created = next(
|
||||
event for event in status['events']
|
||||
if event['job_id'] == job['job_id'] and event['event'] == 'created'
|
||||
)
|
||||
return {
|
||||
'schema_version': 1,
|
||||
'job_id': job['job_id'],
|
||||
'type': 'attachment.process.v1',
|
||||
'attachment_id': status['attachment']['id'],
|
||||
'pipeline_version': job['pipeline_version'],
|
||||
'attempt': attempt if attempt is not None else job['attempt'],
|
||||
'created_at': created['created_at'],
|
||||
}
|
||||
|
||||
|
||||
def publish_raw(messages):
|
||||
text = ''.join(json.dumps(message, separators=(',', ':')) + '\n'
|
||||
for message in messages)
|
||||
code, payload = cli('publish-raw', stdin=text)
|
||||
check(code == 0, 'publish-raw failed: ' + str(payload))
|
||||
return payload
|
||||
|
||||
|
||||
def status_of(attachment_id):
|
||||
code, payload = cli('status', attachment_id)
|
||||
check(code == 0, 'status failed: ' + str(payload))
|
||||
return payload
|
||||
|
||||
|
||||
def flush_outbox(attachment_id):
|
||||
wait_for(lambda: status_of(attachment_id)['outbox']['pending'] == 0,
|
||||
30, 'outbox publish', 1.0)
|
||||
|
||||
|
||||
def new_attachment_id(chat_id, before):
|
||||
rows = cli('find-by-chat', chat_id)[1]
|
||||
for row in reversed(rows):
|
||||
if row['id'] not in before:
|
||||
return row['id']
|
||||
return None
|
||||
|
||||
|
||||
def evaluate(tracker, attachment_id, baseline, expect_failed=False,
|
||||
compare_baseline=True):
|
||||
if attachment_id is None:
|
||||
return {'I1': True, 'I2': True, 'I3': True,
|
||||
'I4': True, 'I5': True, 'I6': True}
|
||||
status = status_of(attachment_id)
|
||||
attachment = status['attachment']
|
||||
jobs = status['jobs']
|
||||
variants = status['variants']
|
||||
ready_version = attachment['ready_pipeline_version']
|
||||
|
||||
records = tracker.records.get(attachment_id, [])
|
||||
reached = any(record[1] == ('failed' if expect_failed else 'ready')
|
||||
for record in records)
|
||||
if not records:
|
||||
reached = attachment['status'] == ('failed' if expect_failed else 'ready')
|
||||
i1 = reached or attachment['status'] in ('ready', 'failed')
|
||||
|
||||
ready_variants = [v for v in variants
|
||||
if v['pipeline_version'] == ready_version]
|
||||
succeeded_ids = {job['job_id'] for job in jobs
|
||||
if job['status'] == 'succeeded'}
|
||||
succeeded_events = {}
|
||||
for event in status['events']:
|
||||
if event['event'] == 'succeeded':
|
||||
succeeded_events[event['job_id']] = (
|
||||
succeeded_events.get(event['job_id'], 0) + 1
|
||||
)
|
||||
i2 = (len(ready_variants) == 3
|
||||
and all(succeeded_events.get(job_id) == 1
|
||||
for job_id in succeeded_ids))
|
||||
|
||||
i3 = True
|
||||
seen_ready = False
|
||||
last_version = 0
|
||||
for _at, state, version in records:
|
||||
if seen_ready and state != 'ready':
|
||||
i3 = False
|
||||
if state == 'ready':
|
||||
seen_ready = True
|
||||
if version is not None:
|
||||
if version < last_version:
|
||||
i3 = False
|
||||
last_version = version
|
||||
|
||||
i4 = bool(succeeded_ids) or expect_failed or attachment['status'] == 'ready'
|
||||
keys = {(variant['pipeline_version'], variant['variant'])
|
||||
for variant in variants}
|
||||
i5 = len(keys) == len(variants)
|
||||
|
||||
if compare_baseline and ready_version == 1 and ready_variants:
|
||||
observed = {variant['variant']: variant['sha256']
|
||||
for variant in ready_variants}
|
||||
i6 = all(observed.get(name) == digest
|
||||
for name, digest in baseline.items())
|
||||
else:
|
||||
i6 = True
|
||||
|
||||
return {'I1': i1, 'I2': i2, 'I3': i3, 'I4': i4,
|
||||
'I5': i5, 'I6': i6}
|
||||
|
||||
|
||||
# --- scenarios -----------------------------------------------------------
|
||||
|
||||
|
||||
def scenario_duplicate(ctx):
|
||||
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
||||
sample_bytes(), 'duplicate.png')
|
||||
ctx['attachment_id'] = attachment['id']
|
||||
ctx['tracker'].track(attachment['id'])
|
||||
wait_ready(ctx['client'], ctx['alice'], attachment['id'])
|
||||
status = status_of(attachment['id'])
|
||||
before = {variant['variant']: variant['s3']
|
||||
for variant in status['variants']}
|
||||
message = build_message(status, status['jobs'][0])
|
||||
publish_raw([message] * 5)
|
||||
wait_for(
|
||||
lambda: len([event for event in status_of(attachment['id'])['events']
|
||||
if event['event'] == 'duplicate_skipped'
|
||||
and event['detail'].get('reason') == 'already_succeeded']) == 5,
|
||||
20, 'five duplicate_skipped events', 1.0,
|
||||
)
|
||||
after_status = status_of(attachment['id'])
|
||||
after = {variant['variant']: variant['s3']
|
||||
for variant in after_status['variants']}
|
||||
check(before == after, 'duplicate delivery rewrote variant objects')
|
||||
return {'attachment_id': attachment['id'],
|
||||
'job_attempts': [job['attempt'] for job in after_status['jobs']],
|
||||
'notes': '5 duplicate deliveries were absorbed by the DB fence'}
|
||||
|
||||
|
||||
def scenario_reorder(ctx):
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
||||
sample_bytes(), 'reorder.png')
|
||||
ctx['attachment_id'] = attachment['id']
|
||||
ctx['tracker'].track(attachment['id'])
|
||||
flush_outbox(attachment['id'])
|
||||
status = status_of(attachment['id'])
|
||||
message = build_message(status, status['jobs'][0])
|
||||
stale = dict(message, attempt=99)
|
||||
publish_raw([stale, message])
|
||||
compose('start', 'worker')
|
||||
wait_ready(ctx['client'], ctx['alice'], attachment['id'])
|
||||
finally:
|
||||
compose('start', 'worker')
|
||||
after_status = status_of(attachment['id'])
|
||||
skipped = [event for event in after_status['events']
|
||||
if event['event'] == 'duplicate_skipped']
|
||||
check(len(skipped) == 2, 'expected exactly two skipped out-of-order messages')
|
||||
check(len([event for event in after_status['events']
|
||||
if event['event'] == 'succeeded']) == 1,
|
||||
'expected exactly one successful effect')
|
||||
return {'attachment_id': attachment['id'],
|
||||
'job_attempts': [job['attempt'] for job in after_status['jobs']],
|
||||
'notes': 'attempt 99 and a duplicate attempt 1 were acked as skips'}
|
||||
|
||||
|
||||
def crash_scenario(ctx, point, filename, expected_attempt,
|
||||
expect_lease_expired=False, expect_duplicate=False):
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
||||
sample_bytes(), filename)
|
||||
ctx['attachment_id'] = attachment['id']
|
||||
ctx['tracker'].track(attachment['id'])
|
||||
starts = worker_start_count()
|
||||
set_fault('fault:' + point + ':' + attachment['id'] + ':1', 'crash')
|
||||
compose('start', 'worker')
|
||||
wait_ready(ctx['client'], ctx['alice'], attachment['id'], 90)
|
||||
finally:
|
||||
compose('start', 'worker')
|
||||
status = status_of(attachment['id'])
|
||||
job = status['jobs'][0]
|
||||
check(job['attempt'] == expected_attempt,
|
||||
'expected attempt ' + str(expected_attempt) +
|
||||
', got ' + str(job['attempt']))
|
||||
check(worker_start_count() >= starts + 2,
|
||||
'worker process did not restart after the injected crash')
|
||||
if expect_lease_expired:
|
||||
check(any(event['event'] == 'lease_expired'
|
||||
for event in status['events']),
|
||||
'missing lease_expired event')
|
||||
if expect_duplicate:
|
||||
check(any(event['event'] == 'duplicate_skipped'
|
||||
and event['detail'].get('reason') == 'already_succeeded'
|
||||
for event in status['events']),
|
||||
'missing already_succeeded duplicate after redelivery')
|
||||
return {'attachment_id': attachment['id'],
|
||||
'job_attempts': [job['attempt']],
|
||||
'notes': 'crashed at ' + point}
|
||||
|
||||
|
||||
def scenario_crash_before_side_effect(ctx):
|
||||
return crash_scenario(ctx, 'worker.before_side_effect',
|
||||
'crash-before.png', 2, expect_lease_expired=True)
|
||||
|
||||
|
||||
def scenario_crash_after_side_effect(ctx):
|
||||
return crash_scenario(ctx, 'worker.after_side_effect',
|
||||
'crash-after.png', 2, expect_lease_expired=True)
|
||||
|
||||
|
||||
def scenario_crash_after_commit_before_ack(ctx):
|
||||
return crash_scenario(ctx, 'worker.after_commit_before_ack',
|
||||
'crash-ack.png', 1, expect_duplicate=True)
|
||||
|
||||
|
||||
def scenario_crash_after_commit_before_publish(ctx):
|
||||
before = {row['id'] for row in cli('find-by-chat', ctx['chat']['id'])[1]}
|
||||
set_fault('fault:api.after_commit_before_publish', 'crash')
|
||||
try:
|
||||
try:
|
||||
ctx['client'].call(
|
||||
'POST',
|
||||
'/api/v1/chats/' + ctx['chat']['id'] +
|
||||
'/attachments?filename=crash-publish.png',
|
||||
(202, 502, 503, 504),
|
||||
actor=ctx['alice'],
|
||||
raw=sample_bytes(),
|
||||
headers={'Content-Type': 'image/png'},
|
||||
)
|
||||
except Failure:
|
||||
# A killed replica may close the connection instead of answering.
|
||||
pass
|
||||
finally:
|
||||
clear_fault('fault:api.after_commit_before_publish')
|
||||
attachment_id = wait_for(
|
||||
lambda: new_attachment_id(ctx['chat']['id'], before), 30,
|
||||
'attachment row after API crash', 1.0,
|
||||
)
|
||||
ctx['attachment_id'] = attachment_id
|
||||
ctx['tracker'].track(attachment_id)
|
||||
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
|
||||
status = status_of(attachment_id)
|
||||
check(len([event for event in status['events']
|
||||
if event['event'] == 'created']) == 1,
|
||||
'expected exactly one created event')
|
||||
return {'attachment_id': attachment_id,
|
||||
'job_attempts': [job['attempt'] for job in status['jobs']],
|
||||
'notes': 'outbox relay published the committed job after API restart'}
|
||||
|
||||
|
||||
def scenario_relay_crash_after_publish(ctx):
|
||||
compose('stop', 'worker')
|
||||
before = {row['id'] for row in cli('find-by-chat', ctx['chat']['id'])[1]}
|
||||
try:
|
||||
set_fault('fault:relay.after_publish_before_mark', 'crash')
|
||||
try:
|
||||
ctx['client'].call(
|
||||
'POST',
|
||||
'/api/v1/chats/' + ctx['chat']['id'] +
|
||||
'/attachments?filename=relay-crash.png',
|
||||
(202, 502, 503, 504),
|
||||
actor=ctx['alice'],
|
||||
raw=sample_bytes(),
|
||||
headers={'Content-Type': 'image/png'},
|
||||
)
|
||||
except Failure:
|
||||
# The relay may crash the handling replica before the reply flushes.
|
||||
pass
|
||||
clear_fault('fault:relay.after_publish_before_mark')
|
||||
attachment_id = wait_for(
|
||||
lambda: new_attachment_id(ctx['chat']['id'], before), 30,
|
||||
'attachment row after relay crash', 1.0,
|
||||
)
|
||||
ctx['attachment_id'] = attachment_id
|
||||
ctx['tracker'].track(attachment_id)
|
||||
wait_for(lambda: status_of(attachment_id)['outbox']['pending'] == 0,
|
||||
60, 'outbox published after relay crash', 1.0)
|
||||
queue_depth = rabbit_queues().get('attachments.process', {}).get(
|
||||
'messages', 0)
|
||||
compose('start', 'worker')
|
||||
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
|
||||
finally:
|
||||
clear_fault('fault:relay.after_publish_before_mark')
|
||||
compose('start', 'worker')
|
||||
status = status_of(attachment_id)
|
||||
check(len([event for event in status['events']
|
||||
if event['event'] == 'succeeded']) == 1,
|
||||
'expected exactly one succeeded event')
|
||||
duplicated = any(event['event'] == 'duplicate_skipped'
|
||||
for event in status['events'])
|
||||
note = ('relay republished the same message; duplicate absorbed'
|
||||
if duplicated else
|
||||
'relay published once in this run; I1/I2 still hold')
|
||||
if queue_depth >= 2:
|
||||
check(duplicated, 'two queued copies but no duplicate_skipped event')
|
||||
return {'attachment_id': attachment_id,
|
||||
'job_attempts': [job['attempt'] for job in status['jobs']],
|
||||
'notes': note}
|
||||
|
||||
|
||||
def scenario_stale_version(ctx):
|
||||
check(int(worker_env('WORKER_CONCURRENCY') or '1') >= 2,
|
||||
'stale_version needs WORKER_CONCURRENCY>=2')
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
||||
sample_bytes(), 'stale.png')
|
||||
attachment_id = attachment['id']
|
||||
ctx['attachment_id'] = attachment_id
|
||||
ctx['tracker'].track(attachment_id)
|
||||
flush_outbox(attachment_id)
|
||||
code, payload = cli('reprocess', attachment_id,
|
||||
'--pipeline-version', '2', '--reason', 'suite')
|
||||
check(code == 0, 'reprocess v2 failed: ' + str(payload))
|
||||
set_fault('fault:worker.after_side_effect:' + attachment_id + ':1',
|
||||
'delay:8000')
|
||||
compose('start', 'worker')
|
||||
v2_etags = None
|
||||
|
||||
def both_terminal():
|
||||
nonlocal v2_etags
|
||||
status = status_of(attachment_id)
|
||||
jobs = {(job['pipeline_version']): job for job in status['jobs']}
|
||||
if '2' in jobs or 2 in jobs:
|
||||
v2_job = jobs.get(2)
|
||||
if v2_job and v2_job['status'] == 'succeeded' and v2_etags is None:
|
||||
v2_etags = {variant['variant']: variant['s3']
|
||||
for variant in status['variants']
|
||||
if variant['pipeline_version'] == 2}
|
||||
v1_job = jobs.get(1)
|
||||
v2_job = jobs.get(2)
|
||||
return (v1_job and v2_job
|
||||
and v1_job['status'] == 'superseded'
|
||||
and v2_job['status'] == 'succeeded')
|
||||
|
||||
wait_for(both_terminal, 90, 'v2 ready and v1 superseded', 1.0)
|
||||
status = status_of(attachment_id)
|
||||
check(status['attachment']['ready_pipeline_version'] == 2,
|
||||
'ready pipeline version is not 2')
|
||||
check(all(variant['pipeline_version'] == 2
|
||||
for variant in status['variants']),
|
||||
'variant rows from v1 survived')
|
||||
after_etags = {variant['variant']: variant['s3']
|
||||
for variant in status['variants']
|
||||
if variant['pipeline_version'] == 2}
|
||||
check(v2_etags == after_etags, 'v1 overwrote v2 variant objects')
|
||||
|
||||
served = ctx['client'].call(
|
||||
'GET',
|
||||
'/api/v1/attachments/' + attachment_id +
|
||||
'/content?variant=thumbnail',
|
||||
actor=ctx['alice'],
|
||||
)
|
||||
v2_thumbnail = next(variant for variant in status['variants']
|
||||
if variant['variant'] == 'thumbnail')
|
||||
check(sha256(served['bytes']) == v2_thumbnail['sha256'],
|
||||
'served thumbnail is not the v2 object')
|
||||
check(sha256(served['bytes']) != ctx['baseline']['thumbnail'],
|
||||
'served thumbnail equals the v1 baseline')
|
||||
|
||||
v1_job = next(job for job in status['jobs']
|
||||
if job['pipeline_version'] == 1)
|
||||
publish_raw([build_message(status, v1_job)])
|
||||
wait_for(
|
||||
lambda: any(event['event'] == 'duplicate_skipped'
|
||||
and event['detail'].get('reason') == 'superseded'
|
||||
for event in status_of(attachment_id)['events']),
|
||||
20, 'superseded duplicate skip', 1.0,
|
||||
)
|
||||
finally:
|
||||
clear_fault('fault:worker.after_side_effect:' +
|
||||
(attachment['id'] if 'attachment' in locals() else ''))
|
||||
compose('start', 'worker')
|
||||
return {'attachment_id': attachment_id,
|
||||
'job_attempts': [job['attempt'] for job in status['jobs']],
|
||||
'notes': 'v1 finished after v2 and was recorded as superseded'}
|
||||
|
||||
|
||||
def scenario_poison(ctx):
|
||||
before = rabbit_queues().get('attachments.process.dlq', {}).get(
|
||||
'messages', 0)
|
||||
publish_raw(['not-json', '{"schema_version":1}'])
|
||||
wait_for(
|
||||
lambda: rabbit_queues().get('attachments.process.dlq', {}).get(
|
||||
'messages', 0) >= before + 2,
|
||||
15, 'two poison messages in DLQ', 0.5,
|
||||
)
|
||||
return {'attachment_id': None, 'job_attempts': [],
|
||||
'notes': 'malformed messages moved to the DLQ without touching jobs'}
|
||||
|
||||
|
||||
def scenario_retry_and_replay(ctx):
|
||||
max_attempts = int(worker_env('JOB_MAX_ATTEMPTS') or '5')
|
||||
base = int(worker_env('JOB_RETRY_BASE_MS') or '500')
|
||||
cap = int(worker_env('JOB_RETRY_MAX_MS') or '4000')
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
||||
sample_bytes(), 'retry.png')
|
||||
attachment_id = attachment['id']
|
||||
ctx['attachment_id'] = attachment_id
|
||||
ctx['tracker'].track(attachment_id)
|
||||
set_fault('fault:worker.before_side_effect:' + attachment_id + ':1',
|
||||
'transient')
|
||||
compose('start', 'worker')
|
||||
failed = wait_ready(ctx['client'], ctx['alice'], attachment_id,
|
||||
90, allow_failed=True)
|
||||
finally:
|
||||
compose('start', 'worker')
|
||||
check(failed['status'] == 'failed', 'transient fault was not exhausted')
|
||||
check(failed['error_code'] == 'retry_exhausted',
|
||||
'expected retry_exhausted, got ' + str(failed['error_code']))
|
||||
status = status_of(attachment_id)
|
||||
retries = [event for event in status['events']
|
||||
if event['event'] == 'retry_scheduled']
|
||||
check(len(retries) == max_attempts - 1,
|
||||
'expected ' + str(max_attempts - 1) + ' retries, got ' +
|
||||
str(len(retries)))
|
||||
for index, event in enumerate(retries):
|
||||
limit = min(cap, base * 2 ** index)
|
||||
check(0 <= event['detail'].get('delay_ms', cap + 1) <= limit,
|
||||
'retry delay exceeds its jitter cap')
|
||||
ctx['client'].call(
|
||||
'GET', '/api/v1/attachments/' + attachment_id + '/content',
|
||||
409, actor=ctx['alice'],
|
||||
)
|
||||
code, payload = cli('replay', ctx['reference_id'], '--reason', 'suite')
|
||||
check(code == 3 and payload.get('reason') in ('no_failed_job',
|
||||
'already_ready'),
|
||||
'replay of a ready attachment was not refused')
|
||||
|
||||
clear_fault('fault:worker.before_side_effect:' + attachment_id + ':1')
|
||||
code, payload = cli('replay', attachment_id, '--reason', 'suite')
|
||||
check(code == 0, 'operator replay failed: ' + str(payload))
|
||||
wait_ready(ctx['client'], ctx['alice'], attachment_id, 60)
|
||||
status = status_of(attachment_id)
|
||||
replayed = [event for event in status['events']
|
||||
if event['event'] == 'replayed']
|
||||
check(len(replayed) == 1 and replayed[0]['detail'].get('operator'),
|
||||
'missing replayed event or operator audit')
|
||||
return {'attachment_id': attachment_id,
|
||||
'job_attempts': [job['attempt'] for job in status['jobs']],
|
||||
'notes': 'transient injected fault exhausted retries; replay recovered'}
|
||||
|
||||
|
||||
def scenario_hung_worker(ctx):
|
||||
timeout_seconds = int(worker_env('JOB_TIMEOUT_SECONDS') or '10')
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
|
||||
sample_bytes(), 'hung.png')
|
||||
attachment_id = attachment['id']
|
||||
ctx['attachment_id'] = attachment_id
|
||||
ctx['tracker'].track(attachment_id)
|
||||
set_fault('fault:worker.after_side_effect:' + attachment_id + ':1',
|
||||
'delay:' + str((timeout_seconds + 5) * 1000))
|
||||
compose('start', 'worker')
|
||||
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
|
||||
finally:
|
||||
compose('start', 'worker')
|
||||
status = status_of(attachment_id)
|
||||
check(any(event['event'] == 'retry_scheduled'
|
||||
and event['detail'].get('code') == 'processing_timeout'
|
||||
for event in status['events']),
|
||||
'missing processing_timeout retry')
|
||||
return {'attachment_id': attachment_id,
|
||||
'job_attempts': [job['attempt'] for job in status['jobs']],
|
||||
'notes': 'hard timeout aborted the hung render and retried'}
|
||||
|
||||
|
||||
def scenario_big_image(ctx):
|
||||
ctx['compare_baseline'] = False
|
||||
data = worst_decode_png()
|
||||
containers = compose('ps', '-q', 'worker').stdout.split()
|
||||
stop = threading.Event()
|
||||
peak = {'mb': 0.0}
|
||||
|
||||
def sample():
|
||||
while not stop.is_set():
|
||||
if containers:
|
||||
result = docker(
|
||||
'stats', '--no-stream', '--format',
|
||||
'{{.Name}} {{.MemUsage}}', *containers,
|
||||
check_result=False,
|
||||
)
|
||||
for line in result.stdout.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
match = re.match(r'([\d.]+)(KiB|MiB|GiB)', parts[1])
|
||||
if not match:
|
||||
continue
|
||||
value = float(match.group(1))
|
||||
if match.group(2) == 'GiB':
|
||||
value *= 1024
|
||||
elif match.group(2) == 'KiB':
|
||||
value /= 1024
|
||||
peak['mb'] = max(peak['mb'], value)
|
||||
stop.wait(0.3)
|
||||
|
||||
sampler = threading.Thread(target=sample, daemon=True)
|
||||
sampler.start()
|
||||
try:
|
||||
def do_upload(index):
|
||||
client = new_client(ctx['base_url'])
|
||||
return upload(client, ctx['alice'], ctx['chat'], data,
|
||||
'big-' + str(index) + '.png')
|
||||
|
||||
with ThreadPoolExecutor(max_workers=2) as pool:
|
||||
attachments = list(pool.map(do_upload, (1, 2)))
|
||||
ctx['attachment_id'] = attachments[0]['id']
|
||||
for attachment in attachments:
|
||||
ctx['tracker'].track(attachment['id'])
|
||||
results = [wait_ready(ctx['client'], ctx['alice'],
|
||||
attachment['id'], 120, allow_failed=True)
|
||||
for attachment in attachments]
|
||||
finally:
|
||||
stop.set()
|
||||
sampler.join(timeout=5)
|
||||
for meta in results:
|
||||
check(meta['status'] in ('ready', 'failed'), 'big image stuck')
|
||||
if meta['status'] == 'failed':
|
||||
check(meta['error_code'] in ('image_too_large', 'invalid_image'),
|
||||
'unexpected big-image error code')
|
||||
for container in containers:
|
||||
result = docker('inspect', '-f', '{{.State.OOMKilled}}', container,
|
||||
check_result=False)
|
||||
check(result.stdout.strip() == 'false', 'worker was OOM-killed')
|
||||
return {'attachment_id': attachments[0]['id'],
|
||||
'job_attempts': [],
|
||||
'notes': 'peak worker memory: %.1f MiB / 1280 MiB' % peak['mb']}
|
||||
|
||||
|
||||
SCENARIOS = {
|
||||
'duplicate': scenario_duplicate,
|
||||
'reorder': scenario_reorder,
|
||||
'crash_before_side_effect': scenario_crash_before_side_effect,
|
||||
'crash_after_side_effect': scenario_crash_after_side_effect,
|
||||
'crash_after_commit_before_ack': scenario_crash_after_commit_before_ack,
|
||||
'crash_after_commit_before_publish': scenario_crash_after_commit_before_publish,
|
||||
'relay_crash_after_publish': scenario_relay_crash_after_publish,
|
||||
'stale_version': scenario_stale_version,
|
||||
'poison': scenario_poison,
|
||||
'retry_and_replay': scenario_retry_and_replay,
|
||||
'hung_worker': scenario_hung_worker,
|
||||
'big_image': scenario_big_image,
|
||||
}
|
||||
|
||||
|
||||
def select(name):
|
||||
if name == 'all':
|
||||
return LEVEL4 + [item for item in TRACK2 if item not in LEVEL4]
|
||||
if name == 'level4':
|
||||
return LEVEL4
|
||||
if name == 'track2':
|
||||
return TRACK2
|
||||
if name in SCENARIOS:
|
||||
return [name]
|
||||
raise SystemExit('unknown scenario: ' + name)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--base-url', default=None)
|
||||
parser.add_argument('--ca-file', default=None)
|
||||
parser.add_argument('--scenario', default='level4')
|
||||
parser.add_argument('--repeat', type=int, default=1)
|
||||
parser.add_argument('--output', default=None)
|
||||
args = parser.parse_args()
|
||||
|
||||
if worker_env('FAULT_INJECTION_ENABLED') != '1':
|
||||
print('fault injection is disabled; restart the stand with:\n ' + HINT,
|
||||
file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
|
||||
client = new_client(args.base_url, args.ca_file)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
clear_faults()
|
||||
reference = upload(client, alice, chat, sample_bytes(), 'reference.png')
|
||||
ready = wait_ready(client, alice, reference['id'], 60)
|
||||
baseline = variant_sha(ready)
|
||||
|
||||
selected = select(args.scenario)
|
||||
results = []
|
||||
failed = 0
|
||||
for name in selected:
|
||||
for repeat in range(1, args.repeat + 1):
|
||||
clear_faults()
|
||||
tracker = Tracker(args.base_url or 'http://localhost:8080', alice,
|
||||
args.ca_file)
|
||||
tracker.start()
|
||||
started = time.monotonic()
|
||||
ctx = {
|
||||
'client': client,
|
||||
'base_url': args.base_url or 'http://localhost:8080',
|
||||
'alice': alice,
|
||||
'chat': chat,
|
||||
'tracker': tracker,
|
||||
'baseline': baseline,
|
||||
'reference_id': reference['id'],
|
||||
'ca_file': args.ca_file,
|
||||
}
|
||||
scenario = {'attachment_id': None, 'job_attempts': [],
|
||||
'notes': ''}
|
||||
passed = True
|
||||
try:
|
||||
scenario.update(SCENARIOS[name](ctx))
|
||||
except Exception as error: # noqa: BLE001 - scenario boundary
|
||||
passed = False
|
||||
scenario['notes'] = type(error).__name__ + ': ' + str(error)
|
||||
finally:
|
||||
tracker.stop()
|
||||
clear_faults()
|
||||
compose('start', 'worker', check_result=False)
|
||||
attachment_id = scenario['attachment_id'] or ctx.get('attachment_id')
|
||||
try:
|
||||
invariants = evaluate(
|
||||
tracker, attachment_id, baseline,
|
||||
compare_baseline=ctx.get('compare_baseline', True),
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
invariants = {key: False for key in
|
||||
('I1', 'I2', 'I3', 'I4', 'I5', 'I6')}
|
||||
passed = passed and all(invariants.values())
|
||||
if not passed:
|
||||
failed += 1
|
||||
result = {
|
||||
'name': name,
|
||||
'repeat': repeat,
|
||||
'passed': passed,
|
||||
'duration_s': round(time.monotonic() - started, 3),
|
||||
'attachment_id': attachment_id,
|
||||
'job_attempts': scenario['job_attempts'],
|
||||
'events_summary': scenario.get('events_summary', {}),
|
||||
'invariants': invariants,
|
||||
'notes': scenario['notes'],
|
||||
}
|
||||
results.append(result)
|
||||
print(('PASS ' if passed else 'FAIL ') + name + ' #' + str(repeat) +
|
||||
' ' + str(result['duration_s']) + 's ' +
|
||||
json.dumps(invariants) + ' ' + result['notes'],
|
||||
flush=True)
|
||||
|
||||
if args.output:
|
||||
path = Path(args.output)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(results, indent=2, ensure_ascii=False) + '\n')
|
||||
print(str(len(results) - failed) + '/' + str(len(results)) + ' scenarios passed')
|
||||
raise SystemExit(1 if failed else 0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Shared stdlib-only helpers for the lab5 demo and fault suite."""
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
|
||||
ENV_FILE = '.env'
|
||||
COMPOSE = ['docker', 'compose', '--env-file', ENV_FILE]
|
||||
|
||||
|
||||
def compose(*args, stdin=None, check_result=True):
|
||||
result = subprocess.run(
|
||||
COMPOSE + list(args),
|
||||
cwd=ROOT,
|
||||
input=stdin,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
)
|
||||
if check_result and result.returncode != 0:
|
||||
detail = (result.stderr or result.stdout).strip().splitlines()
|
||||
raise Failure('compose ' + ' '.join(args) + ' failed: ' +
|
||||
(detail[-1] if detail else 'unknown error'))
|
||||
return result
|
||||
|
||||
|
||||
def docker(*args, check_result=True):
|
||||
result = subprocess.run(
|
||||
['docker'] + list(args), cwd=ROOT, text=True, capture_output=True,
|
||||
)
|
||||
if check_result and result.returncode != 0:
|
||||
detail = (result.stderr or result.stdout).strip().splitlines()
|
||||
raise Failure('docker ' + ' '.join(args) + ' failed: ' +
|
||||
(detail[-1] if detail else 'unknown error'))
|
||||
return result
|
||||
|
||||
|
||||
def cli(*args, stdin=None):
|
||||
"""Run jobs.cli in the API container; returns (exit_code, parsed_json)."""
|
||||
for _ in range(5):
|
||||
result = compose(
|
||||
'exec', '-T', 'api', 'node', 'dist/jobs/jobs.cli.js', *args,
|
||||
stdin=stdin, check_result=False,
|
||||
)
|
||||
text = result.stdout.strip()
|
||||
if text:
|
||||
try:
|
||||
return result.returncode, json.loads(text)
|
||||
except ValueError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise Failure('jobs.cli did not return JSON for ' + ' '.join(args))
|
||||
|
||||
|
||||
def set_fault(key, value):
|
||||
compose('exec', '-T', 'redis', 'redis-cli', 'SET', key, value)
|
||||
|
||||
|
||||
def clear_fault(key):
|
||||
compose('exec', '-T', 'redis', 'redis-cli', 'DEL', key)
|
||||
|
||||
|
||||
def clear_faults():
|
||||
script = (
|
||||
"local keys=redis.call('keys',ARGV[1]) "
|
||||
"for i=1,#keys do redis.call('del',keys[i]) end return #keys"
|
||||
)
|
||||
compose('exec', '-T', 'redis', 'redis-cli', 'EVAL', script, '0', 'fault:*')
|
||||
|
||||
|
||||
def rabbit_queues():
|
||||
result = compose('exec', '-T', 'broker', 'rabbitmqctl', '-q',
|
||||
'-p', 'messenger', 'list_queues',
|
||||
'name', 'messages', 'type')
|
||||
queues = {}
|
||||
for line in result.stdout.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) >= 2 and parts[0] not in ('name', 'Timeout:'):
|
||||
try:
|
||||
queues[parts[0]] = {'messages': int(parts[1]),
|
||||
'type': parts[2] if len(parts) > 2 else ''}
|
||||
except ValueError:
|
||||
continue
|
||||
return queues
|
||||
|
||||
|
||||
def wait_for(predicate, timeout, description='condition', interval=0.5):
|
||||
deadline = time.monotonic() + timeout
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
last = predicate()
|
||||
if last:
|
||||
return last
|
||||
time.sleep(interval)
|
||||
raise Failure('timed out waiting for ' + description)
|
||||
|
||||
|
||||
def upload(client, actor, chat, data, filename='sample.png',
|
||||
content_type='image/png'):
|
||||
result = client.call(
|
||||
'POST',
|
||||
'/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename,
|
||||
202,
|
||||
actor=actor,
|
||||
raw=data,
|
||||
headers={'Content-Type': content_type},
|
||||
schema='Attachment',
|
||||
)
|
||||
check(result['json']['status'] == 'queued', 'upload did not answer queued')
|
||||
return result['json']
|
||||
|
||||
|
||||
def metadata(client, actor, attachment_id):
|
||||
return client.call('GET', '/api/v1/attachments/' + attachment_id,
|
||||
actor=actor, schema='Attachment')['json']
|
||||
|
||||
|
||||
def wait_ready(client, actor, attachment_id, timeout=60,
|
||||
allow_failed=False):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
meta = metadata(client, actor, attachment_id)
|
||||
if meta['status'] == 'failed':
|
||||
if allow_failed:
|
||||
return meta
|
||||
raise Failure('attachment failed: ' +
|
||||
str(meta.get('error_code')))
|
||||
if meta['status'] == 'ready':
|
||||
return meta
|
||||
time.sleep(0.2)
|
||||
raise Failure('attachment did not become ready in ' + str(timeout) + 's')
|
||||
|
||||
|
||||
def sha256(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def variant_sha(meta):
|
||||
return {item['name']: item['sha256'] for item in meta['variants']}
|
||||
|
||||
|
||||
def sample_bytes():
|
||||
return (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
|
||||
|
||||
def worker_env(name):
|
||||
result = compose('exec', '-T', 'worker', 'printenv', name,
|
||||
check_result=False)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def worker_start_count():
|
||||
result = compose('logs', '--no-color', 'worker', check_result=False)
|
||||
return result.stdout.count('worker ready:')
|
||||
|
||||
|
||||
def worker_memory_peak(containers):
|
||||
names = [line for line in containers.splitlines() if line.strip()]
|
||||
if not names:
|
||||
return 'n/a'
|
||||
result = subprocess.run(
|
||||
['docker', 'stats', '--no-stream', '--format',
|
||||
'{{.Name}} {{.MemUsage}}'] + names,
|
||||
text=True, capture_output=True,
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def print_step(number, text):
|
||||
print('[' + str(number) + '] ' + text, flush=True)
|
||||
|
||||
|
||||
def new_client(base_url, ca_file=None):
|
||||
return Client(base_url, ca_file)
|
||||
Executable
+126
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Level-3 demo: work accepted while the worker is down survives restarts."""
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
from stand import ( # noqa: E402
|
||||
Failure,
|
||||
check,
|
||||
cli,
|
||||
compose,
|
||||
new_client,
|
||||
print_step,
|
||||
rabbit_queues,
|
||||
sample_bytes,
|
||||
sha256,
|
||||
upload,
|
||||
wait_for,
|
||||
wait_ready,
|
||||
)
|
||||
from client import arguments, run # noqa: E402
|
||||
from image_probe import jpeg_size # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('lab5 worker-down durability demo')
|
||||
client = new_client(args.base_url, args.ca_file)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
data = sample_bytes()
|
||||
|
||||
print_step(1, 'compose ps')
|
||||
print(compose('ps', '--format',
|
||||
'table {{.Service}}\t{{.Status}}').stdout.strip())
|
||||
|
||||
print_step(2, 'stop worker')
|
||||
compose('stop', 'worker')
|
||||
|
||||
print_step(3, 'upload while worker is down (expect 202 queued)')
|
||||
attachment = upload(client, alice, chat, data, 'durability.png')
|
||||
attachment_id = attachment['id']
|
||||
print('attachment=' + attachment_id + ' status=' + attachment['status'])
|
||||
|
||||
print_step(4, 'content and thumbnail must answer 409 before ready')
|
||||
for query in ('', '?variant=thumbnail'):
|
||||
result = client.call(
|
||||
'GET',
|
||||
'/api/v1/attachments/' + attachment_id + '/content' + query,
|
||||
409,
|
||||
actor=alice,
|
||||
)
|
||||
check(result['status'] == 409, 'content was served before ready')
|
||||
|
||||
print_step(5, 'link the queued attachment to messages')
|
||||
with_attachment = client.message(
|
||||
alice, chat, '', attachments=[attachment_id],
|
||||
)['json']
|
||||
plain = client.message(alice, chat, 'plain while worker is down')['json']
|
||||
check(with_attachment['attachment_ids'] == [attachment_id],
|
||||
'message lost the queued attachment')
|
||||
check(plain['attachment_ids'] == [], 'plain message gained an attachment')
|
||||
print('message=' + with_attachment['id'])
|
||||
|
||||
print_step(6, 'wait for outbox publish and a message in the queue')
|
||||
def published():
|
||||
_code, status = cli('status', attachment_id)
|
||||
queues = rabbit_queues()
|
||||
return (
|
||||
status['outbox']['pending'] == 0
|
||||
and queues.get('attachments.process', {}).get('messages', 0) >= 1
|
||||
)
|
||||
wait_for(published, 30, 'outbox publish and queue depth >= 1')
|
||||
queues = rabbit_queues()
|
||||
print('queue=' + str(queues.get('attachments.process')))
|
||||
|
||||
print_step(7, 'restart broker and API with volumes')
|
||||
compose('restart', 'broker', 'api')
|
||||
compose('up', '-d', '--no-recreate', '--no-deps', '--wait', 'broker', 'api')
|
||||
|
||||
print_step(8, 'the quorum message survived the broker restart')
|
||||
def survived():
|
||||
depth = rabbit_queues().get('attachments.process', {}).get('messages', 0)
|
||||
return depth >= 1
|
||||
wait_for(survived, 60, 'queue depth >= 1 after broker restart')
|
||||
print('queue=' + str(rabbit_queues().get('attachments.process')))
|
||||
|
||||
print_step(9, 'start worker')
|
||||
compose('start', 'worker')
|
||||
|
||||
print_step(10, 'wait ready and verify the thumbnail')
|
||||
ready = wait_ready(client, alice, attachment_id, 60)
|
||||
thumbnail = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0]
|
||||
content = client.call(
|
||||
'GET',
|
||||
'/api/v1/attachments/' + attachment_id + '/content?variant=thumbnail',
|
||||
actor=alice,
|
||||
)
|
||||
check(jpeg_size(content['bytes']) == (256, 160),
|
||||
'thumbnail is not 256x160 JPEG')
|
||||
check(len(content['bytes']) == thumbnail['size_bytes'],
|
||||
'thumbnail size differs from metadata')
|
||||
check(sha256(content['bytes']) == thumbnail['sha256'],
|
||||
'thumbnail checksum differs from metadata')
|
||||
history = client.call(
|
||||
'GET',
|
||||
'/api/v1/chats/' + chat['id'] + '/messages?limit=100',
|
||||
actor=alice,
|
||||
schema='MessagePage',
|
||||
)['json']
|
||||
check(
|
||||
any(attachment_id in item['attachment_ids']
|
||||
for item in history['items']),
|
||||
'message history lost the attachment',
|
||||
)
|
||||
print('ready attachment=' + attachment_id +
|
||||
' thumbnail=256x160 sha256=' + thumbnail['sha256'][:16] + '...')
|
||||
|
||||
print_step(11, 'compose ps')
|
||||
print(compose('ps', '--format',
|
||||
'table {{.Service}}\t{{.Status}}').stdout.strip())
|
||||
print('PASS: worker-down durability demo')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
# Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`.
|
||||
# Semantic and evidence notes: tests/README.md ("Проверка сохранности") and REPORT.md.
|
||||
set -euo pipefail
|
||||
|
||||
compose=(docker compose --env-file .env)
|
||||
"${compose[@]}" restart postgres redis s3 broker api worker haproxy
|
||||
"${compose[@]}" up -d --no-recreate --no-deps --wait postgres redis s3 broker api worker haproxy
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
# Восстанавливает недостающую API-реплику до объявленного числа реплик.
|
||||
set -eu
|
||||
|
||||
env_file="${COMPOSE_ENV_FILE:-.env}"
|
||||
replicas="${API_REPLICAS:-}"
|
||||
if [ -z "$replicas" ] && [ -f "$env_file" ]; then
|
||||
replicas=$(sed -n 's/^API_REPLICAS=//p' "$env_file" | tail -n 1)
|
||||
fi
|
||||
replicas="${replicas:-2}"
|
||||
exec docker compose --env-file "$env_file" up -d --no-deps --scale "api=${replicas}" api
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/sh
|
||||
# Останавливает ровно одну API-реплику стенда — ту, которую обслужила текущая
|
||||
# сессия. TARGET_INSTANCE_ID приходит из теста (значение X-Instance-Id ответа),
|
||||
# а в Docker hostname контейнера равен его короткому ID, поэтому сопоставление
|
||||
# однозначно. Ничего, кроме этой реплики, скрипт не трогает: ни LB, ни БД, ни
|
||||
# тома. Возврат управления — быстрый (SIGTERM -> дрейн DRAIN_DELAY_MS=3s ->
|
||||
# выход), останавливать реплику "навсегда" не требуется.
|
||||
set -eu
|
||||
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
project="${LAB_PROJECT:-mtusi-lab6}"
|
||||
|
||||
target=''
|
||||
for container in $(
|
||||
docker ps \
|
||||
--filter "label=com.docker.compose.project=${project}" \
|
||||
--filter 'label=com.docker.compose.service=api' \
|
||||
--format '{{.ID}}'
|
||||
); do
|
||||
case "$container" in
|
||||
"$TARGET_INSTANCE_ID"*)
|
||||
target="$container"
|
||||
break
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "$target" ]; then
|
||||
printf 'stop-one: no running api replica of project %s whose id starts with %s\n' \
|
||||
"$project" "$TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf 'stop-one: stopping api replica %s\n' "$target" >&2
|
||||
docker stop --time 10 "$target" >/dev/null
|
||||
Reference in New Issue
Block a user