lab6: базовый коммит из lab5 (0790599)

Перенесено дерево lab5 целиком: NestJS-приложение (API + worker,
RabbitMQ, outbox, S3), Dockerfile, compose.yaml, infra/{haproxy,minio,
rabbitmq}, стендовые скрипты, scripts/lab4, scripts/lab5,
evidence/lab4, evidence/lab5 и contracts/extensions.openapi.json.

Задание lab6 оставлено своим: README, REPORT, COURSE, lab.json,
contracts/{openapi,infrastructure,jobs.schema,attachment-states,
examples,observability}, tests, compose.example.yaml, infra/otel,
.github/pull_request_template.md.

Адаптации при переносе:
- docker-проект mtusi-lab5 -> mtusi-lab6 и тег образа
  mtusi-messenger:lab6 (compose.yaml, scripts/stop-one.sh, .env.example,
  infra/README.md), чтобы тома и контейнеры не пересекались с lab5;
- Makefile: цели lab5 сохранены целиком (skeleton lab6 не отличается
  от skeleton lab5);
- .env.example: переменные приложения и стенда lab5 + блок lab6
  (OTEL_*) из skeleton.
This commit is contained in:
2026-11-09 16:44:21 +03:00
parent 470d9d6f1e
commit 6889e77ab4
203 changed files with 27707 additions and 13 deletions
+22
View File
@@ -0,0 +1,22 @@
#!/bin/sh
# Black-box набор приложения (app/test/*.e2e-spec.ts) против поднятого стенда.
# Требует loopback-портов PostgreSQL, Redis и MinIO из compose.yaml.
set -eu
env_file="${COMPOSE_ENV_FILE:-.env}"
compose_dir="$(pwd)"
# shellcheck disable=SC1090
set -a
. "./$env_file"
set +a
export API_BASE_URL="${API_BASE_URL:-http://127.0.0.1:${APP_PUBLIC_PORT:-8080}}"
export DATABASE_URL="postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in $env_file}@127.0.0.1:${POSTGRES_PUBLISHED_PORT:-5432}/${POSTGRES_DB:-messenger}"
export REDIS_URL="redis://127.0.0.1:${REDIS_PUBLISHED_PORT:-6379}"
export E2E_S3_PUBLIC_ENDPOINT="${S3_PUBLIC_ENDPOINT:-http://localhost:9000}"
export E2E_S3_BUCKET="${S3_BUCKET:-messenger-attachments}"
export E2E_COMPOSE_DIR="$compose_dir"
export E2E_COMPOSE_ENV_FILE="$env_file"
cd "$(dirname "$0")/../app"
exec npm run test:e2e -- "$@"
+76
View File
@@ -0,0 +1,76 @@
#!/bin/sh
# Идемпотентно создаёт .env и secrets/* для локального стенда. Значения не печатает.
set -eu
cd "$(dirname "$0")/.."
if [ ! -f .env ]; then
(
umask 077
pg=$(openssl rand -hex 24)
mt=$(openssl rand -hex 16)
sed \
-e "s|^POSTGRES_PASSWORD=.*|POSTGRES_PASSWORD=${pg}|" \
-e "s|^DATABASE_URL=.*|DATABASE_URL=postgresql://messenger:${pg}@postgres:5432/messenger|" \
-e "s|^METRICS_TOKEN=.*|METRICS_TOKEN=${mt}|" \
.env.example > .env
)
echo "created .env"
else
echo "kept .env"
fi
mkdir -p secrets
chmod 700 secrets
write_secret() {
name=$1
value=$2
if [ -s "secrets/$name" ]; then
echo "kept secrets/$name"
return
fi
(umask 133; printf '%s' "$value" > "secrets/$name")
chmod 0644 "secrets/$name"
echo "created secrets/$name"
}
write_secret minio_root_user "root-$(openssl rand -hex 6)"
write_secret minio_root_password "$(openssl rand -hex 24)"
write_secret s3_api_access_key_id "api-$(openssl rand -hex 8)"
write_secret s3_api_secret_access_key "$(openssl rand -hex 24)"
write_secret s3_worker_access_key_id "worker-$(openssl rand -hex 8)"
write_secret s3_worker_secret_access_key "$(openssl rand -hex 24)"
if [ -s secrets/broker_worker_password ] && [ -s secrets/broker_worker_url ]; then
echo "kept secrets/broker_worker_password secrets/broker_worker_url"
else
wp=$(openssl rand -hex 24)
(umask 133; printf '%s' "$wp" > secrets/broker_worker_password)
chmod 0644 secrets/broker_worker_password
(umask 133; printf '%s' "amqp://messenger-worker:${wp}@broker:5672/messenger" > secrets/broker_worker_url)
chmod 0644 secrets/broker_worker_url
echo "created secrets/broker_worker_password secrets/broker_worker_url"
fi
# Broker password lives in one place and is rendered into both files, so the
# URL used by the app and the RabbitMQ config cannot drift apart.
if [ -s secrets/broker_url ] && [ -s secrets/rabbitmq.conf ]; then
echo "kept secrets/broker_url secrets/rabbitmq.conf"
else
bp=$(openssl rand -hex 24)
(umask 133; printf '%s' "amqp://messenger:${bp}@broker:5672/messenger" > secrets/broker_url)
chmod 0644 secrets/broker_url
(
umask 133
{
printf 'default_vhost = messenger\n'
printf 'default_user = messenger\n'
printf 'default_pass = %s\n' "$bp"
printf 'default_permissions.configure = ^(attachments\\.jobs|attachments\\.process|attachments\\.dlx|attachments\\.process\\.dlq)$\n'
printf 'default_permissions.read = ^attachments\\.(jobs|process|dlx|process\\.dlq)$\n'
printf 'default_permissions.write = ^attachments\\.(jobs|process|dlx|process\\.dlq)$\n'
} > secrets/rabbitmq.conf
)
chmod 0644 secrets/rabbitmq.conf
echo "created secrets/broker_url secrets/rabbitmq.conf"
fi
+340
View File
@@ -0,0 +1,340 @@
#!/usr/bin/env python3
"""Upload benchmark: API traffic and peak RSS per upload for proxy vs direct mode.
attachments_bench.py run --mode proxy --size-bytes 1048576 --concurrency 10 \
--uploads-per-worker 5 --repeat 3 --output evidence/lab4/bench/proxy-1048576-c10.json
attachments_bench.py summarize evidence/lab4/bench/*.json > evidence/lab4/summary.md
Run from the repository root with the stand up. Tokens, signed URLs and secrets
are never written to the output.
"""
import argparse
import base64
import hashlib
import http.client
import json
import os
import statistics
import subprocess
import sys
import threading
import time
import urllib.parse
import uuid
from datetime import datetime, timezone
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from pngkit import noise_png # noqa: E402
def sh(*args, check=True):
return subprocess.run(args, check=check, capture_output=True, text=True).stdout.strip()
def read_env(path=".env"):
values = {}
with open(path) as handle:
for line in handle:
line = line.strip()
if line and not line.startswith("#") and "=" in line:
key, _, value = line.partition("=")
values[key] = value
return values
def request(host, port, method, path, headers=None, body=None):
connection = http.client.HTTPConnection(host, port, timeout=120)
try:
connection.request(method, path, body=body, headers=headers or {})
response = connection.getresponse()
payload = response.read()
return response.status, payload
finally:
connection.close()
def api_json(env, method, path, token=None, body=None, basic=None):
headers = {}
data = None
if body is not None:
headers["content-type"] = "application/json"
data = json.dumps(body).encode()
if token:
headers["authorization"] = f"Bearer {token}"
if basic:
headers["authorization"] = "Basic " + base64.b64encode(basic.encode()).decode()
port = int(env.get("APP_PUBLIC_PORT", "8080"))
status, payload = request("127.0.0.1", port, method, path, headers, data)
return status, json.loads(payload) if payload else None
def containers():
ids = sh("docker", "compose", "--env-file", ".env", "ps", "-q", "api").split()
return sorted(ids)
NODE_METRICS = (
"fetch('http://127.0.0.1:'+(process.env.APP_PORT||8080)+'/internal/metrics'+process.argv[1],"
"{method:process.argv[1]?'POST':'GET',headers:{authorization:'Bearer '+process.env.METRICS_TOKEN}})"
".then(r=>r.text()).then(console.log)"
)
def metrics(container):
return json.loads(sh("docker", "exec", container, "node", "-e", NODE_METRICS, ""))
def reset_metrics(container):
sh("docker", "exec", container, "node", "-e", NODE_METRICS, "/reset")
def net_bytes(container):
for line in sh("docker", "exec", container, "cat", "/proc/net/dev").splitlines():
if line.strip().startswith("eth0:"):
fields = line.split(":", 1)[1].split()
return int(fields[0]), int(fields[8])
raise RuntimeError("eth0 not found")
def percentile(values, fraction):
ordered = sorted(values)
if not ordered:
return 0.0
return ordered[min(len(ordered) - 1, max(0, int(round(fraction * len(ordered) + 0.5)) - 1))]
def one_upload(env, mode, token, chat_id, data, seed):
"""Returns (ok, elapsed_ms, outcome); outcome names the failing step and status, e.g. "complete:429"."""
digest = hashlib.sha256(data).hexdigest()
filename = f"bench-{seed}.png"
port = int(env.get("APP_PUBLIC_PORT", "8080"))
started = time.perf_counter()
outcome, attachment = "ok", None
try:
if mode == "proxy":
status, body = request(
"127.0.0.1", port, "POST",
f"/api/v1/chats/{chat_id}/attachments?filename={urllib.parse.quote(filename)}",
{"authorization": f"Bearer {token}", "content-type": "image/png"}, data,
)
if status == 201:
attachment = json.loads(body)
else:
outcome = f"upload:{status}"
else:
headers = {"authorization": f"Bearer {token}", "content-type": "application/json"}
status, body = request(
"127.0.0.1", port, "POST", f"/api/v1/chats/{chat_id}/attachments/direct-uploads", headers,
json.dumps({"filename": filename, "content_type": "image/png",
"size_bytes": len(data), "sha256": digest}).encode(),
)
if status != 201:
outcome = f"init:{status}"
else:
ticket = json.loads(body)
target = urllib.parse.urlsplit(ticket["url"])
put_status, _ = request(
target.hostname, target.port, "PUT", target.path + "?" + target.query,
{**ticket["headers"]}, data,
)
if put_status != 200:
outcome = f"put:{put_status}"
else:
status, body = request(
"127.0.0.1", port, "POST", f"/api/v1/attachments/{ticket['attachment_id']}/complete",
{"authorization": f"Bearer {token}"},
)
if status == 200:
attachment = json.loads(body)
else:
outcome = f"complete:{status}"
except (OSError, http.client.HTTPException, ValueError) as error:
outcome = f"error:{type(error).__name__}"
elapsed_ms = (time.perf_counter() - started) * 1000
if outcome == "ok" and (attachment is None or attachment["sha256"] != digest):
outcome = "checksum_mismatch"
return outcome == "ok", elapsed_ms, outcome
def payload(args, seed):
if args.image:
with open(args.image, "rb") as handle:
return handle.read()
return noise_png(args.size_bytes, seed)
def run_once(env, args, actors, ids):
for container in ids:
reset_metrics(container)
time.sleep(0.2)
baseline = {c: metrics(c) for c in ids}
net_before = {c: net_bytes(c) for c in ids}
latencies, failures = [], []
outcomes = {}
lock = threading.Lock()
def worker(index):
for step in range(args.uploads_per_worker):
seed = index * 1000 + step
data = payload(args, seed)
token, chat_id = actors[index % len(actors)]
ok, elapsed, outcome = one_upload(env, args.mode, token, chat_id, data, seed)
with lock:
(latencies if ok else failures).append(elapsed)
outcomes[outcome] = outcomes.get(outcome, 0) + 1
started = time.perf_counter()
threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)]
for thread in threads:
thread.start()
for thread in threads:
thread.join()
duration = time.perf_counter() - started
time.sleep(0.2)
after = {c: metrics(c) for c in ids}
net_after = {c: net_bytes(c) for c in ids}
replicas = []
for c in ids:
base_rss = baseline[c]["memory_bytes"]["rss"]
peak = after[c]["memory_peak_bytes"]
replicas.append({
"instance_id": after[c]["instance_id"],
"delta_rx_bytes": net_after[c][0] - net_before[c][0],
"delta_tx_bytes": net_after[c][1] - net_before[c][1],
"baseline_rss_bytes": base_rss,
"peak_memory_bytes": peak,
"delta_rss_bytes": peak["rss"] - base_rss,
"counters": {k: v for k, v in after[c]["counters"].items() if k.startswith("attachments.")},
})
total = args.concurrency * args.uploads_per_worker
return {
"uploads": total,
"succeeded": len(latencies),
"failed": len(failures),
"outcomes": outcomes,
"duration_s": round(duration, 3),
"latency_ms": {
"p50": round(percentile(latencies, 0.50), 1),
"p95": round(percentile(latencies, 0.95), 1),
"p99": round(percentile(latencies, 0.99), 1),
},
"replicas": replicas,
"total_delta_rx_bytes": sum(r["delta_rx_bytes"] for r in replicas),
"total_delta_tx_bytes": sum(r["delta_tx_bytes"] for r in replicas),
"rx_bytes_per_upload": round(sum(r["delta_rx_bytes"] for r in replicas) / total),
"tx_bytes_per_upload": round(sum(r["delta_tx_bytes"] for r in replicas) / total),
"max_delta_rss_bytes": max(r["delta_rss_bytes"] for r in replicas),
}
def command_run(args):
if args.image:
args.size_bytes = os.path.getsize(args.image)
if args.size_bytes <= 0:
sys.exit("--size-bytes or --image is required")
env = read_env()
ids = containers()
if not ids:
sys.exit("no api containers found")
# One user per worker: the API caps concurrent uploads per user
# (UPLOAD_MAX_IN_FLIGHT_PER_USER), and real traffic comes from many users.
actors = []
for _ in range(args.concurrency):
name = f"bench_{uuid.uuid4().hex[:10]}"
password = "bench:" + uuid.uuid4().hex
status, user = api_json(env, "POST", "/api/v1/users",
body={"username": name, "display_name": "bench", "password": password})
assert status == 201, status
status, session = api_json(env, "POST", "/api/v1/auth/sessions", basic=f"{name}:{password}")
assert status == 201, status
token = session["token"]
status, chat = api_json(env, "POST", "/api/v1/chats", token, {"title": "bench", "member_ids": [user["id"]]})
assert status == 201, status
actors.append((token, chat["id"]))
# Warm-up: two uploads per replica, not measured.
for seed in range(len(ids) * 2):
token, chat_id = actors[seed % len(actors)]
one_upload(env, args.mode, token, chat_id, payload(args, 900000 + seed), seed)
limits = []
for c in ids:
limits.append(sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}", c))
repeats = [run_once(env, args, actors, ids) for _ in range(args.repeat)]
mem_total = next(int(l.split()[1]) * 1024 for l in open("/proc/meminfo") if l.startswith("MemTotal"))
result = {
"date": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"git": sh("git", "rev-parse", "--short", "HEAD", check=False),
"params": {
"mode": args.mode, "size_bytes": args.size_bytes, "concurrency": args.concurrency,
"uploads_per_worker": args.uploads_per_worker, "repeat": args.repeat,
"users": args.concurrency, "image": os.path.basename(args.image) if args.image else None,
},
"host": {"nproc": os.cpu_count(), "mem_total_bytes": mem_total},
"api_container_limits_nanocpus_memory": limits,
"repeats": repeats,
}
os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True)
with open(args.output, "w") as handle:
json.dump(result, handle, indent=2)
handle.write("\n")
for index, item in enumerate(repeats):
print(f"repeat {index + 1}: ok={item['succeeded']}/{item['uploads']} "
f"rx/upload={item['rx_bytes_per_upload']} tx/upload={item['tx_bytes_per_upload']} "
f"maxΔrss={item['max_delta_rss_bytes']} p50={item['latency_ms']['p50']}ms "
f"outcomes={item['outcomes']}")
def mib(value):
return f"{value / 1048576:.2f}"
def command_summarize(args):
rows = []
for path in args.paths:
with open(path) as handle:
data = json.load(handle)
p = data["params"]
reps = data["repeats"]
med = lambda key: statistics.median(r[key] for r in reps) # noqa: E731
label = p["mode"] + (f" ({p['image']})" if p.get("image") else "")
rows.append((label, p["size_bytes"], p["concurrency"], reps, med, data))
rows.sort(key=lambda r: (r[0].count("("), r[1], r[0], r[2]))
print("| mode | size, MiB | C | ok | failures | API rx/upload, MiB | API tx/upload, MiB | max Δrss, MiB | peak RSS, MiB | p50, ms | p95, ms |")
print("|---|---|---|---|---|---|---|---|---|---|---|")
for mode, size, conc, reps, med, _data in rows:
peak = max(x["peak_memory_bytes"]["rss"] for r in reps for x in r["replicas"])
failed = {}
for rep in reps:
for name, count in rep.get("outcomes", {}).items():
if name != "ok":
failed[name] = failed.get(name, 0) + count
failures = ", ".join(f"{name}×{count}" for name, count in sorted(failed.items())) or "—"
ok = sum(r["succeeded"] for r in reps)
total = sum(r["uploads"] for r in reps)
p50 = statistics.median(r["latency_ms"]["p50"] for r in reps)
p95 = statistics.median(r["latency_ms"]["p95"] for r in reps)
print(f"| {mode} | {mib(size)} | {conc} | {ok}/{total} | {failures} | {mib(med('rx_bytes_per_upload'))} | "
f"{mib(med('tx_bytes_per_upload'))} | {mib(med('max_delta_rss_bytes'))} | {mib(peak)} | {p50:.0f} | {p95:.0f} |")
def main():
parser = argparse.ArgumentParser()
sub = parser.add_subparsers(dest="command", required=True)
run = sub.add_parser("run")
run.add_argument("--mode", choices=["proxy", "direct"], required=True)
run.add_argument("--size-bytes", type=int, default=0)
run.add_argument("--image", help="upload this file instead of generated noise PNGs")
run.add_argument("--concurrency", type=int, default=10)
run.add_argument("--uploads-per-worker", type=int, default=5)
run.add_argument("--repeat", type=int, default=3)
run.add_argument("--output", required=True)
summarize = sub.add_parser("summarize")
summarize.add_argument("paths", nargs="+")
args = parser.parse_args()
{"run": command_run, "summarize": command_summarize}[args.command](args)
if __name__ == "__main__":
main()
+54
View File
@@ -0,0 +1,54 @@
#!/bin/sh
# Проверка приватности S3 вне API: анонимный доступ, права отдельных credentials
# API, закрытая консоль. Секреты не печатаются.
set -u
cd "$(dirname "$0")/../.."
env_file="${COMPOSE_ENV_FILE:-.env}"
# shellcheck disable=SC1090
set -a; . "./$env_file"; set +a
endpoint="${S3_PUBLIC_ENDPOINT:-http://localhost:9000}"
bucket="${S3_BUCKET:-messenger-attachments}"
compose="docker compose --env-file $env_file"
fail=0
report() { # status name
if [ "$1" = ok ]; then echo "PASS $2"; else echo "FAIL $2"; fail=1; fi
}
key=$($compose exec -T postgres psql -U "${POSTGRES_USER:-messenger}" -d "${POSTGRES_DB:-messenger}" -At \
-c "SELECT object_key FROM attachments WHERE status = 'ready' ORDER BY created_at DESC LIMIT 1")
if [ -z "$key" ]; then
echo "FAIL no ready attachment to check; upload one first (make test)"
exit 1
fi
code=$(curl -s -o /dev/null -w '%{http_code}' "$endpoint/$bucket/$key")
[ "$code" = 403 ] && report ok "anonymous GET of a ready object -> 403" || report fail "anonymous GET of a ready object -> $code (want 403)"
code=$(curl -s -o /dev/null -w '%{http_code}' "$endpoint/$bucket/")
[ "$code" = 403 ] && report ok "anonymous list of the bucket -> 403" || report fail "anonymous list of the bucket -> $code (want 403)"
code=$(curl -s -o /dev/null -w '%{http_code}' -X PUT --data-binary x "$endpoint/$bucket/attachments/anonymous-write")
[ "$code" = 403 ] && report ok "anonymous PUT -> 403" || report fail "anonymous PUT -> $code (want 403)"
# The API credentials only, inside a one-off container (secrets come from files).
api_mc() {
$compose run --rm --no-deps -T --entrypoint /bin/sh s3-init -c '
IFS= read -r k < /run/secrets/s3_api_access_key_id
IFS= read -r s < /run/secrets/s3_api_secret_access_key
export MC_CONFIG_DIR=/tmp/.mc MC_HOST_api="http://$k:$s@s3:9000"
exec mc "$@"' sh "$@" >/dev/null 2>&1
}
api_mc mb api/forbidden-bucket && report fail "API credentials can create buckets" || report ok "API credentials cannot create buckets"
api_mc admin info api && report fail "API credentials can use admin API" || report ok "API credentials cannot use admin API"
api_mc stat "api/$bucket/$key" && report ok "API credentials can read attachments/*" || report fail "API credentials cannot read attachments/*"
api_mc stat "api/$bucket/outside-prefix" ; rc=$?
api_mc cp /etc/hostname "api/$bucket/outside-prefix" && report fail "API credentials can write outside attachments/" || report ok "API credentials cannot write outside attachments/"
: "$rc"
port=$($compose port s3 9001 2>/dev/null || true)
[ -z "$port" ] && report ok "console port 9001 is not published" || report fail "console port 9001 is published ($port)"
if curl -s -o /dev/null --max-time 3 http://localhost:9001; then report fail "console reachable on localhost:9001"; else report ok "console not reachable on localhost:9001"; fi
exit "$fail"
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
# Сверка bucket и БД (только чтение): orphans — объект без строки, missing —
# ready-строка без объекта. Код выхода 1, если что-то найдено.
set -eu
cd "$(dirname "$0")/../.."
env_file="${COMPOSE_ENV_FILE:-.env}"
# shellcheck disable=SC1090
set -a; . "./$env_file"; set +a
bucket="${S3_BUCKET:-messenger-attachments}"
compose="docker compose --env-file $env_file"
objects=$(mktemp)
rows=$(mktemp)
trap 'rm -f "$objects" "$rows"' EXIT
$compose run --rm --no-deps -T --entrypoint /bin/sh s3-init -c '
IFS= read -r u < /run/secrets/minio_root_user
IFS= read -r p < /run/secrets/minio_root_password
export MC_CONFIG_DIR=/tmp/.mc MC_HOST_stand="http://$u:$p@s3:9000"
exec mc ls --recursive "stand/$S3_BUCKET"' 2>/dev/null \
| sed -n 's/.* \(attachments\/.*\)$/\1/p' | sort > "$objects"
$compose exec -T postgres psql -U "${POSTGRES_USER:-messenger}" -d "${POSTGRES_DB:-messenger}" -At \
-c "SELECT object_key, status FROM attachments" | sort > "$rows"
echo "bucket: $bucket"
echo "objects: $(wc -l < "$objects" | tr -d ' ')"
echo "rows by status:"
cut -d'|' -f2 "$rows" | sort | uniq -c | sed 's/^ */ /'
orphans=$(cut -d'|' -f1 "$rows" | sort | comm -13 - "$objects")
missing=$(grep '|ready$' "$rows" | cut -d'|' -f1 | sort | comm -23 - "$objects")
echo "orphans: $(printf '%s' "$orphans" | grep -c . || true)"
[ -z "$orphans" ] || printf '%s\n' "$orphans" | sed 's/^/ orphan /'
echo "missing: $(printf '%s' "$missing" | grep -c . || true)"
[ -z "$missing" ] || printf '%s\n' "$missing" | sed 's/^/ missing /'
[ -z "$orphans$missing" ]
+33
View File
@@ -0,0 +1,33 @@
// Peak RSS of one full decode check with the same sharp pipeline as
// app/src/attachments/image/image-decoder.ts, on worst-case inputs near
// MAX_IMAGE_PIXELS. Runs where the API runs (same libvips, VIPS_DISC_THRESHOLD):
// docker compose exec -T api node - < scripts/lab4/decode_memory.mjs # reads files from IMAGES (paths in the container)
// or on the host from app/: IMAGES="a.png b.jpg" node ../scripts/lab4/decode_memory.mjs
import { readFileSync } from 'node:fs';
import { createRequire } from 'node:module';
const sharp = createRequire(`${process.cwd()}/`)('sharp');
sharp.cache(false);
sharp.concurrency(1);
for (const file of (process.env.IMAGES ?? '').split(/\s+/u).filter(Boolean)) {
const data = readFileSync(file);
const meta = await sharp(data).metadata();
const base = process.memoryUsage().rss;
let peak = base;
const timer = setInterval(() => {
peak = Math.max(peak, process.memoryUsage().rss);
}, 5);
const started = Date.now();
let verdict = 'ok';
try {
await sharp(data, { failOn: 'warning', sequentialRead: true, limitInputPixels: 40_000_000 })
.resize(64, 64, { fit: 'inside' })
.toBuffer();
} catch (error) {
verdict = `rejected: ${String(error.message).split('\n')[0]}`;
}
clearInterval(timer);
console.log(
`${meta.format} ${meta.width}x${meta.height} ${meta.channels}ch ${meta.depth}${meta.isProgressive ? ' progressive' : ''}${meta.isPalette ? ' palette' : ''} bytes=${data.length} decode_ms=${Date.now() - started} peak_delta_rss_mib=${((peak - base) / 1048576).toFixed(1)} ${verdict}`,
);
}
+78
View File
@@ -0,0 +1,78 @@
"""Synthetic PNG helpers (stdlib only); same semantics as app/test/support/png.ts."""
import random
import struct
import zlib
SIGNATURE = b"\x89PNG\r\n\x1a\n"
NOISE_WIDTH = 1024
NOISE_ROW = 1 + NOISE_WIDTH * 3
def chunk(kind: bytes, data: bytes) -> bytes:
crc = zlib.crc32(kind + data) & 0xFFFFFFFF
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", crc)
def build_png(width: int, height: int, idat: bytes) -> bytes:
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return SIGNATURE + chunk(b"IHDR", ihdr) + chunk(b"IDAT", idat) + chunk(b"IEND", b"")
def pad_png(data: bytes, size: int) -> bytes:
"""Insert a private ancillary chunk before IEND to reach exactly `size` bytes."""
extra = size - len(data)
if extra == 0:
return data
if extra < 12:
raise ValueError("padding needs at least 12 bytes")
return data[:-12] + chunk(b"prVt", bytes(extra - 12)) + data[-12:]
def _noise_of_height(height: int, rng: random.Random) -> bytes:
raw = bytearray(rng.randbytes(NOISE_ROW * height))
for row in range(height):
raw[row * NOISE_ROW] = 0 # filter type "none"
compressor = zlib.compressobj(0)
return build_png(NOISE_WIDTH, height, compressor.compress(bytes(raw)) + compressor.flush())
def noise_png(size: int, seed: int = 0) -> bytes:
"""Incompressible, structurally valid PNG of exactly `size` bytes."""
rng = random.Random(seed)
height = max(1, size // NOISE_ROW)
while True:
png = _noise_of_height(height, rng)
if len(png) > size or 0 < size - len(png) < 12:
height -= 1
if height < 1:
raise ValueError("size is too small")
continue
return pad_png(png, size)
ADAM7 = [(0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)]
def worst_decode_png(size: int = 10_485_760, width: int = 8000, height: int = 5000) -> bytes:
"""The heaviest valid upload to decode: 40 MP RGBA 16-bit, Adam7-interlaced
(the decoder must hold the whole image), padded to exactly `size` bytes."""
compressor = zlib.compressobj(1)
parts = []
for x0, y0, dx, dy in ADAM7:
pass_width = (width - x0 + dx - 1) // dx if width > x0 else 0
pass_height = (height - y0 + dy - 1) // dy if height > y0 else 0
if pass_width == 0 or pass_height == 0:
continue
row = bytes(1 + pass_width * 8)
for _ in range(pass_height):
parts.append(compressor.compress(row))
parts.append(compressor.flush())
ihdr = struct.pack(">IIBBBBB", width, height, 16, 6, 0, 0, 1)
png = SIGNATURE + chunk(b"IHDR", ihdr) + chunk(b"IDAT", b"".join(parts)) + chunk(b"IEND", b"")
return pad_png(png, size)
if __name__ == "__main__":
import sys
sys.stdout.buffer.write(worst_decode_png())
+36
View File
@@ -0,0 +1,36 @@
#!/bin/sh
# Experiment matrix: sizes x {proxy,direct} at C=10, plus proxy 9.5 MiB at C=20.
# Needs a running stand (make stand-up) and `make init-env` credentials.
set -eu
cd "$(dirname "$0")/../.."
out=evidence/lab4/bench
mkdir -p "$out"
bench() {
mode=$1 size=$2 conc=$3
python3 scripts/lab4/attachments_bench.py run --mode "$mode" --size-bytes "$size" \
--concurrency "$conc" --uploads-per-worker 5 --repeat 3 \
--output "$out/$mode-$size-c$conc.json"
}
for size in 131072 1048576 9961472; do
for mode in proxy direct; do
echo "== $mode size=$size c=10"
bench "$mode" "$size" 10
done
done
echo "== proxy size=9961472 c=20"
bench proxy 9961472 20
# Worst case for memory: every request carries the heaviest valid image
# (40 MP RGBA16 Adam7 PNG padded to 10 MiB, ~340 MiB per decode) and C=32
# saturates UPLOAD_MAX_IN_FLIGHT (16) on both replicas.
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
worst="$tmp/worst-40mp-rgba16-adam7.png"
python3 scripts/lab4/pngkit.py > "$worst"
for mode in proxy direct; do
echo "== $mode worst-case image c=32"
python3 scripts/lab4/attachments_bench.py run --mode "$mode" --image "$worst" \
--concurrency 32 --uploads-per-worker 2 --repeat 3 \
--output "$out/$mode-worst-c32.json"
done
python3 scripts/lab4/attachments_bench.py summarize "$out"/*.json > evidence/lab4/summary.md
cat evidence/lab4/summary.md
+749
View File
@@ -0,0 +1,749 @@
#!/usr/bin/env python3
"""Automated fault injection for lab5 levels 4 and 5 (track 2)."""
import argparse
import json
import re
import sys
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'lab4'))
from stand import ( # noqa: E402
ROOT,
Failure,
check,
clear_fault,
clear_faults,
cli,
compose,
docker,
new_client,
rabbit_queues,
sample_bytes,
set_fault,
sha256,
upload,
variant_sha,
wait_for,
wait_ready,
worker_env,
worker_start_count,
)
from pngkit import worst_decode_png # noqa: E402
HINT = ('FAULT_INJECTION_ENABLED=1 JOB_TIMEOUT_SECONDS=10 '
'JOB_LEASE_SECONDS=8 JOB_RETRY_BASE_MS=500 JOB_RETRY_MAX_MS=4000 '
'make stand-up')
LEVEL4 = [
'crash_after_commit_before_publish',
'crash_after_side_effect',
'crash_after_commit_before_ack',
'retry_and_replay',
'hung_worker',
'poison',
'big_image',
]
TRACK2 = [
'duplicate',
'reorder',
'crash_before_side_effect',
'crash_after_side_effect',
'crash_after_commit_before_ack',
'relay_crash_after_publish',
'stale_version',
]
class Tracker(threading.Thread):
"""Polls metadata while a scenario runs to prove readiness never regresses."""
def __init__(self, base_url, actor, ca_file=None):
super().__init__(daemon=True)
self.client = new_client(base_url, ca_file)
self.actor = actor
self.ids = []
self.records = {}
self.stop_event = threading.Event()
def track(self, attachment_id):
if attachment_id not in self.records:
self.ids.append(attachment_id)
self.records[attachment_id] = []
def run(self):
while not self.stop_event.is_set():
for attachment_id in list(self.ids):
try:
meta = self.client.call(
'GET', '/api/v1/attachments/' + attachment_id,
actor=self.actor, schema='Attachment',
)['json']
self.records[attachment_id].append(
(time.monotonic(), meta['status'],
meta.get('ready_pipeline_version'))
)
except Exception:
pass
self.stop_event.wait(0.2)
def stop(self):
self.stop_event.set()
self.join(timeout=5)
def build_message(status, job, attempt=None):
created = next(
event for event in status['events']
if event['job_id'] == job['job_id'] and event['event'] == 'created'
)
return {
'schema_version': 1,
'job_id': job['job_id'],
'type': 'attachment.process.v1',
'attachment_id': status['attachment']['id'],
'pipeline_version': job['pipeline_version'],
'attempt': attempt if attempt is not None else job['attempt'],
'created_at': created['created_at'],
}
def publish_raw(messages):
text = ''.join(json.dumps(message, separators=(',', ':')) + '\n'
for message in messages)
code, payload = cli('publish-raw', stdin=text)
check(code == 0, 'publish-raw failed: ' + str(payload))
return payload
def status_of(attachment_id):
code, payload = cli('status', attachment_id)
check(code == 0, 'status failed: ' + str(payload))
return payload
def flush_outbox(attachment_id):
wait_for(lambda: status_of(attachment_id)['outbox']['pending'] == 0,
30, 'outbox publish', 1.0)
def new_attachment_id(chat_id, before):
rows = cli('find-by-chat', chat_id)[1]
for row in reversed(rows):
if row['id'] not in before:
return row['id']
return None
def evaluate(tracker, attachment_id, baseline, expect_failed=False,
compare_baseline=True):
if attachment_id is None:
return {'I1': True, 'I2': True, 'I3': True,
'I4': True, 'I5': True, 'I6': True}
status = status_of(attachment_id)
attachment = status['attachment']
jobs = status['jobs']
variants = status['variants']
ready_version = attachment['ready_pipeline_version']
records = tracker.records.get(attachment_id, [])
reached = any(record[1] == ('failed' if expect_failed else 'ready')
for record in records)
if not records:
reached = attachment['status'] == ('failed' if expect_failed else 'ready')
i1 = reached or attachment['status'] in ('ready', 'failed')
ready_variants = [v for v in variants
if v['pipeline_version'] == ready_version]
succeeded_ids = {job['job_id'] for job in jobs
if job['status'] == 'succeeded'}
succeeded_events = {}
for event in status['events']:
if event['event'] == 'succeeded':
succeeded_events[event['job_id']] = (
succeeded_events.get(event['job_id'], 0) + 1
)
i2 = (len(ready_variants) == 3
and all(succeeded_events.get(job_id) == 1
for job_id in succeeded_ids))
i3 = True
seen_ready = False
last_version = 0
for _at, state, version in records:
if seen_ready and state != 'ready':
i3 = False
if state == 'ready':
seen_ready = True
if version is not None:
if version < last_version:
i3 = False
last_version = version
i4 = bool(succeeded_ids) or expect_failed or attachment['status'] == 'ready'
keys = {(variant['pipeline_version'], variant['variant'])
for variant in variants}
i5 = len(keys) == len(variants)
if compare_baseline and ready_version == 1 and ready_variants:
observed = {variant['variant']: variant['sha256']
for variant in ready_variants}
i6 = all(observed.get(name) == digest
for name, digest in baseline.items())
else:
i6 = True
return {'I1': i1, 'I2': i2, 'I3': i3, 'I4': i4,
'I5': i5, 'I6': i6}
# --- scenarios -----------------------------------------------------------
def scenario_duplicate(ctx):
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), 'duplicate.png')
ctx['attachment_id'] = attachment['id']
ctx['tracker'].track(attachment['id'])
wait_ready(ctx['client'], ctx['alice'], attachment['id'])
status = status_of(attachment['id'])
before = {variant['variant']: variant['s3']
for variant in status['variants']}
message = build_message(status, status['jobs'][0])
publish_raw([message] * 5)
wait_for(
lambda: len([event for event in status_of(attachment['id'])['events']
if event['event'] == 'duplicate_skipped'
and event['detail'].get('reason') == 'already_succeeded']) == 5,
20, 'five duplicate_skipped events', 1.0,
)
after_status = status_of(attachment['id'])
after = {variant['variant']: variant['s3']
for variant in after_status['variants']}
check(before == after, 'duplicate delivery rewrote variant objects')
return {'attachment_id': attachment['id'],
'job_attempts': [job['attempt'] for job in after_status['jobs']],
'notes': '5 duplicate deliveries were absorbed by the DB fence'}
def scenario_reorder(ctx):
compose('stop', 'worker')
try:
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), 'reorder.png')
ctx['attachment_id'] = attachment['id']
ctx['tracker'].track(attachment['id'])
flush_outbox(attachment['id'])
status = status_of(attachment['id'])
message = build_message(status, status['jobs'][0])
stale = dict(message, attempt=99)
publish_raw([stale, message])
compose('start', 'worker')
wait_ready(ctx['client'], ctx['alice'], attachment['id'])
finally:
compose('start', 'worker')
after_status = status_of(attachment['id'])
skipped = [event for event in after_status['events']
if event['event'] == 'duplicate_skipped']
check(len(skipped) == 2, 'expected exactly two skipped out-of-order messages')
check(len([event for event in after_status['events']
if event['event'] == 'succeeded']) == 1,
'expected exactly one successful effect')
return {'attachment_id': attachment['id'],
'job_attempts': [job['attempt'] for job in after_status['jobs']],
'notes': 'attempt 99 and a duplicate attempt 1 were acked as skips'}
def crash_scenario(ctx, point, filename, expected_attempt,
expect_lease_expired=False, expect_duplicate=False):
compose('stop', 'worker')
try:
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), filename)
ctx['attachment_id'] = attachment['id']
ctx['tracker'].track(attachment['id'])
starts = worker_start_count()
set_fault('fault:' + point + ':' + attachment['id'] + ':1', 'crash')
compose('start', 'worker')
wait_ready(ctx['client'], ctx['alice'], attachment['id'], 90)
finally:
compose('start', 'worker')
status = status_of(attachment['id'])
job = status['jobs'][0]
check(job['attempt'] == expected_attempt,
'expected attempt ' + str(expected_attempt) +
', got ' + str(job['attempt']))
check(worker_start_count() >= starts + 2,
'worker process did not restart after the injected crash')
if expect_lease_expired:
check(any(event['event'] == 'lease_expired'
for event in status['events']),
'missing lease_expired event')
if expect_duplicate:
check(any(event['event'] == 'duplicate_skipped'
and event['detail'].get('reason') == 'already_succeeded'
for event in status['events']),
'missing already_succeeded duplicate after redelivery')
return {'attachment_id': attachment['id'],
'job_attempts': [job['attempt']],
'notes': 'crashed at ' + point}
def scenario_crash_before_side_effect(ctx):
return crash_scenario(ctx, 'worker.before_side_effect',
'crash-before.png', 2, expect_lease_expired=True)
def scenario_crash_after_side_effect(ctx):
return crash_scenario(ctx, 'worker.after_side_effect',
'crash-after.png', 2, expect_lease_expired=True)
def scenario_crash_after_commit_before_ack(ctx):
return crash_scenario(ctx, 'worker.after_commit_before_ack',
'crash-ack.png', 1, expect_duplicate=True)
def scenario_crash_after_commit_before_publish(ctx):
before = {row['id'] for row in cli('find-by-chat', ctx['chat']['id'])[1]}
set_fault('fault:api.after_commit_before_publish', 'crash')
try:
try:
ctx['client'].call(
'POST',
'/api/v1/chats/' + ctx['chat']['id'] +
'/attachments?filename=crash-publish.png',
(202, 502, 503, 504),
actor=ctx['alice'],
raw=sample_bytes(),
headers={'Content-Type': 'image/png'},
)
except Failure:
# A killed replica may close the connection instead of answering.
pass
finally:
clear_fault('fault:api.after_commit_before_publish')
attachment_id = wait_for(
lambda: new_attachment_id(ctx['chat']['id'], before), 30,
'attachment row after API crash', 1.0,
)
ctx['attachment_id'] = attachment_id
ctx['tracker'].track(attachment_id)
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
status = status_of(attachment_id)
check(len([event for event in status['events']
if event['event'] == 'created']) == 1,
'expected exactly one created event')
return {'attachment_id': attachment_id,
'job_attempts': [job['attempt'] for job in status['jobs']],
'notes': 'outbox relay published the committed job after API restart'}
def scenario_relay_crash_after_publish(ctx):
compose('stop', 'worker')
before = {row['id'] for row in cli('find-by-chat', ctx['chat']['id'])[1]}
try:
set_fault('fault:relay.after_publish_before_mark', 'crash')
try:
ctx['client'].call(
'POST',
'/api/v1/chats/' + ctx['chat']['id'] +
'/attachments?filename=relay-crash.png',
(202, 502, 503, 504),
actor=ctx['alice'],
raw=sample_bytes(),
headers={'Content-Type': 'image/png'},
)
except Failure:
# The relay may crash the handling replica before the reply flushes.
pass
clear_fault('fault:relay.after_publish_before_mark')
attachment_id = wait_for(
lambda: new_attachment_id(ctx['chat']['id'], before), 30,
'attachment row after relay crash', 1.0,
)
ctx['attachment_id'] = attachment_id
ctx['tracker'].track(attachment_id)
wait_for(lambda: status_of(attachment_id)['outbox']['pending'] == 0,
60, 'outbox published after relay crash', 1.0)
queue_depth = rabbit_queues().get('attachments.process', {}).get(
'messages', 0)
compose('start', 'worker')
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
finally:
clear_fault('fault:relay.after_publish_before_mark')
compose('start', 'worker')
status = status_of(attachment_id)
check(len([event for event in status['events']
if event['event'] == 'succeeded']) == 1,
'expected exactly one succeeded event')
duplicated = any(event['event'] == 'duplicate_skipped'
for event in status['events'])
note = ('relay republished the same message; duplicate absorbed'
if duplicated else
'relay published once in this run; I1/I2 still hold')
if queue_depth >= 2:
check(duplicated, 'two queued copies but no duplicate_skipped event')
return {'attachment_id': attachment_id,
'job_attempts': [job['attempt'] for job in status['jobs']],
'notes': note}
def scenario_stale_version(ctx):
check(int(worker_env('WORKER_CONCURRENCY') or '1') >= 2,
'stale_version needs WORKER_CONCURRENCY>=2')
compose('stop', 'worker')
try:
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), 'stale.png')
attachment_id = attachment['id']
ctx['attachment_id'] = attachment_id
ctx['tracker'].track(attachment_id)
flush_outbox(attachment_id)
code, payload = cli('reprocess', attachment_id,
'--pipeline-version', '2', '--reason', 'suite')
check(code == 0, 'reprocess v2 failed: ' + str(payload))
set_fault('fault:worker.after_side_effect:' + attachment_id + ':1',
'delay:8000')
compose('start', 'worker')
v2_etags = None
def both_terminal():
nonlocal v2_etags
status = status_of(attachment_id)
jobs = {(job['pipeline_version']): job for job in status['jobs']}
if '2' in jobs or 2 in jobs:
v2_job = jobs.get(2)
if v2_job and v2_job['status'] == 'succeeded' and v2_etags is None:
v2_etags = {variant['variant']: variant['s3']
for variant in status['variants']
if variant['pipeline_version'] == 2}
v1_job = jobs.get(1)
v2_job = jobs.get(2)
return (v1_job and v2_job
and v1_job['status'] == 'superseded'
and v2_job['status'] == 'succeeded')
wait_for(both_terminal, 90, 'v2 ready and v1 superseded', 1.0)
status = status_of(attachment_id)
check(status['attachment']['ready_pipeline_version'] == 2,
'ready pipeline version is not 2')
check(all(variant['pipeline_version'] == 2
for variant in status['variants']),
'variant rows from v1 survived')
after_etags = {variant['variant']: variant['s3']
for variant in status['variants']
if variant['pipeline_version'] == 2}
check(v2_etags == after_etags, 'v1 overwrote v2 variant objects')
served = ctx['client'].call(
'GET',
'/api/v1/attachments/' + attachment_id +
'/content?variant=thumbnail',
actor=ctx['alice'],
)
v2_thumbnail = next(variant for variant in status['variants']
if variant['variant'] == 'thumbnail')
check(sha256(served['bytes']) == v2_thumbnail['sha256'],
'served thumbnail is not the v2 object')
check(sha256(served['bytes']) != ctx['baseline']['thumbnail'],
'served thumbnail equals the v1 baseline')
v1_job = next(job for job in status['jobs']
if job['pipeline_version'] == 1)
publish_raw([build_message(status, v1_job)])
wait_for(
lambda: any(event['event'] == 'duplicate_skipped'
and event['detail'].get('reason') == 'superseded'
for event in status_of(attachment_id)['events']),
20, 'superseded duplicate skip', 1.0,
)
finally:
clear_fault('fault:worker.after_side_effect:' +
(attachment['id'] if 'attachment' in locals() else ''))
compose('start', 'worker')
return {'attachment_id': attachment_id,
'job_attempts': [job['attempt'] for job in status['jobs']],
'notes': 'v1 finished after v2 and was recorded as superseded'}
def scenario_poison(ctx):
before = rabbit_queues().get('attachments.process.dlq', {}).get(
'messages', 0)
publish_raw(['not-json', '{"schema_version":1}'])
wait_for(
lambda: rabbit_queues().get('attachments.process.dlq', {}).get(
'messages', 0) >= before + 2,
15, 'two poison messages in DLQ', 0.5,
)
return {'attachment_id': None, 'job_attempts': [],
'notes': 'malformed messages moved to the DLQ without touching jobs'}
def scenario_retry_and_replay(ctx):
max_attempts = int(worker_env('JOB_MAX_ATTEMPTS') or '5')
base = int(worker_env('JOB_RETRY_BASE_MS') or '500')
cap = int(worker_env('JOB_RETRY_MAX_MS') or '4000')
compose('stop', 'worker')
try:
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), 'retry.png')
attachment_id = attachment['id']
ctx['attachment_id'] = attachment_id
ctx['tracker'].track(attachment_id)
set_fault('fault:worker.before_side_effect:' + attachment_id + ':1',
'transient')
compose('start', 'worker')
failed = wait_ready(ctx['client'], ctx['alice'], attachment_id,
90, allow_failed=True)
finally:
compose('start', 'worker')
check(failed['status'] == 'failed', 'transient fault was not exhausted')
check(failed['error_code'] == 'retry_exhausted',
'expected retry_exhausted, got ' + str(failed['error_code']))
status = status_of(attachment_id)
retries = [event for event in status['events']
if event['event'] == 'retry_scheduled']
check(len(retries) == max_attempts - 1,
'expected ' + str(max_attempts - 1) + ' retries, got ' +
str(len(retries)))
for index, event in enumerate(retries):
limit = min(cap, base * 2 ** index)
check(0 <= event['detail'].get('delay_ms', cap + 1) <= limit,
'retry delay exceeds its jitter cap')
ctx['client'].call(
'GET', '/api/v1/attachments/' + attachment_id + '/content',
409, actor=ctx['alice'],
)
code, payload = cli('replay', ctx['reference_id'], '--reason', 'suite')
check(code == 3 and payload.get('reason') in ('no_failed_job',
'already_ready'),
'replay of a ready attachment was not refused')
clear_fault('fault:worker.before_side_effect:' + attachment_id + ':1')
code, payload = cli('replay', attachment_id, '--reason', 'suite')
check(code == 0, 'operator replay failed: ' + str(payload))
wait_ready(ctx['client'], ctx['alice'], attachment_id, 60)
status = status_of(attachment_id)
replayed = [event for event in status['events']
if event['event'] == 'replayed']
check(len(replayed) == 1 and replayed[0]['detail'].get('operator'),
'missing replayed event or operator audit')
return {'attachment_id': attachment_id,
'job_attempts': [job['attempt'] for job in status['jobs']],
'notes': 'transient injected fault exhausted retries; replay recovered'}
def scenario_hung_worker(ctx):
timeout_seconds = int(worker_env('JOB_TIMEOUT_SECONDS') or '10')
compose('stop', 'worker')
try:
attachment = upload(ctx['client'], ctx['alice'], ctx['chat'],
sample_bytes(), 'hung.png')
attachment_id = attachment['id']
ctx['attachment_id'] = attachment_id
ctx['tracker'].track(attachment_id)
set_fault('fault:worker.after_side_effect:' + attachment_id + ':1',
'delay:' + str((timeout_seconds + 5) * 1000))
compose('start', 'worker')
wait_ready(ctx['client'], ctx['alice'], attachment_id, 90)
finally:
compose('start', 'worker')
status = status_of(attachment_id)
check(any(event['event'] == 'retry_scheduled'
and event['detail'].get('code') == 'processing_timeout'
for event in status['events']),
'missing processing_timeout retry')
return {'attachment_id': attachment_id,
'job_attempts': [job['attempt'] for job in status['jobs']],
'notes': 'hard timeout aborted the hung render and retried'}
def scenario_big_image(ctx):
ctx['compare_baseline'] = False
data = worst_decode_png()
containers = compose('ps', '-q', 'worker').stdout.split()
stop = threading.Event()
peak = {'mb': 0.0}
def sample():
while not stop.is_set():
if containers:
result = docker(
'stats', '--no-stream', '--format',
'{{.Name}} {{.MemUsage}}', *containers,
check_result=False,
)
for line in result.stdout.splitlines():
parts = line.split()
if len(parts) < 2:
continue
match = re.match(r'([\d.]+)(KiB|MiB|GiB)', parts[1])
if not match:
continue
value = float(match.group(1))
if match.group(2) == 'GiB':
value *= 1024
elif match.group(2) == 'KiB':
value /= 1024
peak['mb'] = max(peak['mb'], value)
stop.wait(0.3)
sampler = threading.Thread(target=sample, daemon=True)
sampler.start()
try:
def do_upload(index):
client = new_client(ctx['base_url'])
return upload(client, ctx['alice'], ctx['chat'], data,
'big-' + str(index) + '.png')
with ThreadPoolExecutor(max_workers=2) as pool:
attachments = list(pool.map(do_upload, (1, 2)))
ctx['attachment_id'] = attachments[0]['id']
for attachment in attachments:
ctx['tracker'].track(attachment['id'])
results = [wait_ready(ctx['client'], ctx['alice'],
attachment['id'], 120, allow_failed=True)
for attachment in attachments]
finally:
stop.set()
sampler.join(timeout=5)
for meta in results:
check(meta['status'] in ('ready', 'failed'), 'big image stuck')
if meta['status'] == 'failed':
check(meta['error_code'] in ('image_too_large', 'invalid_image'),
'unexpected big-image error code')
for container in containers:
result = docker('inspect', '-f', '{{.State.OOMKilled}}', container,
check_result=False)
check(result.stdout.strip() == 'false', 'worker was OOM-killed')
return {'attachment_id': attachments[0]['id'],
'job_attempts': [],
'notes': 'peak worker memory: %.1f MiB / 1280 MiB' % peak['mb']}
SCENARIOS = {
'duplicate': scenario_duplicate,
'reorder': scenario_reorder,
'crash_before_side_effect': scenario_crash_before_side_effect,
'crash_after_side_effect': scenario_crash_after_side_effect,
'crash_after_commit_before_ack': scenario_crash_after_commit_before_ack,
'crash_after_commit_before_publish': scenario_crash_after_commit_before_publish,
'relay_crash_after_publish': scenario_relay_crash_after_publish,
'stale_version': scenario_stale_version,
'poison': scenario_poison,
'retry_and_replay': scenario_retry_and_replay,
'hung_worker': scenario_hung_worker,
'big_image': scenario_big_image,
}
def select(name):
if name == 'all':
return LEVEL4 + [item for item in TRACK2 if item not in LEVEL4]
if name == 'level4':
return LEVEL4
if name == 'track2':
return TRACK2
if name in SCENARIOS:
return [name]
raise SystemExit('unknown scenario: ' + name)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default=None)
parser.add_argument('--ca-file', default=None)
parser.add_argument('--scenario', default='level4')
parser.add_argument('--repeat', type=int, default=1)
parser.add_argument('--output', default=None)
args = parser.parse_args()
if worker_env('FAULT_INJECTION_ENABLED') != '1':
print('fault injection is disabled; restart the stand with:\n ' + HINT,
file=sys.stderr)
raise SystemExit(2)
client = new_client(args.base_url, args.ca_file)
alice, _bob, _eve, chat = client.bootstrap()
clear_faults()
reference = upload(client, alice, chat, sample_bytes(), 'reference.png')
ready = wait_ready(client, alice, reference['id'], 60)
baseline = variant_sha(ready)
selected = select(args.scenario)
results = []
failed = 0
for name in selected:
for repeat in range(1, args.repeat + 1):
clear_faults()
tracker = Tracker(args.base_url or 'http://localhost:8080', alice,
args.ca_file)
tracker.start()
started = time.monotonic()
ctx = {
'client': client,
'base_url': args.base_url or 'http://localhost:8080',
'alice': alice,
'chat': chat,
'tracker': tracker,
'baseline': baseline,
'reference_id': reference['id'],
'ca_file': args.ca_file,
}
scenario = {'attachment_id': None, 'job_attempts': [],
'notes': ''}
passed = True
try:
scenario.update(SCENARIOS[name](ctx))
except Exception as error: # noqa: BLE001 - scenario boundary
passed = False
scenario['notes'] = type(error).__name__ + ': ' + str(error)
finally:
tracker.stop()
clear_faults()
compose('start', 'worker', check_result=False)
attachment_id = scenario['attachment_id'] or ctx.get('attachment_id')
try:
invariants = evaluate(
tracker, attachment_id, baseline,
compare_baseline=ctx.get('compare_baseline', True),
)
except Exception: # noqa: BLE001
invariants = {key: False for key in
('I1', 'I2', 'I3', 'I4', 'I5', 'I6')}
passed = passed and all(invariants.values())
if not passed:
failed += 1
result = {
'name': name,
'repeat': repeat,
'passed': passed,
'duration_s': round(time.monotonic() - started, 3),
'attachment_id': attachment_id,
'job_attempts': scenario['job_attempts'],
'events_summary': scenario.get('events_summary', {}),
'invariants': invariants,
'notes': scenario['notes'],
}
results.append(result)
print(('PASS ' if passed else 'FAIL ') + name + ' #' + str(repeat) +
' ' + str(result['duration_s']) + 's ' +
json.dumps(invariants) + ' ' + result['notes'],
flush=True)
if args.output:
path = Path(args.output)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(results, indent=2, ensure_ascii=False) + '\n')
print(str(len(results) - failed) + '/' + str(len(results)) + ' scenarios passed')
raise SystemExit(1 if failed else 0)
if __name__ == '__main__':
main()
+180
View File
@@ -0,0 +1,180 @@
"""Shared stdlib-only helpers for the lab5 demo and fault suite."""
import hashlib
import json
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check # noqa: E402
ENV_FILE = '.env'
COMPOSE = ['docker', 'compose', '--env-file', ENV_FILE]
def compose(*args, stdin=None, check_result=True):
result = subprocess.run(
COMPOSE + list(args),
cwd=ROOT,
input=stdin,
text=True,
capture_output=True,
)
if check_result and result.returncode != 0:
detail = (result.stderr or result.stdout).strip().splitlines()
raise Failure('compose ' + ' '.join(args) + ' failed: ' +
(detail[-1] if detail else 'unknown error'))
return result
def docker(*args, check_result=True):
result = subprocess.run(
['docker'] + list(args), cwd=ROOT, text=True, capture_output=True,
)
if check_result and result.returncode != 0:
detail = (result.stderr or result.stdout).strip().splitlines()
raise Failure('docker ' + ' '.join(args) + ' failed: ' +
(detail[-1] if detail else 'unknown error'))
return result
def cli(*args, stdin=None):
"""Run jobs.cli in the API container; returns (exit_code, parsed_json)."""
for _ in range(5):
result = compose(
'exec', '-T', 'api', 'node', 'dist/jobs/jobs.cli.js', *args,
stdin=stdin, check_result=False,
)
text = result.stdout.strip()
if text:
try:
return result.returncode, json.loads(text)
except ValueError:
pass
time.sleep(1)
raise Failure('jobs.cli did not return JSON for ' + ' '.join(args))
def set_fault(key, value):
compose('exec', '-T', 'redis', 'redis-cli', 'SET', key, value)
def clear_fault(key):
compose('exec', '-T', 'redis', 'redis-cli', 'DEL', key)
def clear_faults():
script = (
"local keys=redis.call('keys',ARGV[1]) "
"for i=1,#keys do redis.call('del',keys[i]) end return #keys"
)
compose('exec', '-T', 'redis', 'redis-cli', 'EVAL', script, '0', 'fault:*')
def rabbit_queues():
result = compose('exec', '-T', 'broker', 'rabbitmqctl', '-q',
'-p', 'messenger', 'list_queues',
'name', 'messages', 'type')
queues = {}
for line in result.stdout.splitlines():
parts = line.split()
if len(parts) >= 2 and parts[0] not in ('name', 'Timeout:'):
try:
queues[parts[0]] = {'messages': int(parts[1]),
'type': parts[2] if len(parts) > 2 else ''}
except ValueError:
continue
return queues
def wait_for(predicate, timeout, description='condition', interval=0.5):
deadline = time.monotonic() + timeout
last = None
while time.monotonic() < deadline:
last = predicate()
if last:
return last
time.sleep(interval)
raise Failure('timed out waiting for ' + description)
def upload(client, actor, chat, data, filename='sample.png',
content_type='image/png'):
result = client.call(
'POST',
'/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename,
202,
actor=actor,
raw=data,
headers={'Content-Type': content_type},
schema='Attachment',
)
check(result['json']['status'] == 'queued', 'upload did not answer queued')
return result['json']
def metadata(client, actor, attachment_id):
return client.call('GET', '/api/v1/attachments/' + attachment_id,
actor=actor, schema='Attachment')['json']
def wait_ready(client, actor, attachment_id, timeout=60,
allow_failed=False):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
meta = metadata(client, actor, attachment_id)
if meta['status'] == 'failed':
if allow_failed:
return meta
raise Failure('attachment failed: ' +
str(meta.get('error_code')))
if meta['status'] == 'ready':
return meta
time.sleep(0.2)
raise Failure('attachment did not become ready in ' + str(timeout) + 's')
def sha256(data):
return hashlib.sha256(data).hexdigest()
def variant_sha(meta):
return {item['name']: item['sha256'] for item in meta['variants']}
def sample_bytes():
return (ROOT / 'tests/fixtures/sample.png').read_bytes()
def worker_env(name):
result = compose('exec', '-T', 'worker', 'printenv', name,
check_result=False)
return result.stdout.strip()
def worker_start_count():
result = compose('logs', '--no-color', 'worker', check_result=False)
return result.stdout.count('worker ready:')
def worker_memory_peak(containers):
names = [line for line in containers.splitlines() if line.strip()]
if not names:
return 'n/a'
result = subprocess.run(
['docker', 'stats', '--no-stream', '--format',
'{{.Name}} {{.MemUsage}}'] + names,
text=True, capture_output=True,
)
return result.stdout.strip()
def print_step(number, text):
print('[' + str(number) + '] ' + text, flush=True)
def new_client(base_url, ca_file=None):
return Client(base_url, ca_file)
+126
View File
@@ -0,0 +1,126 @@
#!/usr/bin/env python3
"""Level-3 demo: work accepted while the worker is down survives restarts."""
import sys
import time
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent))
from stand import ( # noqa: E402
Failure,
check,
cli,
compose,
new_client,
print_step,
rabbit_queues,
sample_bytes,
sha256,
upload,
wait_for,
wait_ready,
)
from client import arguments, run # noqa: E402
from image_probe import jpeg_size # noqa: E402
def main():
args = arguments('lab5 worker-down durability demo')
client = new_client(args.base_url, args.ca_file)
alice, _bob, _eve, chat = client.bootstrap()
data = sample_bytes()
print_step(1, 'compose ps')
print(compose('ps', '--format',
'table {{.Service}}\t{{.Status}}').stdout.strip())
print_step(2, 'stop worker')
compose('stop', 'worker')
print_step(3, 'upload while worker is down (expect 202 queued)')
attachment = upload(client, alice, chat, data, 'durability.png')
attachment_id = attachment['id']
print('attachment=' + attachment_id + ' status=' + attachment['status'])
print_step(4, 'content and thumbnail must answer 409 before ready')
for query in ('', '?variant=thumbnail'):
result = client.call(
'GET',
'/api/v1/attachments/' + attachment_id + '/content' + query,
409,
actor=alice,
)
check(result['status'] == 409, 'content was served before ready')
print_step(5, 'link the queued attachment to messages')
with_attachment = client.message(
alice, chat, '', attachments=[attachment_id],
)['json']
plain = client.message(alice, chat, 'plain while worker is down')['json']
check(with_attachment['attachment_ids'] == [attachment_id],
'message lost the queued attachment')
check(plain['attachment_ids'] == [], 'plain message gained an attachment')
print('message=' + with_attachment['id'])
print_step(6, 'wait for outbox publish and a message in the queue')
def published():
_code, status = cli('status', attachment_id)
queues = rabbit_queues()
return (
status['outbox']['pending'] == 0
and queues.get('attachments.process', {}).get('messages', 0) >= 1
)
wait_for(published, 30, 'outbox publish and queue depth >= 1')
queues = rabbit_queues()
print('queue=' + str(queues.get('attachments.process')))
print_step(7, 'restart broker and API with volumes')
compose('restart', 'broker', 'api')
compose('up', '-d', '--no-recreate', '--no-deps', '--wait', 'broker', 'api')
print_step(8, 'the quorum message survived the broker restart')
def survived():
depth = rabbit_queues().get('attachments.process', {}).get('messages', 0)
return depth >= 1
wait_for(survived, 60, 'queue depth >= 1 after broker restart')
print('queue=' + str(rabbit_queues().get('attachments.process')))
print_step(9, 'start worker')
compose('start', 'worker')
print_step(10, 'wait ready and verify the thumbnail')
ready = wait_ready(client, alice, attachment_id, 60)
thumbnail = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0]
content = client.call(
'GET',
'/api/v1/attachments/' + attachment_id + '/content?variant=thumbnail',
actor=alice,
)
check(jpeg_size(content['bytes']) == (256, 160),
'thumbnail is not 256x160 JPEG')
check(len(content['bytes']) == thumbnail['size_bytes'],
'thumbnail size differs from metadata')
check(sha256(content['bytes']) == thumbnail['sha256'],
'thumbnail checksum differs from metadata')
history = client.call(
'GET',
'/api/v1/chats/' + chat['id'] + '/messages?limit=100',
actor=alice,
schema='MessagePage',
)['json']
check(
any(attachment_id in item['attachment_ids']
for item in history['items']),
'message history lost the attachment',
)
print('ready attachment=' + attachment_id +
' thumbnail=256x160 sha256=' + thumbnail['sha256'][:16] + '...')
print_step(11, 'compose ps')
print(compose('ps', '--format',
'table {{.Service}}\t{{.Status}}').stdout.strip())
print('PASS: worker-down durability demo')
if __name__ == '__main__':
run(main)
+8
View File
@@ -0,0 +1,8 @@
#!/usr/bin/env bash
# Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`.
# Semantic and evidence notes: tests/README.md ("Проверка сохранности") and REPORT.md.
set -euo pipefail
compose=(docker compose --env-file .env)
"${compose[@]}" restart postgres redis s3 broker api worker haproxy
"${compose[@]}" up -d --no-recreate --no-deps --wait postgres redis s3 broker api worker haproxy
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
# Восстанавливает недостающую API-реплику до объявленного числа реплик.
set -eu
env_file="${COMPOSE_ENV_FILE:-.env}"
replicas="${API_REPLICAS:-}"
if [ -z "$replicas" ] && [ -f "$env_file" ]; then
replicas=$(sed -n 's/^API_REPLICAS=//p' "$env_file" | tail -n 1)
fi
replicas="${replicas:-2}"
exec docker compose --env-file "$env_file" up -d --no-deps --scale "api=${replicas}" api
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
# Останавливает ровно одну API-реплику стенда — ту, которую обслужила текущая
# сессия. TARGET_INSTANCE_ID приходит из теста (значение X-Instance-Id ответа),
# а в Docker hostname контейнера равен его короткому ID, поэтому сопоставление
# однозначно. Ничего, кроме этой реплики, скрипт не трогает: ни LB, ни БД, ни
# тома. Возврат управления — быстрый (SIGTERM -> дрейн DRAIN_DELAY_MS=3s ->
# выход), останавливать реплику "навсегда" не требуется.
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
project="${LAB_PROJECT:-mtusi-lab6}"
target=''
for container in $(
docker ps \
--filter "label=com.docker.compose.project=${project}" \
--filter 'label=com.docker.compose.service=api' \
--format '{{.ID}}'
); do
case "$container" in
"$TARGET_INSTANCE_ID"*)
target="$container"
break
;;
esac
done
if [ -z "$target" ]; then
printf 'stop-one: no running api replica of project %s whose id starts with %s\n' \
"$project" "$TARGET_INSTANCE_ID" >&2
exit 1
fi
printf 'stop-one: stopping api replica %s\n' "$target" >&2
docker stop --time 10 "$target" >/dev/null