Перенести реализацию лабораторной 5 в лабораторную 6

This commit is contained in:
2026-11-21 11:35:52 +03:00
parent 470d9d6f1e
commit 36d84eaf44
53 changed files with 6420 additions and 29 deletions
+32
View File
@@ -0,0 +1,32 @@
# Creates the same local secret files as init-secrets.sh on Windows.
# Existing values are preserved and never printed.
$ErrorActionPreference = 'Stop'
$secretDirectory = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../infra/secrets'))
[IO.Directory]::CreateDirectory($secretDirectory) | Out-Null
function New-RandomHex {
$bytes = New-Object byte[] 24
$randomGenerator = [Security.Cryptography.RandomNumberGenerator]::Create()
try { $randomGenerator.GetBytes($bytes) } finally { $randomGenerator.Dispose() }
return ([BitConverter]::ToString($bytes)).Replace('-', '').ToLowerInvariant()
}
function Write-NewSecret([string]$Name, [string]$Value) {
$secretPath = Join-Path $secretDirectory $Name
if (Test-Path -LiteralPath $secretPath) { Write-Output "keep infra/secrets/$Name" }
else {
[IO.File]::WriteAllText($secretPath, $Value, (New-Object Text.UTF8Encoding $false))
Write-Output "create infra/secrets/$Name"
}
}
Write-NewSecret 'postgres_password.txt' (New-RandomHex)
$databasePassword = [IO.File]::ReadAllText((Join-Path $secretDirectory 'postgres_password.txt')).Trim()
Write-NewSecret 'database_url.txt' "postgresql://messenger:${databasePassword}@db:5432/messenger"
Write-NewSecret 'valkey_password.txt' (New-RandomHex)
$cachePassword = [IO.File]::ReadAllText((Join-Path $secretDirectory 'valkey_password.txt')).Trim()
Write-NewSecret 'cache_url.txt' "redis://:${cachePassword}@valkey:6379/0"
Write-NewSecret 's3_admin_access_key_id.txt' ('admin' + (New-RandomHex).Substring(0,15))
Write-NewSecret 's3_admin_secret_access_key.txt' ((New-RandomHex) + (New-RandomHex))
Write-NewSecret 's3_access_key_id.txt' ('api' + (New-RandomHex).Substring(0,17))
Write-NewSecret 's3_secret_access_key.txt' ((New-RandomHex) + (New-RandomHex))
Write-NewSecret 'broker_password.txt' (New-RandomHex)
$queuePassword = [IO.File]::ReadAllText((Join-Path $secretDirectory 'broker_password.txt')).Trim()
Write-NewSecret 'broker_url.txt' "amqp://messenger:${queuePassword}@broker:5672/"
+51
View File
@@ -0,0 +1,51 @@
#!/bin/sh
# Creates local Docker secrets for a synthetic lab stand.
# Existing files are never overwritten; values are never printed.
set -eu
dir="$(cd "$(dirname "$0")/.." && pwd)/infra/secrets"
mkdir -p "$dir"
# Host access is limited by the directory mode; the files themselves must stay
# readable by the non-root users inside containers (bind-mounted secrets).
chmod 700 "$dir" 2>/dev/null || true
random() {
od -An -tx1 -N24 /dev/urandom | tr -d ' \n'
}
write_new() {
if [ -f "$dir/$1" ]; then
echo "keep infra/secrets/$1"
else
printf '%s' "$2" > "$dir/$1"
chmod 644 "$dir/$1" 2>/dev/null || true
echo "create infra/secrets/$1"
fi
}
if [ ! -f "$dir/postgres_password.txt" ]; then
write_new postgres_password.txt "$(random)"
fi
pg_password="$(cat "$dir/postgres_password.txt")"
write_new database_url.txt "postgresql://messenger:${pg_password}@db:5432/messenger"
if [ ! -f "$dir/valkey_password.txt" ]; then
write_new valkey_password.txt "$(random)"
fi
valkey_password="$(cat "$dir/valkey_password.txt")"
write_new cache_url.txt "redis://:${valkey_password}@valkey:6379/0"
# Lab4: S3 (SeaweedFS). Admin credentials are used only by the s3-init
# bootstrap; the API gets its own key pair restricted to one bucket.
write_new s3_admin_access_key_id.txt "admin$(random | cut -c1-15)"
write_new s3_admin_secret_access_key.txt "$(random)$(random)"
write_new s3_access_key_id.txt "api$(random | cut -c1-17)"
write_new s3_secret_access_key.txt "$(random)$(random)"
# Lab5: RabbitMQ uses a separate local password. The URL is a Docker secret;
# no credentials appear in compose.yaml or the report.
if [ ! -f "$dir/broker_password.txt" ]; then
write_new broker_password.txt "$(random)"
fi
broker_password="$(cat "$dir/broker_password.txt")"
write_new broker_url.txt "amqp://messenger:${broker_password}@broker:5672/"
+18
View File
@@ -0,0 +1,18 @@
#!/bin/sh
# Вариант жёсткого отказа (для сравнения): SIGKILL без graceful shutdown.
# Используется в собственных экспериментах; публичный тест запускается со stop-one.sh.
set -eu
: "${TARGET_INSTANCE_ID:?target API instance alias required}"
. "$(dirname "$0")/lib.sh"
case "$TARGET_INSTANCE_ID" in
*[!0-9a-f]* | "") echo "unexpected instance id format" >&2; exit 2 ;;
esac
for id in $(service_ids api); do
case "$id" in "$TARGET_INSTANCE_ID"*)
docker kill --signal KILL "$id" >/dev/null
echo "killed api container $(short "$id") at $(date -u +%H:%M:%S)"
exit 0 ;;
esac
done
echo "instance $TARGET_INSTANCE_ID is not a running api container" >&2
exit 3
+33
View File
@@ -0,0 +1,33 @@
#!/bin/sh
# Общие функции операторских скриптов. Используют только docker CLI и метки Compose,
# поэтому работают и на хосте, и в контейнере tester (docker.sock).
PROJECT="${COMPOSE_PROJECT:-messenger-lab6}"
# ID контейнеров сервиса (по умолчанию только работающих); -a — включая остановленные.
service_ids() {
# $1 — сервис, $2 — необязательный флаг -a
docker ps ${2:-} -q --no-trunc \
--filter "label=com.docker.compose.project=$PROJECT" \
--filter "label=com.docker.compose.service=$1"
}
# Ждать, пока все переданные контейнеры станут healthy (timeout в секундах первым аргументом).
# В POSIX sh переменные функций глобальны: имена с префиксом _wh_, чтобы не затереть
# переменные вызывающего цикла.
wait_healthy() {
_wh_timeout="$1"; shift
_wh_deadline=$(( $(date +%s) + _wh_timeout ))
for _wh_id in "$@"; do
while :; do
_wh_state=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$_wh_id" 2>/dev/null || echo missing)
case "$_wh_state" in healthy|running) break ;; esac
if [ "$(date +%s)" -ge "$_wh_deadline" ]; then
echo "container $(echo "$_wh_id" | cut -c1-12) not healthy (state=$_wh_state)" >&2
return 1
fi
sleep 1
done
done
}
short() { echo "$1" | cut -c1-12; }
+99
View File
@@ -0,0 +1,99 @@
from __future__ import annotations
import asyncio
import os
from pathlib import Path
import asyncpg
ROOT = Path(__file__).resolve().parents[1]
MIGRATIONS_DIR = ROOT / "migrations"
# Lab3: migrations run once in the "migrate" service before the API replicas
# start. The advisory lock still serialises two accidental parallel runs
# (e.g. a manual run during a rolling update); the second one then sees all
# versions in schema_migrations and only prints SKIP.
MIGRATION_LOCK_ID = 727003
def read_database_url() -> str:
file_name = os.getenv("DATABASE_URL_FILE")
if file_name:
return Path(file_name).read_text(encoding="utf-8").strip()
value = os.getenv("DATABASE_URL")
if not value:
raise RuntimeError("DATABASE_URL or DATABASE_URL_FILE is required.")
return value
def plain_postgres_url(url: str) -> str:
if url.startswith("postgresql+asyncpg://"):
return url.replace("postgresql+asyncpg://", "postgresql://", 1)
return url
async def connect_with_retry(url: str, attempts: int = 30):
for attempt in range(1, attempts + 1):
try:
return await asyncpg.connect(url, timeout=3)
except (OSError, asyncpg.PostgresError) as exc:
if attempt == attempts:
raise
print(f"WAIT database ({type(exc).__name__}), retry {attempt}")
await asyncio.sleep(1)
async def migrate() -> None:
connection = await connect_with_retry(
plain_postgres_url(read_database_url())
)
try:
await connection.execute(
"SELECT pg_advisory_lock($1)",
MIGRATION_LOCK_ID,
)
await connection.execute(
"""
CREATE TABLE IF NOT EXISTS schema_migrations (
version TEXT PRIMARY KEY,
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
)
"""
)
applied_rows = await connection.fetch(
"SELECT version FROM schema_migrations"
)
applied = {row["version"] for row in applied_rows}
for migration_file in sorted(MIGRATIONS_DIR.glob("*.sql")):
version = migration_file.name
if version in applied:
print(f"SKIP {version}")
continue
sql = migration_file.read_text(encoding="utf-8")
print(f"APPLY {version}")
async with connection.transaction():
await connection.execute(sql)
await connection.execute(
"INSERT INTO schema_migrations (version) VALUES ($1)",
version,
)
print("PASS: migrations applied")
finally:
await connection.close()
if __name__ == "__main__":
asyncio.run(migrate())
+31
View File
@@ -0,0 +1,31 @@
#!/bin/sh
# Crash-restart every persistent store after draining the application.
# No re-seed or volume removal; the whole hook is bounded to 85 seconds.
set -eu
. "$(dirname "$0")/lib.sh"
api=$(service_ids api); lb=$(service_ids lb); db=$(service_ids db)
valkey=$(service_ids valkey); s3=$(service_ids s3); broker=$(service_ids broker)
worker=$(service_ids worker); publisher=$(service_ids publisher)
[ -n "$api" ] && [ -n "$db" ] && [ -n "$lb" ] && [ -n "$valkey" ] && [ -n "$s3" ] && [ -n "$broker" ] && [ -n "$worker" ] && [ -n "$publisher" ] || { echo "stack is not running" >&2; exit 2; }
_restart_deadline=$(( $(date +%s) + 85 ))
remaining() {
_restart_left=$(( _restart_deadline - $(date +%s) ))
[ "$_restart_left" -gt 0 ] || { echo "restart deadline exceeded" >&2; exit 1; }
echo "$_restart_left"
}
echo "stopping lb + api + publisher + worker, then all stores"
docker stop --time 10 $lb $api $publisher $worker >/dev/null
# Deliberate crash, not a graceful shutdown of the stores. This tests that
# confirmed data survives process death and avoids spending the public
# hook's entire deadline on S3/RabbitMQ internal shutdown procedures.
# Writers are stopped first; persistent volumes are kept unchanged.
docker kill --signal KILL $db $valkey $s3 $broker >/dev/null
echo "starting db + valkey + s3 + broker with existing volumes"
docker start $db $valkey $s3 $broker >/dev/null
wait_healthy "$(remaining)" $db $valkey $s3 $broker
echo "starting publisher + worker + api"
docker start $publisher $worker $api >/dev/null
wait_healthy "$(remaining)" $api
docker start $lb >/dev/null
wait_healthy "$(remaining)" $lb
echo "restart complete: PostgreSQL + Valkey + S3 + RabbitMQ + publisher + worker + API replicas + LB"
+161
View File
@@ -0,0 +1,161 @@
"""Idempotent bootstrap of the private attachments bucket (service s3-init).
Runs with the S3 ADMIN credentials (only this one-shot container gets them),
before any API replica starts:
1. wait until the S3 gateway answers;
2. create the bucket if it does not exist (a repeated run prints "keep");
3. self-checks, the stand does not start if one fails:
- anonymous (unsigned) GET/PUT/LIST of the bucket -> 403: bucket is private;
- API credentials can PUT/GET/DELETE under the prefix;
- API credentials can NOT create/delete buckets or touch other buckets.
The API identity and its rights are declared in infra/seaweedfs/s3.json.template.
"""
from __future__ import annotations
import os
import sys
import time
import urllib.error
import urllib.request
import uuid
from pathlib import Path
import botocore.session
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
def secret(name: str) -> str:
file_name = os.getenv(f"{name}_FILE")
if file_name:
return Path(file_name).read_text(encoding="utf-8").strip()
value = os.getenv(name)
if not value:
raise RuntimeError(f"{name} or {name}_FILE is required")
return value
ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000")
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
PREFIX = os.getenv("S3_OBJECT_PREFIX", "originals/")
def client(access_key: str, secret_key: str):
return botocore.session.get_session().create_client(
"s3",
endpoint_url=ENDPOINT,
region_name=os.getenv("S3_REGION", "us-east-1"),
aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
config=Config(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=2,
read_timeout=5,
retries={"max_attempts": 1},
),
)
def error_code(exc: ClientError) -> str:
return exc.response.get("Error", {}).get("Code", "")
failures = 0
def check(label: str, ok: bool) -> None:
global failures
failures += not ok
print(("PASS " if ok else "FAIL ") + label, flush=True)
def denied(call) -> bool:
try:
call()
except ClientError as exc:
return error_code(exc) in {"AccessDenied", "403", "Forbidden", "AllAccessDisabled"}
return False
def anonymous_status(method: str, path: str, data: bytes | None = None) -> int:
request = urllib.request.Request(ENDPOINT + path, data=data, method=method)
try:
with urllib.request.urlopen(request, timeout=5) as response:
return response.status
except urllib.error.HTTPError as exc:
return exc.code
def main() -> int:
admin = client(secret("S3_ADMIN_ACCESS_KEY_ID"), secret("S3_ADMIN_SECRET_ACCESS_KEY"))
api = client(secret("S3_ACCESS_KEY_ID"), secret("S3_SECRET_ACCESS_KEY"))
for attempt in range(1, 61):
try:
admin.list_buckets()
break
except (BotoCoreError, ClientError) as exc:
if attempt == 60:
raise
print(f"WAIT s3 ({type(exc).__name__}), retry {attempt}", flush=True)
time.sleep(1)
try:
admin.head_bucket(Bucket=BUCKET)
print(f"keep bucket {BUCKET}", flush=True)
except ClientError as exc:
if error_code(exc) not in {"404", "NoSuchBucket", "NotFound"}:
raise
try:
admin.create_bucket(Bucket=BUCKET)
print(f"create bucket {BUCKET}", flush=True)
except ClientError as create_exc:
# Two bootstrap runs at the same time: the other one won.
if error_code(create_exc) not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
raise
print(f"keep bucket {BUCKET}", flush=True)
probe = f"{PREFIX}bootstrap-probe-{uuid.uuid4().hex}"
check("api credentials: PUT object", not denied(
lambda: api.put_object(Bucket=BUCKET, Key=probe, Body=b"probe")))
check("api credentials: GET object",
api.get_object(Bucket=BUCKET, Key=probe)["Body"].read() == b"probe")
check("anonymous GET object -> 403", anonymous_status("GET", f"/{BUCKET}/{probe}") == 403)
check("anonymous LIST bucket -> 403", anonymous_status("GET", f"/{BUCKET}?list-type=2") == 403)
check("anonymous PUT object -> 403",
anonymous_status("PUT", f"/{BUCKET}/{PREFIX}anonymous-{uuid.uuid4().hex}", b"x") == 403)
other = f"forbidden-{uuid.uuid4().hex[:12]}"
check("api credentials: CreateBucket denied",
denied(lambda: api.create_bucket(Bucket=other)))
check("api credentials: DeleteBucket denied",
denied(lambda: api.delete_bucket(Bucket=BUCKET)))
check("api credentials: DELETE object", not denied(
lambda: api.delete_object(Bucket=BUCKET, Key=probe)))
def foreign_buckets_hidden() -> bool:
# SeaweedFS denies ListBuckets to a bucket-scoped identity; other
# servers may answer with only the allowed bucket. Both are fine.
try:
names = [b["Name"] for b in api.list_buckets().get("Buckets", [])]
except ClientError as exc:
return error_code(exc) in {"AccessDenied", "403", "Forbidden"}
return all(name == BUCKET for name in names)
check("api credentials: ListBuckets shows no foreign buckets", foreign_buckets_hidden())
if failures:
print(f"FAIL: {failures} bucket checks failed; the API must not start", flush=True)
return 1
print("PASS: bucket bootstrap complete (idempotent, private, least privilege)", flush=True)
return 0
if __name__ == "__main__":
sys.exit(main())
+15
View File
@@ -0,0 +1,15 @@
#!/bin/sh
# Восстановление после stop-one.sh: запускает остановленные API-реплики и ждёт healthy.
set -eu
. "$(dirname "$0")/lib.sh"
all=$(service_ids api -a)
running=$(service_ids api)
started=""
for id in $all; do
case " $running " in *" $id "*) continue ;; esac
docker start "$id" >/dev/null
started="$started $id"
echo "started api container $(short "$id")"
done
[ -z "$started" ] || wait_healthy 60 $started
echo "running api replicas: $(service_ids api | wc -l | tr -d ' ')"
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
. "$(dirname "$0")/lib.sh"
worker=$(service_ids worker -a)
[ -n "$worker" ] || { echo "worker has not been created: run docker compose up -d" >&2; exit 2; }
docker start $worker >/dev/null
echo "worker started; pending durable jobs will be consumed"
+21
View File
@@ -0,0 +1,21 @@
#!/bin/sh
# Failover hook: останавливает ОДНУ API-реплику, обслужившую тест, и оставляет её остановленной.
# TARGET_INSTANCE_ID = X-Instance-Id = hostname контейнера = префикс его Docker ID.
# БД, LB и volumes не затрагиваются. Восстановление: scripts/start-api.sh.
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
. "$(dirname "$0")/lib.sh"
case "$TARGET_INSTANCE_ID" in
*[!0-9a-f]* | "") echo "unexpected instance id format" >&2; exit 2 ;;
esac
target=""
for id in $(service_ids api); do
case "$id" in "$TARGET_INSTANCE_ID"*) target="$id" ;; esac
done
if [ -z "$target" ]; then
echo "instance $TARGET_INSTANCE_ID is not a running api container of project $PROJECT" >&2
exit 3
fi
# docker stop = SIGTERM -> graceful drain (~3 s) -> выход; SIGKILL только после 15 s.
docker stop --time 15 "$target" >/dev/null
echo "stopped api container $(short "$target") at $(date -u +%H:%M:%S)"
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
. "$(dirname "$0")/lib.sh"
worker=$(service_ids worker)
[ -n "$worker" ] || { echo "worker is already stopped"; exit 0; }
docker stop --time 10 $worker >/dev/null
echo "worker stopped; publisher and API remain available"
+182
View File
@@ -0,0 +1,182 @@
#!/usr/bin/env python3
"""Run lab5 checks from Windows or Linux with only host Python and Docker.
Start the Compose stand first. Run ``python scripts/test_lab5.py --quick``
for image/API checks, or omit --quick to include deliberate service failures.
Commands run from the repository root regardless of the current directory.
"""
from __future__ import annotations
import argparse
from datetime import datetime, timezone
import os
from pathlib import Path
import shlex
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = ROOT / "evidence"
COMPOSE = ["docker", "compose", "--profile", "tools"]
SERVICES = ("db", "valkey", "s3", "broker", "api", "lb", "publisher", "worker")
class CheckFailed(Exception):
"""A check or a Docker command returned a nonzero exit code."""
def utc_now() -> str:
return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
def command_text(command: list[str]) -> str:
return subprocess.list2cmdline(command) if os.name == "nt" else shlex.join(command)
def run(command: list[str], evidence: str, *, append: bool = False,
stdin_file: Path | None = None) -> str:
"""Stream combined stdout/stderr to the terminal and a UTF-8 evidence log."""
EVIDENCE.mkdir(exist_ok=True)
with (EVIDENCE / evidence).open("a" if append else "w", encoding="utf-8", newline="\n") as log:
def write(value: str) -> None:
print(value, end="", flush=True)
log.write(value)
log.flush()
display = command_text(command)
if stdin_file is not None:
display += " < " + stdin_file.relative_to(ROOT).as_posix()
write(f"\nUTC: {utc_now()}\nCommand: {display}\n")
output: list[str] = []
try:
# Pass an actual file as stdin: no shell redirection, quoting or
# PowerShell encoding conversions are involved.
with stdin_file.open("rb") if stdin_file else open(os.devnull, "rb") as source:
process = subprocess.Popen(
command, cwd=ROOT, stdin=source, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace",
)
try:
assert process.stdout is not None
for line in process.stdout:
write(line)
output.append(line)
code = process.wait()
except BaseException:
process.terminate()
try:
process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
process.wait()
raise
finally:
if process.stdout is not None:
process.stdout.close()
except (OSError, KeyboardInterrupt) as error:
write(f"\nRESULT: ERROR ({type(error).__name__}) at {utc_now()}\n")
raise
write(f"\nRESULT: {'PASS' if code == 0 else 'FAIL'} (exit {code}) at {utc_now()}\n")
if code:
raise CheckFailed(f"{display}: exit {code}; see evidence/{evidence}")
return "".join(output)
def tester(*command: str) -> list[str]:
return [*COMPOSE, "run", "--rm", "-T", "--no-deps", "tester", *command]
def snapshot(stage: str, *, append: bool = True) -> None:
"""Record hook scope without dumping container environment or secrets."""
EVIDENCE.mkdir(exist_ok=True)
with (EVIDENCE / "container-events.txt").open("a" if append else "w", encoding="utf-8") as log:
log.write(f"\n{stage} — {utc_now()}\n")
identifiers = run([*COMPOSE, "ps", "-a", "-q"], "container-events.txt", append=True).split()
if identifiers:
run(["docker", "inspect", "--format",
"{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}",
*identifiers], "container-events.txt", append=True)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--quick", action="store_true",
help="skip recovery, full-stack restart and API failover checks")
args = parser.parse_args()
for stream in (sys.stdout, sys.stderr):
if hasattr(stream, "reconfigure"):
stream.reconfigure(encoding="utf-8", errors="replace")
destructive_started = False
complete = False
exit_code = 0
try:
running = run([*COMPOSE, "ps", "--status", "running", "--services"], "runner-setup.txt")
missing = set(SERVICES) - set(running.split())
if missing:
raise CheckFailed(
"Start the Compose stand first: missing running services "
+ ", ".join(sorted(missing))
+ ". Run docker compose up -d --build --wait --wait-timeout 180."
)
run([*COMPOSE, "ps", "-a"], "stand.txt")
run([*COMPOSE, "build", "tester"], "runner-setup.txt", append=True)
run(tester("python3", "-u", "scripts/check_contract.py"), "check.txt")
run(tester("python3", "-u", "tests/smoke.py"), "smoke.txt")
run(tester("python3", "-u", "student-tests/image_validation_unit.py"), "image-validation.txt")
run(tester("python3", "-u", "student-tests/lab5_images.py"), "images.txt")
run([*COMPOSE, "exec", "-T", "worker", "python", "-u", "-"], "limits.txt",
stdin_file=ROOT / "student-tests" / "lab5_limits.py")
if not args.quick:
# Git clones on Linux may lack executable bits on newly added
# hooks; the public test explicitly requires executable scripts.
run(tester("chmod", "+x", "scripts/restart.sh", "scripts/stop-one.sh"),
"runner-setup.txt", append=True)
destructive_started = True
run(tester("python3", "-u", "student-tests/lab5_recovery.py"), "recovery.txt")
snapshot("Before public persistence", append=False)
try:
run(tester("python3", "-u", "tests/resilience.py", "--mode", "persistence",
"--action-script", "scripts/restart.sh"), "persistence.txt")
finally:
snapshot("After public persistence")
snapshot("Before public failover")
try:
run(tester("python3", "-u", "tests/resilience.py", "--mode", "failover",
"--action-script", "scripts/stop-one.sh"), "failover.txt")
finally:
# Failover deliberately leaves an API replica stopped.
# Restore it after success as well as after a failed check.
try:
snapshot("After public failover, before API restoration")
finally:
run(tester("sh", "scripts/start-api.sh"), "failover.txt", append=True)
snapshot("After API restoration")
run([*COMPOSE, "ps", "-a"], "stand.txt", append=True)
complete = True
except KeyboardInterrupt:
print("\nInterrupted. Completed command logs remain in evidence/.", file=sys.stderr)
exit_code = 130
except (CheckFailed, OSError) as error:
print(f"\nFAIL: {error}", file=sys.stderr)
exit_code = 1
finally:
if destructive_started and not complete:
try:
# Preserve all volumes. Compose may also restart the
# idempotent migration/bootstrap dependencies.
run([*COMPOSE, "start", *SERVICES], "runner-setup.txt", append=True)
except (CheckFailed, OSError, KeyboardInterrupt) as error:
print(f"Service restoration failed: {error}", file=sys.stderr)
exit_code = exit_code or 1
if exit_code == 0:
mode = "quick" if args.quick else "full"
print(f"\nPASS: lab5 {mode} checks. UTF-8 logs: {EVIDENCE}")
return exit_code
if __name__ == "__main__":
raise SystemExit(main())