Перенести реализацию лабораторной 5 в лабораторную 6
This commit is contained in:
@@ -0,0 +1,32 @@
|
||||
# Creates the same local secret files as init-secrets.sh on Windows.
|
||||
# Existing values are preserved and never printed.
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$secretDirectory = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../infra/secrets'))
|
||||
[IO.Directory]::CreateDirectory($secretDirectory) | Out-Null
|
||||
function New-RandomHex {
|
||||
$bytes = New-Object byte[] 24
|
||||
$randomGenerator = [Security.Cryptography.RandomNumberGenerator]::Create()
|
||||
try { $randomGenerator.GetBytes($bytes) } finally { $randomGenerator.Dispose() }
|
||||
return ([BitConverter]::ToString($bytes)).Replace('-', '').ToLowerInvariant()
|
||||
}
|
||||
function Write-NewSecret([string]$Name, [string]$Value) {
|
||||
$secretPath = Join-Path $secretDirectory $Name
|
||||
if (Test-Path -LiteralPath $secretPath) { Write-Output "keep infra/secrets/$Name" }
|
||||
else {
|
||||
[IO.File]::WriteAllText($secretPath, $Value, (New-Object Text.UTF8Encoding $false))
|
||||
Write-Output "create infra/secrets/$Name"
|
||||
}
|
||||
}
|
||||
Write-NewSecret 'postgres_password.txt' (New-RandomHex)
|
||||
$databasePassword = [IO.File]::ReadAllText((Join-Path $secretDirectory 'postgres_password.txt')).Trim()
|
||||
Write-NewSecret 'database_url.txt' "postgresql://messenger:${databasePassword}@db:5432/messenger"
|
||||
Write-NewSecret 'valkey_password.txt' (New-RandomHex)
|
||||
$cachePassword = [IO.File]::ReadAllText((Join-Path $secretDirectory 'valkey_password.txt')).Trim()
|
||||
Write-NewSecret 'cache_url.txt' "redis://:${cachePassword}@valkey:6379/0"
|
||||
Write-NewSecret 's3_admin_access_key_id.txt' ('admin' + (New-RandomHex).Substring(0,15))
|
||||
Write-NewSecret 's3_admin_secret_access_key.txt' ((New-RandomHex) + (New-RandomHex))
|
||||
Write-NewSecret 's3_access_key_id.txt' ('api' + (New-RandomHex).Substring(0,17))
|
||||
Write-NewSecret 's3_secret_access_key.txt' ((New-RandomHex) + (New-RandomHex))
|
||||
Write-NewSecret 'broker_password.txt' (New-RandomHex)
|
||||
$queuePassword = [IO.File]::ReadAllText((Join-Path $secretDirectory 'broker_password.txt')).Trim()
|
||||
Write-NewSecret 'broker_url.txt' "amqp://messenger:${queuePassword}@broker:5672/"
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/bin/sh
|
||||
# Creates local Docker secrets for a synthetic lab stand.
|
||||
# Existing files are never overwritten; values are never printed.
|
||||
set -eu
|
||||
|
||||
dir="$(cd "$(dirname "$0")/.." && pwd)/infra/secrets"
|
||||
mkdir -p "$dir"
|
||||
# Host access is limited by the directory mode; the files themselves must stay
|
||||
# readable by the non-root users inside containers (bind-mounted secrets).
|
||||
chmod 700 "$dir" 2>/dev/null || true
|
||||
|
||||
random() {
|
||||
od -An -tx1 -N24 /dev/urandom | tr -d ' \n'
|
||||
}
|
||||
|
||||
write_new() {
|
||||
if [ -f "$dir/$1" ]; then
|
||||
echo "keep infra/secrets/$1"
|
||||
else
|
||||
printf '%s' "$2" > "$dir/$1"
|
||||
chmod 644 "$dir/$1" 2>/dev/null || true
|
||||
echo "create infra/secrets/$1"
|
||||
fi
|
||||
}
|
||||
|
||||
if [ ! -f "$dir/postgres_password.txt" ]; then
|
||||
write_new postgres_password.txt "$(random)"
|
||||
fi
|
||||
pg_password="$(cat "$dir/postgres_password.txt")"
|
||||
write_new database_url.txt "postgresql://messenger:${pg_password}@db:5432/messenger"
|
||||
|
||||
if [ ! -f "$dir/valkey_password.txt" ]; then
|
||||
write_new valkey_password.txt "$(random)"
|
||||
fi
|
||||
valkey_password="$(cat "$dir/valkey_password.txt")"
|
||||
write_new cache_url.txt "redis://:${valkey_password}@valkey:6379/0"
|
||||
|
||||
# Lab4: S3 (SeaweedFS). Admin credentials are used only by the s3-init
|
||||
# bootstrap; the API gets its own key pair restricted to one bucket.
|
||||
write_new s3_admin_access_key_id.txt "admin$(random | cut -c1-15)"
|
||||
write_new s3_admin_secret_access_key.txt "$(random)$(random)"
|
||||
write_new s3_access_key_id.txt "api$(random | cut -c1-17)"
|
||||
write_new s3_secret_access_key.txt "$(random)$(random)"
|
||||
|
||||
# Lab5: RabbitMQ uses a separate local password. The URL is a Docker secret;
|
||||
# no credentials appear in compose.yaml or the report.
|
||||
if [ ! -f "$dir/broker_password.txt" ]; then
|
||||
write_new broker_password.txt "$(random)"
|
||||
fi
|
||||
broker_password="$(cat "$dir/broker_password.txt")"
|
||||
write_new broker_url.txt "amqp://messenger:${broker_password}@broker:5672/"
|
||||
Executable
+18
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
# Вариант жёсткого отказа (для сравнения): SIGKILL без graceful shutdown.
|
||||
# Используется в собственных экспериментах; публичный тест запускается со stop-one.sh.
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?target API instance alias required}"
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
*[!0-9a-f]* | "") echo "unexpected instance id format" >&2; exit 2 ;;
|
||||
esac
|
||||
for id in $(service_ids api); do
|
||||
case "$id" in "$TARGET_INSTANCE_ID"*)
|
||||
docker kill --signal KILL "$id" >/dev/null
|
||||
echo "killed api container $(short "$id") at $(date -u +%H:%M:%S)"
|
||||
exit 0 ;;
|
||||
esac
|
||||
done
|
||||
echo "instance $TARGET_INSTANCE_ID is not a running api container" >&2
|
||||
exit 3
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/bin/sh
|
||||
# Общие функции операторских скриптов. Используют только docker CLI и метки Compose,
|
||||
# поэтому работают и на хосте, и в контейнере tester (docker.sock).
|
||||
PROJECT="${COMPOSE_PROJECT:-messenger-lab6}"
|
||||
|
||||
# ID контейнеров сервиса (по умолчанию только работающих); -a — включая остановленные.
|
||||
service_ids() {
|
||||
# $1 — сервис, $2 — необязательный флаг -a
|
||||
docker ps ${2:-} -q --no-trunc \
|
||||
--filter "label=com.docker.compose.project=$PROJECT" \
|
||||
--filter "label=com.docker.compose.service=$1"
|
||||
}
|
||||
|
||||
# Ждать, пока все переданные контейнеры станут healthy (timeout в секундах первым аргументом).
|
||||
# В POSIX sh переменные функций глобальны: имена с префиксом _wh_, чтобы не затереть
|
||||
# переменные вызывающего цикла.
|
||||
wait_healthy() {
|
||||
_wh_timeout="$1"; shift
|
||||
_wh_deadline=$(( $(date +%s) + _wh_timeout ))
|
||||
for _wh_id in "$@"; do
|
||||
while :; do
|
||||
_wh_state=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$_wh_id" 2>/dev/null || echo missing)
|
||||
case "$_wh_state" in healthy|running) break ;; esac
|
||||
if [ "$(date +%s)" -ge "$_wh_deadline" ]; then
|
||||
echo "container $(echo "$_wh_id" | cut -c1-12) not healthy (state=$_wh_state)" >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
short() { echo "$1" | cut -c1-12; }
|
||||
@@ -0,0 +1,99 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import asyncpg
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MIGRATIONS_DIR = ROOT / "migrations"
|
||||
# Lab3: migrations run once in the "migrate" service before the API replicas
|
||||
# start. The advisory lock still serialises two accidental parallel runs
|
||||
# (e.g. a manual run during a rolling update); the second one then sees all
|
||||
# versions in schema_migrations and only prints SKIP.
|
||||
MIGRATION_LOCK_ID = 727003
|
||||
|
||||
|
||||
def read_database_url() -> str:
|
||||
file_name = os.getenv("DATABASE_URL_FILE")
|
||||
|
||||
if file_name:
|
||||
return Path(file_name).read_text(encoding="utf-8").strip()
|
||||
|
||||
value = os.getenv("DATABASE_URL")
|
||||
|
||||
if not value:
|
||||
raise RuntimeError("DATABASE_URL or DATABASE_URL_FILE is required.")
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def plain_postgres_url(url: str) -> str:
|
||||
if url.startswith("postgresql+asyncpg://"):
|
||||
return url.replace("postgresql+asyncpg://", "postgresql://", 1)
|
||||
|
||||
return url
|
||||
|
||||
|
||||
async def connect_with_retry(url: str, attempts: int = 30):
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
return await asyncpg.connect(url, timeout=3)
|
||||
except (OSError, asyncpg.PostgresError) as exc:
|
||||
if attempt == attempts:
|
||||
raise
|
||||
print(f"WAIT database ({type(exc).__name__}), retry {attempt}")
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
async def migrate() -> None:
|
||||
connection = await connect_with_retry(
|
||||
plain_postgres_url(read_database_url())
|
||||
)
|
||||
|
||||
try:
|
||||
await connection.execute(
|
||||
"SELECT pg_advisory_lock($1)",
|
||||
MIGRATION_LOCK_ID,
|
||||
)
|
||||
await connection.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
applied_rows = await connection.fetch(
|
||||
"SELECT version FROM schema_migrations"
|
||||
)
|
||||
applied = {row["version"] for row in applied_rows}
|
||||
|
||||
for migration_file in sorted(MIGRATIONS_DIR.glob("*.sql")):
|
||||
version = migration_file.name
|
||||
|
||||
if version in applied:
|
||||
print(f"SKIP {version}")
|
||||
continue
|
||||
|
||||
sql = migration_file.read_text(encoding="utf-8")
|
||||
|
||||
print(f"APPLY {version}")
|
||||
|
||||
async with connection.transaction():
|
||||
await connection.execute(sql)
|
||||
await connection.execute(
|
||||
"INSERT INTO schema_migrations (version) VALUES ($1)",
|
||||
version,
|
||||
)
|
||||
|
||||
print("PASS: migrations applied")
|
||||
finally:
|
||||
await connection.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(migrate())
|
||||
Executable
+31
@@ -0,0 +1,31 @@
|
||||
#!/bin/sh
|
||||
# Crash-restart every persistent store after draining the application.
|
||||
# No re-seed or volume removal; the whole hook is bounded to 85 seconds.
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
api=$(service_ids api); lb=$(service_ids lb); db=$(service_ids db)
|
||||
valkey=$(service_ids valkey); s3=$(service_ids s3); broker=$(service_ids broker)
|
||||
worker=$(service_ids worker); publisher=$(service_ids publisher)
|
||||
[ -n "$api" ] && [ -n "$db" ] && [ -n "$lb" ] && [ -n "$valkey" ] && [ -n "$s3" ] && [ -n "$broker" ] && [ -n "$worker" ] && [ -n "$publisher" ] || { echo "stack is not running" >&2; exit 2; }
|
||||
_restart_deadline=$(( $(date +%s) + 85 ))
|
||||
remaining() {
|
||||
_restart_left=$(( _restart_deadline - $(date +%s) ))
|
||||
[ "$_restart_left" -gt 0 ] || { echo "restart deadline exceeded" >&2; exit 1; }
|
||||
echo "$_restart_left"
|
||||
}
|
||||
echo "stopping lb + api + publisher + worker, then all stores"
|
||||
docker stop --time 10 $lb $api $publisher $worker >/dev/null
|
||||
# Deliberate crash, not a graceful shutdown of the stores. This tests that
|
||||
# confirmed data survives process death and avoids spending the public
|
||||
# hook's entire deadline on S3/RabbitMQ internal shutdown procedures.
|
||||
# Writers are stopped first; persistent volumes are kept unchanged.
|
||||
docker kill --signal KILL $db $valkey $s3 $broker >/dev/null
|
||||
echo "starting db + valkey + s3 + broker with existing volumes"
|
||||
docker start $db $valkey $s3 $broker >/dev/null
|
||||
wait_healthy "$(remaining)" $db $valkey $s3 $broker
|
||||
echo "starting publisher + worker + api"
|
||||
docker start $publisher $worker $api >/dev/null
|
||||
wait_healthy "$(remaining)" $api
|
||||
docker start $lb >/dev/null
|
||||
wait_healthy "$(remaining)" $lb
|
||||
echo "restart complete: PostgreSQL + Valkey + S3 + RabbitMQ + publisher + worker + API replicas + LB"
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Idempotent bootstrap of the private attachments bucket (service s3-init).
|
||||
|
||||
Runs with the S3 ADMIN credentials (only this one-shot container gets them),
|
||||
before any API replica starts:
|
||||
|
||||
1. wait until the S3 gateway answers;
|
||||
2. create the bucket if it does not exist (a repeated run prints "keep");
|
||||
3. self-checks, the stand does not start if one fails:
|
||||
- anonymous (unsigned) GET/PUT/LIST of the bucket -> 403: bucket is private;
|
||||
- API credentials can PUT/GET/DELETE under the prefix;
|
||||
- API credentials can NOT create/delete buckets or touch other buckets.
|
||||
|
||||
The API identity and its rights are declared in infra/seaweedfs/s3.json.template.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import botocore.session
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
|
||||
def secret(name: str) -> str:
|
||||
file_name = os.getenv(f"{name}_FILE")
|
||||
if file_name:
|
||||
return Path(file_name).read_text(encoding="utf-8").strip()
|
||||
value = os.getenv(name)
|
||||
if not value:
|
||||
raise RuntimeError(f"{name} or {name}_FILE is required")
|
||||
return value
|
||||
|
||||
|
||||
ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000")
|
||||
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
PREFIX = os.getenv("S3_OBJECT_PREFIX", "originals/")
|
||||
|
||||
|
||||
def client(access_key: str, secret_key: str):
|
||||
return botocore.session.get_session().create_client(
|
||||
"s3",
|
||||
endpoint_url=ENDPOINT,
|
||||
region_name=os.getenv("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=2,
|
||||
read_timeout=5,
|
||||
retries={"max_attempts": 1},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def error_code(exc: ClientError) -> str:
|
||||
return exc.response.get("Error", {}).get("Code", "")
|
||||
|
||||
|
||||
failures = 0
|
||||
|
||||
|
||||
def check(label: str, ok: bool) -> None:
|
||||
global failures
|
||||
failures += not ok
|
||||
print(("PASS " if ok else "FAIL ") + label, flush=True)
|
||||
|
||||
|
||||
def denied(call) -> bool:
|
||||
try:
|
||||
call()
|
||||
except ClientError as exc:
|
||||
return error_code(exc) in {"AccessDenied", "403", "Forbidden", "AllAccessDisabled"}
|
||||
return False
|
||||
|
||||
|
||||
def anonymous_status(method: str, path: str, data: bytes | None = None) -> int:
|
||||
request = urllib.request.Request(ENDPOINT + path, data=data, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=5) as response:
|
||||
return response.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code
|
||||
|
||||
|
||||
def main() -> int:
|
||||
admin = client(secret("S3_ADMIN_ACCESS_KEY_ID"), secret("S3_ADMIN_SECRET_ACCESS_KEY"))
|
||||
api = client(secret("S3_ACCESS_KEY_ID"), secret("S3_SECRET_ACCESS_KEY"))
|
||||
|
||||
for attempt in range(1, 61):
|
||||
try:
|
||||
admin.list_buckets()
|
||||
break
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
if attempt == 60:
|
||||
raise
|
||||
print(f"WAIT s3 ({type(exc).__name__}), retry {attempt}", flush=True)
|
||||
time.sleep(1)
|
||||
|
||||
try:
|
||||
admin.head_bucket(Bucket=BUCKET)
|
||||
print(f"keep bucket {BUCKET}", flush=True)
|
||||
except ClientError as exc:
|
||||
if error_code(exc) not in {"404", "NoSuchBucket", "NotFound"}:
|
||||
raise
|
||||
try:
|
||||
admin.create_bucket(Bucket=BUCKET)
|
||||
print(f"create bucket {BUCKET}", flush=True)
|
||||
except ClientError as create_exc:
|
||||
# Two bootstrap runs at the same time: the other one won.
|
||||
if error_code(create_exc) not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
|
||||
raise
|
||||
print(f"keep bucket {BUCKET}", flush=True)
|
||||
|
||||
probe = f"{PREFIX}bootstrap-probe-{uuid.uuid4().hex}"
|
||||
|
||||
check("api credentials: PUT object", not denied(
|
||||
lambda: api.put_object(Bucket=BUCKET, Key=probe, Body=b"probe")))
|
||||
check("api credentials: GET object",
|
||||
api.get_object(Bucket=BUCKET, Key=probe)["Body"].read() == b"probe")
|
||||
|
||||
check("anonymous GET object -> 403", anonymous_status("GET", f"/{BUCKET}/{probe}") == 403)
|
||||
check("anonymous LIST bucket -> 403", anonymous_status("GET", f"/{BUCKET}?list-type=2") == 403)
|
||||
check("anonymous PUT object -> 403",
|
||||
anonymous_status("PUT", f"/{BUCKET}/{PREFIX}anonymous-{uuid.uuid4().hex}", b"x") == 403)
|
||||
|
||||
other = f"forbidden-{uuid.uuid4().hex[:12]}"
|
||||
check("api credentials: CreateBucket denied",
|
||||
denied(lambda: api.create_bucket(Bucket=other)))
|
||||
check("api credentials: DeleteBucket denied",
|
||||
denied(lambda: api.delete_bucket(Bucket=BUCKET)))
|
||||
check("api credentials: DELETE object", not denied(
|
||||
lambda: api.delete_object(Bucket=BUCKET, Key=probe)))
|
||||
|
||||
def foreign_buckets_hidden() -> bool:
|
||||
# SeaweedFS denies ListBuckets to a bucket-scoped identity; other
|
||||
# servers may answer with only the allowed bucket. Both are fine.
|
||||
try:
|
||||
names = [b["Name"] for b in api.list_buckets().get("Buckets", [])]
|
||||
except ClientError as exc:
|
||||
return error_code(exc) in {"AccessDenied", "403", "Forbidden"}
|
||||
return all(name == BUCKET for name in names)
|
||||
|
||||
check("api credentials: ListBuckets shows no foreign buckets", foreign_buckets_hidden())
|
||||
|
||||
if failures:
|
||||
print(f"FAIL: {failures} bucket checks failed; the API must not start", flush=True)
|
||||
return 1
|
||||
|
||||
print("PASS: bucket bootstrap complete (idempotent, private, least privilege)", flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/bin/sh
|
||||
# Восстановление после stop-one.sh: запускает остановленные API-реплики и ждёт healthy.
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
all=$(service_ids api -a)
|
||||
running=$(service_ids api)
|
||||
started=""
|
||||
for id in $all; do
|
||||
case " $running " in *" $id "*) continue ;; esac
|
||||
docker start "$id" >/dev/null
|
||||
started="$started $id"
|
||||
echo "started api container $(short "$id")"
|
||||
done
|
||||
[ -z "$started" ] || wait_healthy 60 $started
|
||||
echo "running api replicas: $(service_ids api | wc -l | tr -d ' ')"
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
worker=$(service_ids worker -a)
|
||||
[ -n "$worker" ] || { echo "worker has not been created: run docker compose up -d" >&2; exit 2; }
|
||||
docker start $worker >/dev/null
|
||||
echo "worker started; pending durable jobs will be consumed"
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/sh
|
||||
# Failover hook: останавливает ОДНУ API-реплику, обслужившую тест, и оставляет её остановленной.
|
||||
# TARGET_INSTANCE_ID = X-Instance-Id = hostname контейнера = префикс его Docker ID.
|
||||
# БД, LB и volumes не затрагиваются. Восстановление: scripts/start-api.sh.
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
*[!0-9a-f]* | "") echo "unexpected instance id format" >&2; exit 2 ;;
|
||||
esac
|
||||
target=""
|
||||
for id in $(service_ids api); do
|
||||
case "$id" in "$TARGET_INSTANCE_ID"*) target="$id" ;; esac
|
||||
done
|
||||
if [ -z "$target" ]; then
|
||||
echo "instance $TARGET_INSTANCE_ID is not a running api container of project $PROJECT" >&2
|
||||
exit 3
|
||||
fi
|
||||
# docker stop = SIGTERM -> graceful drain (~3 s) -> выход; SIGKILL только после 15 s.
|
||||
docker stop --time 15 "$target" >/dev/null
|
||||
echo "stopped api container $(short "$target") at $(date -u +%H:%M:%S)"
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
worker=$(service_ids worker)
|
||||
[ -n "$worker" ] || { echo "worker is already stopped"; exit 0; }
|
||||
docker stop --time 10 $worker >/dev/null
|
||||
echo "worker stopped; publisher and API remain available"
|
||||
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run lab5 checks from Windows or Linux with only host Python and Docker.
|
||||
|
||||
Start the Compose stand first. Run ``python scripts/test_lab5.py --quick``
|
||||
for image/API checks, or omit --quick to include deliberate service failures.
|
||||
Commands run from the repository root regardless of the current directory.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = ROOT / "evidence"
|
||||
COMPOSE = ["docker", "compose", "--profile", "tools"]
|
||||
SERVICES = ("db", "valkey", "s3", "broker", "api", "lb", "publisher", "worker")
|
||||
|
||||
|
||||
class CheckFailed(Exception):
|
||||
"""A check or a Docker command returned a nonzero exit code."""
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def command_text(command: list[str]) -> str:
|
||||
return subprocess.list2cmdline(command) if os.name == "nt" else shlex.join(command)
|
||||
|
||||
|
||||
def run(command: list[str], evidence: str, *, append: bool = False,
|
||||
stdin_file: Path | None = None) -> str:
|
||||
"""Stream combined stdout/stderr to the terminal and a UTF-8 evidence log."""
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
with (EVIDENCE / evidence).open("a" if append else "w", encoding="utf-8", newline="\n") as log:
|
||||
def write(value: str) -> None:
|
||||
print(value, end="", flush=True)
|
||||
log.write(value)
|
||||
log.flush()
|
||||
|
||||
display = command_text(command)
|
||||
if stdin_file is not None:
|
||||
display += " < " + stdin_file.relative_to(ROOT).as_posix()
|
||||
write(f"\nUTC: {utc_now()}\nCommand: {display}\n")
|
||||
output: list[str] = []
|
||||
try:
|
||||
# Pass an actual file as stdin: no shell redirection, quoting or
|
||||
# PowerShell encoding conversions are involved.
|
||||
with stdin_file.open("rb") if stdin_file else open(os.devnull, "rb") as source:
|
||||
process = subprocess.Popen(
|
||||
command, cwd=ROOT, stdin=source, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace",
|
||||
)
|
||||
try:
|
||||
assert process.stdout is not None
|
||||
for line in process.stdout:
|
||||
write(line)
|
||||
output.append(line)
|
||||
code = process.wait()
|
||||
except BaseException:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait()
|
||||
raise
|
||||
finally:
|
||||
if process.stdout is not None:
|
||||
process.stdout.close()
|
||||
except (OSError, KeyboardInterrupt) as error:
|
||||
write(f"\nRESULT: ERROR ({type(error).__name__}) at {utc_now()}\n")
|
||||
raise
|
||||
write(f"\nRESULT: {'PASS' if code == 0 else 'FAIL'} (exit {code}) at {utc_now()}\n")
|
||||
if code:
|
||||
raise CheckFailed(f"{display}: exit {code}; see evidence/{evidence}")
|
||||
return "".join(output)
|
||||
|
||||
|
||||
def tester(*command: str) -> list[str]:
|
||||
return [*COMPOSE, "run", "--rm", "-T", "--no-deps", "tester", *command]
|
||||
|
||||
|
||||
def snapshot(stage: str, *, append: bool = True) -> None:
|
||||
"""Record hook scope without dumping container environment or secrets."""
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
with (EVIDENCE / "container-events.txt").open("a" if append else "w", encoding="utf-8") as log:
|
||||
log.write(f"\n{stage} — {utc_now()}\n")
|
||||
identifiers = run([*COMPOSE, "ps", "-a", "-q"], "container-events.txt", append=True).split()
|
||||
if identifiers:
|
||||
run(["docker", "inspect", "--format",
|
||||
"{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}",
|
||||
*identifiers], "container-events.txt", append=True)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--quick", action="store_true",
|
||||
help="skip recovery, full-stack restart and API failover checks")
|
||||
args = parser.parse_args()
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(stream, "reconfigure"):
|
||||
stream.reconfigure(encoding="utf-8", errors="replace")
|
||||
|
||||
destructive_started = False
|
||||
complete = False
|
||||
exit_code = 0
|
||||
try:
|
||||
running = run([*COMPOSE, "ps", "--status", "running", "--services"], "runner-setup.txt")
|
||||
missing = set(SERVICES) - set(running.split())
|
||||
if missing:
|
||||
raise CheckFailed(
|
||||
"Start the Compose stand first: missing running services "
|
||||
+ ", ".join(sorted(missing))
|
||||
+ ". Run docker compose up -d --build --wait --wait-timeout 180."
|
||||
)
|
||||
run([*COMPOSE, "ps", "-a"], "stand.txt")
|
||||
run([*COMPOSE, "build", "tester"], "runner-setup.txt", append=True)
|
||||
run(tester("python3", "-u", "scripts/check_contract.py"), "check.txt")
|
||||
run(tester("python3", "-u", "tests/smoke.py"), "smoke.txt")
|
||||
run(tester("python3", "-u", "student-tests/image_validation_unit.py"), "image-validation.txt")
|
||||
run(tester("python3", "-u", "student-tests/lab5_images.py"), "images.txt")
|
||||
run([*COMPOSE, "exec", "-T", "worker", "python", "-u", "-"], "limits.txt",
|
||||
stdin_file=ROOT / "student-tests" / "lab5_limits.py")
|
||||
|
||||
if not args.quick:
|
||||
# Git clones on Linux may lack executable bits on newly added
|
||||
# hooks; the public test explicitly requires executable scripts.
|
||||
run(tester("chmod", "+x", "scripts/restart.sh", "scripts/stop-one.sh"),
|
||||
"runner-setup.txt", append=True)
|
||||
destructive_started = True
|
||||
run(tester("python3", "-u", "student-tests/lab5_recovery.py"), "recovery.txt")
|
||||
snapshot("Before public persistence", append=False)
|
||||
try:
|
||||
run(tester("python3", "-u", "tests/resilience.py", "--mode", "persistence",
|
||||
"--action-script", "scripts/restart.sh"), "persistence.txt")
|
||||
finally:
|
||||
snapshot("After public persistence")
|
||||
snapshot("Before public failover")
|
||||
try:
|
||||
run(tester("python3", "-u", "tests/resilience.py", "--mode", "failover",
|
||||
"--action-script", "scripts/stop-one.sh"), "failover.txt")
|
||||
finally:
|
||||
# Failover deliberately leaves an API replica stopped.
|
||||
# Restore it after success as well as after a failed check.
|
||||
try:
|
||||
snapshot("After public failover, before API restoration")
|
||||
finally:
|
||||
run(tester("sh", "scripts/start-api.sh"), "failover.txt", append=True)
|
||||
snapshot("After API restoration")
|
||||
run([*COMPOSE, "ps", "-a"], "stand.txt", append=True)
|
||||
complete = True
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted. Completed command logs remain in evidence/.", file=sys.stderr)
|
||||
exit_code = 130
|
||||
except (CheckFailed, OSError) as error:
|
||||
print(f"\nFAIL: {error}", file=sys.stderr)
|
||||
exit_code = 1
|
||||
finally:
|
||||
if destructive_started and not complete:
|
||||
try:
|
||||
# Preserve all volumes. Compose may also restart the
|
||||
# idempotent migration/bootstrap dependencies.
|
||||
run([*COMPOSE, "start", *SERVICES], "runner-setup.txt", append=True)
|
||||
except (CheckFailed, OSError, KeyboardInterrupt) as error:
|
||||
print(f"Service restoration failed: {error}", file=sys.stderr)
|
||||
exit_code = exit_code or 1
|
||||
|
||||
if exit_code == 0:
|
||||
mode = "quick" if args.quick else "full"
|
||||
print(f"\nPASS: lab5 {mode} checks. UTF-8 logs: {EVIDENCE}")
|
||||
return exit_code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user