Files
mtusi2026lab6-fork/scripts/ops.py
T

584 lines
28 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Операции над compose-стендом lab5 (база — lab4). Python 3.9+, только стандартная библиотека.
Работает одинаково в Windows (PowerShell/cmd), Linux и macOS: вызывает
`docker compose` и `docker` без shell-интерполяции. Секреты не печатает.
python scripts/ops.py wait # дождаться N здоровых реплик и LB
python scripts/ops.py ps # реплики: контейнер, X-Instance-Id, health, статус в LB
python scripts/ops.py stop-one # TARGET_INSTANCE_ID=<X-Instance-Id>; оставляет остановленной
python scripts/ops.py restore # запустить остановленные реплики и дождаться UP в LB
python scripts/ops.py restart # hook persistence: API + worker + PostgreSQL + Redis + broker + S3 + LB, volumes целы
python scripts/ops.py wait-s3 # дождаться, пока S3 реально читает/пишет (прогрев после рестарта)
python scripts/ops.py s3-check # приватность бакета вне API, права ключей, закрытые админ-порты
python scripts/ops.py reconcile [--apply] # сверка БД <-> S3 (по умолчанию dry-run)
python scripts/ops.py api-fs # у API read-only rootfs, только tmpfs, без volumes
python scripts/ops.py rolling-update # поочерёдная замена реплик без остановки сервиса
python scripts/ops.py in-network storage # тесты с прямым доступом к БД — внутри compose-сети
python scripts/ops.py in-network expiry # истечение сессии на api-short-ttl (TTL 2 с)
python scripts/ops.py in-network consistency # БД<->S3: сборщик, сироты, обрыв загрузки, внедрённые отказы
lab5:
python scripts/ops.py worker-stop # остановить ВСЕ контейнеры worker (docker stop, graceful)
python scripts/ops.py worker-kill # SIGKILL worker (без перезапуска политикой restart)
python scripts/ops.py worker-start # запустить worker и дождаться healthy
python scripts/ops.py api-restart # перезапуск реплик API (volumes не трогаются)
python scripts/ops.py broker-restart # перезапуск broker (AOF на volume brokerdata)
python scripts/ops.py jobs stats|failed|show ID|replay ID... # src/jobs_admin.py в one-off контейнере
python scripts/ops.py pipeline-faults # внедрённые отказы pipeline (outbox, crash до/после ack, retry, DLQ, replay, lease)
Переменные окружения:
COMPOSE_PROJECT_NAME имя проекта (по умолчанию messenger-lab6, как `name:` в compose.yaml)
ENV_FILE файл переменных (по умолчанию .env, если существует)
API_REPLICAS ожидаемое число реплик (по умолчанию из .env, иначе 2)
APP_PORT, LB_STATS_PORT опубликованные на 127.0.0.1 порты LB (8080, 8404)
STOP_MODE stop-one: stop (SIGTERM, graceful, по умолчанию) | kill (SIGKILL, «падение»)
Каждое действие дописывается в evidence/ops.log (время, контейнер, результат) —
это доказательство того, какой контейнер реально остановлен/заменён.
"""
import csv
import io
import json
import os
import subprocess
import sys
import time
import urllib.error
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OPS_LOG = ROOT / "evidence" / "ops.log"
try: # консоль Windows может быть не в UTF-8
sys.stdout.reconfigure(errors="replace")
sys.stderr.reconfigure(errors="replace")
except AttributeError:
pass
# ---------------------------------------------------------------- settings
def _dotenv():
path = Path(os.environ.get("ENV_FILE", str(ROOT / ".env")))
values = {}
if path.is_file():
for line in path.read_text(encoding="utf-8").splitlines():
line = line.strip()
if line and not line.startswith("#") and "=" in line:
k, v = line.split("=", 1)
values[k.strip()] = v.strip().strip('"').strip("'")
return path if path.is_file() else None, values
ENV_PATH, DOTENV = _dotenv()
def setting(name, default):
return os.environ.get(name) or DOTENV.get(name) or default
PROJECT = setting("COMPOSE_PROJECT_NAME", "messenger-lab6")
REPLICAS = int(setting("API_REPLICAS", "2"))
APP_PORT = int(setting("APP_PORT", "8080"))
STATS_PORT = int(setting("LB_STATS_PORT", "8404"))
LB_URL = "http://127.0.0.1:%d" % APP_PORT
STATS_URL = "http://127.0.0.1:%d/stats;csv" % STATS_PORT
def now_iso():
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
def log(msg):
line = "%s %s" % (now_iso(), msg)
print(line, flush=True)
try:
OPS_LOG.parent.mkdir(exist_ok=True)
with OPS_LOG.open("a", encoding="utf-8") as f:
f.write(line + "\n")
except OSError:
pass
# ---------------------------------------------------------------- docker
def compose_cmd(*args):
cmd = ["docker", "compose", "-p", PROJECT]
if ENV_PATH:
cmd += ["--env-file", str(ENV_PATH)]
return cmd + list(args)
def run(cmd, check=True, capture=False, timeout=300, show=False):
"""capture: вернуть вывод; show: показать вывод в консоли; иначе — тихо."""
out = None if show else (subprocess.PIPE if capture else subprocess.DEVNULL)
result = subprocess.run(
cmd, cwd=ROOT, check=False, timeout=timeout,
stdout=out, stderr=out, text=True,
)
if check and result.returncode != 0:
detail = (result.stderr or "").strip()[-400:] if capture else ""
raise SystemExit("command failed (%d): %s %s" % (result.returncode, " ".join(cmd[:4]), detail))
return result
def compose(*args, timeout=300):
return run(compose_cmd(*args), timeout=timeout, capture=True)
def api_containers(include_stopped=True):
"""[{id, short, name, state, health, image, ip}] — только сервис api, без one-off."""
cmd = ["docker", "ps", "--no-trunc", "--format", "{{.ID}}",
"--filter", "label=com.docker.compose.project=" + PROJECT,
"--filter", "label=com.docker.compose.service=api",
"--filter", "label=com.docker.compose.oneoff=False"]
if include_stopped:
cmd.insert(2, "-a")
ids = [x for x in run(cmd, capture=True).stdout.split() if x]
if not ids:
return []
raw = run(["docker", "inspect"] + ids, capture=True).stdout
result = []
for item in json.loads(raw):
state = item["State"]
nets = item.get("NetworkSettings", {}).get("Networks") or {}
ips = [n.get("IPAddress") for n in nets.values() if n.get("IPAddress")]
result.append({
"id": item["Id"],
"short": item["Id"][:12],
"hostname": item["Config"].get("Hostname", ""),
"name": item["Name"].lstrip("/"),
"state": state.get("Status"),
"health": (state.get("Health") or {}).get("Status", "-"),
"image": item["Config"].get("Image"),
"ip": ips[0] if ips else "",
"created": item.get("Created", "")[:19],
})
result.sort(key=lambda c: c["name"])
return result
def running(containers=None):
return [c for c in (containers if containers is not None else api_containers()) if c["state"] == "running"]
# ---------------------------------------------------------------- load balancer
def lb_servers():
"""[{name, status, addr}] backend api из CSV-статистики HAProxy (слоты с адресом)."""
data = urllib.request.urlopen(STATS_URL, timeout=2).read().decode("utf-8", "replace")
rows = csv.DictReader(io.StringIO(data.lstrip("# ")))
servers = []
for r in rows:
if r.get("pxname") != "api" or r.get("svname") in ("BACKEND", "FRONTEND"):
continue
addr = r.get("addr") or ""
status = r.get("status", "")
if status.startswith("MAINT") and not addr:
continue # пустой слот server-template
servers.append({"name": r["svname"], "status": status, "addr": addr})
return servers
def lb_up_count():
return sum(1 for s in lb_servers() if s["status"].startswith("UP"))
def lb_ready():
try:
with urllib.request.urlopen(LB_URL + "/health/ready", timeout=2) as r:
return r.status == 200
except (urllib.error.URLError, OSError):
return False
def wait_until(predicate, timeout, what, interval=0.3):
deadline = time.monotonic() + timeout
last_exc = None
while time.monotonic() < deadline:
try:
if predicate():
return True
except (urllib.error.URLError, OSError, ValueError, KeyError) as exc:
last_exc = exc
time.sleep(interval)
raise SystemExit("timeout after %ss waiting for %s%s" % (
timeout, what, (" (%s)" % type(last_exc).__name__) if last_exc else ""))
def wait_healthy(ids, timeout=90):
def ok():
states = {c["id"]: c for c in api_containers()}
return all(states.get(i, {}).get("health") == "healthy" for i in ids)
wait_until(ok, timeout, "containers healthy: " + ", ".join(i[:12] for i in ids))
def wait_stack(replicas, timeout=120):
wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= replicas,
timeout, "%d healthy api replicas" % replicas)
wait_until(lambda: lb_up_count() >= replicas, 30, "%d replicas UP in HAProxy" % replicas)
wait_until(lb_ready, 30, "LB /health/ready = 200")
# ---------------------------------------------------------------- commands
def cmd_wait(_args):
t0 = time.monotonic()
wait_stack(REPLICAS)
print("ready: %d api replicas healthy and UP in LB (%.1fs)" % (REPLICAS, time.monotonic() - t0))
def cmd_ps(_args):
containers = api_containers()
try:
by_addr = {s["addr"].rsplit(":", 1)[0]: s for s in lb_servers()}
except (urllib.error.URLError, OSError):
by_addr = {}
print("%-14s %-28s %-9s %-10s %-22s %-6s %s" % ("X-Instance-Id", "container", "state", "health", "image", "lb", "lb_status"))
for c in containers:
s = by_addr.get(c["ip"], {})
print("%-14s %-28s %-9s %-10s %-22s %-6s %s" % (
c["hostname"], c["name"], c["state"], c["health"], c["image"], s.get("name", "-"), s.get("status", "-")))
def find_target(target):
for c in api_containers():
# X-Instance-Id = hostname контейнера = первые 12 символов его ID.
if target and (c["hostname"] == target or c["id"].startswith(target)):
return c
return None
def cmd_stop_one(_args):
target = os.environ.get("TARGET_INSTANCE_ID", "").strip()
if not target:
raise SystemExit("TARGET_INSTANCE_ID is required (X-Instance-Id of the replica to stop)")
mode = os.environ.get("STOP_MODE", "stop").strip().lower()
c = find_target(target)
if c is None:
raise SystemExit("no api container of project %s matches X-Instance-Id %s" % (PROJECT, target))
if c["state"] != "running":
raise SystemExit("container %s is already %s" % (c["name"], c["state"]))
t0 = time.monotonic()
log("stop-one: mode=%s target=%s container=%s ip=%s" % (mode, target, c["name"], c["ip"]))
if mode == "kill":
# «Падение» процесса. Политику restart снимаем, чтобы Docker не поднял
# контейнер сам: тест требует, чтобы реплика оставалась остановленной.
run(["docker", "update", "--restart", "no", c["id"]])
run(["docker", "kill", "--signal", "KILL", c["id"]])
else:
# SIGTERM -> осушение 3 с (readiness 503) -> graceful stop; SIGKILL не позже 20 с.
run(["docker", "stop", "--time", "20", c["id"]], timeout=40)
after = find_target(target) or {}
log("stop-one: container=%s state=%s after %.2fs; left stopped (restore: python scripts/ops.py restore)"
% (c["name"], after.get("state", "removed"), time.monotonic() - t0))
def cmd_restore(_args):
t0 = time.monotonic()
stopped = [c for c in api_containers() if c["state"] != "running"]
for c in stopped: # вернуть политику, снятую в режиме kill
run(["docker", "update", "--restart", "unless-stopped", c["id"]], check=False)
# --no-recreate: не пересоздавать работающие; up запускает остановленные
# и добирает число контейнеров до deploy.replicas.
compose("up", "-d", "--no-deps", "--no-recreate", "api")
wait_stack(REPLICAS)
log("restore: started %s; %d replicas healthy and UP in LB after %.1fs"
% ([c["name"] for c in stopped] or "nothing", REPLICAS, time.monotonic() - t0))
def cmd_restart(_args):
"""Hook для make test-persistence: перезапуск API, worker, ВСЕХ хранилищ
(PostgreSQL, Redis, broker, S3) и LB.
Volumes не удаляются, данные не пересоздаются, bucket/поток не создаются заново.
Порядок: сначала LB, API и worker (новые записи не принимаются, обработка
останавливается graceful), затем хранилища и брокер, затем ожидание реальной
готовности S3, затем API, worker и LB.
"""
t0 = time.monotonic()
before = {c["name"]: c["short"] for c in running()}
stores = ("postgres", "redis", "broker", "s3")
started_before = {svc: container_started(svc) for svc in stores}
compose("stop", "lb", "api", "worker", timeout=180)
compose("restart", *stores, timeout=120)
wait_service_healthy("s3", 60)
wait_service_healthy("broker", 60)
t_warm = time.monotonic()
warm = s3_warmup(70)
log("restart: s3 read/write ready %.1fs after container healthy (%s)" % (time.monotonic() - t_warm, warm))
compose("start", "api", "worker", timeout=120)
wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= REPLICAS,
80, "api healthy after restart")
compose("start", "lb", timeout=60)
log("restart: started_at %s; api %s restarted; worker restarted; lb restarted; %.1fs"
% ("; ".join("%s %s -> %s" % (svc, started_before[svc], container_started(svc)) for svc in stores),
sorted(before), time.monotonic() - t0))
def service_ids(service, include_stopped=False):
cmd = ["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT,
"--filter", "label=com.docker.compose.service=" + service,
"--filter", "label=com.docker.compose.oneoff=False"]
if include_stopped:
cmd.insert(2, "-a")
return run(cmd, capture=True).stdout.split()
def cmd_worker_stop(_args):
t0 = time.monotonic()
ids = service_ids("worker")
# SIGTERM: worker дорабатывает текущие сообщения (stop_grace_period 75 с).
compose("stop", "worker", timeout=120)
log("worker-stop: stopped %s in %.1fs" % ([i[:12] for i in ids], time.monotonic() - t0))
def cmd_worker_kill(_args):
ids = service_ids("worker")
for i in ids:
run(["docker", "update", "--restart", "no", i])
run(["docker", "kill", "--signal", "KILL", i])
log("worker-kill: SIGKILL %s (restart policy disabled until worker-start)" % [i[:12] for i in ids])
def cmd_worker_start(_args):
t0 = time.monotonic()
for i in service_ids("worker", include_stopped=True):
run(["docker", "update", "--restart", "unless-stopped", i], check=False)
compose("up", "-d", "--no-deps", "--no-recreate", "worker", timeout=120)
wait_service_healthy("worker", 90)
log("worker-start: worker healthy after %.1fs" % (time.monotonic() - t0))
def cmd_api_restart(_args):
t0 = time.monotonic()
before = [c["short"] for c in running()]
compose("restart", "api", timeout=120)
wait_stack(REPLICAS)
log("api-restart: %s restarted, %d replicas UP after %.1fs" % (before, REPLICAS, time.monotonic() - t0))
def cmd_broker_restart(_args):
t0 = time.monotonic()
before = container_started("broker")
compose("restart", "broker", timeout=60)
wait_service_healthy("broker", 60)
log("broker-restart: started_at %s -> %s, healthy after %.1fs (volume brokerdata kept)"
% (before, container_started("broker"), time.monotonic() - t0))
def cmd_jobs(args):
run(compose_cmd("run", "--rm", "-T", "--no-deps", "worker", "python", "-m", "src.jobs_admin") + list(args),
show=True, timeout=300)
def faults_compose(*args):
return run(["docker", "compose", "-p", PROJECT] + (["--env-file", str(ENV_PATH)] if ENV_PATH else [])
+ ["-f", "compose.yaml", "-f", "compose.faults.yaml"] + list(args), capture=True, timeout=300)
def cmd_pipeline_faults(_args):
"""Внедрённые отказы pipeline. Worker пересоздаётся с compose.faults.yaml
(маркеры в filename, короткая аренда), поднимаются api-fault-publish/api-fault-dup;
тесты идут внутри сети (нужен доступ к PostgreSQL для журнала job_events).
В конце worker возвращается к обычной конфигурации."""
log("pipeline-faults: recreating worker with compose.faults.yaml")
faults_compose("up", "-d", "--no-deps", "--force-recreate", "worker")
wait_service_healthy("worker", 90)
prof = compose_cmd("--profile", "faults")
fault_api = ["api-fault-publish", "api-fault-dup"]
run(prof + ["up", "-d", "--no-recreate"] + fault_api, show=True, timeout=300)
mount = "%s:/app/student-tests:ro" % (ROOT / "student-tests")
# Тест читает публичную фикстуру tests/fixtures/sample.png; каталога tests/
# в образе нет (Dockerfile копирует только src и migrations) -> монтируем ro.
fixtures = "%s:/app/tests/fixtures:ro" % (ROOT / "tests" / "fixtures")
base = compose_cmd("run", "--rm", "-T", "--no-deps", "-v", mount, "-v", fixtures,
"-e", "BASE_URL=http://lb:8080",
"-e", "FAULT_PUBLISH_URL=http://api-fault-publish:8080",
"-e", "FAULT_DUP_URL=http://api-fault-dup:8080")
try:
for name in fault_api:
run(base + ["api", "python", "student-tests/wait_url.py", "http://%s:8080/health/ready" % name],
show=True, timeout=120)
run(base + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests",
"-p", "test_pipeline_faults.py", "-v"], show=True, timeout=900)
finally:
run(prof + ["rm", "-sf"] + fault_api, check=False, timeout=120)
compose("up", "-d", "--no-deps", "--force-recreate", "worker", timeout=180)
wait_service_healthy("worker", 90)
log("pipeline-faults: worker restored to normal configuration")
def wait_service_healthy(service, timeout):
def ok():
ids = run(["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT,
"--filter", "label=com.docker.compose.service=" + service], capture=True).stdout.split()
if not ids:
return False
out = run(["docker", "inspect", "-f", "{{.State.Health.Status}}", ids[0]], capture=True).stdout.strip()
return out == "healthy"
wait_until(ok, timeout, service + " healthy")
def s3_warmup(timeout=70):
"""One-off контейнер образа API в сетях стенда: put/get/delete пробы + чтение старого объекта."""
out = run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.s3_bootstrap",
"--wait-ready", str(timeout)), capture=True, timeout=timeout + 60)
lines = [x for x in (out.stdout or "").splitlines() if x.startswith("{")]
return lines[-1] if lines else ""
def cmd_wait_s3(_args):
wait_service_healthy("s3", 60)
print(s3_warmup(90))
def cmd_s3_check(_args):
"""Проверка приватного бакета ВНЕ API: анонимные запросы к S3 из сети storage."""
run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.s3_bootstrap",
"--check-only", "--sample-from-db"), show=True, timeout=180)
def cmd_api_fs(_args):
"""Доказательство «на API нет локального хранилища оригиналов»."""
for c in api_containers():
info = json.loads(run(["docker", "inspect", c["id"]], capture=True).stdout)[0]
mounts = [m.get("Type") + ":" + m.get("Destination", "") for m in info.get("Mounts", [])]
print("%s ReadonlyRootfs=%s Tmpfs=%s Mounts=%s" % (
c["name"], info["HostConfig"].get("ReadonlyRootfs"), info["HostConfig"].get("Tmpfs"), mounts or "none"))
def cmd_reconcile(args):
run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.reconcile") + list(args),
show=True, timeout=600)
def container_started(service):
ids = run(["docker", "ps", "-aq", "--filter", "label=com.docker.compose.project=" + PROJECT,
"--filter", "label=com.docker.compose.service=" + service], capture=True).stdout.split()
if not ids:
return "-"
return run(["docker", "inspect", "-f", "{{.State.StartedAt}}", ids[0]], capture=True).stdout.strip()[:23]
def cmd_rolling_update(args):
"""Поочерёдная замена реплик: +1 новая -> healthy и UP в LB -> graceful stop старой.
В любой момент в ротации >= N реплик. Миграции — один раз, до замены, и только
расширяющие (expand): старая версия продолжает работать на новой схеме.
"""
t_all = time.monotonic()
if [c for c in api_containers() if c["state"] != "running"]:
raise SystemExit("some api containers are stopped; run `python scripts/ops.py restore` first")
old = running()
n = len(old)
if n < 1:
raise SystemExit("no running api replicas")
log("rolling-update: start; replicas=%d old=%s" % (n, [c["short"] for c in old]))
if "--no-build" not in args:
t = time.monotonic()
compose("build", "api", "migrate", timeout=900)
log("rolling-update: images built in %.1fs (tag %s)" % (time.monotonic() - t, setting("APP_VERSION", "lab5")))
t = time.monotonic()
out = compose("run", "--rm", "-T", "migrate", timeout=300).stdout.strip().splitlines()
log("rolling-update: migrations once in %.1fs: %s" % (time.monotonic() - t, out[-1] if out else ""))
for i, victim in enumerate(old, 1):
before = {c["id"] for c in api_containers()}
t = time.monotonic()
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=%d" % (n + 1), "api", timeout=180)
new = [c for c in api_containers() if c["id"] not in before]
if len(new) != 1:
raise SystemExit("expected exactly one new container, got %d" % len(new))
fresh = new[0]
wait_healthy([fresh["id"]], timeout=90)
wait_until(lambda: lb_up_count() >= n + 1, 30, "new replica UP in HAProxy")
t_ready = time.monotonic() - t
t = time.monotonic()
run(["docker", "stop", "--time", "20", victim["id"]], timeout=40)
run(["docker", "rm", victim["id"]])
log("rolling-update: step %d/%d new=%s (%s, ready+UP in %.1fs) replaced old=%s (graceful stop %.1fs)"
% (i, n, fresh["short"], fresh["image"], t_ready, victim["short"], time.monotonic() - t))
wait_stack(n)
log("rolling-update: done in %.1fs; replicas now %s" % (time.monotonic() - t_all, [c["short"] for c in running()]))
def cmd_in_network(args):
"""Тесты, которым нужен прямой доступ к PostgreSQL/Redis или экземпляр api-short-ttl.
Запускаются в one-off контейнере образа API внутри сети backend: порты БД и
реплик наружу не публикуются. One-off контейнер не получает DNS-имя `api`,
поэтому LB его не видит.
"""
which = args[0] if args else ""
mount = "%s:/app/student-tests:ro" % (ROOT / "student-tests")
base = compose_cmd("run", "--rm", "-T", "--no-deps", "-v", mount)
if which == "storage":
for pattern in ("test_storage_*.py", "test_session_cache.py"):
run(base + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests", "-p", pattern, "-v"],
show=True, timeout=600)
elif which == "consistency":
prof = compose_cmd("--profile", "faults")
faults = ["api-fault-s3", "api-fault-db", "api-fault-crash"]
run(prof + ["up", "-d", "--no-recreate"] + faults, show=True, timeout=300)
env = ["-e", "BASE_URL=http://lb:8080",
"-e", "FAULT_S3_URL=http://api-fault-s3:8080",
"-e", "FAULT_DB_URL=http://api-fault-db:8080",
"-e", "FAULT_CRASH_URL=http://api-fault-crash:8080"]
try:
for name in faults:
run(base + env + ["api", "python", "student-tests/wait_url.py", "http://%s:8080/health/ready" % name],
show=True, timeout=120)
run(base + env + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests",
"-p", "test_attachment_consistency.py", "-v"], show=True, timeout=600)
finally:
run(prof + ["rm", "-sf"] + faults, check=False, timeout=120)
elif which == "expiry":
prof = compose_cmd("--profile", "ttl")
run(prof + ["up", "-d", "--no-recreate", "api-short-ttl"], show=True, timeout=300)
env = ["-e", "EXPIRY_BASE_URL=http://api-short-ttl:8080", "-e", "BASE_URL=http://lb:8080"]
try:
run(base + env + ["api", "python", "student-tests/wait_url.py", "http://api-short-ttl:8080/health/ready"],
show=True, timeout=120)
run(base + env + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests",
"-p", "test_expiry.py", "-v"], show=True, timeout=300)
finally:
run(prof + ["rm", "-sf", "api-short-ttl"], check=False, timeout=120)
else:
raise SystemExit("usage: python scripts/ops.py in-network storage|expiry|consistency")
COMMANDS = {
"wait": cmd_wait,
"ps": cmd_ps,
"stop-one": cmd_stop_one,
"restore": cmd_restore,
"restart": cmd_restart,
"rolling-update": cmd_rolling_update,
"in-network": cmd_in_network,
"wait-s3": cmd_wait_s3,
"s3-check": cmd_s3_check,
"reconcile": cmd_reconcile,
"api-fs": cmd_api_fs,
"worker-stop": cmd_worker_stop,
"worker-kill": cmd_worker_kill,
"worker-start": cmd_worker_start,
"api-restart": cmd_api_restart,
"broker-restart": cmd_broker_restart,
"jobs": cmd_jobs,
"pipeline-faults": cmd_pipeline_faults,
}
def main():
if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS:
print(__doc__)
sys.exit(2)
COMMANDS[sys.argv[1]](sys.argv[2:])
if __name__ == "__main__":
main()