474 lines
22 KiB
Python
474 lines
22 KiB
Python
#!/usr/bin/env python3
|
||
"""Операции над compose-стендом lab4. Python 3.9+, только стандартная библиотека.
|
||
|
||
Работает одинаково в Windows (PowerShell/cmd), Linux и macOS: вызывает
|
||
`docker compose` и `docker` без shell-интерполяции. Секреты не печатает.
|
||
|
||
python scripts/ops.py wait # дождаться N здоровых реплик и LB
|
||
python scripts/ops.py ps # реплики: контейнер, X-Instance-Id, health, статус в LB
|
||
python scripts/ops.py stop-one # TARGET_INSTANCE_ID=<X-Instance-Id>; оставляет остановленной
|
||
python scripts/ops.py restore # запустить остановленные реплики и дождаться UP в LB
|
||
python scripts/ops.py restart # hook persistence: API + PostgreSQL + Redis + S3 + LB, volumes целы
|
||
python scripts/ops.py wait-s3 # дождаться, пока S3 реально читает/пишет (прогрев после рестарта)
|
||
python scripts/ops.py s3-check # приватность бакета вне API, права ключей, закрытые админ-порты
|
||
python scripts/ops.py reconcile [--apply] # сверка БД <-> S3 (по умолчанию dry-run)
|
||
python scripts/ops.py api-fs # у API read-only rootfs, только tmpfs, без volumes
|
||
python scripts/ops.py rolling-update # поочерёдная замена реплик без остановки сервиса
|
||
python scripts/ops.py in-network storage # тесты с прямым доступом к БД — внутри compose-сети
|
||
python scripts/ops.py in-network expiry # истечение сессии на api-short-ttl (TTL 2 с)
|
||
python scripts/ops.py in-network consistency # БД<->S3: сборщик, сироты, обрыв загрузки, внедрённые отказы
|
||
|
||
Переменные окружения:
|
||
COMPOSE_PROJECT_NAME имя проекта (по умолчанию messenger-lab4, как `name:` в compose.yaml)
|
||
ENV_FILE файл переменных (по умолчанию .env, если существует)
|
||
API_REPLICAS ожидаемое число реплик (по умолчанию из .env, иначе 2)
|
||
APP_PORT, LB_STATS_PORT опубликованные на 127.0.0.1 порты LB (8080, 8404)
|
||
STOP_MODE stop-one: stop (SIGTERM, graceful, по умолчанию) | kill (SIGKILL, «падение»)
|
||
|
||
Каждое действие дописывается в evidence/ops.log (время, контейнер, результат) —
|
||
это доказательство того, какой контейнер реально остановлен/заменён.
|
||
"""
|
||
import csv
|
||
import io
|
||
import json
|
||
import os
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import urllib.error
|
||
import urllib.request
|
||
from datetime import datetime, timezone
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
OPS_LOG = ROOT / "evidence" / "ops.log"
|
||
|
||
try: # консоль Windows может быть не в UTF-8
|
||
sys.stdout.reconfigure(errors="replace")
|
||
sys.stderr.reconfigure(errors="replace")
|
||
except AttributeError:
|
||
pass
|
||
|
||
|
||
# ---------------------------------------------------------------- settings
|
||
|
||
def _dotenv():
|
||
path = Path(os.environ.get("ENV_FILE", str(ROOT / ".env")))
|
||
values = {}
|
||
if path.is_file():
|
||
for line in path.read_text(encoding="utf-8").splitlines():
|
||
line = line.strip()
|
||
if line and not line.startswith("#") and "=" in line:
|
||
k, v = line.split("=", 1)
|
||
values[k.strip()] = v.strip().strip('"').strip("'")
|
||
return path if path.is_file() else None, values
|
||
|
||
|
||
ENV_PATH, DOTENV = _dotenv()
|
||
|
||
|
||
def setting(name, default):
|
||
return os.environ.get(name) or DOTENV.get(name) or default
|
||
|
||
|
||
PROJECT = setting("COMPOSE_PROJECT_NAME", "messenger-lab4")
|
||
REPLICAS = int(setting("API_REPLICAS", "2"))
|
||
APP_PORT = int(setting("APP_PORT", "8080"))
|
||
STATS_PORT = int(setting("LB_STATS_PORT", "8404"))
|
||
LB_URL = "http://127.0.0.1:%d" % APP_PORT
|
||
STATS_URL = "http://127.0.0.1:%d/stats;csv" % STATS_PORT
|
||
|
||
|
||
def now_iso():
|
||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
||
|
||
|
||
def log(msg):
|
||
line = "%s %s" % (now_iso(), msg)
|
||
print(line, flush=True)
|
||
try:
|
||
OPS_LOG.parent.mkdir(exist_ok=True)
|
||
with OPS_LOG.open("a", encoding="utf-8") as f:
|
||
f.write(line + "\n")
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
# ---------------------------------------------------------------- docker
|
||
|
||
def compose_cmd(*args):
|
||
cmd = ["docker", "compose", "-p", PROJECT]
|
||
if ENV_PATH:
|
||
cmd += ["--env-file", str(ENV_PATH)]
|
||
return cmd + list(args)
|
||
|
||
|
||
def run(cmd, check=True, capture=False, timeout=300, show=False):
|
||
"""capture: вернуть вывод; show: показать вывод в консоли; иначе — тихо."""
|
||
out = None if show else (subprocess.PIPE if capture else subprocess.DEVNULL)
|
||
result = subprocess.run(
|
||
cmd, cwd=ROOT, check=False, timeout=timeout,
|
||
stdout=out, stderr=out, text=True,
|
||
)
|
||
if check and result.returncode != 0:
|
||
detail = (result.stderr or "").strip()[-400:] if capture else ""
|
||
raise SystemExit("command failed (%d): %s %s" % (result.returncode, " ".join(cmd[:4]), detail))
|
||
return result
|
||
|
||
|
||
def compose(*args, timeout=300):
|
||
return run(compose_cmd(*args), timeout=timeout, capture=True)
|
||
|
||
|
||
def api_containers(include_stopped=True):
|
||
"""[{id, short, name, state, health, image, ip}] — только сервис api, без one-off."""
|
||
cmd = ["docker", "ps", "--no-trunc", "--format", "{{.ID}}",
|
||
"--filter", "label=com.docker.compose.project=" + PROJECT,
|
||
"--filter", "label=com.docker.compose.service=api",
|
||
"--filter", "label=com.docker.compose.oneoff=False"]
|
||
if include_stopped:
|
||
cmd.insert(2, "-a")
|
||
ids = [x for x in run(cmd, capture=True).stdout.split() if x]
|
||
if not ids:
|
||
return []
|
||
raw = run(["docker", "inspect"] + ids, capture=True).stdout
|
||
result = []
|
||
for item in json.loads(raw):
|
||
state = item["State"]
|
||
nets = item.get("NetworkSettings", {}).get("Networks") or {}
|
||
ips = [n.get("IPAddress") for n in nets.values() if n.get("IPAddress")]
|
||
result.append({
|
||
"id": item["Id"],
|
||
"short": item["Id"][:12],
|
||
"hostname": item["Config"].get("Hostname", ""),
|
||
"name": item["Name"].lstrip("/"),
|
||
"state": state.get("Status"),
|
||
"health": (state.get("Health") or {}).get("Status", "-"),
|
||
"image": item["Config"].get("Image"),
|
||
"ip": ips[0] if ips else "",
|
||
"created": item.get("Created", "")[:19],
|
||
})
|
||
result.sort(key=lambda c: c["name"])
|
||
return result
|
||
|
||
|
||
def running(containers=None):
|
||
return [c for c in (containers if containers is not None else api_containers()) if c["state"] == "running"]
|
||
|
||
|
||
# ---------------------------------------------------------------- load balancer
|
||
|
||
def lb_servers():
|
||
"""[{name, status, addr}] backend api из CSV-статистики HAProxy (слоты с адресом)."""
|
||
data = urllib.request.urlopen(STATS_URL, timeout=2).read().decode("utf-8", "replace")
|
||
rows = csv.DictReader(io.StringIO(data.lstrip("# ")))
|
||
servers = []
|
||
for r in rows:
|
||
if r.get("pxname") != "api" or r.get("svname") in ("BACKEND", "FRONTEND"):
|
||
continue
|
||
addr = r.get("addr") or ""
|
||
status = r.get("status", "")
|
||
if status.startswith("MAINT") and not addr:
|
||
continue # пустой слот server-template
|
||
servers.append({"name": r["svname"], "status": status, "addr": addr})
|
||
return servers
|
||
|
||
|
||
def lb_up_count():
|
||
return sum(1 for s in lb_servers() if s["status"].startswith("UP"))
|
||
|
||
|
||
def lb_ready():
|
||
try:
|
||
with urllib.request.urlopen(LB_URL + "/health/ready", timeout=2) as r:
|
||
return r.status == 200
|
||
except (urllib.error.URLError, OSError):
|
||
return False
|
||
|
||
|
||
def wait_until(predicate, timeout, what, interval=0.3):
|
||
deadline = time.monotonic() + timeout
|
||
last_exc = None
|
||
while time.monotonic() < deadline:
|
||
try:
|
||
if predicate():
|
||
return True
|
||
except (urllib.error.URLError, OSError, ValueError, KeyError) as exc:
|
||
last_exc = exc
|
||
time.sleep(interval)
|
||
raise SystemExit("timeout after %ss waiting for %s%s" % (
|
||
timeout, what, (" (%s)" % type(last_exc).__name__) if last_exc else ""))
|
||
|
||
|
||
def wait_healthy(ids, timeout=90):
|
||
def ok():
|
||
states = {c["id"]: c for c in api_containers()}
|
||
return all(states.get(i, {}).get("health") == "healthy" for i in ids)
|
||
wait_until(ok, timeout, "containers healthy: " + ", ".join(i[:12] for i in ids))
|
||
|
||
|
||
def wait_stack(replicas, timeout=120):
|
||
wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= replicas,
|
||
timeout, "%d healthy api replicas" % replicas)
|
||
wait_until(lambda: lb_up_count() >= replicas, 30, "%d replicas UP in HAProxy" % replicas)
|
||
wait_until(lb_ready, 30, "LB /health/ready = 200")
|
||
|
||
|
||
# ---------------------------------------------------------------- commands
|
||
|
||
def cmd_wait(_args):
|
||
t0 = time.monotonic()
|
||
wait_stack(REPLICAS)
|
||
print("ready: %d api replicas healthy and UP in LB (%.1fs)" % (REPLICAS, time.monotonic() - t0))
|
||
|
||
|
||
def cmd_ps(_args):
|
||
containers = api_containers()
|
||
try:
|
||
by_addr = {s["addr"].rsplit(":", 1)[0]: s for s in lb_servers()}
|
||
except (urllib.error.URLError, OSError):
|
||
by_addr = {}
|
||
print("%-14s %-28s %-9s %-10s %-22s %-6s %s" % ("X-Instance-Id", "container", "state", "health", "image", "lb", "lb_status"))
|
||
for c in containers:
|
||
s = by_addr.get(c["ip"], {})
|
||
print("%-14s %-28s %-9s %-10s %-22s %-6s %s" % (
|
||
c["hostname"], c["name"], c["state"], c["health"], c["image"], s.get("name", "-"), s.get("status", "-")))
|
||
|
||
|
||
def find_target(target):
|
||
for c in api_containers():
|
||
# X-Instance-Id = hostname контейнера = первые 12 символов его ID.
|
||
if target and (c["hostname"] == target or c["id"].startswith(target)):
|
||
return c
|
||
return None
|
||
|
||
|
||
def cmd_stop_one(_args):
|
||
target = os.environ.get("TARGET_INSTANCE_ID", "").strip()
|
||
if not target:
|
||
raise SystemExit("TARGET_INSTANCE_ID is required (X-Instance-Id of the replica to stop)")
|
||
mode = os.environ.get("STOP_MODE", "stop").strip().lower()
|
||
c = find_target(target)
|
||
if c is None:
|
||
raise SystemExit("no api container of project %s matches X-Instance-Id %s" % (PROJECT, target))
|
||
if c["state"] != "running":
|
||
raise SystemExit("container %s is already %s" % (c["name"], c["state"]))
|
||
t0 = time.monotonic()
|
||
log("stop-one: mode=%s target=%s container=%s ip=%s" % (mode, target, c["name"], c["ip"]))
|
||
if mode == "kill":
|
||
# «Падение» процесса. Политику restart снимаем, чтобы Docker не поднял
|
||
# контейнер сам: тест требует, чтобы реплика оставалась остановленной.
|
||
run(["docker", "update", "--restart", "no", c["id"]])
|
||
run(["docker", "kill", "--signal", "KILL", c["id"]])
|
||
else:
|
||
# SIGTERM -> осушение 3 с (readiness 503) -> graceful stop; SIGKILL не позже 20 с.
|
||
run(["docker", "stop", "--time", "20", c["id"]], timeout=40)
|
||
after = find_target(target) or {}
|
||
log("stop-one: container=%s state=%s after %.2fs; left stopped (restore: python scripts/ops.py restore)"
|
||
% (c["name"], after.get("state", "removed"), time.monotonic() - t0))
|
||
|
||
|
||
def cmd_restore(_args):
|
||
t0 = time.monotonic()
|
||
stopped = [c for c in api_containers() if c["state"] != "running"]
|
||
for c in stopped: # вернуть политику, снятую в режиме kill
|
||
run(["docker", "update", "--restart", "unless-stopped", c["id"]], check=False)
|
||
# --no-recreate: не пересоздавать работающие; up запускает остановленные
|
||
# и добирает число контейнеров до deploy.replicas.
|
||
compose("up", "-d", "--no-deps", "--no-recreate", "api")
|
||
wait_stack(REPLICAS)
|
||
log("restore: started %s; %d replicas healthy and UP in LB after %.1fs"
|
||
% ([c["name"] for c in stopped] or "nothing", REPLICAS, time.monotonic() - t0))
|
||
|
||
|
||
def cmd_restart(_args):
|
||
"""Hook для make test-persistence: перезапуск API, ВСЕХ хранилищ (PostgreSQL,
|
||
Redis, S3) и LB.
|
||
|
||
Volumes не удаляются, данные не пересоздаются, bucket не создаётся заново.
|
||
Порядок: сначала LB и API (новые записи не принимаются), затем хранилища,
|
||
затем ожидание реальной готовности S3 к чтению/записи (SeaweedFS отвечает
|
||
/healthz сразу, но первые операции после рестарта ждут 15–30 с), затем API и LB.
|
||
"""
|
||
t0 = time.monotonic()
|
||
before = {c["name"]: c["short"] for c in running()}
|
||
started_before = {svc: container_started(svc) for svc in ("postgres", "redis", "s3")}
|
||
compose("stop", "lb", "api", timeout=120)
|
||
compose("restart", "postgres", "redis", "s3", timeout=120)
|
||
wait_service_healthy("s3", 60)
|
||
t_warm = time.monotonic()
|
||
warm = s3_warmup(70)
|
||
log("restart: s3 read/write ready %.1fs after container healthy (%s)" % (time.monotonic() - t_warm, warm))
|
||
compose("start", "api", timeout=120)
|
||
wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= REPLICAS,
|
||
80, "api healthy after restart")
|
||
compose("start", "lb", timeout=60)
|
||
log("restart: started_at postgres %s -> %s; redis %s -> %s; s3 %s -> %s; api %s restarted; lb restarted; %.1fs"
|
||
% (started_before["postgres"], container_started("postgres"), started_before["redis"],
|
||
container_started("redis"), started_before["s3"], container_started("s3"),
|
||
sorted(before), time.monotonic() - t0))
|
||
|
||
|
||
def wait_service_healthy(service, timeout):
|
||
def ok():
|
||
ids = run(["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT,
|
||
"--filter", "label=com.docker.compose.service=" + service], capture=True).stdout.split()
|
||
if not ids:
|
||
return False
|
||
out = run(["docker", "inspect", "-f", "{{.State.Health.Status}}", ids[0]], capture=True).stdout.strip()
|
||
return out == "healthy"
|
||
wait_until(ok, timeout, service + " healthy")
|
||
|
||
|
||
def s3_warmup(timeout=70):
|
||
"""One-off контейнер образа API в сетях стенда: put/get/delete пробы + чтение старого объекта."""
|
||
out = run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.s3_bootstrap",
|
||
"--wait-ready", str(timeout)), capture=True, timeout=timeout + 60)
|
||
lines = [x for x in (out.stdout or "").splitlines() if x.startswith("{")]
|
||
return lines[-1] if lines else ""
|
||
|
||
|
||
def cmd_wait_s3(_args):
|
||
wait_service_healthy("s3", 60)
|
||
print(s3_warmup(90))
|
||
|
||
|
||
def cmd_s3_check(_args):
|
||
"""Проверка приватного бакета ВНЕ API: анонимные запросы к S3 из сети storage."""
|
||
run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.s3_bootstrap",
|
||
"--check-only", "--sample-from-db"), show=True, timeout=180)
|
||
|
||
|
||
def cmd_api_fs(_args):
|
||
"""Доказательство «на API нет локального хранилища оригиналов»."""
|
||
for c in api_containers():
|
||
info = json.loads(run(["docker", "inspect", c["id"]], capture=True).stdout)[0]
|
||
mounts = [m.get("Type") + ":" + m.get("Destination", "") for m in info.get("Mounts", [])]
|
||
print("%s ReadonlyRootfs=%s Tmpfs=%s Mounts=%s" % (
|
||
c["name"], info["HostConfig"].get("ReadonlyRootfs"), info["HostConfig"].get("Tmpfs"), mounts or "none"))
|
||
|
||
|
||
def cmd_reconcile(args):
|
||
run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.reconcile") + list(args),
|
||
show=True, timeout=600)
|
||
|
||
|
||
def container_started(service):
|
||
ids = run(["docker", "ps", "-aq", "--filter", "label=com.docker.compose.project=" + PROJECT,
|
||
"--filter", "label=com.docker.compose.service=" + service], capture=True).stdout.split()
|
||
if not ids:
|
||
return "-"
|
||
return run(["docker", "inspect", "-f", "{{.State.StartedAt}}", ids[0]], capture=True).stdout.strip()[:23]
|
||
|
||
|
||
def cmd_rolling_update(args):
|
||
"""Поочерёдная замена реплик: +1 новая -> healthy и UP в LB -> graceful stop старой.
|
||
|
||
В любой момент в ротации >= N реплик. Миграции — один раз, до замены, и только
|
||
расширяющие (expand): старая версия продолжает работать на новой схеме.
|
||
"""
|
||
t_all = time.monotonic()
|
||
if [c for c in api_containers() if c["state"] != "running"]:
|
||
raise SystemExit("some api containers are stopped; run `python scripts/ops.py restore` first")
|
||
old = running()
|
||
n = len(old)
|
||
if n < 1:
|
||
raise SystemExit("no running api replicas")
|
||
log("rolling-update: start; replicas=%d old=%s" % (n, [c["short"] for c in old]))
|
||
if "--no-build" not in args:
|
||
t = time.monotonic()
|
||
compose("build", "api", "migrate", timeout=900)
|
||
log("rolling-update: images built in %.1fs (tag %s)" % (time.monotonic() - t, setting("APP_VERSION", "lab4")))
|
||
t = time.monotonic()
|
||
out = compose("run", "--rm", "-T", "migrate", timeout=300).stdout.strip().splitlines()
|
||
log("rolling-update: migrations once in %.1fs: %s" % (time.monotonic() - t, out[-1] if out else ""))
|
||
for i, victim in enumerate(old, 1):
|
||
before = {c["id"] for c in api_containers()}
|
||
t = time.monotonic()
|
||
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=%d" % (n + 1), "api", timeout=180)
|
||
new = [c for c in api_containers() if c["id"] not in before]
|
||
if len(new) != 1:
|
||
raise SystemExit("expected exactly one new container, got %d" % len(new))
|
||
fresh = new[0]
|
||
wait_healthy([fresh["id"]], timeout=90)
|
||
wait_until(lambda: lb_up_count() >= n + 1, 30, "new replica UP in HAProxy")
|
||
t_ready = time.monotonic() - t
|
||
t = time.monotonic()
|
||
run(["docker", "stop", "--time", "20", victim["id"]], timeout=40)
|
||
run(["docker", "rm", victim["id"]])
|
||
log("rolling-update: step %d/%d new=%s (%s, ready+UP in %.1fs) replaced old=%s (graceful stop %.1fs)"
|
||
% (i, n, fresh["short"], fresh["image"], t_ready, victim["short"], time.monotonic() - t))
|
||
wait_stack(n)
|
||
log("rolling-update: done in %.1fs; replicas now %s" % (time.monotonic() - t_all, [c["short"] for c in running()]))
|
||
|
||
|
||
def cmd_in_network(args):
|
||
"""Тесты, которым нужен прямой доступ к PostgreSQL/Redis или экземпляр api-short-ttl.
|
||
|
||
Запускаются в one-off контейнере образа API внутри сети backend: порты БД и
|
||
реплик наружу не публикуются. One-off контейнер не получает DNS-имя `api`,
|
||
поэтому LB его не видит.
|
||
"""
|
||
which = args[0] if args else ""
|
||
mount = "%s:/app/student-tests:ro" % (ROOT / "student-tests")
|
||
base = compose_cmd("run", "--rm", "-T", "--no-deps", "-v", mount)
|
||
if which == "storage":
|
||
for pattern in ("test_storage_*.py", "test_session_cache.py"):
|
||
run(base + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests", "-p", pattern, "-v"],
|
||
show=True, timeout=600)
|
||
elif which == "consistency":
|
||
prof = compose_cmd("--profile", "faults")
|
||
faults = ["api-fault-s3", "api-fault-db", "api-fault-crash"]
|
||
run(prof + ["up", "-d", "--no-recreate"] + faults, show=True, timeout=300)
|
||
env = ["-e", "BASE_URL=http://lb:8080",
|
||
"-e", "FAULT_S3_URL=http://api-fault-s3:8080",
|
||
"-e", "FAULT_DB_URL=http://api-fault-db:8080",
|
||
"-e", "FAULT_CRASH_URL=http://api-fault-crash:8080"]
|
||
try:
|
||
for name in faults:
|
||
run(base + env + ["api", "python", "student-tests/wait_url.py", "http://%s:8080/health/ready" % name],
|
||
show=True, timeout=120)
|
||
run(base + env + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests",
|
||
"-p", "test_attachment_consistency.py", "-v"], show=True, timeout=600)
|
||
finally:
|
||
run(prof + ["rm", "-sf"] + faults, check=False, timeout=120)
|
||
elif which == "expiry":
|
||
prof = compose_cmd("--profile", "ttl")
|
||
run(prof + ["up", "-d", "--no-recreate", "api-short-ttl"], show=True, timeout=300)
|
||
env = ["-e", "EXPIRY_BASE_URL=http://api-short-ttl:8080", "-e", "BASE_URL=http://lb:8080"]
|
||
try:
|
||
run(base + env + ["api", "python", "student-tests/wait_url.py", "http://api-short-ttl:8080/health/ready"],
|
||
show=True, timeout=120)
|
||
run(base + env + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests",
|
||
"-p", "test_expiry.py", "-v"], show=True, timeout=300)
|
||
finally:
|
||
run(prof + ["rm", "-sf", "api-short-ttl"], check=False, timeout=120)
|
||
else:
|
||
raise SystemExit("usage: python scripts/ops.py in-network storage|expiry|consistency")
|
||
|
||
|
||
COMMANDS = {
|
||
"wait": cmd_wait,
|
||
"ps": cmd_ps,
|
||
"stop-one": cmd_stop_one,
|
||
"restore": cmd_restore,
|
||
"restart": cmd_restart,
|
||
"rolling-update": cmd_rolling_update,
|
||
"in-network": cmd_in_network,
|
||
"wait-s3": cmd_wait_s3,
|
||
"s3-check": cmd_s3_check,
|
||
"reconcile": cmd_reconcile,
|
||
"api-fs": cmd_api_fs,
|
||
}
|
||
|
||
|
||
def main():
|
||
if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS:
|
||
print(__doc__)
|
||
sys.exit(2)
|
||
COMMANDS[sys.argv[1]](sys.argv[2:])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|