forked from mtusi2026/mtusi2026lab6
Лабораторная работа 6
This commit is contained in:
@@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lab6: журнал переходов alert'ов Prometheus (inactive -> pending -> firing -> resolved).
|
||||
|
||||
python scripts/alerts_watch.py --duration 600 > evidence/alerts_timeline.txt
|
||||
|
||||
Опрашивает http://127.0.0.1:9090/api/v1/alerts и печатает строку при каждом
|
||||
изменении состояния (UTC-время, имя, состояние, значение). «resolved» = alert
|
||||
исчез из списка активных. Только стандартная библиотека.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from urllib.request import urlopen
|
||||
|
||||
|
||||
def now():
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--prometheus", default="http://127.0.0.1:9090")
|
||||
p.add_argument("--duration", type=float, default=600)
|
||||
p.add_argument("--interval", type=float, default=5)
|
||||
args = p.parse_args()
|
||||
seen = {}
|
||||
end = time.monotonic() + args.duration
|
||||
print("%s watch start %s" % (now(), args.prometheus), flush=True)
|
||||
while time.monotonic() < end:
|
||||
try:
|
||||
data = json.load(urlopen(args.prometheus + "/api/v1/alerts", timeout=5))["data"]["alerts"]
|
||||
except Exception as exc:
|
||||
print("%s prometheus unavailable: %s" % (now(), type(exc).__name__), flush=True)
|
||||
time.sleep(args.interval)
|
||||
continue
|
||||
current = {a["labels"]["alertname"]: a for a in data}
|
||||
for name, a in current.items():
|
||||
if seen.get(name) != a["state"]:
|
||||
print("%s %-22s %-8s value=%s active_since=%s" % (now(), name, a["state"], a.get("value"),
|
||||
a.get("activeAt")), flush=True)
|
||||
seen[name] = a["state"]
|
||||
for name in [n for n in seen if n not in current]:
|
||||
print("%s %-22s resolved" % (now(), name), flush=True)
|
||||
del seen[name]
|
||||
time.sleep(args.interval)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,230 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Прогон всех проверок lab5 (и унаследованных lab2-4) на запущенном compose-стенде с записью в evidence/.
|
||||
|
||||
python scripts/collect_evidence.py # ~18-22 минуты
|
||||
python scripts/collect_evidence.py --quick # ~10-12 минут: без kill-серии, rolling update и нагрузки
|
||||
|
||||
Перед запуском: стенд поднят (`docker compose --env-file .env up -d --build`,
|
||||
`python scripts/ops.py wait`). Скрипт останавливает и восстанавливает реплики,
|
||||
перезапускает хранилища и S3 (volumes сохраняются), останавливает S3 и делает
|
||||
rolling update lab5 -> lab5.1 -> lab5 (откат), останавливает worker/broker, поэтому запускайте его только на учебном стенде.
|
||||
Токены и пароли в файлы не пишутся (тесты держат их в памяти).
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EV = ROOT / "evidence"
|
||||
PY = sys.executable or "python"
|
||||
WIN = os.name == "nt"
|
||||
HOOK_STOP = str(ROOT / "scripts" / ("stop-one.cmd" if WIN else "stop-one.sh"))
|
||||
HOOK_RESTART = str(ROOT / "scripts" / ("restart.cmd" if WIN else "restart.sh"))
|
||||
BASE = "http://127.0.0.1:" + os.environ.get("APP_PORT", "8080")
|
||||
RESULTS = []
|
||||
|
||||
try:
|
||||
sys.stdout.reconfigure(errors="replace")
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
|
||||
def stamp():
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||
|
||||
|
||||
def step(title, cmd, out_file, append=False, env=None, timeout=900):
|
||||
"""Выполнить команду, сохранить «$ команда + вывод» в evidence/<out_file>."""
|
||||
print("\n== %s\n$ %s" % (title, " ".join(cmd)), flush=True)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
proc = subprocess.run(cmd, cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
|
||||
timeout=timeout, env=dict(os.environ, **(env or {})),
|
||||
encoding="utf-8", errors="replace")
|
||||
output, rc = proc.stdout, proc.returncode
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
output, rc = (exc.stdout or "") + "\nTIMEOUT", 124
|
||||
elapsed = time.monotonic() - started
|
||||
shown = " ".join(c if c != PY else "python" for c in cmd).replace(str(ROOT) + os.sep, "")
|
||||
text = "# %s | %s | exit %d | %.1fs\n$ %s\n%s\n" % (stamp(), title, rc, elapsed, shown, output.rstrip())
|
||||
path = EV / out_file
|
||||
with path.open("a" if append else "w", encoding="utf-8") as f:
|
||||
f.write(text + ("\n" if append else ""))
|
||||
print(output.rstrip()[-1500:], flush=True)
|
||||
RESULTS.append((title, rc, out_file))
|
||||
return rc, output
|
||||
|
||||
|
||||
def instances_through_lb(n=20):
|
||||
"""Одна сессия, n запросов через LB -> какие реплики отвечали."""
|
||||
import base64
|
||||
import uuid
|
||||
name = "inst_" + uuid.uuid4().hex[:12]
|
||||
pw = "Inst-" + uuid.uuid4().hex
|
||||
body = json.dumps({"username": name, "display_name": "Inst", "password": pw}).encode()
|
||||
urlopen(Request(BASE + "/api/v1/users", data=body, headers={"Content-Type": "application/json"}), timeout=10)
|
||||
cred = base64.b64encode((name + ":" + pw).encode()).decode()
|
||||
sess = json.loads(urlopen(Request(BASE + "/api/v1/auth/sessions", data=b"", method="POST",
|
||||
headers={"Authorization": "Basic " + cred}), timeout=10).read())
|
||||
lines = []
|
||||
for i in range(n):
|
||||
with urlopen(Request(BASE + "/api/v1/users/me", headers={"Authorization": "Bearer " + sess["token"]}), timeout=5) as r:
|
||||
lines.append("%2d %s X-Instance-Id=%s X-Request-Id=%s" % (
|
||||
i + 1, r.status, r.headers.get("X-Instance-Id"), r.headers.get("X-Request-Id")))
|
||||
return lines
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--quick", action="store_true", help="без SIGKILL-серии, rolling update и нагрузки")
|
||||
args = ap.parse_args()
|
||||
EV.mkdir(exist_ok=True)
|
||||
(EV / "ops.log").write_text("", encoding="utf-8")
|
||||
dc = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
step("Контракт (offline)", [PY, "scripts/check_contract.py"], "check.txt")
|
||||
step("Compose config", dc + ["config", "--quiet"], "check.txt", append=True)
|
||||
step("Стенд: контейнеры", dc + ["ps", "-a"], "stand.txt")
|
||||
step("Стенд: реплики и статус в LB", [PY, "scripts/ops.py", "ps"], "stand.txt", append=True)
|
||||
step("Стенд: лимиты CPU/RAM", ["docker", "stats", "--no-stream", "--format",
|
||||
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"], "stand.txt", append=True)
|
||||
step("Стенд: опубликованные порты (только lb)", dc + ["ps", "--format", "table {{.Service}}\t{{.Ports}}"],
|
||||
"stand.txt", append=True)
|
||||
step("Образ SeaweedFS (закреплённая версия)", dc + ["images", "s3"], "stand.txt", append=True)
|
||||
step("lab5: broker (Redis AOF) и worker", dc + ["ps", "broker", "worker"], "stand.txt", append=True)
|
||||
step("lab5: конфигурация durability брокера", dc + ["exec", "-T", "broker", "sh", "-c",
|
||||
"REDISCLI_AUTH=\"$(tr -d '\\r\\n' < /run/secrets/broker_password)\" redis-cli CONFIG GET appendfsync; "
|
||||
"REDISCLI_AUTH=\"$(tr -d '\\r\\n' < /run/secrets/broker_password)\" redis-cli CONFIG GET maxmemory-policy"],
|
||||
"stand.txt", append=True)
|
||||
|
||||
# Bucket bootstrap: первый запуск был при up; повтор — no-op.
|
||||
step("S3 bootstrap: лог первого запуска s3-init", dc + ["logs", "--no-color", "s3-init"], "s3_bootstrap.txt")
|
||||
step("S3 bootstrap: повторный запуск (идемпотентность)", dc + ["run", "--rm", "-T", "s3-init"],
|
||||
"s3_bootstrap.txt", append=True)
|
||||
|
||||
lines = instances_through_lb()
|
||||
(EV / "instances.txt").write_text("# %s | одна сессия, 20 запросов GET /users/me через LB\n%s\nразных реплик: %d\n"
|
||||
% (stamp(), "\n".join(lines), len({l.split("X-Instance-Id=")[1].split()[0] for l in lines})),
|
||||
encoding="utf-8")
|
||||
print("\n".join(lines))
|
||||
|
||||
step("Публичный smoke", [PY, "tests/smoke.py", "--base-url", BASE], "smoke.txt")
|
||||
step("Собственные тесты (HTTP через LB, включая test_attachments.py)",
|
||||
[PY, "student-tests/run_all.py", "--base-url", BASE], "student-tests.txt")
|
||||
step("Тесты с доступом к БД (внутри compose-сети)", [PY, "scripts/ops.py", "in-network", "storage"],
|
||||
"student-tests.txt", append=True)
|
||||
step("Истечение сессии (api-short-ttl, TTL 2 с)", [PY, "scripts/ops.py", "in-network", "expiry"],
|
||||
"student-tests.txt", append=True)
|
||||
step("Согласованность БД <-> S3, обрыв загрузки, внедрённые отказы (api-fault-*)",
|
||||
[PY, "scripts/ops.py", "in-network", "consistency"], "consistency.txt")
|
||||
step("Сверка БД <-> S3 (dry-run)", [PY, "scripts/ops.py", "reconcile"], "reconcile.txt")
|
||||
step("Приватный бакет вне API, права ключей, закрытые админ-порты", [PY, "scripts/ops.py", "s3-check"],
|
||||
"s3_private.txt")
|
||||
step("На API нет хранилища оригиналов: read-only rootfs, только tmpfs, без volumes",
|
||||
[PY, "scripts/ops.py", "api-fs"], "s3_private.txt", append=True)
|
||||
step("API: файлы в /tmp (tmpfs) после загрузок",
|
||||
dc + ["exec", "-T", "api", "sh", "-c", "df -h /tmp; echo files_in_tmp=$(find /tmp -type f | wc -l)"],
|
||||
"s3_private.txt", append=True)
|
||||
|
||||
step("Пик памяти: 10 одновременных загрузок по 9.5 MiB",
|
||||
[PY, "student-tests/upload_memory.py", "--base-url", BASE, "--concurrency", "10", "--rounds", "3",
|
||||
"--docker", "--label", "c=10, MAX_CONCURRENT_UPLOADS=4", "--output", "evidence/upload_memory_c10.json"],
|
||||
"upload_memory.txt")
|
||||
step("Пик памяти: 20 одновременных загрузок",
|
||||
[PY, "student-tests/upload_memory.py", "--base-url", BASE, "--concurrency", "20", "--rounds", "2",
|
||||
"--docker", "--label", "c=20, MAX_CONCURRENT_UPLOADS=4", "--output", "evidence/upload_memory_c20.json"],
|
||||
"upload_memory.txt", append=True)
|
||||
|
||||
# Публичный тест останавливает реплику, обслужившую его первое сообщение.
|
||||
# Round-robin детерминирован, поэтому для второй реплики сдвигаем очередь
|
||||
# лишними запросами, пока цель не окажется другой (README: «повторить для второй»).
|
||||
targets = []
|
||||
for i in range(1, 5):
|
||||
if len(set(targets)) >= 2:
|
||||
break
|
||||
for _ in range(i - 1):
|
||||
urlopen(BASE + "/health/live", timeout=5).read()
|
||||
ops_log = EV / "ops.log"
|
||||
stops_before = sum(1 for l in ops_log.read_text(encoding="utf-8").splitlines()
|
||||
if "stop-one: mode=" in l) if ops_log.exists() else 0
|
||||
step("Failover #%d: реплики до" % i, [PY, "scripts/ops.py", "ps"], "failover.txt", append=i > 1)
|
||||
step("Failover #%d: make test-failover (вкл. скачивание файла через другую реплику)" % i,
|
||||
[PY, "tests/resilience.py", "--mode", "failover", "--action-script", HOOK_STOP, "--base-url", BASE],
|
||||
"failover.txt", append=True)
|
||||
step("Failover #%d: реплики после (остановлена именно цель теста)" % i, [PY, "scripts/ops.py", "ps"],
|
||||
"failover.txt", append=True)
|
||||
step("Failover #%d: restore" % i, [PY, "scripts/ops.py", "restore"], "failover.txt", append=True)
|
||||
stops = [l for l in ops_log.read_text(encoding="utf-8").splitlines() if "stop-one: mode=" in l] \
|
||||
if ops_log.exists() else []
|
||||
# Только остановка из ЭТОЙ попытки: если тест упал до hook'а, новой строки нет,
|
||||
# и цель прошлой попытки не должна засчитываться повторно.
|
||||
if len(stops) > stops_before:
|
||||
targets.append(stops[-1].split("target=")[1].split()[0])
|
||||
with (EV / "failover.txt").open("a", encoding="utf-8") as f:
|
||||
f.write("# остановленные публичным тестом реплики: %s\n" % targets)
|
||||
|
||||
step("Серия отказов: каждая реплика, graceful stop", [PY, "student-tests/failover_series.py", "--base-url", BASE,
|
||||
"--output", "evidence/failover_series.json"], "failover_series.txt")
|
||||
if not args.quick:
|
||||
step("Серия отказов: каждая реплика, SIGKILL", [PY, "student-tests/failover_series.py", "--base-url", BASE,
|
||||
"--mode", "kill", "--output", "evidence/failover_series_kill.json"],
|
||||
"failover_series.txt", append=True)
|
||||
|
||||
step("Недоступность S3 и восстановление", [PY, "student-tests/s3_outage.py", "--base-url", BASE,
|
||||
"--output", "evidence/s3_outage.json"], "s3_outage.txt")
|
||||
|
||||
# lab5: очередь и worker
|
||||
step("Варианты изображений по фактическим байтам (test_pipeline.py)",
|
||||
[PY, "-m", "unittest", "discover", "-s", "student-tests", "-p", "test_pipeline.py", "-v"],
|
||||
"pipeline.txt", env={"BASE_URL": BASE})
|
||||
step("Сценарий защиты 3: worker остановлен -> 202 -> рестарт API и broker -> worker -> ready",
|
||||
[PY, "student-tests/worker_outage.py", "--base-url", BASE, "--output", "evidence/worker_outage.json"],
|
||||
"worker_outage.txt")
|
||||
step("Уровень 4: внедрённые отказы pipeline (outbox, crash до/после ack, retry, DLQ, replay, lease)",
|
||||
[PY, "scripts/ops.py", "pipeline-faults"], "pipeline_faults.txt", timeout=1200)
|
||||
step("Очередь после сценариев: задания, outbox, DLQ, lag", [PY, "scripts/ops.py", "jobs", "stats"],
|
||||
"pipeline_faults.txt", append=True)
|
||||
step("Реестр failed (DLQ)", [PY, "scripts/ops.py", "jobs", "failed", "--limit", "10"], "pipeline_faults.txt",
|
||||
append=True)
|
||||
step("Worker: последние строки лога", dc + ["logs", "--no-color", "--tail", "80", "worker"], "worker_log_sample.txt")
|
||||
|
||||
if not args.quick:
|
||||
step("Rolling update lab5 -> lab5.1 под нагрузкой",
|
||||
[PY, "student-tests/rolling_update_load.py", "--base-url", BASE, "--output", "evidence/rolling_update.json"],
|
||||
"rolling_update.txt", env={"APP_VERSION": "lab5.1"})
|
||||
step("Rolling update обратно lab5.1 -> lab5 (откат) под нагрузкой",
|
||||
[PY, "student-tests/rolling_update_load.py", "--base-url", BASE, "--output", "evidence/rolling_back.json"],
|
||||
"rolling_update.txt", append=True, env={"APP_VERSION": "lab5"})
|
||||
step("Реплики после обновлений", [PY, "scripts/ops.py", "ps"], "rolling_update.txt", append=True)
|
||||
|
||||
fmt = ["ps", "-a", "--format", "table {{.Name}}\t{{.Status}}\t{{.RunningFor}}"]
|
||||
step("Persistence: контейнеры до", dc + fmt, "persistence.txt")
|
||||
step("Persistence: make test-persistence (API + worker + PostgreSQL + Redis + broker + S3 + LB)",
|
||||
[PY, "tests/resilience.py", "--mode", "persistence", "--action-script", HOOK_RESTART, "--base-url", BASE],
|
||||
"persistence.txt", append=True)
|
||||
step("Persistence: контейнеры после (Status = Up N seconds)", dc + fmt, "persistence.txt", append=True)
|
||||
step("Persistence: сверка БД <-> S3 после рестарта", [PY, "scripts/ops.py", "reconcile"], "persistence.txt",
|
||||
append=True)
|
||||
|
||||
if not args.quick:
|
||||
step("Нагрузка 30 с, concurrency 8", [PY, "tests/load.py", "--base-url", BASE, "--duration", "30",
|
||||
"--concurrency", "8", "--output", "evidence/load.json"], "load.txt")
|
||||
step("CPU/RAM после нагрузки", ["docker", "stats", "--no-stream", "--format",
|
||||
"table {{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}\t{{.MemPerc}}"], "load.txt", append=True)
|
||||
|
||||
step("Лог LB (последние строки, без секретов)", dc + ["logs", "--no-color", "--tail", "60", "lb"], "lb_log_sample.txt")
|
||||
|
||||
summary = ["# %s" % stamp()] + ["%-4s %-70s evidence/%s" % ("OK" if rc == 0 else "FAIL", t, f) for t, rc, f in RESULTS]
|
||||
(EV / "SUMMARY.txt").write_text("\n".join(summary) + "\n", encoding="utf-8")
|
||||
print("\n" + "\n".join(summary))
|
||||
sys.exit(0 if all(rc == 0 for _, rc, _ in RESULTS) else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,226 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lab6: контролируемый запрос и ограниченные отказы для проверки наблюдаемости.
|
||||
|
||||
Только стандартная библиотека; Docker управляется через scripts/ops.py / docker compose.
|
||||
Печатает JSON-строки таймлайна (ID, статусы, счётчики) — без токенов и паролей.
|
||||
|
||||
python scripts/incident.py probe [--filename photo.png]
|
||||
Один upload: печатает X-Request-Id, attachment_id и время до ready.
|
||||
Затем в Grafana -> Explore -> Tempo (TraceQL):
|
||||
{ span.messenger.request_id = "<X-Request-Id>" }
|
||||
и в Loki: {service_name=~"messenger-.+"} | request_id="<X-Request-Id>"
|
||||
|
||||
python scripts/incident.py worker-stop [--duration 150]
|
||||
Инцидент 1: worker остановлен, upload'ы продолжаются -> растут queue depth и
|
||||
возраст старейшего задания -> QueueLagGrowing firing; worker запущен ->
|
||||
очередь разбирается -> resolved.
|
||||
|
||||
python scripts/incident.py db-lock [--duration 180]
|
||||
Инцидент 2: «медленная БД». В PostgreSQL держится ACCESS EXCLUSIVE lock на
|
||||
таблице messages: запросы к сообщениям ждут до statement_timeout (5 с) и
|
||||
получают 503 -> ApiHighErrorRate firing. В trace — SQL span ~5 с с ошибкой
|
||||
QueryCanceled (а не долгий handler). Остальные маршруты работают.
|
||||
|
||||
python scripts/incident.py telemetry-down [--duration 90] [--services otel-collector]
|
||||
Инцидент 3: telemetry backend недоступен. Под нагрузкой останавливаются
|
||||
указанные сервисы; проверяется, что API отвечает, и RSS API не растёт.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ROOT = HERE.parent
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
import common # noqa: E402
|
||||
from imagegen import sample_png # noqa: E402
|
||||
from test_attachments import upload # noqa: E402
|
||||
|
||||
PY = sys.executable
|
||||
T0 = time.monotonic()
|
||||
|
||||
|
||||
def note(step, **kw):
|
||||
kw = dict(step=step, t=round(time.monotonic() - T0, 1), **kw)
|
||||
print(json.dumps(kw, ensure_ascii=False), flush=True)
|
||||
|
||||
|
||||
def sh(cmd, check=True):
|
||||
note("exec", cmd=cmd)
|
||||
return subprocess.run(shlex.split(cmd), check=check, stdout=subprocess.DEVNULL if check else None)
|
||||
|
||||
|
||||
def setup_chat():
|
||||
owner = common.actor("obs")
|
||||
chat = common.create_chat(owner)
|
||||
return owner, chat
|
||||
|
||||
|
||||
def one_upload(owner, chat, filename="photo.png", data=None):
|
||||
rid = "probe-" + uuid.uuid4().hex[:12]
|
||||
r = upload(owner["_token"], chat["id"], data or sample_png(), filename=filename,
|
||||
headers={"X-Request-Id": rid})
|
||||
return r, rid
|
||||
|
||||
|
||||
def cmd_probe(args):
|
||||
owner, chat = setup_chat()
|
||||
started = time.monotonic()
|
||||
r, rid = one_upload(owner, chat, args.filename)
|
||||
note("upload", status=r.status, request_id=rid, attachment_id=(r.json or {}).get("id"),
|
||||
instance=r.headers.get("X-Instance-Id"))
|
||||
if r.status != 202:
|
||||
return 1
|
||||
ready = common.wait_ready(owner["_token"], r.json["id"], timeout=args.timeout, allow_failed=True)
|
||||
note("final", status=(ready or {}).get("status"), seconds_to_final=round(time.monotonic() - started, 2))
|
||||
print("\nTempo (TraceQL): { span.messenger.request_id = \"%s\" }" % rid)
|
||||
print("Loki: {service_name=~\"messenger-.+\"} | request_id=\"%s\"" % rid)
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_worker_stop(args):
|
||||
owner, chat = setup_chat()
|
||||
sh(args.stop_worker)
|
||||
ids, end = [], time.monotonic() + args.duration
|
||||
while time.monotonic() < end:
|
||||
r, rid = one_upload(owner, chat)
|
||||
ids.append(r.json["id"] if r.status == 202 else None)
|
||||
note("upload_while_worker_down", status=r.status, request_id=rid)
|
||||
time.sleep(args.interval)
|
||||
sh(args.start_worker)
|
||||
started = time.monotonic()
|
||||
pending = [i for i in ids if i]
|
||||
while pending and time.monotonic() - started < 300:
|
||||
pending = [i for i in pending if common.request("GET", "/api/v1/attachments/" + i,
|
||||
token=owner["_token"]).json.get("status") != "ready"]
|
||||
time.sleep(2)
|
||||
note("drained", uploads=len(ids), not_ready=len(pending), drain_seconds=round(time.monotonic() - started, 1))
|
||||
return 0 if not pending else 1
|
||||
|
||||
|
||||
class Load:
|
||||
"""Постоянная нагрузка: POST сообщения + GET истории (closed loop, N потоков)."""
|
||||
|
||||
def __init__(self, owner, chat, threads):
|
||||
self.owner, self.chat, self.stop = owner, chat, threading.Event()
|
||||
self.counts, self.lock = {}, threading.Lock()
|
||||
self.threads = [threading.Thread(target=self.run, daemon=True) for _ in range(threads)]
|
||||
|
||||
def run(self):
|
||||
path = "/api/v1/chats/%s/messages" % self.chat["id"]
|
||||
while not self.stop.is_set():
|
||||
for method, payload in (("POST", {"text": "load"}), ("GET", None)):
|
||||
try:
|
||||
st = common.request(method, path, token=self.owner["_token"], payload=payload, timeout=15).status
|
||||
except Exception:
|
||||
st = "conn_error"
|
||||
key = "%s %s" % (method, st)
|
||||
with self.lock:
|
||||
self.counts[key] = self.counts.get(key, 0) + 1
|
||||
time.sleep(0.05)
|
||||
|
||||
def snapshot(self):
|
||||
with self.lock:
|
||||
snap, self.counts = self.counts, {}
|
||||
return snap
|
||||
|
||||
def __enter__(self):
|
||||
for t in self.threads:
|
||||
t.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
self.stop.set()
|
||||
|
||||
|
||||
def cmd_db_lock(args):
|
||||
owner, chat = setup_chat()
|
||||
lock_sql = "BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(%d); COMMIT;" % args.duration
|
||||
cmd = args.compose + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
||||
"-v", "ON_ERROR_STOP=1", "-c", lock_sql]
|
||||
with Load(owner, chat, args.threads) as load:
|
||||
time.sleep(args.warmup)
|
||||
note("baseline", **load.snapshot())
|
||||
note("db_lock_start", duration_s=args.duration)
|
||||
proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL)
|
||||
while proc.poll() is None:
|
||||
time.sleep(10)
|
||||
note("during_lock", **load.snapshot())
|
||||
note("db_lock_end", rc=proc.returncode)
|
||||
time.sleep(args.warmup)
|
||||
note("after", **load.snapshot())
|
||||
return 0
|
||||
|
||||
|
||||
def rss_of(service):
|
||||
out = subprocess.run(["docker", "compose", "-p", args_project(), "ps", "-q", service],
|
||||
capture_output=True, text=True).stdout.split()
|
||||
if not out:
|
||||
return {}
|
||||
stats = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"] + out,
|
||||
capture_output=True, text=True).stdout.strip().splitlines()
|
||||
return dict(line.split(" ", 1) for line in stats)
|
||||
|
||||
|
||||
def args_project():
|
||||
import os
|
||||
return os.environ.get("COMPOSE_PROJECT_NAME", "messenger-lab6")
|
||||
|
||||
|
||||
def cmd_telemetry_down(args):
|
||||
owner, chat = setup_chat()
|
||||
services = args.services.split(",")
|
||||
with Load(owner, chat, args.threads) as load:
|
||||
time.sleep(args.warmup)
|
||||
note("baseline", **load.snapshot(), memory=rss_of("api"))
|
||||
subprocess.run(args.compose + ["stop"] + services, check=True, stdout=subprocess.DEVNULL)
|
||||
note("telemetry_stopped", services=services)
|
||||
end = time.monotonic() + args.duration
|
||||
while time.monotonic() < end:
|
||||
time.sleep(15)
|
||||
note("telemetry_down", **load.snapshot(), memory=rss_of("api"))
|
||||
subprocess.run(args.compose + ["start"] + services, check=True, stdout=subprocess.DEVNULL)
|
||||
note("telemetry_started", services=services)
|
||||
time.sleep(args.warmup)
|
||||
note("after", **load.snapshot(), memory=rss_of("api"))
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
sub = p.add_subparsers(dest="cmd", required=True)
|
||||
pr = sub.add_parser("probe")
|
||||
pr.add_argument("--filename", default="photo.png")
|
||||
pr.add_argument("--timeout", type=float, default=60)
|
||||
ws = sub.add_parser("worker-stop")
|
||||
ws.add_argument("--duration", type=float, default=150)
|
||||
ws.add_argument("--interval", type=float, default=5)
|
||||
ws.add_argument("--stop-worker", default="%s scripts/ops.py worker-stop" % shlex.quote(PY))
|
||||
ws.add_argument("--start-worker", default="%s scripts/ops.py worker-start" % shlex.quote(PY))
|
||||
for name in ("db-lock", "telemetry-down"):
|
||||
sp = sub.add_parser(name)
|
||||
sp.add_argument("--duration", type=int, default=180 if name == "db-lock" else 90)
|
||||
sp.add_argument("--threads", type=int, default=4)
|
||||
sp.add_argument("--warmup", type=float, default=20)
|
||||
sp.add_argument("--compose", type=shlex.split, default=shlex.split("docker compose --env-file .env"))
|
||||
if name == "telemetry-down":
|
||||
sp.add_argument("--services", default="otel-collector")
|
||||
args = p.parse_args()
|
||||
common.BASE_URL = args.base_url.rstrip("/")
|
||||
import test_attachments
|
||||
test_attachments.BASE_URL = common.BASE_URL
|
||||
fn = {"probe": cmd_probe, "worker-stop": cmd_worker_stop, "db-lock": cmd_db_lock,
|
||||
"telemetry-down": cmd_telemetry_down}[args.cmd]
|
||||
sys.exit(fn(args))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,87 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Создаёт локальные учебные секреты в ./secrets (каталог в .gitignore).
|
||||
|
||||
lab5: broker_password и broker_url (Redis Streams с AOF).
|
||||
Повторный запуск ничего не перезаписывает: пароль PostgreSQL задаётся только
|
||||
при первой инициализации volume, и смена файла сломала бы подключение.
|
||||
Значения секретов не печатаются.
|
||||
|
||||
lab4: две пары S3-ключей — bootstrap-admin (только для one-shot s3-init) и
|
||||
messenger-api (Read/Write/List одного бакета). s3_config.json — identities
|
||||
SeaweedFS, собирается из этих ключей (пересоздаётся, если удалён: ключи S3 не
|
||||
привязаны к данным в volume, их можно сменить: удалить 5 файлов s3_* и
|
||||
перезапустить init_secrets.py + docker compose up -d --force-recreate s3 s3-init api).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DIR = ROOT / "secrets"
|
||||
|
||||
|
||||
def write(name: str, value: str) -> None:
|
||||
path = DIR / name
|
||||
if path.exists():
|
||||
print("keep secrets/%s" % name)
|
||||
return
|
||||
# newline="\n": на Windows иначе пишется CRLF, и `$(cat ...)` в контейнере Redis
|
||||
# получает пароль с лишним \r -> AUTH клиента не совпадает (lab5: брокер недоступен).
|
||||
with path.open("w", encoding="utf-8", newline="\n") as handle:
|
||||
handle.write(value + "\n")
|
||||
# Каталог 0700, файлы 0644: внутри контейнера их читают непривилегированные
|
||||
# пользователи (appuser, redis), а на хосте каталог закрыт.
|
||||
os.chmod(path, 0o644)
|
||||
print("create secrets/%s" % name)
|
||||
|
||||
|
||||
def read(name: str) -> str:
|
||||
return (DIR / name).read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
DIR.mkdir(exist_ok=True)
|
||||
try:
|
||||
os.chmod(DIR, 0o700)
|
||||
except OSError:
|
||||
pass
|
||||
write("postgres_password", secrets.token_hex(24))
|
||||
write("redis_password", secrets.token_hex(24))
|
||||
write("database_url", "postgresql://messenger:%s@postgres:5432/messenger" % read("postgres_password"))
|
||||
write("cache_url", "redis://:%s@redis:6379/0" % read("redis_password"))
|
||||
# lab5: брокер (отдельный Redis с AOF). URL содержит пароль — только в secrets/.
|
||||
write("broker_password", secrets.token_hex(24))
|
||||
write("broker_url", "redis://:%s@broker:6379/0" % read("broker_password"))
|
||||
# lab6: пароль администратора Grafana (логин admin). Посмотреть: cat secrets/grafana_admin_password
|
||||
write("grafana_admin_password", secrets.token_urlsafe(18))
|
||||
# lab4: S3 (SeaweedFS). Access key — ASCII 20 символов, secret — 40 (как у AWS).
|
||||
write("s3_admin_access_key_id", "ADM" + secrets.token_hex(8).upper() + "X")
|
||||
write("s3_admin_secret_access_key", secrets.token_urlsafe(30)[:40])
|
||||
write("s3_access_key_id", "API" + secrets.token_hex(8).upper() + "X")
|
||||
write("s3_secret_access_key", secrets.token_urlsafe(30)[:40])
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
config = {
|
||||
"identities": [
|
||||
{
|
||||
"name": "bootstrap_admin",
|
||||
"credentials": [{"accessKey": read("s3_admin_access_key_id"),
|
||||
"secretKey": read("s3_admin_secret_access_key")}],
|
||||
"actions": ["Admin", "Read", "List", "Tagging", "Write"],
|
||||
},
|
||||
{
|
||||
# Только свой бакет: чтение, запись/удаление объектов, листинг.
|
||||
# Создать/удалить бакет или прочитать чужой этими ключами нельзя.
|
||||
"name": "messenger_api",
|
||||
"credentials": [{"accessKey": read("s3_access_key_id"),
|
||||
"secretKey": read("s3_secret_access_key")}],
|
||||
"actions": ["Read:" + bucket, "Write:" + bucket, "List:" + bucket],
|
||||
},
|
||||
]
|
||||
# Identity "anonymous" не объявлена -> анонимные запросы получают 403.
|
||||
}
|
||||
write("s3_config.json", json.dumps(config, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+583
@@ -0,0 +1,583 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Операции над compose-стендом lab5 (база — lab4). Python 3.9+, только стандартная библиотека.
|
||||
|
||||
Работает одинаково в Windows (PowerShell/cmd), Linux и macOS: вызывает
|
||||
`docker compose` и `docker` без shell-интерполяции. Секреты не печатает.
|
||||
|
||||
python scripts/ops.py wait # дождаться N здоровых реплик и LB
|
||||
python scripts/ops.py ps # реплики: контейнер, X-Instance-Id, health, статус в LB
|
||||
python scripts/ops.py stop-one # TARGET_INSTANCE_ID=<X-Instance-Id>; оставляет остановленной
|
||||
python scripts/ops.py restore # запустить остановленные реплики и дождаться UP в LB
|
||||
python scripts/ops.py restart # hook persistence: API + worker + PostgreSQL + Redis + broker + S3 + LB, volumes целы
|
||||
python scripts/ops.py wait-s3 # дождаться, пока S3 реально читает/пишет (прогрев после рестарта)
|
||||
python scripts/ops.py s3-check # приватность бакета вне API, права ключей, закрытые админ-порты
|
||||
python scripts/ops.py reconcile [--apply] # сверка БД <-> S3 (по умолчанию dry-run)
|
||||
python scripts/ops.py api-fs # у API read-only rootfs, только tmpfs, без volumes
|
||||
python scripts/ops.py rolling-update # поочерёдная замена реплик без остановки сервиса
|
||||
python scripts/ops.py in-network storage # тесты с прямым доступом к БД — внутри compose-сети
|
||||
python scripts/ops.py in-network expiry # истечение сессии на api-short-ttl (TTL 2 с)
|
||||
python scripts/ops.py in-network consistency # БД<->S3: сборщик, сироты, обрыв загрузки, внедрённые отказы
|
||||
|
||||
lab5:
|
||||
python scripts/ops.py worker-stop # остановить ВСЕ контейнеры worker (docker stop, graceful)
|
||||
python scripts/ops.py worker-kill # SIGKILL worker (без перезапуска политикой restart)
|
||||
python scripts/ops.py worker-start # запустить worker и дождаться healthy
|
||||
python scripts/ops.py api-restart # перезапуск реплик API (volumes не трогаются)
|
||||
python scripts/ops.py broker-restart # перезапуск broker (AOF на volume brokerdata)
|
||||
python scripts/ops.py jobs stats|failed|show ID|replay ID... # src/jobs_admin.py в one-off контейнере
|
||||
python scripts/ops.py pipeline-faults # внедрённые отказы pipeline (outbox, crash до/после ack, retry, DLQ, replay, lease)
|
||||
|
||||
Переменные окружения:
|
||||
COMPOSE_PROJECT_NAME имя проекта (по умолчанию messenger-lab6, как `name:` в compose.yaml)
|
||||
ENV_FILE файл переменных (по умолчанию .env, если существует)
|
||||
API_REPLICAS ожидаемое число реплик (по умолчанию из .env, иначе 2)
|
||||
APP_PORT, LB_STATS_PORT опубликованные на 127.0.0.1 порты LB (8080, 8404)
|
||||
STOP_MODE stop-one: stop (SIGTERM, graceful, по умолчанию) | kill (SIGKILL, «падение»)
|
||||
|
||||
Каждое действие дописывается в evidence/ops.log (время, контейнер, результат) —
|
||||
это доказательство того, какой контейнер реально остановлен/заменён.
|
||||
"""
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OPS_LOG = ROOT / "evidence" / "ops.log"
|
||||
|
||||
try: # консоль Windows может быть не в UTF-8
|
||||
sys.stdout.reconfigure(errors="replace")
|
||||
sys.stderr.reconfigure(errors="replace")
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- settings
|
||||
|
||||
def _dotenv():
|
||||
path = Path(os.environ.get("ENV_FILE", str(ROOT / ".env")))
|
||||
values = {}
|
||||
if path.is_file():
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if line and not line.startswith("#") and "=" in line:
|
||||
k, v = line.split("=", 1)
|
||||
values[k.strip()] = v.strip().strip('"').strip("'")
|
||||
return path if path.is_file() else None, values
|
||||
|
||||
|
||||
ENV_PATH, DOTENV = _dotenv()
|
||||
|
||||
|
||||
def setting(name, default):
|
||||
return os.environ.get(name) or DOTENV.get(name) or default
|
||||
|
||||
|
||||
PROJECT = setting("COMPOSE_PROJECT_NAME", "messenger-lab6")
|
||||
REPLICAS = int(setting("API_REPLICAS", "2"))
|
||||
APP_PORT = int(setting("APP_PORT", "8080"))
|
||||
STATS_PORT = int(setting("LB_STATS_PORT", "8404"))
|
||||
LB_URL = "http://127.0.0.1:%d" % APP_PORT
|
||||
STATS_URL = "http://127.0.0.1:%d/stats;csv" % STATS_PORT
|
||||
|
||||
|
||||
def now_iso():
|
||||
return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
|
||||
|
||||
|
||||
def log(msg):
|
||||
line = "%s %s" % (now_iso(), msg)
|
||||
print(line, flush=True)
|
||||
try:
|
||||
OPS_LOG.parent.mkdir(exist_ok=True)
|
||||
with OPS_LOG.open("a", encoding="utf-8") as f:
|
||||
f.write(line + "\n")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- docker
|
||||
|
||||
def compose_cmd(*args):
|
||||
cmd = ["docker", "compose", "-p", PROJECT]
|
||||
if ENV_PATH:
|
||||
cmd += ["--env-file", str(ENV_PATH)]
|
||||
return cmd + list(args)
|
||||
|
||||
|
||||
def run(cmd, check=True, capture=False, timeout=300, show=False):
|
||||
"""capture: вернуть вывод; show: показать вывод в консоли; иначе — тихо."""
|
||||
out = None if show else (subprocess.PIPE if capture else subprocess.DEVNULL)
|
||||
result = subprocess.run(
|
||||
cmd, cwd=ROOT, check=False, timeout=timeout,
|
||||
stdout=out, stderr=out, text=True,
|
||||
)
|
||||
if check and result.returncode != 0:
|
||||
detail = (result.stderr or "").strip()[-400:] if capture else ""
|
||||
raise SystemExit("command failed (%d): %s %s" % (result.returncode, " ".join(cmd[:4]), detail))
|
||||
return result
|
||||
|
||||
|
||||
def compose(*args, timeout=300):
|
||||
return run(compose_cmd(*args), timeout=timeout, capture=True)
|
||||
|
||||
|
||||
def api_containers(include_stopped=True):
|
||||
"""[{id, short, name, state, health, image, ip}] — только сервис api, без one-off."""
|
||||
cmd = ["docker", "ps", "--no-trunc", "--format", "{{.ID}}",
|
||||
"--filter", "label=com.docker.compose.project=" + PROJECT,
|
||||
"--filter", "label=com.docker.compose.service=api",
|
||||
"--filter", "label=com.docker.compose.oneoff=False"]
|
||||
if include_stopped:
|
||||
cmd.insert(2, "-a")
|
||||
ids = [x for x in run(cmd, capture=True).stdout.split() if x]
|
||||
if not ids:
|
||||
return []
|
||||
raw = run(["docker", "inspect"] + ids, capture=True).stdout
|
||||
result = []
|
||||
for item in json.loads(raw):
|
||||
state = item["State"]
|
||||
nets = item.get("NetworkSettings", {}).get("Networks") or {}
|
||||
ips = [n.get("IPAddress") for n in nets.values() if n.get("IPAddress")]
|
||||
result.append({
|
||||
"id": item["Id"],
|
||||
"short": item["Id"][:12],
|
||||
"hostname": item["Config"].get("Hostname", ""),
|
||||
"name": item["Name"].lstrip("/"),
|
||||
"state": state.get("Status"),
|
||||
"health": (state.get("Health") or {}).get("Status", "-"),
|
||||
"image": item["Config"].get("Image"),
|
||||
"ip": ips[0] if ips else "",
|
||||
"created": item.get("Created", "")[:19],
|
||||
})
|
||||
result.sort(key=lambda c: c["name"])
|
||||
return result
|
||||
|
||||
|
||||
def running(containers=None):
|
||||
return [c for c in (containers if containers is not None else api_containers()) if c["state"] == "running"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- load balancer
|
||||
|
||||
def lb_servers():
|
||||
"""[{name, status, addr}] backend api из CSV-статистики HAProxy (слоты с адресом)."""
|
||||
data = urllib.request.urlopen(STATS_URL, timeout=2).read().decode("utf-8", "replace")
|
||||
rows = csv.DictReader(io.StringIO(data.lstrip("# ")))
|
||||
servers = []
|
||||
for r in rows:
|
||||
if r.get("pxname") != "api" or r.get("svname") in ("BACKEND", "FRONTEND"):
|
||||
continue
|
||||
addr = r.get("addr") or ""
|
||||
status = r.get("status", "")
|
||||
if status.startswith("MAINT") and not addr:
|
||||
continue # пустой слот server-template
|
||||
servers.append({"name": r["svname"], "status": status, "addr": addr})
|
||||
return servers
|
||||
|
||||
|
||||
def lb_up_count():
|
||||
return sum(1 for s in lb_servers() if s["status"].startswith("UP"))
|
||||
|
||||
|
||||
def lb_ready():
|
||||
try:
|
||||
with urllib.request.urlopen(LB_URL + "/health/ready", timeout=2) as r:
|
||||
return r.status == 200
|
||||
except (urllib.error.URLError, OSError):
|
||||
return False
|
||||
|
||||
|
||||
def wait_until(predicate, timeout, what, interval=0.3):
|
||||
deadline = time.monotonic() + timeout
|
||||
last_exc = None
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if predicate():
|
||||
return True
|
||||
except (urllib.error.URLError, OSError, ValueError, KeyError) as exc:
|
||||
last_exc = exc
|
||||
time.sleep(interval)
|
||||
raise SystemExit("timeout after %ss waiting for %s%s" % (
|
||||
timeout, what, (" (%s)" % type(last_exc).__name__) if last_exc else ""))
|
||||
|
||||
|
||||
def wait_healthy(ids, timeout=90):
|
||||
def ok():
|
||||
states = {c["id"]: c for c in api_containers()}
|
||||
return all(states.get(i, {}).get("health") == "healthy" for i in ids)
|
||||
wait_until(ok, timeout, "containers healthy: " + ", ".join(i[:12] for i in ids))
|
||||
|
||||
|
||||
def wait_stack(replicas, timeout=120):
|
||||
wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= replicas,
|
||||
timeout, "%d healthy api replicas" % replicas)
|
||||
wait_until(lambda: lb_up_count() >= replicas, 30, "%d replicas UP in HAProxy" % replicas)
|
||||
wait_until(lb_ready, 30, "LB /health/ready = 200")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- commands
|
||||
|
||||
def cmd_wait(_args):
|
||||
t0 = time.monotonic()
|
||||
wait_stack(REPLICAS)
|
||||
print("ready: %d api replicas healthy and UP in LB (%.1fs)" % (REPLICAS, time.monotonic() - t0))
|
||||
|
||||
|
||||
def cmd_ps(_args):
|
||||
containers = api_containers()
|
||||
try:
|
||||
by_addr = {s["addr"].rsplit(":", 1)[0]: s for s in lb_servers()}
|
||||
except (urllib.error.URLError, OSError):
|
||||
by_addr = {}
|
||||
print("%-14s %-28s %-9s %-10s %-22s %-6s %s" % ("X-Instance-Id", "container", "state", "health", "image", "lb", "lb_status"))
|
||||
for c in containers:
|
||||
s = by_addr.get(c["ip"], {})
|
||||
print("%-14s %-28s %-9s %-10s %-22s %-6s %s" % (
|
||||
c["hostname"], c["name"], c["state"], c["health"], c["image"], s.get("name", "-"), s.get("status", "-")))
|
||||
|
||||
|
||||
def find_target(target):
|
||||
for c in api_containers():
|
||||
# X-Instance-Id = hostname контейнера = первые 12 символов его ID.
|
||||
if target and (c["hostname"] == target or c["id"].startswith(target)):
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def cmd_stop_one(_args):
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "").strip()
|
||||
if not target:
|
||||
raise SystemExit("TARGET_INSTANCE_ID is required (X-Instance-Id of the replica to stop)")
|
||||
mode = os.environ.get("STOP_MODE", "stop").strip().lower()
|
||||
c = find_target(target)
|
||||
if c is None:
|
||||
raise SystemExit("no api container of project %s matches X-Instance-Id %s" % (PROJECT, target))
|
||||
if c["state"] != "running":
|
||||
raise SystemExit("container %s is already %s" % (c["name"], c["state"]))
|
||||
t0 = time.monotonic()
|
||||
log("stop-one: mode=%s target=%s container=%s ip=%s" % (mode, target, c["name"], c["ip"]))
|
||||
if mode == "kill":
|
||||
# «Падение» процесса. Политику restart снимаем, чтобы Docker не поднял
|
||||
# контейнер сам: тест требует, чтобы реплика оставалась остановленной.
|
||||
run(["docker", "update", "--restart", "no", c["id"]])
|
||||
run(["docker", "kill", "--signal", "KILL", c["id"]])
|
||||
else:
|
||||
# SIGTERM -> осушение 3 с (readiness 503) -> graceful stop; SIGKILL не позже 20 с.
|
||||
run(["docker", "stop", "--time", "20", c["id"]], timeout=40)
|
||||
after = find_target(target) or {}
|
||||
log("stop-one: container=%s state=%s after %.2fs; left stopped (restore: python scripts/ops.py restore)"
|
||||
% (c["name"], after.get("state", "removed"), time.monotonic() - t0))
|
||||
|
||||
|
||||
def cmd_restore(_args):
|
||||
t0 = time.monotonic()
|
||||
stopped = [c for c in api_containers() if c["state"] != "running"]
|
||||
for c in stopped: # вернуть политику, снятую в режиме kill
|
||||
run(["docker", "update", "--restart", "unless-stopped", c["id"]], check=False)
|
||||
# --no-recreate: не пересоздавать работающие; up запускает остановленные
|
||||
# и добирает число контейнеров до deploy.replicas.
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "api")
|
||||
wait_stack(REPLICAS)
|
||||
log("restore: started %s; %d replicas healthy and UP in LB after %.1fs"
|
||||
% ([c["name"] for c in stopped] or "nothing", REPLICAS, time.monotonic() - t0))
|
||||
|
||||
|
||||
def cmd_restart(_args):
|
||||
"""Hook для make test-persistence: перезапуск API, worker, ВСЕХ хранилищ
|
||||
(PostgreSQL, Redis, broker, S3) и LB.
|
||||
|
||||
Volumes не удаляются, данные не пересоздаются, bucket/поток не создаются заново.
|
||||
Порядок: сначала LB, API и worker (новые записи не принимаются, обработка
|
||||
останавливается graceful), затем хранилища и брокер, затем ожидание реальной
|
||||
готовности S3, затем API, worker и LB.
|
||||
"""
|
||||
t0 = time.monotonic()
|
||||
before = {c["name"]: c["short"] for c in running()}
|
||||
stores = ("postgres", "redis", "broker", "s3")
|
||||
started_before = {svc: container_started(svc) for svc in stores}
|
||||
compose("stop", "lb", "api", "worker", timeout=180)
|
||||
compose("restart", *stores, timeout=120)
|
||||
wait_service_healthy("s3", 60)
|
||||
wait_service_healthy("broker", 60)
|
||||
t_warm = time.monotonic()
|
||||
warm = s3_warmup(70)
|
||||
log("restart: s3 read/write ready %.1fs after container healthy (%s)" % (time.monotonic() - t_warm, warm))
|
||||
compose("start", "api", "worker", timeout=120)
|
||||
wait_until(lambda: len([c for c in running() if c["health"] == "healthy"]) >= REPLICAS,
|
||||
80, "api healthy after restart")
|
||||
compose("start", "lb", timeout=60)
|
||||
log("restart: started_at %s; api %s restarted; worker restarted; lb restarted; %.1fs"
|
||||
% ("; ".join("%s %s -> %s" % (svc, started_before[svc], container_started(svc)) for svc in stores),
|
||||
sorted(before), time.monotonic() - t0))
|
||||
|
||||
|
||||
def service_ids(service, include_stopped=False):
|
||||
cmd = ["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT,
|
||||
"--filter", "label=com.docker.compose.service=" + service,
|
||||
"--filter", "label=com.docker.compose.oneoff=False"]
|
||||
if include_stopped:
|
||||
cmd.insert(2, "-a")
|
||||
return run(cmd, capture=True).stdout.split()
|
||||
|
||||
|
||||
def cmd_worker_stop(_args):
|
||||
t0 = time.monotonic()
|
||||
ids = service_ids("worker")
|
||||
# SIGTERM: worker дорабатывает текущие сообщения (stop_grace_period 75 с).
|
||||
compose("stop", "worker", timeout=120)
|
||||
log("worker-stop: stopped %s in %.1fs" % ([i[:12] for i in ids], time.monotonic() - t0))
|
||||
|
||||
|
||||
def cmd_worker_kill(_args):
|
||||
ids = service_ids("worker")
|
||||
for i in ids:
|
||||
run(["docker", "update", "--restart", "no", i])
|
||||
run(["docker", "kill", "--signal", "KILL", i])
|
||||
log("worker-kill: SIGKILL %s (restart policy disabled until worker-start)" % [i[:12] for i in ids])
|
||||
|
||||
|
||||
def cmd_worker_start(_args):
|
||||
t0 = time.monotonic()
|
||||
for i in service_ids("worker", include_stopped=True):
|
||||
run(["docker", "update", "--restart", "unless-stopped", i], check=False)
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "worker", timeout=120)
|
||||
wait_service_healthy("worker", 90)
|
||||
log("worker-start: worker healthy after %.1fs" % (time.monotonic() - t0))
|
||||
|
||||
|
||||
def cmd_api_restart(_args):
|
||||
t0 = time.monotonic()
|
||||
before = [c["short"] for c in running()]
|
||||
compose("restart", "api", timeout=120)
|
||||
wait_stack(REPLICAS)
|
||||
log("api-restart: %s restarted, %d replicas UP after %.1fs" % (before, REPLICAS, time.monotonic() - t0))
|
||||
|
||||
|
||||
def cmd_broker_restart(_args):
|
||||
t0 = time.monotonic()
|
||||
before = container_started("broker")
|
||||
compose("restart", "broker", timeout=60)
|
||||
wait_service_healthy("broker", 60)
|
||||
log("broker-restart: started_at %s -> %s, healthy after %.1fs (volume brokerdata kept)"
|
||||
% (before, container_started("broker"), time.monotonic() - t0))
|
||||
|
||||
|
||||
def cmd_jobs(args):
|
||||
run(compose_cmd("run", "--rm", "-T", "--no-deps", "worker", "python", "-m", "src.jobs_admin") + list(args),
|
||||
show=True, timeout=300)
|
||||
|
||||
|
||||
def faults_compose(*args):
|
||||
return run(["docker", "compose", "-p", PROJECT] + (["--env-file", str(ENV_PATH)] if ENV_PATH else [])
|
||||
+ ["-f", "compose.yaml", "-f", "compose.faults.yaml"] + list(args), capture=True, timeout=300)
|
||||
|
||||
|
||||
def cmd_pipeline_faults(_args):
|
||||
"""Внедрённые отказы pipeline. Worker пересоздаётся с compose.faults.yaml
|
||||
(маркеры в filename, короткая аренда), поднимаются api-fault-publish/api-fault-dup;
|
||||
тесты идут внутри сети (нужен доступ к PostgreSQL для журнала job_events).
|
||||
В конце worker возвращается к обычной конфигурации."""
|
||||
log("pipeline-faults: recreating worker with compose.faults.yaml")
|
||||
faults_compose("up", "-d", "--no-deps", "--force-recreate", "worker")
|
||||
wait_service_healthy("worker", 90)
|
||||
prof = compose_cmd("--profile", "faults")
|
||||
fault_api = ["api-fault-publish", "api-fault-dup"]
|
||||
run(prof + ["up", "-d", "--no-recreate"] + fault_api, show=True, timeout=300)
|
||||
mount = "%s:/app/student-tests:ro" % (ROOT / "student-tests")
|
||||
# Тест читает публичную фикстуру tests/fixtures/sample.png; каталога tests/
|
||||
# в образе нет (Dockerfile копирует только src и migrations) -> монтируем ro.
|
||||
fixtures = "%s:/app/tests/fixtures:ro" % (ROOT / "tests" / "fixtures")
|
||||
base = compose_cmd("run", "--rm", "-T", "--no-deps", "-v", mount, "-v", fixtures,
|
||||
"-e", "BASE_URL=http://lb:8080",
|
||||
"-e", "FAULT_PUBLISH_URL=http://api-fault-publish:8080",
|
||||
"-e", "FAULT_DUP_URL=http://api-fault-dup:8080")
|
||||
try:
|
||||
for name in fault_api:
|
||||
run(base + ["api", "python", "student-tests/wait_url.py", "http://%s:8080/health/ready" % name],
|
||||
show=True, timeout=120)
|
||||
run(base + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests",
|
||||
"-p", "test_pipeline_faults.py", "-v"], show=True, timeout=900)
|
||||
finally:
|
||||
run(prof + ["rm", "-sf"] + fault_api, check=False, timeout=120)
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "worker", timeout=180)
|
||||
wait_service_healthy("worker", 90)
|
||||
log("pipeline-faults: worker restored to normal configuration")
|
||||
|
||||
|
||||
def wait_service_healthy(service, timeout):
|
||||
def ok():
|
||||
ids = run(["docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT,
|
||||
"--filter", "label=com.docker.compose.service=" + service], capture=True).stdout.split()
|
||||
if not ids:
|
||||
return False
|
||||
out = run(["docker", "inspect", "-f", "{{.State.Health.Status}}", ids[0]], capture=True).stdout.strip()
|
||||
return out == "healthy"
|
||||
wait_until(ok, timeout, service + " healthy")
|
||||
|
||||
|
||||
def s3_warmup(timeout=70):
|
||||
"""One-off контейнер образа API в сетях стенда: put/get/delete пробы + чтение старого объекта."""
|
||||
out = run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.s3_bootstrap",
|
||||
"--wait-ready", str(timeout)), capture=True, timeout=timeout + 60)
|
||||
lines = [x for x in (out.stdout or "").splitlines() if x.startswith("{")]
|
||||
return lines[-1] if lines else ""
|
||||
|
||||
|
||||
def cmd_wait_s3(_args):
|
||||
wait_service_healthy("s3", 60)
|
||||
print(s3_warmup(90))
|
||||
|
||||
|
||||
def cmd_s3_check(_args):
|
||||
"""Проверка приватного бакета ВНЕ API: анонимные запросы к S3 из сети storage."""
|
||||
run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.s3_bootstrap",
|
||||
"--check-only", "--sample-from-db"), show=True, timeout=180)
|
||||
|
||||
|
||||
def cmd_api_fs(_args):
|
||||
"""Доказательство «на API нет локального хранилища оригиналов»."""
|
||||
for c in api_containers():
|
||||
info = json.loads(run(["docker", "inspect", c["id"]], capture=True).stdout)[0]
|
||||
mounts = [m.get("Type") + ":" + m.get("Destination", "") for m in info.get("Mounts", [])]
|
||||
print("%s ReadonlyRootfs=%s Tmpfs=%s Mounts=%s" % (
|
||||
c["name"], info["HostConfig"].get("ReadonlyRootfs"), info["HostConfig"].get("Tmpfs"), mounts or "none"))
|
||||
|
||||
|
||||
def cmd_reconcile(args):
|
||||
run(compose_cmd("run", "--rm", "-T", "--no-deps", "api", "python", "-m", "src.reconcile") + list(args),
|
||||
show=True, timeout=600)
|
||||
|
||||
|
||||
def container_started(service):
|
||||
ids = run(["docker", "ps", "-aq", "--filter", "label=com.docker.compose.project=" + PROJECT,
|
||||
"--filter", "label=com.docker.compose.service=" + service], capture=True).stdout.split()
|
||||
if not ids:
|
||||
return "-"
|
||||
return run(["docker", "inspect", "-f", "{{.State.StartedAt}}", ids[0]], capture=True).stdout.strip()[:23]
|
||||
|
||||
|
||||
def cmd_rolling_update(args):
|
||||
"""Поочерёдная замена реплик: +1 новая -> healthy и UP в LB -> graceful stop старой.
|
||||
|
||||
В любой момент в ротации >= N реплик. Миграции — один раз, до замены, и только
|
||||
расширяющие (expand): старая версия продолжает работать на новой схеме.
|
||||
"""
|
||||
t_all = time.monotonic()
|
||||
if [c for c in api_containers() if c["state"] != "running"]:
|
||||
raise SystemExit("some api containers are stopped; run `python scripts/ops.py restore` first")
|
||||
old = running()
|
||||
n = len(old)
|
||||
if n < 1:
|
||||
raise SystemExit("no running api replicas")
|
||||
log("rolling-update: start; replicas=%d old=%s" % (n, [c["short"] for c in old]))
|
||||
if "--no-build" not in args:
|
||||
t = time.monotonic()
|
||||
compose("build", "api", "migrate", timeout=900)
|
||||
log("rolling-update: images built in %.1fs (tag %s)" % (time.monotonic() - t, setting("APP_VERSION", "lab5")))
|
||||
t = time.monotonic()
|
||||
out = compose("run", "--rm", "-T", "migrate", timeout=300).stdout.strip().splitlines()
|
||||
log("rolling-update: migrations once in %.1fs: %s" % (time.monotonic() - t, out[-1] if out else ""))
|
||||
for i, victim in enumerate(old, 1):
|
||||
before = {c["id"] for c in api_containers()}
|
||||
t = time.monotonic()
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", "api=%d" % (n + 1), "api", timeout=180)
|
||||
new = [c for c in api_containers() if c["id"] not in before]
|
||||
if len(new) != 1:
|
||||
raise SystemExit("expected exactly one new container, got %d" % len(new))
|
||||
fresh = new[0]
|
||||
wait_healthy([fresh["id"]], timeout=90)
|
||||
wait_until(lambda: lb_up_count() >= n + 1, 30, "new replica UP in HAProxy")
|
||||
t_ready = time.monotonic() - t
|
||||
t = time.monotonic()
|
||||
run(["docker", "stop", "--time", "20", victim["id"]], timeout=40)
|
||||
run(["docker", "rm", victim["id"]])
|
||||
log("rolling-update: step %d/%d new=%s (%s, ready+UP in %.1fs) replaced old=%s (graceful stop %.1fs)"
|
||||
% (i, n, fresh["short"], fresh["image"], t_ready, victim["short"], time.monotonic() - t))
|
||||
wait_stack(n)
|
||||
log("rolling-update: done in %.1fs; replicas now %s" % (time.monotonic() - t_all, [c["short"] for c in running()]))
|
||||
|
||||
|
||||
def cmd_in_network(args):
|
||||
"""Тесты, которым нужен прямой доступ к PostgreSQL/Redis или экземпляр api-short-ttl.
|
||||
|
||||
Запускаются в one-off контейнере образа API внутри сети backend: порты БД и
|
||||
реплик наружу не публикуются. One-off контейнер не получает DNS-имя `api`,
|
||||
поэтому LB его не видит.
|
||||
"""
|
||||
which = args[0] if args else ""
|
||||
mount = "%s:/app/student-tests:ro" % (ROOT / "student-tests")
|
||||
base = compose_cmd("run", "--rm", "-T", "--no-deps", "-v", mount)
|
||||
if which == "storage":
|
||||
for pattern in ("test_storage_*.py", "test_session_cache.py"):
|
||||
run(base + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests", "-p", pattern, "-v"],
|
||||
show=True, timeout=600)
|
||||
elif which == "consistency":
|
||||
prof = compose_cmd("--profile", "faults")
|
||||
faults = ["api-fault-s3", "api-fault-db", "api-fault-crash"]
|
||||
run(prof + ["up", "-d", "--no-recreate"] + faults, show=True, timeout=300)
|
||||
env = ["-e", "BASE_URL=http://lb:8080",
|
||||
"-e", "FAULT_S3_URL=http://api-fault-s3:8080",
|
||||
"-e", "FAULT_DB_URL=http://api-fault-db:8080",
|
||||
"-e", "FAULT_CRASH_URL=http://api-fault-crash:8080"]
|
||||
try:
|
||||
for name in faults:
|
||||
run(base + env + ["api", "python", "student-tests/wait_url.py", "http://%s:8080/health/ready" % name],
|
||||
show=True, timeout=120)
|
||||
run(base + env + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests",
|
||||
"-p", "test_attachment_consistency.py", "-v"], show=True, timeout=600)
|
||||
finally:
|
||||
run(prof + ["rm", "-sf"] + faults, check=False, timeout=120)
|
||||
elif which == "expiry":
|
||||
prof = compose_cmd("--profile", "ttl")
|
||||
run(prof + ["up", "-d", "--no-recreate", "api-short-ttl"], show=True, timeout=300)
|
||||
env = ["-e", "EXPIRY_BASE_URL=http://api-short-ttl:8080", "-e", "BASE_URL=http://lb:8080"]
|
||||
try:
|
||||
run(base + env + ["api", "python", "student-tests/wait_url.py", "http://api-short-ttl:8080/health/ready"],
|
||||
show=True, timeout=120)
|
||||
run(base + env + ["api", "python", "-m", "unittest", "discover", "-s", "student-tests",
|
||||
"-p", "test_expiry.py", "-v"], show=True, timeout=300)
|
||||
finally:
|
||||
run(prof + ["rm", "-sf", "api-short-ttl"], check=False, timeout=120)
|
||||
else:
|
||||
raise SystemExit("usage: python scripts/ops.py in-network storage|expiry|consistency")
|
||||
|
||||
|
||||
COMMANDS = {
|
||||
"wait": cmd_wait,
|
||||
"ps": cmd_ps,
|
||||
"stop-one": cmd_stop_one,
|
||||
"restore": cmd_restore,
|
||||
"restart": cmd_restart,
|
||||
"rolling-update": cmd_rolling_update,
|
||||
"in-network": cmd_in_network,
|
||||
"wait-s3": cmd_wait_s3,
|
||||
"s3-check": cmd_s3_check,
|
||||
"reconcile": cmd_reconcile,
|
||||
"api-fs": cmd_api_fs,
|
||||
"worker-stop": cmd_worker_stop,
|
||||
"worker-kill": cmd_worker_kill,
|
||||
"worker-start": cmd_worker_start,
|
||||
"api-restart": cmd_api_restart,
|
||||
"broker-restart": cmd_broker_restart,
|
||||
"jobs": cmd_jobs,
|
||||
"pipeline-faults": cmd_pipeline_faults,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2 or sys.argv[1] not in COMMANDS:
|
||||
print(__doc__)
|
||||
sys.exit(2)
|
||||
COMMANDS[sys.argv[1]](sys.argv[2:])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,9 @@
|
||||
@echo off
|
||||
rem Windows-вариант restart.sh: API + PostgreSQL + Redis + S3 + LB, volumes сохраняются.
|
||||
rem python tests\resilience.py --mode persistence --action-script scripts\restart.cmd
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
set PY=python
|
||||
where python >nul 2>nul || set PY=py -3
|
||||
%PY% scripts\ops.py restart
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
# Hook для make test-persistence: перезапускает API (все реплики), PostgreSQL,
|
||||
# Redis, S3 (SeaweedFS) и LB, сохраняя volumes pgdata и s3data. Никаких down -v /
|
||||
# flush / повторного seed / пересоздания бакета. После рестарта S3 ждёт, пока
|
||||
# реально работают запись и чтение (прогрев SeaweedFS 15–30 с), затем поднимает API.
|
||||
# Возвращает 0, когда API снова healthy; готовность LB тест ждёт сам.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
PY=$(command -v python3 || command -v python)
|
||||
exec "$PY" scripts/ops.py restart
|
||||
@@ -0,0 +1,93 @@
|
||||
-- Обоснование индекса истории: EXPLAIN (ANALYZE, BUFFERS) на >= 10 000 сообщений.
|
||||
-- Всё выполняется в ОДНОЙ транзакции и откатывается (ROLLBACK): синтетические
|
||||
-- данные и временное удаление индекса не остаются в БД.
|
||||
-- Запуск (compose):
|
||||
-- docker compose exec -T postgres psql -U messenger -d messenger -v ON_ERROR_STOP=1 \
|
||||
-- < scripts/sql/explain_history.sql > evidence/explain_history.txt
|
||||
-- Внимание: DROP INDEX внутри транзакции берёт эксклюзивную блокировку messages
|
||||
-- до ROLLBACK (несколько секунд) — запускать на учебном стенде.
|
||||
|
||||
\set ON_ERROR_STOP 1
|
||||
\timing off
|
||||
SET TimeZone = 'UTC';
|
||||
-- Убираем мёртвые версии строк от прошлых запусков (VACUUM нельзя внутри транзакции).
|
||||
VACUUM (ANALYZE) messages, chat_members, chats, users;
|
||||
BEGIN;
|
||||
|
||||
INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES ('00000000-0000-4000-8000-00000000e001', 'explain_alice', 'Explain', 'x'),
|
||||
('00000000-0000-4000-8000-00000000e002', 'explain_bob', 'Explain', 'x');
|
||||
|
||||
-- 1 большой чат (12 000 сообщений) + 40 чатов по 1 000 = 52 000 сообщений.
|
||||
INSERT INTO chats (id, title, created_at)
|
||||
SELECT CASE WHEN g = 0 THEN '00000000-0000-4000-8000-0000000c0000'::uuid ELSE gen_random_uuid() END,
|
||||
'explain-' || g, timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
|
||||
FROM generate_series(0, 40) AS g;
|
||||
|
||||
INSERT INTO chat_members (chat_id, user_id, position)
|
||||
SELECT c.id, u.id, u.pos
|
||||
FROM chats c
|
||||
CROSS JOIN (VALUES ('00000000-0000-4000-8000-00000000e001'::uuid, 0),
|
||||
('00000000-0000-4000-8000-00000000e002'::uuid, 1)) AS u(id, pos)
|
||||
WHERE c.title LIKE 'explain-%';
|
||||
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), c.id,
|
||||
CASE WHEN g % 2 = 0 THEN '00000000-0000-4000-8000-00000000e001'::uuid
|
||||
ELSE '00000000-0000-4000-8000-00000000e002'::uuid END,
|
||||
'synthetic message ' || g,
|
||||
timestamptz '2026-01-02 00:00:00+00' + g * interval '1 millisecond' + (c.created_at - timestamptz '2026-01-01 00:00:00+00')
|
||||
FROM chats c
|
||||
CROSS JOIN LATERAL generate_series(1, CASE WHEN c.title = 'explain-0' THEN 12000 ELSE 1000 END) AS g
|
||||
WHERE c.title LIKE 'explain-%';
|
||||
|
||||
ANALYZE users, chats, chat_members, messages;
|
||||
|
||||
\echo '=== dataset'
|
||||
SELECT count(*) AS total_messages,
|
||||
count(*) FILTER (WHERE chat_id = '00000000-0000-4000-8000-0000000c0000') AS big_chat_messages
|
||||
FROM messages;
|
||||
|
||||
-- Позиция курсора после 10 000-го сообщения большого чата (как в next_cursor).
|
||||
SELECT created_at AS cur_ts, id AS cur_id
|
||||
FROM messages
|
||||
WHERE chat_id = '00000000-0000-4000-8000-0000000c0000'
|
||||
ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset
|
||||
|
||||
\echo '=== 1. first page (no cursor), LIMIT 51 = limit+1'
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING)
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = '00000000-0000-4000-8000-0000000c0000'
|
||||
ORDER BY created_at, id
|
||||
LIMIT 51;
|
||||
|
||||
\echo '=== 2. deep page: keyset cursor after message #10000 (query used by API)'
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING)
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = '00000000-0000-4000-8000-0000000c0000'
|
||||
AND (created_at, id) > (:'cur_ts'::timestamptz, :'cur_id'::uuid)
|
||||
ORDER BY created_at, id
|
||||
LIMIT 51;
|
||||
|
||||
\echo '=== 3. for comparison: OFFSET pagination to the same page (NOT used by API)'
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING)
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = '00000000-0000-4000-8000-0000000c0000'
|
||||
ORDER BY created_at, id
|
||||
OFFSET 10000 LIMIT 51;
|
||||
|
||||
\echo '=== 4. same keyset query WITHOUT messages_chat_created_id_idx (index dropped inside the transaction)'
|
||||
DROP INDEX messages_chat_created_id_idx;
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS, TIMING)
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = '00000000-0000-4000-8000-0000000c0000'
|
||||
AND (created_at, id) > (:'cur_ts'::timestamptz, :'cur_id'::uuid)
|
||||
ORDER BY created_at, id
|
||||
LIMIT 51;
|
||||
|
||||
ROLLBACK;
|
||||
\echo '=== rolled back: synthetic data removed, index restored'
|
||||
@@ -0,0 +1,9 @@
|
||||
@echo off
|
||||
rem Windows-вариант stop-one.sh: TARGET_INSTANCE_ID -> один контейнер api, остаётся остановленным.
|
||||
rem python tests\resilience.py --mode failover --action-script scripts\stop-one.cmd
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
set PY=python
|
||||
where python >nul 2>nul || set PY=py -3
|
||||
%PY% scripts\ops.py stop-one
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
# Hook для make test-failover: останавливает ОДНУ API-реплику, которую тест передал
|
||||
# в TARGET_INSTANCE_ID (= X-Instance-Id = hostname контейнера = короткий ID), и
|
||||
# оставляет её остановленной. По умолчанию graceful (docker stop: SIGTERM ->
|
||||
# осушение 3 с -> graceful shutdown); STOP_MODE=kill — «падение» (SIGKILL).
|
||||
# БД, Redis, S3, LB и volumes не трогает. Восстановление: python3 scripts/ops.py restore
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
PY=$(command -v python3 || command -v python)
|
||||
exec "$PY" scripts/ops.py stop-one
|
||||
Reference in New Issue
Block a user