Reject disturbed reconciler measurements and preserve every series
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
## 1. Достоверность драйвера
|
||||
|
||||
- [ ] 1.1 Добавить новые каталоги, полный provenance, отказ от disturbed и checkpoint.
|
||||
- [ ] 1.2 Проверить отказные пути драйвера, lint/check; закоммитить до опытов.
|
||||
- [x] 1.1 Добавить новые каталоги, полный provenance, отказ от disturbed и checkpoint.
|
||||
- [x] 1.2 Проверить отказные пути драйвера, lint/check; закоммитить до опытов.
|
||||
|
||||
## 2. Корректность
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ SETTINGS = (
|
||||
"LB_STATS_PORT",
|
||||
"BASE_URL",
|
||||
"EVIDENCE_DIR",
|
||||
"RECONCILE_EVIDENCE_DIR",
|
||||
"NO_PROXY",
|
||||
"S3_CPUS",
|
||||
"S3_MEMORY",
|
||||
|
||||
+52
-25
@@ -19,7 +19,6 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
client = __import__("client")
|
||||
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
OUT = ROOT / "evidence" / "reconcile"
|
||||
GRACE = 60
|
||||
FLAGS = ("--grace", str(GRACE), "--skew", "0")
|
||||
INTENT_AGE = 300
|
||||
@@ -99,13 +98,18 @@ def inject(kind, count, ids=(), age=None):
|
||||
|
||||
|
||||
def head():
|
||||
return must(["git", "rev-parse", "--short", "HEAD"]).strip()
|
||||
return must(["git", "rev-parse", "HEAD"]).strip()
|
||||
|
||||
|
||||
def save(name, data):
|
||||
path = OUT / name
|
||||
def evidence_root():
|
||||
return ROOT / os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile")
|
||||
|
||||
|
||||
def save(name, data, *, stamp=True):
|
||||
path = evidence_root() / name
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
|
||||
stamped = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head(), **data}
|
||||
path.write_text(json.dumps(stamped if stamp else data, ensure_ascii=False, indent=2) + "\n")
|
||||
return path
|
||||
|
||||
|
||||
@@ -153,10 +157,10 @@ def faults(c, check, pre):
|
||||
"bitflip": [linked[3], ids[9], ids[10]],
|
||||
}
|
||||
damaged = {i for group in targets.values() for i in group}
|
||||
ready = OUT / "ready-before.json"
|
||||
ready = evidence_root() / "ready-before.json"
|
||||
oracle("--grace", str(GRACE), "--deep", "--save-ready", str(ready))
|
||||
untouched = {k: v for k, v in json.loads(ready.read_text()).items() if k not in damaged}
|
||||
save("ready-untouched.json", untouched)
|
||||
save("ready-untouched.json", untouched, stamp=False)
|
||||
started = time.monotonic()
|
||||
injected = {
|
||||
"orphan_stale": inject("orphan", 3),
|
||||
@@ -171,7 +175,9 @@ def faults(c, check, pre):
|
||||
injected["pending_fresh"] = inject("stale-pending", 2, age=0)
|
||||
expected = {name: len(keys) for name, keys in injected.items()}
|
||||
expected.update(quarantined=pre["counts"]["quarantined"])
|
||||
before = oracle("--grace", str(GRACE), "--deep", "--output", str(OUT / "check-before.json"))
|
||||
before = oracle(
|
||||
"--grace", str(GRACE), "--deep", "--output", str(evidence_root() / "check-before.json")
|
||||
)
|
||||
dry = reconcile("--dry-run", "--deep", *FLAGS)
|
||||
save("dry-run.json", dry)
|
||||
after_dry = oracle("--grace", str(GRACE), "--deep")
|
||||
@@ -216,9 +222,9 @@ def faults(c, check, pre):
|
||||
str(GRACE),
|
||||
"--deep",
|
||||
"--expect-ready",
|
||||
str(OUT / "ready-untouched.json"),
|
||||
str(evidence_root() / "ready-untouched.json"),
|
||||
"--output",
|
||||
str(OUT / "check-after.json"),
|
||||
str(evidence_root() / "check-after.json"),
|
||||
)
|
||||
tolerated = ("orphan_fresh", "pending_fresh")
|
||||
check(
|
||||
@@ -406,7 +412,7 @@ def uploader(base, token, chat, *, deadline, results, lock):
|
||||
|
||||
def under_load(c, check, base, seconds, threads):
|
||||
alice, _, _, chat = c.bootstrap()
|
||||
ready = OUT / "ready-load.json"
|
||||
ready = evidence_root() / "ready-load.json"
|
||||
left = oracle("--grace", str(GRACE), "--save-ready", str(ready))["counts"]
|
||||
leftover = {
|
||||
"orphan_stale": left["orphan_fresh"] + left["orphan_stale"],
|
||||
@@ -436,7 +442,7 @@ def under_load(c, check, base, seconds, threads):
|
||||
runs.append(reconcile("--apply", *FLAGS))
|
||||
created = {att["id"]: att["sha256"] for status, att in results if att}
|
||||
expected = dict(json.loads(ready.read_text()), **created)
|
||||
save("ready-load.json", expected)
|
||||
save("ready-load.json", expected, stamp=False)
|
||||
after = oracle(
|
||||
"--grace",
|
||||
str(GRACE),
|
||||
@@ -444,7 +450,7 @@ def under_load(c, check, base, seconds, threads):
|
||||
"--expect-ready",
|
||||
str(ready),
|
||||
"--output",
|
||||
str(OUT / "check-under-load.json"),
|
||||
str(evidence_root() / "check-under-load.json"),
|
||||
)
|
||||
statuses = {}
|
||||
for status, _ in results:
|
||||
@@ -495,7 +501,7 @@ def under_load(c, check, base, seconds, threads):
|
||||
|
||||
|
||||
def demo(args):
|
||||
OUT.mkdir(parents=True, exist_ok=True)
|
||||
evidence_root().mkdir(parents=True, exist_ok=False)
|
||||
c = client.Client(args.base_url, timeout=30)
|
||||
check = Checks()
|
||||
pre = oracle("--grace", str(GRACE))
|
||||
@@ -608,19 +614,21 @@ def wait_quiet(args, pinned):
|
||||
quiet = screen(first, sample(), 0.0, pinned, args, started=False)
|
||||
calm = 0 if quiet["disturbed"] else calm + 1
|
||||
sys.stderr.write(f"waiting for quiet: {json.dumps(quiet)}\n")
|
||||
return calm == 3
|
||||
|
||||
|
||||
def measure(flags, env, args, pinned):
|
||||
rejected = []
|
||||
for _ in range(args.retries + 1):
|
||||
for attempt in range(args.retries + 1):
|
||||
first = sample()
|
||||
report = reconcile(*flags, "--samples", "3", env=env)
|
||||
report["screen"] = screen(first, sample(), report["cost"]["cpu_seconds"], pinned, args)
|
||||
if not report["screen"]["disturbed"]:
|
||||
break
|
||||
return report, rejected
|
||||
rejected.append(report)
|
||||
wait_quiet(args, pinned)
|
||||
return report, rejected
|
||||
if attempt < args.retries and not wait_quiet(args, pinned):
|
||||
break
|
||||
return None, rejected
|
||||
|
||||
|
||||
def seed(count, env):
|
||||
@@ -709,13 +717,12 @@ def table(rows):
|
||||
|
||||
|
||||
def cost(args):
|
||||
out = OUT / "cost"
|
||||
(out / "disturbed").mkdir(parents=True, exist_ok=True)
|
||||
for stale in [*out.glob("n*-*.json"), *(out / "disturbed").glob("*.json")]:
|
||||
stale.unlink()
|
||||
out = evidence_root() / "cost"
|
||||
out.mkdir(parents=True, exist_ok=False)
|
||||
(out / "disturbed").mkdir()
|
||||
rows = int(psql("SELECT count(*) FROM attachments"))
|
||||
if rows > args.sizes[0]:
|
||||
raise SystemExit(f"на стенде уже {rows} вложений: make reset-data && make up && make test")
|
||||
raise SystemExit(f"на стенде уже {rows} вложений: используйте новый отдельный project")
|
||||
snapshot = out / "host-docker-stats.txt"
|
||||
snapshot.write_text(host("before the series"))
|
||||
env = pins(args)
|
||||
@@ -735,7 +742,9 @@ def cost(args):
|
||||
plan["seeds"].append(dict(seed(n - rows, env), target=n))
|
||||
rows = n
|
||||
time.sleep(args.settle)
|
||||
wait_quiet(args, pinned)
|
||||
save("cost/plan.json", plan)
|
||||
if not wait_quiet(args, pinned):
|
||||
raise SystemExit("no quiet measurement window before timeout")
|
||||
modes = [("shallow", ["--dry-run"])]
|
||||
if n <= args.deep_max:
|
||||
modes.append(("deep", ["--dry-run", "--deep"]))
|
||||
@@ -748,10 +757,23 @@ def cost(args):
|
||||
for number, bad in enumerate(dropped, 1):
|
||||
save(f"cost/disturbed/{name}-try{number}.json", bad)
|
||||
rejected[(n, mode)] = rejected.get((n, mode), 0) + len(dropped)
|
||||
(out / f"{name}.json").write_text(json.dumps(report, indent=2) + "\n")
|
||||
if report is None:
|
||||
raise SystemExit(f"{name}: no clean run after retries; rejected preserved")
|
||||
if (
|
||||
report["exit_code"]
|
||||
or report["errors"]
|
||||
or report["skipped"]
|
||||
or report["scanned"] != {"objects": n, "rows": n}
|
||||
or any(report["found"].values())
|
||||
):
|
||||
save(f"cost/invalid/{name}.json", report)
|
||||
raise SystemExit(f"{name}: inconsistent dataset or failed pass")
|
||||
save(f"cost/{name}.json", report)
|
||||
results.setdefault((n, mode), []).append(report)
|
||||
save("cost/summary.json", {**plan, "results": summarize(results, rejected)})
|
||||
sys.stderr.write(f"{name}: {report['cost']['seconds']} s {report['screen']}\n")
|
||||
finally:
|
||||
save("cost/summary.json", {**plan, "results": summarize(results, rejected)})
|
||||
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres")
|
||||
with snapshot.open("a") as handle:
|
||||
handle.write("\n" + host("after the series"))
|
||||
@@ -784,7 +806,12 @@ def main():
|
||||
k.add_argument("--start-pinned", type=float, default=0.5, help="the same on pinned cores")
|
||||
k.add_argument("--retries", type=int, default=3, help="reruns of a disturbed run")
|
||||
k.add_argument("--quiet-timeout", type=float, default=900.0)
|
||||
for command in (d, k):
|
||||
command.add_argument(
|
||||
"--out", default=os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile")
|
||||
)
|
||||
args = p.parse_args()
|
||||
os.environ["RECONCILE_EVIDENCE_DIR"] = args.out
|
||||
if args.command == "demo":
|
||||
demo(args)
|
||||
else:
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def track(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
|
||||
script = Path(__file__).resolve().parents[2] / "scripts/reconcile_track.py"
|
||||
spec = importlib.util.spec_from_file_location("reconcile_measurements", script)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("disturbed", "quiet", "accepted", "discarded"),
|
||||
[
|
||||
([False], True, True, 0),
|
||||
([True, False], True, True, 1),
|
||||
([True, True], True, False, 2),
|
||||
([True], False, False, 1),
|
||||
],
|
||||
)
|
||||
def test_only_clean_measurements_are_accepted(
|
||||
track: ModuleType,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
*,
|
||||
disturbed: list[bool],
|
||||
quiet: bool,
|
||||
accepted: bool,
|
||||
discarded: int,
|
||||
) -> None:
|
||||
reports = [{"cost": {"cpu_seconds": 1}, "attempt": i} for i in range(len(disturbed))]
|
||||
monkeypatch.setattr(track, "sample", lambda: None)
|
||||
monkeypatch.setattr(track, "reconcile", Mock(side_effect=reports))
|
||||
monkeypatch.setattr(track, "screen", Mock(side_effect=[{"disturbed": d} for d in disturbed]))
|
||||
monkeypatch.setattr(track, "wait_quiet", lambda *_: quiet)
|
||||
chosen, rejected = track.measure([], {}, SimpleNamespace(retries=1), set())
|
||||
assert (chosen is not None) == accepted
|
||||
assert len(rejected) == discarded
|
||||
assert all(r["screen"]["disturbed"] for r in rejected)
|
||||
if chosen is not None:
|
||||
assert not chosen["screen"]["disturbed"]
|
||||
assert chosen not in rejected
|
||||
|
||||
|
||||
def test_existing_series_is_preserved_before_docker_access(
|
||||
track: ModuleType, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
monkeypatch.setenv("RECONCILE_EVIDENCE_DIR", str(tmp_path))
|
||||
folder = tmp_path / "cost"
|
||||
folder.mkdir()
|
||||
raw = folder / "n1000-shallow-1.json"
|
||||
raw.write_text("previous evidence")
|
||||
query = Mock(side_effect=AssertionError("must not access the stand"))
|
||||
monkeypatch.setattr(track, "psql", query)
|
||||
with pytest.raises(FileExistsError):
|
||||
track.cost(SimpleNamespace(sizes=[1000]))
|
||||
assert raw.read_text() == "previous evidence"
|
||||
query.assert_not_called()
|
||||
|
||||
|
||||
def test_quiet_deadline_is_not_success(track: ModuleType, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(track.time, "monotonic", lambda: 10)
|
||||
assert not track.wait_quiet(SimpleNamespace(quiet_timeout=0), set())
|
||||
Reference in New Issue
Block a user