Reject disturbed reconciler measurements and preserve every series

This commit is contained in:
Artyom
2026-11-06 02:29:53 +03:00
parent 751aaed4b1
commit 827b783be3
4 changed files with 124 additions and 27 deletions
@@ -1,7 +1,7 @@
## 1. Достоверность драйвера
- [ ] 1.1 Добавить новые каталоги, полный provenance, отказ от disturbed и checkpoint.
- [ ] 1.2 Проверить отказные пути драйвера, lint/check; закоммитить до опытов.
- [x] 1.1 Добавить новые каталоги, полный provenance, отказ от disturbed и checkpoint.
- [x] 1.2 Проверить отказные пути драйвера, lint/check; закоммитить до опытов.
## 2. Корректность
+1
View File
@@ -15,6 +15,7 @@ SETTINGS = (
"LB_STATS_PORT",
"BASE_URL",
"EVIDENCE_DIR",
"RECONCILE_EVIDENCE_DIR",
"NO_PROXY",
"S3_CPUS",
"S3_MEMORY",
+52 -25
View File
@@ -19,7 +19,6 @@ ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
client = __import__("client")
FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes()
OUT = ROOT / "evidence" / "reconcile"
GRACE = 60
FLAGS = ("--grace", str(GRACE), "--skew", "0")
INTENT_AGE = 300
@@ -99,13 +98,18 @@ def inject(kind, count, ids=(), age=None):
def head():
return must(["git", "rev-parse", "--short", "HEAD"]).strip()
return must(["git", "rev-parse", "HEAD"]).strip()
def save(name, data):
path = OUT / name
def evidence_root():
return ROOT / os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile")
def save(name, data, *, stamp=True):
path = evidence_root() / name
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n")
stamped = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "commit": head(), **data}
path.write_text(json.dumps(stamped if stamp else data, ensure_ascii=False, indent=2) + "\n")
return path
@@ -153,10 +157,10 @@ def faults(c, check, pre):
"bitflip": [linked[3], ids[9], ids[10]],
}
damaged = {i for group in targets.values() for i in group}
ready = OUT / "ready-before.json"
ready = evidence_root() / "ready-before.json"
oracle("--grace", str(GRACE), "--deep", "--save-ready", str(ready))
untouched = {k: v for k, v in json.loads(ready.read_text()).items() if k not in damaged}
save("ready-untouched.json", untouched)
save("ready-untouched.json", untouched, stamp=False)
started = time.monotonic()
injected = {
"orphan_stale": inject("orphan", 3),
@@ -171,7 +175,9 @@ def faults(c, check, pre):
injected["pending_fresh"] = inject("stale-pending", 2, age=0)
expected = {name: len(keys) for name, keys in injected.items()}
expected.update(quarantined=pre["counts"]["quarantined"])
before = oracle("--grace", str(GRACE), "--deep", "--output", str(OUT / "check-before.json"))
before = oracle(
"--grace", str(GRACE), "--deep", "--output", str(evidence_root() / "check-before.json")
)
dry = reconcile("--dry-run", "--deep", *FLAGS)
save("dry-run.json", dry)
after_dry = oracle("--grace", str(GRACE), "--deep")
@@ -216,9 +222,9 @@ def faults(c, check, pre):
str(GRACE),
"--deep",
"--expect-ready",
str(OUT / "ready-untouched.json"),
str(evidence_root() / "ready-untouched.json"),
"--output",
str(OUT / "check-after.json"),
str(evidence_root() / "check-after.json"),
)
tolerated = ("orphan_fresh", "pending_fresh")
check(
@@ -406,7 +412,7 @@ def uploader(base, token, chat, *, deadline, results, lock):
def under_load(c, check, base, seconds, threads):
alice, _, _, chat = c.bootstrap()
ready = OUT / "ready-load.json"
ready = evidence_root() / "ready-load.json"
left = oracle("--grace", str(GRACE), "--save-ready", str(ready))["counts"]
leftover = {
"orphan_stale": left["orphan_fresh"] + left["orphan_stale"],
@@ -436,7 +442,7 @@ def under_load(c, check, base, seconds, threads):
runs.append(reconcile("--apply", *FLAGS))
created = {att["id"]: att["sha256"] for status, att in results if att}
expected = dict(json.loads(ready.read_text()), **created)
save("ready-load.json", expected)
save("ready-load.json", expected, stamp=False)
after = oracle(
"--grace",
str(GRACE),
@@ -444,7 +450,7 @@ def under_load(c, check, base, seconds, threads):
"--expect-ready",
str(ready),
"--output",
str(OUT / "check-under-load.json"),
str(evidence_root() / "check-under-load.json"),
)
statuses = {}
for status, _ in results:
@@ -495,7 +501,7 @@ def under_load(c, check, base, seconds, threads):
def demo(args):
OUT.mkdir(parents=True, exist_ok=True)
evidence_root().mkdir(parents=True, exist_ok=False)
c = client.Client(args.base_url, timeout=30)
check = Checks()
pre = oracle("--grace", str(GRACE))
@@ -608,19 +614,21 @@ def wait_quiet(args, pinned):
quiet = screen(first, sample(), 0.0, pinned, args, started=False)
calm = 0 if quiet["disturbed"] else calm + 1
sys.stderr.write(f"waiting for quiet: {json.dumps(quiet)}\n")
return calm == 3
def measure(flags, env, args, pinned):
rejected = []
for _ in range(args.retries + 1):
for attempt in range(args.retries + 1):
first = sample()
report = reconcile(*flags, "--samples", "3", env=env)
report["screen"] = screen(first, sample(), report["cost"]["cpu_seconds"], pinned, args)
if not report["screen"]["disturbed"]:
break
return report, rejected
rejected.append(report)
wait_quiet(args, pinned)
return report, rejected
if attempt < args.retries and not wait_quiet(args, pinned):
break
return None, rejected
def seed(count, env):
@@ -709,13 +717,12 @@ def table(rows):
def cost(args):
out = OUT / "cost"
(out / "disturbed").mkdir(parents=True, exist_ok=True)
for stale in [*out.glob("n*-*.json"), *(out / "disturbed").glob("*.json")]:
stale.unlink()
out = evidence_root() / "cost"
out.mkdir(parents=True, exist_ok=False)
(out / "disturbed").mkdir()
rows = int(psql("SELECT count(*) FROM attachments"))
if rows > args.sizes[0]:
raise SystemExit(f"на стенде уже {rows} вложений: make reset-data && make up && make test")
raise SystemExit(f"на стенде уже {rows} вложений: используйте новый отдельный project")
snapshot = out / "host-docker-stats.txt"
snapshot.write_text(host("before the series"))
env = pins(args)
@@ -735,7 +742,9 @@ def cost(args):
plan["seeds"].append(dict(seed(n - rows, env), target=n))
rows = n
time.sleep(args.settle)
wait_quiet(args, pinned)
save("cost/plan.json", plan)
if not wait_quiet(args, pinned):
raise SystemExit("no quiet measurement window before timeout")
modes = [("shallow", ["--dry-run"])]
if n <= args.deep_max:
modes.append(("deep", ["--dry-run", "--deep"]))
@@ -748,10 +757,23 @@ def cost(args):
for number, bad in enumerate(dropped, 1):
save(f"cost/disturbed/{name}-try{number}.json", bad)
rejected[(n, mode)] = rejected.get((n, mode), 0) + len(dropped)
(out / f"{name}.json").write_text(json.dumps(report, indent=2) + "\n")
if report is None:
raise SystemExit(f"{name}: no clean run after retries; rejected preserved")
if (
report["exit_code"]
or report["errors"]
or report["skipped"]
or report["scanned"] != {"objects": n, "rows": n}
or any(report["found"].values())
):
save(f"cost/invalid/{name}.json", report)
raise SystemExit(f"{name}: inconsistent dataset or failed pass")
save(f"cost/{name}.json", report)
results.setdefault((n, mode), []).append(report)
save("cost/summary.json", {**plan, "results": summarize(results, rejected)})
sys.stderr.write(f"{name}: {report['cost']['seconds']} s {report['screen']}\n")
finally:
save("cost/summary.json", {**plan, "results": summarize(results, rejected)})
compose("up", "-d", "--wait", "--no-deps", "s3", "postgres")
with snapshot.open("a") as handle:
handle.write("\n" + host("after the series"))
@@ -784,7 +806,12 @@ def main():
k.add_argument("--start-pinned", type=float, default=0.5, help="the same on pinned cores")
k.add_argument("--retries", type=int, default=3, help="reruns of a disturbed run")
k.add_argument("--quiet-timeout", type=float, default=900.0)
for command in (d, k):
command.add_argument(
"--out", default=os.environ.get("RECONCILE_EVIDENCE_DIR", "evidence/reconcile")
)
args = p.parse_args()
os.environ["RECONCILE_EVIDENCE_DIR"] = args.out
if args.command == "demo":
demo(args)
else:
@@ -0,0 +1,69 @@
import importlib.util
from pathlib import Path
from types import ModuleType, SimpleNamespace
from unittest.mock import Mock
import pytest
@pytest.fixture
def track(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
script = Path(__file__).resolve().parents[2] / "scripts/reconcile_track.py"
spec = importlib.util.spec_from_file_location("reconcile_measurements", script)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
@pytest.mark.parametrize(
("disturbed", "quiet", "accepted", "discarded"),
[
([False], True, True, 0),
([True, False], True, True, 1),
([True, True], True, False, 2),
([True], False, False, 1),
],
)
def test_only_clean_measurements_are_accepted(
track: ModuleType,
monkeypatch: pytest.MonkeyPatch,
*,
disturbed: list[bool],
quiet: bool,
accepted: bool,
discarded: int,
) -> None:
reports = [{"cost": {"cpu_seconds": 1}, "attempt": i} for i in range(len(disturbed))]
monkeypatch.setattr(track, "sample", lambda: None)
monkeypatch.setattr(track, "reconcile", Mock(side_effect=reports))
monkeypatch.setattr(track, "screen", Mock(side_effect=[{"disturbed": d} for d in disturbed]))
monkeypatch.setattr(track, "wait_quiet", lambda *_: quiet)
chosen, rejected = track.measure([], {}, SimpleNamespace(retries=1), set())
assert (chosen is not None) == accepted
assert len(rejected) == discarded
assert all(r["screen"]["disturbed"] for r in rejected)
if chosen is not None:
assert not chosen["screen"]["disturbed"]
assert chosen not in rejected
def test_existing_series_is_preserved_before_docker_access(
track: ModuleType, monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
monkeypatch.setenv("RECONCILE_EVIDENCE_DIR", str(tmp_path))
folder = tmp_path / "cost"
folder.mkdir()
raw = folder / "n1000-shallow-1.json"
raw.write_text("previous evidence")
query = Mock(side_effect=AssertionError("must not access the stand"))
monkeypatch.setattr(track, "psql", query)
with pytest.raises(FileExistsError):
track.cost(SimpleNamespace(sizes=[1000]))
assert raw.read_text() == "previous evidence"
query.assert_not_called()
def test_quiet_deadline_is_not_success(track: ModuleType, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(track.time, "monotonic", lambda: 10)
assert not track.wait_quiet(SimpleNamespace(quiet_timeout=0), set())