Files
mtusi2026lab4/tests/reconcile_cost_check.py
T
2026-11-08 18:17:48 +03:00

561 lines
14 KiB
Python

import csv
import hashlib
import json
import math
from pathlib import Path
import random
import statistics
import subprocess
import sys
import time
import uuid
from client import Client, arguments, check, run
import reconcile_check as helpers
ROOT = Path(__file__).resolve().parents[1]
def options(parser):
parser.add_argument(
"--sizes",
type=int,
nargs="+",
default=[10, 100, 1000],
)
parser.add_argument(
"--repeats",
type=int,
default=3,
)
parser.add_argument(
"--max-seconds",
type=int,
default=600,
)
def digest_metadata(chat_id):
snapshot = helpers.snapshot(chat_id)
return hashlib.sha256(
snapshot.encode()
).hexdigest()
def environment_info():
ids = helpers.command([
"docker",
"compose",
"ps",
"-q",
"api",
"postgres",
"minio",
]).decode().split()
containers = json.loads(
helpers.command([
"docker",
"inspect",
*ids,
])
)
cpu_count = int(helpers.command([
"docker",
"info",
"--format",
"{{.NCPU}}",
]))
memory_bytes = int(helpers.command([
"docker",
"info",
"--format",
"{{.MemTotal}}",
]))
worker_image = helpers.command([
"docker",
"image",
"inspect",
"--format",
"{{.Id}}",
"mtusi4-reconciler:local",
]).decode().strip()
# Сохраняем только параметры окружения,
# необходимые для воспроизводимости.
# Credentials и полный docker inspect не записываем.
services = []
for container in containers:
services.append({
"service": container["Config"]["Labels"].get(
"com.docker.compose.service"
),
"image_id": container["Image"],
"memory_limit_bytes": (
container["HostConfig"]["Memory"]
),
"nano_cpus": (
container["HostConfig"]["NanoCpus"]
),
})
return {
"docker_cpus": cpu_count,
"docker_memory_bytes": memory_bytes,
"worker_image_id": worker_image,
"worker_memory_limit_bytes": 256 * 1024 * 1024,
"worker_cpu_limit": 1,
"services": services,
}
def main():
args = arguments(
"Measure reconciler traversal cost",
options,
)
sizes = sorted(set(args.sizes))
check(
all(1 <= size <= 5000 for size in sizes),
"Sizes must be between 1 and 5000",
)
check(
1 <= args.repeats <= 10,
"Repeats must be between 1 and 10",
)
check(
args.max_seconds >= 1,
"max-seconds must be positive",
)
# Проверяет две API-реплики и отключённую очистку.
helpers.BUCKET = helpers.inspect_api()
fixture = (
ROOT / "tests/fixtures/sample.png"
).read_bytes()
fixture_sha = hashlib.sha256(fixture).hexdigest()
directory = (
ROOT
/ "results"
/ ("reconcile-cost-" + uuid.uuid4().hex)
)
backups = directory / "empty-backups"
backups.mkdir(parents=True, exist_ok=False)
report = {
"passed": False,
"hypothesis": (
"Traversal calls grow approximately linearly "
"with attachment count; wall time also "
"contains fixed overhead."
),
"sizes": sizes,
"repeats": args.repeats,
"grace_seconds": 60,
"fixture_size_bytes": len(fixture),
"fixture_sha256": fixture_sha,
"environment": environment_info(),
"groups": [],
"samples": [],
"summary": [],
"method": (
"Separate chats; API uploads; real grace wait; "
"one warm-up per size; shuffled measured rounds; "
"dry-run only."
),
}
groups = []
client = Client(
args.base_url,
args.ca_file,
timeout=30,
)
def save():
(directory / "summary.json").write_text(
json.dumps(
report,
ensure_ascii=False,
indent=2,
) + "\n",
encoding="utf-8",
)
fields = [
"attachments",
"trial",
"worker_seconds",
"launcher_seconds",
"sql_calls",
"s3_calls",
"s3_list_calls",
"s3_head_calls",
]
with (directory / "samples.csv").open(
"w",
encoding="utf-8",
newline="",
) as stream:
writer = csv.DictWriter(
stream,
fieldnames=fields,
)
writer.writeheader()
writer.writerows(report["samples"])
def scan(group, label):
path = directory / (
f'n{group["n"]}-{label}.json'
)
started = time.monotonic()
result = subprocess.run(
[
sys.executable,
"-X",
"utf8",
str(ROOT / "scripts/reconcile.py"),
"--dry-run",
"--chat-id",
group["chat_id"],
"--grace-seconds",
"60",
"--max-seconds",
str(args.max_seconds),
"--backup-dir",
str(backups),
"--output",
str(path),
],
cwd=ROOT,
capture_output=True,
text=True,
timeout=args.max_seconds + 180,
)
wall = time.monotonic() - started
check(
result.returncode == 0,
"Scan failed: " + str(path),
)
data = json.loads(
path.read_text(encoding="utf-8")
)
check(
data["status"] == "completed"
and data["mode"] == "dry-run",
"Unexpected scan status",
)
check(
data.get("chat_id") == group["chat_id"],
"Wrong scan scope",
)
counts = data["counts"]
# Каждый объект и каждая запись должны пройти
# полноценную проверку после grace period.
for field in (
"objects_scanned",
"referenced_objects",
"metadata_scanned",
"present_metadata",
):
check(
counts.get(field, 0) == group["n"],
"Incomplete scan: " + field,
)
for field in (
"fresh_objects",
"fresh_metadata",
"busy",
"missing_objects",
"unresolved_missing",
"size_mismatches",
"would_delete_orphans",
"deleted_orphans",
"restored",
):
check(
counts.get(field, 0) == 0,
"Unexpected outcome: " + field,
)
calls = data["s3_operation_calls"]
check(
calls.get("put_object", 0) == 0
and calls.get("delete_object", 0) == 0,
"Dry-run attempted mutations",
)
return {
"attachments": group["n"],
"trial": label,
"worker_seconds": data["elapsed_seconds"],
"launcher_seconds": round(wall, 3),
"sql_calls": data["sql_statement_calls"],
"s3_calls": sum(calls.values()),
"s3_list_calls": calls.get(
"list_objects_v2", 0
),
"s3_head_calls": calls.get(
"head_object", 0
),
}
save()
try:
setup_started = time.monotonic()
# Каждый размер получает отдельный чат.
for size in sizes:
alice, bob, _, chat = client.bootstrap()
chat_id = str(uuid.UUID(chat["id"]))
entry = {
"attachments": size,
"chat_id": chat_id,
"uploaded": 0,
}
report["groups"].append(entry)
save()
sample_attachments = []
# Последовательные загрузки.
# Автоматических повторов POST нет.
for index in range(size):
attachment, content = client.attachment(
alice,
chat,
)
check(
content == fixture,
"Fixture changed during experiment",
)
# Для контрольного скачивания сохраняем
# первое и последнее вложение.
if index in {0, size - 1}:
sample_attachments.append(attachment)
entry["uploaded"] = index + 1
if (
(index + 1) % 100 == 0
or index + 1 == size
):
print(
f"Preparing N={size}: "
f"{index + 1}/{size}",
flush=True,
)
save()
message = client.message(
alice,
chat,
"Reconciler cost experiment",
attachments=[
attachment["id"]
for attachment in sample_attachments
],
)["json"]
groups.append({
"n": size,
"chat_id": chat_id,
"bob": bob,
"samples": sample_attachments,
"message": message,
"metadata_digest": digest_metadata(chat_id),
})
report["setup_seconds"] = round(
time.monotonic() - setup_started,
3,
)
save()
print(
"All uploads completed. Waiting 65 seconds "
"for the real grace period...",
flush=True,
)
time.sleep(65)
# Прогрев сохраняется отдельными JSON,
# но не включается в итоговую статистику.
for group in groups:
scan(group, "warmup")
print(
f'Warm-up N={group["n"]}: complete',
flush=True,
)
# В каждом раунде меняем порядок объёмов.
# Фиксированный seed делает порядок повторяемым.
rng = random.Random(1729)
for trial in range(1, args.repeats + 1):
order = list(groups)
rng.shuffle(order)
for group in order:
sample = scan(
group,
f"run-{trial}",
)
report["samples"].append(sample)
save()
print(
f'N={group["n"]}, run={trial}: '
f'worker={sample["worker_seconds"]}s, '
f'launcher={sample["launcher_seconds"]}s, '
f'SQL={sample["sql_calls"]}, '
f'S3={sample["s3_calls"]}',
flush=True,
)
# Контроль после измерений.
for group in groups:
check(
digest_metadata(group["chat_id"])
== group["metadata_digest"],
"Metadata changed during benchmark",
)
for attachment in group["samples"]:
downloaded = client.call(
"GET",
"/api/v1/attachments/"
+ attachment["id"]
+ "/content",
actor=group["bob"],
)
check(
downloaded["bytes"] == fixture,
"Sample original changed",
)
page = client.call(
"GET",
"/api/v1/chats/"
+ group["chat_id"]
+ "/messages?limit=100",
actor=group["bob"],
schema="MessagePage",
)["json"]
check(
group["message"] in page["items"],
"Sample message changed",
)
for size in sizes:
samples = [
sample
for sample in report["samples"]
if sample["attachments"] == size
]
worker_times = [
sample["worker_seconds"]
for sample in samples
]
launcher_times = [
sample["launcher_seconds"]
for sample in samples
]
report["summary"].append({
"attachments": size,
"worker_seconds_min": min(worker_times),
"worker_seconds_median": round(
statistics.median(worker_times),
3,
),
"worker_seconds_max": max(worker_times),
"launcher_seconds_median": round(
statistics.median(launcher_times),
3,
),
"sql_calls_observed": sorted({
sample["sql_calls"]
for sample in samples
}),
"s3_calls_observed": sorted({
sample["s3_calls"]
for sample in samples
}),
"sql_calls_expected_for_current_algorithm": (
5 * size
+ math.ceil(size / 500)
+ 2
),
"s3_calls_expected_for_current_algorithm": (
size
+ math.ceil(size / 500)
+ 1
),
})
report["passed"] = True
finally:
save()
print(json.dumps(
report["summary"],
ensure_ascii=False,
indent=2,
))
print("Reports: " + str(directory))
print(
"PASS: reconciler traversal cost "
"experiment completed"
)
if __name__ == "__main__":
run(main)