forked from mtusi2026/mtusi2026lab4
561 lines
14 KiB
Python
561 lines
14 KiB
Python
import csv
|
|
import hashlib
|
|
import json
|
|
import math
|
|
from pathlib import Path
|
|
import random
|
|
import statistics
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import uuid
|
|
|
|
from client import Client, arguments, check, run
|
|
import reconcile_check as helpers
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
|
|
|
|
def options(parser):
|
|
parser.add_argument(
|
|
"--sizes",
|
|
type=int,
|
|
nargs="+",
|
|
default=[10, 100, 1000],
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--repeats",
|
|
type=int,
|
|
default=3,
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--max-seconds",
|
|
type=int,
|
|
default=600,
|
|
)
|
|
|
|
|
|
def digest_metadata(chat_id):
|
|
snapshot = helpers.snapshot(chat_id)
|
|
|
|
return hashlib.sha256(
|
|
snapshot.encode()
|
|
).hexdigest()
|
|
|
|
|
|
def environment_info():
|
|
ids = helpers.command([
|
|
"docker",
|
|
"compose",
|
|
"ps",
|
|
"-q",
|
|
"api",
|
|
"postgres",
|
|
"minio",
|
|
]).decode().split()
|
|
|
|
containers = json.loads(
|
|
helpers.command([
|
|
"docker",
|
|
"inspect",
|
|
*ids,
|
|
])
|
|
)
|
|
|
|
cpu_count = int(helpers.command([
|
|
"docker",
|
|
"info",
|
|
"--format",
|
|
"{{.NCPU}}",
|
|
]))
|
|
|
|
memory_bytes = int(helpers.command([
|
|
"docker",
|
|
"info",
|
|
"--format",
|
|
"{{.MemTotal}}",
|
|
]))
|
|
|
|
worker_image = helpers.command([
|
|
"docker",
|
|
"image",
|
|
"inspect",
|
|
"--format",
|
|
"{{.Id}}",
|
|
"mtusi4-reconciler:local",
|
|
]).decode().strip()
|
|
|
|
# Сохраняем только параметры окружения,
|
|
# необходимые для воспроизводимости.
|
|
# Credentials и полный docker inspect не записываем.
|
|
services = []
|
|
|
|
for container in containers:
|
|
services.append({
|
|
"service": container["Config"]["Labels"].get(
|
|
"com.docker.compose.service"
|
|
),
|
|
"image_id": container["Image"],
|
|
"memory_limit_bytes": (
|
|
container["HostConfig"]["Memory"]
|
|
),
|
|
"nano_cpus": (
|
|
container["HostConfig"]["NanoCpus"]
|
|
),
|
|
})
|
|
|
|
return {
|
|
"docker_cpus": cpu_count,
|
|
"docker_memory_bytes": memory_bytes,
|
|
"worker_image_id": worker_image,
|
|
"worker_memory_limit_bytes": 256 * 1024 * 1024,
|
|
"worker_cpu_limit": 1,
|
|
"services": services,
|
|
}
|
|
|
|
|
|
def main():
|
|
args = arguments(
|
|
"Measure reconciler traversal cost",
|
|
options,
|
|
)
|
|
|
|
sizes = sorted(set(args.sizes))
|
|
|
|
check(
|
|
all(1 <= size <= 5000 for size in sizes),
|
|
"Sizes must be between 1 and 5000",
|
|
)
|
|
|
|
check(
|
|
1 <= args.repeats <= 10,
|
|
"Repeats must be between 1 and 10",
|
|
)
|
|
|
|
check(
|
|
args.max_seconds >= 1,
|
|
"max-seconds must be positive",
|
|
)
|
|
|
|
# Проверяет две API-реплики и отключённую очистку.
|
|
helpers.BUCKET = helpers.inspect_api()
|
|
|
|
fixture = (
|
|
ROOT / "tests/fixtures/sample.png"
|
|
).read_bytes()
|
|
|
|
fixture_sha = hashlib.sha256(fixture).hexdigest()
|
|
|
|
directory = (
|
|
ROOT
|
|
/ "results"
|
|
/ ("reconcile-cost-" + uuid.uuid4().hex)
|
|
)
|
|
|
|
backups = directory / "empty-backups"
|
|
backups.mkdir(parents=True, exist_ok=False)
|
|
|
|
report = {
|
|
"passed": False,
|
|
"hypothesis": (
|
|
"Traversal calls grow approximately linearly "
|
|
"with attachment count; wall time also "
|
|
"contains fixed overhead."
|
|
),
|
|
"sizes": sizes,
|
|
"repeats": args.repeats,
|
|
"grace_seconds": 60,
|
|
"fixture_size_bytes": len(fixture),
|
|
"fixture_sha256": fixture_sha,
|
|
"environment": environment_info(),
|
|
"groups": [],
|
|
"samples": [],
|
|
"summary": [],
|
|
"method": (
|
|
"Separate chats; API uploads; real grace wait; "
|
|
"one warm-up per size; shuffled measured rounds; "
|
|
"dry-run only."
|
|
),
|
|
}
|
|
|
|
groups = []
|
|
|
|
client = Client(
|
|
args.base_url,
|
|
args.ca_file,
|
|
timeout=30,
|
|
)
|
|
|
|
def save():
|
|
(directory / "summary.json").write_text(
|
|
json.dumps(
|
|
report,
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
fields = [
|
|
"attachments",
|
|
"trial",
|
|
"worker_seconds",
|
|
"launcher_seconds",
|
|
"sql_calls",
|
|
"s3_calls",
|
|
"s3_list_calls",
|
|
"s3_head_calls",
|
|
]
|
|
|
|
with (directory / "samples.csv").open(
|
|
"w",
|
|
encoding="utf-8",
|
|
newline="",
|
|
) as stream:
|
|
writer = csv.DictWriter(
|
|
stream,
|
|
fieldnames=fields,
|
|
)
|
|
|
|
writer.writeheader()
|
|
writer.writerows(report["samples"])
|
|
|
|
def scan(group, label):
|
|
path = directory / (
|
|
f'n{group["n"]}-{label}.json'
|
|
)
|
|
|
|
started = time.monotonic()
|
|
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-X",
|
|
"utf8",
|
|
str(ROOT / "scripts/reconcile.py"),
|
|
"--dry-run",
|
|
"--chat-id",
|
|
group["chat_id"],
|
|
"--grace-seconds",
|
|
"60",
|
|
"--max-seconds",
|
|
str(args.max_seconds),
|
|
"--backup-dir",
|
|
str(backups),
|
|
"--output",
|
|
str(path),
|
|
],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=args.max_seconds + 180,
|
|
)
|
|
|
|
wall = time.monotonic() - started
|
|
|
|
check(
|
|
result.returncode == 0,
|
|
"Scan failed: " + str(path),
|
|
)
|
|
|
|
data = json.loads(
|
|
path.read_text(encoding="utf-8")
|
|
)
|
|
|
|
check(
|
|
data["status"] == "completed"
|
|
and data["mode"] == "dry-run",
|
|
"Unexpected scan status",
|
|
)
|
|
|
|
check(
|
|
data.get("chat_id") == group["chat_id"],
|
|
"Wrong scan scope",
|
|
)
|
|
|
|
counts = data["counts"]
|
|
|
|
# Каждый объект и каждая запись должны пройти
|
|
# полноценную проверку после grace period.
|
|
for field in (
|
|
"objects_scanned",
|
|
"referenced_objects",
|
|
"metadata_scanned",
|
|
"present_metadata",
|
|
):
|
|
check(
|
|
counts.get(field, 0) == group["n"],
|
|
"Incomplete scan: " + field,
|
|
)
|
|
|
|
for field in (
|
|
"fresh_objects",
|
|
"fresh_metadata",
|
|
"busy",
|
|
"missing_objects",
|
|
"unresolved_missing",
|
|
"size_mismatches",
|
|
"would_delete_orphans",
|
|
"deleted_orphans",
|
|
"restored",
|
|
):
|
|
check(
|
|
counts.get(field, 0) == 0,
|
|
"Unexpected outcome: " + field,
|
|
)
|
|
|
|
calls = data["s3_operation_calls"]
|
|
|
|
check(
|
|
calls.get("put_object", 0) == 0
|
|
and calls.get("delete_object", 0) == 0,
|
|
"Dry-run attempted mutations",
|
|
)
|
|
|
|
return {
|
|
"attachments": group["n"],
|
|
"trial": label,
|
|
"worker_seconds": data["elapsed_seconds"],
|
|
"launcher_seconds": round(wall, 3),
|
|
"sql_calls": data["sql_statement_calls"],
|
|
"s3_calls": sum(calls.values()),
|
|
"s3_list_calls": calls.get(
|
|
"list_objects_v2", 0
|
|
),
|
|
"s3_head_calls": calls.get(
|
|
"head_object", 0
|
|
),
|
|
}
|
|
|
|
save()
|
|
|
|
try:
|
|
setup_started = time.monotonic()
|
|
|
|
# Каждый размер получает отдельный чат.
|
|
for size in sizes:
|
|
alice, bob, _, chat = client.bootstrap()
|
|
chat_id = str(uuid.UUID(chat["id"]))
|
|
|
|
entry = {
|
|
"attachments": size,
|
|
"chat_id": chat_id,
|
|
"uploaded": 0,
|
|
}
|
|
|
|
report["groups"].append(entry)
|
|
save()
|
|
|
|
sample_attachments = []
|
|
|
|
# Последовательные загрузки.
|
|
# Автоматических повторов POST нет.
|
|
for index in range(size):
|
|
attachment, content = client.attachment(
|
|
alice,
|
|
chat,
|
|
)
|
|
|
|
check(
|
|
content == fixture,
|
|
"Fixture changed during experiment",
|
|
)
|
|
|
|
# Для контрольного скачивания сохраняем
|
|
# первое и последнее вложение.
|
|
if index in {0, size - 1}:
|
|
sample_attachments.append(attachment)
|
|
|
|
entry["uploaded"] = index + 1
|
|
|
|
if (
|
|
(index + 1) % 100 == 0
|
|
or index + 1 == size
|
|
):
|
|
print(
|
|
f"Preparing N={size}: "
|
|
f"{index + 1}/{size}",
|
|
flush=True,
|
|
)
|
|
|
|
save()
|
|
|
|
message = client.message(
|
|
alice,
|
|
chat,
|
|
"Reconciler cost experiment",
|
|
attachments=[
|
|
attachment["id"]
|
|
for attachment in sample_attachments
|
|
],
|
|
)["json"]
|
|
|
|
groups.append({
|
|
"n": size,
|
|
"chat_id": chat_id,
|
|
"bob": bob,
|
|
"samples": sample_attachments,
|
|
"message": message,
|
|
"metadata_digest": digest_metadata(chat_id),
|
|
})
|
|
|
|
report["setup_seconds"] = round(
|
|
time.monotonic() - setup_started,
|
|
3,
|
|
)
|
|
|
|
save()
|
|
|
|
print(
|
|
"All uploads completed. Waiting 65 seconds "
|
|
"for the real grace period...",
|
|
flush=True,
|
|
)
|
|
|
|
time.sleep(65)
|
|
|
|
# Прогрев сохраняется отдельными JSON,
|
|
# но не включается в итоговую статистику.
|
|
for group in groups:
|
|
scan(group, "warmup")
|
|
|
|
print(
|
|
f'Warm-up N={group["n"]}: complete',
|
|
flush=True,
|
|
)
|
|
|
|
# В каждом раунде меняем порядок объёмов.
|
|
# Фиксированный seed делает порядок повторяемым.
|
|
rng = random.Random(1729)
|
|
|
|
for trial in range(1, args.repeats + 1):
|
|
order = list(groups)
|
|
rng.shuffle(order)
|
|
|
|
for group in order:
|
|
sample = scan(
|
|
group,
|
|
f"run-{trial}",
|
|
)
|
|
|
|
report["samples"].append(sample)
|
|
save()
|
|
|
|
print(
|
|
f'N={group["n"]}, run={trial}: '
|
|
f'worker={sample["worker_seconds"]}s, '
|
|
f'launcher={sample["launcher_seconds"]}s, '
|
|
f'SQL={sample["sql_calls"]}, '
|
|
f'S3={sample["s3_calls"]}',
|
|
flush=True,
|
|
)
|
|
|
|
# Контроль после измерений.
|
|
for group in groups:
|
|
check(
|
|
digest_metadata(group["chat_id"])
|
|
== group["metadata_digest"],
|
|
"Metadata changed during benchmark",
|
|
)
|
|
|
|
for attachment in group["samples"]:
|
|
downloaded = client.call(
|
|
"GET",
|
|
"/api/v1/attachments/"
|
|
+ attachment["id"]
|
|
+ "/content",
|
|
actor=group["bob"],
|
|
)
|
|
|
|
check(
|
|
downloaded["bytes"] == fixture,
|
|
"Sample original changed",
|
|
)
|
|
|
|
page = client.call(
|
|
"GET",
|
|
"/api/v1/chats/"
|
|
+ group["chat_id"]
|
|
+ "/messages?limit=100",
|
|
actor=group["bob"],
|
|
schema="MessagePage",
|
|
)["json"]
|
|
|
|
check(
|
|
group["message"] in page["items"],
|
|
"Sample message changed",
|
|
)
|
|
|
|
for size in sizes:
|
|
samples = [
|
|
sample
|
|
for sample in report["samples"]
|
|
if sample["attachments"] == size
|
|
]
|
|
|
|
worker_times = [
|
|
sample["worker_seconds"]
|
|
for sample in samples
|
|
]
|
|
|
|
launcher_times = [
|
|
sample["launcher_seconds"]
|
|
for sample in samples
|
|
]
|
|
|
|
report["summary"].append({
|
|
"attachments": size,
|
|
"worker_seconds_min": min(worker_times),
|
|
"worker_seconds_median": round(
|
|
statistics.median(worker_times),
|
|
3,
|
|
),
|
|
"worker_seconds_max": max(worker_times),
|
|
"launcher_seconds_median": round(
|
|
statistics.median(launcher_times),
|
|
3,
|
|
),
|
|
"sql_calls_observed": sorted({
|
|
sample["sql_calls"]
|
|
for sample in samples
|
|
}),
|
|
"s3_calls_observed": sorted({
|
|
sample["s3_calls"]
|
|
for sample in samples
|
|
}),
|
|
"sql_calls_expected_for_current_algorithm": (
|
|
5 * size
|
|
+ math.ceil(size / 500)
|
|
+ 2
|
|
),
|
|
"s3_calls_expected_for_current_algorithm": (
|
|
size
|
|
+ math.ceil(size / 500)
|
|
+ 1
|
|
),
|
|
})
|
|
|
|
report["passed"] = True
|
|
|
|
finally:
|
|
save()
|
|
|
|
print(json.dumps(
|
|
report["summary"],
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
))
|
|
|
|
print("Reports: " + str(directory))
|
|
|
|
print(
|
|
"PASS: reconciler traversal cost "
|
|
"experiment completed"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run(main) |