Files
mtusi2026lab4/bench/fill_objects.py
T
ricoromaandClaude Opus 5.5 8d4cf7eebf feat: add reconcile strategies, deep check, rate limit and cost profile
- reconcile --strategy head, --deep (checksum_mismatch), restore needs the same SHA-256
- [reconciler] max_objects_per_second (2000), pause interrupted by signal
- --apply stops rechecks once the limit is exceeded (unchecked)
- pass cost in the summary: S3 objects and requests, bytes, DB sessions, phases
- S3 client counters, cleaner exits with os._exit after main()
- fault tests, hook args, driver reconcile-signal --pace
- bench profile reconcile / reconcile_quick: fill outside the API, gateway metrics, background load
- sync spec deltas, rules, AGENTS.md, PLAN.md

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-23 04:52:56 +03:00

117 lines
4.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Загрузчик объектов профиля сверки: прямой PUT в S3 стенда мимо API.
Работает только в контейнере `mtusi-bench-fill` на образе стенда
(`mtusi-messenger:bench`): botocore образа, `src` не импортируется, каталог
`bench/` смонтирован только на чтение, номера, ключи и содержимое объектов
— генератор набора `bench/reconcile.py` (design change
`track2-reconcile-cost`, решение 3). Ключи пользователя API — окружение
контейнера (`S3__ACCESS_KEY_ID`, `S3__SECRET_ACCESS_KEY`: `docker run -e
<имя>` без значения в argv); в вывод они не попадают.
Объекты — номера диапазона [--start, --end), у которых по модели набора
есть объект; несколько процессов по нескольку потоков, у каждого процесса
свой клиент. Вывод — одна строка `fill: <объектов> objects, <PUT/с>
put/s, <с> s`; ошибка любого PUT — код 1.
"""
import argparse
import os
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from multiprocessing import Pool
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parent))
import reconcile # noqa: E402
KEY_ENV = ("S3__ACCESS_KEY_ID", "S3__SECRET_ACCESS_KEY")
def client(endpoint: str, threads: int) -> Any:
import botocore.session
from botocore.config import Config
session = botocore.session.get_session()
return session.create_client(
"s3",
endpoint_url=endpoint,
region_name="us-east-1",
aws_access_key_id=os.environ[KEY_ENV[0]],
aws_secret_access_key=os.environ[KEY_ENV[1]],
config=Config(
s3={"addressing_style": "path"},
retries={"total_max_attempts": 3},
max_pool_connections=threads,
),
)
def put_chunk(job: tuple[dict[str, Any], list[int]]) -> int:
"""Один процесс: свой клиент, потоки по номерам; число записанных."""
args, indices = job
s3 = client(args["endpoint"], args["threads"])
def put(index: int) -> None:
attachment_id = reconcile.object_id(args["seed"], index)
s3.put_object(
Bucket=args["bucket"],
Key=reconcile.object_key(args["namespace"], attachment_id),
Body=reconcile.content(args["seed"], index, args["size"]),
ContentType=reconcile.CONTENT_TYPE,
)
with ThreadPoolExecutor(max_workers=args["threads"]) as pool:
for _ in pool.map(put, indices):
pass
return len(indices)
def main(argv: list[str]) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--endpoint", required=True)
parser.add_argument("--bucket", required=True)
parser.add_argument("--namespace", required=True)
parser.add_argument("--seed", type=int, required=True)
parser.add_argument("--start", type=int, required=True)
parser.add_argument("--end", type=int, required=True)
parser.add_argument("--size", type=int, required=True)
parser.add_argument("--force", choices=(reconcile.FORCE_OK,))
parser.add_argument("--processes", type=int, default=3)
parser.add_argument("--threads", type=int, default=24)
args = parser.parse_args(argv)
if not Path("/.dockerenv").exists():
print("fill: run only in the bench container", flush=True)
return 2
if any(not os.environ.get(name) for name in KEY_ENV):
print(f"fill: {' and '.join(KEY_ENV)} are required", flush=True)
return 2
indices = reconcile.object_indices(args.start, args.end, args.force)
parts = [indices[i :: args.processes] for i in range(args.processes)]
shared = {
key: getattr(args, key)
for key in ("endpoint", "bucket", "namespace", "seed", "size")
}
shared["threads"] = args.threads
started = time.monotonic()
try:
with Pool(args.processes) as pool:
done = sum(pool.map(put_chunk, [(shared, p) for p in parts if p]))
except Exception as exc:
# без текста исключения botocore: в нём может быть адрес и заголовки
print(f"fill: put failed ({type(exc).__name__})", flush=True)
return 1
seconds = time.monotonic() - started
print(
f"fill: {done} objects, {done / seconds if seconds else 0:.0f} "
f"put/s, {seconds:.1f} s",
flush=True,
)
return 0 if done == len(indices) else 1
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))