Files
mtusi2026lab6/bench/fill_objects.py
T
ricoromaandClaude Opus 5.5 9cc6c0c7a6 chore: port lab 5 solution into the lab 6 skeleton
Source: mtusi2026lab5-safronov, solution/bvt2301-safronov, 046ec9e.
Contracts, public tests and evidence of lab 5 are not carried over.
The compose project, image and CI prefix are renamed to lab6; the first
broker start defect is recorded in KNOWN_ISSUES.md; PLAN.md outlines the
OpenSpec changes of lab 6.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-11 04:46:56 +03:00

117 lines
4.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Загрузчик объектов профиля сверки: прямой PUT в S3 стенда мимо API.
Работает только в контейнере `mtusi-bench-fill` на образе стенда
(`mtusi-messenger:bench`): botocore образа, `src` не импортируется, каталог
`bench/` смонтирован только на чтение, номера, ключи и содержимое объектов
— генератор набора `bench/reconcile.py` (design change
`track2-reconcile-cost`, решение 3). Ключи пользователя API — окружение
контейнера (`S3__ACCESS_KEY_ID`, `S3__SECRET_ACCESS_KEY`: `docker run -e
<имя>` без значения в argv); в вывод они не попадают.
Объекты — номера диапазона [--start, --end), у которых по модели набора
есть объект; несколько процессов по нескольку потоков, у каждого процесса
свой клиент. Вывод — одна строка `fill: <объектов> objects, <PUT/с>
put/s, <с> s`; ошибка любого PUT — код 1.
"""
import argparse
import os
import sys
import time
from concurrent.futures import ThreadPoolExecutor
from multiprocessing import Pool
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parent))
import reconcile # noqa: E402
KEY_ENV = ("S3__ACCESS_KEY_ID", "S3__SECRET_ACCESS_KEY")
def client(endpoint: str, threads: int) -> Any:
import botocore.session
from botocore.config import Config
session = botocore.session.get_session()
return session.create_client(
"s3",
endpoint_url=endpoint,
region_name="us-east-1",
aws_access_key_id=os.environ[KEY_ENV[0]],
aws_secret_access_key=os.environ[KEY_ENV[1]],
config=Config(
s3={"addressing_style": "path"},
retries={"total_max_attempts": 3},
max_pool_connections=threads,
),
)
def put_chunk(job: tuple[dict[str, Any], list[int]]) -> int:
"""Один процесс: свой клиент, потоки по номерам; число записанных."""
args, indices = job
s3 = client(args["endpoint"], args["threads"])
def put(index: int) -> None:
attachment_id = reconcile.object_id(args["seed"], index)
s3.put_object(
Bucket=args["bucket"],
Key=reconcile.object_key(args["namespace"], attachment_id),
Body=reconcile.content(args["seed"], index, args["size"]),
ContentType=reconcile.CONTENT_TYPE,
)
with ThreadPoolExecutor(max_workers=args["threads"]) as pool:
for _ in pool.map(put, indices):
pass
return len(indices)
def main(argv: list[str]) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--endpoint", required=True)
parser.add_argument("--bucket", required=True)
parser.add_argument("--namespace", required=True)
parser.add_argument("--seed", type=int, required=True)
parser.add_argument("--start", type=int, required=True)
parser.add_argument("--end", type=int, required=True)
parser.add_argument("--size", type=int, required=True)
parser.add_argument("--force", choices=(reconcile.FORCE_OK,))
parser.add_argument("--processes", type=int, default=3)
parser.add_argument("--threads", type=int, default=24)
args = parser.parse_args(argv)
if not Path("/.dockerenv").exists():
print("fill: run only in the bench container", flush=True)
return 2
if any(not os.environ.get(name) for name in KEY_ENV):
print(f"fill: {' and '.join(KEY_ENV)} are required", flush=True)
return 2
indices = reconcile.object_indices(args.start, args.end, args.force)
parts = [indices[i :: args.processes] for i in range(args.processes)]
shared = {
key: getattr(args, key)
for key in ("endpoint", "bucket", "namespace", "seed", "size")
}
shared["threads"] = args.threads
started = time.monotonic()
try:
with Pool(args.processes) as pool:
done = sum(pool.map(put_chunk, [(shared, p) for p in parts if p]))
except Exception as exc:
# без текста исключения botocore: в нём может быть адрес и заголовки
print(f"fill: put failed ({type(exc).__name__})", flush=True)
return 1
seconds = time.monotonic() - started
print(
f"fill: {done} objects, {done / seconds if seconds else 0:.0f} "
f"put/s, {seconds:.1f} s",
flush=True,
)
return 0 if done == len(indices) else 1
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))