Source: mtusi2026lab5-safronov, solution/bvt2301-safronov, 046ec9e. Contracts, public tests and evidence of lab 5 are not carried over. The compose project, image and CI prefix are renamed to lab6; the first broker start defect is recorded in KNOWN_ISSUES.md; PLAN.md outlines the OpenSpec changes of lab 6. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
117 lines
4.8 KiB
Python
117 lines
4.8 KiB
Python
"""Загрузчик объектов профиля сверки: прямой PUT в S3 стенда мимо API.
|
||
|
||
Работает только в контейнере `mtusi-bench-fill` на образе стенда
|
||
(`mtusi-messenger:bench`): botocore образа, `src` не импортируется, каталог
|
||
`bench/` смонтирован только на чтение, номера, ключи и содержимое объектов
|
||
— генератор набора `bench/reconcile.py` (design change
|
||
`track2-reconcile-cost`, решение 3). Ключи пользователя API — окружение
|
||
контейнера (`S3__ACCESS_KEY_ID`, `S3__SECRET_ACCESS_KEY`: `docker run -e
|
||
<имя>` без значения в argv); в вывод они не попадают.
|
||
|
||
Объекты — номера диапазона [--start, --end), у которых по модели набора
|
||
есть объект; несколько процессов по нескольку потоков, у каждого процесса
|
||
свой клиент. Вывод — одна строка `fill: <объектов> objects, <PUT/с>
|
||
put/s, <с> s`; ошибка любого PUT — код 1.
|
||
"""
|
||
|
||
import argparse
|
||
import os
|
||
import sys
|
||
import time
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from multiprocessing import Pool
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||
|
||
import reconcile # noqa: E402
|
||
|
||
KEY_ENV = ("S3__ACCESS_KEY_ID", "S3__SECRET_ACCESS_KEY")
|
||
|
||
|
||
def client(endpoint: str, threads: int) -> Any:
|
||
import botocore.session
|
||
from botocore.config import Config
|
||
|
||
session = botocore.session.get_session()
|
||
return session.create_client(
|
||
"s3",
|
||
endpoint_url=endpoint,
|
||
region_name="us-east-1",
|
||
aws_access_key_id=os.environ[KEY_ENV[0]],
|
||
aws_secret_access_key=os.environ[KEY_ENV[1]],
|
||
config=Config(
|
||
s3={"addressing_style": "path"},
|
||
retries={"total_max_attempts": 3},
|
||
max_pool_connections=threads,
|
||
),
|
||
)
|
||
|
||
|
||
def put_chunk(job: tuple[dict[str, Any], list[int]]) -> int:
|
||
"""Один процесс: свой клиент, потоки по номерам; число записанных."""
|
||
args, indices = job
|
||
s3 = client(args["endpoint"], args["threads"])
|
||
|
||
def put(index: int) -> None:
|
||
attachment_id = reconcile.object_id(args["seed"], index)
|
||
s3.put_object(
|
||
Bucket=args["bucket"],
|
||
Key=reconcile.object_key(args["namespace"], attachment_id),
|
||
Body=reconcile.content(args["seed"], index, args["size"]),
|
||
ContentType=reconcile.CONTENT_TYPE,
|
||
)
|
||
|
||
with ThreadPoolExecutor(max_workers=args["threads"]) as pool:
|
||
for _ in pool.map(put, indices):
|
||
pass
|
||
return len(indices)
|
||
|
||
|
||
def main(argv: list[str]) -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
parser.add_argument("--endpoint", required=True)
|
||
parser.add_argument("--bucket", required=True)
|
||
parser.add_argument("--namespace", required=True)
|
||
parser.add_argument("--seed", type=int, required=True)
|
||
parser.add_argument("--start", type=int, required=True)
|
||
parser.add_argument("--end", type=int, required=True)
|
||
parser.add_argument("--size", type=int, required=True)
|
||
parser.add_argument("--force", choices=(reconcile.FORCE_OK,))
|
||
parser.add_argument("--processes", type=int, default=3)
|
||
parser.add_argument("--threads", type=int, default=24)
|
||
args = parser.parse_args(argv)
|
||
if not Path("/.dockerenv").exists():
|
||
print("fill: run only in the bench container", flush=True)
|
||
return 2
|
||
if any(not os.environ.get(name) for name in KEY_ENV):
|
||
print(f"fill: {' and '.join(KEY_ENV)} are required", flush=True)
|
||
return 2
|
||
indices = reconcile.object_indices(args.start, args.end, args.force)
|
||
parts = [indices[i :: args.processes] for i in range(args.processes)]
|
||
shared = {
|
||
key: getattr(args, key)
|
||
for key in ("endpoint", "bucket", "namespace", "seed", "size")
|
||
}
|
||
shared["threads"] = args.threads
|
||
started = time.monotonic()
|
||
try:
|
||
with Pool(args.processes) as pool:
|
||
done = sum(pool.map(put_chunk, [(shared, p) for p in parts if p]))
|
||
except Exception as exc:
|
||
# без текста исключения botocore: в нём может быть адрес и заголовки
|
||
print(f"fill: put failed ({type(exc).__name__})", flush=True)
|
||
return 1
|
||
seconds = time.monotonic() - started
|
||
print(
|
||
f"fill: {done} objects, {done / seconds if seconds else 0:.0f} "
|
||
f"put/s, {seconds:.1f} s",
|
||
flush=True,
|
||
)
|
||
return 0 if done == len(indices) else 1
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main(sys.argv[1:]))
|