Files
mtusi2026labs/student-tests/tools/seed_storage.py
T

140 lines
4.6 KiB
Python

import argparse
import asyncio
import hashlib
import json
import time
from datetime import UTC, datetime, timedelta
from io import BytesIO
from uuid import uuid4
from PIL import Image
from sqlalchemy import func, insert, make_url, select
from sqlalchemy.ext.asyncio import create_async_engine
from messenger.attachments.config import AttachmentsSettings
from messenger.attachments.infrastructure.postgres import AttachmentRow
from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client
from messenger.identity.infrastructure.postgres import UserRow
from messenger.messaging.infrastructure.postgres import ChatMemberRow
from messenger.shared.config import PlatformSettings
MODELS = (UserRow, ChatMemberRow, AttachmentRow)
COLUMNS = (
"id",
"chat_id",
"owner_id",
"filename",
"content_type",
"size_bytes",
"sha256",
"width",
"height",
"object_key",
"status",
"created_at",
"updated_at",
)
def picture() -> bytes:
buffer = BytesIO()
Image.effect_noise((8, 8), 64).convert("RGB").save(buffer, "PNG")
return buffer.getvalue()
async def seed(count: int, concurrency: int) -> dict[str, object]:
platform, settings = PlatformSettings(), AttachmentsSettings()
body = picture()
digest = hashlib.sha256(body).hexdigest()
engine = create_async_engine(
make_url(platform.database_url.get_secret_value()).set(drivername="postgresql+asyncpg"),
hide_parameters=True,
)
try:
async with engine.connect() as connection:
member = (
(
await connection.execute(
select(ChatMemberRow.chat_id, ChatMemberRow.user_id).limit(1)
)
)
.mappings()
.first()
)
if member is None:
raise SystemExit("нет ни одного чата: сначала make test")
ids = [uuid4() for _ in range(count)]
pending = iter(ids)
completed = 0
started = time.monotonic()
async with s3_client(
endpoint=settings.s3_endpoint,
region=settings.s3_region,
access_key_id=settings.s3_access_key_id.get_secret_value(),
secret_access_key=settings.s3_secret_access_key.get_secret_value(),
connect_timeout=settings.s3_connect_timeout_seconds,
read_timeout=settings.s3_read_timeout_seconds,
max_attempts=3,
) as client:
storage = S3ObjectStorage(client, settings.s3_bucket)
async def worker() -> None:
nonlocal completed
for attachment_id in pending:
await storage.put(f"attachments/{attachment_id}", body, "image/png")
completed += 1
if completed % 10000 == 0 or completed == count:
print(json.dumps({"progress": completed, "target": count}), flush=True)
await asyncio.gather(*(worker() for _ in range(concurrency)))
stored = time.monotonic()
stamp = datetime.now(UTC) - timedelta(hours=2)
records = [
(
attachment_id,
member["chat_id"],
member["user_id"],
f"seed-{number}.png",
"image/png",
len(body),
digest,
8,
8,
f"attachments/{attachment_id}",
"ready",
stamp,
stamp,
)
for number, attachment_id in enumerate(ids)
]
async with engine.begin() as connection:
for start in range(0, count, 1000):
await connection.execute(
insert(AttachmentRow),
[dict(zip(COLUMNS, row, strict=True)) for row in records[start : start + 1000]],
)
total = await connection.scalar(select(func.count()).select_from(AttachmentRow))
finally:
await engine.dispose()
return {
"added": count,
"rows_total": total,
"object_bytes": len(body),
"seconds": {
"s3_put": round(stored - started, 3),
"pg_insert": round(time.monotonic() - stored, 3),
},
}
def main() -> None:
parser = argparse.ArgumentParser(description="Наполнить bucket и attachments N вложениями")
parser.add_argument("--count", type=int, required=True)
parser.add_argument("--concurrency", type=int, default=32)
args = parser.parse_args()
print(json.dumps(asyncio.run(seed(args.count, args.concurrency))))
if __name__ == "__main__":
main()