Files
mtusi2026lab2/student-tests/tools/seed_storage.py
T
Artyom 12db77133f Forward the lab 5 foundation while preserving lab 6 telemetry
Source: immutable lab5 snapshot f88e9ba43cdfd7eb6a5a7ae5feb4247b4e820071; forward the allowed diff from 64d5c67. Carry SQLAlchemy, Alembic, reliability fixes, versioned jobs, infrastructure and student tests without changing protected coursework files.

Validation: make check, validate, lint; 515 student tests passed with one Linux-only skip; public smoke with telemetry enabled and repeat migration passed. Stop the owned stack and retain volumes. Preserve the existing OTel work in progress; grade implementation remains a separate phase.
2026-11-06 12:29:35 +03:00

140 lines
4.6 KiB
Python

import argparse
import asyncio
import hashlib
import json
import time
from datetime import UTC, datetime, timedelta
from io import BytesIO
from uuid import uuid4
from PIL import Image
from sqlalchemy import func, insert, make_url, select
from sqlalchemy.ext.asyncio import create_async_engine
from messenger.attachments.config import AttachmentsSettings
from messenger.attachments.infrastructure.postgres import AttachmentRow
from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client
from messenger.identity.infrastructure.postgres import UserRow
from messenger.messaging.infrastructure.postgres import ChatMemberRow
from messenger.shared.config import PlatformSettings
MODELS = (UserRow, ChatMemberRow, AttachmentRow)
COLUMNS = (
"id",
"chat_id",
"owner_id",
"filename",
"content_type",
"size_bytes",
"sha256",
"width",
"height",
"object_key",
"status",
"created_at",
"updated_at",
)
def picture() -> bytes:
buffer = BytesIO()
Image.effect_noise((8, 8), 64).convert("RGB").save(buffer, "PNG")
return buffer.getvalue()
async def seed(count: int, concurrency: int) -> dict[str, object]:
platform, settings = PlatformSettings(), AttachmentsSettings()
body = picture()
digest = hashlib.sha256(body).hexdigest()
engine = create_async_engine(
make_url(platform.database_url.get_secret_value()).set(drivername="postgresql+asyncpg"),
hide_parameters=True,
)
try:
async with engine.connect() as connection:
member = (
(
await connection.execute(
select(ChatMemberRow.chat_id, ChatMemberRow.user_id).limit(1)
)
)
.mappings()
.first()
)
if member is None:
raise SystemExit("нет ни одного чата: сначала make test")
ids = [uuid4() for _ in range(count)]
pending = iter(ids)
completed = 0
started = time.monotonic()
async with s3_client(
endpoint=settings.s3_endpoint,
region=settings.s3_region,
access_key_id=settings.s3_access_key_id.get_secret_value(),
secret_access_key=settings.s3_secret_access_key.get_secret_value(),
connect_timeout=settings.s3_connect_timeout_seconds,
read_timeout=settings.s3_read_timeout_seconds,
max_attempts=3,
) as client:
storage = S3ObjectStorage(client, settings.s3_bucket)
async def worker() -> None:
nonlocal completed
for attachment_id in pending:
await storage.put(f"attachments/{attachment_id}", body, "image/png")
completed += 1
if completed % 10000 == 0 or completed == count:
print(json.dumps({"progress": completed, "target": count}), flush=True)
await asyncio.gather(*(worker() for _ in range(concurrency)))
stored = time.monotonic()
stamp = datetime.now(UTC) - timedelta(hours=2)
records = [
(
attachment_id,
member["chat_id"],
member["user_id"],
f"seed-{number}.png",
"image/png",
len(body),
digest,
8,
8,
f"attachments/{attachment_id}",
"ready",
stamp,
stamp,
)
for number, attachment_id in enumerate(ids)
]
async with engine.begin() as connection:
for start in range(0, count, 1000):
await connection.execute(
insert(AttachmentRow),
[dict(zip(COLUMNS, row, strict=True)) for row in records[start : start + 1000]],
)
total = await connection.scalar(select(func.count()).select_from(AttachmentRow))
finally:
await engine.dispose()
return {
"added": count,
"rows_total": total,
"object_bytes": len(body),
"seconds": {
"s3_put": round(stored - started, 3),
"pg_insert": round(time.monotonic() - stored, 3),
},
}
def main() -> None:
parser = argparse.ArgumentParser(description="Наполнить bucket и attachments N вложениями")
parser.add_argument("--count", type=int, required=True)
parser.add_argument("--concurrency", type=int, default=32)
args = parser.parse_args()
print(json.dumps(asyncio.run(seed(args.count, args.concurrency))))
if __name__ == "__main__":
main()