Source: immutable lab5 snapshot f88e9ba43cdfd7eb6a5a7ae5feb4247b4e820071; forward the allowed diff from 64d5c67. Carry SQLAlchemy, Alembic, reliability fixes, versioned jobs, infrastructure and student tests without changing protected coursework files. Validation: make check, validate, lint; 515 student tests passed with one Linux-only skip; public smoke with telemetry enabled and repeat migration passed. Stop the owned stack and retain volumes. Preserve the existing OTel work in progress; grade implementation remains a separate phase.
140 lines
4.6 KiB
Python
140 lines
4.6 KiB
Python
import argparse
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import time
|
|
from datetime import UTC, datetime, timedelta
|
|
from io import BytesIO
|
|
from uuid import uuid4
|
|
|
|
from PIL import Image
|
|
from sqlalchemy import func, insert, make_url, select
|
|
from sqlalchemy.ext.asyncio import create_async_engine
|
|
|
|
from messenger.attachments.config import AttachmentsSettings
|
|
from messenger.attachments.infrastructure.postgres import AttachmentRow
|
|
from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client
|
|
from messenger.identity.infrastructure.postgres import UserRow
|
|
from messenger.messaging.infrastructure.postgres import ChatMemberRow
|
|
from messenger.shared.config import PlatformSettings
|
|
|
|
MODELS = (UserRow, ChatMemberRow, AttachmentRow)
|
|
COLUMNS = (
|
|
"id",
|
|
"chat_id",
|
|
"owner_id",
|
|
"filename",
|
|
"content_type",
|
|
"size_bytes",
|
|
"sha256",
|
|
"width",
|
|
"height",
|
|
"object_key",
|
|
"status",
|
|
"created_at",
|
|
"updated_at",
|
|
)
|
|
|
|
|
|
def picture() -> bytes:
|
|
buffer = BytesIO()
|
|
Image.effect_noise((8, 8), 64).convert("RGB").save(buffer, "PNG")
|
|
return buffer.getvalue()
|
|
|
|
|
|
async def seed(count: int, concurrency: int) -> dict[str, object]:
|
|
platform, settings = PlatformSettings(), AttachmentsSettings()
|
|
body = picture()
|
|
digest = hashlib.sha256(body).hexdigest()
|
|
engine = create_async_engine(
|
|
make_url(platform.database_url.get_secret_value()).set(drivername="postgresql+asyncpg"),
|
|
hide_parameters=True,
|
|
)
|
|
try:
|
|
async with engine.connect() as connection:
|
|
member = (
|
|
(
|
|
await connection.execute(
|
|
select(ChatMemberRow.chat_id, ChatMemberRow.user_id).limit(1)
|
|
)
|
|
)
|
|
.mappings()
|
|
.first()
|
|
)
|
|
if member is None:
|
|
raise SystemExit("нет ни одного чата: сначала make test")
|
|
ids = [uuid4() for _ in range(count)]
|
|
pending = iter(ids)
|
|
completed = 0
|
|
started = time.monotonic()
|
|
async with s3_client(
|
|
endpoint=settings.s3_endpoint,
|
|
region=settings.s3_region,
|
|
access_key_id=settings.s3_access_key_id.get_secret_value(),
|
|
secret_access_key=settings.s3_secret_access_key.get_secret_value(),
|
|
connect_timeout=settings.s3_connect_timeout_seconds,
|
|
read_timeout=settings.s3_read_timeout_seconds,
|
|
max_attempts=3,
|
|
) as client:
|
|
storage = S3ObjectStorage(client, settings.s3_bucket)
|
|
|
|
async def worker() -> None:
|
|
nonlocal completed
|
|
for attachment_id in pending:
|
|
await storage.put(f"attachments/{attachment_id}", body, "image/png")
|
|
completed += 1
|
|
if completed % 10000 == 0 or completed == count:
|
|
print(json.dumps({"progress": completed, "target": count}), flush=True)
|
|
|
|
await asyncio.gather(*(worker() for _ in range(concurrency)))
|
|
stored = time.monotonic()
|
|
stamp = datetime.now(UTC) - timedelta(hours=2)
|
|
records = [
|
|
(
|
|
attachment_id,
|
|
member["chat_id"],
|
|
member["user_id"],
|
|
f"seed-{number}.png",
|
|
"image/png",
|
|
len(body),
|
|
digest,
|
|
8,
|
|
8,
|
|
f"attachments/{attachment_id}",
|
|
"ready",
|
|
stamp,
|
|
stamp,
|
|
)
|
|
for number, attachment_id in enumerate(ids)
|
|
]
|
|
async with engine.begin() as connection:
|
|
for start in range(0, count, 1000):
|
|
await connection.execute(
|
|
insert(AttachmentRow),
|
|
[dict(zip(COLUMNS, row, strict=True)) for row in records[start : start + 1000]],
|
|
)
|
|
total = await connection.scalar(select(func.count()).select_from(AttachmentRow))
|
|
finally:
|
|
await engine.dispose()
|
|
return {
|
|
"added": count,
|
|
"rows_total": total,
|
|
"object_bytes": len(body),
|
|
"seconds": {
|
|
"s3_put": round(stored - started, 3),
|
|
"pg_insert": round(time.monotonic() - stored, 3),
|
|
},
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Наполнить bucket и attachments N вложениями")
|
|
parser.add_argument("--count", type=int, required=True)
|
|
parser.add_argument("--concurrency", type=int, default=32)
|
|
args = parser.parse_args()
|
|
print(json.dumps(asyncio.run(seed(args.count, args.concurrency))))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|