Files
mtusi2026lab2/student-tests/tools/seed_storage.py
T
ArtyomandClaude Opus 5.5 a335f654ed Add the track 2 fault injection, invariant check and experiment driver
reconcile-inject.sh and reconcile_check.py come from the prepared kit:
the first breaks either side on purpose (an object without a row, a
removed object, another size, the same size with other bytes, an old
intent with and without its object), the second is an oracle outside
the API that classifies every row and object from psql and rc, hashes
objects with --deep and reports ready attachments lost against a saved
set.

reconcile_track.py demo uploads through the balancer, links some
attachments to messages, injects each class on both sides with old and
fresh variants, and checks that a deep dry-run finds exactly what was
injected and changes nothing, that apply fixes exactly that and a
second apply does nothing, that broken rows and their message links
survive in quarantine with metadata 200 and content 404, that a
restored object heals, that a second pass is skipped while the lock is
held, and that passes in a loop during 75 s of uploads lose nothing.
Intents are injected 300 s old, between the reconciler grace and the
API sweep grace, so only the reconciler removes them.

reconcile_track.py cost snapshots docker stats of the whole host, pins
S3, PostgreSQL and the reconciler to their own CPUs, fills the stand to
each N with seed_storage.py run in the API image (parallel PutObject and
COPY), and records three dry runs per N and mode with a median table.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
2026-10-29 03:23:59 +03:00

113 lines
3.5 KiB
Python

import argparse
import asyncio
import hashlib
import json
import time
from datetime import UTC, datetime, timedelta
from io import BytesIO
from uuid import uuid4
import asyncpg
from PIL import Image
from messenger.attachments.config import AttachmentsSettings
from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client
from messenger.shared.config import PlatformSettings
COLUMNS = (
"id",
"chat_id",
"owner_id",
"filename",
"content_type",
"size_bytes",
"sha256",
"width",
"height",
"object_key",
"status",
"created_at",
"updated_at",
)
def picture() -> bytes:
buffer = BytesIO()
Image.effect_noise((8, 8), 64).convert("RGB").save(buffer, "PNG")
return buffer.getvalue()
async def seed(count: int, concurrency: int) -> dict[str, object]:
platform, settings = PlatformSettings(), AttachmentsSettings()
body = picture()
digest = hashlib.sha256(body).hexdigest()
connection = await asyncpg.connect(platform.database_url.get_secret_value())
try:
member = await connection.fetchrow("SELECT chat_id, user_id FROM chat_members LIMIT 1")
if member is None:
raise SystemExit("нет ни одного чата: сначала make test")
ids = [uuid4() for _ in range(count)]
pending = iter(ids)
started = time.monotonic()
async with s3_client(
endpoint=settings.s3_endpoint,
region=settings.s3_region,
access_key_id=settings.s3_access_key_id.get_secret_value(),
secret_access_key=settings.s3_secret_access_key.get_secret_value(),
connect_timeout=settings.s3_connect_timeout_seconds,
read_timeout=settings.s3_read_timeout_seconds,
max_attempts=3,
) as client:
storage = S3ObjectStorage(client, settings.s3_bucket)
async def worker() -> None:
for attachment_id in pending:
await storage.put(f"attachments/{attachment_id}", body, "image/png")
await asyncio.gather(*(worker() for _ in range(concurrency)))
stored = time.monotonic()
stamp = datetime.now(UTC) - timedelta(hours=2)
records = [
(
attachment_id,
member["chat_id"],
member["user_id"],
f"seed-{number}.png",
"image/png",
len(body),
digest,
8,
8,
f"attachments/{attachment_id}",
"ready",
stamp,
stamp,
)
for number, attachment_id in enumerate(ids)
]
await connection.copy_records_to_table("attachments", records=records, columns=COLUMNS)
total = await connection.fetchval("SELECT count(*) FROM attachments")
finally:
await connection.close()
return {
"added": count,
"rows_total": total,
"object_bytes": len(body),
"seconds": {
"s3_put": round(stored - started, 3),
"pg_copy": round(time.monotonic() - stored, 3),
},
}
def main() -> None:
parser = argparse.ArgumentParser(description="Наполнить bucket и attachments N вложениями")
parser.add_argument("--count", type=int, required=True)
parser.add_argument("--concurrency", type=int, default=32)
args = parser.parse_args()
print(json.dumps(asyncio.run(seed(args.count, args.concurrency))))
if __name__ == "__main__":
main()