reconcile-inject.sh and reconcile_check.py come from the prepared kit: the first breaks either side on purpose (an object without a row, a removed object, another size, the same size with other bytes, an old intent with and without its object), the second is an oracle outside the API that classifies every row and object from psql and rc, hashes objects with --deep and reports ready attachments lost against a saved set. reconcile_track.py demo uploads through the balancer, links some attachments to messages, injects each class on both sides with old and fresh variants, and checks that a deep dry-run finds exactly what was injected and changes nothing, that apply fixes exactly that and a second apply does nothing, that broken rows and their message links survive in quarantine with metadata 200 and content 404, that a restored object heals, that a second pass is skipped while the lock is held, and that passes in a loop during 75 s of uploads lose nothing. Intents are injected 300 s old, between the reconciler grace and the API sweep grace, so only the reconciler removes them. reconcile_track.py cost snapshots docker stats of the whole host, pins S3, PostgreSQL and the reconciler to their own CPUs, fills the stand to each N with seed_storage.py run in the API image (parallel PutObject and COPY), and records three dry runs per N and mode with a median table. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01V1d3RLHNBBMA6UqBLLJv27
113 lines
3.5 KiB
Python
113 lines
3.5 KiB
Python
import argparse
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import time
|
|
from datetime import UTC, datetime, timedelta
|
|
from io import BytesIO
|
|
from uuid import uuid4
|
|
|
|
import asyncpg
|
|
from PIL import Image
|
|
|
|
from messenger.attachments.config import AttachmentsSettings
|
|
from messenger.attachments.infrastructure.s3 import S3ObjectStorage, s3_client
|
|
from messenger.shared.config import PlatformSettings
|
|
|
|
COLUMNS = (
|
|
"id",
|
|
"chat_id",
|
|
"owner_id",
|
|
"filename",
|
|
"content_type",
|
|
"size_bytes",
|
|
"sha256",
|
|
"width",
|
|
"height",
|
|
"object_key",
|
|
"status",
|
|
"created_at",
|
|
"updated_at",
|
|
)
|
|
|
|
|
|
def picture() -> bytes:
|
|
buffer = BytesIO()
|
|
Image.effect_noise((8, 8), 64).convert("RGB").save(buffer, "PNG")
|
|
return buffer.getvalue()
|
|
|
|
|
|
async def seed(count: int, concurrency: int) -> dict[str, object]:
|
|
platform, settings = PlatformSettings(), AttachmentsSettings()
|
|
body = picture()
|
|
digest = hashlib.sha256(body).hexdigest()
|
|
connection = await asyncpg.connect(platform.database_url.get_secret_value())
|
|
try:
|
|
member = await connection.fetchrow("SELECT chat_id, user_id FROM chat_members LIMIT 1")
|
|
if member is None:
|
|
raise SystemExit("нет ни одного чата: сначала make test")
|
|
ids = [uuid4() for _ in range(count)]
|
|
pending = iter(ids)
|
|
started = time.monotonic()
|
|
async with s3_client(
|
|
endpoint=settings.s3_endpoint,
|
|
region=settings.s3_region,
|
|
access_key_id=settings.s3_access_key_id.get_secret_value(),
|
|
secret_access_key=settings.s3_secret_access_key.get_secret_value(),
|
|
connect_timeout=settings.s3_connect_timeout_seconds,
|
|
read_timeout=settings.s3_read_timeout_seconds,
|
|
max_attempts=3,
|
|
) as client:
|
|
storage = S3ObjectStorage(client, settings.s3_bucket)
|
|
|
|
async def worker() -> None:
|
|
for attachment_id in pending:
|
|
await storage.put(f"attachments/{attachment_id}", body, "image/png")
|
|
|
|
await asyncio.gather(*(worker() for _ in range(concurrency)))
|
|
stored = time.monotonic()
|
|
stamp = datetime.now(UTC) - timedelta(hours=2)
|
|
records = [
|
|
(
|
|
attachment_id,
|
|
member["chat_id"],
|
|
member["user_id"],
|
|
f"seed-{number}.png",
|
|
"image/png",
|
|
len(body),
|
|
digest,
|
|
8,
|
|
8,
|
|
f"attachments/{attachment_id}",
|
|
"ready",
|
|
stamp,
|
|
stamp,
|
|
)
|
|
for number, attachment_id in enumerate(ids)
|
|
]
|
|
await connection.copy_records_to_table("attachments", records=records, columns=COLUMNS)
|
|
total = await connection.fetchval("SELECT count(*) FROM attachments")
|
|
finally:
|
|
await connection.close()
|
|
return {
|
|
"added": count,
|
|
"rows_total": total,
|
|
"object_bytes": len(body),
|
|
"seconds": {
|
|
"s3_put": round(stored - started, 3),
|
|
"pg_copy": round(time.monotonic() - stored, 3),
|
|
},
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
parser = argparse.ArgumentParser(description="Наполнить bucket и attachments N вложениями")
|
|
parser.add_argument("--count", type=int, required=True)
|
|
parser.add_argument("--concurrency", type=int, default=32)
|
|
args = parser.parse_args()
|
|
print(json.dumps(asyncio.run(seed(args.count, args.concurrency))))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|