forked from mtusi2026/mtusi2026lab5
91 lines
4.4 KiB
Python
91 lines
4.4 KiB
Python
"""Run inside worker: migrate a real lab4 database and process its existing file."""
|
|
import asyncio
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import os
|
|
import subprocess
|
|
from urllib.parse import urlsplit, urlunsplit
|
|
from uuid import uuid4
|
|
|
|
import asyncpg
|
|
from PIL import Image
|
|
from pydantic import SecretStr
|
|
from sqlalchemy import select
|
|
from sqlalchemy.ext.asyncio import async_sessionmaker
|
|
|
|
from src.core.db.models import Attachment, AttachmentJob
|
|
from src.core.db.session import create_engine
|
|
from src.core.settings import settings
|
|
from src.core.storage import make_storage
|
|
from src.pipeline.dispatcher import cleanup
|
|
from src.pipeline.worker import Envelope, process_job
|
|
|
|
|
|
async def main():
|
|
previous = settings.database_url.get_secret_value()
|
|
database = 'migration_' + uuid4().hex
|
|
parts = urlsplit(previous)
|
|
url = urlunsplit(parts._replace(path='/' + database))
|
|
control = await asyncpg.connect(previous)
|
|
storage = make_storage(role='worker')
|
|
attachment_id, user_id, chat_id = uuid4(), uuid4(), uuid4()
|
|
key = 'originals/' + str(attachment_id)
|
|
engine = None
|
|
try:
|
|
await control.execute('CREATE DATABASE ' + database)
|
|
env = dict(os.environ, DATABASE_URL=url)
|
|
def migrate(target):
|
|
result = subprocess.run(['alembic', 'upgrade', target], env=env, capture_output=True)
|
|
assert result.returncode == 0, 'migration failed: ' + result.stderr.decode()[-500:]
|
|
await asyncio.to_thread(migrate, '008')
|
|
buffer = io.BytesIO()
|
|
Image.new('RGB', (320, 200), 'orange').save(buffer, 'PNG')
|
|
data = buffer.getvalue()
|
|
await asyncio.to_thread(storage.put_object, settings.s3_bucket, key, io.BytesIO(data), len(data), 'image/png')
|
|
connection = await asyncpg.connect(url)
|
|
try:
|
|
await connection.execute("INSERT INTO users (id,username,display_name,password_hash,created_at) VALUES ($1,'migration_user','Migration','unused',now())", user_id)
|
|
await connection.execute("INSERT INTO chats (id,title,created_at) VALUES ($1,'Migration',now())", chat_id)
|
|
await connection.execute("INSERT INTO chat_members (chat_id,user_id) VALUES ($1,$2)", chat_id, user_id)
|
|
await connection.execute("""INSERT INTO attachments
|
|
(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,created_at,updated_at)
|
|
VALUES ($1,$2,$3,'legacy.png',$4,'image/png',$5,$6,now(),now())""",
|
|
attachment_id, chat_id, user_id, key, len(data), hashlib.sha256(data).hexdigest())
|
|
finally:
|
|
await connection.close()
|
|
await asyncio.to_thread(migrate, '009')
|
|
settings.database_url = SecretStr(url)
|
|
engine = create_engine()
|
|
factory = async_sessionmaker(engine, expire_on_commit=False)
|
|
async with factory() as session:
|
|
attachment = await session.get(Attachment, attachment_id)
|
|
job = await session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == attachment_id))
|
|
assert attachment.status == 'queued' and attachment.staging_key == key
|
|
message = Envelope(schema_version=1,job_id=str(job.id),type='attachment.process.v1',
|
|
attachment_id=str(attachment_id),pipeline_version=1,attempt=1,created_at=job.created_at.isoformat())
|
|
assert await process_job(factory, message)
|
|
await cleanup(factory)
|
|
async with factory() as session:
|
|
attachment = await session.get(Attachment, attachment_id)
|
|
assert attachment.status == 'ready' and len(attachment.variants) == 3
|
|
assert attachment.staging_key is None
|
|
response = storage.get_object(settings.s3_bucket, key)
|
|
try:
|
|
assert response.read() == data, 'cleanup removed/replaced legacy original'
|
|
finally:
|
|
response.close(); response.release_conn()
|
|
print(json.dumps(dict(result='PASS', migration='008 -> 009', legacy_original_unchanged=True,
|
|
variants=3, staging_cleanup_safe=True)))
|
|
finally:
|
|
settings.database_url = SecretStr(previous)
|
|
if engine:
|
|
await engine.dispose()
|
|
await control.execute('DROP DATABASE IF EXISTS ' + database + ' WITH (FORCE)')
|
|
await control.close()
|
|
for object_key in [key] + ['variants/' + str(attachment_id) + '/v1/' + name for name in ('thumbnail','crop','optimized')]:
|
|
await asyncio.to_thread(storage.remove_object, settings.s3_bucket, object_key)
|
|
|
|
|
|
asyncio.run(main())
|