Выполнил уровень 5, трек 2: сравнил индексы и проверил восстановление
This commit is contained in:
@@ -0,0 +1,107 @@
|
||||
"""Повторяемое исследование SQL, сбор CPU/RAM, backup и восстановление в новый том."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
os.chdir(ROOT)
|
||||
|
||||
|
||||
def command(args):
|
||||
result=subprocess.run(args,capture_output=True,text=True,encoding='utf-8',errors='replace',check=True)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def compose(*args):
|
||||
return command(['docker','compose',*args])
|
||||
|
||||
|
||||
def database_query(service,database,query):
|
||||
output=compose('exec','-T',service,'psql','-U','messenger','-d',database,'-Atc',query)
|
||||
return json.loads(output)
|
||||
|
||||
|
||||
def main():
|
||||
evidence=ROOT/'evidence'
|
||||
evidence.mkdir(exist_ok=True)
|
||||
stats=[]
|
||||
stop=threading.Event()
|
||||
def collect():
|
||||
while not stop.is_set():
|
||||
raw=command(['docker','stats','--no-stream','--format','{{json .}}',compose('ps','-q','db')])
|
||||
stats.append({'elapsed_seconds':round(time.monotonic()-started,3),'docker':json.loads(raw)})
|
||||
stop.wait(1)
|
||||
started=time.monotonic()
|
||||
collector=threading.Thread(target=collect)
|
||||
collector.start()
|
||||
try:
|
||||
source=(ROOT/'student-tests/research_worker.py').read_text(encoding='utf-8')
|
||||
proc=subprocess.run(['docker','compose','exec','-T','api','python','-'],input=source,capture_output=True,text=True,encoding='utf-8',check=True)
|
||||
result=json.loads(proc.stdout)
|
||||
finally:
|
||||
stop.set()
|
||||
collector.join()
|
||||
result['total_research_seconds']=time.monotonic()-started
|
||||
result['resources']=stats
|
||||
result['postgres_version']=compose('exec','-T','db','postgres','--version')
|
||||
(evidence/'research.json').write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
||||
print('SQL-исследование завершено: 100 000 сообщений, три индекса.')
|
||||
# Дамп содержит синтетические данные; бинарный архив остаётся вне Git.
|
||||
folder=ROOT/'backups'
|
||||
folder.mkdir(exist_ok=True)
|
||||
archive='research_'+uuid.uuid4().hex[:10]+'.dump'
|
||||
before=time.perf_counter()
|
||||
compose('exec','-T','db','pg_dump','-U','messenger','-d',result['database'],'-Fc','-f','/tmp/'+archive)
|
||||
db_id=compose('ps','-q','db')
|
||||
command(['docker','cp',db_id+':/tmp/'+archive,str(folder/archive)])
|
||||
backup_seconds=time.perf_counter()-before
|
||||
# Каждый запуск получает новый отдельный том; существующие тома не удаляются.
|
||||
restore_volume='lyakhonov_lab2_restore_'+uuid.uuid4().hex[:10]
|
||||
restore_name=restore_volume.replace('_','-')
|
||||
command(['docker','volume','create',restore_volume])
|
||||
before=time.perf_counter()
|
||||
password_file=str(ROOT/'secrets/db_password')
|
||||
command(['docker','run','-d','--name',restore_name,'--network','lyakhonov-lab2_default',
|
||||
'--cpus','1','--memory','512m','-e','POSTGRES_USER=messenger','-e','POSTGRES_DB=messenger',
|
||||
'-e','POSTGRES_PASSWORD_FILE=/run/secrets/db_password','--mount','type=bind,src='+password_file+',dst=/run/secrets/db_password,readonly',
|
||||
'-v',restore_volume+':/var/lib/postgresql/data','postgres:16.6-alpine'])
|
||||
deadline=time.monotonic()+60
|
||||
while True:
|
||||
ready=subprocess.run(['docker','exec',restore_name,'pg_isready','-h','127.0.0.1','-U','messenger','-d','messenger'],capture_output=True)
|
||||
if ready.returncode==0:
|
||||
break
|
||||
if time.monotonic()>deadline:
|
||||
raise RuntimeError('новое хранилище не готово')
|
||||
time.sleep(0.5)
|
||||
command(['docker','cp',str(folder/archive),restore_name+':/tmp/'+archive])
|
||||
restore_start=time.perf_counter()
|
||||
command(['docker','exec',restore_name,'pg_restore','-U','messenger','-d','messenger','--exit-on-error','--no-owner','/tmp/'+archive])
|
||||
restore_seconds=time.perf_counter()-restore_start
|
||||
provision_restore_seconds=time.perf_counter()-before
|
||||
counts="SELECT json_build_object('users',(SELECT count(*) FROM users),'chats',(SELECT count(*) FROM chats),'members',(SELECT count(*) FROM chat_members),'messages',(SELECT count(*) FROM messages),'sessions',(SELECT count(*) FROM sessions))"
|
||||
fingerprint="SELECT json_build_object('count',count(*),'digest',md5(string_agg(md5(id::text || chat_id::text || sender_id::text || text || created_at::text),'' ORDER BY id))) FROM messages"
|
||||
samples="SELECT json_agg(t ORDER BY chat_id,id) FROM (SELECT * FROM (SELECT id,chat_id,sender_id,text,created_at,row_number() OVER(PARTITION BY chat_id ORDER BY created_at,id) AS n,count(*) OVER(PARTITION BY chat_id) AS total FROM messages) s WHERE n=1 OR n=total OR n=total/2) t"
|
||||
def restored(query):
|
||||
return json.loads(command(['docker','exec',restore_name,'psql','-U','messenger','-d','messenger','-Atc',query]))
|
||||
checks={}
|
||||
for label,query in [('counts',counts),('all_message_fingerprint',fingerprint),('selected_messages',samples)]:
|
||||
original=database_query('db',result['database'],query)
|
||||
recovered=restored(query)
|
||||
assert original==recovered, 'расхождение после восстановления: '+label
|
||||
checks[label]={'equal':True,'value':original}
|
||||
recovery={'source_database':result['database'],'restore_container':restore_name,'new_volume':restore_volume,
|
||||
'archive_file':'backups/'+archive,'archive_bytes':(folder/archive).stat().st_size,
|
||||
'backup_seconds':backup_seconds,'restore_seconds':restore_seconds,'provision_and_restore_seconds':provision_restore_seconds,
|
||||
'checks':checks,'snapshot_message_loss':0,'passed':True}
|
||||
(evidence/'recovery.json').write_text(json.dumps(recovery,ensure_ascii=False,indent=2)+'\n',encoding='utf-8')
|
||||
print('PASS: backup восстановлен в новый том; количество, все сообщения и выборка совпали.')
|
||||
# Контейнер исследования останавливается; том сохраняется для защиты.
|
||||
command(['docker','stop',restore_name])
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user