Files
mtusi2026lab2/student-tests/research_worker.py
T

93 lines
5.2 KiB
Python

"""Сравнение индексов в отдельной БД, не затрагивающее данные API."""
import json
from pathlib import Path
import statistics
import time
import uuid
import psycopg
from psycopg import sql
from psycopg.conninfo import make_conninfo
from psycopg.rows import dict_row
from src.config import setting
def percentile(values, p):
ordered=sorted(values)
return ordered[min(len(ordered)-1, int((len(ordered)-1)*p))]
def main():
name='lab2_research_'+uuid.uuid4().hex[:10]
url=setting('DATABASE_URL')
with psycopg.connect(url,autocommit=True) as admin:
admin.execute(sql.SQL('CREATE DATABASE {}').format(sql.Identifier(name)))
conn=psycopg.connect(make_conninfo(url,dbname=name),row_factory=dict_row,autocommit=True)
conn.execute("SET statement_timeout='30s'")
for path in sorted(Path('/app/migrations').glob('*.sql')):
conn.execute(path.read_text(encoding='utf-8'))
user=str(uuid.uuid4())
conn.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,'research','Синтетический пользователь','не используется для входа')",[user])
sizes=[70000,20000,9000]+[100]*10
chats=[]
seed_started=time.perf_counter()
with conn.transaction():
for i,size in enumerate(sizes):
chat=str(uuid.uuid4())
chats.append(chat)
conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s)',[chat,'Исследование '+str(i)])
conn.execute('INSERT INTO chat_members VALUES (%s,%s)',[chat,user])
conn.execute("INSERT INTO messages(id,chat_id,sender_id,text,created_at) SELECT gen_random_uuid(),%s,%s,repeat('x',128) || n, timestamptz '2026-01-01' + n * interval '1 microsecond' FROM generate_series(1,%s) n",[chat,user,size])
seed_seconds=time.perf_counter()-seed_started
conn.execute('DROP INDEX messages_history_idx')
conn.execute('VACUUM ANALYZE messages')
variants=[('chat_only','CREATE INDEX research_idx ON messages(chat_id)'),
('composite','CREATE INDEX research_idx ON messages(chat_id,created_at,id)'),
('covering','CREATE INDEX research_idx ON messages(chat_id,created_at,id) INCLUDE(sender_id,text)')]
result={'database':name,'rows':sum(sizes),'chat_sizes':sizes,'seed_seconds':seed_seconds,
'concurrency':1,'warmup_queries_per_case':3,'queries_per_case':100,'repeats_write':3,'variants':[]}
query='SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT 51'
result['query']=query
for label,ddl in variants:
started=time.perf_counter()
conn.execute(ddl)
build_seconds=time.perf_counter()-started
conn.execute('VACUUM ANALYZE messages')
cases=[]
for chat,size in zip(chats[:4],sizes[:4]):
for depth in [0.1,0.5,0.9]:
# OFFSET нужен только для выбора фиксированной точки эксперимента.
marker=conn.execute('SELECT created_at,id FROM messages WHERE chat_id=%s ORDER BY created_at,id OFFSET %s LIMIT 1',[chat,int(size*depth)]).fetchone()
params=[chat,marker['created_at'],marker['id']]
for _ in range(3):
conn.execute(query,params).fetchall()
timings=[]
for _ in range(100):
start=time.perf_counter()
rows=conn.execute(query,params).fetchall()
timings.append((time.perf_counter()-start)*1000)
assert len(rows)==min(51,size-int(size*depth)-1)
plan=conn.execute('EXPLAIN (ANALYZE,BUFFERS,FORMAT JSON) '+query,params).fetchone()['QUERY PLAN'][0]
cases.append({'chat_size':size,'depth':depth,'successful_queries':100,'errors':0,
'duration_seconds':sum(timings)/1000,'successful_rps':100000/sum(timings),
'p50_ms':percentile(timings,0.5),'p95_ms':percentile(timings,0.95),'p99_ms':percentile(timings,0.99),'plan':plan})
writes=[]
for trial in range(3):
start=time.perf_counter()
with conn.transaction():
conn.execute("INSERT INTO messages(id,chat_id,sender_id,text) SELECT gen_random_uuid(),%s,%s,'write_probe' FROM generate_series(1,1000)",[chats[0],user])
writes.append(time.perf_counter()-start)
conn.execute("DELETE FROM messages WHERE text='write_probe'")
index_bytes=conn.execute("SELECT pg_relation_size('research_idx') AS n").fetchone()['n']
result['variants'].append({'name':label,'ddl':ddl,'build_seconds':build_seconds,'index_bytes':index_bytes,
'write_1000_commit_seconds':writes,'write_median_rows_per_second':1000/statistics.median(writes),'cases':cases})
conn.execute('DROP INDEX research_idx')
conn.execute('CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id)')
conn.execute('VACUUM ANALYZE messages')
result['final_count']=conn.execute('SELECT count(*) AS n FROM messages').fetchone()['n']
assert result['final_count']==100000
conn.close()
print(json.dumps(result,ensure_ascii=False,default=str))
main()