93 lines
5.2 KiB
Python
93 lines
5.2 KiB
Python
"""Сравнение индексов в отдельной БД, не затрагивающее данные API."""
|
|
import json
|
|
from pathlib import Path
|
|
import statistics
|
|
import time
|
|
import uuid
|
|
import psycopg
|
|
from psycopg import sql
|
|
from psycopg.conninfo import make_conninfo
|
|
from psycopg.rows import dict_row
|
|
from src.config import setting
|
|
|
|
|
|
def percentile(values, p):
|
|
ordered=sorted(values)
|
|
return ordered[min(len(ordered)-1, int((len(ordered)-1)*p))]
|
|
|
|
|
|
def main():
|
|
name='lab2_research_'+uuid.uuid4().hex[:10]
|
|
url=setting('DATABASE_URL')
|
|
with psycopg.connect(url,autocommit=True) as admin:
|
|
admin.execute(sql.SQL('CREATE DATABASE {}').format(sql.Identifier(name)))
|
|
conn=psycopg.connect(make_conninfo(url,dbname=name),row_factory=dict_row,autocommit=True)
|
|
conn.execute("SET statement_timeout='30s'")
|
|
for path in sorted(Path('/app/migrations').glob('*.sql')):
|
|
conn.execute(path.read_text(encoding='utf-8'))
|
|
user=str(uuid.uuid4())
|
|
conn.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,'research','Синтетический пользователь','не используется для входа')",[user])
|
|
sizes=[70000,20000,9000]+[100]*10
|
|
chats=[]
|
|
seed_started=time.perf_counter()
|
|
with conn.transaction():
|
|
for i,size in enumerate(sizes):
|
|
chat=str(uuid.uuid4())
|
|
chats.append(chat)
|
|
conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s)',[chat,'Исследование '+str(i)])
|
|
conn.execute('INSERT INTO chat_members VALUES (%s,%s)',[chat,user])
|
|
conn.execute("INSERT INTO messages(id,chat_id,sender_id,text,created_at) SELECT gen_random_uuid(),%s,%s,repeat('x',128) || n, timestamptz '2026-01-01' + n * interval '1 microsecond' FROM generate_series(1,%s) n",[chat,user,size])
|
|
seed_seconds=time.perf_counter()-seed_started
|
|
conn.execute('DROP INDEX messages_history_idx')
|
|
conn.execute('VACUUM ANALYZE messages')
|
|
variants=[('chat_only','CREATE INDEX research_idx ON messages(chat_id)'),
|
|
('composite','CREATE INDEX research_idx ON messages(chat_id,created_at,id)'),
|
|
('covering','CREATE INDEX research_idx ON messages(chat_id,created_at,id) INCLUDE(sender_id,text)')]
|
|
result={'database':name,'rows':sum(sizes),'chat_sizes':sizes,'seed_seconds':seed_seconds,
|
|
'concurrency':1,'warmup_queries_per_case':3,'queries_per_case':100,'repeats_write':3,'variants':[]}
|
|
query='SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s) ORDER BY created_at,id LIMIT 51'
|
|
result['query']=query
|
|
for label,ddl in variants:
|
|
started=time.perf_counter()
|
|
conn.execute(ddl)
|
|
build_seconds=time.perf_counter()-started
|
|
conn.execute('VACUUM ANALYZE messages')
|
|
cases=[]
|
|
for chat,size in zip(chats[:4],sizes[:4]):
|
|
for depth in [0.1,0.5,0.9]:
|
|
# OFFSET нужен только для выбора фиксированной точки эксперимента.
|
|
marker=conn.execute('SELECT created_at,id FROM messages WHERE chat_id=%s ORDER BY created_at,id OFFSET %s LIMIT 1',[chat,int(size*depth)]).fetchone()
|
|
params=[chat,marker['created_at'],marker['id']]
|
|
for _ in range(3):
|
|
conn.execute(query,params).fetchall()
|
|
timings=[]
|
|
for _ in range(100):
|
|
start=time.perf_counter()
|
|
rows=conn.execute(query,params).fetchall()
|
|
timings.append((time.perf_counter()-start)*1000)
|
|
assert len(rows)==min(51,size-int(size*depth)-1)
|
|
plan=conn.execute('EXPLAIN (ANALYZE,BUFFERS,FORMAT JSON) '+query,params).fetchone()['QUERY PLAN'][0]
|
|
cases.append({'chat_size':size,'depth':depth,'successful_queries':100,'errors':0,
|
|
'duration_seconds':sum(timings)/1000,'successful_rps':100000/sum(timings),
|
|
'p50_ms':percentile(timings,0.5),'p95_ms':percentile(timings,0.95),'p99_ms':percentile(timings,0.99),'plan':plan})
|
|
writes=[]
|
|
for trial in range(3):
|
|
start=time.perf_counter()
|
|
with conn.transaction():
|
|
conn.execute("INSERT INTO messages(id,chat_id,sender_id,text) SELECT gen_random_uuid(),%s,%s,'write_probe' FROM generate_series(1,1000)",[chats[0],user])
|
|
writes.append(time.perf_counter()-start)
|
|
conn.execute("DELETE FROM messages WHERE text='write_probe'")
|
|
index_bytes=conn.execute("SELECT pg_relation_size('research_idx') AS n").fetchone()['n']
|
|
result['variants'].append({'name':label,'ddl':ddl,'build_seconds':build_seconds,'index_bytes':index_bytes,
|
|
'write_1000_commit_seconds':writes,'write_median_rows_per_second':1000/statistics.median(writes),'cases':cases})
|
|
conn.execute('DROP INDEX research_idx')
|
|
conn.execute('CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id)')
|
|
conn.execute('VACUUM ANALYZE messages')
|
|
result['final_count']=conn.execute('SELECT count(*) AS n FROM messages').fetchone()['n']
|
|
assert result['final_count']==100000
|
|
conn.close()
|
|
print(json.dumps(result,ensure_ascii=False,default=str))
|
|
|
|
|
|
main()
|