ЛР 2: PostgreSQL, сессии в Valkey, кэш истории (трек 1)

This commit is contained in:
Нурлыгаянова Екатерина
2026-10-29 11:29:38 +03:00
parent 65003b6119
commit 1503b5db81
132 changed files with 15114 additions and 20 deletions
+595
View File
@@ -0,0 +1,595 @@
package main
import (
"bytes"
"context"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"encoding/json"
"flag"
"fmt"
"io"
"net/http"
"net/url"
"os"
"sort"
"strconv"
"strings"
"sync"
"time"
"github.com/jackc/pgx/v5"
"github.com/redis/go-redis/v9"
)
type message struct {
ID string `json:"id"`
CreatedAt string `json:"created_at"`
}
type pos struct {
t time.Time
id string
}
func posOf(m message) pos {
t, _ := time.Parse(time.RFC3339Nano, m.CreatedAt)
return pos{t, m.ID}
}
func (a pos) after(b pos) bool {
return a.t.After(b.t) || (a.t.Equal(b.t) && a.id > b.id)
}
type ackedMsg struct {
p pos
ack time.Time
}
type acked struct {
mu sync.Mutex
msgs []ackedMsg
}
func (a *acked) add(m message, at time.Time) {
a.mu.Lock()
a.msgs = append(a.msgs, ackedMsg{posOf(m), at})
a.mu.Unlock()
}
func (a *acked) missing(start time.Time, after *pos, items []message) int {
seen := make(map[string]bool, len(items))
for _, m := range items {
seen[m.ID] = true
}
a.mu.Lock()
defer a.mu.Unlock()
n := 0
for _, m := range a.msgs {
if !m.ack.Before(start) || (after != nil && !m.p.after(*after)) {
continue
}
if !seen[m.p.id] {
n++
}
}
return n
}
type page struct {
Items []message `json:"items"`
NextCursor *string `json:"next_cursor"`
}
type client struct {
base string
token string
http *http.Client
}
func (c *client) do(method, path string, body any, out any) (int, error) {
var rd io.Reader
if body != nil {
b, _ := json.Marshal(body)
rd = bytes.NewReader(b)
}
req, err := http.NewRequest(method, c.base+path, rd)
if err != nil {
return 0, err
}
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
if c.token != "" {
req.Header.Set("Authorization", "Bearer "+c.token)
}
resp, err := c.http.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()
data, _ := io.ReadAll(resp.Body)
if out != nil && resp.StatusCode < 300 {
if err := json.Unmarshal(data, out); err != nil {
return resp.StatusCode, err
}
}
return resp.StatusCode, nil
}
func randHex(n int) string {
b := make([]byte, n)
_, _ = rand.Read(b)
return hex.EncodeToString(b)
}
func mustOK(code int, err error, want int, what string) {
if err != nil || code != want {
fmt.Fprintf(os.Stderr, "setup %s: status %d err %v\n", what, code, err)
os.Exit(1)
}
}
// register создаёт синтетического пользователя и входит через Basic.
func register(base string, hc *http.Client, name string) (*client, string) {
c := &client{base: base, http: hc}
pw := "bench-" + randHex(12)
var u struct {
ID string `json:"id"`
}
code, err := c.do("POST", "/api/v1/users", map[string]string{
"username": name, "display_name": name, "password": pw,
}, &u)
mustOK(code, err, 201, "register")
req, _ := http.NewRequest("POST", base+"/api/v1/auth/sessions", nil)
req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(name+":"+pw)))
resp, err := hc.Do(req)
if err != nil {
mustOK(0, err, 201, "login")
}
defer resp.Body.Close()
var s struct {
Token string `json:"token"`
}
_ = json.NewDecoder(resp.Body).Decode(&s)
mustOK(resp.StatusCode, nil, 201, "login")
c.token = s.Token
return c, u.ID
}
type second struct {
reads, readErr, writes, writeErr, rawChecks, rawViolations int
staleChecks, staleReads int
readLat []float64
}
type recorder struct {
mu sync.Mutex
start time.Time
secs map[int]*second
status map[string]int
reads []float64
writes []float64
}
func (r *recorder) at(t time.Time) *second {
i := int(t.Sub(r.start) / time.Second)
s, ok := r.secs[i]
if !ok {
s = &second{}
r.secs[i] = s
}
return s
}
func (r *recorder) read(t time.Time, lat time.Duration, code int, err error) {
r.mu.Lock()
defer r.mu.Unlock()
s := r.at(t)
if err != nil || code != 200 {
s.readErr++
r.status[statusKey("read", code, err)]++
return
}
ms := float64(lat.Microseconds()) / 1000
s.reads++
s.readLat = append(s.readLat, ms)
r.reads = append(r.reads, ms)
}
func (r *recorder) write(t time.Time, lat time.Duration, code int, err error) {
r.mu.Lock()
defer r.mu.Unlock()
s := r.at(t)
if err != nil || code != 201 {
s.writeErr++
r.status[statusKey("write", code, err)]++
return
}
s.writes++
r.writes = append(r.writes, float64(lat.Microseconds())/1000)
}
func (r *recorder) stale(t time.Time, violated bool) {
r.mu.Lock()
defer r.mu.Unlock()
s := r.at(t)
s.staleChecks++
if violated {
s.staleReads++
}
}
func (r *recorder) raw(t time.Time, violated bool) {
r.mu.Lock()
defer r.mu.Unlock()
s := r.at(t)
s.rawChecks++
if violated {
s.rawViolations++
}
}
func statusKey(kind string, code int, err error) string {
if err != nil {
return kind + "_net_error"
}
return fmt.Sprintf("%s_%d", kind, code)
}
func pct(xs []float64, p float64) float64 {
if len(xs) == 0 {
return 0
}
s := append([]float64(nil), xs...)
sort.Float64s(s)
i := int(p*float64(len(s))+0.5) - 1
if i < 0 {
i = 0
}
if i >= len(s) {
i = len(s) - 1
}
return s[i]
}
func tail(c *client, chat, cursor string) (ids map[string]bool, last string, code int, err error) {
ids = map[string]bool{}
last = cursor
for i := 0; i < 50; i++ {
path := "/api/v1/chats/" + chat + "/messages?limit=100"
if last != "" {
path += "&cursor=" + url.QueryEscape(last)
}
var p page
code, err = c.do("GET", path, nil, &p)
if err != nil || code != 200 {
return ids, last, code, err
}
for _, m := range p.Items {
ids[m.ID] = true
}
if p.NextCursor == nil {
return ids, last, code, nil
}
last = *p.NextCursor
}
return ids, last, code, nil
}
func main() {
base := flag.String("base-url", "http://localhost:8080", "API base URL")
readers := flag.Int("readers", 32, "читателей горячего чата (closed loop)")
writers := flag.Int("writers", 2, "писателей с проверкой read-after-write")
duration := flag.Duration("duration", 30*time.Second, "длительность замера")
warmup := flag.Duration("warmup", 3*time.Second, "прогрев до замера (не входит в результат)")
interval := flag.Duration("write-interval", 100*time.Millisecond, "пауза между записями одного писателя")
seed := flag.Int("seed", 150, "сообщений в чате до начала")
label := flag.String("label", "", "метка прогона (режим кэша)")
out := flag.String("out", "", "куда записать JSON-результат")
dbFile := flag.String("db-url-file", "", "файл с DSN PostgreSQL для pg_stat_statements")
cacheFile := flag.String("cache-url-file", "", "файл с URL valkey-cache для INFO stats")
flag.Parse()
ctx := context.Background()
stats := openStats(ctx, *dbFile, *cacheFile)
hc := &http.Client{
Timeout: 5 * time.Second,
Transport: &http.Transport{MaxIdleConnsPerHost: *readers + *writers + 4},
}
sfx := randHex(4)
owner, ownerID := register(*base, hc, "bench_o_"+sfx)
members := []string{ownerID}
clients := []*client{owner}
for i := 0; i < *writers; i++ {
c, id := register(*base, hc, fmt.Sprintf("bench_w%d_%s", i, sfx))
clients = append(clients, c)
members = append(members, id)
}
var chat struct {
ID string `json:"id"`
}
code, err := owner.do("POST", "/api/v1/chats", map[string]any{"title": "bench " + sfx, "member_ids": members}, &chat)
mustOK(code, err, 201, "chat")
for i := 0; i < *seed; i++ {
code, err := owner.do("POST", "/api/v1/chats/"+chat.ID+"/messages", map[string]string{"text": fmt.Sprintf("seed %d", i)}, nil)
mustOK(code, err, 201, "seed")
}
rec := &recorder{secs: map[int]*second{}, status: map[string]int{}}
stop := make(chan struct{})
measuring := make(chan struct{})
var wg sync.WaitGroup
ack := &acked{}
for i := 0; i < *readers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
cursor := ""
var cursorPos *pos
for {
select {
case <-stop:
return
default:
}
path := "/api/v1/chats/" + chat.ID + "/messages?limit=100"
if cursor != "" {
path += "&cursor=" + url.QueryEscape(cursor)
}
var p page
t0 := time.Now()
code, err := owner.do("GET", path, nil, &p)
ok := err == nil && code == 200
if closed(measuring) {
rec.read(t0, time.Since(t0), code, err)
if ok && p.NextCursor == nil {
rec.stale(t0, ack.missing(t0, cursorPos, p.Items) > 0)
}
}
if ok && p.NextCursor != nil && len(p.Items) > 0 {
cursor = *p.NextCursor
last := posOf(p.Items[len(p.Items)-1])
cursorPos = &last
}
}
}()
}
for w := 0; w < *writers; w++ {
wg.Add(1)
c := clients[w+1]
go func() {
defer wg.Done()
cursor := ""
n := 0
for {
select {
case <-stop:
return
case <-time.After(*interval):
}
var m message
t0 := time.Now()
code, err := c.do("POST", "/api/v1/chats/"+chat.ID+"/messages", map[string]string{"text": fmt.Sprintf("raw %s %d", sfx, n)}, &m)
n++
isMeasuring := closed(measuring)
if isMeasuring {
rec.write(t0, time.Since(t0), code, err)
}
if err != nil || code != 201 {
continue
}
ack.add(m, time.Now())
ids, last, rc, rerr := tail(c, chat.ID, cursor)
if rerr != nil || rc != 200 {
continue
}
cursor = last
if isMeasuring {
rec.raw(t0, !ids[m.ID])
}
}
}()
}
time.Sleep(*warmup)
stats.reset(ctx)
rec.mu.Lock()
rec.start = time.Now()
rec.mu.Unlock()
close(measuring)
time.Sleep(*duration)
close(stop)
wg.Wait()
sqlStats, cacheStats := stats.collect(ctx)
type secOut struct {
S int `json:"s"`
Reads int `json:"reads"`
ReadErrors int `json:"read_errors"`
ReadP95Ms float64 `json:"read_p95_ms"`
Writes int `json:"writes"`
WriteErrors int `json:"write_errors"`
RAWViolations int `json:"raw_violations"`
StaleReads int `json:"stale_reads"`
}
var timeline []secOut
totals := second{}
for i := 0; i < int(*duration/time.Second)+1; i++ {
s, ok := rec.secs[i]
if !ok {
continue
}
timeline = append(timeline, secOut{i, s.reads, s.readErr, pct(s.readLat, 0.95), s.writes, s.writeErr, s.rawViolations, s.staleReads})
totals.reads += s.reads
totals.readErr += s.readErr
totals.writes += s.writes
totals.writeErr += s.writeErr
totals.rawChecks += s.rawChecks
totals.rawViolations += s.rawViolations
totals.staleChecks += s.staleChecks
totals.staleReads += s.staleReads
}
secs := duration.Seconds()
result := map[string]any{
"label": *label,
"date_utc": time.Now().UTC().Format(time.RFC3339),
"duration_s": secs,
"warmup_s": warmup.Seconds(),
"readers": *readers,
"writers": *writers,
"write_interval_ms": interval.Milliseconds(),
"seed_messages": *seed,
"reads_ok": totals.reads,
"read_errors": totals.readErr,
"read_rps": float64(totals.reads) / secs,
"read_p50_ms": pct(rec.reads, 0.50),
"read_p95_ms": pct(rec.reads, 0.95),
"read_p99_ms": pct(rec.reads, 0.99),
"writes_ok": totals.writes,
"write_errors": totals.writeErr,
"write_p50_ms": pct(rec.writes, 0.50),
"write_p95_ms": pct(rec.writes, 0.95),
"raw_checks": totals.rawChecks,
"raw_violations": totals.rawViolations,
"stale_checks": totals.staleChecks,
"stale_reads": totals.staleReads,
"error_statuses": rec.status,
"sql": sqlStats,
"cache": cacheStats,
"timeline": timeline,
}
if n, ok := sqlStats["history_page_calls"].(int64); ok && totals.reads > 0 {
m := sqlStats["membership_calls"].(int64)
result["sql_history_per_read"] = float64(n) / float64(totals.reads)
result["sql_read_path_per_read"] = float64(n+m) / float64(totals.reads)
}
b, _ := json.MarshalIndent(result, "", " ")
if *out != "" {
if err := os.WriteFile(*out, b, 0o644); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
short := map[string]any{}
for k, v := range result {
if k != "timeline" {
short[k] = v
}
}
sb, _ := json.Marshal(short)
fmt.Println(string(sb))
}
func closed(ch chan struct{}) bool {
select {
case <-ch:
return true
default:
return false
}
}
type statsConn struct {
db *pgx.Conn
cache *redis.Client
}
func readSecret(path string) string {
b, err := os.ReadFile(path)
if err != nil {
fmt.Fprintf(os.Stderr, "read %s: %v\n", path, err)
os.Exit(1)
}
return strings.TrimSpace(string(b))
}
func openStats(ctx context.Context, dbFile, cacheFile string) *statsConn {
s := &statsConn{}
if dbFile != "" {
conn, err := pgx.Connect(ctx, readSecret(dbFile))
if err != nil {
fmt.Fprintln(os.Stderr, "stats db connect failed")
os.Exit(1)
}
if _, err := conn.Exec(ctx, `CREATE EXTENSION IF NOT EXISTS pg_stat_statements`); err != nil {
fmt.Fprintf(os.Stderr, "pg_stat_statements: %v\n", err)
os.Exit(1)
}
s.db = conn
}
if cacheFile != "" {
if opts, err := redis.ParseURL(readSecret(cacheFile)); err == nil {
s.cache = redis.NewClient(opts)
}
}
return s
}
func (s *statsConn) reset(ctx context.Context) {
if s.db != nil {
if _, err := s.db.Exec(ctx, `SELECT pg_stat_statements_reset()`); err != nil {
fmt.Fprintf(os.Stderr, "pg_stat_statements_reset: %v\n", err)
}
}
if s.cache != nil {
_ = s.cache.ConfigResetStat(ctx).Err()
}
}
func (s *statsConn) collect(ctx context.Context) (map[string]any, map[string]any) {
sqlOut := map[string]any{}
if s.db != nil {
rows, err := s.db.Query(ctx, `
SELECT query, calls, total_exec_time
FROM pg_stat_statements
WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database())
AND query NOT ILIKE '%pg_stat_statements%'`)
if err == nil {
var total, history, member, insert int64
var historyMs float64
for rows.Next() {
var q string
var calls int64
var ms float64
if rows.Scan(&q, &calls, &ms) != nil {
continue
}
total += calls
switch {
case strings.Contains(q, "FROM messages m"):
history += calls
historyMs += ms
case strings.Contains(q, "FROM chat_members WHERE chat_id"):
member += calls
case strings.Contains(q, "INSERT INTO messages"):
insert += calls
}
}
rows.Close()
sqlOut["total_calls"] = total
sqlOut["history_page_calls"] = history
sqlOut["history_page_total_ms"] = historyMs
sqlOut["membership_calls"] = member
sqlOut["insert_message_calls"] = insert
}
}
cacheOut := map[string]any{}
if s.cache != nil {
info, err := s.cache.Info(ctx, "stats").Result()
if err != nil {
cacheOut["error"] = "unavailable"
}
for _, line := range strings.Split(info, "\n") {
k, v, ok := strings.Cut(strings.TrimSpace(line), ":")
if ok && (k == "keyspace_hits" || k == "keyspace_misses" || k == "evicted_keys") {
n, _ := strconv.ParseInt(v, 10, 64)
cacheOut[k] = n
}
}
}
return sqlOut, cacheOut
}