forked from mtusi2026/mtusi2026lab2
ЛР 2: PostgreSQL, сессии в Valkey, кэш истории (трек 1)
This commit is contained in:
@@ -0,0 +1,595 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
type message struct {
|
||||
ID string `json:"id"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
}
|
||||
|
||||
type pos struct {
|
||||
t time.Time
|
||||
id string
|
||||
}
|
||||
|
||||
func posOf(m message) pos {
|
||||
t, _ := time.Parse(time.RFC3339Nano, m.CreatedAt)
|
||||
return pos{t, m.ID}
|
||||
}
|
||||
|
||||
func (a pos) after(b pos) bool {
|
||||
return a.t.After(b.t) || (a.t.Equal(b.t) && a.id > b.id)
|
||||
}
|
||||
|
||||
type ackedMsg struct {
|
||||
p pos
|
||||
ack time.Time
|
||||
}
|
||||
|
||||
type acked struct {
|
||||
mu sync.Mutex
|
||||
msgs []ackedMsg
|
||||
}
|
||||
|
||||
func (a *acked) add(m message, at time.Time) {
|
||||
a.mu.Lock()
|
||||
a.msgs = append(a.msgs, ackedMsg{posOf(m), at})
|
||||
a.mu.Unlock()
|
||||
}
|
||||
|
||||
func (a *acked) missing(start time.Time, after *pos, items []message) int {
|
||||
seen := make(map[string]bool, len(items))
|
||||
for _, m := range items {
|
||||
seen[m.ID] = true
|
||||
}
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
n := 0
|
||||
for _, m := range a.msgs {
|
||||
if !m.ack.Before(start) || (after != nil && !m.p.after(*after)) {
|
||||
continue
|
||||
}
|
||||
if !seen[m.p.id] {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
type page struct {
|
||||
Items []message `json:"items"`
|
||||
NextCursor *string `json:"next_cursor"`
|
||||
}
|
||||
|
||||
type client struct {
|
||||
base string
|
||||
token string
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
func (c *client) do(method, path string, body any, out any) (int, error) {
|
||||
var rd io.Reader
|
||||
if body != nil {
|
||||
b, _ := json.Marshal(body)
|
||||
rd = bytes.NewReader(b)
|
||||
}
|
||||
req, err := http.NewRequest(method, c.base+path, rd)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if c.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||
}
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, _ := io.ReadAll(resp.Body)
|
||||
if out != nil && resp.StatusCode < 300 {
|
||||
if err := json.Unmarshal(data, out); err != nil {
|
||||
return resp.StatusCode, err
|
||||
}
|
||||
}
|
||||
return resp.StatusCode, nil
|
||||
}
|
||||
|
||||
func randHex(n int) string {
|
||||
b := make([]byte, n)
|
||||
_, _ = rand.Read(b)
|
||||
return hex.EncodeToString(b)
|
||||
}
|
||||
|
||||
func mustOK(code int, err error, want int, what string) {
|
||||
if err != nil || code != want {
|
||||
fmt.Fprintf(os.Stderr, "setup %s: status %d err %v\n", what, code, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// register создаёт синтетического пользователя и входит через Basic.
|
||||
func register(base string, hc *http.Client, name string) (*client, string) {
|
||||
c := &client{base: base, http: hc}
|
||||
pw := "bench-" + randHex(12)
|
||||
var u struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
code, err := c.do("POST", "/api/v1/users", map[string]string{
|
||||
"username": name, "display_name": name, "password": pw,
|
||||
}, &u)
|
||||
mustOK(code, err, 201, "register")
|
||||
|
||||
req, _ := http.NewRequest("POST", base+"/api/v1/auth/sessions", nil)
|
||||
req.Header.Set("Authorization", "Basic "+base64.StdEncoding.EncodeToString([]byte(name+":"+pw)))
|
||||
resp, err := hc.Do(req)
|
||||
if err != nil {
|
||||
mustOK(0, err, 201, "login")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
var s struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
_ = json.NewDecoder(resp.Body).Decode(&s)
|
||||
mustOK(resp.StatusCode, nil, 201, "login")
|
||||
c.token = s.Token
|
||||
return c, u.ID
|
||||
}
|
||||
|
||||
type second struct {
|
||||
reads, readErr, writes, writeErr, rawChecks, rawViolations int
|
||||
staleChecks, staleReads int
|
||||
readLat []float64
|
||||
}
|
||||
|
||||
type recorder struct {
|
||||
mu sync.Mutex
|
||||
start time.Time
|
||||
secs map[int]*second
|
||||
status map[string]int
|
||||
reads []float64
|
||||
writes []float64
|
||||
}
|
||||
|
||||
func (r *recorder) at(t time.Time) *second {
|
||||
i := int(t.Sub(r.start) / time.Second)
|
||||
s, ok := r.secs[i]
|
||||
if !ok {
|
||||
s = &second{}
|
||||
r.secs[i] = s
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (r *recorder) read(t time.Time, lat time.Duration, code int, err error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
s := r.at(t)
|
||||
if err != nil || code != 200 {
|
||||
s.readErr++
|
||||
r.status[statusKey("read", code, err)]++
|
||||
return
|
||||
}
|
||||
ms := float64(lat.Microseconds()) / 1000
|
||||
s.reads++
|
||||
s.readLat = append(s.readLat, ms)
|
||||
r.reads = append(r.reads, ms)
|
||||
}
|
||||
|
||||
func (r *recorder) write(t time.Time, lat time.Duration, code int, err error) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
s := r.at(t)
|
||||
if err != nil || code != 201 {
|
||||
s.writeErr++
|
||||
r.status[statusKey("write", code, err)]++
|
||||
return
|
||||
}
|
||||
s.writes++
|
||||
r.writes = append(r.writes, float64(lat.Microseconds())/1000)
|
||||
}
|
||||
|
||||
func (r *recorder) stale(t time.Time, violated bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
s := r.at(t)
|
||||
s.staleChecks++
|
||||
if violated {
|
||||
s.staleReads++
|
||||
}
|
||||
}
|
||||
|
||||
func (r *recorder) raw(t time.Time, violated bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
s := r.at(t)
|
||||
s.rawChecks++
|
||||
if violated {
|
||||
s.rawViolations++
|
||||
}
|
||||
}
|
||||
|
||||
func statusKey(kind string, code int, err error) string {
|
||||
if err != nil {
|
||||
return kind + "_net_error"
|
||||
}
|
||||
return fmt.Sprintf("%s_%d", kind, code)
|
||||
}
|
||||
|
||||
func pct(xs []float64, p float64) float64 {
|
||||
if len(xs) == 0 {
|
||||
return 0
|
||||
}
|
||||
s := append([]float64(nil), xs...)
|
||||
sort.Float64s(s)
|
||||
i := int(p*float64(len(s))+0.5) - 1
|
||||
if i < 0 {
|
||||
i = 0
|
||||
}
|
||||
if i >= len(s) {
|
||||
i = len(s) - 1
|
||||
}
|
||||
return s[i]
|
||||
}
|
||||
|
||||
func tail(c *client, chat, cursor string) (ids map[string]bool, last string, code int, err error) {
|
||||
ids = map[string]bool{}
|
||||
last = cursor
|
||||
for i := 0; i < 50; i++ {
|
||||
path := "/api/v1/chats/" + chat + "/messages?limit=100"
|
||||
if last != "" {
|
||||
path += "&cursor=" + url.QueryEscape(last)
|
||||
}
|
||||
var p page
|
||||
code, err = c.do("GET", path, nil, &p)
|
||||
if err != nil || code != 200 {
|
||||
return ids, last, code, err
|
||||
}
|
||||
for _, m := range p.Items {
|
||||
ids[m.ID] = true
|
||||
}
|
||||
if p.NextCursor == nil {
|
||||
return ids, last, code, nil
|
||||
}
|
||||
last = *p.NextCursor
|
||||
}
|
||||
return ids, last, code, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
base := flag.String("base-url", "http://localhost:8080", "API base URL")
|
||||
readers := flag.Int("readers", 32, "читателей горячего чата (closed loop)")
|
||||
writers := flag.Int("writers", 2, "писателей с проверкой read-after-write")
|
||||
duration := flag.Duration("duration", 30*time.Second, "длительность замера")
|
||||
warmup := flag.Duration("warmup", 3*time.Second, "прогрев до замера (не входит в результат)")
|
||||
interval := flag.Duration("write-interval", 100*time.Millisecond, "пауза между записями одного писателя")
|
||||
seed := flag.Int("seed", 150, "сообщений в чате до начала")
|
||||
label := flag.String("label", "", "метка прогона (режим кэша)")
|
||||
out := flag.String("out", "", "куда записать JSON-результат")
|
||||
dbFile := flag.String("db-url-file", "", "файл с DSN PostgreSQL для pg_stat_statements")
|
||||
cacheFile := flag.String("cache-url-file", "", "файл с URL valkey-cache для INFO stats")
|
||||
flag.Parse()
|
||||
ctx := context.Background()
|
||||
stats := openStats(ctx, *dbFile, *cacheFile)
|
||||
|
||||
hc := &http.Client{
|
||||
Timeout: 5 * time.Second,
|
||||
Transport: &http.Transport{MaxIdleConnsPerHost: *readers + *writers + 4},
|
||||
}
|
||||
sfx := randHex(4)
|
||||
owner, ownerID := register(*base, hc, "bench_o_"+sfx)
|
||||
members := []string{ownerID}
|
||||
clients := []*client{owner}
|
||||
for i := 0; i < *writers; i++ {
|
||||
c, id := register(*base, hc, fmt.Sprintf("bench_w%d_%s", i, sfx))
|
||||
clients = append(clients, c)
|
||||
members = append(members, id)
|
||||
}
|
||||
var chat struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
code, err := owner.do("POST", "/api/v1/chats", map[string]any{"title": "bench " + sfx, "member_ids": members}, &chat)
|
||||
mustOK(code, err, 201, "chat")
|
||||
for i := 0; i < *seed; i++ {
|
||||
code, err := owner.do("POST", "/api/v1/chats/"+chat.ID+"/messages", map[string]string{"text": fmt.Sprintf("seed %d", i)}, nil)
|
||||
mustOK(code, err, 201, "seed")
|
||||
}
|
||||
|
||||
rec := &recorder{secs: map[int]*second{}, status: map[string]int{}}
|
||||
stop := make(chan struct{})
|
||||
measuring := make(chan struct{})
|
||||
var wg sync.WaitGroup
|
||||
ack := &acked{}
|
||||
|
||||
for i := 0; i < *readers; i++ {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
cursor := ""
|
||||
var cursorPos *pos
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
default:
|
||||
}
|
||||
path := "/api/v1/chats/" + chat.ID + "/messages?limit=100"
|
||||
if cursor != "" {
|
||||
path += "&cursor=" + url.QueryEscape(cursor)
|
||||
}
|
||||
var p page
|
||||
t0 := time.Now()
|
||||
code, err := owner.do("GET", path, nil, &p)
|
||||
ok := err == nil && code == 200
|
||||
if closed(measuring) {
|
||||
rec.read(t0, time.Since(t0), code, err)
|
||||
if ok && p.NextCursor == nil {
|
||||
rec.stale(t0, ack.missing(t0, cursorPos, p.Items) > 0)
|
||||
}
|
||||
}
|
||||
if ok && p.NextCursor != nil && len(p.Items) > 0 {
|
||||
cursor = *p.NextCursor
|
||||
last := posOf(p.Items[len(p.Items)-1])
|
||||
cursorPos = &last
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
for w := 0; w < *writers; w++ {
|
||||
wg.Add(1)
|
||||
c := clients[w+1]
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
cursor := ""
|
||||
n := 0
|
||||
for {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-time.After(*interval):
|
||||
}
|
||||
var m message
|
||||
t0 := time.Now()
|
||||
code, err := c.do("POST", "/api/v1/chats/"+chat.ID+"/messages", map[string]string{"text": fmt.Sprintf("raw %s %d", sfx, n)}, &m)
|
||||
n++
|
||||
isMeasuring := closed(measuring)
|
||||
if isMeasuring {
|
||||
rec.write(t0, time.Since(t0), code, err)
|
||||
}
|
||||
if err != nil || code != 201 {
|
||||
continue
|
||||
}
|
||||
ack.add(m, time.Now())
|
||||
ids, last, rc, rerr := tail(c, chat.ID, cursor)
|
||||
if rerr != nil || rc != 200 {
|
||||
continue
|
||||
}
|
||||
cursor = last
|
||||
if isMeasuring {
|
||||
rec.raw(t0, !ids[m.ID])
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
time.Sleep(*warmup)
|
||||
stats.reset(ctx)
|
||||
rec.mu.Lock()
|
||||
rec.start = time.Now()
|
||||
rec.mu.Unlock()
|
||||
close(measuring)
|
||||
time.Sleep(*duration)
|
||||
close(stop)
|
||||
wg.Wait()
|
||||
sqlStats, cacheStats := stats.collect(ctx)
|
||||
|
||||
type secOut struct {
|
||||
S int `json:"s"`
|
||||
Reads int `json:"reads"`
|
||||
ReadErrors int `json:"read_errors"`
|
||||
ReadP95Ms float64 `json:"read_p95_ms"`
|
||||
Writes int `json:"writes"`
|
||||
WriteErrors int `json:"write_errors"`
|
||||
RAWViolations int `json:"raw_violations"`
|
||||
StaleReads int `json:"stale_reads"`
|
||||
}
|
||||
var timeline []secOut
|
||||
totals := second{}
|
||||
for i := 0; i < int(*duration/time.Second)+1; i++ {
|
||||
s, ok := rec.secs[i]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
timeline = append(timeline, secOut{i, s.reads, s.readErr, pct(s.readLat, 0.95), s.writes, s.writeErr, s.rawViolations, s.staleReads})
|
||||
totals.reads += s.reads
|
||||
totals.readErr += s.readErr
|
||||
totals.writes += s.writes
|
||||
totals.writeErr += s.writeErr
|
||||
totals.rawChecks += s.rawChecks
|
||||
totals.rawViolations += s.rawViolations
|
||||
totals.staleChecks += s.staleChecks
|
||||
totals.staleReads += s.staleReads
|
||||
}
|
||||
secs := duration.Seconds()
|
||||
result := map[string]any{
|
||||
"label": *label,
|
||||
"date_utc": time.Now().UTC().Format(time.RFC3339),
|
||||
"duration_s": secs,
|
||||
"warmup_s": warmup.Seconds(),
|
||||
"readers": *readers,
|
||||
"writers": *writers,
|
||||
"write_interval_ms": interval.Milliseconds(),
|
||||
"seed_messages": *seed,
|
||||
"reads_ok": totals.reads,
|
||||
"read_errors": totals.readErr,
|
||||
"read_rps": float64(totals.reads) / secs,
|
||||
"read_p50_ms": pct(rec.reads, 0.50),
|
||||
"read_p95_ms": pct(rec.reads, 0.95),
|
||||
"read_p99_ms": pct(rec.reads, 0.99),
|
||||
"writes_ok": totals.writes,
|
||||
"write_errors": totals.writeErr,
|
||||
"write_p50_ms": pct(rec.writes, 0.50),
|
||||
"write_p95_ms": pct(rec.writes, 0.95),
|
||||
"raw_checks": totals.rawChecks,
|
||||
"raw_violations": totals.rawViolations,
|
||||
"stale_checks": totals.staleChecks,
|
||||
"stale_reads": totals.staleReads,
|
||||
"error_statuses": rec.status,
|
||||
"sql": sqlStats,
|
||||
"cache": cacheStats,
|
||||
"timeline": timeline,
|
||||
}
|
||||
if n, ok := sqlStats["history_page_calls"].(int64); ok && totals.reads > 0 {
|
||||
m := sqlStats["membership_calls"].(int64)
|
||||
result["sql_history_per_read"] = float64(n) / float64(totals.reads)
|
||||
result["sql_read_path_per_read"] = float64(n+m) / float64(totals.reads)
|
||||
}
|
||||
b, _ := json.MarshalIndent(result, "", " ")
|
||||
if *out != "" {
|
||||
if err := os.WriteFile(*out, b, 0o644); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
short := map[string]any{}
|
||||
for k, v := range result {
|
||||
if k != "timeline" {
|
||||
short[k] = v
|
||||
}
|
||||
}
|
||||
sb, _ := json.Marshal(short)
|
||||
fmt.Println(string(sb))
|
||||
}
|
||||
|
||||
func closed(ch chan struct{}) bool {
|
||||
select {
|
||||
case <-ch:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type statsConn struct {
|
||||
db *pgx.Conn
|
||||
cache *redis.Client
|
||||
}
|
||||
|
||||
func readSecret(path string) string {
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "read %s: %v\n", path, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
|
||||
func openStats(ctx context.Context, dbFile, cacheFile string) *statsConn {
|
||||
s := &statsConn{}
|
||||
if dbFile != "" {
|
||||
conn, err := pgx.Connect(ctx, readSecret(dbFile))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "stats db connect failed")
|
||||
os.Exit(1)
|
||||
}
|
||||
if _, err := conn.Exec(ctx, `CREATE EXTENSION IF NOT EXISTS pg_stat_statements`); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "pg_stat_statements: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
s.db = conn
|
||||
}
|
||||
if cacheFile != "" {
|
||||
if opts, err := redis.ParseURL(readSecret(cacheFile)); err == nil {
|
||||
s.cache = redis.NewClient(opts)
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *statsConn) reset(ctx context.Context) {
|
||||
if s.db != nil {
|
||||
if _, err := s.db.Exec(ctx, `SELECT pg_stat_statements_reset()`); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "pg_stat_statements_reset: %v\n", err)
|
||||
}
|
||||
}
|
||||
if s.cache != nil {
|
||||
_ = s.cache.ConfigResetStat(ctx).Err()
|
||||
}
|
||||
}
|
||||
|
||||
func (s *statsConn) collect(ctx context.Context) (map[string]any, map[string]any) {
|
||||
sqlOut := map[string]any{}
|
||||
if s.db != nil {
|
||||
rows, err := s.db.Query(ctx, `
|
||||
SELECT query, calls, total_exec_time
|
||||
FROM pg_stat_statements
|
||||
WHERE dbid = (SELECT oid FROM pg_database WHERE datname = current_database())
|
||||
AND query NOT ILIKE '%pg_stat_statements%'`)
|
||||
if err == nil {
|
||||
var total, history, member, insert int64
|
||||
var historyMs float64
|
||||
for rows.Next() {
|
||||
var q string
|
||||
var calls int64
|
||||
var ms float64
|
||||
if rows.Scan(&q, &calls, &ms) != nil {
|
||||
continue
|
||||
}
|
||||
total += calls
|
||||
switch {
|
||||
case strings.Contains(q, "FROM messages m"):
|
||||
history += calls
|
||||
historyMs += ms
|
||||
case strings.Contains(q, "FROM chat_members WHERE chat_id"):
|
||||
member += calls
|
||||
case strings.Contains(q, "INSERT INTO messages"):
|
||||
insert += calls
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
sqlOut["total_calls"] = total
|
||||
sqlOut["history_page_calls"] = history
|
||||
sqlOut["history_page_total_ms"] = historyMs
|
||||
sqlOut["membership_calls"] = member
|
||||
sqlOut["insert_message_calls"] = insert
|
||||
}
|
||||
}
|
||||
cacheOut := map[string]any{}
|
||||
if s.cache != nil {
|
||||
info, err := s.cache.Info(ctx, "stats").Result()
|
||||
if err != nil {
|
||||
cacheOut["error"] = "unavailable"
|
||||
}
|
||||
for _, line := range strings.Split(info, "\n") {
|
||||
k, v, ok := strings.Cut(strings.TrimSpace(line), ":")
|
||||
if ok && (k == "keyspace_hits" || k == "keyspace_misses" || k == "evicted_keys") {
|
||||
n, _ := strconv.ParseInt(v, 10, 64)
|
||||
cacheOut[k] = n
|
||||
}
|
||||
}
|
||||
}
|
||||
return sqlOut, cacheOut
|
||||
}
|
||||
Reference in New Issue
Block a user