Files
mtusi2026lab5/tools/loadgen/scrape.go
T

140 lines
3.7 KiB
Go

package main
import (
"context"
"encoding/csv"
"encoding/json"
"io"
"net"
"net/http"
"os"
"strings"
"time"
)
// scraper samples the HAProxy statistics and every replica's expvar once a
// second; replicas are found through Docker DNS, as HAProxy finds them.
type scraper struct {
statsURL string
apiHost string
debugPort string
http *http.Client
}
type scrapeLine struct {
T float64 `json:"t"`
HAProxy []map[string]string `json:"haproxy,omitempty"`
Replicas []json.RawMessage `json:"replicas,omitempty"`
}
// haproxyFields are the columns of the statistics CSV that are kept.
var haproxyFields = []string{"svname", "status", "scur", "qcur", "stot", "rate", "rtime", "ttime", "hrsp_5xx", "chkfail", "econ", "eresp", "wretr", "wredis"}
func (s scraper) run(ctx context.Context, start time.Time, path string) error {
f, err := os.Create(path)
if err != nil {
return err
}
defer f.Close()
enc := json.NewEncoder(f)
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return nil
case <-ticker.C:
}
line := scrapeLine{T: time.Since(start).Seconds(), HAProxy: s.haproxy(ctx), Replicas: s.replicas(ctx)}
if err := enc.Encode(line); err != nil {
return err
}
}
}
func (s scraper) get(ctx context.Context, url string) ([]byte, error) {
ctx, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := s.http.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return io.ReadAll(io.LimitReader(resp.Body, 1<<20))
}
// haproxy keeps the rows of the replicas backend.
func (s scraper) haproxy(ctx context.Context) []map[string]string {
raw, err := s.get(ctx, s.statsURL)
if err != nil {
return nil
}
rows, err := csv.NewReader(strings.NewReader(strings.TrimPrefix(string(raw), "# "))).ReadAll()
if err != nil || len(rows) < 2 {
return nil
}
header := rows[0]
col := map[string]int{}
for i, name := range header {
col[name] = i
}
var out []map[string]string
for _, row := range rows[1:] {
if len(row) != len(header) || row[col["pxname"]] != "replicas" || strings.HasPrefix(row[col["status"]], "MAINT") {
continue
}
kept := map[string]string{}
for _, name := range haproxyFields {
if i, ok := col[name]; ok {
kept[name] = row[i]
}
}
out = append(out, kept)
}
return out
}
// replicas returns the expvar documents of the running replicas, trimmed to
// the fields the study uses.
func (s scraper) replicas(ctx context.Context) []json.RawMessage {
addrs, err := net.DefaultResolver.LookupHost(ctx, s.apiHost)
if err != nil {
return nil
}
var out []json.RawMessage
for _, addr := range addrs {
raw, err := s.get(ctx, "http://"+net.JoinHostPort(addr, s.debugPort)+"/debug/vars")
if err != nil {
continue
}
var vars struct {
Instance string `json:"instance"`
GOMAXPROCS int `json:"gomaxprocs"`
Pool json.RawMessage `json:"pgxpool"`
Mem struct {
HeapAlloc uint64 `json:"HeapAlloc"`
Sys uint64 `json:"Sys"`
NumGC uint32 `json:"NumGC"`
PauseTotalNs uint64 `json:"PauseTotalNs"`
GCCPUFraction float64 `json:"GCCPUFraction"`
} `json:"memstats"`
}
if json.Unmarshal(raw, &vars) != nil {
continue
}
kept, err := json.Marshal(map[string]any{
"addr": addr, "instance": vars.Instance, "gomaxprocs": vars.GOMAXPROCS,
"pgxpool": vars.Pool, "heap_alloc": vars.Mem.HeapAlloc, "sys": vars.Mem.Sys,
"num_gc": vars.Mem.NumGC, "gc_pause_total_ns": vars.Mem.PauseTotalNs, "gc_cpu_fraction": vars.Mem.GCCPUFraction,
})
if err == nil {
out = append(out, kept)
}
}
return out
}