140 lines
3.7 KiB
Go
140 lines
3.7 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"encoding/csv"
|
|
"encoding/json"
|
|
"io"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// scraper samples the HAProxy statistics and every replica's expvar once a
|
|
// second; replicas are found through Docker DNS, as HAProxy finds them.
|
|
type scraper struct {
|
|
statsURL string
|
|
apiHost string
|
|
debugPort string
|
|
http *http.Client
|
|
}
|
|
|
|
type scrapeLine struct {
|
|
T float64 `json:"t"`
|
|
HAProxy []map[string]string `json:"haproxy,omitempty"`
|
|
Replicas []json.RawMessage `json:"replicas,omitempty"`
|
|
}
|
|
|
|
// haproxyFields are the columns of the statistics CSV that are kept.
|
|
var haproxyFields = []string{"svname", "status", "scur", "qcur", "stot", "rate", "rtime", "ttime", "hrsp_5xx", "chkfail", "econ", "eresp", "wretr", "wredis"}
|
|
|
|
func (s scraper) run(ctx context.Context, start time.Time, path string) error {
|
|
f, err := os.Create(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
enc := json.NewEncoder(f)
|
|
ticker := time.NewTicker(time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return nil
|
|
case <-ticker.C:
|
|
}
|
|
line := scrapeLine{T: time.Since(start).Seconds(), HAProxy: s.haproxy(ctx), Replicas: s.replicas(ctx)}
|
|
if err := enc.Encode(line); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
|
|
func (s scraper) get(ctx context.Context, url string) ([]byte, error) {
|
|
ctx, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
|
|
defer cancel()
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
resp, err := s.http.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
return io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
|
}
|
|
|
|
// haproxy keeps the rows of the replicas backend.
|
|
func (s scraper) haproxy(ctx context.Context) []map[string]string {
|
|
raw, err := s.get(ctx, s.statsURL)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
rows, err := csv.NewReader(strings.NewReader(strings.TrimPrefix(string(raw), "# "))).ReadAll()
|
|
if err != nil || len(rows) < 2 {
|
|
return nil
|
|
}
|
|
header := rows[0]
|
|
col := map[string]int{}
|
|
for i, name := range header {
|
|
col[name] = i
|
|
}
|
|
var out []map[string]string
|
|
for _, row := range rows[1:] {
|
|
if len(row) != len(header) || row[col["pxname"]] != "replicas" || strings.HasPrefix(row[col["status"]], "MAINT") {
|
|
continue
|
|
}
|
|
kept := map[string]string{}
|
|
for _, name := range haproxyFields {
|
|
if i, ok := col[name]; ok {
|
|
kept[name] = row[i]
|
|
}
|
|
}
|
|
out = append(out, kept)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// replicas returns the expvar documents of the running replicas, trimmed to
|
|
// the fields the study uses.
|
|
func (s scraper) replicas(ctx context.Context) []json.RawMessage {
|
|
addrs, err := net.DefaultResolver.LookupHost(ctx, s.apiHost)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
var out []json.RawMessage
|
|
for _, addr := range addrs {
|
|
raw, err := s.get(ctx, "http://"+net.JoinHostPort(addr, s.debugPort)+"/debug/vars")
|
|
if err != nil {
|
|
continue
|
|
}
|
|
var vars struct {
|
|
Instance string `json:"instance"`
|
|
GOMAXPROCS int `json:"gomaxprocs"`
|
|
Pool json.RawMessage `json:"pgxpool"`
|
|
Mem struct {
|
|
HeapAlloc uint64 `json:"HeapAlloc"`
|
|
Sys uint64 `json:"Sys"`
|
|
NumGC uint32 `json:"NumGC"`
|
|
PauseTotalNs uint64 `json:"PauseTotalNs"`
|
|
GCCPUFraction float64 `json:"GCCPUFraction"`
|
|
} `json:"memstats"`
|
|
}
|
|
if json.Unmarshal(raw, &vars) != nil {
|
|
continue
|
|
}
|
|
kept, err := json.Marshal(map[string]any{
|
|
"addr": addr, "instance": vars.Instance, "gomaxprocs": vars.GOMAXPROCS,
|
|
"pgxpool": vars.Pool, "heap_alloc": vars.Mem.HeapAlloc, "sys": vars.Mem.Sys,
|
|
"num_gc": vars.Mem.NumGC, "gc_pause_total_ns": vars.Mem.PauseTotalNs, "gc_cpu_fraction": vars.Mem.GCCPUFraction,
|
|
})
|
|
if err == nil {
|
|
out = append(out, kept)
|
|
}
|
|
}
|
|
return out
|
|
}
|