Files

367 lines
13 KiB
Go

package main
import (
"encoding/json"
"flag"
"fmt"
"html"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
)
type latency struct {
P50 float64 `json:"p50"`
P95 float64 `json:"p95"`
P99 float64 `json:"p99"`
}
type inputResult struct {
Label string `json:"label"`
Timestamp string `json:"timestamp"`
DurationSeconds float64 `json:"duration_seconds"`
WarmupSeconds float64 `json:"warmup_seconds"`
Concurrency int `json:"concurrency"`
Replicas int `json:"replicas"`
APICPUs string `json:"api_cpus_per_replica"`
APIMemory string `json:"api_memory_per_replica"`
DBMaxConns int `json:"db_max_connections_per_replica"`
SuccessfulRequests int `json:"successful_requests"`
SuccessfulRPS float64 `json:"successful_rps"`
Errors map[string]int `json:"errors"`
LatencyMS latency `json:"latency_ms"`
Instances []string `json:"instances"`
ObservedInstances int `json:"observed_instances"`
Prometheus map[string]float64 `json:"prometheus"`
}
type rawResult struct {
Label string `json:"label"`
Timestamp string `json:"timestamp"`
DurationSeconds float64 `json:"duration_seconds"`
WarmupSeconds float64 `json:"warmup_seconds"`
Concurrency int `json:"concurrency"`
Replicas int `json:"replicas"`
APICPUs string `json:"api_cpus_per_replica"`
APIMemory string `json:"api_memory_per_replica"`
DBMaxConns int `json:"db_max_connections_per_replica"`
SuccessfulRequests int `json:"successful_requests"`
SuccessfulRPS float64 `json:"successful_rps"`
Errors map[string]int `json:"errors"`
LatencyMS latency `json:"latency_ms"`
ObservedInstances int `json:"observed_instances"`
Prometheus map[string]float64 `json:"prometheus"`
}
type aggregate struct {
Label string `json:"label"`
Repeats int `json:"repeats"`
Concurrency int `json:"concurrency"`
Replicas int `json:"replicas"`
APICPUs string `json:"api_cpus_per_replica"`
APIMemory string `json:"api_memory_per_replica"`
DBMaxConns int `json:"db_max_connections_per_replica"`
SuccessfulRPSMedian float64 `json:"successful_rps_median"`
SuccessfulRPSMin float64 `json:"successful_rps_min"`
SuccessfulRPSMax float64 `json:"successful_rps_max"`
LatencyP50MedianMS float64 `json:"latency_p50_median_ms"`
LatencyP95MedianMS float64 `json:"latency_p95_median_ms"`
LatencyP99MedianMS float64 `json:"latency_p99_median_ms"`
TotalErrors int `json:"total_errors"`
ObservedInstancesMin int `json:"observed_instances_min"`
APICPUCoresMedian float64 `json:"api_cpu_cores_median"`
APIResidentBytesMedian float64 `json:"api_resident_bytes_median"`
PostgresConnectionsMedian float64 `json:"postgres_connections_median"`
PrometheusHealthyMin float64 `json:"prometheus_healthy_replicas_min"`
}
type report struct {
GeneratedAt string `json:"generated_at"`
Method string `json:"method"`
Raw []rawResult `json:"raw"`
Summary []aggregate `json:"summary"`
}
func main() {
inputDirectory := flag.String("input", "evidence", "directory with experiment JSON files")
outputPath := flag.String("output", "evidence/lab3-scale-results.json", "combined JSON output")
chartPath := flag.String("chart", "evidence/lab3-scale-rps.svg", "SVG chart output")
flag.Parse()
results, err := readResults(*inputDirectory)
if err != nil {
fatal(err)
}
aggregates := summarize(results)
if err := writeJSON(*outputPath, report{
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
Method: "closed-loop HTTP through Nginx; median of three isolated repeats",
Raw: sanitize(results),
Summary: aggregates,
}); err != nil {
fatal(err)
}
if err := writeChart(*chartPath, aggregates); err != nil {
fatal(err)
}
}
func readResults(directory string) ([]inputResult, error) {
var paths []string
for _, pattern := range []string{"lab3-fixed-*.json", "lab3-growing-*.json", "lab3-pool-*.json"} {
matches, err := filepath.Glob(filepath.Join(directory, pattern))
if err != nil {
return nil, err
}
paths = append(paths, matches...)
}
if len(paths) == 0 {
return nil, fmt.Errorf("no experiment results in %s", directory)
}
results := make([]inputResult, 0, len(paths))
for _, path := range paths {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var result inputResult
if err := json.Unmarshal(data, &result); err != nil {
return nil, fmt.Errorf("decode %s: %w", path, err)
}
results = append(results, result)
}
sort.Slice(results, func(i, j int) bool { return results[i].Label < results[j].Label })
return results, nil
}
func sanitize(results []inputResult) []rawResult {
raw := make([]rawResult, 0, len(results))
for _, value := range results {
raw = append(raw, rawResult{
Label: value.Label, Timestamp: value.Timestamp,
DurationSeconds: value.DurationSeconds, WarmupSeconds: value.WarmupSeconds,
Concurrency: value.Concurrency, Replicas: value.Replicas,
APICPUs: value.APICPUs, APIMemory: value.APIMemory, DBMaxConns: value.DBMaxConns,
SuccessfulRequests: value.SuccessfulRequests, SuccessfulRPS: value.SuccessfulRPS,
Errors: value.Errors, LatencyMS: value.LatencyMS,
ObservedInstances: instanceCount(value), Prometheus: value.Prometheus,
})
}
return raw
}
func summarize(results []inputResult) []aggregate {
groups := make(map[string][]inputResult)
for _, value := range results {
groups[withoutRepeat(value.Label)] = append(groups[withoutRepeat(value.Label)], value)
}
labels := make([]string, 0, len(groups))
for label := range groups {
labels = append(labels, label)
}
sort.Strings(labels)
aggregates := make([]aggregate, 0, len(labels))
for _, label := range labels {
values := groups[label]
first := values[0]
rps := numbers(values, func(value inputResult) float64 { return value.SuccessfulRPS })
p50 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P50 })
p95 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P95 })
p99 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P99 })
cpu := metricNumbers(values, "api_cpu_cores")
memory := metricNumbers(values, "api_resident_bytes")
connections := metricNumbers(values, "postgres_connections")
healthy := metricNumbers(values, "healthy_replicas")
aggregate := aggregate{
Label: label, Repeats: len(values), Concurrency: first.Concurrency, Replicas: first.Replicas,
APICPUs: first.APICPUs, APIMemory: first.APIMemory, DBMaxConns: first.DBMaxConns,
SuccessfulRPSMedian: median(rps), SuccessfulRPSMin: minimum(rps), SuccessfulRPSMax: maximum(rps),
LatencyP50MedianMS: median(p50), LatencyP95MedianMS: median(p95), LatencyP99MedianMS: median(p99),
ObservedInstancesMin: instanceCount(first), APICPUCoresMedian: median(cpu),
APIResidentBytesMedian: median(memory), PostgresConnectionsMedian: median(connections),
PrometheusHealthyMin: minimum(healthy),
}
for _, value := range values {
if count := instanceCount(value); count < aggregate.ObservedInstancesMin {
aggregate.ObservedInstancesMin = count
}
for _, count := range value.Errors {
aggregate.TotalErrors += count
}
}
aggregates = append(aggregates, aggregate)
}
return aggregates
}
func instanceCount(value inputResult) int {
if value.ObservedInstances > 0 {
return value.ObservedInstances
}
return len(value.Instances)
}
func withoutRepeat(label string) string {
index := strings.LastIndex(label, "-n")
if index == -1 {
return label
}
if _, err := strconv.Atoi(label[index+2:]); err != nil {
return label
}
return label[:index]
}
func numbers(values []inputResult, value func(inputResult) float64) []float64 {
result := make([]float64, 0, len(values))
for _, item := range values {
result = append(result, value(item))
}
return result
}
func metricNumbers(values []inputResult, name string) []float64 {
result := make([]float64, 0, len(values))
for _, item := range values {
if value, ok := item.Prometheus[name]; ok {
result = append(result, value)
}
}
return result
}
func median(values []float64) float64 {
if len(values) == 0 {
return 0
}
values = append([]float64(nil), values...)
sort.Float64s(values)
middle := len(values) / 2
if len(values)%2 == 1 {
return values[middle]
}
return (values[middle-1] + values[middle]) / 2
}
func minimum(values []float64) float64 {
if len(values) == 0 {
return 0
}
minimum := values[0]
for _, value := range values[1:] {
if value < minimum {
minimum = value
}
}
return minimum
}
func maximum(values []float64) float64 {
if len(values) == 0 {
return 0
}
maximum := values[0]
for _, value := range values[1:] {
if value > maximum {
maximum = value
}
}
return maximum
}
func writeJSON(path string, value any) error {
data, err := json.MarshalIndent(value, "", " ")
if err != nil {
return err
}
return os.WriteFile(path, append(data, '\n'), 0o644)
}
func writeChart(path string, values []aggregate) error {
const width, height = 960, 560
const left, right, top, bottom = 85.0, 240.0, 45.0, 70.0
plotWidth := width - left - right
plotHeight := height - top - bottom
var selected []aggregate
var maxRPS float64
for _, value := range values {
if strings.HasPrefix(value.Label, "fixed-") || strings.HasPrefix(value.Label, "growing-") {
selected = append(selected, value)
if value.SuccessfulRPSMedian > maxRPS {
maxRPS = value.SuccessfulRPSMedian
}
}
}
if maxRPS == 0 {
return fmt.Errorf("no scale results for chart")
}
maxRPS *= 1.1
var output strings.Builder
fmt.Fprintf(&output, `<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d" viewBox="0 0 %d %d">`, width, height, width, height)
output.WriteString(`<rect width="100%" height="100%" fill="white"/><style>text{font-family:system-ui,sans-serif;fill:#202124}.axis{stroke:#5f6368;stroke-width:1}.grid{stroke:#dadce0;stroke-width:1}.line{fill:none;stroke-width:2.5}.point{stroke:white;stroke-width:1.5}</style>`)
fmt.Fprintf(&output, `<text x="%g" y="25" font-size="18" font-weight="600">Median successful RPS by replica count</text>`, left)
for step := 0; step <= 5; step++ {
y := top + plotHeight*float64(5-step)/5
value := maxRPS * float64(step) / 5
fmt.Fprintf(&output, `<line class="grid" x1="%g" y1="%g" x2="%g" y2="%g"/><text x="%g" y="%g" text-anchor="end" font-size="12">%.0f</text>`, left, y, left+plotWidth, y, left-10, y+4, value)
}
fmt.Fprintf(&output, `<line class="axis" x1="%g" y1="%g" x2="%g" y2="%g"/><line class="axis" x1="%g" y1="%g" x2="%g" y2="%g"/>`, left, top, left, top+plotHeight, left, top+plotHeight, left+plotWidth, top+plotHeight)
for index, replicas := range []int{1, 2, 4} {
x := left + plotWidth*float64(index)/2
fmt.Fprintf(&output, `<text x="%g" y="%g" text-anchor="middle" font-size="13">%d</text>`, x, top+plotHeight+25, replicas)
}
fmt.Fprintf(&output, `<text x="%g" y="%d" text-anchor="middle" font-size="13">API replicas</text>`, left+plotWidth/2, height-15)
colors := []string{"#1a73e8", "#d93025", "#188038", "#9334e6", "#f29900", "#00897b"}
type series struct {
name string
values []aggregate
}
groups := make(map[string][]aggregate)
for _, value := range selected {
name := strings.Split(value.Label, "-r")[0] + ", c=" + strconv.Itoa(value.Concurrency)
groups[name] = append(groups[name], value)
}
var names []string
for name := range groups {
names = append(names, name)
}
sort.Strings(names)
allSeries := make([]series, 0, len(names))
for _, name := range names {
points := groups[name]
sort.Slice(points, func(i, j int) bool { return points[i].Replicas < points[j].Replicas })
allSeries = append(allSeries, series{name: name, values: points})
}
for index, current := range allSeries {
color := colors[index%len(colors)]
var coordinates []string
for _, point := range current.values {
xIndex := map[int]float64{1: 0, 2: 1, 4: 2}[point.Replicas]
x := left + plotWidth*xIndex/2
y := top + plotHeight*(1-point.SuccessfulRPSMedian/maxRPS)
coordinates = append(coordinates, fmt.Sprintf("%g,%g", x, y))
fmt.Fprintf(&output, `<circle class="point" cx="%g" cy="%g" r="4" fill="%s"/>`, x, y, color)
}
fmt.Fprintf(&output, `<polyline class="line" points="%s" stroke="%s"/>`, strings.Join(coordinates, " "), color)
legendY := top + 20 + float64(index*28)
fmt.Fprintf(&output, `<line x1="%g" y1="%g" x2="%g" y2="%g" stroke="%s" stroke-width="3"/><text x="%g" y="%g" font-size="12">%s</text>`, left+plotWidth+25, legendY, left+plotWidth+50, legendY, color, left+plotWidth+58, legendY+4, html.EscapeString(current.name))
}
output.WriteString(`</svg>`)
return os.WriteFile(path, []byte(output.String()), 0o644)
}
func fatal(err error) {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}