ops: add balanced replicated stack

This commit is contained in:
2026-10-02 14:46:44 +03:00
parent 563aa26a92
commit 14d7660bd2
14 changed files with 1654 additions and 0 deletions
+44
View File
@@ -0,0 +1,44 @@
FROM golang:1.26.5-alpine3.24 AS build
WORKDIR /src
COPY . .
RUN go build -o /out/messenger ./cmd/messenger
RUN go build -o /out/migrate ./cmd/migrate
RUN go build -o /out/lab3experiment ./cmd/lab3experiment
RUN go build -o /out/lab3summary ./cmd/lab3summary
FROM build AS test
CMD ["go", "test", "./internal/postgres", "./internal/session"]
FROM swaggerapi/swagger-ui:v5.31.2 AS swagger
COPY contracts/openapi.json /spec/openapi.json
ENV SWAGGER_JSON=/spec/openapi.json
FROM prom/prometheus:v3.13.3 AS prometheus
COPY monitoring/prometheus.yml /etc/prometheus/prometheus.yml
FROM grafana/grafana:13.1.6 AS grafana
COPY monitoring/grafana/provisioning /etc/grafana/provisioning
COPY monitoring/grafana/dashboards /etc/grafana/dashboards
FROM nginx:1.30.5-alpine AS loadbalancer
COPY nginx/nginx.conf /etc/nginx/nginx.conf
FROM alpine:3.24.2
RUN addgroup -S app && adduser -S -G app -u 10001 app
COPY --from=build --chown=app:app /out/messenger /app/messenger
COPY --from=build --chown=app:app /out/migrate /app/migrate
COPY --from=build --chown=app:app /out/lab3experiment /app/lab3experiment
COPY --from=build --chown=app:app /out/lab3summary /app/lab3summary
USER app
WORKDIR /app
HEALTHCHECK --interval=10s --timeout=2s --start-period=5s --retries=3 \
CMD wget -q -O /dev/null "http://127.0.0.1:${APP_PORT}/health/ready" || exit 1
CMD ["/app/messenger"]
+522
View File
@@ -0,0 +1,522 @@
package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"net"
"net/http"
"net/url"
"os"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/google/uuid"
)
type configuration struct {
baseURL string
prometheusURL string
label string
duration time.Duration
warmup time.Duration
concurrency int
replicas int
apiCPUs string
apiMemory string
dbMaxConns int
output string
}
type apiClient struct {
baseURL string
client *http.Client
token string
chatID string
}
type collector struct {
mu sync.Mutex
latencies []time.Duration
errors map[string]int
instances map[string]struct{}
}
type result struct {
Label string `json:"label"`
Timestamp string `json:"timestamp"`
GOOS string `json:"goos"`
GOARCH string `json:"goarch"`
DurationSeconds float64 `json:"duration_seconds"`
WarmupSeconds float64 `json:"warmup_seconds"`
Concurrency int `json:"concurrency"`
Replicas int `json:"replicas"`
APICPUs string `json:"api_cpus_per_replica"`
APIMemory string `json:"api_memory_per_replica"`
DBMaxConns int `json:"db_max_connections_per_replica"`
SuccessfulRequests int `json:"successful_requests"`
SuccessfulRPS float64 `json:"successful_rps"`
Errors map[string]int `json:"errors"`
LatencyMS latencySummary `json:"latency_ms"`
ObservedInstances int `json:"observed_instances"`
Prometheus map[string]float64 `json:"prometheus"`
}
type latencySummary struct {
P50 float64 `json:"p50"`
P95 float64 `json:"p95"`
P99 float64 `json:"p99"`
}
type runtimeSnapshot struct {
apiCPUSeconds float64
apiResident float64
healthyReplicas float64
nginxRequests float64
}
func main() {
if err := run(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run() error {
settings := parseFlags()
client := &apiClient{
baseURL: strings.TrimRight(settings.baseURL, "/"),
client: &http.Client{Transport: &http.Transport{
MaxIdleConns: settings.concurrency * 2,
MaxIdleConnsPerHost: settings.concurrency,
MaxConnsPerHost: settings.concurrency,
IdleConnTimeout: 30 * time.Second,
}, Timeout: 10 * time.Second},
}
if err := client.waitForReplicas(settings.replicas); err != nil {
return err
}
if err := client.bootstrap(); err != nil {
return err
}
if settings.warmup > 0 {
warmup := newCollector()
if err := executeLoad(client, settings.warmup, settings.concurrency, warmup); err != nil {
return fmt.Errorf("warmup: %w", err)
}
}
var before runtimeSnapshot
if settings.prometheusURL != "" {
snapshot, err := collectRuntimeSnapshot()
if err != nil {
return fmt.Errorf("collect metrics before load: %w", err)
}
before = snapshot
}
measurements := newCollector()
started := time.Now()
if err := executeLoad(client, settings.duration, settings.concurrency, measurements); err != nil {
return err
}
elapsed := time.Since(started)
summary := measurements.result(settings, elapsed)
if settings.prometheusURL != "" {
after, err := collectRuntimeSnapshot()
if err != nil {
return fmt.Errorf("collect metrics after load: %w", err)
}
summary.Prometheus, err = prometheusMeasurement(settings.prometheusURL, before, after, elapsed)
if err != nil {
return err
}
}
return writeResult(settings.output, summary)
}
func (c *apiClient) waitForReplicas(expected int) error {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
defer cancel()
instances := make(map[string]struct{}, expected)
for ctx.Err() == nil {
request, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/health/ready", http.NoBody)
if err != nil {
return err
}
response, err := c.client.Do(request)
if err == nil {
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
_ = response.Body.Close()
if response.StatusCode == http.StatusOK {
if instanceID := response.Header.Get("X-Instance-Id"); instanceID != "" {
instances[instanceID] = struct{}{}
}
if len(instances) == expected {
return nil
}
}
}
time.Sleep(100 * time.Millisecond)
}
return fmt.Errorf("nginx discovered %d of %d API replicas: %w", len(instances), expected, ctx.Err())
}
func parseFlags() configuration {
var settings configuration
flag.StringVar(&settings.baseURL, "base-url", "http://localhost:8080", "public Nginx base URL")
flag.StringVar(&settings.prometheusURL, "prometheus-url", "", "Prometheus base URL")
flag.StringVar(&settings.label, "label", "manual", "experiment point label")
flag.DurationVar(&settings.duration, "duration", 30*time.Second, "measurement duration")
flag.DurationVar(&settings.warmup, "warmup", 5*time.Second, "warmup duration")
flag.IntVar(&settings.concurrency, "concurrency", 16, "concurrent clients")
flag.IntVar(&settings.replicas, "replicas", 2, "API replica count")
flag.StringVar(&settings.apiCPUs, "api-cpus", "1", "CPU limit per API replica")
flag.StringVar(&settings.apiMemory, "api-memory", "256m", "memory limit per API replica")
flag.IntVar(&settings.dbMaxConns, "db-max-conns", 10, "database pool limit per API replica")
flag.StringVar(&settings.output, "output", "", "JSON output path; stdout when empty")
flag.Parse()
if settings.duration <= 0 || settings.warmup < 0 || settings.concurrency < 1 || settings.replicas < 1 || settings.dbMaxConns < 1 {
fmt.Fprintln(os.Stderr, "duration, warmup and concurrency must be positive")
os.Exit(2)
}
return settings
}
func (c *apiClient) bootstrap() error {
username := "scale_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:20]
password := "Synthetic-" + uuid.NewString()
var user struct {
ID string `json:"id"`
}
if err := c.doJSON(http.MethodPost, "/api/v1/users", "", "", map[string]string{
"username": username, "display_name": "Scale experiment", "password": password,
}, &user, http.StatusCreated); err != nil {
return fmt.Errorf("register: %w", err)
}
request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", http.NoBody)
if err != nil {
return err
}
request.SetBasicAuth(username, password)
response, err := c.client.Do(request)
if err != nil {
return fmt.Errorf("login: %w", err)
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusCreated {
return responseError("login", response)
}
var session struct {
Token string `json:"token"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil {
return fmt.Errorf("decode session: %w", err)
}
c.token = session.Token
var chat struct {
ID string `json:"id"`
}
if err := c.doJSON(http.MethodPost, "/api/v1/chats", c.token, "", map[string]any{
"title": "Scale experiment", "member_ids": []string{user.ID},
}, &chat, http.StatusCreated); err != nil {
return fmt.Errorf("create chat: %w", err)
}
c.chatID = chat.ID
return nil
}
func executeLoad(client *apiClient, duration time.Duration, concurrency int, measurements *collector) error {
ctx, cancel := context.WithTimeout(context.Background(), duration)
defer cancel()
var sequence atomic.Uint64
var group sync.WaitGroup
for range concurrency {
group.Add(1)
go func() {
defer group.Done()
for ctx.Err() == nil {
number := sequence.Add(1)
started := time.Now()
instanceID, status, err := client.createMessage(ctx, number)
if err != nil && ctx.Err() != nil {
return
}
measurements.observe(time.Since(started), instanceID, status, err)
}
}()
}
group.Wait()
if len(measurements.latencies) == 0 {
return errors.New("no successful requests")
}
return nil
}
func (c *apiClient) createMessage(ctx context.Context, sequence uint64) (string, int, error) {
payload, err := json.Marshal(map[string]string{"text": fmt.Sprintf("scale message %d", sequence)})
if err != nil {
return "", 0, err
}
request, err := http.NewRequestWithContext(
ctx, http.MethodPost, c.baseURL+"/api/v1/chats/"+c.chatID+"/messages", bytes.NewReader(payload),
)
if err != nil {
return "", 0, err
}
request.Header.Set("Authorization", "Bearer "+c.token)
request.Header.Set("Content-Type", "application/json")
request.Header.Set("Idempotency-Key", uuid.NewString())
response, err := c.client.Do(request)
if err != nil {
return "", 0, err
}
defer func() { _ = response.Body.Close() }()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20))
if response.StatusCode != http.StatusCreated {
return response.Header.Get("X-Instance-Id"), response.StatusCode, fmt.Errorf("unexpected status")
}
return response.Header.Get("X-Instance-Id"), response.StatusCode, nil
}
func newCollector() *collector {
return &collector{errors: make(map[string]int), instances: make(map[string]struct{})}
}
func (c *collector) observe(latency time.Duration, instanceID string, status int, err error) {
c.mu.Lock()
defer c.mu.Unlock()
if err != nil {
key := "network"
if status != 0 {
key = strconv.Itoa(status)
}
c.errors[key]++
return
}
c.latencies = append(c.latencies, latency)
if instanceID != "" {
c.instances[instanceID] = struct{}{}
}
}
func (c *collector) result(settings configuration, elapsed time.Duration) result {
sort.Slice(c.latencies, func(i, j int) bool { return c.latencies[i] < c.latencies[j] })
return result{
Label: settings.label, Timestamp: time.Now().UTC().Format(time.RFC3339),
GOOS: runtime.GOOS, GOARCH: runtime.GOARCH,
DurationSeconds: elapsed.Seconds(), WarmupSeconds: settings.warmup.Seconds(),
Concurrency: settings.concurrency, Replicas: settings.replicas,
APICPUs: settings.apiCPUs, APIMemory: settings.apiMemory, DBMaxConns: settings.dbMaxConns,
SuccessfulRequests: len(c.latencies),
SuccessfulRPS: float64(len(c.latencies)) / elapsed.Seconds(), Errors: c.errors,
LatencyMS: latencySummary{
P50: percentile(c.latencies, 0.50), P95: percentile(c.latencies, 0.95), P99: percentile(c.latencies, 0.99),
},
ObservedInstances: len(c.instances),
}
}
func percentile(values []time.Duration, percentile float64) float64 {
if len(values) == 0 {
return 0
}
index := int(float64(len(values)-1) * percentile)
return float64(values[index].Microseconds()) / 1000
}
func collectRuntimeSnapshot() (runtimeSnapshot, error) {
hosts, err := net.LookupHost("api")
if err != nil {
return runtimeSnapshot{}, fmt.Errorf("resolve API replicas: %w", err)
}
uniqueHosts := make(map[string]struct{}, len(hosts))
var snapshot runtimeSnapshot
for _, host := range hosts {
if _, duplicate := uniqueHosts[host]; duplicate {
continue
}
uniqueHosts[host] = struct{}{}
metrics, err := scrapeMetrics("http://" + net.JoinHostPort(host, "8080") + "/metrics")
if err != nil {
return runtimeSnapshot{}, fmt.Errorf("scrape API %s: %w", host, err)
}
cpuSeconds, err := requiredMetric(metrics, "process_cpu_seconds_total")
if err != nil {
return runtimeSnapshot{}, err
}
resident, err := requiredMetric(metrics, "process_resident_memory_bytes")
if err != nil {
return runtimeSnapshot{}, err
}
snapshot.apiCPUSeconds += cpuSeconds
snapshot.apiResident += resident
snapshot.healthyReplicas++
}
nginxMetrics, err := scrapeMetrics("http://nginx-exporter:9113/metrics")
if err != nil {
return runtimeSnapshot{}, fmt.Errorf("scrape Nginx exporter: %w", err)
}
snapshot.nginxRequests, err = requiredMetric(nginxMetrics, "nginx_http_requests_total")
if err != nil {
return runtimeSnapshot{}, err
}
return snapshot, nil
}
func scrapeMetrics(endpoint string) (map[string]float64, error) {
client := &http.Client{Timeout: 5 * time.Second}
response, err := client.Get(endpoint)
if err != nil {
return nil, err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("status %d", response.StatusCode)
}
metrics := make(map[string]float64)
scanner := bufio.NewScanner(io.LimitReader(response.Body, 8<<20))
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" || strings.HasPrefix(line, "#") {
continue
}
fields := strings.Fields(line)
if len(fields) < 2 {
continue
}
name, _, _ := strings.Cut(fields[0], "{")
value, err := strconv.ParseFloat(fields[len(fields)-1], 64)
if err != nil {
return nil, fmt.Errorf("parse metric %s: %w", name, err)
}
metrics[name] += value
}
if err := scanner.Err(); err != nil {
return nil, err
}
return metrics, nil
}
func requiredMetric(metrics map[string]float64, name string) (float64, error) {
value, found := metrics[name]
if !found {
return 0, fmt.Errorf("metric %s not found", name)
}
return value, nil
}
func prometheusMeasurement(baseURL string, before, after runtimeSnapshot, elapsed time.Duration) (map[string]float64, error) {
postgresConnections, err := prometheusQuery(baseURL, `sum(pg_stat_activity_count{datname="messenger"})`)
if err != nil {
return nil, fmt.Errorf("query PostgreSQL connections: %w", err)
}
seconds := elapsed.Seconds()
return map[string]float64{
"api_cpu_cores": (after.apiCPUSeconds - before.apiCPUSeconds) / seconds,
"api_resident_bytes": after.apiResident,
"healthy_replicas": after.healthyReplicas,
"nginx_rps": (after.nginxRequests - before.nginxRequests) / seconds,
"postgres_connections": postgresConnections,
}, nil
}
func prometheusQuery(baseURL, query string) (float64, error) {
endpoint := strings.TrimRight(baseURL, "/") + "/api/v1/query?query=" + url.QueryEscape(query)
client := &http.Client{Timeout: 5 * time.Second}
response, err := client.Get(endpoint)
if err != nil {
return 0, err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != http.StatusOK {
return 0, fmt.Errorf("prometheus status %d", response.StatusCode)
}
var payload struct {
Data struct {
Result []struct {
Value []json.RawMessage `json:"value"`
} `json:"result"`
} `json:"data"`
}
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&payload); err != nil {
return 0, err
}
var total float64
for _, item := range payload.Data.Result {
if len(item.Value) != 2 {
continue
}
var encoded string
if err := json.Unmarshal(item.Value[1], &encoded); err != nil {
return 0, err
}
value, err := strconv.ParseFloat(encoded, 64)
if err != nil {
return 0, err
}
total += value
}
return total, nil
}
func (c *apiClient) doJSON(method, path, token, idempotencyKey string, body, target any, expected int) error {
payload, err := json.Marshal(body)
if err != nil {
return err
}
request, err := http.NewRequest(method, c.baseURL+path, bytes.NewReader(payload))
if err != nil {
return err
}
request.Header.Set("Content-Type", "application/json")
if token != "" {
request.Header.Set("Authorization", "Bearer "+token)
}
if idempotencyKey != "" {
request.Header.Set("Idempotency-Key", idempotencyKey)
}
response, err := c.client.Do(request)
if err != nil {
return err
}
defer func() { _ = response.Body.Close() }()
if response.StatusCode != expected {
return responseError(method+" "+path, response)
}
return json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(target)
}
func responseError(action string, response *http.Response) error {
body, _ := io.ReadAll(io.LimitReader(response.Body, 1<<20))
return fmt.Errorf("%s status=%d body=%s", action, response.StatusCode, bytes.TrimSpace(body))
}
func writeResult(path string, value result) error {
data, err := json.MarshalIndent(value, "", " ")
if err != nil {
return err
}
data = append(data, '\n')
if path == "" {
_, err = os.Stdout.Write(data)
return err
}
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return err
}
return os.WriteFile(path, data, 0o644)
}
+366
View File
@@ -0,0 +1,366 @@
package main
import (
"encoding/json"
"flag"
"fmt"
"html"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
)
type latency struct {
P50 float64 `json:"p50"`
P95 float64 `json:"p95"`
P99 float64 `json:"p99"`
}
type inputResult struct {
Label string `json:"label"`
Timestamp string `json:"timestamp"`
DurationSeconds float64 `json:"duration_seconds"`
WarmupSeconds float64 `json:"warmup_seconds"`
Concurrency int `json:"concurrency"`
Replicas int `json:"replicas"`
APICPUs string `json:"api_cpus_per_replica"`
APIMemory string `json:"api_memory_per_replica"`
DBMaxConns int `json:"db_max_connections_per_replica"`
SuccessfulRequests int `json:"successful_requests"`
SuccessfulRPS float64 `json:"successful_rps"`
Errors map[string]int `json:"errors"`
LatencyMS latency `json:"latency_ms"`
Instances []string `json:"instances"`
ObservedInstances int `json:"observed_instances"`
Prometheus map[string]float64 `json:"prometheus"`
}
type rawResult struct {
Label string `json:"label"`
Timestamp string `json:"timestamp"`
DurationSeconds float64 `json:"duration_seconds"`
WarmupSeconds float64 `json:"warmup_seconds"`
Concurrency int `json:"concurrency"`
Replicas int `json:"replicas"`
APICPUs string `json:"api_cpus_per_replica"`
APIMemory string `json:"api_memory_per_replica"`
DBMaxConns int `json:"db_max_connections_per_replica"`
SuccessfulRequests int `json:"successful_requests"`
SuccessfulRPS float64 `json:"successful_rps"`
Errors map[string]int `json:"errors"`
LatencyMS latency `json:"latency_ms"`
ObservedInstances int `json:"observed_instances"`
Prometheus map[string]float64 `json:"prometheus"`
}
type aggregate struct {
Label string `json:"label"`
Repeats int `json:"repeats"`
Concurrency int `json:"concurrency"`
Replicas int `json:"replicas"`
APICPUs string `json:"api_cpus_per_replica"`
APIMemory string `json:"api_memory_per_replica"`
DBMaxConns int `json:"db_max_connections_per_replica"`
SuccessfulRPSMedian float64 `json:"successful_rps_median"`
SuccessfulRPSMin float64 `json:"successful_rps_min"`
SuccessfulRPSMax float64 `json:"successful_rps_max"`
LatencyP50MedianMS float64 `json:"latency_p50_median_ms"`
LatencyP95MedianMS float64 `json:"latency_p95_median_ms"`
LatencyP99MedianMS float64 `json:"latency_p99_median_ms"`
TotalErrors int `json:"total_errors"`
ObservedInstancesMin int `json:"observed_instances_min"`
APICPUCoresMedian float64 `json:"api_cpu_cores_median"`
APIResidentBytesMedian float64 `json:"api_resident_bytes_median"`
PostgresConnectionsMedian float64 `json:"postgres_connections_median"`
PrometheusHealthyMin float64 `json:"prometheus_healthy_replicas_min"`
}
type report struct {
GeneratedAt string `json:"generated_at"`
Method string `json:"method"`
Raw []rawResult `json:"raw"`
Summary []aggregate `json:"summary"`
}
func main() {
inputDirectory := flag.String("input", "evidence", "directory with experiment JSON files")
outputPath := flag.String("output", "evidence/lab3-scale-results.json", "combined JSON output")
chartPath := flag.String("chart", "evidence/lab3-scale-rps.svg", "SVG chart output")
flag.Parse()
results, err := readResults(*inputDirectory)
if err != nil {
fatal(err)
}
aggregates := summarize(results)
if err := writeJSON(*outputPath, report{
GeneratedAt: time.Now().UTC().Format(time.RFC3339),
Method: "closed-loop HTTP through Nginx; median of three isolated repeats",
Raw: sanitize(results),
Summary: aggregates,
}); err != nil {
fatal(err)
}
if err := writeChart(*chartPath, aggregates); err != nil {
fatal(err)
}
}
func readResults(directory string) ([]inputResult, error) {
var paths []string
for _, pattern := range []string{"lab3-fixed-*.json", "lab3-growing-*.json", "lab3-pool-*.json"} {
matches, err := filepath.Glob(filepath.Join(directory, pattern))
if err != nil {
return nil, err
}
paths = append(paths, matches...)
}
if len(paths) == 0 {
return nil, fmt.Errorf("no experiment results in %s", directory)
}
results := make([]inputResult, 0, len(paths))
for _, path := range paths {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var result inputResult
if err := json.Unmarshal(data, &result); err != nil {
return nil, fmt.Errorf("decode %s: %w", path, err)
}
results = append(results, result)
}
sort.Slice(results, func(i, j int) bool { return results[i].Label < results[j].Label })
return results, nil
}
func sanitize(results []inputResult) []rawResult {
raw := make([]rawResult, 0, len(results))
for _, value := range results {
raw = append(raw, rawResult{
Label: value.Label, Timestamp: value.Timestamp,
DurationSeconds: value.DurationSeconds, WarmupSeconds: value.WarmupSeconds,
Concurrency: value.Concurrency, Replicas: value.Replicas,
APICPUs: value.APICPUs, APIMemory: value.APIMemory, DBMaxConns: value.DBMaxConns,
SuccessfulRequests: value.SuccessfulRequests, SuccessfulRPS: value.SuccessfulRPS,
Errors: value.Errors, LatencyMS: value.LatencyMS,
ObservedInstances: instanceCount(value), Prometheus: value.Prometheus,
})
}
return raw
}
func summarize(results []inputResult) []aggregate {
groups := make(map[string][]inputResult)
for _, value := range results {
groups[withoutRepeat(value.Label)] = append(groups[withoutRepeat(value.Label)], value)
}
labels := make([]string, 0, len(groups))
for label := range groups {
labels = append(labels, label)
}
sort.Strings(labels)
aggregates := make([]aggregate, 0, len(labels))
for _, label := range labels {
values := groups[label]
first := values[0]
rps := numbers(values, func(value inputResult) float64 { return value.SuccessfulRPS })
p50 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P50 })
p95 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P95 })
p99 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P99 })
cpu := metricNumbers(values, "api_cpu_cores")
memory := metricNumbers(values, "api_resident_bytes")
connections := metricNumbers(values, "postgres_connections")
healthy := metricNumbers(values, "healthy_replicas")
aggregate := aggregate{
Label: label, Repeats: len(values), Concurrency: first.Concurrency, Replicas: first.Replicas,
APICPUs: first.APICPUs, APIMemory: first.APIMemory, DBMaxConns: first.DBMaxConns,
SuccessfulRPSMedian: median(rps), SuccessfulRPSMin: minimum(rps), SuccessfulRPSMax: maximum(rps),
LatencyP50MedianMS: median(p50), LatencyP95MedianMS: median(p95), LatencyP99MedianMS: median(p99),
ObservedInstancesMin: instanceCount(first), APICPUCoresMedian: median(cpu),
APIResidentBytesMedian: median(memory), PostgresConnectionsMedian: median(connections),
PrometheusHealthyMin: minimum(healthy),
}
for _, value := range values {
if count := instanceCount(value); count < aggregate.ObservedInstancesMin {
aggregate.ObservedInstancesMin = count
}
for _, count := range value.Errors {
aggregate.TotalErrors += count
}
}
aggregates = append(aggregates, aggregate)
}
return aggregates
}
func instanceCount(value inputResult) int {
if value.ObservedInstances > 0 {
return value.ObservedInstances
}
return len(value.Instances)
}
func withoutRepeat(label string) string {
index := strings.LastIndex(label, "-n")
if index == -1 {
return label
}
if _, err := strconv.Atoi(label[index+2:]); err != nil {
return label
}
return label[:index]
}
func numbers(values []inputResult, value func(inputResult) float64) []float64 {
result := make([]float64, 0, len(values))
for _, item := range values {
result = append(result, value(item))
}
return result
}
func metricNumbers(values []inputResult, name string) []float64 {
result := make([]float64, 0, len(values))
for _, item := range values {
if value, ok := item.Prometheus[name]; ok {
result = append(result, value)
}
}
return result
}
func median(values []float64) float64 {
if len(values) == 0 {
return 0
}
values = append([]float64(nil), values...)
sort.Float64s(values)
middle := len(values) / 2
if len(values)%2 == 1 {
return values[middle]
}
return (values[middle-1] + values[middle]) / 2
}
func minimum(values []float64) float64 {
if len(values) == 0 {
return 0
}
minimum := values[0]
for _, value := range values[1:] {
if value < minimum {
minimum = value
}
}
return minimum
}
func maximum(values []float64) float64 {
if len(values) == 0 {
return 0
}
maximum := values[0]
for _, value := range values[1:] {
if value > maximum {
maximum = value
}
}
return maximum
}
func writeJSON(path string, value any) error {
data, err := json.MarshalIndent(value, "", " ")
if err != nil {
return err
}
return os.WriteFile(path, append(data, '\n'), 0o644)
}
func writeChart(path string, values []aggregate) error {
const width, height = 960, 560
const left, right, top, bottom = 85.0, 240.0, 45.0, 70.0
plotWidth := width - left - right
plotHeight := height - top - bottom
var selected []aggregate
var maxRPS float64
for _, value := range values {
if strings.HasPrefix(value.Label, "fixed-") || strings.HasPrefix(value.Label, "growing-") {
selected = append(selected, value)
if value.SuccessfulRPSMedian > maxRPS {
maxRPS = value.SuccessfulRPSMedian
}
}
}
if maxRPS == 0 {
return fmt.Errorf("no scale results for chart")
}
maxRPS *= 1.1
var output strings.Builder
fmt.Fprintf(&output, `<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d" viewBox="0 0 %d %d">`, width, height, width, height)
output.WriteString(`<rect width="100%" height="100%" fill="white"/><style>text{font-family:system-ui,sans-serif;fill:#202124}.axis{stroke:#5f6368;stroke-width:1}.grid{stroke:#dadce0;stroke-width:1}.line{fill:none;stroke-width:2.5}.point{stroke:white;stroke-width:1.5}</style>`)
fmt.Fprintf(&output, `<text x="%g" y="25" font-size="18" font-weight="600">Median successful RPS by replica count</text>`, left)
for step := 0; step <= 5; step++ {
y := top + plotHeight*float64(5-step)/5
value := maxRPS * float64(step) / 5
fmt.Fprintf(&output, `<line class="grid" x1="%g" y1="%g" x2="%g" y2="%g"/><text x="%g" y="%g" text-anchor="end" font-size="12">%.0f</text>`, left, y, left+plotWidth, y, left-10, y+4, value)
}
fmt.Fprintf(&output, `<line class="axis" x1="%g" y1="%g" x2="%g" y2="%g"/><line class="axis" x1="%g" y1="%g" x2="%g" y2="%g"/>`, left, top, left, top+plotHeight, left, top+plotHeight, left+plotWidth, top+plotHeight)
for index, replicas := range []int{1, 2, 4} {
x := left + plotWidth*float64(index)/2
fmt.Fprintf(&output, `<text x="%g" y="%g" text-anchor="middle" font-size="13">%d</text>`, x, top+plotHeight+25, replicas)
}
fmt.Fprintf(&output, `<text x="%g" y="%d" text-anchor="middle" font-size="13">API replicas</text>`, left+plotWidth/2, height-15)
colors := []string{"#1a73e8", "#d93025", "#188038", "#9334e6", "#f29900", "#00897b"}
type series struct {
name string
values []aggregate
}
groups := make(map[string][]aggregate)
for _, value := range selected {
name := strings.Split(value.Label, "-r")[0] + ", c=" + strconv.Itoa(value.Concurrency)
groups[name] = append(groups[name], value)
}
var names []string
for name := range groups {
names = append(names, name)
}
sort.Strings(names)
allSeries := make([]series, 0, len(names))
for _, name := range names {
points := groups[name]
sort.Slice(points, func(i, j int) bool { return points[i].Replicas < points[j].Replicas })
allSeries = append(allSeries, series{name: name, values: points})
}
for index, current := range allSeries {
color := colors[index%len(colors)]
var coordinates []string
for _, point := range current.values {
xIndex := map[int]float64{1: 0, 2: 1, 4: 2}[point.Replicas]
x := left + plotWidth*xIndex/2
y := top + plotHeight*(1-point.SuccessfulRPSMedian/maxRPS)
coordinates = append(coordinates, fmt.Sprintf("%g,%g", x, y))
fmt.Fprintf(&output, `<circle class="point" cx="%g" cy="%g" r="4" fill="%s"/>`, x, y, color)
}
fmt.Fprintf(&output, `<polyline class="line" points="%s" stroke="%s"/>`, strings.Join(coordinates, " "), color)
legendY := top + 20 + float64(index*28)
fmt.Fprintf(&output, `<line x1="%g" y1="%g" x2="%g" y2="%g" stroke="%s" stroke-width="3"/><text x="%g" y="%g" font-size="12">%s</text>`, left+plotWidth+25, legendY, left+plotWidth+50, legendY, color, left+plotWidth+58, legendY+4, html.EscapeString(current.name))
}
output.WriteString(`</svg>`)
return os.WriteFile(path, []byte(output.String()), 0o644)
}
func fatal(err error) {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
+261
View File
@@ -0,0 +1,261 @@
services:
postgres:
image: postgres:17.6-alpine3.22
environment:
POSTGRES_USER: ${POSTGRES_USER}
POSTGRES_DB: ${POSTGRES_DB}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"]
interval: 5s
timeout: 3s
retries: 20
cpus: ${POSTGRES_CPUS}
mem_limit: ${POSTGRES_MEMORY}
restart: unless-stopped
valkey:
image: valkey/valkey:8.1.4-alpine
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec", "--requirepass", "${VALKEY_PASSWORD}"]
environment:
VALKEY_PASSWORD: ${VALKEY_PASSWORD}
volumes:
- valkey_data:/data
healthcheck:
test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$VALKEY_PASSWORD\" ping | grep -q PONG"]
interval: 5s
timeout: 3s
retries: 20
cpus: ${VALKEY_CPUS}
mem_limit: ${VALKEY_MEMORY}
restart: unless-stopped
migrate:
build: .
image: ${API_IMAGE:-mtusi-messenger:lab3}
command: ["/app/migrate"]
environment:
DATABASE_URL: ${DATABASE_URL}
DB_MIN_CONNS: ${DB_MIN_CONNS}
DB_MAX_CONNS: ${DB_MAX_CONNS}
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
MIGRATION_TIMEOUT_SECONDS: ${MIGRATION_TIMEOUT_SECONDS}
depends_on:
postgres:
condition: service_healthy
healthcheck:
disable: true
cpus: ${MIGRATE_CPUS}
mem_limit: ${MIGRATE_MEMORY}
restart: "no"
api:
build: .
image: ${API_IMAGE:-mtusi-messenger:lab3}
environment:
APP_HOST: ${APP_HOST}
APP_PORT: ${APP_PORT}
DATABASE_URL: ${DATABASE_URL}
CACHE_URL: ${CACHE_URL}
SESSION_STORE: ${SESSION_STORE}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS}
DB_MIN_CONNS: ${DB_MIN_CONNS}
DB_MAX_CONNS: ${DB_MAX_CONNS}
DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS}
DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS}
DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS}
DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS}
REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS}
HTTP_READ_HEADER_TIMEOUT_SECONDS: ${HTTP_READ_HEADER_TIMEOUT_SECONDS}
HTTP_READ_TIMEOUT_SECONDS: ${HTTP_READ_TIMEOUT_SECONDS}
HTTP_WRITE_TIMEOUT_SECONDS: ${HTTP_WRITE_TIMEOUT_SECONDS}
HTTP_IDLE_TIMEOUT_SECONDS: ${HTTP_IDLE_TIMEOUT_SECONDS}
HTTP_SHUTDOWN_TIMEOUT_SECONDS: ${HTTP_SHUTDOWN_TIMEOUT_SECONDS}
PASSWORD_MEMORY_KIB: ${PASSWORD_MEMORY_KIB}
PASSWORD_ITERATIONS: ${PASSWORD_ITERATIONS}
PASSWORD_PARALLELISM: ${PASSWORD_PARALLELISM}
PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH}
PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH}
SWAGGER_PORT: ${SWAGGER_PORT}
expose:
- "8080"
depends_on:
migrate:
condition: service_completed_successfully
valkey:
condition: service_healthy
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"]
interval: 2s
timeout: 1s
start_period: 5s
retries: 2
deploy:
replicas: ${API_REPLICAS:-2}
cpus: ${API_CPUS}
mem_limit: ${API_MEMORY}
stop_grace_period: 8s
restart: unless-stopped
nginx:
build:
context: .
target: loadbalancer
image: ${NGINX_IMAGE:-mtusi-nginx:lab3}
ports:
- "${API_BIND_ADDRESS}:8080"
depends_on:
api:
condition: service_healthy
healthcheck:
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081/stub_status || exit 1"]
interval: 2s
timeout: 1s
retries: 10
cpus: ${NGINX_CPUS}
mem_limit: ${NGINX_MEMORY}
restart: unless-stopped
swagger:
build:
context: .
target: swagger
image: ${SWAGGER_IMAGE:-mtusi-swagger:lab3}
ports:
- "${SWAGGER_BIND_ADDRESS}:8080"
depends_on:
nginx:
condition: service_healthy
cpus: ${SWAGGER_CPUS}
mem_limit: ${SWAGGER_MEMORY}
restart: unless-stopped
tests:
profiles: ["tests"]
build:
context: .
target: test
environment:
TEST_DATABASE_URL: ${DATABASE_URL}
TEST_CACHE_URL: ${CACHE_URL}
depends_on:
migrate:
condition: service_completed_successfully
valkey:
condition: service_healthy
cpus: ${TEST_CPUS}
mem_limit: ${TEST_MEMORY}
experiment:
profiles: ["research"]
build: .
image: ${API_IMAGE:-mtusi-messenger:lab3}
command:
- /app/lab3experiment
- -base-url=http://nginx:8080
- -prometheus-url=http://prometheus:9090
- -output=/output/lab3-scale.json
volumes:
- ./evidence:/output
depends_on:
nginx:
condition: service_healthy
prometheus:
condition: service_healthy
cpus: ${TEST_CPUS}
mem_limit: ${TEST_MEMORY}
postgres-exporter:
image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1
environment:
DATA_SOURCE_URI: postgres:5432/${POSTGRES_DB}?sslmode=disable
DATA_SOURCE_USER: ${POSTGRES_USER}
DATA_SOURCE_PASS: ${POSTGRES_PASSWORD}
PG_EXPORTER_COLLECTION_TIMEOUT: 5s
depends_on:
postgres:
condition: service_healthy
cpus: ${EXPORTER_CPUS}
mem_limit: ${EXPORTER_MEMORY}
restart: unless-stopped
valkey-exporter:
image: oliver006/redis_exporter:v1.89.0-alpine
environment:
REDIS_ADDR: redis://valkey:6379
REDIS_USER: default
REDIS_PASSWORD: ${VALKEY_PASSWORD}
depends_on:
valkey:
condition: service_healthy
cpus: ${EXPORTER_CPUS}
mem_limit: ${EXPORTER_MEMORY}
restart: unless-stopped
nginx-exporter:
image: nginx/nginx-prometheus-exporter:1.5.1
command: ["--nginx.scrape-uri=http://nginx:8081/stub_status"]
depends_on:
nginx:
condition: service_healthy
cpus: ${EXPORTER_CPUS}
mem_limit: ${EXPORTER_MEMORY}
restart: unless-stopped
prometheus:
build:
context: .
target: prometheus
image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab3}
command:
- --config.file=/etc/prometheus/prometheus.yml
- --storage.tsdb.path=/prometheus
- --storage.tsdb.retention.time=7d
volumes:
- prometheus_data:/prometheus
ports:
- "${PROMETHEUS_BIND_ADDRESS}:9090"
healthcheck:
test: ["CMD", "promtool", "query", "instant", "http://127.0.0.1:9090", "up"]
interval: 5s
timeout: 3s
retries: 20
depends_on:
- api
- nginx-exporter
- postgres-exporter
- valkey-exporter
cpus: ${PROMETHEUS_CPUS}
mem_limit: ${PROMETHEUS_MEMORY}
restart: unless-stopped
grafana:
build:
context: .
target: grafana
image: ${GRAFANA_IMAGE:-mtusi-grafana:lab3}
environment:
GF_SECURITY_ADMIN_USER: admin
GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
GF_USERS_ALLOW_SIGN_UP: "false"
volumes:
- grafana_data:/var/lib/grafana
ports:
- "${GRAFANA_BIND_ADDRESS}:3000"
depends_on:
- prometheus
cpus: ${GRAFANA_CPUS}
mem_limit: ${GRAFANA_MEMORY}
restart: unless-stopped
volumes:
postgres_data:
valkey_data:
prometheus_data:
grafana_data:
+79
View File
@@ -0,0 +1,79 @@
# Адрес и внутренний порт HTTP-сервера.
APP_HOST=0.0.0.0
APP_PORT=8080
# Адреса публикации сервисов на локальной машине.
API_BIND_ADDRESS=127.0.0.1:8080
SWAGGER_BIND_ADDRESS=127.0.0.1:8081
PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090
GRAFANA_BIND_ADDRESS=127.0.0.1:3000
# Время жизни сессии, идемпотентности и используемое хранилище.
SESSION_TTL_SECONDS=3600
IDEMPOTENCY_TTL_SECONDS=86400
SESSION_STORE=valkey
# Число API-реплик за Nginx.
API_REPLICAS=2
# Учётные данные локального PostgreSQL и строка подключения API.
POSTGRES_USER=messenger
POSTGRES_DB=messenger
POSTGRES_PASSWORD=local-postgres-password
DATABASE_URL=postgres://messenger:local-postgres-password@postgres:5432/messenger?sslmode=disable
# Пароль локального Valkey и строка подключения API.
VALKEY_PASSWORD=local-valkey-password
CACHE_URL=redis://default:local-valkey-password@valkey:6379/0
# Размер пула и обслуживание соединений PostgreSQL.
DB_MIN_CONNS=1
DB_MAX_CONNS=10
DB_MAX_CONNECTION_AGE_SECONDS=1800
DB_MAX_CONNECTION_IDLE_SECONDS=300
DB_HEALTH_CHECK_SECONDS=30
# Таймауты зависимостей, миграций и HTTP-запросов.
DEPENDENCY_TIMEOUT_SECONDS=5
MIGRATION_TIMEOUT_SECONDS=30
REQUEST_TIMEOUT_SECONDS=5
HTTP_READ_HEADER_TIMEOUT_SECONDS=5
HTTP_READ_TIMEOUT_SECONDS=10
HTTP_WRITE_TIMEOUT_SECONDS=10
HTTP_IDLE_TIMEOUT_SECONDS=60
HTTP_SHUTDOWN_TIMEOUT_SECONDS=5
# Параметры Argon2id для новых хешей паролей.
PASSWORD_MEMORY_KIB=65536
PASSWORD_ITERATIONS=3
PASSWORD_PARALLELISM=2
PASSWORD_SALT_LENGTH=16
PASSWORD_KEY_LENGTH=32
# Порт Swagger внутри конфигурации CORS API.
SWAGGER_PORT=8081
# Ограничения ресурсов контейнеров.
API_CPUS=1
API_MEMORY=256m
MIGRATE_CPUS=0.5
MIGRATE_MEMORY=128m
NGINX_CPUS=0.5
NGINX_MEMORY=128m
SWAGGER_CPUS=0.25
SWAGGER_MEMORY=128m
TEST_CPUS=2
TEST_MEMORY=512m
EXPORTER_CPUS=0.25
EXPORTER_MEMORY=128m
PROMETHEUS_CPUS=1
PROMETHEUS_MEMORY=512m
GRAFANA_CPUS=0.5
GRAFANA_MEMORY=256m
POSTGRES_CPUS=2
POSTGRES_MEMORY=1g
VALKEY_CPUS=1
VALKEY_MEMORY=256m
# Пароль локального администратора Grafana.
GRAFANA_ADMIN_PASSWORD=local-grafana-password
@@ -0,0 +1,99 @@
{
"annotations": {"list": []},
"editable": false,
"graphTooltip": 1,
"panels": [
{
"type": "timeseries", "title": "API requests per second", "id": 1,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0},
"targets": [{"expr": "sum(rate(messenger_http_requests_total[1m]))", "legendFormat": "total", "refId": "A"}]
},
{
"type": "timeseries", "title": "API RPS by replica", "id": 2,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0},
"targets": [{"expr": "sum by (instance) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{instance}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "API responses by status class", "id": 3,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 8},
"targets": [{"expr": "sum by (status_class) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{status_class}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "Healthy API replicas", "id": 4,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8},
"targets": [{"expr": "sum(up{job=\"messenger\"})", "legendFormat": "replicas", "refId": "A"}]
},
{
"type": "timeseries", "title": "API latency p95 by replica and route", "id": 5,
"gridPos": {"h": 8, "w": 24, "x": 0, "y": 16},
"fieldConfig": {"defaults": {"unit": "s"}, "overrides": []},
"targets": [{"expr": "histogram_quantile(0.95, sum by (le, instance, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", "legendFormat": "{{instance}} {{route}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "API in flight", "id": 6,
"gridPos": {"h": 8, "w": 8, "x": 0, "y": 24},
"targets": [{"expr": "messenger_http_requests_in_flight", "legendFormat": "{{instance}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "Go heap by replica", "id": 7,
"gridPos": {"h": 8, "w": 8, "x": 8, "y": 24},
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
"targets": [{"expr": "go_memstats_heap_alloc_bytes{job=\"messenger\"}", "legendFormat": "{{instance}}", "refId": "A"}]
},
{
"type": "timeseries", "title": "Idempotency replays", "id": 8,
"gridPos": {"h": 8, "w": 8, "x": 16, "y": 24},
"targets": [{"expr": "sum(rate(messenger_http_idempotency_replays_total[1m]))", "legendFormat": "replays/s", "refId": "A"}]
},
{
"type": "timeseries", "title": "Nginx connections and requests", "id": 9,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 32},
"targets": [
{"expr": "nginx_connections_active", "legendFormat": "active connections", "refId": "A"},
{"expr": "rate(nginx_http_requests_total[1m])", "legendFormat": "requests/s", "refId": "B"}
]
},
{
"type": "timeseries", "title": "PostgreSQL connections", "id": 10,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 32},
"targets": [{"expr": "sum(pg_stat_activity_count{datname=\"messenger\"})", "legendFormat": "connections", "refId": "A"}]
},
{
"type": "timeseries", "title": "PostgreSQL transactions", "id": 11,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 40},
"targets": [
{"expr": "rate(pg_stat_database_xact_commit{datname=\"messenger\"}[1m])", "legendFormat": "commit/s", "refId": "A"},
{"expr": "rate(pg_stat_database_xact_rollback{datname=\"messenger\"}[1m])", "legendFormat": "rollback/s", "refId": "B"}
]
},
{
"type": "timeseries", "title": "PostgreSQL database size", "id": 12,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 40},
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
"targets": [{"expr": "pg_database_size_bytes{datname=\"messenger\"}", "legendFormat": "database", "refId": "A"}]
},
{
"type": "timeseries", "title": "Valkey memory", "id": 13,
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 48},
"fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []},
"targets": [{"expr": "redis_memory_used_bytes", "legendFormat": "used", "refId": "A"}]
},
{
"type": "timeseries", "title": "Valkey keys and commands", "id": 14,
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 48},
"targets": [
{"expr": "sum(redis_db_keys)", "legendFormat": "keys", "refId": "A"},
{"expr": "rate(redis_commands_processed_total[1m])", "legendFormat": "commands/s", "refId": "B"}
]
}
],
"refresh": "5s",
"schemaVersion": 41,
"tags": ["highload", "lab3", "nginx", "failover"],
"templating": {"list": []},
"time": {"from": "now-15m", "to": "now"},
"timezone": "browser",
"title": "Messenger lab3 overview",
"uid": "messenger-overview",
"version": 2
}
@@ -0,0 +1,10 @@
apiVersion: 1
providers:
- name: Messenger
folder: Highload
type: file
disableDeletion: true
editable: false
options:
path: /etc/grafana/dashboards
@@ -0,0 +1,10 @@
apiVersion: 1
datasources:
- name: Prometheus
type: prometheus
uid: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false
+23
View File
@@ -0,0 +1,23 @@
global:
scrape_interval: 5s
evaluation_interval: 5s
scrape_configs:
- job_name: messenger
dns_sd_configs:
- names: ["api"]
type: A
port: 8080
refresh_interval: 2s
- job_name: nginx
static_configs:
- targets: ["nginx-exporter:9113"]
- job_name: postgres
static_configs:
- targets: ["postgres-exporter:9187"]
- job_name: valkey
static_configs:
- targets: ["valkey-exporter:9121"]
+55
View File
@@ -0,0 +1,55 @@
user nginx;
worker_processes auto;
error_log /dev/stderr notice;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
resolver 127.0.0.11 valid=2s ipv6=off;
log_format structured escape=json
'{"time":"$time_iso8601","request_id":"$http_x_request_id",'
'"method":"$request_method","uri":"$request_uri","status":$status,'
'"request_time":$request_time,"upstream":"$upstream_addr",'
'"upstream_status":"$upstream_status","upstream_time":"$upstream_response_time"}';
access_log /dev/stdout structured;
upstream messenger {
zone messenger 64k;
server api:8080 resolve max_fails=1 fail_timeout=3s;
keepalive 32;
}
server {
listen 8080;
location / {
proxy_pass http://messenger;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_connect_timeout 1s;
proxy_send_timeout 10s;
proxy_read_timeout 10s;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 3s;
}
}
server {
listen 8081;
access_log off;
location = /stub_status {
stub_status;
}
}
}
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
docker compose stop nginx api valkey postgres
docker compose start postgres valkey
docker compose run --rm migrate
docker compose up -d --wait --wait-timeout 90
+57
View File
@@ -0,0 +1,57 @@
#!/bin/sh
set -eu
wait_for_replicas() {
expected=$1
deadline=$(($(date +%s) + 90))
while [ "$(date +%s)" -lt "$deadline" ]; do
containers=$(docker compose ps -q api)
count=0
healthy=0
for container in $containers; do
count=$((count + 1))
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container")
if [ "$status" = "healthy" ]; then
healthy=$((healthy + 1))
fi
done
if [ "$count" -eq "$expected" ] && [ "$healthy" -eq "$expected" ]; then
return 0
fi
sleep 1
done
printf '%s\n' "API replicas did not become healthy: expected $expected" >&2
return 1
}
old_containers=$(docker compose ps -q api)
old_count=0
for container in $old_containers; do
old_count=$((old_count + 1))
done
docker compose build
docker compose up -d postgres valkey
docker compose run --rm migrate
if [ "$old_count" -lt 2 ]; then
docker compose up -d --scale api=2 --wait --wait-timeout 120
exit 0
fi
docker compose up -d --no-deps --no-recreate --scale api=3 api
wait_for_replicas 3
for container in $old_containers; do
docker stop --time 6 "$container" >/dev/null
docker rm "$container" >/dev/null
docker compose up -d --no-deps --no-recreate --scale api=3 api
wait_for_replicas 3
done
docker compose up -d --no-deps --no-recreate --scale api=2 api
wait_for_replicas 2
docker compose up -d --no-build --scale api=2 --wait --wait-timeout 120
+104
View File
@@ -0,0 +1,104 @@
#!/bin/sh
set -eu
duration=${DURATION:-15s}
warmup=${WARMUP:-3s}
concurrencies=${CONCURRENCIES:-"8 32 64"}
repeats=${REPEATS:-3}
current_project=
restore_default_stack() {
if [ -n "$current_project" ]; then
env COMPOSE_PROJECT_NAME="$current_project" docker compose --profile research down -v --remove-orphans || true
fi
docker compose up -d --scale api=2 --wait --wait-timeout 120
}
run_configuration() {
series=$1
replicas=$2
api_cpus=$3
api_memory=$4
db_max_conns=$5
repeat=$6
point_concurrencies=$7
project="lab3exp_${series}_r${replicas}_n${repeat}"
pending=false
for concurrency in $point_concurrencies; do
label="${series}-r${replicas}-c${concurrency}-n${repeat}"
if [ ! -s "evidence/lab3-${label}.json" ]; then
pending=true
fi
done
if [ "$pending" = false ]; then
echo "Skipping completed configuration ${series}-r${replicas}-n${repeat}"
return
fi
current_project=$project
compose() {
env \
COMPOSE_PROJECT_NAME="$project" \
API_REPLICAS="$replicas" \
API_CPUS="$api_cpus" \
API_MEMORY="$api_memory" \
DB_MIN_CONNS=0 \
DB_MAX_CONNS="$db_max_conns" \
docker compose "$@"
}
compose up -d --scale api="$replicas" --wait --wait-timeout 120 \
postgres valkey migrate api nginx postgres-exporter valkey-exporter nginx-exporter prometheus
for concurrency in $point_concurrencies; do
label="${series}-r${replicas}-c${concurrency}-n${repeat}"
output="lab3-${label}.json"
if [ -s "evidence/$output" ]; then
echo "Skipping completed point $label"
continue
fi
compose --profile research run --rm experiment \
/app/lab3experiment \
-base-url=http://nginx:8080 \
-prometheus-url=http://prometheus:9090 \
-label="$label" \
-duration="$duration" \
-warmup="$warmup" \
-concurrency="$concurrency" \
-replicas="$replicas" \
-api-cpus="$api_cpus" \
-api-memory="$api_memory" \
-db-max-conns="$db_max_conns" \
-output="/output/$output"
done
compose --profile research down -v --remove-orphans
current_project=
}
trap restore_default_stack EXIT INT TERM
docker compose build api nginx swagger prometheus grafana
docker compose stop
repeat=1
while [ "$repeat" -le "$repeats" ]; do
run_configuration fixed 1 2 1g 16 "$repeat" "$concurrencies"
run_configuration fixed 2 1 512m 8 "$repeat" "$concurrencies"
run_configuration fixed 4 0.5 256m 4 "$repeat" "$concurrencies"
run_configuration growing 1 1 256m 8 "$repeat" "$concurrencies"
run_configuration growing 2 1 256m 8 "$repeat" "$concurrencies"
run_configuration growing 4 1 256m 8 "$repeat" "$concurrencies"
run_configuration pool-baseline 4 0.5 256m 10 "$repeat" "64"
run_configuration pool-tuned 4 0.5 256m 4 "$repeat" "64"
repeat=$((repeat + 1))
done
docker compose --profile research run --no-deps --rm experiment \
/app/lab3summary \
-input=/output \
-output=/output/lab3-scale-results.json \
-chart=/output/lab3-scale-rps.svg
rm -f evidence/lab3-fixed-*.json evidence/lab3-growing-*.json evidence/lab3-pool-*.json
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
for container in $(docker compose ps -q api); do
instance=$(docker inspect --format '{{.Config.Hostname}}' "$container")
if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then
docker stop --time 6 "$container" >/dev/null
exit 0
fi
done
printf '%s\n' "API instance not found: $TARGET_INSTANCE_ID" >&2
exit 1