diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2a723b5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,44 @@ +FROM golang:1.26.5-alpine3.24 AS build + +WORKDIR /src +COPY . . +RUN go build -o /out/messenger ./cmd/messenger +RUN go build -o /out/migrate ./cmd/migrate +RUN go build -o /out/lab3experiment ./cmd/lab3experiment +RUN go build -o /out/lab3summary ./cmd/lab3summary + +FROM build AS test + +CMD ["go", "test", "./internal/postgres", "./internal/session"] + +FROM swaggerapi/swagger-ui:v5.31.2 AS swagger + +COPY contracts/openapi.json /spec/openapi.json +ENV SWAGGER_JSON=/spec/openapi.json + +FROM prom/prometheus:v3.13.3 AS prometheus + +COPY monitoring/prometheus.yml /etc/prometheus/prometheus.yml + +FROM grafana/grafana:13.1.6 AS grafana + +COPY monitoring/grafana/provisioning /etc/grafana/provisioning +COPY monitoring/grafana/dashboards /etc/grafana/dashboards + +FROM nginx:1.30.5-alpine AS loadbalancer + +COPY nginx/nginx.conf /etc/nginx/nginx.conf + +FROM alpine:3.24.2 + +RUN addgroup -S app && adduser -S -G app -u 10001 app +COPY --from=build --chown=app:app /out/messenger /app/messenger +COPY --from=build --chown=app:app /out/migrate /app/migrate +COPY --from=build --chown=app:app /out/lab3experiment /app/lab3experiment +COPY --from=build --chown=app:app /out/lab3summary /app/lab3summary + +USER app +WORKDIR /app +HEALTHCHECK --interval=10s --timeout=2s --start-period=5s --retries=3 \ + CMD wget -q -O /dev/null "http://127.0.0.1:${APP_PORT}/health/ready" || exit 1 +CMD ["/app/messenger"] diff --git a/cmd/lab3experiment/main.go b/cmd/lab3experiment/main.go new file mode 100644 index 0000000..efc40d0 --- /dev/null +++ b/cmd/lab3experiment/main.go @@ -0,0 +1,522 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/google/uuid" +) + +type configuration struct { + baseURL string + prometheusURL string + label string + duration time.Duration + warmup time.Duration + concurrency int + replicas int + apiCPUs string + apiMemory string + dbMaxConns int + output string +} + +type apiClient struct { + baseURL string + client *http.Client + token string + chatID string +} + +type collector struct { + mu sync.Mutex + latencies []time.Duration + errors map[string]int + instances map[string]struct{} +} + +type result struct { + Label string `json:"label"` + Timestamp string `json:"timestamp"` + GOOS string `json:"goos"` + GOARCH string `json:"goarch"` + DurationSeconds float64 `json:"duration_seconds"` + WarmupSeconds float64 `json:"warmup_seconds"` + Concurrency int `json:"concurrency"` + Replicas int `json:"replicas"` + APICPUs string `json:"api_cpus_per_replica"` + APIMemory string `json:"api_memory_per_replica"` + DBMaxConns int `json:"db_max_connections_per_replica"` + SuccessfulRequests int `json:"successful_requests"` + SuccessfulRPS float64 `json:"successful_rps"` + Errors map[string]int `json:"errors"` + LatencyMS latencySummary `json:"latency_ms"` + ObservedInstances int `json:"observed_instances"` + Prometheus map[string]float64 `json:"prometheus"` +} + +type latencySummary struct { + P50 float64 `json:"p50"` + P95 float64 `json:"p95"` + P99 float64 `json:"p99"` +} + +type runtimeSnapshot struct { + apiCPUSeconds float64 + apiResident float64 + healthyReplicas float64 + nginxRequests float64 +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run() error { + settings := parseFlags() + client := &apiClient{ + baseURL: strings.TrimRight(settings.baseURL, "/"), + client: &http.Client{Transport: &http.Transport{ + MaxIdleConns: settings.concurrency * 2, + MaxIdleConnsPerHost: settings.concurrency, + MaxConnsPerHost: settings.concurrency, + IdleConnTimeout: 30 * time.Second, + }, Timeout: 10 * time.Second}, + } + if err := client.waitForReplicas(settings.replicas); err != nil { + return err + } + if err := client.bootstrap(); err != nil { + return err + } + if settings.warmup > 0 { + warmup := newCollector() + if err := executeLoad(client, settings.warmup, settings.concurrency, warmup); err != nil { + return fmt.Errorf("warmup: %w", err) + } + } + + var before runtimeSnapshot + if settings.prometheusURL != "" { + snapshot, err := collectRuntimeSnapshot() + if err != nil { + return fmt.Errorf("collect metrics before load: %w", err) + } + before = snapshot + } + measurements := newCollector() + started := time.Now() + if err := executeLoad(client, settings.duration, settings.concurrency, measurements); err != nil { + return err + } + elapsed := time.Since(started) + summary := measurements.result(settings, elapsed) + if settings.prometheusURL != "" { + after, err := collectRuntimeSnapshot() + if err != nil { + return fmt.Errorf("collect metrics after load: %w", err) + } + summary.Prometheus, err = prometheusMeasurement(settings.prometheusURL, before, after, elapsed) + if err != nil { + return err + } + } + return writeResult(settings.output, summary) +} + +func (c *apiClient) waitForReplicas(expected int) error { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + instances := make(map[string]struct{}, expected) + for ctx.Err() == nil { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, c.baseURL+"/health/ready", http.NoBody) + if err != nil { + return err + } + response, err := c.client.Do(request) + if err == nil { + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20)) + _ = response.Body.Close() + if response.StatusCode == http.StatusOK { + if instanceID := response.Header.Get("X-Instance-Id"); instanceID != "" { + instances[instanceID] = struct{}{} + } + if len(instances) == expected { + return nil + } + } + } + time.Sleep(100 * time.Millisecond) + } + return fmt.Errorf("nginx discovered %d of %d API replicas: %w", len(instances), expected, ctx.Err()) +} + +func parseFlags() configuration { + var settings configuration + flag.StringVar(&settings.baseURL, "base-url", "http://localhost:8080", "public Nginx base URL") + flag.StringVar(&settings.prometheusURL, "prometheus-url", "", "Prometheus base URL") + flag.StringVar(&settings.label, "label", "manual", "experiment point label") + flag.DurationVar(&settings.duration, "duration", 30*time.Second, "measurement duration") + flag.DurationVar(&settings.warmup, "warmup", 5*time.Second, "warmup duration") + flag.IntVar(&settings.concurrency, "concurrency", 16, "concurrent clients") + flag.IntVar(&settings.replicas, "replicas", 2, "API replica count") + flag.StringVar(&settings.apiCPUs, "api-cpus", "1", "CPU limit per API replica") + flag.StringVar(&settings.apiMemory, "api-memory", "256m", "memory limit per API replica") + flag.IntVar(&settings.dbMaxConns, "db-max-conns", 10, "database pool limit per API replica") + flag.StringVar(&settings.output, "output", "", "JSON output path; stdout when empty") + flag.Parse() + if settings.duration <= 0 || settings.warmup < 0 || settings.concurrency < 1 || settings.replicas < 1 || settings.dbMaxConns < 1 { + fmt.Fprintln(os.Stderr, "duration, warmup and concurrency must be positive") + os.Exit(2) + } + return settings +} + +func (c *apiClient) bootstrap() error { + username := "scale_" + strings.ReplaceAll(uuid.NewString(), "-", "")[:20] + password := "Synthetic-" + uuid.NewString() + var user struct { + ID string `json:"id"` + } + if err := c.doJSON(http.MethodPost, "/api/v1/users", "", "", map[string]string{ + "username": username, "display_name": "Scale experiment", "password": password, + }, &user, http.StatusCreated); err != nil { + return fmt.Errorf("register: %w", err) + } + + request, err := http.NewRequest(http.MethodPost, c.baseURL+"/api/v1/auth/sessions", http.NoBody) + if err != nil { + return err + } + request.SetBasicAuth(username, password) + response, err := c.client.Do(request) + if err != nil { + return fmt.Errorf("login: %w", err) + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusCreated { + return responseError("login", response) + } + var session struct { + Token string `json:"token"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil { + return fmt.Errorf("decode session: %w", err) + } + c.token = session.Token + + var chat struct { + ID string `json:"id"` + } + if err := c.doJSON(http.MethodPost, "/api/v1/chats", c.token, "", map[string]any{ + "title": "Scale experiment", "member_ids": []string{user.ID}, + }, &chat, http.StatusCreated); err != nil { + return fmt.Errorf("create chat: %w", err) + } + c.chatID = chat.ID + return nil +} + +func executeLoad(client *apiClient, duration time.Duration, concurrency int, measurements *collector) error { + ctx, cancel := context.WithTimeout(context.Background(), duration) + defer cancel() + var sequence atomic.Uint64 + var group sync.WaitGroup + for range concurrency { + group.Add(1) + go func() { + defer group.Done() + for ctx.Err() == nil { + number := sequence.Add(1) + started := time.Now() + instanceID, status, err := client.createMessage(ctx, number) + if err != nil && ctx.Err() != nil { + return + } + measurements.observe(time.Since(started), instanceID, status, err) + } + }() + } + group.Wait() + if len(measurements.latencies) == 0 { + return errors.New("no successful requests") + } + return nil +} + +func (c *apiClient) createMessage(ctx context.Context, sequence uint64) (string, int, error) { + payload, err := json.Marshal(map[string]string{"text": fmt.Sprintf("scale message %d", sequence)}) + if err != nil { + return "", 0, err + } + request, err := http.NewRequestWithContext( + ctx, http.MethodPost, c.baseURL+"/api/v1/chats/"+c.chatID+"/messages", bytes.NewReader(payload), + ) + if err != nil { + return "", 0, err + } + request.Header.Set("Authorization", "Bearer "+c.token) + request.Header.Set("Content-Type", "application/json") + request.Header.Set("Idempotency-Key", uuid.NewString()) + response, err := c.client.Do(request) + if err != nil { + return "", 0, err + } + defer func() { _ = response.Body.Close() }() + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 1<<20)) + if response.StatusCode != http.StatusCreated { + return response.Header.Get("X-Instance-Id"), response.StatusCode, fmt.Errorf("unexpected status") + } + return response.Header.Get("X-Instance-Id"), response.StatusCode, nil +} + +func newCollector() *collector { + return &collector{errors: make(map[string]int), instances: make(map[string]struct{})} +} + +func (c *collector) observe(latency time.Duration, instanceID string, status int, err error) { + c.mu.Lock() + defer c.mu.Unlock() + if err != nil { + key := "network" + if status != 0 { + key = strconv.Itoa(status) + } + c.errors[key]++ + return + } + c.latencies = append(c.latencies, latency) + if instanceID != "" { + c.instances[instanceID] = struct{}{} + } +} + +func (c *collector) result(settings configuration, elapsed time.Duration) result { + sort.Slice(c.latencies, func(i, j int) bool { return c.latencies[i] < c.latencies[j] }) + return result{ + Label: settings.label, Timestamp: time.Now().UTC().Format(time.RFC3339), + GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, + DurationSeconds: elapsed.Seconds(), WarmupSeconds: settings.warmup.Seconds(), + Concurrency: settings.concurrency, Replicas: settings.replicas, + APICPUs: settings.apiCPUs, APIMemory: settings.apiMemory, DBMaxConns: settings.dbMaxConns, + SuccessfulRequests: len(c.latencies), + SuccessfulRPS: float64(len(c.latencies)) / elapsed.Seconds(), Errors: c.errors, + LatencyMS: latencySummary{ + P50: percentile(c.latencies, 0.50), P95: percentile(c.latencies, 0.95), P99: percentile(c.latencies, 0.99), + }, + ObservedInstances: len(c.instances), + } +} + +func percentile(values []time.Duration, percentile float64) float64 { + if len(values) == 0 { + return 0 + } + index := int(float64(len(values)-1) * percentile) + return float64(values[index].Microseconds()) / 1000 +} + +func collectRuntimeSnapshot() (runtimeSnapshot, error) { + hosts, err := net.LookupHost("api") + if err != nil { + return runtimeSnapshot{}, fmt.Errorf("resolve API replicas: %w", err) + } + uniqueHosts := make(map[string]struct{}, len(hosts)) + var snapshot runtimeSnapshot + for _, host := range hosts { + if _, duplicate := uniqueHosts[host]; duplicate { + continue + } + uniqueHosts[host] = struct{}{} + metrics, err := scrapeMetrics("http://" + net.JoinHostPort(host, "8080") + "/metrics") + if err != nil { + return runtimeSnapshot{}, fmt.Errorf("scrape API %s: %w", host, err) + } + cpuSeconds, err := requiredMetric(metrics, "process_cpu_seconds_total") + if err != nil { + return runtimeSnapshot{}, err + } + resident, err := requiredMetric(metrics, "process_resident_memory_bytes") + if err != nil { + return runtimeSnapshot{}, err + } + snapshot.apiCPUSeconds += cpuSeconds + snapshot.apiResident += resident + snapshot.healthyReplicas++ + } + nginxMetrics, err := scrapeMetrics("http://nginx-exporter:9113/metrics") + if err != nil { + return runtimeSnapshot{}, fmt.Errorf("scrape Nginx exporter: %w", err) + } + snapshot.nginxRequests, err = requiredMetric(nginxMetrics, "nginx_http_requests_total") + if err != nil { + return runtimeSnapshot{}, err + } + return snapshot, nil +} + +func scrapeMetrics(endpoint string) (map[string]float64, error) { + client := &http.Client{Timeout: 5 * time.Second} + response, err := client.Get(endpoint) + if err != nil { + return nil, err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("status %d", response.StatusCode) + } + metrics := make(map[string]float64) + scanner := bufio.NewScanner(io.LimitReader(response.Body, 8<<20)) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + fields := strings.Fields(line) + if len(fields) < 2 { + continue + } + name, _, _ := strings.Cut(fields[0], "{") + value, err := strconv.ParseFloat(fields[len(fields)-1], 64) + if err != nil { + return nil, fmt.Errorf("parse metric %s: %w", name, err) + } + metrics[name] += value + } + if err := scanner.Err(); err != nil { + return nil, err + } + return metrics, nil +} + +func requiredMetric(metrics map[string]float64, name string) (float64, error) { + value, found := metrics[name] + if !found { + return 0, fmt.Errorf("metric %s not found", name) + } + return value, nil +} + +func prometheusMeasurement(baseURL string, before, after runtimeSnapshot, elapsed time.Duration) (map[string]float64, error) { + postgresConnections, err := prometheusQuery(baseURL, `sum(pg_stat_activity_count{datname="messenger"})`) + if err != nil { + return nil, fmt.Errorf("query PostgreSQL connections: %w", err) + } + seconds := elapsed.Seconds() + return map[string]float64{ + "api_cpu_cores": (after.apiCPUSeconds - before.apiCPUSeconds) / seconds, + "api_resident_bytes": after.apiResident, + "healthy_replicas": after.healthyReplicas, + "nginx_rps": (after.nginxRequests - before.nginxRequests) / seconds, + "postgres_connections": postgresConnections, + }, nil +} + +func prometheusQuery(baseURL, query string) (float64, error) { + endpoint := strings.TrimRight(baseURL, "/") + "/api/v1/query?query=" + url.QueryEscape(query) + client := &http.Client{Timeout: 5 * time.Second} + response, err := client.Get(endpoint) + if err != nil { + return 0, err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != http.StatusOK { + return 0, fmt.Errorf("prometheus status %d", response.StatusCode) + } + var payload struct { + Data struct { + Result []struct { + Value []json.RawMessage `json:"value"` + } `json:"result"` + } `json:"data"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&payload); err != nil { + return 0, err + } + var total float64 + for _, item := range payload.Data.Result { + if len(item.Value) != 2 { + continue + } + var encoded string + if err := json.Unmarshal(item.Value[1], &encoded); err != nil { + return 0, err + } + value, err := strconv.ParseFloat(encoded, 64) + if err != nil { + return 0, err + } + total += value + } + return total, nil +} + +func (c *apiClient) doJSON(method, path, token, idempotencyKey string, body, target any, expected int) error { + payload, err := json.Marshal(body) + if err != nil { + return err + } + request, err := http.NewRequest(method, c.baseURL+path, bytes.NewReader(payload)) + if err != nil { + return err + } + request.Header.Set("Content-Type", "application/json") + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + if idempotencyKey != "" { + request.Header.Set("Idempotency-Key", idempotencyKey) + } + response, err := c.client.Do(request) + if err != nil { + return err + } + defer func() { _ = response.Body.Close() }() + if response.StatusCode != expected { + return responseError(method+" "+path, response) + } + return json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(target) +} + +func responseError(action string, response *http.Response) error { + body, _ := io.ReadAll(io.LimitReader(response.Body, 1<<20)) + return fmt.Errorf("%s status=%d body=%s", action, response.StatusCode, bytes.TrimSpace(body)) +} + +func writeResult(path string, value result) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + data = append(data, '\n') + if path == "" { + _, err = os.Stdout.Write(data) + return err + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + return os.WriteFile(path, data, 0o644) +} diff --git a/cmd/lab3summary/main.go b/cmd/lab3summary/main.go new file mode 100644 index 0000000..8a3b6c4 --- /dev/null +++ b/cmd/lab3summary/main.go @@ -0,0 +1,366 @@ +package main + +import ( + "encoding/json" + "flag" + "fmt" + "html" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +type latency struct { + P50 float64 `json:"p50"` + P95 float64 `json:"p95"` + P99 float64 `json:"p99"` +} + +type inputResult struct { + Label string `json:"label"` + Timestamp string `json:"timestamp"` + DurationSeconds float64 `json:"duration_seconds"` + WarmupSeconds float64 `json:"warmup_seconds"` + Concurrency int `json:"concurrency"` + Replicas int `json:"replicas"` + APICPUs string `json:"api_cpus_per_replica"` + APIMemory string `json:"api_memory_per_replica"` + DBMaxConns int `json:"db_max_connections_per_replica"` + SuccessfulRequests int `json:"successful_requests"` + SuccessfulRPS float64 `json:"successful_rps"` + Errors map[string]int `json:"errors"` + LatencyMS latency `json:"latency_ms"` + Instances []string `json:"instances"` + ObservedInstances int `json:"observed_instances"` + Prometheus map[string]float64 `json:"prometheus"` +} + +type rawResult struct { + Label string `json:"label"` + Timestamp string `json:"timestamp"` + DurationSeconds float64 `json:"duration_seconds"` + WarmupSeconds float64 `json:"warmup_seconds"` + Concurrency int `json:"concurrency"` + Replicas int `json:"replicas"` + APICPUs string `json:"api_cpus_per_replica"` + APIMemory string `json:"api_memory_per_replica"` + DBMaxConns int `json:"db_max_connections_per_replica"` + SuccessfulRequests int `json:"successful_requests"` + SuccessfulRPS float64 `json:"successful_rps"` + Errors map[string]int `json:"errors"` + LatencyMS latency `json:"latency_ms"` + ObservedInstances int `json:"observed_instances"` + Prometheus map[string]float64 `json:"prometheus"` +} + +type aggregate struct { + Label string `json:"label"` + Repeats int `json:"repeats"` + Concurrency int `json:"concurrency"` + Replicas int `json:"replicas"` + APICPUs string `json:"api_cpus_per_replica"` + APIMemory string `json:"api_memory_per_replica"` + DBMaxConns int `json:"db_max_connections_per_replica"` + SuccessfulRPSMedian float64 `json:"successful_rps_median"` + SuccessfulRPSMin float64 `json:"successful_rps_min"` + SuccessfulRPSMax float64 `json:"successful_rps_max"` + LatencyP50MedianMS float64 `json:"latency_p50_median_ms"` + LatencyP95MedianMS float64 `json:"latency_p95_median_ms"` + LatencyP99MedianMS float64 `json:"latency_p99_median_ms"` + TotalErrors int `json:"total_errors"` + ObservedInstancesMin int `json:"observed_instances_min"` + APICPUCoresMedian float64 `json:"api_cpu_cores_median"` + APIResidentBytesMedian float64 `json:"api_resident_bytes_median"` + PostgresConnectionsMedian float64 `json:"postgres_connections_median"` + PrometheusHealthyMin float64 `json:"prometheus_healthy_replicas_min"` +} + +type report struct { + GeneratedAt string `json:"generated_at"` + Method string `json:"method"` + Raw []rawResult `json:"raw"` + Summary []aggregate `json:"summary"` +} + +func main() { + inputDirectory := flag.String("input", "evidence", "directory with experiment JSON files") + outputPath := flag.String("output", "evidence/lab3-scale-results.json", "combined JSON output") + chartPath := flag.String("chart", "evidence/lab3-scale-rps.svg", "SVG chart output") + flag.Parse() + + results, err := readResults(*inputDirectory) + if err != nil { + fatal(err) + } + aggregates := summarize(results) + if err := writeJSON(*outputPath, report{ + GeneratedAt: time.Now().UTC().Format(time.RFC3339), + Method: "closed-loop HTTP through Nginx; median of three isolated repeats", + Raw: sanitize(results), + Summary: aggregates, + }); err != nil { + fatal(err) + } + if err := writeChart(*chartPath, aggregates); err != nil { + fatal(err) + } +} + +func readResults(directory string) ([]inputResult, error) { + var paths []string + for _, pattern := range []string{"lab3-fixed-*.json", "lab3-growing-*.json", "lab3-pool-*.json"} { + matches, err := filepath.Glob(filepath.Join(directory, pattern)) + if err != nil { + return nil, err + } + paths = append(paths, matches...) + } + if len(paths) == 0 { + return nil, fmt.Errorf("no experiment results in %s", directory) + } + + results := make([]inputResult, 0, len(paths)) + for _, path := range paths { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var result inputResult + if err := json.Unmarshal(data, &result); err != nil { + return nil, fmt.Errorf("decode %s: %w", path, err) + } + results = append(results, result) + } + sort.Slice(results, func(i, j int) bool { return results[i].Label < results[j].Label }) + return results, nil +} + +func sanitize(results []inputResult) []rawResult { + raw := make([]rawResult, 0, len(results)) + for _, value := range results { + raw = append(raw, rawResult{ + Label: value.Label, Timestamp: value.Timestamp, + DurationSeconds: value.DurationSeconds, WarmupSeconds: value.WarmupSeconds, + Concurrency: value.Concurrency, Replicas: value.Replicas, + APICPUs: value.APICPUs, APIMemory: value.APIMemory, DBMaxConns: value.DBMaxConns, + SuccessfulRequests: value.SuccessfulRequests, SuccessfulRPS: value.SuccessfulRPS, + Errors: value.Errors, LatencyMS: value.LatencyMS, + ObservedInstances: instanceCount(value), Prometheus: value.Prometheus, + }) + } + return raw +} + +func summarize(results []inputResult) []aggregate { + groups := make(map[string][]inputResult) + for _, value := range results { + groups[withoutRepeat(value.Label)] = append(groups[withoutRepeat(value.Label)], value) + } + + labels := make([]string, 0, len(groups)) + for label := range groups { + labels = append(labels, label) + } + sort.Strings(labels) + + aggregates := make([]aggregate, 0, len(labels)) + for _, label := range labels { + values := groups[label] + first := values[0] + rps := numbers(values, func(value inputResult) float64 { return value.SuccessfulRPS }) + p50 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P50 }) + p95 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P95 }) + p99 := numbers(values, func(value inputResult) float64 { return value.LatencyMS.P99 }) + cpu := metricNumbers(values, "api_cpu_cores") + memory := metricNumbers(values, "api_resident_bytes") + connections := metricNumbers(values, "postgres_connections") + healthy := metricNumbers(values, "healthy_replicas") + aggregate := aggregate{ + Label: label, Repeats: len(values), Concurrency: first.Concurrency, Replicas: first.Replicas, + APICPUs: first.APICPUs, APIMemory: first.APIMemory, DBMaxConns: first.DBMaxConns, + SuccessfulRPSMedian: median(rps), SuccessfulRPSMin: minimum(rps), SuccessfulRPSMax: maximum(rps), + LatencyP50MedianMS: median(p50), LatencyP95MedianMS: median(p95), LatencyP99MedianMS: median(p99), + ObservedInstancesMin: instanceCount(first), APICPUCoresMedian: median(cpu), + APIResidentBytesMedian: median(memory), PostgresConnectionsMedian: median(connections), + PrometheusHealthyMin: minimum(healthy), + } + for _, value := range values { + if count := instanceCount(value); count < aggregate.ObservedInstancesMin { + aggregate.ObservedInstancesMin = count + } + for _, count := range value.Errors { + aggregate.TotalErrors += count + } + } + aggregates = append(aggregates, aggregate) + } + return aggregates +} + +func instanceCount(value inputResult) int { + if value.ObservedInstances > 0 { + return value.ObservedInstances + } + return len(value.Instances) +} + +func withoutRepeat(label string) string { + index := strings.LastIndex(label, "-n") + if index == -1 { + return label + } + if _, err := strconv.Atoi(label[index+2:]); err != nil { + return label + } + return label[:index] +} + +func numbers(values []inputResult, value func(inputResult) float64) []float64 { + result := make([]float64, 0, len(values)) + for _, item := range values { + result = append(result, value(item)) + } + return result +} + +func metricNumbers(values []inputResult, name string) []float64 { + result := make([]float64, 0, len(values)) + for _, item := range values { + if value, ok := item.Prometheus[name]; ok { + result = append(result, value) + } + } + return result +} + +func median(values []float64) float64 { + if len(values) == 0 { + return 0 + } + values = append([]float64(nil), values...) + sort.Float64s(values) + middle := len(values) / 2 + if len(values)%2 == 1 { + return values[middle] + } + return (values[middle-1] + values[middle]) / 2 +} + +func minimum(values []float64) float64 { + if len(values) == 0 { + return 0 + } + minimum := values[0] + for _, value := range values[1:] { + if value < minimum { + minimum = value + } + } + return minimum +} + +func maximum(values []float64) float64 { + if len(values) == 0 { + return 0 + } + maximum := values[0] + for _, value := range values[1:] { + if value > maximum { + maximum = value + } + } + return maximum +} + +func writeJSON(path string, value any) error { + data, err := json.MarshalIndent(value, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(data, '\n'), 0o644) +} + +func writeChart(path string, values []aggregate) error { + const width, height = 960, 560 + const left, right, top, bottom = 85.0, 240.0, 45.0, 70.0 + plotWidth := width - left - right + plotHeight := height - top - bottom + + var selected []aggregate + var maxRPS float64 + for _, value := range values { + if strings.HasPrefix(value.Label, "fixed-") || strings.HasPrefix(value.Label, "growing-") { + selected = append(selected, value) + if value.SuccessfulRPSMedian > maxRPS { + maxRPS = value.SuccessfulRPSMedian + } + } + } + if maxRPS == 0 { + return fmt.Errorf("no scale results for chart") + } + maxRPS *= 1.1 + + var output strings.Builder + fmt.Fprintf(&output, ``, width, height, width, height) + output.WriteString(``) + fmt.Fprintf(&output, `Median successful RPS by replica count`, left) + + for step := 0; step <= 5; step++ { + y := top + plotHeight*float64(5-step)/5 + value := maxRPS * float64(step) / 5 + fmt.Fprintf(&output, `%.0f`, left, y, left+plotWidth, y, left-10, y+4, value) + } + fmt.Fprintf(&output, ``, left, top, left, top+plotHeight, left, top+plotHeight, left+plotWidth, top+plotHeight) + for index, replicas := range []int{1, 2, 4} { + x := left + plotWidth*float64(index)/2 + fmt.Fprintf(&output, `%d`, x, top+plotHeight+25, replicas) + } + fmt.Fprintf(&output, `API replicas`, left+plotWidth/2, height-15) + + colors := []string{"#1a73e8", "#d93025", "#188038", "#9334e6", "#f29900", "#00897b"} + type series struct { + name string + values []aggregate + } + groups := make(map[string][]aggregate) + for _, value := range selected { + name := strings.Split(value.Label, "-r")[0] + ", c=" + strconv.Itoa(value.Concurrency) + groups[name] = append(groups[name], value) + } + var names []string + for name := range groups { + names = append(names, name) + } + sort.Strings(names) + allSeries := make([]series, 0, len(names)) + for _, name := range names { + points := groups[name] + sort.Slice(points, func(i, j int) bool { return points[i].Replicas < points[j].Replicas }) + allSeries = append(allSeries, series{name: name, values: points}) + } + for index, current := range allSeries { + color := colors[index%len(colors)] + var coordinates []string + for _, point := range current.values { + xIndex := map[int]float64{1: 0, 2: 1, 4: 2}[point.Replicas] + x := left + plotWidth*xIndex/2 + y := top + plotHeight*(1-point.SuccessfulRPSMedian/maxRPS) + coordinates = append(coordinates, fmt.Sprintf("%g,%g", x, y)) + fmt.Fprintf(&output, ``, x, y, color) + } + fmt.Fprintf(&output, ``, strings.Join(coordinates, " "), color) + legendY := top + 20 + float64(index*28) + fmt.Fprintf(&output, `%s`, left+plotWidth+25, legendY, left+plotWidth+50, legendY, color, left+plotWidth+58, legendY+4, html.EscapeString(current.name)) + } + output.WriteString(``) + return os.WriteFile(path, []byte(output.String()), 0o644) +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) +} diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..3bae319 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,261 @@ +services: + postgres: + image: postgres:17.6-alpine3.22 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${POSTGRES_CPUS} + mem_limit: ${POSTGRES_MEMORY} + restart: unless-stopped + + valkey: + image: valkey/valkey:8.1.4-alpine + command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec", "--requirepass", "${VALKEY_PASSWORD}"] + environment: + VALKEY_PASSWORD: ${VALKEY_PASSWORD} + volumes: + - valkey_data:/data + healthcheck: + test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$VALKEY_PASSWORD\" ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 20 + cpus: ${VALKEY_CPUS} + mem_limit: ${VALKEY_MEMORY} + restart: unless-stopped + + migrate: + build: . + image: ${API_IMAGE:-mtusi-messenger:lab3} + command: ["/app/migrate"] + environment: + DATABASE_URL: ${DATABASE_URL} + DB_MIN_CONNS: ${DB_MIN_CONNS} + DB_MAX_CONNS: ${DB_MAX_CONNS} + DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} + DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} + DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} + DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} + MIGRATION_TIMEOUT_SECONDS: ${MIGRATION_TIMEOUT_SECONDS} + depends_on: + postgres: + condition: service_healthy + healthcheck: + disable: true + cpus: ${MIGRATE_CPUS} + mem_limit: ${MIGRATE_MEMORY} + restart: "no" + + api: + build: . + image: ${API_IMAGE:-mtusi-messenger:lab3} + environment: + APP_HOST: ${APP_HOST} + APP_PORT: ${APP_PORT} + DATABASE_URL: ${DATABASE_URL} + CACHE_URL: ${CACHE_URL} + SESSION_STORE: ${SESSION_STORE} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS} + DB_MIN_CONNS: ${DB_MIN_CONNS} + DB_MAX_CONNS: ${DB_MAX_CONNS} + DB_MAX_CONNECTION_AGE_SECONDS: ${DB_MAX_CONNECTION_AGE_SECONDS} + DB_MAX_CONNECTION_IDLE_SECONDS: ${DB_MAX_CONNECTION_IDLE_SECONDS} + DB_HEALTH_CHECK_SECONDS: ${DB_HEALTH_CHECK_SECONDS} + DEPENDENCY_TIMEOUT_SECONDS: ${DEPENDENCY_TIMEOUT_SECONDS} + REQUEST_TIMEOUT_SECONDS: ${REQUEST_TIMEOUT_SECONDS} + HTTP_READ_HEADER_TIMEOUT_SECONDS: ${HTTP_READ_HEADER_TIMEOUT_SECONDS} + HTTP_READ_TIMEOUT_SECONDS: ${HTTP_READ_TIMEOUT_SECONDS} + HTTP_WRITE_TIMEOUT_SECONDS: ${HTTP_WRITE_TIMEOUT_SECONDS} + HTTP_IDLE_TIMEOUT_SECONDS: ${HTTP_IDLE_TIMEOUT_SECONDS} + HTTP_SHUTDOWN_TIMEOUT_SECONDS: ${HTTP_SHUTDOWN_TIMEOUT_SECONDS} + PASSWORD_MEMORY_KIB: ${PASSWORD_MEMORY_KIB} + PASSWORD_ITERATIONS: ${PASSWORD_ITERATIONS} + PASSWORD_PARALLELISM: ${PASSWORD_PARALLELISM} + PASSWORD_SALT_LENGTH: ${PASSWORD_SALT_LENGTH} + PASSWORD_KEY_LENGTH: ${PASSWORD_KEY_LENGTH} + SWAGGER_PORT: ${SWAGGER_PORT} + expose: + - "8080" + depends_on: + migrate: + condition: service_completed_successfully + valkey: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:${APP_PORT}/health/ready || exit 1"] + interval: 2s + timeout: 1s + start_period: 5s + retries: 2 + deploy: + replicas: ${API_REPLICAS:-2} + cpus: ${API_CPUS} + mem_limit: ${API_MEMORY} + stop_grace_period: 8s + restart: unless-stopped + + nginx: + build: + context: . + target: loadbalancer + image: ${NGINX_IMAGE:-mtusi-nginx:lab3} + ports: + - "${API_BIND_ADDRESS}:8080" + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081/stub_status || exit 1"] + interval: 2s + timeout: 1s + retries: 10 + cpus: ${NGINX_CPUS} + mem_limit: ${NGINX_MEMORY} + restart: unless-stopped + + swagger: + build: + context: . + target: swagger + image: ${SWAGGER_IMAGE:-mtusi-swagger:lab3} + ports: + - "${SWAGGER_BIND_ADDRESS}:8080" + depends_on: + nginx: + condition: service_healthy + cpus: ${SWAGGER_CPUS} + mem_limit: ${SWAGGER_MEMORY} + restart: unless-stopped + + tests: + profiles: ["tests"] + build: + context: . + target: test + environment: + TEST_DATABASE_URL: ${DATABASE_URL} + TEST_CACHE_URL: ${CACHE_URL} + depends_on: + migrate: + condition: service_completed_successfully + valkey: + condition: service_healthy + cpus: ${TEST_CPUS} + mem_limit: ${TEST_MEMORY} + + experiment: + profiles: ["research"] + build: . + image: ${API_IMAGE:-mtusi-messenger:lab3} + command: + - /app/lab3experiment + - -base-url=http://nginx:8080 + - -prometheus-url=http://prometheus:9090 + - -output=/output/lab3-scale.json + volumes: + - ./evidence:/output + depends_on: + nginx: + condition: service_healthy + prometheus: + condition: service_healthy + cpus: ${TEST_CPUS} + mem_limit: ${TEST_MEMORY} + + postgres-exporter: + image: quay.io/prometheuscommunity/postgres-exporter:v0.20.1 + environment: + DATA_SOURCE_URI: postgres:5432/${POSTGRES_DB}?sslmode=disable + DATA_SOURCE_USER: ${POSTGRES_USER} + DATA_SOURCE_PASS: ${POSTGRES_PASSWORD} + PG_EXPORTER_COLLECTION_TIMEOUT: 5s + depends_on: + postgres: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + + valkey-exporter: + image: oliver006/redis_exporter:v1.89.0-alpine + environment: + REDIS_ADDR: redis://valkey:6379 + REDIS_USER: default + REDIS_PASSWORD: ${VALKEY_PASSWORD} + depends_on: + valkey: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + + nginx-exporter: + image: nginx/nginx-prometheus-exporter:1.5.1 + command: ["--nginx.scrape-uri=http://nginx:8081/stub_status"] + depends_on: + nginx: + condition: service_healthy + cpus: ${EXPORTER_CPUS} + mem_limit: ${EXPORTER_MEMORY} + restart: unless-stopped + + prometheus: + build: + context: . + target: prometheus + image: ${PROMETHEUS_IMAGE:-mtusi-prometheus:lab3} + command: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=7d + volumes: + - prometheus_data:/prometheus + ports: + - "${PROMETHEUS_BIND_ADDRESS}:9090" + healthcheck: + test: ["CMD", "promtool", "query", "instant", "http://127.0.0.1:9090", "up"] + interval: 5s + timeout: 3s + retries: 20 + depends_on: + - api + - nginx-exporter + - postgres-exporter + - valkey-exporter + cpus: ${PROMETHEUS_CPUS} + mem_limit: ${PROMETHEUS_MEMORY} + restart: unless-stopped + + grafana: + build: + context: . + target: grafana + image: ${GRAFANA_IMAGE:-mtusi-grafana:lab3} + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD} + GF_USERS_ALLOW_SIGN_UP: "false" + volumes: + - grafana_data:/var/lib/grafana + ports: + - "${GRAFANA_BIND_ADDRESS}:3000" + depends_on: + - prometheus + cpus: ${GRAFANA_CPUS} + mem_limit: ${GRAFANA_MEMORY} + restart: unless-stopped + +volumes: + postgres_data: + valkey_data: + prometheus_data: + grafana_data: diff --git a/lab3.env.example b/lab3.env.example new file mode 100644 index 0000000..447b3b7 --- /dev/null +++ b/lab3.env.example @@ -0,0 +1,79 @@ +# Адрес и внутренний порт HTTP-сервера. +APP_HOST=0.0.0.0 +APP_PORT=8080 + +# Адреса публикации сервисов на локальной машине. +API_BIND_ADDRESS=127.0.0.1:8080 +SWAGGER_BIND_ADDRESS=127.0.0.1:8081 +PROMETHEUS_BIND_ADDRESS=127.0.0.1:9090 +GRAFANA_BIND_ADDRESS=127.0.0.1:3000 + +# Время жизни сессии, идемпотентности и используемое хранилище. +SESSION_TTL_SECONDS=3600 +IDEMPOTENCY_TTL_SECONDS=86400 +SESSION_STORE=valkey + +# Число API-реплик за Nginx. +API_REPLICAS=2 + +# Учётные данные локального PostgreSQL и строка подключения API. +POSTGRES_USER=messenger +POSTGRES_DB=messenger +POSTGRES_PASSWORD=local-postgres-password +DATABASE_URL=postgres://messenger:local-postgres-password@postgres:5432/messenger?sslmode=disable + +# Пароль локального Valkey и строка подключения API. +VALKEY_PASSWORD=local-valkey-password +CACHE_URL=redis://default:local-valkey-password@valkey:6379/0 + +# Размер пула и обслуживание соединений PostgreSQL. +DB_MIN_CONNS=1 +DB_MAX_CONNS=10 +DB_MAX_CONNECTION_AGE_SECONDS=1800 +DB_MAX_CONNECTION_IDLE_SECONDS=300 +DB_HEALTH_CHECK_SECONDS=30 + +# Таймауты зависимостей, миграций и HTTP-запросов. +DEPENDENCY_TIMEOUT_SECONDS=5 +MIGRATION_TIMEOUT_SECONDS=30 +REQUEST_TIMEOUT_SECONDS=5 +HTTP_READ_HEADER_TIMEOUT_SECONDS=5 +HTTP_READ_TIMEOUT_SECONDS=10 +HTTP_WRITE_TIMEOUT_SECONDS=10 +HTTP_IDLE_TIMEOUT_SECONDS=60 +HTTP_SHUTDOWN_TIMEOUT_SECONDS=5 + +# Параметры Argon2id для новых хешей паролей. +PASSWORD_MEMORY_KIB=65536 +PASSWORD_ITERATIONS=3 +PASSWORD_PARALLELISM=2 +PASSWORD_SALT_LENGTH=16 +PASSWORD_KEY_LENGTH=32 + +# Порт Swagger внутри конфигурации CORS API. +SWAGGER_PORT=8081 + +# Ограничения ресурсов контейнеров. +API_CPUS=1 +API_MEMORY=256m +MIGRATE_CPUS=0.5 +MIGRATE_MEMORY=128m +NGINX_CPUS=0.5 +NGINX_MEMORY=128m +SWAGGER_CPUS=0.25 +SWAGGER_MEMORY=128m +TEST_CPUS=2 +TEST_MEMORY=512m +EXPORTER_CPUS=0.25 +EXPORTER_MEMORY=128m +PROMETHEUS_CPUS=1 +PROMETHEUS_MEMORY=512m +GRAFANA_CPUS=0.5 +GRAFANA_MEMORY=256m +POSTGRES_CPUS=2 +POSTGRES_MEMORY=1g +VALKEY_CPUS=1 +VALKEY_MEMORY=256m + +# Пароль локального администратора Grafana. +GRAFANA_ADMIN_PASSWORD=local-grafana-password diff --git a/monitoring/grafana/dashboards/messenger.json b/monitoring/grafana/dashboards/messenger.json new file mode 100644 index 0000000..ad0b521 --- /dev/null +++ b/monitoring/grafana/dashboards/messenger.json @@ -0,0 +1,99 @@ +{ + "annotations": {"list": []}, + "editable": false, + "graphTooltip": 1, + "panels": [ + { + "type": "timeseries", "title": "API requests per second", "id": 1, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}, + "targets": [{"expr": "sum(rate(messenger_http_requests_total[1m]))", "legendFormat": "total", "refId": "A"}] + }, + { + "type": "timeseries", "title": "API RPS by replica", "id": 2, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}, + "targets": [{"expr": "sum by (instance) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{instance}}", "refId": "A"}] + }, + { + "type": "timeseries", "title": "API responses by status class", "id": 3, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 8}, + "targets": [{"expr": "sum by (status_class) (rate(messenger_http_requests_total[1m]))", "legendFormat": "{{status_class}}", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Healthy API replicas", "id": 4, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}, + "targets": [{"expr": "sum(up{job=\"messenger\"})", "legendFormat": "replicas", "refId": "A"}] + }, + { + "type": "timeseries", "title": "API latency p95 by replica and route", "id": 5, + "gridPos": {"h": 8, "w": 24, "x": 0, "y": 16}, + "fieldConfig": {"defaults": {"unit": "s"}, "overrides": []}, + "targets": [{"expr": "histogram_quantile(0.95, sum by (le, instance, route) (rate(messenger_http_request_duration_seconds_bucket[5m])))", "legendFormat": "{{instance}} {{route}}", "refId": "A"}] + }, + { + "type": "timeseries", "title": "API in flight", "id": 6, + "gridPos": {"h": 8, "w": 8, "x": 0, "y": 24}, + "targets": [{"expr": "messenger_http_requests_in_flight", "legendFormat": "{{instance}}", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Go heap by replica", "id": 7, + "gridPos": {"h": 8, "w": 8, "x": 8, "y": 24}, + "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, + "targets": [{"expr": "go_memstats_heap_alloc_bytes{job=\"messenger\"}", "legendFormat": "{{instance}}", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Idempotency replays", "id": 8, + "gridPos": {"h": 8, "w": 8, "x": 16, "y": 24}, + "targets": [{"expr": "sum(rate(messenger_http_idempotency_replays_total[1m]))", "legendFormat": "replays/s", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Nginx connections and requests", "id": 9, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 32}, + "targets": [ + {"expr": "nginx_connections_active", "legendFormat": "active connections", "refId": "A"}, + {"expr": "rate(nginx_http_requests_total[1m])", "legendFormat": "requests/s", "refId": "B"} + ] + }, + { + "type": "timeseries", "title": "PostgreSQL connections", "id": 10, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 32}, + "targets": [{"expr": "sum(pg_stat_activity_count{datname=\"messenger\"})", "legendFormat": "connections", "refId": "A"}] + }, + { + "type": "timeseries", "title": "PostgreSQL transactions", "id": 11, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 40}, + "targets": [ + {"expr": "rate(pg_stat_database_xact_commit{datname=\"messenger\"}[1m])", "legendFormat": "commit/s", "refId": "A"}, + {"expr": "rate(pg_stat_database_xact_rollback{datname=\"messenger\"}[1m])", "legendFormat": "rollback/s", "refId": "B"} + ] + }, + { + "type": "timeseries", "title": "PostgreSQL database size", "id": 12, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 40}, + "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, + "targets": [{"expr": "pg_database_size_bytes{datname=\"messenger\"}", "legendFormat": "database", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Valkey memory", "id": 13, + "gridPos": {"h": 8, "w": 12, "x": 0, "y": 48}, + "fieldConfig": {"defaults": {"unit": "bytes"}, "overrides": []}, + "targets": [{"expr": "redis_memory_used_bytes", "legendFormat": "used", "refId": "A"}] + }, + { + "type": "timeseries", "title": "Valkey keys and commands", "id": 14, + "gridPos": {"h": 8, "w": 12, "x": 12, "y": 48}, + "targets": [ + {"expr": "sum(redis_db_keys)", "legendFormat": "keys", "refId": "A"}, + {"expr": "rate(redis_commands_processed_total[1m])", "legendFormat": "commands/s", "refId": "B"} + ] + } + ], + "refresh": "5s", + "schemaVersion": 41, + "tags": ["highload", "lab3", "nginx", "failover"], + "templating": {"list": []}, + "time": {"from": "now-15m", "to": "now"}, + "timezone": "browser", + "title": "Messenger lab3 overview", + "uid": "messenger-overview", + "version": 2 +} diff --git a/monitoring/grafana/provisioning/dashboards/messenger.yml b/monitoring/grafana/provisioning/dashboards/messenger.yml new file mode 100644 index 0000000..fa7934c --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/messenger.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: Messenger + folder: Highload + type: file + disableDeletion: true + editable: false + options: + path: /etc/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..8704d9e --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml new file mode 100644 index 0000000..2c0881e --- /dev/null +++ b/monitoring/prometheus.yml @@ -0,0 +1,23 @@ +global: + scrape_interval: 5s + evaluation_interval: 5s + +scrape_configs: + - job_name: messenger + dns_sd_configs: + - names: ["api"] + type: A + port: 8080 + refresh_interval: 2s + + - job_name: nginx + static_configs: + - targets: ["nginx-exporter:9113"] + + - job_name: postgres + static_configs: + - targets: ["postgres-exporter:9187"] + + - job_name: valkey + static_configs: + - targets: ["valkey-exporter:9121"] diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..9dcebe8 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,55 @@ +user nginx; +worker_processes auto; +error_log /dev/stderr notice; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + resolver 127.0.0.11 valid=2s ipv6=off; + + log_format structured escape=json + '{"time":"$time_iso8601","request_id":"$http_x_request_id",' + '"method":"$request_method","uri":"$request_uri","status":$status,' + '"request_time":$request_time,"upstream":"$upstream_addr",' + '"upstream_status":"$upstream_status","upstream_time":"$upstream_response_time"}'; + + access_log /dev/stdout structured; + + upstream messenger { + zone messenger 64k; + server api:8080 resolve max_fails=1 fail_timeout=3s; + keepalive 32; + } + + server { + listen 8080; + + location / { + proxy_pass http://messenger; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 1s; + proxy_send_timeout 10s; + proxy_read_timeout 10s; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 3s; + } + } + + server { + listen 8081; + access_log off; + + location = /stub_status { + stub_status; + } + } +} diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 0000000..cacb521 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +set -eu + +docker compose stop nginx api valkey postgres +docker compose start postgres valkey +docker compose run --rm migrate +docker compose up -d --wait --wait-timeout 90 diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 0000000..e32e297 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,57 @@ +#!/bin/sh + +set -eu + +wait_for_replicas() { + expected=$1 + deadline=$(($(date +%s) + 90)) + + while [ "$(date +%s)" -lt "$deadline" ]; do + containers=$(docker compose ps -q api) + count=0 + healthy=0 + for container in $containers; do + count=$((count + 1)) + status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container") + if [ "$status" = "healthy" ]; then + healthy=$((healthy + 1)) + fi + done + if [ "$count" -eq "$expected" ] && [ "$healthy" -eq "$expected" ]; then + return 0 + fi + sleep 1 + done + + printf '%s\n' "API replicas did not become healthy: expected $expected" >&2 + return 1 +} + +old_containers=$(docker compose ps -q api) +old_count=0 +for container in $old_containers; do + old_count=$((old_count + 1)) +done + +docker compose build +docker compose up -d postgres valkey +docker compose run --rm migrate + +if [ "$old_count" -lt 2 ]; then + docker compose up -d --scale api=2 --wait --wait-timeout 120 + exit 0 +fi + +docker compose up -d --no-deps --no-recreate --scale api=3 api +wait_for_replicas 3 + +for container in $old_containers; do + docker stop --time 6 "$container" >/dev/null + docker rm "$container" >/dev/null + docker compose up -d --no-deps --no-recreate --scale api=3 api + wait_for_replicas 3 +done + +docker compose up -d --no-deps --no-recreate --scale api=2 api +wait_for_replicas 2 +docker compose up -d --no-build --scale api=2 --wait --wait-timeout 120 diff --git a/scripts/run-scale-experiment.sh b/scripts/run-scale-experiment.sh new file mode 100755 index 0000000..092713c --- /dev/null +++ b/scripts/run-scale-experiment.sh @@ -0,0 +1,104 @@ +#!/bin/sh + +set -eu + +duration=${DURATION:-15s} +warmup=${WARMUP:-3s} +concurrencies=${CONCURRENCIES:-"8 32 64"} +repeats=${REPEATS:-3} +current_project= + +restore_default_stack() { + if [ -n "$current_project" ]; then + env COMPOSE_PROJECT_NAME="$current_project" docker compose --profile research down -v --remove-orphans || true + fi + docker compose up -d --scale api=2 --wait --wait-timeout 120 +} + +run_configuration() { + series=$1 + replicas=$2 + api_cpus=$3 + api_memory=$4 + db_max_conns=$5 + repeat=$6 + point_concurrencies=$7 + project="lab3exp_${series}_r${replicas}_n${repeat}" + + pending=false + for concurrency in $point_concurrencies; do + label="${series}-r${replicas}-c${concurrency}-n${repeat}" + if [ ! -s "evidence/lab3-${label}.json" ]; then + pending=true + fi + done + if [ "$pending" = false ]; then + echo "Skipping completed configuration ${series}-r${replicas}-n${repeat}" + return + fi + + current_project=$project + + compose() { + env \ + COMPOSE_PROJECT_NAME="$project" \ + API_REPLICAS="$replicas" \ + API_CPUS="$api_cpus" \ + API_MEMORY="$api_memory" \ + DB_MIN_CONNS=0 \ + DB_MAX_CONNS="$db_max_conns" \ + docker compose "$@" + } + + compose up -d --scale api="$replicas" --wait --wait-timeout 120 \ + postgres valkey migrate api nginx postgres-exporter valkey-exporter nginx-exporter prometheus + for concurrency in $point_concurrencies; do + label="${series}-r${replicas}-c${concurrency}-n${repeat}" + output="lab3-${label}.json" + if [ -s "evidence/$output" ]; then + echo "Skipping completed point $label" + continue + fi + compose --profile research run --rm experiment \ + /app/lab3experiment \ + -base-url=http://nginx:8080 \ + -prometheus-url=http://prometheus:9090 \ + -label="$label" \ + -duration="$duration" \ + -warmup="$warmup" \ + -concurrency="$concurrency" \ + -replicas="$replicas" \ + -api-cpus="$api_cpus" \ + -api-memory="$api_memory" \ + -db-max-conns="$db_max_conns" \ + -output="/output/$output" + done + compose --profile research down -v --remove-orphans + current_project= +} + +trap restore_default_stack EXIT INT TERM +docker compose build api nginx swagger prometheus grafana +docker compose stop + +repeat=1 +while [ "$repeat" -le "$repeats" ]; do + run_configuration fixed 1 2 1g 16 "$repeat" "$concurrencies" + run_configuration fixed 2 1 512m 8 "$repeat" "$concurrencies" + run_configuration fixed 4 0.5 256m 4 "$repeat" "$concurrencies" + + run_configuration growing 1 1 256m 8 "$repeat" "$concurrencies" + run_configuration growing 2 1 256m 8 "$repeat" "$concurrencies" + run_configuration growing 4 1 256m 8 "$repeat" "$concurrencies" + + run_configuration pool-baseline 4 0.5 256m 10 "$repeat" "64" + run_configuration pool-tuned 4 0.5 256m 4 "$repeat" "64" + repeat=$((repeat + 1)) +done + +docker compose --profile research run --no-deps --rm experiment \ + /app/lab3summary \ + -input=/output \ + -output=/output/lab3-scale-results.json \ + -chart=/output/lab3-scale-rps.svg +rm -f evidence/lab3-fixed-*.json evidence/lab3-growing-*.json evidence/lab3-pool-*.json diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 0000000..5327251 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +set -eu + +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" + +for container in $(docker compose ps -q api); do + instance=$(docker inspect --format '{{.Config.Hostname}}' "$container") + if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then + docker stop --time 6 "$container" >/dev/null + exit 0 + fi +done + +printf '%s\n' "API instance not found: $TARGET_INSTANCE_ID" >&2 +exit 1