forked from mtusi2026/mtusi2026lab4
677 lines
22 KiB
Go
677 lines
22 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"image"
|
|
"image/jpeg"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/config"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/database"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/objectstore"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/postgres"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab4/internal/reconcile"
|
|
)
|
|
|
|
const experimentPassword = "Lab4-experiment-password"
|
|
|
|
type apiClient struct {
|
|
baseURL string
|
|
client *http.Client
|
|
token string
|
|
}
|
|
|
|
type attachment struct {
|
|
ID string `json:"id"`
|
|
}
|
|
|
|
type setup struct {
|
|
client *apiClient
|
|
chatID string
|
|
}
|
|
|
|
func main() {
|
|
if err := run(os.Args[1:], os.Stdout); err != nil {
|
|
fmt.Fprintln(os.Stderr, err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func run(arguments []string, output io.Writer) error {
|
|
if len(arguments) == 0 {
|
|
return errors.New("usage: lab4experiment memory|consistency [flags]")
|
|
}
|
|
switch arguments[0] {
|
|
case "memory":
|
|
return runMemory(arguments[1:], output)
|
|
case "consistency":
|
|
return runConsistency(arguments[1:], output)
|
|
default:
|
|
return errors.New("usage: lab4experiment memory|consistency [flags]")
|
|
}
|
|
}
|
|
|
|
type memoryResult struct {
|
|
Replicas int `json:"replicas"`
|
|
Concurrency int `json:"concurrency"`
|
|
Rounds int `json:"rounds"`
|
|
SettleTimeMillis int64 `json:"settle_time_ms"`
|
|
ImageBytes int `json:"image_bytes"`
|
|
UploadRateBytes int64 `json:"upload_rate_bytes_per_second"`
|
|
SuccessfulUploads int64 `json:"successful_uploads"`
|
|
FailedUploads int64 `json:"failed_uploads"`
|
|
DurationMillis int64 `json:"duration_ms"`
|
|
HeapBaselineBytes int64 `json:"heap_baseline_bytes"`
|
|
HeapPeakBytes int64 `json:"heap_peak_bytes"`
|
|
HeapDeltaBytes int64 `json:"heap_delta_bytes"`
|
|
RSSBaselineBytes int64 `json:"rss_baseline_bytes"`
|
|
RSSPeakBytes int64 `json:"rss_peak_bytes"`
|
|
RSSDeltaBytes int64 `json:"rss_delta_bytes"`
|
|
}
|
|
|
|
func runMemory(arguments []string, output io.Writer) error {
|
|
flags := flag.NewFlagSet("memory", flag.ContinueOnError)
|
|
flags.SetOutput(io.Discard)
|
|
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
|
|
prometheusURL := flags.String("prometheus-url", "http://prometheus:9090", "Prometheus base URL")
|
|
replicas := flags.Int("replicas", 2, "expected API replicas")
|
|
concurrency := flags.Int("concurrency", 12, "number of concurrent uploads")
|
|
rounds := flags.Int("rounds", 4, "number of upload waves")
|
|
width := flags.Int("width", 2048, "image width")
|
|
height := flags.Int("height", 2048, "image height")
|
|
uploadRate := flags.Int64("upload-rate", 1536*1024, "bytes per second for each upload")
|
|
settleTime := flags.Duration("settle-time", 3*time.Second, "delay before the baseline sample")
|
|
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
|
return errors.New("invalid memory experiment flags")
|
|
}
|
|
if *replicas < 1 || *concurrency < 10 || *rounds < 1 || *width < 1 || *height < 1 || *uploadRate < 1 || *settleTime < 0 {
|
|
return errors.New("concurrency must be at least 10 and rounds/dimensions/rate must be positive")
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
|
defer cancel()
|
|
fixture, err := jpegFixture(*width, *height)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := waitForReplicas(ctx, *prometheusURL, *replicas); err != nil {
|
|
return err
|
|
}
|
|
settle := time.NewTimer(*settleTime)
|
|
defer settle.Stop()
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-settle.C:
|
|
}
|
|
heapBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
rssBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("process_resident_memory_bytes"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
environment, err := prepare(ctx, *baseURL)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
var successful atomic.Int64
|
|
var failed atomic.Int64
|
|
var firstError struct {
|
|
sync.Mutex
|
|
err error
|
|
}
|
|
heapPeak := heapBaseline
|
|
rssPeak := rssBaseline
|
|
loadDone := make(chan struct{})
|
|
sampleDone := make(chan error, 1)
|
|
go sampleMemory(ctx, *prometheusURL, *replicas, loadDone, &heapPeak, &rssPeak, sampleDone)
|
|
started := time.Now()
|
|
for round := range *rounds {
|
|
var workers sync.WaitGroup
|
|
workers.Add(*concurrency)
|
|
for index := range *concurrency {
|
|
go func(round, index int) {
|
|
defer workers.Done()
|
|
reader := newRateReader(fixture, *uploadRate)
|
|
filename := fmt.Sprintf("memory-%02d-%02d.jpg", round, index)
|
|
if _, err := environment.client.upload(ctx, environment.chatID, filename, reader, int64(len(fixture))); err != nil {
|
|
failed.Add(1)
|
|
firstError.Lock()
|
|
if firstError.err == nil {
|
|
firstError.err = err
|
|
}
|
|
firstError.Unlock()
|
|
return
|
|
}
|
|
successful.Add(1)
|
|
}(round, index)
|
|
}
|
|
workers.Wait()
|
|
}
|
|
close(loadDone)
|
|
if err := <-sampleDone; err != nil {
|
|
return err
|
|
}
|
|
result := memoryResult{
|
|
Replicas: *replicas, Concurrency: *concurrency, Rounds: *rounds, SettleTimeMillis: settleTime.Milliseconds(),
|
|
ImageBytes: len(fixture), UploadRateBytes: *uploadRate,
|
|
SuccessfulUploads: successful.Load(), FailedUploads: failed.Load(), DurationMillis: time.Since(started).Milliseconds(),
|
|
HeapBaselineBytes: heapBaseline, HeapPeakBytes: heapPeak, HeapDeltaBytes: heapPeak - heapBaseline,
|
|
RSSBaselineBytes: rssBaseline, RSSPeakBytes: rssPeak, RSSDeltaBytes: rssPeak - rssBaseline,
|
|
}
|
|
if result.FailedUploads > 0 {
|
|
return fmt.Errorf("%d uploads failed: %w", result.FailedUploads, firstError.err)
|
|
}
|
|
return writeJSON(output, result)
|
|
}
|
|
|
|
func sampleMemory(
|
|
ctx context.Context,
|
|
prometheusURL string,
|
|
expectedReplicas int,
|
|
done <-chan struct{},
|
|
heapPeak *int64,
|
|
rssPeak *int64,
|
|
result chan<- error,
|
|
) {
|
|
ticker := time.NewTicker(500 * time.Millisecond)
|
|
defer ticker.Stop()
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
result <- ctx.Err()
|
|
return
|
|
case <-done:
|
|
result <- nil
|
|
return
|
|
case <-ticker.C:
|
|
replicas, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery())
|
|
if err != nil {
|
|
result <- err
|
|
return
|
|
}
|
|
if replicas != int64(expectedReplicas) {
|
|
result <- fmt.Errorf("active API replicas = %d, expected %d", replicas, expectedReplicas)
|
|
return
|
|
}
|
|
heap, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes"))
|
|
if err != nil {
|
|
result <- err
|
|
return
|
|
}
|
|
rss, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("process_resident_memory_bytes"))
|
|
if err != nil {
|
|
result <- err
|
|
return
|
|
}
|
|
*heapPeak = max(*heapPeak, heap)
|
|
*rssPeak = max(*rssPeak, rss)
|
|
}
|
|
}
|
|
}
|
|
|
|
type consistencyResult struct {
|
|
PageSize int `json:"page_size"`
|
|
SeededConsistent int `json:"seeded_consistent"`
|
|
FreshApply reconciliationRun `json:"fresh_apply"`
|
|
DryRun reconciliationRun `json:"dry_run"`
|
|
Apply reconciliationRun `json:"apply"`
|
|
RepeatedApply reconciliationRun `json:"repeated_apply"`
|
|
FreshPreserved bool `json:"fresh_preserved"`
|
|
OrphanDeleted bool `json:"orphan_deleted"`
|
|
UnusedDeleted bool `json:"unused_metadata_deleted"`
|
|
InUsePreserved bool `json:"in_use_metadata_preserved"`
|
|
AbandonedDeleted bool `json:"abandoned_attachment_deleted"`
|
|
TotalDurationMillis int64 `json:"total_duration_ms"`
|
|
}
|
|
|
|
type reconciliationRun struct {
|
|
DurationMillis int64 `json:"duration_ms"`
|
|
Report reconcile.Report `json:"report"`
|
|
}
|
|
|
|
func runConsistency(arguments []string, output io.Writer) error {
|
|
flags := flag.NewFlagSet("consistency", flag.ContinueOnError)
|
|
flags.SetOutput(io.Discard)
|
|
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
|
|
pageSize := flags.Int("page-size", 10, "reconciliation page size")
|
|
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
|
return errors.New("invalid consistency experiment flags")
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
|
defer cancel()
|
|
settings, err := config.LoadReconciliation()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pool, err := database.Open(ctx, database.Config{
|
|
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
|
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
|
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer pool.Close()
|
|
objects, err := objectstore.New(objectstore.Config{
|
|
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
|
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
|
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
metadata := postgres.New(pool)
|
|
clean, err := reconcile.New(metadata, objects, reconcile.Config{
|
|
Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0, UnreferencedRetention: time.Hour,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
initial, err := clean.Run(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if hasAnomalies(initial) {
|
|
return errors.New("consistency experiment requires a stack without existing attachment anomalies")
|
|
}
|
|
|
|
started := time.Now()
|
|
environment, err := prepare(ctx, *baseURL)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
fixture, err := jpegFixture(320, 200)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
seededConsistent := *pageSize + 1
|
|
for index := range seededConsistent {
|
|
filename := fmt.Sprintf("consistent-%03d.jpg", index)
|
|
if _, err := environment.client.upload(ctx, environment.chatID, filename, bytes.NewReader(fixture), int64(len(fixture))); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
unused, err := environment.client.upload(ctx, environment.chatID, "unused-missing.jpg", bytes.NewReader(fixture), int64(len(fixture)))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
inUse, err := environment.client.upload(ctx, environment.chatID, "in-use-missing.jpg", bytes.NewReader(fixture), int64(len(fixture)))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := environment.client.createMessage(ctx, environment.chatID, inUse.ID); err != nil {
|
|
return err
|
|
}
|
|
abandoned, err := environment.client.upload(ctx, environment.chatID, "abandoned.jpg", bytes.NewReader(fixture), int64(len(fixture)))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if _, err := pool.Exec(ctx, `UPDATE attachments SET status = 'pending', ready_at = NULL WHERE id = $1`, abandoned.ID); err != nil {
|
|
return fmt.Errorf("prepare abandoned pending attachment: %w", err)
|
|
}
|
|
keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := objects.Delete(ctx, keys[unused.ID]); err != nil {
|
|
return err
|
|
}
|
|
if err := objects.Delete(ctx, keys[inUse.ID]); err != nil {
|
|
return err
|
|
}
|
|
orphanKey := "attachments/experiment-" + uuid.NewString() + "/original"
|
|
if err := objects.Put(ctx, orphanKey, "image/jpeg", fixture); err != nil {
|
|
return err
|
|
}
|
|
|
|
fresh, err := reconcile.New(metadata, objects, reconcile.Config{
|
|
Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour,
|
|
UnreferencedRetention: time.Hour, Apply: true,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
freshRun, err := measuredReconciliation(ctx, fresh)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
freshPreserved := objectExists(ctx, objects, orphanKey) &&
|
|
attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID) &&
|
|
attachmentExists(ctx, pool, abandoned.ID)
|
|
if _, err := pool.Exec(ctx, `
|
|
UPDATE attachments
|
|
SET created_at = clock_timestamp() - INTERVAL '2 hours'
|
|
WHERE id = $1`, abandoned.ID); err != nil {
|
|
return fmt.Errorf("age abandoned attachment: %w", err)
|
|
}
|
|
|
|
dryRun, err := reconcile.New(metadata, objects, reconcile.Config{
|
|
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
dryRunResult, err := measuredReconciliation(ctx, dryRun)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
apply, err := reconcile.New(metadata, objects, reconcile.Config{
|
|
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, Apply: true,
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
applyResult, err := measuredReconciliation(ctx, apply)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
repeatedResult, err := measuredReconciliation(ctx, apply)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
result := consistencyResult{
|
|
PageSize: *pageSize, SeededConsistent: seededConsistent,
|
|
FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult,
|
|
FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey),
|
|
UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID),
|
|
AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]),
|
|
TotalDurationMillis: time.Since(started).Milliseconds(),
|
|
}
|
|
if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted ||
|
|
repeatedResult.Report.DeletedOrphanObjects != 0 || repeatedResult.Report.DeletedUnreferencedRows != 0 {
|
|
return fmt.Errorf("unexpected reconciliation result: %+v", result)
|
|
}
|
|
return writeJSON(output, result)
|
|
}
|
|
|
|
func measuredReconciliation(ctx context.Context, reconciler *reconcile.Reconciler) (reconciliationRun, error) {
|
|
started := time.Now()
|
|
report, err := reconciler.Run(ctx)
|
|
return reconciliationRun{DurationMillis: time.Since(started).Milliseconds(), Report: report}, err
|
|
}
|
|
|
|
func hasAnomalies(report reconcile.Report) bool {
|
|
return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 ||
|
|
report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 ||
|
|
report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 ||
|
|
report.OldUnreferencedAttachments != 0 || report.DeletingAttachments != 0
|
|
}
|
|
|
|
func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) {
|
|
rows, err := pool.Query(ctx, `SELECT id::text, object_key FROM attachments WHERE id::text = ANY($1)`, ids)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("query attachment keys: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
keys := make(map[string]string, len(ids))
|
|
for rows.Next() {
|
|
var id string
|
|
var key string
|
|
if err := rows.Scan(&id, &key); err != nil {
|
|
return nil, fmt.Errorf("scan attachment key: %w", err)
|
|
}
|
|
keys[id] = key
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, fmt.Errorf("iterate attachment keys: %w", err)
|
|
}
|
|
if len(keys) != len(ids) {
|
|
return nil, errors.New("not all attachment keys were found")
|
|
}
|
|
return keys, nil
|
|
}
|
|
|
|
func attachmentExists(ctx context.Context, pool *pgxpool.Pool, id string) bool {
|
|
var exists bool
|
|
return pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM attachments WHERE id = $1)`, id).Scan(&exists) == nil && exists
|
|
}
|
|
|
|
func objectExists(ctx context.Context, objects *objectstore.Store, key string) bool {
|
|
_, err := objects.Stat(ctx, key)
|
|
return err == nil
|
|
}
|
|
|
|
func prepare(ctx context.Context, baseURL string) (setup, error) {
|
|
client := &apiClient{baseURL: strings.TrimRight(baseURL, "/"), client: &http.Client{Timeout: 90 * time.Second}}
|
|
username := "lab4_" + strings.ReplaceAll(uuid.NewString()[:18], "-", "")
|
|
var user struct {
|
|
ID string `json:"id"`
|
|
}
|
|
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/users", map[string]string{
|
|
"username": username, "display_name": "Lab 4 experiment", "password": experimentPassword,
|
|
}, &user, ""); err != nil {
|
|
return setup{}, err
|
|
}
|
|
credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + experimentPassword))
|
|
var session struct {
|
|
Token string `json:"token"`
|
|
}
|
|
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/auth/sessions", nil, &session, "Basic "+credentials); err != nil {
|
|
return setup{}, err
|
|
}
|
|
client.token = session.Token
|
|
var chat struct {
|
|
ID string `json:"id"`
|
|
}
|
|
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/chats", map[string]any{
|
|
"title": "Lab 4 experiment", "member_ids": []string{user.ID},
|
|
}, &chat, "Bearer "+client.token); err != nil {
|
|
return setup{}, err
|
|
}
|
|
return setup{client: client, chatID: chat.ID}, nil
|
|
}
|
|
|
|
func (c *apiClient) upload(
|
|
ctx context.Context,
|
|
chatID string,
|
|
filename string,
|
|
body io.Reader,
|
|
contentLength int64,
|
|
) (attachment, error) {
|
|
path := "/api/v1/chats/" + chatID + "/attachments?filename=" + filename
|
|
request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, body)
|
|
if err != nil {
|
|
return attachment{}, err
|
|
}
|
|
request.ContentLength = contentLength
|
|
request.Header.Set("Authorization", "Bearer "+c.token)
|
|
request.Header.Set("Content-Type", "image/jpeg")
|
|
response, err := c.client.Do(request)
|
|
if err != nil {
|
|
return attachment{}, err
|
|
}
|
|
defer func() { _ = response.Body.Close() }()
|
|
if response.StatusCode != http.StatusCreated {
|
|
message, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
|
return attachment{}, fmt.Errorf("upload status %d: %s", response.StatusCode, message)
|
|
}
|
|
var result attachment
|
|
if err := json.NewDecoder(response.Body).Decode(&result); err != nil {
|
|
return attachment{}, err
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (c *apiClient) createMessage(ctx context.Context, chatID, attachmentID string) error {
|
|
return c.jsonRequest(ctx, http.MethodPost, "/api/v1/chats/"+chatID+"/messages", map[string]any{
|
|
"text": "", "attachment_ids": []string{attachmentID},
|
|
}, nil, "Bearer "+c.token, "Idempotency-Key", uuid.NewString())
|
|
}
|
|
|
|
func (c *apiClient) jsonRequest(
|
|
ctx context.Context,
|
|
method string,
|
|
path string,
|
|
payload any,
|
|
result any,
|
|
authorization string,
|
|
extraHeader ...string,
|
|
) error {
|
|
var body io.Reader
|
|
if payload != nil {
|
|
encoded, err := json.Marshal(payload)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
body = bytes.NewReader(encoded)
|
|
}
|
|
request, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if payload != nil {
|
|
request.Header.Set("Content-Type", "application/json")
|
|
}
|
|
if authorization != "" {
|
|
request.Header.Set("Authorization", authorization)
|
|
}
|
|
for index := 0; index+1 < len(extraHeader); index += 2 {
|
|
request.Header.Set(extraHeader[index], extraHeader[index+1])
|
|
}
|
|
response, err := c.client.Do(request)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer func() { _ = response.Body.Close() }()
|
|
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
|
message, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
|
return fmt.Errorf("%s %s status %d: %s", method, path, response.StatusCode, message)
|
|
}
|
|
if result == nil {
|
|
return nil
|
|
}
|
|
return json.NewDecoder(response.Body).Decode(result)
|
|
}
|
|
|
|
func waitForReplicas(ctx context.Context, prometheusURL string, expected int) error {
|
|
ticker := time.NewTicker(time.Second)
|
|
defer ticker.Stop()
|
|
for {
|
|
count, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery())
|
|
if err == nil && count == int64(expected) {
|
|
return nil
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return fmt.Errorf("wait for %d Prometheus API targets: %w", expected, ctx.Err())
|
|
case <-ticker.C:
|
|
}
|
|
}
|
|
}
|
|
|
|
func activeReplicasQuery() string {
|
|
return `count(last_over_time(process_resident_memory_bytes{job="messenger"}[10s]))`
|
|
}
|
|
|
|
func freshMetricQuery(metric string) string {
|
|
return fmt.Sprintf(`sum(last_over_time(%s{job="messenger"}[10s]))`, metric)
|
|
}
|
|
|
|
func prometheusValue(ctx context.Context, baseURL, query string) (int64, error) {
|
|
request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(baseURL, "/")+"/api/v1/query", nil)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
values := request.URL.Query()
|
|
values.Set("query", query)
|
|
request.URL.RawQuery = values.Encode()
|
|
response, err := http.DefaultClient.Do(request)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
defer func() { _ = response.Body.Close() }()
|
|
if response.StatusCode != http.StatusOK {
|
|
return 0, fmt.Errorf("prometheus status %d", response.StatusCode)
|
|
}
|
|
var payload struct {
|
|
Status string `json:"status"`
|
|
Data struct {
|
|
Result []struct {
|
|
Value []json.RawMessage `json:"value"`
|
|
} `json:"result"`
|
|
} `json:"data"`
|
|
}
|
|
if err := json.NewDecoder(response.Body).Decode(&payload); err != nil {
|
|
return 0, err
|
|
}
|
|
if payload.Status != "success" || len(payload.Data.Result) != 1 || len(payload.Data.Result[0].Value) != 2 {
|
|
return 0, errors.New("prometheus query returned no scalar value")
|
|
}
|
|
var raw string
|
|
if err := json.Unmarshal(payload.Data.Result[0].Value[1], &raw); err != nil {
|
|
return 0, err
|
|
}
|
|
value, err := strconv.ParseFloat(raw, 64)
|
|
return int64(value), err
|
|
}
|
|
|
|
func jpegFixture(width, height int) ([]byte, error) {
|
|
pixels := image.NewRGBA(image.Rect(0, 0, width, height))
|
|
state := uint64(0x9e3779b97f4a7c15)
|
|
for index := range pixels.Pix {
|
|
state ^= state << 13
|
|
state ^= state >> 7
|
|
state ^= state << 17
|
|
pixels.Pix[index] = byte(state >> 24)
|
|
}
|
|
var encoded bytes.Buffer
|
|
if err := jpeg.Encode(&encoded, pixels, &jpeg.Options{Quality: 90}); err != nil {
|
|
return nil, err
|
|
}
|
|
return encoded.Bytes(), nil
|
|
}
|
|
|
|
type rateReader struct {
|
|
reader *bytes.Reader
|
|
rate int64
|
|
started time.Time
|
|
read int64
|
|
}
|
|
|
|
func newRateReader(data []byte, rate int64) *rateReader {
|
|
return &rateReader{reader: bytes.NewReader(data), rate: rate, started: time.Now()}
|
|
}
|
|
|
|
func (r *rateReader) Read(buffer []byte) (int, error) {
|
|
for {
|
|
allowed := time.Since(r.started).Nanoseconds()*r.rate/int64(time.Second) - r.read
|
|
if allowed > 0 {
|
|
if int64(len(buffer)) > allowed {
|
|
buffer = buffer[:allowed]
|
|
}
|
|
count, err := r.reader.Read(buffer)
|
|
r.read += int64(count)
|
|
return count, err
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func writeJSON(output io.Writer, value any) error {
|
|
encoder := json.NewEncoder(output)
|
|
encoder.SetIndent("", " ")
|
|
return encoder.Encode(value)
|
|
}
|