forked from mtusi2026/mtusi2026lab5
test: verify image pipeline recovery
This commit is contained in:
@@ -1,676 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/reconcile"
|
||||
)
|
||||
|
||||
const experimentPassword = "Lab4-experiment-password"
|
||||
|
||||
type apiClient struct {
|
||||
baseURL string
|
||||
client *http.Client
|
||||
token string
|
||||
}
|
||||
|
||||
type attachment struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
type setup struct {
|
||||
client *apiClient
|
||||
chatID string
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(os.Args[1:], os.Stdout); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(arguments []string, output io.Writer) error {
|
||||
if len(arguments) == 0 {
|
||||
return errors.New("usage: lab4experiment memory|consistency [flags]")
|
||||
}
|
||||
switch arguments[0] {
|
||||
case "memory":
|
||||
return runMemory(arguments[1:], output)
|
||||
case "consistency":
|
||||
return runConsistency(arguments[1:], output)
|
||||
default:
|
||||
return errors.New("usage: lab4experiment memory|consistency [flags]")
|
||||
}
|
||||
}
|
||||
|
||||
type memoryResult struct {
|
||||
Replicas int `json:"replicas"`
|
||||
Concurrency int `json:"concurrency"`
|
||||
Rounds int `json:"rounds"`
|
||||
SettleTimeMillis int64 `json:"settle_time_ms"`
|
||||
ImageBytes int `json:"image_bytes"`
|
||||
UploadRateBytes int64 `json:"upload_rate_bytes_per_second"`
|
||||
SuccessfulUploads int64 `json:"successful_uploads"`
|
||||
FailedUploads int64 `json:"failed_uploads"`
|
||||
DurationMillis int64 `json:"duration_ms"`
|
||||
HeapBaselineBytes int64 `json:"heap_baseline_bytes"`
|
||||
HeapPeakBytes int64 `json:"heap_peak_bytes"`
|
||||
HeapDeltaBytes int64 `json:"heap_delta_bytes"`
|
||||
RSSBaselineBytes int64 `json:"rss_baseline_bytes"`
|
||||
RSSPeakBytes int64 `json:"rss_peak_bytes"`
|
||||
RSSDeltaBytes int64 `json:"rss_delta_bytes"`
|
||||
}
|
||||
|
||||
func runMemory(arguments []string, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("memory", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
|
||||
prometheusURL := flags.String("prometheus-url", "http://prometheus:9090", "Prometheus base URL")
|
||||
replicas := flags.Int("replicas", 2, "expected API replicas")
|
||||
concurrency := flags.Int("concurrency", 12, "number of concurrent uploads")
|
||||
rounds := flags.Int("rounds", 4, "number of upload waves")
|
||||
width := flags.Int("width", 2048, "image width")
|
||||
height := flags.Int("height", 2048, "image height")
|
||||
uploadRate := flags.Int64("upload-rate", 1536*1024, "bytes per second for each upload")
|
||||
settleTime := flags.Duration("settle-time", 3*time.Second, "delay before the baseline sample")
|
||||
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
||||
return errors.New("invalid memory experiment flags")
|
||||
}
|
||||
if *replicas < 1 || *concurrency < 10 || *rounds < 1 || *width < 1 || *height < 1 || *uploadRate < 1 || *settleTime < 0 {
|
||||
return errors.New("concurrency must be at least 10 and rounds/dimensions/rate must be positive")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute)
|
||||
defer cancel()
|
||||
fixture, err := jpegFixture(*width, *height)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := waitForReplicas(ctx, *prometheusURL, *replicas); err != nil {
|
||||
return err
|
||||
}
|
||||
settle := time.NewTimer(*settleTime)
|
||||
defer settle.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-settle.C:
|
||||
}
|
||||
heapBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rssBaseline, err := prometheusValue(ctx, *prometheusURL, freshMetricQuery("process_resident_memory_bytes"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
environment, err := prepare(ctx, *baseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var successful atomic.Int64
|
||||
var failed atomic.Int64
|
||||
var firstError struct {
|
||||
sync.Mutex
|
||||
err error
|
||||
}
|
||||
heapPeak := heapBaseline
|
||||
rssPeak := rssBaseline
|
||||
loadDone := make(chan struct{})
|
||||
sampleDone := make(chan error, 1)
|
||||
go sampleMemory(ctx, *prometheusURL, *replicas, loadDone, &heapPeak, &rssPeak, sampleDone)
|
||||
started := time.Now()
|
||||
for round := range *rounds {
|
||||
var workers sync.WaitGroup
|
||||
workers.Add(*concurrency)
|
||||
for index := range *concurrency {
|
||||
go func(round, index int) {
|
||||
defer workers.Done()
|
||||
reader := newRateReader(fixture, *uploadRate)
|
||||
filename := fmt.Sprintf("memory-%02d-%02d.jpg", round, index)
|
||||
if _, err := environment.client.upload(ctx, environment.chatID, filename, reader, int64(len(fixture))); err != nil {
|
||||
failed.Add(1)
|
||||
firstError.Lock()
|
||||
if firstError.err == nil {
|
||||
firstError.err = err
|
||||
}
|
||||
firstError.Unlock()
|
||||
return
|
||||
}
|
||||
successful.Add(1)
|
||||
}(round, index)
|
||||
}
|
||||
workers.Wait()
|
||||
}
|
||||
close(loadDone)
|
||||
if err := <-sampleDone; err != nil {
|
||||
return err
|
||||
}
|
||||
result := memoryResult{
|
||||
Replicas: *replicas, Concurrency: *concurrency, Rounds: *rounds, SettleTimeMillis: settleTime.Milliseconds(),
|
||||
ImageBytes: len(fixture), UploadRateBytes: *uploadRate,
|
||||
SuccessfulUploads: successful.Load(), FailedUploads: failed.Load(), DurationMillis: time.Since(started).Milliseconds(),
|
||||
HeapBaselineBytes: heapBaseline, HeapPeakBytes: heapPeak, HeapDeltaBytes: heapPeak - heapBaseline,
|
||||
RSSBaselineBytes: rssBaseline, RSSPeakBytes: rssPeak, RSSDeltaBytes: rssPeak - rssBaseline,
|
||||
}
|
||||
if result.FailedUploads > 0 {
|
||||
return fmt.Errorf("%d uploads failed: %w", result.FailedUploads, firstError.err)
|
||||
}
|
||||
return writeJSON(output, result)
|
||||
}
|
||||
|
||||
func sampleMemory(
|
||||
ctx context.Context,
|
||||
prometheusURL string,
|
||||
expectedReplicas int,
|
||||
done <-chan struct{},
|
||||
heapPeak *int64,
|
||||
rssPeak *int64,
|
||||
result chan<- error,
|
||||
) {
|
||||
ticker := time.NewTicker(500 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
result <- ctx.Err()
|
||||
return
|
||||
case <-done:
|
||||
result <- nil
|
||||
return
|
||||
case <-ticker.C:
|
||||
replicas, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery())
|
||||
if err != nil {
|
||||
result <- err
|
||||
return
|
||||
}
|
||||
if replicas != int64(expectedReplicas) {
|
||||
result <- fmt.Errorf("active API replicas = %d, expected %d", replicas, expectedReplicas)
|
||||
return
|
||||
}
|
||||
heap, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("go_memstats_heap_alloc_bytes"))
|
||||
if err != nil {
|
||||
result <- err
|
||||
return
|
||||
}
|
||||
rss, err := prometheusValue(ctx, prometheusURL, freshMetricQuery("process_resident_memory_bytes"))
|
||||
if err != nil {
|
||||
result <- err
|
||||
return
|
||||
}
|
||||
*heapPeak = max(*heapPeak, heap)
|
||||
*rssPeak = max(*rssPeak, rss)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type consistencyResult struct {
|
||||
PageSize int `json:"page_size"`
|
||||
SeededConsistent int `json:"seeded_consistent"`
|
||||
FreshApply reconciliationRun `json:"fresh_apply"`
|
||||
DryRun reconciliationRun `json:"dry_run"`
|
||||
Apply reconciliationRun `json:"apply"`
|
||||
RepeatedApply reconciliationRun `json:"repeated_apply"`
|
||||
FreshPreserved bool `json:"fresh_preserved"`
|
||||
OrphanDeleted bool `json:"orphan_deleted"`
|
||||
UnusedDeleted bool `json:"unused_metadata_deleted"`
|
||||
InUsePreserved bool `json:"in_use_metadata_preserved"`
|
||||
AbandonedDeleted bool `json:"abandoned_attachment_deleted"`
|
||||
TotalDurationMillis int64 `json:"total_duration_ms"`
|
||||
}
|
||||
|
||||
type reconciliationRun struct {
|
||||
DurationMillis int64 `json:"duration_ms"`
|
||||
Report reconcile.Report `json:"report"`
|
||||
}
|
||||
|
||||
func runConsistency(arguments []string, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("consistency", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
|
||||
pageSize := flags.Int("page-size", 10, "reconciliation page size")
|
||||
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
||||
return errors.New("invalid consistency experiment flags")
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
settings, err := config.LoadReconciliation()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool, err := database.Open(ctx, database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
objects, err := objectstore.New(objectstore.Config{
|
||||
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
metadata := postgres.New(pool)
|
||||
clean, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, GraceTime: 0, UnreferencedRetention: time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
initial, err := clean.Run(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if hasAnomalies(initial) {
|
||||
return errors.New("consistency experiment requires a stack without existing attachment anomalies")
|
||||
}
|
||||
|
||||
started := time.Now()
|
||||
environment, err := prepare(ctx, *baseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fixture, err := jpegFixture(320, 200)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
seededConsistent := *pageSize + 1
|
||||
for index := range seededConsistent {
|
||||
filename := fmt.Sprintf("consistent-%03d.jpg", index)
|
||||
if _, err := environment.client.upload(ctx, environment.chatID, filename, bytes.NewReader(fixture), int64(len(fixture))); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
unused, err := environment.client.upload(ctx, environment.chatID, "unused-missing.jpg", bytes.NewReader(fixture), int64(len(fixture)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
inUse, err := environment.client.upload(ctx, environment.chatID, "in-use-missing.jpg", bytes.NewReader(fixture), int64(len(fixture)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := environment.client.createMessage(ctx, environment.chatID, inUse.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
abandoned, err := environment.client.upload(ctx, environment.chatID, "abandoned.jpg", bytes.NewReader(fixture), int64(len(fixture)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE attachments SET status = 'pending', ready_at = NULL WHERE id = $1`, abandoned.ID); err != nil {
|
||||
return fmt.Errorf("prepare abandoned pending attachment: %w", err)
|
||||
}
|
||||
keys, err := attachmentKeys(ctx, pool, []string{unused.ID, inUse.ID, abandoned.ID})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := objects.Delete(ctx, keys[unused.ID]); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := objects.Delete(ctx, keys[inUse.ID]); err != nil {
|
||||
return err
|
||||
}
|
||||
orphanKey := "attachments/experiment-" + uuid.NewString() + "/original"
|
||||
if err := objects.Put(ctx, orphanKey, "image/jpeg", fixture); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fresh, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, GraceTime: time.Hour,
|
||||
UnreferencedRetention: time.Hour, Apply: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
freshRun, err := measuredReconciliation(ctx, fresh)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
freshPreserved := objectExists(ctx, objects, orphanKey) &&
|
||||
attachmentExists(ctx, pool, unused.ID) && attachmentExists(ctx, pool, inUse.ID) &&
|
||||
attachmentExists(ctx, pool, abandoned.ID)
|
||||
if _, err := pool.Exec(ctx, `
|
||||
UPDATE attachments
|
||||
SET created_at = clock_timestamp() - INTERVAL '2 hours'
|
||||
WHERE id = $1`, abandoned.ID); err != nil {
|
||||
return fmt.Errorf("age abandoned attachment: %w", err)
|
||||
}
|
||||
|
||||
dryRun, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dryRunResult, err := measuredReconciliation(ctx, dryRun)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
apply, err := reconcile.New(metadata, objects, reconcile.Config{
|
||||
Prefix: "attachments/", PageSize: *pageSize, UnreferencedRetention: time.Hour, Apply: true,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
applyResult, err := measuredReconciliation(ctx, apply)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
repeatedResult, err := measuredReconciliation(ctx, apply)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
result := consistencyResult{
|
||||
PageSize: *pageSize, SeededConsistent: seededConsistent,
|
||||
FreshApply: freshRun, DryRun: dryRunResult, Apply: applyResult, RepeatedApply: repeatedResult,
|
||||
FreshPreserved: freshPreserved, OrphanDeleted: !objectExists(ctx, objects, orphanKey),
|
||||
UnusedDeleted: !attachmentExists(ctx, pool, unused.ID), InUsePreserved: attachmentExists(ctx, pool, inUse.ID),
|
||||
AbandonedDeleted: !attachmentExists(ctx, pool, abandoned.ID) && !objectExists(ctx, objects, keys[abandoned.ID]),
|
||||
TotalDurationMillis: time.Since(started).Milliseconds(),
|
||||
}
|
||||
if !result.FreshPreserved || !result.OrphanDeleted || !result.UnusedDeleted || !result.InUsePreserved || !result.AbandonedDeleted ||
|
||||
repeatedResult.Report.DeletedOrphanObjects != 0 || repeatedResult.Report.DeletedUnreferencedRows != 0 {
|
||||
return fmt.Errorf("unexpected reconciliation result: %+v", result)
|
||||
}
|
||||
return writeJSON(output, result)
|
||||
}
|
||||
|
||||
func measuredReconciliation(ctx context.Context, reconciler *reconcile.Reconciler) (reconciliationRun, error) {
|
||||
started := time.Now()
|
||||
report, err := reconciler.Run(ctx)
|
||||
return reconciliationRun{DurationMillis: time.Since(started).Milliseconds(), Report: report}, err
|
||||
}
|
||||
|
||||
func hasAnomalies(report reconcile.Report) bool {
|
||||
return report.FreshOrphanObjects != 0 || report.OldOrphanObjects != 0 ||
|
||||
report.FreshMissingObjects != 0 || report.ReferencedMissingObjects != 0 ||
|
||||
report.OldUnreferencedMetadata != 0 || report.OldPendingAttachments != 0 ||
|
||||
report.OldUnreferencedAttachments != 0 || report.DeletingAttachments != 0
|
||||
}
|
||||
|
||||
func attachmentKeys(ctx context.Context, pool *pgxpool.Pool, ids []string) (map[string]string, error) {
|
||||
rows, err := pool.Query(ctx, `SELECT id::text, object_key FROM attachments WHERE id::text = ANY($1)`, ids)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query attachment keys: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
keys := make(map[string]string, len(ids))
|
||||
for rows.Next() {
|
||||
var id string
|
||||
var key string
|
||||
if err := rows.Scan(&id, &key); err != nil {
|
||||
return nil, fmt.Errorf("scan attachment key: %w", err)
|
||||
}
|
||||
keys[id] = key
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, fmt.Errorf("iterate attachment keys: %w", err)
|
||||
}
|
||||
if len(keys) != len(ids) {
|
||||
return nil, errors.New("not all attachment keys were found")
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func attachmentExists(ctx context.Context, pool *pgxpool.Pool, id string) bool {
|
||||
var exists bool
|
||||
return pool.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM attachments WHERE id = $1)`, id).Scan(&exists) == nil && exists
|
||||
}
|
||||
|
||||
func objectExists(ctx context.Context, objects *objectstore.Store, key string) bool {
|
||||
_, err := objects.Stat(ctx, key)
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func prepare(ctx context.Context, baseURL string) (setup, error) {
|
||||
client := &apiClient{baseURL: strings.TrimRight(baseURL, "/"), client: &http.Client{Timeout: 90 * time.Second}}
|
||||
username := "lab4_" + strings.ReplaceAll(uuid.NewString()[:18], "-", "")
|
||||
var user struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/users", map[string]string{
|
||||
"username": username, "display_name": "Lab 4 experiment", "password": experimentPassword,
|
||||
}, &user, ""); err != nil {
|
||||
return setup{}, err
|
||||
}
|
||||
credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + experimentPassword))
|
||||
var session struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/auth/sessions", nil, &session, "Basic "+credentials); err != nil {
|
||||
return setup{}, err
|
||||
}
|
||||
client.token = session.Token
|
||||
var chat struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
if err := client.jsonRequest(ctx, http.MethodPost, "/api/v1/chats", map[string]any{
|
||||
"title": "Lab 4 experiment", "member_ids": []string{user.ID},
|
||||
}, &chat, "Bearer "+client.token); err != nil {
|
||||
return setup{}, err
|
||||
}
|
||||
return setup{client: client, chatID: chat.ID}, nil
|
||||
}
|
||||
|
||||
func (c *apiClient) upload(
|
||||
ctx context.Context,
|
||||
chatID string,
|
||||
filename string,
|
||||
body io.Reader,
|
||||
contentLength int64,
|
||||
) (attachment, error) {
|
||||
path := "/api/v1/chats/" + chatID + "/attachments?filename=" + filename
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+path, body)
|
||||
if err != nil {
|
||||
return attachment{}, err
|
||||
}
|
||||
request.ContentLength = contentLength
|
||||
request.Header.Set("Authorization", "Bearer "+c.token)
|
||||
request.Header.Set("Content-Type", "image/jpeg")
|
||||
response, err := c.client.Do(request)
|
||||
if err != nil {
|
||||
return attachment{}, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
message, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return attachment{}, fmt.Errorf("upload status %d: %s", response.StatusCode, message)
|
||||
}
|
||||
var result attachment
|
||||
if err := json.NewDecoder(response.Body).Decode(&result); err != nil {
|
||||
return attachment{}, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *apiClient) createMessage(ctx context.Context, chatID, attachmentID string) error {
|
||||
return c.jsonRequest(ctx, http.MethodPost, "/api/v1/chats/"+chatID+"/messages", map[string]any{
|
||||
"text": "", "attachment_ids": []string{attachmentID},
|
||||
}, nil, "Bearer "+c.token, "Idempotency-Key", uuid.NewString())
|
||||
}
|
||||
|
||||
func (c *apiClient) jsonRequest(
|
||||
ctx context.Context,
|
||||
method string,
|
||||
path string,
|
||||
payload any,
|
||||
result any,
|
||||
authorization string,
|
||||
extraHeader ...string,
|
||||
) error {
|
||||
var body io.Reader
|
||||
if payload != nil {
|
||||
encoded, err := json.Marshal(payload)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body = bytes.NewReader(encoded)
|
||||
}
|
||||
request, err := http.NewRequestWithContext(ctx, method, c.baseURL+path, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if payload != nil {
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if authorization != "" {
|
||||
request.Header.Set("Authorization", authorization)
|
||||
}
|
||||
for index := 0; index+1 < len(extraHeader); index += 2 {
|
||||
request.Header.Set(extraHeader[index], extraHeader[index+1])
|
||||
}
|
||||
response, err := c.client.Do(request)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode < 200 || response.StatusCode >= 300 {
|
||||
message, _ := io.ReadAll(io.LimitReader(response.Body, 4096))
|
||||
return fmt.Errorf("%s %s status %d: %s", method, path, response.StatusCode, message)
|
||||
}
|
||||
if result == nil {
|
||||
return nil
|
||||
}
|
||||
return json.NewDecoder(response.Body).Decode(result)
|
||||
}
|
||||
|
||||
func waitForReplicas(ctx context.Context, prometheusURL string, expected int) error {
|
||||
ticker := time.NewTicker(time.Second)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
count, err := prometheusValue(ctx, prometheusURL, activeReplicasQuery())
|
||||
if err == nil && count == int64(expected) {
|
||||
return nil
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return fmt.Errorf("wait for %d Prometheus API targets: %w", expected, ctx.Err())
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func activeReplicasQuery() string {
|
||||
return `count(last_over_time(process_resident_memory_bytes{job="messenger"}[10s]))`
|
||||
}
|
||||
|
||||
func freshMetricQuery(metric string) string {
|
||||
return fmt.Sprintf(`sum(last_over_time(%s{job="messenger"}[10s]))`, metric)
|
||||
}
|
||||
|
||||
func prometheusValue(ctx context.Context, baseURL, query string) (int64, error) {
|
||||
request, err := http.NewRequestWithContext(ctx, http.MethodGet, strings.TrimRight(baseURL, "/")+"/api/v1/query", nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
values := request.URL.Query()
|
||||
values.Set("query", query)
|
||||
request.URL.RawQuery = values.Encode()
|
||||
response, err := http.DefaultClient.Do(request)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return 0, fmt.Errorf("prometheus status %d", response.StatusCode)
|
||||
}
|
||||
var payload struct {
|
||||
Status string `json:"status"`
|
||||
Data struct {
|
||||
Result []struct {
|
||||
Value []json.RawMessage `json:"value"`
|
||||
} `json:"result"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.NewDecoder(response.Body).Decode(&payload); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if payload.Status != "success" || len(payload.Data.Result) != 1 || len(payload.Data.Result[0].Value) != 2 {
|
||||
return 0, errors.New("prometheus query returned no scalar value")
|
||||
}
|
||||
var raw string
|
||||
if err := json.Unmarshal(payload.Data.Result[0].Value[1], &raw); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
value, err := strconv.ParseFloat(raw, 64)
|
||||
return int64(value), err
|
||||
}
|
||||
|
||||
func jpegFixture(width, height int) ([]byte, error) {
|
||||
pixels := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||
state := uint64(0x9e3779b97f4a7c15)
|
||||
for index := range pixels.Pix {
|
||||
state ^= state << 13
|
||||
state ^= state >> 7
|
||||
state ^= state << 17
|
||||
pixels.Pix[index] = byte(state >> 24)
|
||||
}
|
||||
var encoded bytes.Buffer
|
||||
if err := jpeg.Encode(&encoded, pixels, &jpeg.Options{Quality: 90}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return encoded.Bytes(), nil
|
||||
}
|
||||
|
||||
type rateReader struct {
|
||||
reader *bytes.Reader
|
||||
rate int64
|
||||
started time.Time
|
||||
read int64
|
||||
}
|
||||
|
||||
func newRateReader(data []byte, rate int64) *rateReader {
|
||||
return &rateReader{reader: bytes.NewReader(data), rate: rate, started: time.Now()}
|
||||
}
|
||||
|
||||
func (r *rateReader) Read(buffer []byte) (int, error) {
|
||||
for {
|
||||
allowed := time.Since(r.started).Nanoseconds()*r.rate/int64(time.Second) - r.read
|
||||
if allowed > 0 {
|
||||
if int64(len(buffer)) > allowed {
|
||||
buffer = buffer[:allowed]
|
||||
}
|
||||
count, err := r.reader.Read(buffer)
|
||||
r.read += int64(count)
|
||||
return count, err
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
func writeJSON(output io.Writer, value any) error {
|
||||
encoder := json.NewEncoder(output)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(value)
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
)
|
||||
|
||||
type apiState struct {
|
||||
BaseURL string `json:"base_url"`
|
||||
Token string `json:"token"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
}
|
||||
|
||||
type apiUser struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
type apiSession struct {
|
||||
Token string `json:"token"`
|
||||
}
|
||||
|
||||
type apiChat struct {
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
func runAPIProduce(arguments []string, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("api-produce", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
baseURL := flags.String("base-url", "http://nginx:8080", "messenger base URL")
|
||||
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
||||
return errors.New("usage: lab5experiment api-produce [-base-url URL]")
|
||||
}
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12]
|
||||
password := "Lab5-" + uuid.NewString()
|
||||
owner, err := registerAPIUser(client, *baseURL, "qa_"+suffix, password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
member, err := registerAPIUser(client, *baseURL, "qb_"+suffix, password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
token, err := createAPISession(client, *baseURL, "qa_"+suffix, password)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chat, err := createAPIChat(client, *baseURL, token, owner.ID, member.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fixture, err := pngFixture(320, 200)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var attachment model.Attachment
|
||||
response, err := apiRequest(
|
||||
client, http.MethodPost,
|
||||
*baseURL+"/api/v1/chats/"+chat.ID+"/attachments?filename=queued.png",
|
||||
token, "image/png", fixture, &attachment,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if response.StatusCode != http.StatusAccepted || attachment.Status != model.AttachmentQueued {
|
||||
return fmt.Errorf("upload returned status=%d attachment_status=%s", response.StatusCode, attachment.Status)
|
||||
}
|
||||
messageBody, err := json.Marshal(map[string]any{"text": "", "attachment_ids": []string{attachment.ID}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response, err = apiRequestWithHeaders(
|
||||
client, http.MethodPost, *baseURL+"/api/v1/chats/"+chat.ID+"/messages",
|
||||
token, "application/json", messageBody, nil,
|
||||
map[string]string{"Idempotency-Key": uuid.NewString()},
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
return fmt.Errorf("link queued attachment returned %d", response.StatusCode)
|
||||
}
|
||||
response, err = apiRequest(client, http.MethodGet, *baseURL+"/api/v1/attachments/"+attachment.ID+"/content", token, "", nil, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if response.StatusCode != http.StatusConflict {
|
||||
return fmt.Errorf("queued content returned %d, expected 409", response.StatusCode)
|
||||
}
|
||||
return json.NewEncoder(output).Encode(apiState{BaseURL: *baseURL, Token: token, AttachmentID: attachment.ID})
|
||||
}
|
||||
|
||||
func runAPIVerify(arguments []string, input io.Reader, output io.Writer) error {
|
||||
flags := flag.NewFlagSet("api-verify", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
waitTimeout := flags.Duration("wait-timeout", time.Minute, "ready wait timeout")
|
||||
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 {
|
||||
return errors.New("usage: lab5experiment api-verify [-wait-timeout 1m]")
|
||||
}
|
||||
var state apiState
|
||||
if err := json.NewDecoder(input).Decode(&state); err != nil {
|
||||
return fmt.Errorf("decode API scenario state: %w", err)
|
||||
}
|
||||
client := &http.Client{Timeout: 10 * time.Second}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), *waitTimeout)
|
||||
defer cancel()
|
||||
var attachment model.Attachment
|
||||
for {
|
||||
response, err := apiRequest(client, http.MethodGet, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID, state.Token, "", nil, &attachment)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("attachment metadata returned %d", response.StatusCode)
|
||||
}
|
||||
if attachment.Status == model.AttachmentReady {
|
||||
break
|
||||
}
|
||||
if attachment.Status == model.AttachmentFailed {
|
||||
return fmt.Errorf("attachment failed with %v", attachment.ErrorCode)
|
||||
}
|
||||
if err := wait(ctx, 250*time.Millisecond); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if len(attachment.Variants) != 3 {
|
||||
return fmt.Errorf("public variants = %d, expected 3", len(attachment.Variants))
|
||||
}
|
||||
response, original, err := apiBytes(client, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID+"/content", state.Token)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fixture, err := pngFixture(320, 200)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if response.StatusCode != http.StatusOK || !bytes.Equal(original, fixture) {
|
||||
return errors.New("original changed after worker and broker restart")
|
||||
}
|
||||
response, thumbnail, err := apiBytes(
|
||||
client, state.BaseURL+"/api/v1/attachments/"+state.AttachmentID+"/content?variant=thumbnail", state.Token,
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if response.StatusCode != http.StatusOK || response.Header.Get("Content-Type") != "image/jpeg" {
|
||||
return errors.New("thumbnail response is invalid")
|
||||
}
|
||||
imageConfig, format, err := image.DecodeConfig(bytes.NewReader(thumbnail))
|
||||
if err != nil || format != "jpeg" || imageConfig.Width != 256 || imageConfig.Height != 160 {
|
||||
return errors.New("thumbnail dimensions are invalid")
|
||||
}
|
||||
return json.NewEncoder(output).Encode(map[string]any{
|
||||
"accepted_while_worker_stopped": true,
|
||||
"accepted_before_outbox_publish": true,
|
||||
"linked_while_queued": true,
|
||||
"content_conflict_while_queued": true,
|
||||
"ready_after_api_broker_restart": true,
|
||||
"original_preserved": true,
|
||||
"thumbnail_dimensions": "256x160",
|
||||
})
|
||||
}
|
||||
|
||||
func registerAPIUser(client *http.Client, baseURL, username, password string) (apiUser, error) {
|
||||
body, err := json.Marshal(map[string]string{
|
||||
"username": username, "display_name": username, "password": password,
|
||||
})
|
||||
if err != nil {
|
||||
return apiUser{}, err
|
||||
}
|
||||
var user apiUser
|
||||
response, err := apiRequest(client, http.MethodPost, baseURL+"/api/v1/users", "", "application/json", body, &user)
|
||||
if err != nil {
|
||||
return apiUser{}, err
|
||||
}
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
return apiUser{}, fmt.Errorf("register user returned %d", response.StatusCode)
|
||||
}
|
||||
return user, nil
|
||||
}
|
||||
|
||||
func createAPISession(client *http.Client, baseURL, username, password string) (string, error) {
|
||||
credentials := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
|
||||
request, err := http.NewRequest(http.MethodPost, baseURL+"/api/v1/auth/sessions", nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
request.Header.Set("Authorization", "Basic "+credentials)
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
return "", fmt.Errorf("create session returned %d", response.StatusCode)
|
||||
}
|
||||
var session apiSession
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&session); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return session.Token, nil
|
||||
}
|
||||
|
||||
func createAPIChat(client *http.Client, baseURL, token, ownerID, memberID string) (apiChat, error) {
|
||||
body, err := json.Marshal(map[string]any{
|
||||
"title": "Lab 5 resilience", "member_ids": []string{ownerID, memberID},
|
||||
})
|
||||
if err != nil {
|
||||
return apiChat{}, err
|
||||
}
|
||||
var chat apiChat
|
||||
response, err := apiRequest(client, http.MethodPost, baseURL+"/api/v1/chats", token, "application/json", body, &chat)
|
||||
if err != nil {
|
||||
return apiChat{}, err
|
||||
}
|
||||
if response.StatusCode != http.StatusCreated {
|
||||
return apiChat{}, fmt.Errorf("create chat returned %d", response.StatusCode)
|
||||
}
|
||||
return chat, nil
|
||||
}
|
||||
|
||||
func apiRequest(
|
||||
client *http.Client,
|
||||
method string,
|
||||
requestURL string,
|
||||
token string,
|
||||
contentType string,
|
||||
body []byte,
|
||||
result any,
|
||||
) (*http.Response, error) {
|
||||
return apiRequestWithHeaders(client, method, requestURL, token, contentType, body, result, nil)
|
||||
}
|
||||
|
||||
func apiRequestWithHeaders(
|
||||
client *http.Client,
|
||||
method string,
|
||||
requestURL string,
|
||||
token string,
|
||||
contentType string,
|
||||
body []byte,
|
||||
result any,
|
||||
headers map[string]string,
|
||||
) (*http.Response, error) {
|
||||
request, err := http.NewRequest(method, requestURL, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if token != "" {
|
||||
request.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
if contentType != "" {
|
||||
request.Header.Set("Content-Type", contentType)
|
||||
}
|
||||
for name, value := range headers {
|
||||
request.Header.Set(name, value)
|
||||
}
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if result != nil && response.StatusCode < http.StatusBadRequest {
|
||||
if err := json.NewDecoder(io.LimitReader(response.Body, 16<<20)).Decode(result); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return response, nil
|
||||
}
|
||||
|
||||
func apiBytes(client *http.Client, requestURL, token string) (*http.Response, []byte, error) {
|
||||
request, err := http.NewRequest(http.MethodGet, requestURL, nil)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
request.Header.Set("Authorization", "Bearer "+token)
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, 16<<20))
|
||||
return response, body, err
|
||||
}
|
||||
@@ -0,0 +1,605 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/png"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
|
||||
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
|
||||
)
|
||||
|
||||
const (
|
||||
experimentLease = 2 * time.Second
|
||||
maxAttempts = 5
|
||||
)
|
||||
|
||||
type result struct {
|
||||
StartedAt string `json:"started_at"`
|
||||
DurationMillis int64 `json:"duration_ms"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
FaultScenarios []scenario `json:"fault_scenarios"`
|
||||
DuplicateDeliveries int `json:"duplicate_deliveries"`
|
||||
V1ObjectCount int `json:"v1_object_count"`
|
||||
V2ObjectCount int `json:"v2_object_count"`
|
||||
AcceptedJobs int `json:"accepted_jobs"`
|
||||
TerminalJobs int `json:"terminal_jobs"`
|
||||
ReadyJobs int `json:"ready_jobs"`
|
||||
FailedJobs int `json:"failed_jobs"`
|
||||
PermanentErrorCode string `json:"permanent_error_code"`
|
||||
ReplayAttempt int `json:"replay_attempt"`
|
||||
InvariantResults map[string]bool `json:"invariant_results"`
|
||||
}
|
||||
|
||||
type scenario struct {
|
||||
Name string `json:"name"`
|
||||
AttemptAfterRecovery int `json:"attempt_after_recovery"`
|
||||
ObjectsBeforeRecovery int `json:"objects_before_recovery"`
|
||||
ObjectsAfterRecovery int `json:"objects_after_recovery"`
|
||||
FinalStatus string `json:"final_status"`
|
||||
}
|
||||
|
||||
type experiment struct {
|
||||
settings config.WorkerConfig
|
||||
repository *postgres.Store
|
||||
objects *objectstore.Store
|
||||
broker *redisstream.Broker
|
||||
dispatcher *outbox.Dispatcher
|
||||
worker *worker.Worker
|
||||
ownerID string
|
||||
chatID string
|
||||
fixture []byte
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := run(os.Args[1:], os.Stdout); err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(arguments []string, output io.Writer) error {
|
||||
if len(arguments) > 0 {
|
||||
switch arguments[0] {
|
||||
case "api-produce":
|
||||
return runAPIProduce(arguments[1:], output)
|
||||
case "api-verify":
|
||||
return runAPIVerify(arguments[1:], os.Stdin, output)
|
||||
}
|
||||
}
|
||||
flags := flag.NewFlagSet("lab5experiment", flag.ContinueOnError)
|
||||
flags.SetOutput(io.Discard)
|
||||
timeout := flags.Duration("timeout", 2*time.Minute, "experiment timeout")
|
||||
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 || *timeout <= 0 {
|
||||
return errors.New("usage: lab5experiment [-timeout 2m]")
|
||||
}
|
||||
started := time.Now()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
|
||||
defer cancel()
|
||||
experiment, closeExperiment, err := newExperiment(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closeExperiment()
|
||||
|
||||
report, err := experiment.run(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
report.StartedAt = started.UTC().Format(time.RFC3339)
|
||||
report.DurationMillis = time.Since(started).Milliseconds()
|
||||
encoder := json.NewEncoder(output)
|
||||
encoder.SetIndent("", " ")
|
||||
return encoder.Encode(report)
|
||||
}
|
||||
|
||||
func newExperiment(ctx context.Context) (*experiment, func(), error) {
|
||||
settings, err := config.LoadWorker()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
pool, err := database.Open(ctx, database.Config{
|
||||
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
||||
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
||||
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
objects, err := objectstore.New(objectstore.Config{
|
||||
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
||||
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
||||
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
|
||||
})
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
redisOptions, err := redis.ParseURL(settings.BrokerURL)
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
client := redis.NewClient(redisOptions)
|
||||
if err := client.Ping(ctx).Err(); err != nil {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
consumer := "lab5experiment-" + uuid.NewString()
|
||||
broker, err := redisstream.New(client, redisstream.Config{
|
||||
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer,
|
||||
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: time.Second,
|
||||
ClaimMinIdle: experimentLease, ReadCount: 32,
|
||||
})
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := broker.Ensure(ctx); err != nil {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
repository := postgres.New(pool)
|
||||
dispatcher, err := outbox.New(repository, broker, nil, outbox.Config{
|
||||
BatchSize: 32, LeaseDuration: 10 * time.Second, PollInterval: time.Second,
|
||||
RecoveryInterval: 10 * time.Second, RetryDelay: time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels))
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
processorWorker, err := worker.New(
|
||||
slog.New(slog.NewTextHandler(io.Discard, nil)), repository, broker, objects, processor, nil,
|
||||
worker.Config{
|
||||
WorkerID: consumer, Concurrency: 1,
|
||||
LeaseDuration: experimentLease, ProcessingTimeout: time.Second,
|
||||
ReclaimInterval: experimentLease,
|
||||
MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second,
|
||||
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
|
||||
MemoryBytes: settings.MemoryBytes,
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
fixture, err := pngFixture(320, 200)
|
||||
if err != nil {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
value := &experiment{
|
||||
settings: settings, repository: repository, objects: objects, broker: broker,
|
||||
dispatcher: dispatcher, worker: processorWorker, fixture: fixture,
|
||||
}
|
||||
if err := value.seed(ctx); err != nil {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
return nil, nil, err
|
||||
}
|
||||
closeExperiment := func() {
|
||||
pool.Close()
|
||||
_ = client.Close()
|
||||
}
|
||||
return value, closeExperiment, nil
|
||||
}
|
||||
|
||||
func (e *experiment) seed(ctx context.Context) error {
|
||||
suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12]
|
||||
owner, err := e.repository.CreateUser(ctx, "l5a_"+suffix, "Lab 5 A", "$argon2id$experiment")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
member, err := e.repository.CreateUser(ctx, "l5b_"+suffix, "Lab 5 B", "$argon2id$experiment")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chat, err := e.repository.CreateChat(ctx, "Lab 5 experiment", []string{owner.ID, member.ID}, owner.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
e.ownerID = owner.ID
|
||||
e.chatID = chat.ID
|
||||
return nil
|
||||
}
|
||||
|
||||
func (e *experiment) run(ctx context.Context) (result, error) {
|
||||
report := result{
|
||||
PipelineVersion: 1,
|
||||
InvariantResults: map[string]bool{
|
||||
"accepted_jobs_reach_terminal_state": false,
|
||||
"duplicate_effect_is_unique": false,
|
||||
"old_pipeline_cannot_replace_new": false,
|
||||
},
|
||||
}
|
||||
for _, fault := range []worker.FaultPoint{worker.FaultBeforeSideEffect, worker.FaultAfterSideEffect} {
|
||||
observed, err := e.runLeaseRecovery(ctx, fault)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
report.FaultScenarios = append(report.FaultScenarios, observed)
|
||||
report.AcceptedJobs++
|
||||
if observed.FinalStatus == model.AttachmentReady {
|
||||
report.ReadyJobs++
|
||||
report.TerminalJobs++
|
||||
}
|
||||
}
|
||||
permanentCode, replayAttempt, err := e.runPermanentFailureReplay(ctx)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
report.AcceptedJobs++
|
||||
report.TerminalJobs++
|
||||
report.FailedJobs++
|
||||
report.PermanentErrorCode = permanentCode
|
||||
report.ReplayAttempt = replayAttempt
|
||||
|
||||
commitScenario, envelope, attachmentID, err := e.runAfterCommit(ctx)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
report.FaultScenarios = append(report.FaultScenarios, commitScenario)
|
||||
report.AcceptedJobs++
|
||||
if commitScenario.FinalStatus == model.AttachmentReady {
|
||||
report.ReadyJobs++
|
||||
report.TerminalJobs++
|
||||
}
|
||||
|
||||
payload, err := redisstream.EncodeEnvelope(envelope)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
for range 2 {
|
||||
if _, err := e.broker.PublishConfirmed(ctx, payload); err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
}
|
||||
duplicateDeliveries, err := e.readDeliveries(ctx, 2)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
for _, delivery := range duplicateDeliveries {
|
||||
if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
}
|
||||
report.DuplicateDeliveries = len(duplicateDeliveries)
|
||||
report.V1ObjectCount, err = e.objectCount(ctx, attachmentID)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
if report.V1ObjectCount != 4 {
|
||||
return result{}, fmt.Errorf("duplicate delivery produced %d objects, expected 4", report.V1ObjectCount)
|
||||
}
|
||||
report.InvariantResults["duplicate_effect_is_unique"] = true
|
||||
|
||||
oldPayload := payload
|
||||
v2Job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: envelope.PipelineVersion + 1}
|
||||
if _, err := e.repository.QueueAttachmentPipelineVersion(ctx, attachmentID, v2Job, "", ""); err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
v2Delivery, err := e.readDelivery(ctx)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
if err := e.worker.ProcessDelivery(ctx, v2Delivery, worker.FaultNone); err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
if _, err := e.broker.PublishConfirmed(ctx, oldPayload); err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
staleDelivery, err := e.readDelivery(ctx)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
if err := e.worker.ProcessDelivery(ctx, staleDelivery, worker.FaultNone); err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
if metadata.Status != model.AttachmentReady || metadata.PipelineVersion != v2Job.PipelineVersion || len(metadata.Variants) != 3 {
|
||||
return result{}, errors.New("new pipeline version was not preserved after stale delivery")
|
||||
}
|
||||
report.V2ObjectCount, err = e.objectCount(ctx, attachmentID)
|
||||
if err != nil {
|
||||
return result{}, err
|
||||
}
|
||||
if report.V2ObjectCount != 8 {
|
||||
return result{}, fmt.Errorf("versioned pipeline produced %d objects, expected 8", report.V2ObjectCount)
|
||||
}
|
||||
report.InvariantResults["old_pipeline_cannot_replace_new"] = true
|
||||
report.AcceptedJobs++
|
||||
report.ReadyJobs++
|
||||
report.TerminalJobs++
|
||||
report.InvariantResults["accepted_jobs_reach_terminal_state"] = report.AcceptedJobs == report.TerminalJobs
|
||||
return report, nil
|
||||
}
|
||||
|
||||
func (e *experiment) runPermanentFailureReplay(ctx context.Context) (string, int, error) {
|
||||
attachmentID, _, _, err := e.queueAttachmentData(ctx, []byte("corrupted image"), "image/png")
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
delivery, err := e.readDelivery(ctx)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil {
|
||||
return "", 0, errors.New("corrupted staging object did not reach failed")
|
||||
}
|
||||
firstCode := *metadata.ErrorCode
|
||||
replayed, err := e.repository.ReplayFailedAttachment(
|
||||
ctx, attachmentID, "lab5experiment", "verify explicit replay fencing",
|
||||
)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
replayDelivery, err := e.readDelivery(ctx)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if err := e.worker.ProcessDelivery(ctx, replayDelivery, worker.FaultNone); err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
metadata, err = e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
|
||||
if err != nil {
|
||||
return "", 0, err
|
||||
}
|
||||
if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil || *metadata.ErrorCode != firstCode {
|
||||
return "", 0, errors.New("replayed permanent failure changed terminal classification")
|
||||
}
|
||||
return firstCode, replayed.Attempt, nil
|
||||
}
|
||||
|
||||
func (e *experiment) runLeaseRecovery(ctx context.Context, fault worker.FaultPoint) (scenario, error) {
|
||||
attachmentID, _, err := e.queueAttachment(ctx)
|
||||
if err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
delivery, err := e.readDelivery(ctx)
|
||||
if err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
if err := e.worker.ProcessDelivery(ctx, delivery, fault); !errors.Is(err, worker.ErrInjectedFailure) {
|
||||
return scenario{}, fmt.Errorf("fault %s returned %v", fault, err)
|
||||
}
|
||||
before, err := e.objectCount(ctx, attachmentID)
|
||||
if err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
if fault == worker.FaultBeforeSideEffect && before != 0 {
|
||||
return scenario{}, errors.New("objects were written before the injected side effect")
|
||||
}
|
||||
if fault == worker.FaultAfterSideEffect && before != 4 {
|
||||
return scenario{}, fmt.Errorf("side effect wrote %d objects, expected 4", before)
|
||||
}
|
||||
if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
recovered, err := e.repository.RecoverExpiredAttachmentJobs(ctx, 10, maxAttempts, time.Now())
|
||||
if err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
if recovered.Requeued != 1 {
|
||||
return scenario{}, fmt.Errorf("requeued jobs = %d, expected 1", recovered.Requeued)
|
||||
}
|
||||
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
retry, err := e.readDelivery(ctx)
|
||||
if err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
if err := e.worker.ProcessDelivery(ctx, retry, worker.FaultNone); err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
if err := e.ackReclaimed(ctx); err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
|
||||
if err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
after, err := e.objectCount(ctx, attachmentID)
|
||||
if err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
job, err := e.repository.AttachmentJob(ctx, retry.Envelope.JobID.String())
|
||||
if err != nil {
|
||||
return scenario{}, err
|
||||
}
|
||||
return scenario{
|
||||
Name: string(fault), AttemptAfterRecovery: job.Attempt,
|
||||
ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (e *experiment) runAfterCommit(ctx context.Context) (scenario, redisstream.Envelope, string, error) {
|
||||
attachmentID, _, err := e.queueAttachment(ctx)
|
||||
if err != nil {
|
||||
return scenario{}, redisstream.Envelope{}, "", err
|
||||
}
|
||||
delivery, err := e.readDelivery(ctx)
|
||||
if err != nil {
|
||||
return scenario{}, redisstream.Envelope{}, "", err
|
||||
}
|
||||
if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultAfterCommit); !errors.Is(err, worker.ErrInjectedFailure) {
|
||||
return scenario{}, redisstream.Envelope{}, "", fmt.Errorf("after_commit returned %v", err)
|
||||
}
|
||||
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
|
||||
if err != nil {
|
||||
return scenario{}, redisstream.Envelope{}, "", err
|
||||
}
|
||||
before, err := e.objectCount(ctx, attachmentID)
|
||||
if err != nil {
|
||||
return scenario{}, redisstream.Envelope{}, "", err
|
||||
}
|
||||
if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil {
|
||||
return scenario{}, redisstream.Envelope{}, "", err
|
||||
}
|
||||
if err := e.ackReclaimed(ctx); err != nil {
|
||||
return scenario{}, redisstream.Envelope{}, "", err
|
||||
}
|
||||
after, err := e.objectCount(ctx, attachmentID)
|
||||
if err != nil {
|
||||
return scenario{}, redisstream.Envelope{}, "", err
|
||||
}
|
||||
return scenario{
|
||||
Name: string(worker.FaultAfterCommit), AttemptAfterRecovery: delivery.Envelope.Attempt,
|
||||
ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status,
|
||||
}, delivery.Envelope, attachmentID, nil
|
||||
}
|
||||
|
||||
func (e *experiment) queueAttachment(ctx context.Context) (string, model.AttachmentJob, error) {
|
||||
attachmentID, _, job, err := e.queueAttachmentData(ctx, e.fixture, "image/png")
|
||||
return attachmentID, job, err
|
||||
}
|
||||
|
||||
func (e *experiment) queueAttachmentData(
|
||||
ctx context.Context,
|
||||
data []byte,
|
||||
contentType string,
|
||||
) (string, string, model.AttachmentJob, error) {
|
||||
attachmentID := uuid.NewString()
|
||||
stagingKey := "staging/" + attachmentID + "/experiment"
|
||||
digest := sha256.Sum256(data)
|
||||
_, err := e.repository.CreatePendingAttachment(ctx, model.Attachment{
|
||||
ID: attachmentID, ChatID: e.chatID, OwnerID: e.ownerID, Filename: "experiment.png",
|
||||
ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]),
|
||||
ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending, PipelineVersion: 1,
|
||||
})
|
||||
if err != nil {
|
||||
return "", "", model.AttachmentJob{}, err
|
||||
}
|
||||
if err := e.objects.Put(ctx, stagingKey, contentType, data); err != nil {
|
||||
return "", "", model.AttachmentJob{}, err
|
||||
}
|
||||
job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 1, Attempt: 1}
|
||||
if _, err := e.repository.QueueAttachmentWithJobAndOutbox(ctx, attachmentID, stagingKey, job, "", ""); err != nil {
|
||||
return "", "", model.AttachmentJob{}, err
|
||||
}
|
||||
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
|
||||
return "", "", model.AttachmentJob{}, err
|
||||
}
|
||||
return attachmentID, stagingKey, job, nil
|
||||
}
|
||||
|
||||
func (e *experiment) readDelivery(ctx context.Context) (redisstream.Delivery, error) {
|
||||
deliveries, err := e.readDeliveries(ctx, 1)
|
||||
if err != nil {
|
||||
return redisstream.Delivery{}, err
|
||||
}
|
||||
return deliveries[0], nil
|
||||
}
|
||||
|
||||
func (e *experiment) readDeliveries(ctx context.Context, expected int) ([]redisstream.Delivery, error) {
|
||||
var result []redisstream.Delivery
|
||||
for len(result) < expected {
|
||||
deliveries, err := e.broker.Read(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, deliveries...)
|
||||
}
|
||||
if len(result) != expected {
|
||||
return nil, fmt.Errorf("deliveries = %d, expected %d", len(result), expected)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (e *experiment) ackReclaimed(ctx context.Context) error {
|
||||
deliveries, _, err := e.broker.Reclaim(ctx, "0-0")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(deliveries) != 1 {
|
||||
return fmt.Errorf("reclaimed deliveries = %d, expected 1", len(deliveries))
|
||||
}
|
||||
return e.worker.ProcessDelivery(ctx, deliveries[0], worker.FaultNone)
|
||||
}
|
||||
|
||||
func (e *experiment) objectCount(ctx context.Context, attachmentID string) (int, error) {
|
||||
objects, next, err := e.objects.List(ctx, "attachments/"+attachmentID+"/", "", 100)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if next != "" {
|
||||
return 0, errors.New("unexpected object pagination")
|
||||
}
|
||||
return len(objects), nil
|
||||
}
|
||||
|
||||
func pngFixture(width, height int) ([]byte, error) {
|
||||
value := image.NewNRGBA(image.Rect(0, 0, width, height))
|
||||
for y := range height {
|
||||
for x := range width {
|
||||
value.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255})
|
||||
}
|
||||
}
|
||||
var output bytes.Buffer
|
||||
if err := png.Encode(&output, value); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return output.Bytes(), nil
|
||||
}
|
||||
|
||||
func wait(ctx context.Context, duration time.Duration) error {
|
||||
timer := time.NewTimer(duration)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-timer.C:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user