Files

608 lines
19 KiB
Go

package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"flag"
"fmt"
"image"
"image/color"
"image/png"
"io"
"log/slog"
"os"
"strings"
"time"
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/broker/redisstream"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/config"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/database"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/transform"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/messenger/attachment/worker"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/model"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/objectstore"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/outbox"
"gitea.privcloud.tech/dizi/mtusi2026lab5/internal/postgres"
)
const (
experimentLease = 2 * time.Second
maxAttempts = 5
)
type result struct {
StartedAt string `json:"started_at"`
DurationMillis int64 `json:"duration_ms"`
PipelineVersion int `json:"pipeline_version"`
FaultScenarios []scenario `json:"fault_scenarios"`
DuplicateDeliveries int `json:"duplicate_deliveries"`
V1ObjectCount int `json:"v1_object_count"`
V2ObjectCount int `json:"v2_object_count"`
AcceptedJobs int `json:"accepted_jobs"`
TerminalJobs int `json:"terminal_jobs"`
ReadyJobs int `json:"ready_jobs"`
FailedJobs int `json:"failed_jobs"`
PermanentErrorCode string `json:"permanent_error_code"`
ReplayAttempt int `json:"replay_attempt"`
InvariantResults map[string]bool `json:"invariant_results"`
}
type scenario struct {
Name string `json:"name"`
AttemptAfterRecovery int `json:"attempt_after_recovery"`
ObjectsBeforeRecovery int `json:"objects_before_recovery"`
ObjectsAfterRecovery int `json:"objects_after_recovery"`
FinalStatus string `json:"final_status"`
}
type experiment struct {
settings config.WorkerConfig
repository *postgres.Store
objects *objectstore.Store
broker *redisstream.Broker
dispatcher *outbox.Dispatcher
worker *worker.Worker
ownerID string
chatID string
fixture []byte
}
func main() {
if err := run(os.Args[1:], os.Stdout); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func run(arguments []string, output io.Writer) error {
if len(arguments) > 0 {
switch arguments[0] {
case "api-produce":
return runAPIProduce(arguments[1:], output)
case "api-verify":
return runAPIVerify(arguments[1:], os.Stdin, output)
}
}
flags := flag.NewFlagSet("lab5experiment", flag.ContinueOnError)
flags.SetOutput(io.Discard)
timeout := flags.Duration("timeout", 2*time.Minute, "experiment timeout")
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 || *timeout <= 0 {
return errors.New("usage: lab5experiment [-timeout 2m]")
}
started := time.Now()
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
defer cancel()
experiment, closeExperiment, err := newExperiment(ctx)
if err != nil {
return err
}
defer closeExperiment()
report, err := experiment.run(ctx)
if err != nil {
return err
}
report.StartedAt = started.UTC().Format(time.RFC3339)
report.DurationMillis = time.Since(started).Milliseconds()
encoder := json.NewEncoder(output)
encoder.SetIndent("", " ")
return encoder.Encode(report)
}
func newExperiment(ctx context.Context) (*experiment, func(), error) {
settings, err := config.LoadWorker()
if err != nil {
return nil, nil, err
}
pool, err := database.Open(ctx, database.Config{
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
})
if err != nil {
return nil, nil, err
}
objects, err := objectstore.New(objectstore.Config{
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout,
})
if err != nil {
pool.Close()
return nil, nil, err
}
redisOptions, err := redis.ParseURL(settings.BrokerURL)
if err != nil {
pool.Close()
return nil, nil, err
}
client := redis.NewClient(redisOptions)
if err := client.Ping(ctx).Err(); err != nil {
pool.Close()
_ = client.Close()
return nil, nil, err
}
consumer := "lab5experiment-" + uuid.NewString()
broker, err := redisstream.New(client, redisstream.Config{
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer,
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: time.Second,
ClaimMinIdle: experimentLease, ReadCount: 32,
})
if err != nil {
pool.Close()
_ = client.Close()
return nil, nil, err
}
if err := broker.Ensure(ctx); err != nil {
pool.Close()
_ = client.Close()
return nil, nil, err
}
repository := postgres.New(pool)
dispatcher, err := outbox.New(repository, broker, nil, outbox.Config{
BatchSize: 32, LeaseDuration: 10 * time.Second, PollInterval: time.Second,
RecoveryInterval: 10 * time.Second, RetryDelay: time.Second,
Retention: 24 * time.Hour, CleanupInterval: time.Hour, CleanupBatchSize: 1000,
})
if err != nil {
pool.Close()
_ = client.Close()
return nil, nil, err
}
processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels))
if err != nil {
pool.Close()
_ = client.Close()
return nil, nil, err
}
processorWorker, err := worker.New(
slog.New(slog.NewTextHandler(io.Discard, nil)), repository, broker, objects, processor, nil,
worker.Config{
WorkerID: consumer, Concurrency: 1,
LeaseDuration: experimentLease, HeartbeatInterval: 500 * time.Millisecond,
ProcessingTimeout: time.Second,
ReclaimInterval: experimentLease,
MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second,
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
MemoryBytes: settings.MemoryBytes,
},
)
if err != nil {
pool.Close()
_ = client.Close()
return nil, nil, err
}
fixture, err := pngFixture(320, 200)
if err != nil {
pool.Close()
_ = client.Close()
return nil, nil, err
}
value := &experiment{
settings: settings, repository: repository, objects: objects, broker: broker,
dispatcher: dispatcher, worker: processorWorker, fixture: fixture,
}
if err := value.seed(ctx); err != nil {
pool.Close()
_ = client.Close()
return nil, nil, err
}
closeExperiment := func() {
pool.Close()
_ = client.Close()
}
return value, closeExperiment, nil
}
func (e *experiment) seed(ctx context.Context) error {
suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12]
owner, err := e.repository.CreateUser(ctx, "l5a_"+suffix, "Lab 5 A", "$argon2id$experiment")
if err != nil {
return err
}
member, err := e.repository.CreateUser(ctx, "l5b_"+suffix, "Lab 5 B", "$argon2id$experiment")
if err != nil {
return err
}
chat, err := e.repository.CreateChat(ctx, "Lab 5 experiment", []string{owner.ID, member.ID}, owner.ID)
if err != nil {
return err
}
e.ownerID = owner.ID
e.chatID = chat.ID
return nil
}
func (e *experiment) run(ctx context.Context) (result, error) {
report := result{
PipelineVersion: 1,
InvariantResults: map[string]bool{
"accepted_jobs_reach_terminal_state": false,
"duplicate_effect_is_unique": false,
"old_pipeline_cannot_replace_new": false,
},
}
for _, fault := range []worker.FaultPoint{worker.FaultBeforeSideEffect, worker.FaultAfterSideEffect} {
observed, err := e.runLeaseRecovery(ctx, fault)
if err != nil {
return result{}, err
}
report.FaultScenarios = append(report.FaultScenarios, observed)
report.AcceptedJobs++
if observed.FinalStatus == model.AttachmentReady {
report.ReadyJobs++
report.TerminalJobs++
}
}
permanentCode, replayAttempt, err := e.runPermanentFailureReplay(ctx)
if err != nil {
return result{}, err
}
report.AcceptedJobs++
report.TerminalJobs++
report.FailedJobs++
report.PermanentErrorCode = permanentCode
report.ReplayAttempt = replayAttempt
commitScenario, envelope, attachmentID, err := e.runAfterCommit(ctx)
if err != nil {
return result{}, err
}
report.FaultScenarios = append(report.FaultScenarios, commitScenario)
report.AcceptedJobs++
if commitScenario.FinalStatus == model.AttachmentReady {
report.ReadyJobs++
report.TerminalJobs++
}
payload, err := redisstream.EncodeEnvelope(envelope)
if err != nil {
return result{}, err
}
for range 2 {
if _, err := e.broker.PublishConfirmed(ctx, payload); err != nil {
return result{}, err
}
}
duplicateDeliveries, err := e.readDeliveries(ctx, 2)
if err != nil {
return result{}, err
}
for _, delivery := range duplicateDeliveries {
if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil {
return result{}, err
}
}
report.DuplicateDeliveries = len(duplicateDeliveries)
report.V1ObjectCount, err = e.objectCount(ctx, attachmentID)
if err != nil {
return result{}, err
}
if report.V1ObjectCount != 4 {
return result{}, fmt.Errorf("duplicate delivery produced %d objects, expected 4", report.V1ObjectCount)
}
report.InvariantResults["duplicate_effect_is_unique"] = true
oldPayload := payload
v2Job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: envelope.PipelineVersion + 1}
if _, err := e.repository.QueueAttachmentPipelineVersion(ctx, attachmentID, v2Job, "", ""); err != nil {
return result{}, err
}
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
return result{}, err
}
v2Delivery, err := e.readDelivery(ctx)
if err != nil {
return result{}, err
}
if err := e.worker.ProcessDelivery(ctx, v2Delivery, worker.FaultNone); err != nil {
return result{}, err
}
if _, err := e.broker.PublishConfirmed(ctx, oldPayload); err != nil {
return result{}, err
}
staleDelivery, err := e.readDelivery(ctx)
if err != nil {
return result{}, err
}
if err := e.worker.ProcessDelivery(ctx, staleDelivery, worker.FaultNone); err != nil {
return result{}, err
}
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
if err != nil {
return result{}, err
}
if metadata.Status != model.AttachmentReady || metadata.PipelineVersion != v2Job.PipelineVersion || len(metadata.Variants) != 3 {
return result{}, errors.New("new pipeline version was not preserved after stale delivery")
}
report.V2ObjectCount, err = e.objectCount(ctx, attachmentID)
if err != nil {
return result{}, err
}
if report.V2ObjectCount != 8 {
return result{}, fmt.Errorf("versioned pipeline produced %d objects, expected 8", report.V2ObjectCount)
}
report.InvariantResults["old_pipeline_cannot_replace_new"] = true
report.AcceptedJobs++
report.ReadyJobs++
report.TerminalJobs++
report.InvariantResults["accepted_jobs_reach_terminal_state"] = report.AcceptedJobs == report.TerminalJobs
return report, nil
}
func (e *experiment) runPermanentFailureReplay(ctx context.Context) (string, int, error) {
attachmentID, _, _, err := e.queueAttachmentData(ctx, []byte("corrupted image"), "image/png")
if err != nil {
return "", 0, err
}
delivery, err := e.readDelivery(ctx)
if err != nil {
return "", 0, err
}
if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil {
return "", 0, err
}
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
if err != nil {
return "", 0, err
}
if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil {
return "", 0, errors.New("corrupted staging object did not reach failed")
}
firstCode := *metadata.ErrorCode
replayed, err := e.repository.ReplayFailedAttachment(
ctx, attachmentID, "lab5experiment", "verify explicit replay fencing",
)
if err != nil {
return "", 0, err
}
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
return "", 0, err
}
replayDelivery, err := e.readDelivery(ctx)
if err != nil {
return "", 0, err
}
if err := e.worker.ProcessDelivery(ctx, replayDelivery, worker.FaultNone); err != nil {
return "", 0, err
}
metadata, err = e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
if err != nil {
return "", 0, err
}
if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil || *metadata.ErrorCode != firstCode {
return "", 0, errors.New("replayed permanent failure changed terminal classification")
}
return firstCode, replayed.Attempt, nil
}
func (e *experiment) runLeaseRecovery(ctx context.Context, fault worker.FaultPoint) (scenario, error) {
attachmentID, _, err := e.queueAttachment(ctx)
if err != nil {
return scenario{}, err
}
delivery, err := e.readDelivery(ctx)
if err != nil {
return scenario{}, err
}
if err := e.worker.ProcessDelivery(ctx, delivery, fault); !errors.Is(err, worker.ErrInjectedFailure) {
return scenario{}, fmt.Errorf("fault %s returned %v", fault, err)
}
before, err := e.objectCount(ctx, attachmentID)
if err != nil {
return scenario{}, err
}
if fault == worker.FaultBeforeSideEffect && before != 0 {
return scenario{}, errors.New("objects were written before the injected side effect")
}
if fault == worker.FaultAfterSideEffect && before != 4 {
return scenario{}, fmt.Errorf("side effect wrote %d objects, expected 4", before)
}
if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil {
return scenario{}, err
}
recovered, err := e.repository.RecoverExpiredAttachmentJobs(ctx, 10, maxAttempts, time.Now())
if err != nil {
return scenario{}, err
}
if recovered.Requeued != 1 {
return scenario{}, fmt.Errorf("requeued jobs = %d, expected 1", recovered.Requeued)
}
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
return scenario{}, err
}
retry, err := e.readDelivery(ctx)
if err != nil {
return scenario{}, err
}
if err := e.worker.ProcessDelivery(ctx, retry, worker.FaultNone); err != nil {
return scenario{}, err
}
if err := e.ackReclaimed(ctx); err != nil {
return scenario{}, err
}
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
if err != nil {
return scenario{}, err
}
after, err := e.objectCount(ctx, attachmentID)
if err != nil {
return scenario{}, err
}
job, err := e.repository.AttachmentJob(ctx, retry.Envelope.JobID.String())
if err != nil {
return scenario{}, err
}
return scenario{
Name: string(fault), AttemptAfterRecovery: job.Attempt,
ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status,
}, nil
}
func (e *experiment) runAfterCommit(ctx context.Context) (scenario, redisstream.Envelope, string, error) {
attachmentID, _, err := e.queueAttachment(ctx)
if err != nil {
return scenario{}, redisstream.Envelope{}, "", err
}
delivery, err := e.readDelivery(ctx)
if err != nil {
return scenario{}, redisstream.Envelope{}, "", err
}
if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultAfterCommit); !errors.Is(err, worker.ErrInjectedFailure) {
return scenario{}, redisstream.Envelope{}, "", fmt.Errorf("after_commit returned %v", err)
}
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
if err != nil {
return scenario{}, redisstream.Envelope{}, "", err
}
before, err := e.objectCount(ctx, attachmentID)
if err != nil {
return scenario{}, redisstream.Envelope{}, "", err
}
if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil {
return scenario{}, redisstream.Envelope{}, "", err
}
if err := e.ackReclaimed(ctx); err != nil {
return scenario{}, redisstream.Envelope{}, "", err
}
after, err := e.objectCount(ctx, attachmentID)
if err != nil {
return scenario{}, redisstream.Envelope{}, "", err
}
return scenario{
Name: string(worker.FaultAfterCommit), AttemptAfterRecovery: delivery.Envelope.Attempt,
ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status,
}, delivery.Envelope, attachmentID, nil
}
func (e *experiment) queueAttachment(ctx context.Context) (string, model.AttachmentJob, error) {
attachmentID, _, job, err := e.queueAttachmentData(ctx, e.fixture, "image/png")
return attachmentID, job, err
}
func (e *experiment) queueAttachmentData(
ctx context.Context,
data []byte,
contentType string,
) (string, string, model.AttachmentJob, error) {
attachmentID := uuid.NewString()
stagingKey := "staging/" + attachmentID + "/experiment"
digest := sha256.Sum256(data)
_, err := e.repository.CreatePendingAttachment(ctx, model.Attachment{
ID: attachmentID, ChatID: e.chatID, OwnerID: e.ownerID, Filename: "experiment.png",
ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]),
ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending, PipelineVersion: 1,
})
if err != nil {
return "", "", model.AttachmentJob{}, err
}
if err := e.objects.Put(ctx, stagingKey, contentType, data); err != nil {
return "", "", model.AttachmentJob{}, err
}
job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 1, Attempt: 1}
if _, err := e.repository.QueueAttachmentWithJobAndOutbox(ctx, attachmentID, stagingKey, job, "", ""); err != nil {
return "", "", model.AttachmentJob{}, err
}
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
return "", "", model.AttachmentJob{}, err
}
return attachmentID, stagingKey, job, nil
}
func (e *experiment) readDelivery(ctx context.Context) (redisstream.Delivery, error) {
deliveries, err := e.readDeliveries(ctx, 1)
if err != nil {
return redisstream.Delivery{}, err
}
return deliveries[0], nil
}
func (e *experiment) readDeliveries(ctx context.Context, expected int) ([]redisstream.Delivery, error) {
var result []redisstream.Delivery
for len(result) < expected {
deliveries, err := e.broker.Read(ctx)
if err != nil {
return nil, err
}
result = append(result, deliveries...)
}
if len(result) != expected {
return nil, fmt.Errorf("deliveries = %d, expected %d", len(result), expected)
}
return result, nil
}
func (e *experiment) ackReclaimed(ctx context.Context) error {
deliveries, _, err := e.broker.Reclaim(ctx, "0-0")
if err != nil {
return err
}
if len(deliveries) != 1 {
return fmt.Errorf("reclaimed deliveries = %d, expected 1", len(deliveries))
}
return e.worker.ProcessDelivery(ctx, deliveries[0], worker.FaultNone)
}
func (e *experiment) objectCount(ctx context.Context, attachmentID string) (int, error) {
objects, next, err := e.objects.List(ctx, "attachments/"+attachmentID+"/", "", 100)
if err != nil {
return 0, err
}
if next != "" {
return 0, errors.New("unexpected object pagination")
}
return len(objects), nil
}
func pngFixture(width, height int) ([]byte, error) {
value := image.NewNRGBA(image.Rect(0, 0, width, height))
for y := range height {
for x := range width {
value.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255})
}
}
var output bytes.Buffer
if err := png.Encode(&output, value); err != nil {
return nil, err
}
return output.Bytes(), nil
}
func wait(ctx context.Context, duration time.Duration) error {
timer := time.NewTimer(duration)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}