package main import ( "bytes" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "flag" "fmt" "image" "image/color" "image/png" "io" "log/slog" "os" "strings" "time" "github.com/google/uuid" "github.com/redis/go-redis/v9" "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/broker/redisstream" "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/config" "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/database" "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/transform" "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/messenger/attachment/worker" "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/model" "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/objectstore" "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/outbox" "gitea.privcloud.tech/dizi/mtusi2026lab6/internal/postgres" ) const ( experimentLease = 2 * time.Second maxAttempts = 5 ) type result struct { StartedAt string `json:"started_at"` DurationMillis int64 `json:"duration_ms"` PipelineVersion int `json:"pipeline_version"` FaultScenarios []scenario `json:"fault_scenarios"` DuplicateDeliveries int `json:"duplicate_deliveries"` V1ObjectCount int `json:"v1_object_count"` V2ObjectCount int `json:"v2_object_count"` AcceptedJobs int `json:"accepted_jobs"` TerminalJobs int `json:"terminal_jobs"` ReadyJobs int `json:"ready_jobs"` FailedJobs int `json:"failed_jobs"` PermanentErrorCode string `json:"permanent_error_code"` ReplayAttempt int `json:"replay_attempt"` InvariantResults map[string]bool `json:"invariant_results"` } type scenario struct { Name string `json:"name"` AttemptAfterRecovery int `json:"attempt_after_recovery"` ObjectsBeforeRecovery int `json:"objects_before_recovery"` ObjectsAfterRecovery int `json:"objects_after_recovery"` FinalStatus string `json:"final_status"` } type experiment struct { settings config.WorkerConfig repository *postgres.Store objects *objectstore.Store broker *redisstream.Broker dispatcher *outbox.Dispatcher worker *worker.Worker ownerID string chatID string fixture []byte } func main() { if err := run(os.Args[1:], os.Stdout); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } } func run(arguments []string, output io.Writer) error { if len(arguments) > 0 { switch arguments[0] { case "api-produce": return runAPIProduce(arguments[1:], output) case "api-verify": return runAPIVerify(arguments[1:], os.Stdin, output) } } flags := flag.NewFlagSet("lab5experiment", flag.ContinueOnError) flags.SetOutput(io.Discard) timeout := flags.Duration("timeout", 2*time.Minute, "experiment timeout") if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 || *timeout <= 0 { return errors.New("usage: lab5experiment [-timeout 2m]") } started := time.Now() ctx, cancel := context.WithTimeout(context.Background(), *timeout) defer cancel() experiment, closeExperiment, err := newExperiment(ctx) if err != nil { return err } defer closeExperiment() report, err := experiment.run(ctx) if err != nil { return err } report.StartedAt = started.UTC().Format(time.RFC3339) report.DurationMillis = time.Since(started).Milliseconds() encoder := json.NewEncoder(output) encoder.SetIndent("", " ") return encoder.Encode(report) } func newExperiment(ctx context.Context) (*experiment, func(), error) { settings, err := config.LoadWorker() if err != nil { return nil, nil, err } pool, err := database.Open(ctx, database.Config{ URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns, MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle, HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout, }) if err != nil { return nil, nil, err } objects, err := objectstore.New(objectstore.Config{ Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket, AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey, UseTLS: settings.S3UseTLS, Timeout: settings.S3Timeout, }) if err != nil { pool.Close() return nil, nil, err } redisOptions, err := redis.ParseURL(settings.BrokerURL) if err != nil { pool.Close() return nil, nil, err } client := redis.NewClient(redisOptions) if err := client.Ping(ctx).Err(); err != nil { pool.Close() _ = client.Close() return nil, nil, err } consumer := "lab5experiment-" + uuid.NewString() broker, err := redisstream.New(client, redisstream.Config{ Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer, FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: time.Second, ClaimMinIdle: experimentLease, ReadCount: 32, }) if err != nil { pool.Close() _ = client.Close() return nil, nil, err } if err := broker.Ensure(ctx); err != nil { pool.Close() _ = client.Close() return nil, nil, err } repository := postgres.New(pool) dispatcher, err := outbox.New(repository, broker, nil, outbox.Config{ BatchSize: 32, LeaseDuration: 10 * time.Second, PollInterval: time.Second, RecoveryInterval: 10 * time.Second, RetryDelay: time.Second, Retention: 24 * time.Hour, CleanupInterval: time.Hour, CleanupBatchSize: 1000, }) if err != nil { pool.Close() _ = client.Close() return nil, nil, err } processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels)) if err != nil { pool.Close() _ = client.Close() return nil, nil, err } processorWorker, err := worker.New( slog.New(slog.NewTextHandler(io.Discard, nil)), repository, broker, objects, processor, nil, worker.Config{ WorkerID: consumer, Concurrency: 1, LeaseDuration: experimentLease, HeartbeatInterval: 500 * time.Millisecond, ProcessingTimeout: time.Second, ReclaimInterval: experimentLease, MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second, MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels, MemoryBytes: settings.MemoryBytes, }, ) if err != nil { pool.Close() _ = client.Close() return nil, nil, err } fixture, err := pngFixture(320, 200) if err != nil { pool.Close() _ = client.Close() return nil, nil, err } value := &experiment{ settings: settings, repository: repository, objects: objects, broker: broker, dispatcher: dispatcher, worker: processorWorker, fixture: fixture, } if err := value.seed(ctx); err != nil { pool.Close() _ = client.Close() return nil, nil, err } closeExperiment := func() { pool.Close() _ = client.Close() } return value, closeExperiment, nil } func (e *experiment) seed(ctx context.Context) error { suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12] owner, err := e.repository.CreateUser(ctx, "l5a_"+suffix, "Lab 5 A", "$argon2id$experiment") if err != nil { return err } member, err := e.repository.CreateUser(ctx, "l5b_"+suffix, "Lab 5 B", "$argon2id$experiment") if err != nil { return err } chat, err := e.repository.CreateChat(ctx, "Lab 5 experiment", []string{owner.ID, member.ID}, owner.ID) if err != nil { return err } e.ownerID = owner.ID e.chatID = chat.ID return nil } func (e *experiment) run(ctx context.Context) (result, error) { report := result{ PipelineVersion: 1, InvariantResults: map[string]bool{ "accepted_jobs_reach_terminal_state": false, "duplicate_effect_is_unique": false, "old_pipeline_cannot_replace_new": false, }, } for _, fault := range []worker.FaultPoint{worker.FaultBeforeSideEffect, worker.FaultAfterSideEffect} { observed, err := e.runLeaseRecovery(ctx, fault) if err != nil { return result{}, err } report.FaultScenarios = append(report.FaultScenarios, observed) report.AcceptedJobs++ if observed.FinalStatus == model.AttachmentReady { report.ReadyJobs++ report.TerminalJobs++ } } permanentCode, replayAttempt, err := e.runPermanentFailureReplay(ctx) if err != nil { return result{}, err } report.AcceptedJobs++ report.TerminalJobs++ report.FailedJobs++ report.PermanentErrorCode = permanentCode report.ReplayAttempt = replayAttempt commitScenario, envelope, attachmentID, err := e.runAfterCommit(ctx) if err != nil { return result{}, err } report.FaultScenarios = append(report.FaultScenarios, commitScenario) report.AcceptedJobs++ if commitScenario.FinalStatus == model.AttachmentReady { report.ReadyJobs++ report.TerminalJobs++ } payload, err := redisstream.EncodeEnvelope(envelope) if err != nil { return result{}, err } for range 2 { if _, err := e.broker.PublishConfirmed(ctx, payload); err != nil { return result{}, err } } duplicateDeliveries, err := e.readDeliveries(ctx, 2) if err != nil { return result{}, err } for _, delivery := range duplicateDeliveries { if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil { return result{}, err } } report.DuplicateDeliveries = len(duplicateDeliveries) report.V1ObjectCount, err = e.objectCount(ctx, attachmentID) if err != nil { return result{}, err } if report.V1ObjectCount != 4 { return result{}, fmt.Errorf("duplicate delivery produced %d objects, expected 4", report.V1ObjectCount) } report.InvariantResults["duplicate_effect_is_unique"] = true oldPayload := payload v2Job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: envelope.PipelineVersion + 1} if _, err := e.repository.QueueAttachmentPipelineVersion(ctx, attachmentID, v2Job, "", ""); err != nil { return result{}, err } if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { return result{}, err } v2Delivery, err := e.readDelivery(ctx) if err != nil { return result{}, err } if err := e.worker.ProcessDelivery(ctx, v2Delivery, worker.FaultNone); err != nil { return result{}, err } if _, err := e.broker.PublishConfirmed(ctx, oldPayload); err != nil { return result{}, err } staleDelivery, err := e.readDelivery(ctx) if err != nil { return result{}, err } if err := e.worker.ProcessDelivery(ctx, staleDelivery, worker.FaultNone); err != nil { return result{}, err } metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) if err != nil { return result{}, err } if metadata.Status != model.AttachmentReady || metadata.PipelineVersion != v2Job.PipelineVersion || len(metadata.Variants) != 3 { return result{}, errors.New("new pipeline version was not preserved after stale delivery") } report.V2ObjectCount, err = e.objectCount(ctx, attachmentID) if err != nil { return result{}, err } if report.V2ObjectCount != 8 { return result{}, fmt.Errorf("versioned pipeline produced %d objects, expected 8", report.V2ObjectCount) } report.InvariantResults["old_pipeline_cannot_replace_new"] = true report.AcceptedJobs++ report.ReadyJobs++ report.TerminalJobs++ report.InvariantResults["accepted_jobs_reach_terminal_state"] = report.AcceptedJobs == report.TerminalJobs return report, nil } func (e *experiment) runPermanentFailureReplay(ctx context.Context) (string, int, error) { attachmentID, _, _, err := e.queueAttachmentData(ctx, []byte("corrupted image"), "image/png") if err != nil { return "", 0, err } delivery, err := e.readDelivery(ctx) if err != nil { return "", 0, err } if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil { return "", 0, err } metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) if err != nil { return "", 0, err } if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil { return "", 0, errors.New("corrupted staging object did not reach failed") } firstCode := *metadata.ErrorCode replayed, err := e.repository.ReplayFailedAttachment( ctx, attachmentID, "lab5experiment", "verify explicit replay fencing", ) if err != nil { return "", 0, err } if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { return "", 0, err } replayDelivery, err := e.readDelivery(ctx) if err != nil { return "", 0, err } if err := e.worker.ProcessDelivery(ctx, replayDelivery, worker.FaultNone); err != nil { return "", 0, err } metadata, err = e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) if err != nil { return "", 0, err } if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil || *metadata.ErrorCode != firstCode { return "", 0, errors.New("replayed permanent failure changed terminal classification") } return firstCode, replayed.Attempt, nil } func (e *experiment) runLeaseRecovery(ctx context.Context, fault worker.FaultPoint) (scenario, error) { attachmentID, _, err := e.queueAttachment(ctx) if err != nil { return scenario{}, err } delivery, err := e.readDelivery(ctx) if err != nil { return scenario{}, err } if err := e.worker.ProcessDelivery(ctx, delivery, fault); !errors.Is(err, worker.ErrInjectedFailure) { return scenario{}, fmt.Errorf("fault %s returned %v", fault, err) } before, err := e.objectCount(ctx, attachmentID) if err != nil { return scenario{}, err } if fault == worker.FaultBeforeSideEffect && before != 0 { return scenario{}, errors.New("objects were written before the injected side effect") } if fault == worker.FaultAfterSideEffect && before != 4 { return scenario{}, fmt.Errorf("side effect wrote %d objects, expected 4", before) } if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil { return scenario{}, err } recovered, err := e.repository.RecoverExpiredAttachmentJobs(ctx, 10, maxAttempts, time.Now()) if err != nil { return scenario{}, err } if recovered.Requeued != 1 { return scenario{}, fmt.Errorf("requeued jobs = %d, expected 1", recovered.Requeued) } if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { return scenario{}, err } retry, err := e.readDelivery(ctx) if err != nil { return scenario{}, err } if err := e.worker.ProcessDelivery(ctx, retry, worker.FaultNone); err != nil { return scenario{}, err } if err := e.ackReclaimed(ctx); err != nil { return scenario{}, err } metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) if err != nil { return scenario{}, err } after, err := e.objectCount(ctx, attachmentID) if err != nil { return scenario{}, err } job, err := e.repository.AttachmentJob(ctx, retry.Envelope.JobID.String()) if err != nil { return scenario{}, err } return scenario{ Name: string(fault), AttemptAfterRecovery: job.Attempt, ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status, }, nil } func (e *experiment) runAfterCommit(ctx context.Context) (scenario, redisstream.Envelope, string, error) { attachmentID, _, err := e.queueAttachment(ctx) if err != nil { return scenario{}, redisstream.Envelope{}, "", err } delivery, err := e.readDelivery(ctx) if err != nil { return scenario{}, redisstream.Envelope{}, "", err } if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultAfterCommit); !errors.Is(err, worker.ErrInjectedFailure) { return scenario{}, redisstream.Envelope{}, "", fmt.Errorf("after_commit returned %v", err) } metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID) if err != nil { return scenario{}, redisstream.Envelope{}, "", err } before, err := e.objectCount(ctx, attachmentID) if err != nil { return scenario{}, redisstream.Envelope{}, "", err } if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil { return scenario{}, redisstream.Envelope{}, "", err } if err := e.ackReclaimed(ctx); err != nil { return scenario{}, redisstream.Envelope{}, "", err } after, err := e.objectCount(ctx, attachmentID) if err != nil { return scenario{}, redisstream.Envelope{}, "", err } return scenario{ Name: string(worker.FaultAfterCommit), AttemptAfterRecovery: delivery.Envelope.Attempt, ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status, }, delivery.Envelope, attachmentID, nil } func (e *experiment) queueAttachment(ctx context.Context) (string, model.AttachmentJob, error) { attachmentID, _, job, err := e.queueAttachmentData(ctx, e.fixture, "image/png") return attachmentID, job, err } func (e *experiment) queueAttachmentData( ctx context.Context, data []byte, contentType string, ) (string, string, model.AttachmentJob, error) { attachmentID := uuid.NewString() stagingKey := "staging/" + attachmentID + "/experiment" digest := sha256.Sum256(data) _, err := e.repository.CreatePendingAttachment(ctx, model.Attachment{ ID: attachmentID, ChatID: e.chatID, OwnerID: e.ownerID, Filename: "experiment.png", ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending, PipelineVersion: 1, }) if err != nil { return "", "", model.AttachmentJob{}, err } if err := e.objects.Put(ctx, stagingKey, contentType, data); err != nil { return "", "", model.AttachmentJob{}, err } job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 1, Attempt: 1} if _, err := e.repository.QueueAttachmentWithJobAndOutbox(ctx, attachmentID, stagingKey, job, "", ""); err != nil { return "", "", model.AttachmentJob{}, err } if _, err := e.dispatcher.DispatchOnce(ctx); err != nil { return "", "", model.AttachmentJob{}, err } return attachmentID, stagingKey, job, nil } func (e *experiment) readDelivery(ctx context.Context) (redisstream.Delivery, error) { deliveries, err := e.readDeliveries(ctx, 1) if err != nil { return redisstream.Delivery{}, err } return deliveries[0], nil } func (e *experiment) readDeliveries(ctx context.Context, expected int) ([]redisstream.Delivery, error) { var result []redisstream.Delivery for len(result) < expected { deliveries, err := e.broker.Read(ctx) if err != nil { return nil, err } result = append(result, deliveries...) } if len(result) != expected { return nil, fmt.Errorf("deliveries = %d, expected %d", len(result), expected) } return result, nil } func (e *experiment) ackReclaimed(ctx context.Context) error { deliveries, _, err := e.broker.Reclaim(ctx, "0-0") if err != nil { return err } if len(deliveries) != 1 { return fmt.Errorf("reclaimed deliveries = %d, expected 1", len(deliveries)) } return e.worker.ProcessDelivery(ctx, deliveries[0], worker.FaultNone) } func (e *experiment) objectCount(ctx context.Context, attachmentID string) (int, error) { objects, next, err := e.objects.List(ctx, "attachments/"+attachmentID+"/", "", 100) if err != nil { return 0, err } if next != "" { return 0, errors.New("unexpected object pagination") } return len(objects), nil } func pngFixture(width, height int) ([]byte, error) { value := image.NewNRGBA(image.Rect(0, 0, width, height)) for y := range height { for x := range width { value.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255}) } } var output bytes.Buffer if err := png.Encode(&output, value); err != nil { return nil, err } return output.Bytes(), nil } func wait(ctx context.Context, duration time.Duration) error { timer := time.NewTimer(duration) defer timer.Stop() select { case <-ctx.Done(): return ctx.Err() case <-timer.C: return nil } }