forked from mtusi2026/mtusi2026lab7
618 lines
20 KiB
Go
618 lines
20 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"flag"
|
|
"fmt"
|
|
"image"
|
|
"image/color"
|
|
"image/png"
|
|
"io"
|
|
"log/slog"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/redis/go-redis/v9"
|
|
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/broker/redisstream"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/config"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/database"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment/transform"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/messenger/attachment/worker"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/model"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/objectstore"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/outbox"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/postgres"
|
|
"gitea.privcloud.tech/dizi/mtusi2026lab7/internal/security/tlsconfig"
|
|
)
|
|
|
|
const (
|
|
experimentLease = 2 * time.Second
|
|
maxAttempts = 5
|
|
)
|
|
|
|
type result struct {
|
|
StartedAt string `json:"started_at"`
|
|
DurationMillis int64 `json:"duration_ms"`
|
|
PipelineVersion int `json:"pipeline_version"`
|
|
FaultScenarios []scenario `json:"fault_scenarios"`
|
|
DuplicateDeliveries int `json:"duplicate_deliveries"`
|
|
V1ObjectCount int `json:"v1_object_count"`
|
|
V2ObjectCount int `json:"v2_object_count"`
|
|
AcceptedJobs int `json:"accepted_jobs"`
|
|
TerminalJobs int `json:"terminal_jobs"`
|
|
ReadyJobs int `json:"ready_jobs"`
|
|
FailedJobs int `json:"failed_jobs"`
|
|
PermanentErrorCode string `json:"permanent_error_code"`
|
|
ReplayAttempt int `json:"replay_attempt"`
|
|
InvariantResults map[string]bool `json:"invariant_results"`
|
|
}
|
|
|
|
type scenario struct {
|
|
Name string `json:"name"`
|
|
AttemptAfterRecovery int `json:"attempt_after_recovery"`
|
|
ObjectsBeforeRecovery int `json:"objects_before_recovery"`
|
|
ObjectsAfterRecovery int `json:"objects_after_recovery"`
|
|
FinalStatus string `json:"final_status"`
|
|
}
|
|
|
|
type experiment struct {
|
|
settings config.WorkerConfig
|
|
repository *postgres.Store
|
|
objects *objectstore.Store
|
|
broker *redisstream.Broker
|
|
dispatcher *outbox.Dispatcher
|
|
worker *worker.Worker
|
|
ownerID string
|
|
chatID string
|
|
fixture []byte
|
|
}
|
|
|
|
func main() {
|
|
if err := run(os.Args[1:], os.Stdout); err != nil {
|
|
fmt.Fprintln(os.Stderr, err)
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
func run(arguments []string, output io.Writer) error {
|
|
if len(arguments) > 0 {
|
|
switch arguments[0] {
|
|
case "api-produce":
|
|
return runAPIProduce(arguments[1:], output)
|
|
case "api-verify":
|
|
return runAPIVerify(arguments[1:], os.Stdin, output)
|
|
}
|
|
}
|
|
flags := flag.NewFlagSet("lab5experiment", flag.ContinueOnError)
|
|
flags.SetOutput(io.Discard)
|
|
timeout := flags.Duration("timeout", 2*time.Minute, "experiment timeout")
|
|
if err := flags.Parse(arguments); err != nil || flags.NArg() != 0 || *timeout <= 0 {
|
|
return errors.New("usage: lab5experiment [-timeout 2m]")
|
|
}
|
|
started := time.Now()
|
|
ctx, cancel := context.WithTimeout(context.Background(), *timeout)
|
|
defer cancel()
|
|
experiment, closeExperiment, err := newExperiment(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer closeExperiment()
|
|
|
|
report, err := experiment.run(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
report.StartedAt = started.UTC().Format(time.RFC3339)
|
|
report.DurationMillis = time.Since(started).Milliseconds()
|
|
encoder := json.NewEncoder(output)
|
|
encoder.SetIndent("", " ")
|
|
return encoder.Encode(report)
|
|
}
|
|
|
|
func newExperiment(ctx context.Context) (*experiment, func(), error) {
|
|
settings, err := config.LoadWorker()
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
pool, err := database.Open(ctx, database.Config{
|
|
URL: settings.DatabaseURL, MinConnections: settings.DBMinConns, MaxConnections: settings.DBMaxConns,
|
|
MaxConnectionAge: settings.DBMaxConnectionAge, MaxConnectionIdle: settings.DBMaxConnectionIdle,
|
|
HealthCheckPeriod: settings.DBHealthCheckPeriod, ConnectTimeout: settings.DependencyTimeout,
|
|
})
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
objects, err := objectstore.New(objectstore.Config{
|
|
Endpoint: settings.S3Endpoint, Region: settings.S3Region, Bucket: settings.S3Bucket,
|
|
AccessKey: settings.S3AccessKey, SecretKey: settings.S3SecretKey,
|
|
UseTLS: settings.S3UseTLS, CAPath: settings.S3CAPath, Timeout: settings.S3Timeout,
|
|
})
|
|
if err != nil {
|
|
pool.Close()
|
|
return nil, nil, err
|
|
}
|
|
redisOptions, err := redis.ParseURL(settings.BrokerURL)
|
|
if err != nil {
|
|
pool.Close()
|
|
return nil, nil, err
|
|
}
|
|
brokerTLS, err := tlsconfig.LoadClient(tlsconfig.ClientConfig{
|
|
CAPath: settings.BrokerTLS.CAPath, CertificatePath: settings.BrokerTLS.CertificatePath,
|
|
PrivateKeyPath: settings.BrokerTLS.PrivateKeyPath, ServerName: settings.BrokerTLS.ServerName,
|
|
})
|
|
if err != nil {
|
|
pool.Close()
|
|
return nil, nil, err
|
|
}
|
|
redisOptions.TLSConfig = brokerTLS
|
|
client := redis.NewClient(redisOptions)
|
|
if err := client.Ping(ctx).Err(); err != nil {
|
|
pool.Close()
|
|
_ = client.Close()
|
|
return nil, nil, err
|
|
}
|
|
consumer := "lab5experiment-" + uuid.NewString()
|
|
broker, err := redisstream.New(client, redisstream.Config{
|
|
Stream: settings.Stream, Group: settings.ConsumerGroup, Consumer: consumer,
|
|
FailedStream: settings.FailedStream, FailedMaxLen: settings.FailedStreamMaxLength, Block: time.Second,
|
|
ClaimMinIdle: experimentLease, ReadCount: 32,
|
|
})
|
|
if err != nil {
|
|
pool.Close()
|
|
_ = client.Close()
|
|
return nil, nil, err
|
|
}
|
|
if err := broker.Ensure(ctx); err != nil {
|
|
pool.Close()
|
|
_ = client.Close()
|
|
return nil, nil, err
|
|
}
|
|
repository := postgres.New(pool)
|
|
dispatcher, err := outbox.New(repository, broker, nil, outbox.Config{
|
|
BatchSize: 32, LeaseDuration: 10 * time.Second, PollInterval: time.Second,
|
|
RecoveryInterval: 10 * time.Second, RetryDelay: time.Second,
|
|
Retention: 24 * time.Hour, CleanupInterval: time.Hour, CleanupBatchSize: 1000,
|
|
})
|
|
if err != nil {
|
|
pool.Close()
|
|
_ = client.Close()
|
|
return nil, nil, err
|
|
}
|
|
processor, err := transform.NewProcessor(transform.DefaultConfig(settings.AttachmentMaxBytes, settings.AttachmentMaxPixels))
|
|
if err != nil {
|
|
pool.Close()
|
|
_ = client.Close()
|
|
return nil, nil, err
|
|
}
|
|
processorWorker, err := worker.New(
|
|
slog.New(slog.NewTextHandler(io.Discard, nil)), repository, broker, objects, processor, nil,
|
|
worker.Config{
|
|
WorkerID: consumer, Concurrency: 1,
|
|
LeaseDuration: experimentLease, HeartbeatInterval: 500 * time.Millisecond,
|
|
ProcessingTimeout: time.Second,
|
|
ReclaimInterval: experimentLease,
|
|
MaxAttempts: maxAttempts, RetryBase: time.Second, RetryMax: 4 * time.Second,
|
|
MaxBytes: settings.AttachmentMaxBytes, MaxPixels: settings.AttachmentMaxPixels,
|
|
MemoryBytes: settings.MemoryBytes,
|
|
},
|
|
)
|
|
if err != nil {
|
|
pool.Close()
|
|
_ = client.Close()
|
|
return nil, nil, err
|
|
}
|
|
fixture, err := pngFixture(320, 200)
|
|
if err != nil {
|
|
pool.Close()
|
|
_ = client.Close()
|
|
return nil, nil, err
|
|
}
|
|
value := &experiment{
|
|
settings: settings, repository: repository, objects: objects, broker: broker,
|
|
dispatcher: dispatcher, worker: processorWorker, fixture: fixture,
|
|
}
|
|
if err := value.seed(ctx); err != nil {
|
|
pool.Close()
|
|
_ = client.Close()
|
|
return nil, nil, err
|
|
}
|
|
closeExperiment := func() {
|
|
pool.Close()
|
|
_ = client.Close()
|
|
}
|
|
return value, closeExperiment, nil
|
|
}
|
|
|
|
func (e *experiment) seed(ctx context.Context) error {
|
|
suffix := strings.ReplaceAll(uuid.NewString(), "-", "")[:12]
|
|
owner, err := e.repository.CreateUser(ctx, "l5a_"+suffix, "Lab 5 A", "$argon2id$experiment")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
member, err := e.repository.CreateUser(ctx, "l5b_"+suffix, "Lab 5 B", "$argon2id$experiment")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
chat, err := e.repository.CreateChat(ctx, "Lab 5 experiment", []string{owner.ID, member.ID}, owner.ID)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
e.ownerID = owner.ID
|
|
e.chatID = chat.ID
|
|
return nil
|
|
}
|
|
|
|
func (e *experiment) run(ctx context.Context) (result, error) {
|
|
report := result{
|
|
PipelineVersion: 1,
|
|
InvariantResults: map[string]bool{
|
|
"accepted_jobs_reach_terminal_state": false,
|
|
"duplicate_effect_is_unique": false,
|
|
"old_pipeline_cannot_replace_new": false,
|
|
},
|
|
}
|
|
for _, fault := range []worker.FaultPoint{worker.FaultBeforeSideEffect, worker.FaultAfterSideEffect} {
|
|
observed, err := e.runLeaseRecovery(ctx, fault)
|
|
if err != nil {
|
|
return result{}, err
|
|
}
|
|
report.FaultScenarios = append(report.FaultScenarios, observed)
|
|
report.AcceptedJobs++
|
|
if observed.FinalStatus == model.AttachmentReady {
|
|
report.ReadyJobs++
|
|
report.TerminalJobs++
|
|
}
|
|
}
|
|
permanentCode, replayAttempt, err := e.runPermanentFailureReplay(ctx)
|
|
if err != nil {
|
|
return result{}, err
|
|
}
|
|
report.AcceptedJobs++
|
|
report.TerminalJobs++
|
|
report.FailedJobs++
|
|
report.PermanentErrorCode = permanentCode
|
|
report.ReplayAttempt = replayAttempt
|
|
|
|
commitScenario, envelope, attachmentID, err := e.runAfterCommit(ctx)
|
|
if err != nil {
|
|
return result{}, err
|
|
}
|
|
report.FaultScenarios = append(report.FaultScenarios, commitScenario)
|
|
report.AcceptedJobs++
|
|
if commitScenario.FinalStatus == model.AttachmentReady {
|
|
report.ReadyJobs++
|
|
report.TerminalJobs++
|
|
}
|
|
|
|
payload, err := redisstream.EncodeEnvelope(envelope)
|
|
if err != nil {
|
|
return result{}, err
|
|
}
|
|
for range 2 {
|
|
if _, err := e.broker.PublishConfirmed(ctx, payload); err != nil {
|
|
return result{}, err
|
|
}
|
|
}
|
|
duplicateDeliveries, err := e.readDeliveries(ctx, 2)
|
|
if err != nil {
|
|
return result{}, err
|
|
}
|
|
for _, delivery := range duplicateDeliveries {
|
|
if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil {
|
|
return result{}, err
|
|
}
|
|
}
|
|
report.DuplicateDeliveries = len(duplicateDeliveries)
|
|
report.V1ObjectCount, err = e.objectCount(ctx, attachmentID)
|
|
if err != nil {
|
|
return result{}, err
|
|
}
|
|
if report.V1ObjectCount != 4 {
|
|
return result{}, fmt.Errorf("duplicate delivery produced %d objects, expected 4", report.V1ObjectCount)
|
|
}
|
|
report.InvariantResults["duplicate_effect_is_unique"] = true
|
|
|
|
oldPayload := payload
|
|
v2Job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: envelope.PipelineVersion + 1}
|
|
if _, err := e.repository.QueueAttachmentPipelineVersion(ctx, attachmentID, v2Job, "", ""); err != nil {
|
|
return result{}, err
|
|
}
|
|
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
|
|
return result{}, err
|
|
}
|
|
v2Delivery, err := e.readDelivery(ctx)
|
|
if err != nil {
|
|
return result{}, err
|
|
}
|
|
if err := e.worker.ProcessDelivery(ctx, v2Delivery, worker.FaultNone); err != nil {
|
|
return result{}, err
|
|
}
|
|
if _, err := e.broker.PublishConfirmed(ctx, oldPayload); err != nil {
|
|
return result{}, err
|
|
}
|
|
staleDelivery, err := e.readDelivery(ctx)
|
|
if err != nil {
|
|
return result{}, err
|
|
}
|
|
if err := e.worker.ProcessDelivery(ctx, staleDelivery, worker.FaultNone); err != nil {
|
|
return result{}, err
|
|
}
|
|
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
|
|
if err != nil {
|
|
return result{}, err
|
|
}
|
|
if metadata.Status != model.AttachmentReady || metadata.PipelineVersion != v2Job.PipelineVersion || len(metadata.Variants) != 3 {
|
|
return result{}, errors.New("new pipeline version was not preserved after stale delivery")
|
|
}
|
|
report.V2ObjectCount, err = e.objectCount(ctx, attachmentID)
|
|
if err != nil {
|
|
return result{}, err
|
|
}
|
|
if report.V2ObjectCount != 8 {
|
|
return result{}, fmt.Errorf("versioned pipeline produced %d objects, expected 8", report.V2ObjectCount)
|
|
}
|
|
report.InvariantResults["old_pipeline_cannot_replace_new"] = true
|
|
report.AcceptedJobs++
|
|
report.ReadyJobs++
|
|
report.TerminalJobs++
|
|
report.InvariantResults["accepted_jobs_reach_terminal_state"] = report.AcceptedJobs == report.TerminalJobs
|
|
return report, nil
|
|
}
|
|
|
|
func (e *experiment) runPermanentFailureReplay(ctx context.Context) (string, int, error) {
|
|
attachmentID, _, _, err := e.queueAttachmentData(ctx, []byte("corrupted image"), "image/png")
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
delivery, err := e.readDelivery(ctx)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultNone); err != nil {
|
|
return "", 0, err
|
|
}
|
|
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil {
|
|
return "", 0, errors.New("corrupted staging object did not reach failed")
|
|
}
|
|
firstCode := *metadata.ErrorCode
|
|
replayed, err := e.repository.ReplayFailedAttachment(
|
|
ctx, attachmentID, "lab5experiment", "verify explicit replay fencing",
|
|
)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
|
|
return "", 0, err
|
|
}
|
|
replayDelivery, err := e.readDelivery(ctx)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
if err := e.worker.ProcessDelivery(ctx, replayDelivery, worker.FaultNone); err != nil {
|
|
return "", 0, err
|
|
}
|
|
metadata, err = e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
|
|
if err != nil {
|
|
return "", 0, err
|
|
}
|
|
if metadata.Status != model.AttachmentFailed || metadata.ErrorCode == nil || *metadata.ErrorCode != firstCode {
|
|
return "", 0, errors.New("replayed permanent failure changed terminal classification")
|
|
}
|
|
return firstCode, replayed.Attempt, nil
|
|
}
|
|
|
|
func (e *experiment) runLeaseRecovery(ctx context.Context, fault worker.FaultPoint) (scenario, error) {
|
|
attachmentID, _, err := e.queueAttachment(ctx)
|
|
if err != nil {
|
|
return scenario{}, err
|
|
}
|
|
delivery, err := e.readDelivery(ctx)
|
|
if err != nil {
|
|
return scenario{}, err
|
|
}
|
|
if err := e.worker.ProcessDelivery(ctx, delivery, fault); !errors.Is(err, worker.ErrInjectedFailure) {
|
|
return scenario{}, fmt.Errorf("fault %s returned %v", fault, err)
|
|
}
|
|
before, err := e.objectCount(ctx, attachmentID)
|
|
if err != nil {
|
|
return scenario{}, err
|
|
}
|
|
if fault == worker.FaultBeforeSideEffect && before != 0 {
|
|
return scenario{}, errors.New("objects were written before the injected side effect")
|
|
}
|
|
if fault == worker.FaultAfterSideEffect && before != 4 {
|
|
return scenario{}, fmt.Errorf("side effect wrote %d objects, expected 4", before)
|
|
}
|
|
if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil {
|
|
return scenario{}, err
|
|
}
|
|
recovered, err := e.repository.RecoverExpiredAttachmentJobs(ctx, 10, maxAttempts, time.Now())
|
|
if err != nil {
|
|
return scenario{}, err
|
|
}
|
|
if recovered.Requeued != 1 {
|
|
return scenario{}, fmt.Errorf("requeued jobs = %d, expected 1", recovered.Requeued)
|
|
}
|
|
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
|
|
return scenario{}, err
|
|
}
|
|
retry, err := e.readDelivery(ctx)
|
|
if err != nil {
|
|
return scenario{}, err
|
|
}
|
|
if err := e.worker.ProcessDelivery(ctx, retry, worker.FaultNone); err != nil {
|
|
return scenario{}, err
|
|
}
|
|
if err := e.ackReclaimed(ctx); err != nil {
|
|
return scenario{}, err
|
|
}
|
|
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
|
|
if err != nil {
|
|
return scenario{}, err
|
|
}
|
|
after, err := e.objectCount(ctx, attachmentID)
|
|
if err != nil {
|
|
return scenario{}, err
|
|
}
|
|
job, err := e.repository.AttachmentJob(ctx, retry.Envelope.JobID.String())
|
|
if err != nil {
|
|
return scenario{}, err
|
|
}
|
|
return scenario{
|
|
Name: string(fault), AttemptAfterRecovery: job.Attempt,
|
|
ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status,
|
|
}, nil
|
|
}
|
|
|
|
func (e *experiment) runAfterCommit(ctx context.Context) (scenario, redisstream.Envelope, string, error) {
|
|
attachmentID, _, err := e.queueAttachment(ctx)
|
|
if err != nil {
|
|
return scenario{}, redisstream.Envelope{}, "", err
|
|
}
|
|
delivery, err := e.readDelivery(ctx)
|
|
if err != nil {
|
|
return scenario{}, redisstream.Envelope{}, "", err
|
|
}
|
|
if err := e.worker.ProcessDelivery(ctx, delivery, worker.FaultAfterCommit); !errors.Is(err, worker.ErrInjectedFailure) {
|
|
return scenario{}, redisstream.Envelope{}, "", fmt.Errorf("after_commit returned %v", err)
|
|
}
|
|
metadata, err := e.repository.AttachmentForUser(ctx, attachmentID, e.ownerID)
|
|
if err != nil {
|
|
return scenario{}, redisstream.Envelope{}, "", err
|
|
}
|
|
before, err := e.objectCount(ctx, attachmentID)
|
|
if err != nil {
|
|
return scenario{}, redisstream.Envelope{}, "", err
|
|
}
|
|
if err := wait(ctx, experimentLease+250*time.Millisecond); err != nil {
|
|
return scenario{}, redisstream.Envelope{}, "", err
|
|
}
|
|
if err := e.ackReclaimed(ctx); err != nil {
|
|
return scenario{}, redisstream.Envelope{}, "", err
|
|
}
|
|
after, err := e.objectCount(ctx, attachmentID)
|
|
if err != nil {
|
|
return scenario{}, redisstream.Envelope{}, "", err
|
|
}
|
|
return scenario{
|
|
Name: string(worker.FaultAfterCommit), AttemptAfterRecovery: delivery.Envelope.Attempt,
|
|
ObjectsBeforeRecovery: before, ObjectsAfterRecovery: after, FinalStatus: metadata.Status,
|
|
}, delivery.Envelope, attachmentID, nil
|
|
}
|
|
|
|
func (e *experiment) queueAttachment(ctx context.Context) (string, model.AttachmentJob, error) {
|
|
attachmentID, _, job, err := e.queueAttachmentData(ctx, e.fixture, "image/png")
|
|
return attachmentID, job, err
|
|
}
|
|
|
|
func (e *experiment) queueAttachmentData(
|
|
ctx context.Context,
|
|
data []byte,
|
|
contentType string,
|
|
) (string, string, model.AttachmentJob, error) {
|
|
attachmentID := uuid.NewString()
|
|
stagingKey := "staging/" + attachmentID + "/experiment"
|
|
digest := sha256.Sum256(data)
|
|
_, err := e.repository.CreatePendingAttachment(ctx, model.Attachment{
|
|
ID: attachmentID, ChatID: e.chatID, OwnerID: e.ownerID, Filename: "experiment.png",
|
|
ContentType: contentType, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]),
|
|
ObjectKey: stagingKey, StagingKey: stagingKey, Status: model.AttachmentPending, PipelineVersion: 1,
|
|
})
|
|
if err != nil {
|
|
return "", "", model.AttachmentJob{}, err
|
|
}
|
|
if err := e.objects.Put(ctx, stagingKey, contentType, data); err != nil {
|
|
return "", "", model.AttachmentJob{}, err
|
|
}
|
|
job := model.AttachmentJob{ID: uuid.NewString(), PipelineVersion: 1, Attempt: 1}
|
|
if _, err := e.repository.QueueAttachmentWithJobAndOutbox(ctx, attachmentID, stagingKey, job, "", ""); err != nil {
|
|
return "", "", model.AttachmentJob{}, err
|
|
}
|
|
if _, err := e.dispatcher.DispatchOnce(ctx); err != nil {
|
|
return "", "", model.AttachmentJob{}, err
|
|
}
|
|
return attachmentID, stagingKey, job, nil
|
|
}
|
|
|
|
func (e *experiment) readDelivery(ctx context.Context) (redisstream.Delivery, error) {
|
|
deliveries, err := e.readDeliveries(ctx, 1)
|
|
if err != nil {
|
|
return redisstream.Delivery{}, err
|
|
}
|
|
return deliveries[0], nil
|
|
}
|
|
|
|
func (e *experiment) readDeliveries(ctx context.Context, expected int) ([]redisstream.Delivery, error) {
|
|
var result []redisstream.Delivery
|
|
for len(result) < expected {
|
|
deliveries, err := e.broker.Read(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
result = append(result, deliveries...)
|
|
}
|
|
if len(result) != expected {
|
|
return nil, fmt.Errorf("deliveries = %d, expected %d", len(result), expected)
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func (e *experiment) ackReclaimed(ctx context.Context) error {
|
|
deliveries, _, err := e.broker.Reclaim(ctx, "0-0")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if len(deliveries) != 1 {
|
|
return fmt.Errorf("reclaimed deliveries = %d, expected 1", len(deliveries))
|
|
}
|
|
return e.worker.ProcessDelivery(ctx, deliveries[0], worker.FaultNone)
|
|
}
|
|
|
|
func (e *experiment) objectCount(ctx context.Context, attachmentID string) (int, error) {
|
|
objects, next, err := e.objects.List(ctx, "attachments/"+attachmentID+"/", "", 100)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if next != "" {
|
|
return 0, errors.New("unexpected object pagination")
|
|
}
|
|
return len(objects), nil
|
|
}
|
|
|
|
func pngFixture(width, height int) ([]byte, error) {
|
|
value := image.NewNRGBA(image.Rect(0, 0, width, height))
|
|
for y := range height {
|
|
for x := range width {
|
|
value.SetNRGBA(x, y, color.NRGBA{R: uint8(x), G: uint8(y), B: uint8(x + y), A: 255})
|
|
}
|
|
}
|
|
var output bytes.Buffer
|
|
if err := png.Encode(&output, value); err != nil {
|
|
return nil, err
|
|
}
|
|
return output.Bytes(), nil
|
|
}
|
|
|
|
func wait(ctx context.Context, duration time.Duration) error {
|
|
timer := time.NewTimer(duration)
|
|
defer timer.Stop()
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-timer.C:
|
|
return nil
|
|
}
|
|
}
|