diff --git a/cmd/worker/main.go b/cmd/worker/main.go index cbcbd49..08bccc2 100644 --- a/cmd/worker/main.go +++ b/cmd/worker/main.go @@ -7,19 +7,34 @@ import ( "fmt" "os" "os/signal" + "strings" "syscall" + "time" + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" "github.com/joho/godotenv" "mtusi2026-messenger/internal/config" + "mtusi2026-messenger/internal/db" "mtusi2026-messenger/internal/logger" "mtusi2026-messenger/internal/postgres" s3store "mtusi2026-messenger/internal/s3" "mtusi2026-messenger/internal/worker" ) +var ( + errConflictingReplayFlags = errors.New("replay and list-failed cannot be combined") + errInvalidReplayID = errors.New("replay must be an attachment UUID") + errMissingReplayAudit = errors.New("replay requires nonempty operator and reason") +) + func main() { maxPixels := flag.Int("process-image", 0, "process an image from stdin with the given pixel limit") + replay := flag.String("replay", "", "replay a failed attachment by UUID; requires retained staging and no existing variants") + operator := flag.String("operator", "", "operator requesting replay") + reason := flag.String("reason", "", "reason for replay") + listFailed := flag.Bool("list-failed", false, "list up to 100 failed attachments") flag.Parse() if *maxPixels > 0 { @@ -35,44 +50,107 @@ func main() { return } - if err := run(); err != nil { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stop() + + var err error + switch { + case *listFailed && *replay != "": + err = errConflictingReplayFlags + case *listFailed: + err = runListFailed(ctx) + case *replay != "": + err = runReplay(ctx, *replay, *operator, *reason) + default: + err = runWorker(ctx) + } + + if err != nil { fmt.Fprintln(os.Stderr, "worker:", err) os.Exit(1) } } -func run() error { - if err := godotenv.Load(); err != nil && !os.IsNotExist(err) { - return err - } +func runListFailed(ctx context.Context) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() - cfg := config.Load() - log := logger.New(cfg.Logging.Level) - databaseURL, err := cfg.Database.URL() + _, pool, err := loadDatabase(ctx) if err != nil { return err } + defer pool.Close() + + rows, err := db.New(pool).ListFailedAttachments(ctx, 100) + if err != nil { + return err + } + + for _, row := range rows { + code := "-" + if row.ErrorCode != nil { + code = *row.ErrorCode + } + + fmt.Printf("%s attempts=%d error_code=%s\n", row.ID, row.Attempts, code) + } + + return nil +} + +func runReplay(ctx context.Context, replay, operator, reason string) error { + replayID, err := uuid.Parse(replay) + if err != nil { + return errInvalidReplayID + } + + operator, reason = strings.TrimSpace(operator), strings.TrimSpace(reason) + if operator == "" || reason == "" { + return errMissingReplayAudit + } + + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + cfg, pool, err := loadDatabase(ctx) + if err != nil { + return err + } + defer pool.Close() + + log := logger.New(cfg.Logging.Level) + storage, err := s3store.NewAPI(ctx, cfg.S3, log) + if err != nil { + return err + } + + if err := worker.Replay(ctx, pool, storage, replayID, operator, reason); err != nil { + return err + } + + log.Info("attachment replay queued", "attachment_id", replayID, "operator", operator) + return nil +} + +func runWorker(ctx context.Context) error { + cfg, pool, err := loadDatabase(ctx) + if err != nil { + return err + } + defer pool.Close() brokerURL, err := cfg.Broker.URL() if err != nil { return err } - ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer stop() - - pool, err := postgres.New(ctx, databaseURL) - if err != nil { - return err - } - defer pool.Close() - + log := logger.New(cfg.Logging.Level) storage, err := s3store.NewAPI(ctx, cfg.S3, log) if err != nil { return err } - worker.New( + w := worker.New( pool, storage, brokerURL, @@ -80,7 +158,27 @@ func run() error { cfg.S3.MaxUploadBytes, cfg.S3.MaxImagePixels, log, - ).Run(ctx) + ) + w.Run(ctx) return nil } + +func loadDatabase(ctx context.Context) (config.Config, *pgxpool.Pool, error) { + if err := godotenv.Load(); err != nil && !os.IsNotExist(err) { + return config.Config{}, nil, err + } + + cfg := config.Load() + databaseURL, err := cfg.Database.URL() + if err != nil { + return config.Config{}, nil, err + } + + pool, err := postgres.New(ctx, databaseURL) + if err != nil { + return config.Config{}, nil, err + } + + return cfg, pool, nil +} diff --git a/internal/worker/job.go b/internal/worker/job.go index 66b7775..7a2bfe7 100644 --- a/internal/worker/job.go +++ b/internal/worker/job.go @@ -12,6 +12,7 @@ import ( "mtusi2026-messenger/internal/db" "mtusi2026-messenger/internal/model" + s3store "mtusi2026-messenger/internal/s3" ) var ( @@ -128,13 +129,13 @@ func (w *Worker) duplicate(ctx context.Context, job model.AttachmentJob) error { return err } - if err := w.verify(ctx, row.S3Key, row.SizeBytes, row.Sha256); err != nil { + if err := verify(ctx, w.s3, row.S3Key, row.SizeBytes, row.Sha256); err != nil { return err } hasThumbnail := false for _, variant := range variants { - if err := w.verify(ctx, variant.S3Key, variant.SizeBytes, variant.Sha256); err != nil { + if err := verify(ctx, w.s3, variant.S3Key, variant.SizeBytes, variant.Sha256); err != nil { return err } @@ -161,8 +162,8 @@ func (w *Worker) duplicate(ctx context.Context, job model.AttachmentJob) error { return errAttemptUnavailable } -func (w *Worker) verify(ctx context.Context, key string, size int, checksum string) error { - object, err := w.s3.Get(ctx, key) +func verify(ctx context.Context, storage *s3store.Client, key string, size int, checksum string) error { + object, err := storage.Get(ctx, key) if err != nil { return err } diff --git a/internal/worker/replay.go b/internal/worker/replay.go new file mode 100644 index 0000000..daccce4 --- /dev/null +++ b/internal/worker/replay.go @@ -0,0 +1,86 @@ +package worker + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + + "mtusi2026-messenger/internal/db" + s3store "mtusi2026-messenger/internal/s3" +) + +var ( + errAttachmentNotReplayable = errors.New("attachment is not replayable: expected failed with staging and no existing variants") + errMissingPipelineJob = errors.New("attachment has no pipeline job") + errReplayNotUpdated = errors.New("attachment replay did not update one row") +) + +type txBeginner interface { + Begin(ctx context.Context) (pgx.Tx, error) +} + +func Replay( + ctx context.Context, + pool txBeginner, + storage *s3store.Client, + id uuid.UUID, + operator, reason string, +) error { + tx, err := pool.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(context.Background()) + + q := db.New(tx) + row, err := q.LockFailedAttachmentForReplay(ctx, id) + if errors.Is(err, pgx.ErrNoRows) { + return errAttachmentNotReplayable + } + if err != nil { + return err + } + if row.JobID == nil { + return errMissingPipelineJob + } + + if err := verify(ctx, storage, *row.StagingKey, row.SizeBytes, row.Sha256); err != nil { + return fmt.Errorf("verify replay source: %w", err) + } + + n, err := q.ReplayAttachmentJob(ctx, id) + if err != nil { + return err + } + if n != 1 { + return errReplayNotUpdated + } + + if err := q.CreateAttachmentReplay(ctx, db.CreateAttachmentReplayParams{ + AttachmentID: id, + Attempt: row.Attempts + 1, + PreviousErrorCode: row.ErrorCode, + Operator: operator, + Reason: reason, + }); err != nil { + return err + } + + if _, err := q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{ + AttachmentID: id, + Attempt: row.Attempts + 1, + AvailableAt: time.Now(), + }); err != nil { + return err + } + + if err := tx.Commit(ctx); err != nil { + return err + } + + return nil +} diff --git a/student-tests/worker_test.go b/student-tests/worker_test.go index fd50e0c..4faad7c 100644 --- a/student-tests/worker_test.go +++ b/student-tests/worker_test.go @@ -272,3 +272,87 @@ func TestWorkerProcessingTimeout(t *testing.T) { t.Fatal("parent worker restarted after the image timeout") } } + +func TestWorkerReplay(t *testing.T) { + user := createUser(t) + chat := createChat(t, user, user.ID) + stopService(t, "worker") + if workerQueue(t) != "0:0" { + t.Fatal("run on an idle stand with an empty attachment queue") + } + att, source := workerUpload(t, user, chat.ID) + id := att.ID.String() + where := "attachment_id='" + id + "'" + + testWorker := "worker-replay-" + uuid.NewString() + t.Cleanup(func() { + if testWorker != "" { + workerCommand(t, "rm", "-f", testWorker) + } + }) + workerCommand(t, "compose", "run", "-d", "--no-deps", "--name", testWorker, + "-e", "JOB_TIMEOUT_SECONDS=3", "-e", "JOB_MAX_ATTEMPTS=1", + "-e", "S3_ENDPOINT=http://127.0.0.1:1", "worker") + waitAttachment(t, user, att.ID, "failed", 15*time.Second) + workerCommand(t, "rm", "-f", testWorker) + testWorker = "" + + runCLI := func(args ...string) (string, error) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "docker", append([]string{ + "compose", "run", "--rm", "--no-deps", "worker", + }, args...)...) + cmd.Dir = ".." + out, err := cmd.CombinedOutput() + return string(out), err + } + replayArgs := []string{"-replay", id, "-operator", "synthetic-operator", "-reason", "S3 restored"} + if out, err := runCLI("-list-failed"); err != nil || !strings.Contains(out, id) { + t.Fatalf("failed registry did not list attachment: %v\n%s", err, out) + } + if out, err := runCLI("-replay", id); err == nil || !strings.Contains(out, "nonempty operator and reason") { + t.Fatalf("replay without audit details was accepted: %v\n%s", err, out) + } + + // A stale DB key must not make a missing S3 source replayable. + key := workerSQL(t, "SELECT staging_key FROM attachments WHERE id='"+id+"'") + workerSQL(t, "UPDATE attachments SET staging_key='staging/"+uuid.NewString()+"' WHERE id='"+id+"'") + out, err := runCLI(replayArgs...) + workerSQL(t, "UPDATE attachments SET staging_key='"+key+"' WHERE id='"+id+"'") + if err == nil || !strings.Contains(out, "verify replay source") { + t.Fatalf("missing S3 source was accepted: %v\n%s", err, out) + } + if workerSQL(t, "SELECT status||':'||attempts||':'||retry_base FROM attachments WHERE id='"+id+"'") != "failed:1:0" || + workerSQL(t, "SELECT count(*) FROM attachment_replays WHERE "+where) != "0" || + workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where) != "1" { + t.Fatal("rejected replay changed attachment, audit or outbox") + } + + if out, err := runCLI(replayArgs...); err != nil { + t.Fatalf("replay failed: %v\n%s", err, out) + } + if workerSQL(t, "SELECT status||':'||attempts||':'||retry_base FROM attachments WHERE id='"+id+"'") != "queued:1:1" || + workerSQL(t, "SELECT attempt||':'||operator||':'||reason||':'||previous_error_code FROM attachment_replays WHERE "+where) != + "2:synthetic-operator:S3 restored:retry_exhausted" || + workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where) != "2" { + t.Fatal("replay did not atomically queue the next attempt with audit and outbox") + } + if out, err := runCLI(replayArgs...); err == nil || !strings.Contains(out, "not replayable") { + t.Fatalf("queued replay was accepted twice: %v\n%s", err, out) + } + + compose(t, "start", "worker") + ready := waitAttachment(t, user, att.ID, "ready", 30*time.Second) + if len(ready.Variants) != 3 || !bytes.Equal(workerContent(t, user, att.ID, "original"), source) { + t.Fatal("replay did not complete the same attachment with unchanged original") + } + if out, err := runCLI(replayArgs...); err == nil || !strings.Contains(out, "not replayable") { + t.Fatalf("ready replay was accepted: %v\n%s", err, out) + } + if workerSQL(t, "SELECT count(*) FROM attachment_replays WHERE "+where) != "1" || + workerSQL(t, "SELECT attempts FROM attachments WHERE id='"+id+"'") != "2" { + t.Fatal("duplicate replay changed audit or attempt counter") + } +}