feat(worker): add safe failed attachment replay CLI
This commit is contained in:
+117
-19
@@ -7,19 +7,34 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/joho/godotenv"
|
||||
|
||||
"mtusi2026-messenger/internal/config"
|
||||
"mtusi2026-messenger/internal/db"
|
||||
"mtusi2026-messenger/internal/logger"
|
||||
"mtusi2026-messenger/internal/postgres"
|
||||
s3store "mtusi2026-messenger/internal/s3"
|
||||
"mtusi2026-messenger/internal/worker"
|
||||
)
|
||||
|
||||
var (
|
||||
errConflictingReplayFlags = errors.New("replay and list-failed cannot be combined")
|
||||
errInvalidReplayID = errors.New("replay must be an attachment UUID")
|
||||
errMissingReplayAudit = errors.New("replay requires nonempty operator and reason")
|
||||
)
|
||||
|
||||
func main() {
|
||||
maxPixels := flag.Int("process-image", 0, "process an image from stdin with the given pixel limit")
|
||||
replay := flag.String("replay", "", "replay a failed attachment by UUID; requires retained staging and no existing variants")
|
||||
operator := flag.String("operator", "", "operator requesting replay")
|
||||
reason := flag.String("reason", "", "reason for replay")
|
||||
listFailed := flag.Bool("list-failed", false, "list up to 100 failed attachments")
|
||||
flag.Parse()
|
||||
|
||||
if *maxPixels > 0 {
|
||||
@@ -35,44 +50,107 @@ func main() {
|
||||
return
|
||||
}
|
||||
|
||||
if err := run(); err != nil {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
var err error
|
||||
switch {
|
||||
case *listFailed && *replay != "":
|
||||
err = errConflictingReplayFlags
|
||||
case *listFailed:
|
||||
err = runListFailed(ctx)
|
||||
case *replay != "":
|
||||
err = runReplay(ctx, *replay, *operator, *reason)
|
||||
default:
|
||||
err = runWorker(ctx)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "worker:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
if err := godotenv.Load(); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
func runListFailed(ctx context.Context) error {
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cfg := config.Load()
|
||||
log := logger.New(cfg.Logging.Level)
|
||||
databaseURL, err := cfg.Database.URL()
|
||||
_, pool, err := loadDatabase(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
rows, err := db.New(pool).ListFailedAttachments(ctx, 100)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, row := range rows {
|
||||
code := "-"
|
||||
if row.ErrorCode != nil {
|
||||
code = *row.ErrorCode
|
||||
}
|
||||
|
||||
fmt.Printf("%s attempts=%d error_code=%s\n", row.ID, row.Attempts, code)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func runReplay(ctx context.Context, replay, operator, reason string) error {
|
||||
replayID, err := uuid.Parse(replay)
|
||||
if err != nil {
|
||||
return errInvalidReplayID
|
||||
}
|
||||
|
||||
operator, reason = strings.TrimSpace(operator), strings.TrimSpace(reason)
|
||||
if operator == "" || reason == "" {
|
||||
return errMissingReplayAudit
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
|
||||
cfg, pool, err := loadDatabase(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
log := logger.New(cfg.Logging.Level)
|
||||
storage, err := s3store.NewAPI(ctx, cfg.S3, log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := worker.Replay(ctx, pool, storage, replayID, operator, reason); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info("attachment replay queued", "attachment_id", replayID, "operator", operator)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runWorker(ctx context.Context) error {
|
||||
cfg, pool, err := loadDatabase(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
brokerURL, err := cfg.Broker.URL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
pool, err := postgres.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
log := logger.New(cfg.Logging.Level)
|
||||
storage, err := s3store.NewAPI(ctx, cfg.S3, log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
worker.New(
|
||||
w := worker.New(
|
||||
pool,
|
||||
storage,
|
||||
brokerURL,
|
||||
@@ -80,7 +158,27 @@ func run() error {
|
||||
cfg.S3.MaxUploadBytes,
|
||||
cfg.S3.MaxImagePixels,
|
||||
log,
|
||||
).Run(ctx)
|
||||
)
|
||||
w.Run(ctx)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadDatabase(ctx context.Context) (config.Config, *pgxpool.Pool, error) {
|
||||
if err := godotenv.Load(); err != nil && !os.IsNotExist(err) {
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
|
||||
cfg := config.Load()
|
||||
databaseURL, err := cfg.Database.URL()
|
||||
if err != nil {
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
|
||||
pool, err := postgres.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
return config.Config{}, nil, err
|
||||
}
|
||||
|
||||
return cfg, pool, nil
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"mtusi2026-messenger/internal/db"
|
||||
"mtusi2026-messenger/internal/model"
|
||||
s3store "mtusi2026-messenger/internal/s3"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -128,13 +129,13 @@ func (w *Worker) duplicate(ctx context.Context, job model.AttachmentJob) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := w.verify(ctx, row.S3Key, row.SizeBytes, row.Sha256); err != nil {
|
||||
if err := verify(ctx, w.s3, row.S3Key, row.SizeBytes, row.Sha256); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
hasThumbnail := false
|
||||
for _, variant := range variants {
|
||||
if err := w.verify(ctx, variant.S3Key, variant.SizeBytes, variant.Sha256); err != nil {
|
||||
if err := verify(ctx, w.s3, variant.S3Key, variant.SizeBytes, variant.Sha256); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -161,8 +162,8 @@ func (w *Worker) duplicate(ctx context.Context, job model.AttachmentJob) error {
|
||||
return errAttemptUnavailable
|
||||
}
|
||||
|
||||
func (w *Worker) verify(ctx context.Context, key string, size int, checksum string) error {
|
||||
object, err := w.s3.Get(ctx, key)
|
||||
func verify(ctx context.Context, storage *s3store.Client, key string, size int, checksum string) error {
|
||||
object, err := storage.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"mtusi2026-messenger/internal/db"
|
||||
s3store "mtusi2026-messenger/internal/s3"
|
||||
)
|
||||
|
||||
var (
|
||||
errAttachmentNotReplayable = errors.New("attachment is not replayable: expected failed with staging and no existing variants")
|
||||
errMissingPipelineJob = errors.New("attachment has no pipeline job")
|
||||
errReplayNotUpdated = errors.New("attachment replay did not update one row")
|
||||
)
|
||||
|
||||
type txBeginner interface {
|
||||
Begin(ctx context.Context) (pgx.Tx, error)
|
||||
}
|
||||
|
||||
func Replay(
|
||||
ctx context.Context,
|
||||
pool txBeginner,
|
||||
storage *s3store.Client,
|
||||
id uuid.UUID,
|
||||
operator, reason string,
|
||||
) error {
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
q := db.New(tx)
|
||||
row, err := q.LockFailedAttachmentForReplay(ctx, id)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return errAttachmentNotReplayable
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if row.JobID == nil {
|
||||
return errMissingPipelineJob
|
||||
}
|
||||
|
||||
if err := verify(ctx, storage, *row.StagingKey, row.SizeBytes, row.Sha256); err != nil {
|
||||
return fmt.Errorf("verify replay source: %w", err)
|
||||
}
|
||||
|
||||
n, err := q.ReplayAttachmentJob(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n != 1 {
|
||||
return errReplayNotUpdated
|
||||
}
|
||||
|
||||
if err := q.CreateAttachmentReplay(ctx, db.CreateAttachmentReplayParams{
|
||||
AttachmentID: id,
|
||||
Attempt: row.Attempts + 1,
|
||||
PreviousErrorCode: row.ErrorCode,
|
||||
Operator: operator,
|
||||
Reason: reason,
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err := q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{
|
||||
AttachmentID: id,
|
||||
Attempt: row.Attempts + 1,
|
||||
AvailableAt: time.Now(),
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -272,3 +272,87 @@ func TestWorkerProcessingTimeout(t *testing.T) {
|
||||
t.Fatal("parent worker restarted after the image timeout")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWorkerReplay(t *testing.T) {
|
||||
user := createUser(t)
|
||||
chat := createChat(t, user, user.ID)
|
||||
stopService(t, "worker")
|
||||
if workerQueue(t) != "0:0" {
|
||||
t.Fatal("run on an idle stand with an empty attachment queue")
|
||||
}
|
||||
att, source := workerUpload(t, user, chat.ID)
|
||||
id := att.ID.String()
|
||||
where := "attachment_id='" + id + "'"
|
||||
|
||||
testWorker := "worker-replay-" + uuid.NewString()
|
||||
t.Cleanup(func() {
|
||||
if testWorker != "" {
|
||||
workerCommand(t, "rm", "-f", testWorker)
|
||||
}
|
||||
})
|
||||
workerCommand(t, "compose", "run", "-d", "--no-deps", "--name", testWorker,
|
||||
"-e", "JOB_TIMEOUT_SECONDS=3", "-e", "JOB_MAX_ATTEMPTS=1",
|
||||
"-e", "S3_ENDPOINT=http://127.0.0.1:1", "worker")
|
||||
waitAttachment(t, user, att.ID, "failed", 15*time.Second)
|
||||
workerCommand(t, "rm", "-f", testWorker)
|
||||
testWorker = ""
|
||||
|
||||
runCLI := func(args ...string) (string, error) {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
cmd := exec.CommandContext(ctx, "docker", append([]string{
|
||||
"compose", "run", "--rm", "--no-deps", "worker",
|
||||
}, args...)...)
|
||||
cmd.Dir = ".."
|
||||
out, err := cmd.CombinedOutput()
|
||||
return string(out), err
|
||||
}
|
||||
replayArgs := []string{"-replay", id, "-operator", "synthetic-operator", "-reason", "S3 restored"}
|
||||
if out, err := runCLI("-list-failed"); err != nil || !strings.Contains(out, id) {
|
||||
t.Fatalf("failed registry did not list attachment: %v\n%s", err, out)
|
||||
}
|
||||
if out, err := runCLI("-replay", id); err == nil || !strings.Contains(out, "nonempty operator and reason") {
|
||||
t.Fatalf("replay without audit details was accepted: %v\n%s", err, out)
|
||||
}
|
||||
|
||||
// A stale DB key must not make a missing S3 source replayable.
|
||||
key := workerSQL(t, "SELECT staging_key FROM attachments WHERE id='"+id+"'")
|
||||
workerSQL(t, "UPDATE attachments SET staging_key='staging/"+uuid.NewString()+"' WHERE id='"+id+"'")
|
||||
out, err := runCLI(replayArgs...)
|
||||
workerSQL(t, "UPDATE attachments SET staging_key='"+key+"' WHERE id='"+id+"'")
|
||||
if err == nil || !strings.Contains(out, "verify replay source") {
|
||||
t.Fatalf("missing S3 source was accepted: %v\n%s", err, out)
|
||||
}
|
||||
if workerSQL(t, "SELECT status||':'||attempts||':'||retry_base FROM attachments WHERE id='"+id+"'") != "failed:1:0" ||
|
||||
workerSQL(t, "SELECT count(*) FROM attachment_replays WHERE "+where) != "0" ||
|
||||
workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where) != "1" {
|
||||
t.Fatal("rejected replay changed attachment, audit or outbox")
|
||||
}
|
||||
|
||||
if out, err := runCLI(replayArgs...); err != nil {
|
||||
t.Fatalf("replay failed: %v\n%s", err, out)
|
||||
}
|
||||
if workerSQL(t, "SELECT status||':'||attempts||':'||retry_base FROM attachments WHERE id='"+id+"'") != "queued:1:1" ||
|
||||
workerSQL(t, "SELECT attempt||':'||operator||':'||reason||':'||previous_error_code FROM attachment_replays WHERE "+where) !=
|
||||
"2:synthetic-operator:S3 restored:retry_exhausted" ||
|
||||
workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where) != "2" {
|
||||
t.Fatal("replay did not atomically queue the next attempt with audit and outbox")
|
||||
}
|
||||
if out, err := runCLI(replayArgs...); err == nil || !strings.Contains(out, "not replayable") {
|
||||
t.Fatalf("queued replay was accepted twice: %v\n%s", err, out)
|
||||
}
|
||||
|
||||
compose(t, "start", "worker")
|
||||
ready := waitAttachment(t, user, att.ID, "ready", 30*time.Second)
|
||||
if len(ready.Variants) != 3 || !bytes.Equal(workerContent(t, user, att.ID, "original"), source) {
|
||||
t.Fatal("replay did not complete the same attachment with unchanged original")
|
||||
}
|
||||
if out, err := runCLI(replayArgs...); err == nil || !strings.Contains(out, "not replayable") {
|
||||
t.Fatalf("ready replay was accepted: %v\n%s", err, out)
|
||||
}
|
||||
if workerSQL(t, "SELECT count(*) FROM attachment_replays WHERE "+where) != "1" ||
|
||||
workerSQL(t, "SELECT attempts FROM attachments WHERE id='"+id+"'") != "2" {
|
||||
t.Fatal("duplicate replay changed audit or attempt counter")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user