feat(worker): add safe failed attachment replay CLI

This commit is contained in:
kk10
2026-11-16 20:35:48 +03:00
parent f2000a7883
commit 6e09da0b46
4 changed files with 292 additions and 23 deletions
+117 -19
View File
@@ -7,19 +7,34 @@ import (
"fmt"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/joho/godotenv"
"mtusi2026-messenger/internal/config"
"mtusi2026-messenger/internal/db"
"mtusi2026-messenger/internal/logger"
"mtusi2026-messenger/internal/postgres"
s3store "mtusi2026-messenger/internal/s3"
"mtusi2026-messenger/internal/worker"
)
var (
errConflictingReplayFlags = errors.New("replay and list-failed cannot be combined")
errInvalidReplayID = errors.New("replay must be an attachment UUID")
errMissingReplayAudit = errors.New("replay requires nonempty operator and reason")
)
func main() {
maxPixels := flag.Int("process-image", 0, "process an image from stdin with the given pixel limit")
replay := flag.String("replay", "", "replay a failed attachment by UUID; requires retained staging and no existing variants")
operator := flag.String("operator", "", "operator requesting replay")
reason := flag.String("reason", "", "reason for replay")
listFailed := flag.Bool("list-failed", false, "list up to 100 failed attachments")
flag.Parse()
if *maxPixels > 0 {
@@ -35,44 +50,107 @@ func main() {
return
}
if err := run(); err != nil {
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
var err error
switch {
case *listFailed && *replay != "":
err = errConflictingReplayFlags
case *listFailed:
err = runListFailed(ctx)
case *replay != "":
err = runReplay(ctx, *replay, *operator, *reason)
default:
err = runWorker(ctx)
}
if err != nil {
fmt.Fprintln(os.Stderr, "worker:", err)
os.Exit(1)
}
}
func run() error {
if err := godotenv.Load(); err != nil && !os.IsNotExist(err) {
return err
}
func runListFailed(ctx context.Context) error {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
cfg := config.Load()
log := logger.New(cfg.Logging.Level)
databaseURL, err := cfg.Database.URL()
_, pool, err := loadDatabase(ctx)
if err != nil {
return err
}
defer pool.Close()
rows, err := db.New(pool).ListFailedAttachments(ctx, 100)
if err != nil {
return err
}
for _, row := range rows {
code := "-"
if row.ErrorCode != nil {
code = *row.ErrorCode
}
fmt.Printf("%s attempts=%d error_code=%s\n", row.ID, row.Attempts, code)
}
return nil
}
func runReplay(ctx context.Context, replay, operator, reason string) error {
replayID, err := uuid.Parse(replay)
if err != nil {
return errInvalidReplayID
}
operator, reason = strings.TrimSpace(operator), strings.TrimSpace(reason)
if operator == "" || reason == "" {
return errMissingReplayAudit
}
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
cfg, pool, err := loadDatabase(ctx)
if err != nil {
return err
}
defer pool.Close()
log := logger.New(cfg.Logging.Level)
storage, err := s3store.NewAPI(ctx, cfg.S3, log)
if err != nil {
return err
}
if err := worker.Replay(ctx, pool, storage, replayID, operator, reason); err != nil {
return err
}
log.Info("attachment replay queued", "attachment_id", replayID, "operator", operator)
return nil
}
func runWorker(ctx context.Context) error {
cfg, pool, err := loadDatabase(ctx)
if err != nil {
return err
}
defer pool.Close()
brokerURL, err := cfg.Broker.URL()
if err != nil {
return err
}
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
pool, err := postgres.New(ctx, databaseURL)
if err != nil {
return err
}
defer pool.Close()
log := logger.New(cfg.Logging.Level)
storage, err := s3store.NewAPI(ctx, cfg.S3, log)
if err != nil {
return err
}
worker.New(
w := worker.New(
pool,
storage,
brokerURL,
@@ -80,7 +158,27 @@ func run() error {
cfg.S3.MaxUploadBytes,
cfg.S3.MaxImagePixels,
log,
).Run(ctx)
)
w.Run(ctx)
return nil
}
func loadDatabase(ctx context.Context) (config.Config, *pgxpool.Pool, error) {
if err := godotenv.Load(); err != nil && !os.IsNotExist(err) {
return config.Config{}, nil, err
}
cfg := config.Load()
databaseURL, err := cfg.Database.URL()
if err != nil {
return config.Config{}, nil, err
}
pool, err := postgres.New(ctx, databaseURL)
if err != nil {
return config.Config{}, nil, err
}
return cfg, pool, nil
}
+5 -4
View File
@@ -12,6 +12,7 @@ import (
"mtusi2026-messenger/internal/db"
"mtusi2026-messenger/internal/model"
s3store "mtusi2026-messenger/internal/s3"
)
var (
@@ -128,13 +129,13 @@ func (w *Worker) duplicate(ctx context.Context, job model.AttachmentJob) error {
return err
}
if err := w.verify(ctx, row.S3Key, row.SizeBytes, row.Sha256); err != nil {
if err := verify(ctx, w.s3, row.S3Key, row.SizeBytes, row.Sha256); err != nil {
return err
}
hasThumbnail := false
for _, variant := range variants {
if err := w.verify(ctx, variant.S3Key, variant.SizeBytes, variant.Sha256); err != nil {
if err := verify(ctx, w.s3, variant.S3Key, variant.SizeBytes, variant.Sha256); err != nil {
return err
}
@@ -161,8 +162,8 @@ func (w *Worker) duplicate(ctx context.Context, job model.AttachmentJob) error {
return errAttemptUnavailable
}
func (w *Worker) verify(ctx context.Context, key string, size int, checksum string) error {
object, err := w.s3.Get(ctx, key)
func verify(ctx context.Context, storage *s3store.Client, key string, size int, checksum string) error {
object, err := storage.Get(ctx, key)
if err != nil {
return err
}
+86
View File
@@ -0,0 +1,86 @@
package worker
import (
"context"
"errors"
"fmt"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"mtusi2026-messenger/internal/db"
s3store "mtusi2026-messenger/internal/s3"
)
var (
errAttachmentNotReplayable = errors.New("attachment is not replayable: expected failed with staging and no existing variants")
errMissingPipelineJob = errors.New("attachment has no pipeline job")
errReplayNotUpdated = errors.New("attachment replay did not update one row")
)
type txBeginner interface {
Begin(ctx context.Context) (pgx.Tx, error)
}
func Replay(
ctx context.Context,
pool txBeginner,
storage *s3store.Client,
id uuid.UUID,
operator, reason string,
) error {
tx, err := pool.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(context.Background())
q := db.New(tx)
row, err := q.LockFailedAttachmentForReplay(ctx, id)
if errors.Is(err, pgx.ErrNoRows) {
return errAttachmentNotReplayable
}
if err != nil {
return err
}
if row.JobID == nil {
return errMissingPipelineJob
}
if err := verify(ctx, storage, *row.StagingKey, row.SizeBytes, row.Sha256); err != nil {
return fmt.Errorf("verify replay source: %w", err)
}
n, err := q.ReplayAttachmentJob(ctx, id)
if err != nil {
return err
}
if n != 1 {
return errReplayNotUpdated
}
if err := q.CreateAttachmentReplay(ctx, db.CreateAttachmentReplayParams{
AttachmentID: id,
Attempt: row.Attempts + 1,
PreviousErrorCode: row.ErrorCode,
Operator: operator,
Reason: reason,
}); err != nil {
return err
}
if _, err := q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{
AttachmentID: id,
Attempt: row.Attempts + 1,
AvailableAt: time.Now(),
}); err != nil {
return err
}
if err := tx.Commit(ctx); err != nil {
return err
}
return nil
}
+84
View File
@@ -272,3 +272,87 @@ func TestWorkerProcessingTimeout(t *testing.T) {
t.Fatal("parent worker restarted after the image timeout")
}
}
func TestWorkerReplay(t *testing.T) {
user := createUser(t)
chat := createChat(t, user, user.ID)
stopService(t, "worker")
if workerQueue(t) != "0:0" {
t.Fatal("run on an idle stand with an empty attachment queue")
}
att, source := workerUpload(t, user, chat.ID)
id := att.ID.String()
where := "attachment_id='" + id + "'"
testWorker := "worker-replay-" + uuid.NewString()
t.Cleanup(func() {
if testWorker != "" {
workerCommand(t, "rm", "-f", testWorker)
}
})
workerCommand(t, "compose", "run", "-d", "--no-deps", "--name", testWorker,
"-e", "JOB_TIMEOUT_SECONDS=3", "-e", "JOB_MAX_ATTEMPTS=1",
"-e", "S3_ENDPOINT=http://127.0.0.1:1", "worker")
waitAttachment(t, user, att.ID, "failed", 15*time.Second)
workerCommand(t, "rm", "-f", testWorker)
testWorker = ""
runCLI := func(args ...string) (string, error) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, "docker", append([]string{
"compose", "run", "--rm", "--no-deps", "worker",
}, args...)...)
cmd.Dir = ".."
out, err := cmd.CombinedOutput()
return string(out), err
}
replayArgs := []string{"-replay", id, "-operator", "synthetic-operator", "-reason", "S3 restored"}
if out, err := runCLI("-list-failed"); err != nil || !strings.Contains(out, id) {
t.Fatalf("failed registry did not list attachment: %v\n%s", err, out)
}
if out, err := runCLI("-replay", id); err == nil || !strings.Contains(out, "nonempty operator and reason") {
t.Fatalf("replay without audit details was accepted: %v\n%s", err, out)
}
// A stale DB key must not make a missing S3 source replayable.
key := workerSQL(t, "SELECT staging_key FROM attachments WHERE id='"+id+"'")
workerSQL(t, "UPDATE attachments SET staging_key='staging/"+uuid.NewString()+"' WHERE id='"+id+"'")
out, err := runCLI(replayArgs...)
workerSQL(t, "UPDATE attachments SET staging_key='"+key+"' WHERE id='"+id+"'")
if err == nil || !strings.Contains(out, "verify replay source") {
t.Fatalf("missing S3 source was accepted: %v\n%s", err, out)
}
if workerSQL(t, "SELECT status||':'||attempts||':'||retry_base FROM attachments WHERE id='"+id+"'") != "failed:1:0" ||
workerSQL(t, "SELECT count(*) FROM attachment_replays WHERE "+where) != "0" ||
workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where) != "1" {
t.Fatal("rejected replay changed attachment, audit or outbox")
}
if out, err := runCLI(replayArgs...); err != nil {
t.Fatalf("replay failed: %v\n%s", err, out)
}
if workerSQL(t, "SELECT status||':'||attempts||':'||retry_base FROM attachments WHERE id='"+id+"'") != "queued:1:1" ||
workerSQL(t, "SELECT attempt||':'||operator||':'||reason||':'||previous_error_code FROM attachment_replays WHERE "+where) !=
"2:synthetic-operator:S3 restored:retry_exhausted" ||
workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where) != "2" {
t.Fatal("replay did not atomically queue the next attempt with audit and outbox")
}
if out, err := runCLI(replayArgs...); err == nil || !strings.Contains(out, "not replayable") {
t.Fatalf("queued replay was accepted twice: %v\n%s", err, out)
}
compose(t, "start", "worker")
ready := waitAttachment(t, user, att.ID, "ready", 30*time.Second)
if len(ready.Variants) != 3 || !bytes.Equal(workerContent(t, user, att.ID, "original"), source) {
t.Fatal("replay did not complete the same attachment with unchanged original")
}
if out, err := runCLI(replayArgs...); err == nil || !strings.Contains(out, "not replayable") {
t.Fatalf("ready replay was accepted: %v\n%s", err, out)
}
if workerSQL(t, "SELECT count(*) FROM attachment_replays WHERE "+where) != "1" ||
workerSQL(t, "SELECT attempts FROM attachments WHERE id='"+id+"'") != "2" {
t.Fatal("duplicate replay changed audit or attempt counter")
}
}