Files
mtusi2026lab6/internal/worker/pipeline_integration_test.go
T

582 lines
21 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package worker_test
// Интеграционный fault-injection (трек 5): настоящий PostgreSQL со всеми
// миграциями, S3 — в памяти, broker — вызовы Handle с телом из outbox (так же,
// как его опубликовал бы relay). Запуск:
//
// TEST_DATABASE_URL=postgres://user@host:5432/postgres go test ./internal/worker/ -run Pipeline -v
//
// Для каждого сценария создаётся отдельная база, после каждого шага проверяются
// инварианты (checkInvariants).
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"image"
"image/color"
"image/png"
"io"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/charawein/mtusi2026lab6/internal/domain"
"github.com/charawein/mtusi2026lab6/internal/repository/postgres"
"github.com/charawein/mtusi2026lab6/internal/storage"
"github.com/charawein/mtusi2026lab6/internal/worker"
)
// ---------- in-memory S3 ----------
type memStore struct {
mu sync.Mutex
objs map[string][]byte
puts map[string]int
failPut int // сколько ближайших Put завершить временной ошибкой
}
func newMemStore() *memStore { return &memStore{objs: map[string][]byte{}, puts: map[string]int{}} }
func (m *memStore) Put(_ context.Context, key string, r io.Reader, size int64, _ string) error {
m.mu.Lock()
defer m.mu.Unlock()
if m.failPut > 0 {
m.failPut--
return errors.New("s3 put: 503 slow down")
}
b, err := io.ReadAll(r)
if err != nil {
return err
}
m.objs[key] = b
m.puts[key]++
return nil
}
func (m *memStore) Get(_ context.Context, key string) (io.ReadCloser, int64, error) {
m.mu.Lock()
defer m.mu.Unlock()
b, ok := m.objs[key]
if !ok {
return nil, 0, storage.ErrNotFound
}
return io.NopCloser(bytes.NewReader(b)), int64(len(b)), nil
}
func (m *memStore) Stat(_ context.Context, key string) (storage.Object, error) {
m.mu.Lock()
defer m.mu.Unlock()
b, ok := m.objs[key]
if !ok {
return storage.Object{}, storage.ErrNotFound
}
return storage.Object{Key: key, Size: int64(len(b))}, nil
}
func (m *memStore) Delete(_ context.Context, key string) error {
m.mu.Lock()
defer m.mu.Unlock()
delete(m.objs, key)
return nil
}
func (m *memStore) List(context.Context, string, func(storage.Object) error) error { return nil }
func (m *memStore) Copy(_ context.Context, src, dst string) error {
m.mu.Lock()
defer m.mu.Unlock()
b, ok := m.objs[src]
if !ok {
return storage.ErrNotFound
}
m.objs[dst] = bytes.Clone(b)
m.puts[dst]++
return nil
}
func (m *memStore) get(key string) ([]byte, bool) {
m.mu.Lock()
defer m.mu.Unlock()
b, ok := m.objs[key]
return b, ok
}
// Детерминированный «WebP» для тестов: RIFF/WEBP-заголовок + хэш пикселей.
func fakeWebP(_ context.Context, img image.Image, q int) ([]byte, error) {
h := sha256.New()
b := img.Bounds()
fmt.Fprintf(h, "%d:%d:%d", b.Dx(), b.Dy(), q)
for y := b.Min.Y; y < b.Max.Y; y += 7 {
for x := b.Min.X; x < b.Max.X; x += 7 {
r, g, bb, _ := img.At(x, y).RGBA()
fmt.Fprintf(h, "%d%d%d", r, g, bb)
}
}
return append([]byte("RIFF\x00\x00\x00\x00WEBP"), h.Sum(nil)...), nil
}
// ---------- окружение ----------
type env struct {
t *testing.T
ctx context.Context
pool *pgxpool.Pool
repo *postgres.PipelineRepository
att *postgres.AttachmentRepository
store *memStore
chat string
owner string
}
type crash struct{ code int }
func setup(t *testing.T) *env {
t.Helper()
admin := os.Getenv("TEST_DATABASE_URL")
if admin == "" {
t.Skip("TEST_DATABASE_URL not set (integration test needs PostgreSQL)")
}
ctx := context.Background()
conn, err := pgx.Connect(ctx, admin)
if err != nil {
t.Fatal(err)
}
db := "lab5_it_" + strings.ReplaceAll(uuid.NewString()[:8], "-", "")
if _, err := conn.Exec(ctx, "CREATE DATABASE "+db); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_, _ = conn.Exec(context.Background(), "DROP DATABASE IF EXISTS "+db+" WITH (FORCE)")
conn.Close(context.Background())
})
cfg, err := pgxpool.ParseConfig(admin)
if err != nil {
t.Fatal(err)
}
cfg.ConnConfig.Database = db
pool, err := pgxpool.NewWithConfig(ctx, cfg)
if err != nil {
t.Fatal(err)
}
t.Cleanup(pool.Close)
files, _ := filepath.Glob("../../migrations/*.up.sql")
sort.Strings(files)
for _, f := range files {
sql, _ := os.ReadFile(f)
if _, err := pool.Exec(ctx, string(sql)); err != nil {
t.Fatalf("%s: %v", f, err)
}
}
e := &env{t: t, ctx: ctx, pool: pool, repo: postgres.NewPipelineRepository(pool),
att: postgres.NewAttachmentRepository(pool), store: newMemStore(), owner: uuid.NewString()}
e.chat = uuid.NewString()
e.exec(`INSERT INTO users (id, username, display_name, password_hash) VALUES ($1::uuid, $2, 'U', 'x')`, e.owner, "u_"+e.owner[:8])
e.exec(`INSERT INTO chats (id, title) VALUES ($1, 'c')`, e.chat)
e.exec(`INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)`, e.chat, e.owner)
return e
}
func (e *env) exec(sql string, args ...any) {
e.t.Helper()
if _, err := e.pool.Exec(e.ctx, sql, args...); err != nil {
e.t.Fatalf("%s: %v", sql, err)
}
}
func (e *env) scalar(sql string, args ...any) string {
e.t.Helper()
var s string
if err := e.pool.QueryRow(e.ctx, sql, args...).Scan(&s); err != nil {
e.t.Fatalf("%s: %v", sql, err)
}
return s
}
func pngBytes(w, h int, seed uint8) []byte {
img := image.NewRGBA(image.Rect(0, 0, w, h))
for y := 0; y < h; y++ {
for x := 0; x < w; x++ {
img.Set(x, y, color.RGBA{uint8(x) + seed, uint8(y), seed, 255})
}
}
var b bytes.Buffer
_ = png.Encode(&b, img)
return b.Bytes()
}
// upload — как API: намерение, staging-объект, MarkQueued (jobs + outbox).
func (e *env) upload(data []byte) *domain.Attachment {
e.t.Helper()
id := uuid.NewString()
sum := sha256.Sum256(data)
a := &domain.Attachment{ID: id, ChatID: e.chat, OwnerID: e.owner, Filename: "f.png",
ObjectKey: "s/" + e.chat + "/" + id, CreatedAt: time.Now(), UpdatedAt: time.Now(),
ContentType: "image/png", SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(sum[:])}
if err := e.att.CreateIntent(e.ctx, a); err != nil {
e.t.Fatal(err)
}
_ = e.store.Put(e.ctx, a.ObjectKey, bytes.NewReader(data), int64(len(data)), "image/png")
if _, err := e.att.MarkQueued(e.ctx, a, 1, 3); err != nil {
e.t.Fatal(err)
}
return a
}
// relay — опубликовать созревшие сообщения outbox; вернуть их тела.
func (e *env) relay(afterPublish func()) [][]byte {
e.t.Helper()
var out [][]byte
_, err := e.repo.PublishDue(e.ctx, 100, func(_ context.Context, m postgres.OutboxMessage) error {
out = append(out, m.Payload)
return nil
}, afterPublish)
if err != nil {
e.t.Fatal(err)
}
return out
}
func (e *env) worker(lease time.Duration, faults ...string) *worker.Worker {
pts := map[string]bool{}
for _, f := range faults {
pts[f] = true
}
w := worker.New(e.repo, e.store, fakeWebP, worker.Config{
Owner: "w-" + uuid.NewString()[:6], Lease: lease, JobTimeout: lease,
MaxBytes: 10 << 20, MaxPixels: 40_000_000, PixelBudget: 60_000_000, WebPQuality: 80,
RetryBase: 200 * time.Millisecond, RetryMax: 400 * time.Millisecond,
FaultPoints: pts, FaultDir: e.t.TempDir(), DeleteStaging: true,
})
worker.SetExit(w, func(code int) { panic(crash{code}) })
return w
}
// handle — доставка с перехватом «падения процесса».
func handle(w *worker.Worker, body []byte) (d worker.Decision, crashed bool) {
defer func() {
if r := recover(); r != nil {
if _, ok := r.(crash); !ok {
panic(r)
}
crashed = true
}
}()
return w.Handle(context.Background(), body), false
}
type state struct {
Status, ErrorCode, JobState string
Version, Attempt, Variants int
}
func (e *env) state(a *domain.Attachment) state {
var s state
err := e.pool.QueryRow(e.ctx, `
SELECT a.status, COALESCE(a.error_code, ''), a.pipeline_version,
(SELECT count(*) FROM attachment_variants v WHERE v.attachment_id = a.id),
COALESCE((SELECT state FROM jobs j WHERE j.attachment_id = a.id ORDER BY pipeline_version DESC LIMIT 1), ''),
COALESCE((SELECT attempt FROM jobs j WHERE j.attachment_id = a.id ORDER BY pipeline_version DESC LIMIT 1), 0)
FROM attachments a WHERE a.id = $1`, a.ID).
Scan(&s.Status, &s.ErrorCode, &s.Version, &s.Variants, &s.JobState, &s.Attempt)
if err != nil {
e.t.Fatal(err)
}
return s
}
// checkInvariants — инварианты pipeline после любого шага:
//
// I1 не больше одного варианта на (attachment, version, name) — PK, проверяем явно;
// I2 ready ⇒ опубликованная версия ≥ 1, варианты этой версии полные (3) и
// объекты в S3 совпадают с метаданными (размер и SHA-256);
// I3 ready ⇒ оригинал в конечном prefix, байты равны принятым (SHA-256 из 202);
// I4 нет processing с истёкшим lease после sweeper (проверяется в сценариях);
// I5 опубликованная версия никогда не уменьшается (проверяется по истории в сценариях).
func (e *env) checkInvariants(a *domain.Attachment) {
e.t.Helper()
if dup := e.scalar(`SELECT count(*) FROM (SELECT 1 FROM attachment_variants WHERE attachment_id = $1
GROUP BY pipeline_version, name HAVING count(*) > 1) d`, a.ID); dup != "0" {
e.t.Fatalf("I1: duplicated variants: %s", dup)
}
s := e.state(a)
if s.Status != "ready" {
return
}
if s.Version < 1 {
e.t.Fatalf("I2: ready without published version")
}
rows, err := e.pool.Query(e.ctx, `SELECT object_key, size_bytes, sha256 FROM attachment_variants
WHERE attachment_id = $1 AND pipeline_version = $2`, a.ID, s.Version)
if err != nil {
e.t.Fatal(err)
}
n := 0
for rows.Next() {
var key, sum string
var size int64
_ = rows.Scan(&key, &size, &sum)
b, ok := e.store.get(key)
got := sha256.Sum256(b)
if !ok || int64(len(b)) != size || hex.EncodeToString(got[:]) != sum {
e.t.Fatalf("I2: variant object %s does not match metadata", key)
}
n++
}
rows.Close()
if n != 3 {
e.t.Fatalf("I2: ready with %d variants of version %d", n, s.Version)
}
key := e.scalar(`SELECT object_key FROM attachments WHERE id = $1`, a.ID)
b, ok := e.store.get(key)
got := sha256.Sum256(b)
if !strings.HasPrefix(key, "o/") || !ok || hex.EncodeToString(got[:]) != a.SHA256 {
e.t.Fatalf("I3: published original %s missing or changed", key)
}
}
func (e *env) events(a *domain.Attachment) string {
rows, _ := e.pool.Query(e.ctx, `SELECT event FROM job_events e JOIN jobs j USING (job_id)
WHERE j.attachment_id = $1 ORDER BY e.id`, a.ID)
defer rows.Close()
var ev []string
for rows.Next() {
var s string
_ = rows.Scan(&s)
ev = append(ev, s)
}
return strings.Join(ev, ",")
}
func expect(t *testing.T, cond bool, format string, args ...any) {
t.Helper()
if !cond {
t.Fatalf(format, args...)
}
}
// ---------- сценарии ----------
func TestPipelineHappyPathAndDuplicate(t *testing.T) {
e := setup(t)
a := e.upload(pngBytes(320, 200, 1))
msgs := e.relay(nil)
expect(t, len(msgs) == 1, "outbox messages: %d", len(msgs))
w := e.worker(5 * time.Second)
d, _ := handle(w, msgs[0])
expect(t, d == worker.Ack, "decision %v", d)
s := e.state(a)
expect(t, s.Status == "ready" && s.Version == 1 && s.Variants == 3 && s.JobState == "done", "state %+v", s)
e.checkInvariants(a)
_, staged := e.store.get("s/" + e.chat + "/" + a.ID)
expect(t, !staged, "staging must be removed after publish")
puts := len(e.store.puts)
// Дубль после done (повторная доставка того же сообщения).
d, _ = handle(w, msgs[0])
expect(t, d == worker.Ack && len(e.store.puts) == puts, "duplicate must be a no-op ack")
expect(t, e.state(a).Attempt == 1, "duplicate must not start a new attempt")
e.checkInvariants(a)
}
func TestPipelineConcurrentDuplicateIsBusy(t *testing.T) {
e := setup(t)
a := e.upload(pngBytes(64, 64, 2))
msg := e.relay(nil)[0]
// Первый worker захватил lease и «работает» (имитация: claim без завершения).
out, _, _, err := e.repo.Claim(e.ctx, e.scalar(`SELECT job_id::text FROM jobs WHERE attachment_id=$1`, a.ID), "slow-worker", time.Minute)
expect(t, err == nil && out == postgres.Claimed, "claim %v %v", out, err)
d, _ := handle(e.worker(time.Minute), msg)
expect(t, d == worker.Ack, "duplicate during valid lease must be acked")
expect(t, e.state(a).Attempt == 1 && e.state(a).Variants == 0, "duplicate must not process: %+v", e.state(a))
}
func TestPipelineCrashAfterSideEffectBeforeCommit(t *testing.T) {
e := setup(t)
a := e.upload(pngBytes(300, 240, 3))
msg := e.relay(nil)[0]
_, crashed := handle(e.worker(time.Second, "worker.after_side_effect"), msg)
expect(t, crashed, "fault must crash the worker")
s := e.state(a)
expect(t, s.Status == "processing" && s.Variants == 0, "after crash: %+v", s)
puts := len(e.store.puts)
expect(t, puts >= 4, "side effects (3 variants + original copy) written before crash: %d", puts)
// Перезапущенный broker доставит то же сообщение снова, но lease ещё жив → ack дубля;
// восстановление делает sweeper после истечения lease.
d, _ := handle(e.worker(time.Second), msg)
expect(t, d == worker.Ack && e.state(a).Variants == 0, "redelivery during lease: %+v", e.state(a))
time.Sleep(1100 * time.Millisecond)
n, err := e.repo.SweepExpiredLeases(e.ctx)
expect(t, err == nil && n == 1, "sweep %d %v", n, err)
msgs := e.relay(nil)
expect(t, len(msgs) == 1, "requeued message: %d", len(msgs))
d, _ = handle(e.worker(time.Second), msgs[0])
s = e.state(a)
expect(t, d == worker.Ack && s.Status == "ready" && s.Attempt == 2 && s.Variants == 3, "after recovery: %+v", s)
// Повтор перезаписал те же ключи, не создал новых объектов.
keys := 0
for k := range e.store.puts {
if strings.HasPrefix(k, "v/") {
keys++
}
}
expect(t, keys == 3, "variant object keys: %d (must stay 3)", keys)
e.checkInvariants(a)
expect(t, strings.Contains(e.events(a), "lease_expired_requeued"), "events: %s", e.events(a))
}
func TestPipelineCrashAfterCommitBeforeAck(t *testing.T) {
e := setup(t)
a := e.upload(pngBytes(100, 100, 4))
msg := e.relay(nil)[0]
_, crashed := handle(e.worker(5*time.Second, "worker.after_commit"), msg)
expect(t, crashed && e.state(a).Status == "ready", "commit before crash: %+v", e.state(a))
puts := len(e.store.puts)
d, _ := handle(e.worker(5*time.Second), msg) // broker доставляет снова
expect(t, d == worker.Ack && len(e.store.puts) == puts && e.state(a).Attempt == 1, "redelivery after commit must be no-op")
e.checkInvariants(a)
}
func TestPipelineRelayCrashAfterPublishDuplicates(t *testing.T) {
e := setup(t)
a := e.upload(pngBytes(80, 60, 5))
var first [][]byte
func() {
defer func() { _ = recover() }()
first = e.relay(func() { panic(crash{137}) }) // confirm получен, COMMIT нет
}()
again := e.relay(nil) // outbox не помечен → публикуется ещё раз
expect(t, len(again) == 1, "message must be republished after relay crash")
w := e.worker(5 * time.Second)
handle(w, again[0])
handle(w, again[0]) // и ещё дубль
s := e.state(a)
expect(t, s.Status == "ready" && s.Attempt == 1 && s.Variants == 3, "duplicate publish: %+v", s)
e.checkInvariants(a)
_ = first
}
func TestPipelineVersionReorderAndStaleJob(t *testing.T) {
e := setup(t)
a := e.upload(pngBytes(320, 200, 6))
v1msg := e.relay(nil)[0]
// Reorder: v2 поставлена и обработана РАНЬШЕ, чем дошла v1.
_, err := e.repo.Reprocess(e.ctx, a.ID, 2, 3)
expect(t, err == nil, "reprocess: %v", err)
v2msg := e.relay(nil)[0]
handle(e.worker(5*time.Second), v2msg)
s := e.state(a)
expect(t, s.Status == "ready" && s.Version == 2, "v2 published: %+v", s)
thumbV2 := e.scalar(`SELECT sha256 FROM attachment_variants WHERE attachment_id=$1 AND pipeline_version=2 AND name='thumbnail'`, a.ID)
// Устаревшая v1 приходит позже: не должна перезаписать v2.
d, _ := handle(e.worker(5*time.Second), v1msg)
s = e.state(a)
expect(t, d == worker.Ack && s.Version == 2, "stale v1 overwrote v2: %+v", s)
expect(t, e.scalar(`SELECT count(*) FROM attachment_variants WHERE attachment_id=$1 AND pipeline_version=1`, a.ID) == "0",
"stale v1 must be a no-op when v2 is already published")
e.checkInvariants(a)
// Гонка: v1 УЖЕ обрабатывалась (lease), v3 успела опубликоваться, v1 фиксируется позже.
b := e.upload(pngBytes(200, 100, 7))
b1 := e.relay(nil)[0]
jobV1 := e.scalar(`SELECT job_id::text FROM jobs WHERE attachment_id=$1 AND pipeline_version=1`, b.ID)
out, j1, _, _ := e.repo.Claim(e.ctx, jobV1, "slow", time.Minute)
expect(t, out == postgres.Claimed, "claim v1")
_, _ = e.repo.Reprocess(e.ctx, b.ID, 3, 3)
handle(e.worker(5*time.Second), e.relay(nil)[0])
expect(t, e.state(b).Version == 3, "v3 published")
res, err := e.repo.Complete(e.ctx, jobV1, "slow", j1.Attempt, "o/"+e.chat+"/"+b.ID, []domain.Variant{
{Name: "thumbnail", ContentType: "image/jpeg", SizeBytes: 1, Width: 1, Height: 1, SHA256: strings.Repeat("a", 64), ObjectKey: "v/x/p1/thumbnail.jpg"}})
expect(t, err == nil && res.Applied && !res.Published && res.NewerPublished == 3, "late v1 commit: %+v %v", res, err)
expect(t, e.state(b).Version == 3, "late v1 commit downgraded version")
_ = b1
expect(t, e.scalar(`SELECT sha256 FROM attachment_variants WHERE attachment_id=$1 AND pipeline_version=2 AND name='thumbnail'`, a.ID) == thumbV2,
"v2 result changed")
}
func TestPipelineLostLeaseFencing(t *testing.T) {
e := setup(t)
a := e.upload(pngBytes(90, 90, 8))
jobID := e.scalar(`SELECT job_id::text FROM jobs WHERE attachment_id=$1`, a.ID)
e.relay(nil)
_, ja, _, _ := e.repo.Claim(e.ctx, jobID, "A", 300*time.Millisecond) // A «завис»
time.Sleep(400 * time.Millisecond)
_, _ = e.repo.SweepExpiredLeases(e.ctx)
handle(e.worker(5*time.Second), e.relay(nil)[0]) // B завершает попытку 2
expect(t, e.state(a).Status == "ready" && e.state(a).Attempt == 2, "B finished: %+v", e.state(a))
res, err := e.repo.Complete(e.ctx, jobID, "A", ja.Attempt, "o/zz", nil) // A «проснулся»
expect(t, err == nil && res.LostLease, "zombie A must be fenced: %+v %v", res, err)
e.checkInvariants(a)
}
func TestPipelineTransientRetryWithBackoff(t *testing.T) {
e := setup(t)
a := e.upload(pngBytes(120, 80, 9))
msg := e.relay(nil)[0]
e.store.failPut = 1
d, _ := handle(e.worker(5*time.Second), msg)
s := e.state(a)
expect(t, d == worker.Ack && s.Status == "queued" && s.JobState == "queued", "retry scheduled: %+v", s)
expect(t, len(e.relay(nil)) == 0, "retry must wait for backoff")
d, _ = handle(e.worker(5*time.Second), msg) // ранняя повторная доставка
expect(t, d == worker.Ack && e.state(a).Attempt == 1, "early delivery must not run: %+v", e.state(a))
time.Sleep(500 * time.Millisecond)
msgs := e.relay(nil)
var job domain.Job
expect(t, len(msgs) == 1 && json.Unmarshal(msgs[0], &job) == nil && job.Attempt == 2, "retry message attempt 2: %s", msgs)
handle(e.worker(5*time.Second), msgs[0])
expect(t, e.state(a).Status == "ready" && e.state(a).Attempt == 2, "after retry: %+v", e.state(a))
e.checkInvariants(a)
}
func TestPipelinePermanentFailureAndReplay(t *testing.T) {
e := setup(t)
good := pngBytes(50, 40, 10)
a := e.upload(good)
// Порча staging-объекта: checksum не совпадёт → постоянная ошибка, без повторов.
key := "s/" + e.chat + "/" + a.ID
_ = e.store.Put(e.ctx, key, bytes.NewReader([]byte("garbage")), 7, "image/png")
handle(e.worker(5*time.Second), e.relay(nil)[0])
s := e.state(a)
expect(t, s.Status == "failed" && s.ErrorCode == "checksum_mismatch" && s.Attempt == 1, "permanent: %+v", s)
// Replay успешной работы запрещён; failed — разрешён (с аудитом).
_, err := e.repo.Replay(e.ctx, a.ID, 1, "tester")
expect(t, err == nil, "replay failed job: %v", err)
_ = e.store.Put(e.ctx, key, bytes.NewReader(good), int64(len(good)), "image/png") // исходник восстановлен
handle(e.worker(5*time.Second), e.relay(nil)[0])
expect(t, e.state(a).Status == "ready", "after replay: %+v", e.state(a))
_, err = e.repo.Replay(e.ctx, a.ID, 1, "tester")
expect(t, err != nil, "replay of done job must be refused")
expect(t, strings.Contains(e.events(a), "replay,") && strings.Contains(e.events(a), "replay_refused"), "audit: %s", e.events(a))
e.checkInvariants(a)
}
func TestPipelineRetryExhausted(t *testing.T) {
e := setup(t)
a := e.upload(pngBytes(30, 30, 11))
e.store.failPut = 100
msg := e.relay(nil)[0]
for i := 0; i < 3; i++ {
handle(e.worker(5*time.Second), msg)
time.Sleep(450 * time.Millisecond)
if m := e.relay(nil); len(m) > 0 {
msg = m[0]
}
}
s := e.state(a)
expect(t, s.Status == "failed" && s.ErrorCode == "retry_exhausted" && s.Attempt == 3, "exhausted: %+v", s)
jobs, _ := e.repo.FailedJobs(e.ctx, 10)
expect(t, len(jobs) == 1, "failed registry: %d", len(jobs))
}