582 lines
21 KiB
Go
582 lines
21 KiB
Go
package worker_test
|
||
|
||
// Интеграционный fault-injection (трек 5): настоящий PostgreSQL со всеми
|
||
// миграциями, S3 — в памяти, broker — вызовы Handle с телом из outbox (так же,
|
||
// как его опубликовал бы relay). Запуск:
|
||
//
|
||
// TEST_DATABASE_URL=postgres://user@host:5432/postgres go test ./internal/worker/ -run Pipeline -v
|
||
//
|
||
// Для каждого сценария создаётся отдельная база, после каждого шага проверяются
|
||
// инварианты (checkInvariants).
|
||
|
||
import (
|
||
"bytes"
|
||
"context"
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"image"
|
||
"image/color"
|
||
"image/png"
|
||
"io"
|
||
"os"
|
||
"path/filepath"
|
||
"sort"
|
||
"strings"
|
||
"sync"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/google/uuid"
|
||
"github.com/jackc/pgx/v5"
|
||
"github.com/jackc/pgx/v5/pgxpool"
|
||
|
||
"github.com/charawein/mtusi2026lab6/internal/domain"
|
||
"github.com/charawein/mtusi2026lab6/internal/repository/postgres"
|
||
"github.com/charawein/mtusi2026lab6/internal/storage"
|
||
"github.com/charawein/mtusi2026lab6/internal/worker"
|
||
)
|
||
|
||
// ---------- in-memory S3 ----------
|
||
|
||
type memStore struct {
|
||
mu sync.Mutex
|
||
objs map[string][]byte
|
||
puts map[string]int
|
||
failPut int // сколько ближайших Put завершить временной ошибкой
|
||
}
|
||
|
||
func newMemStore() *memStore { return &memStore{objs: map[string][]byte{}, puts: map[string]int{}} }
|
||
|
||
func (m *memStore) Put(_ context.Context, key string, r io.Reader, size int64, _ string) error {
|
||
m.mu.Lock()
|
||
defer m.mu.Unlock()
|
||
if m.failPut > 0 {
|
||
m.failPut--
|
||
return errors.New("s3 put: 503 slow down")
|
||
}
|
||
b, err := io.ReadAll(r)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
m.objs[key] = b
|
||
m.puts[key]++
|
||
return nil
|
||
}
|
||
func (m *memStore) Get(_ context.Context, key string) (io.ReadCloser, int64, error) {
|
||
m.mu.Lock()
|
||
defer m.mu.Unlock()
|
||
b, ok := m.objs[key]
|
||
if !ok {
|
||
return nil, 0, storage.ErrNotFound
|
||
}
|
||
return io.NopCloser(bytes.NewReader(b)), int64(len(b)), nil
|
||
}
|
||
func (m *memStore) Stat(_ context.Context, key string) (storage.Object, error) {
|
||
m.mu.Lock()
|
||
defer m.mu.Unlock()
|
||
b, ok := m.objs[key]
|
||
if !ok {
|
||
return storage.Object{}, storage.ErrNotFound
|
||
}
|
||
return storage.Object{Key: key, Size: int64(len(b))}, nil
|
||
}
|
||
func (m *memStore) Delete(_ context.Context, key string) error {
|
||
m.mu.Lock()
|
||
defer m.mu.Unlock()
|
||
delete(m.objs, key)
|
||
return nil
|
||
}
|
||
func (m *memStore) List(context.Context, string, func(storage.Object) error) error { return nil }
|
||
func (m *memStore) Copy(_ context.Context, src, dst string) error {
|
||
m.mu.Lock()
|
||
defer m.mu.Unlock()
|
||
b, ok := m.objs[src]
|
||
if !ok {
|
||
return storage.ErrNotFound
|
||
}
|
||
m.objs[dst] = bytes.Clone(b)
|
||
m.puts[dst]++
|
||
return nil
|
||
}
|
||
func (m *memStore) get(key string) ([]byte, bool) {
|
||
m.mu.Lock()
|
||
defer m.mu.Unlock()
|
||
b, ok := m.objs[key]
|
||
return b, ok
|
||
}
|
||
|
||
// Детерминированный «WebP» для тестов: RIFF/WEBP-заголовок + хэш пикселей.
|
||
func fakeWebP(_ context.Context, img image.Image, q int) ([]byte, error) {
|
||
h := sha256.New()
|
||
b := img.Bounds()
|
||
fmt.Fprintf(h, "%d:%d:%d", b.Dx(), b.Dy(), q)
|
||
for y := b.Min.Y; y < b.Max.Y; y += 7 {
|
||
for x := b.Min.X; x < b.Max.X; x += 7 {
|
||
r, g, bb, _ := img.At(x, y).RGBA()
|
||
fmt.Fprintf(h, "%d%d%d", r, g, bb)
|
||
}
|
||
}
|
||
return append([]byte("RIFF\x00\x00\x00\x00WEBP"), h.Sum(nil)...), nil
|
||
}
|
||
|
||
// ---------- окружение ----------
|
||
|
||
type env struct {
|
||
t *testing.T
|
||
ctx context.Context
|
||
pool *pgxpool.Pool
|
||
repo *postgres.PipelineRepository
|
||
att *postgres.AttachmentRepository
|
||
store *memStore
|
||
chat string
|
||
owner string
|
||
}
|
||
|
||
type crash struct{ code int }
|
||
|
||
func setup(t *testing.T) *env {
|
||
t.Helper()
|
||
admin := os.Getenv("TEST_DATABASE_URL")
|
||
if admin == "" {
|
||
t.Skip("TEST_DATABASE_URL not set (integration test needs PostgreSQL)")
|
||
}
|
||
ctx := context.Background()
|
||
conn, err := pgx.Connect(ctx, admin)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
db := "lab5_it_" + strings.ReplaceAll(uuid.NewString()[:8], "-", "")
|
||
if _, err := conn.Exec(ctx, "CREATE DATABASE "+db); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
t.Cleanup(func() {
|
||
_, _ = conn.Exec(context.Background(), "DROP DATABASE IF EXISTS "+db+" WITH (FORCE)")
|
||
conn.Close(context.Background())
|
||
})
|
||
cfg, err := pgxpool.ParseConfig(admin)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
cfg.ConnConfig.Database = db
|
||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||
if err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
t.Cleanup(pool.Close)
|
||
files, _ := filepath.Glob("../../migrations/*.up.sql")
|
||
sort.Strings(files)
|
||
for _, f := range files {
|
||
sql, _ := os.ReadFile(f)
|
||
if _, err := pool.Exec(ctx, string(sql)); err != nil {
|
||
t.Fatalf("%s: %v", f, err)
|
||
}
|
||
}
|
||
e := &env{t: t, ctx: ctx, pool: pool, repo: postgres.NewPipelineRepository(pool),
|
||
att: postgres.NewAttachmentRepository(pool), store: newMemStore(), owner: uuid.NewString()}
|
||
e.chat = uuid.NewString()
|
||
e.exec(`INSERT INTO users (id, username, display_name, password_hash) VALUES ($1::uuid, $2, 'U', 'x')`, e.owner, "u_"+e.owner[:8])
|
||
e.exec(`INSERT INTO chats (id, title) VALUES ($1, 'c')`, e.chat)
|
||
e.exec(`INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)`, e.chat, e.owner)
|
||
return e
|
||
}
|
||
|
||
func (e *env) exec(sql string, args ...any) {
|
||
e.t.Helper()
|
||
if _, err := e.pool.Exec(e.ctx, sql, args...); err != nil {
|
||
e.t.Fatalf("%s: %v", sql, err)
|
||
}
|
||
}
|
||
|
||
func (e *env) scalar(sql string, args ...any) string {
|
||
e.t.Helper()
|
||
var s string
|
||
if err := e.pool.QueryRow(e.ctx, sql, args...).Scan(&s); err != nil {
|
||
e.t.Fatalf("%s: %v", sql, err)
|
||
}
|
||
return s
|
||
}
|
||
|
||
func pngBytes(w, h int, seed uint8) []byte {
|
||
img := image.NewRGBA(image.Rect(0, 0, w, h))
|
||
for y := 0; y < h; y++ {
|
||
for x := 0; x < w; x++ {
|
||
img.Set(x, y, color.RGBA{uint8(x) + seed, uint8(y), seed, 255})
|
||
}
|
||
}
|
||
var b bytes.Buffer
|
||
_ = png.Encode(&b, img)
|
||
return b.Bytes()
|
||
}
|
||
|
||
// upload — как API: намерение, staging-объект, MarkQueued (jobs + outbox).
|
||
func (e *env) upload(data []byte) *domain.Attachment {
|
||
e.t.Helper()
|
||
id := uuid.NewString()
|
||
sum := sha256.Sum256(data)
|
||
a := &domain.Attachment{ID: id, ChatID: e.chat, OwnerID: e.owner, Filename: "f.png",
|
||
ObjectKey: "s/" + e.chat + "/" + id, CreatedAt: time.Now(), UpdatedAt: time.Now(),
|
||
ContentType: "image/png", SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(sum[:])}
|
||
if err := e.att.CreateIntent(e.ctx, a); err != nil {
|
||
e.t.Fatal(err)
|
||
}
|
||
_ = e.store.Put(e.ctx, a.ObjectKey, bytes.NewReader(data), int64(len(data)), "image/png")
|
||
if _, err := e.att.MarkQueued(e.ctx, a, 1, 3); err != nil {
|
||
e.t.Fatal(err)
|
||
}
|
||
return a
|
||
}
|
||
|
||
// relay — опубликовать созревшие сообщения outbox; вернуть их тела.
|
||
func (e *env) relay(afterPublish func()) [][]byte {
|
||
e.t.Helper()
|
||
var out [][]byte
|
||
_, err := e.repo.PublishDue(e.ctx, 100, func(_ context.Context, m postgres.OutboxMessage) error {
|
||
out = append(out, m.Payload)
|
||
return nil
|
||
}, afterPublish)
|
||
if err != nil {
|
||
e.t.Fatal(err)
|
||
}
|
||
return out
|
||
}
|
||
|
||
func (e *env) worker(lease time.Duration, faults ...string) *worker.Worker {
|
||
pts := map[string]bool{}
|
||
for _, f := range faults {
|
||
pts[f] = true
|
||
}
|
||
w := worker.New(e.repo, e.store, fakeWebP, worker.Config{
|
||
Owner: "w-" + uuid.NewString()[:6], Lease: lease, JobTimeout: lease,
|
||
MaxBytes: 10 << 20, MaxPixels: 40_000_000, PixelBudget: 60_000_000, WebPQuality: 80,
|
||
RetryBase: 200 * time.Millisecond, RetryMax: 400 * time.Millisecond,
|
||
FaultPoints: pts, FaultDir: e.t.TempDir(), DeleteStaging: true,
|
||
})
|
||
worker.SetExit(w, func(code int) { panic(crash{code}) })
|
||
return w
|
||
}
|
||
|
||
// handle — доставка с перехватом «падения процесса».
|
||
func handle(w *worker.Worker, body []byte) (d worker.Decision, crashed bool) {
|
||
defer func() {
|
||
if r := recover(); r != nil {
|
||
if _, ok := r.(crash); !ok {
|
||
panic(r)
|
||
}
|
||
crashed = true
|
||
}
|
||
}()
|
||
return w.Handle(context.Background(), body), false
|
||
}
|
||
|
||
type state struct {
|
||
Status, ErrorCode, JobState string
|
||
Version, Attempt, Variants int
|
||
}
|
||
|
||
func (e *env) state(a *domain.Attachment) state {
|
||
var s state
|
||
err := e.pool.QueryRow(e.ctx, `
|
||
SELECT a.status, COALESCE(a.error_code, ''), a.pipeline_version,
|
||
(SELECT count(*) FROM attachment_variants v WHERE v.attachment_id = a.id),
|
||
COALESCE((SELECT state FROM jobs j WHERE j.attachment_id = a.id ORDER BY pipeline_version DESC LIMIT 1), ''),
|
||
COALESCE((SELECT attempt FROM jobs j WHERE j.attachment_id = a.id ORDER BY pipeline_version DESC LIMIT 1), 0)
|
||
FROM attachments a WHERE a.id = $1`, a.ID).
|
||
Scan(&s.Status, &s.ErrorCode, &s.Version, &s.Variants, &s.JobState, &s.Attempt)
|
||
if err != nil {
|
||
e.t.Fatal(err)
|
||
}
|
||
return s
|
||
}
|
||
|
||
// checkInvariants — инварианты pipeline после любого шага:
|
||
//
|
||
// I1 не больше одного варианта на (attachment, version, name) — PK, проверяем явно;
|
||
// I2 ready ⇒ опубликованная версия ≥ 1, варианты этой версии полные (3) и
|
||
// объекты в S3 совпадают с метаданными (размер и SHA-256);
|
||
// I3 ready ⇒ оригинал в конечном prefix, байты равны принятым (SHA-256 из 202);
|
||
// I4 нет processing с истёкшим lease после sweeper (проверяется в сценариях);
|
||
// I5 опубликованная версия никогда не уменьшается (проверяется по истории в сценариях).
|
||
func (e *env) checkInvariants(a *domain.Attachment) {
|
||
e.t.Helper()
|
||
if dup := e.scalar(`SELECT count(*) FROM (SELECT 1 FROM attachment_variants WHERE attachment_id = $1
|
||
GROUP BY pipeline_version, name HAVING count(*) > 1) d`, a.ID); dup != "0" {
|
||
e.t.Fatalf("I1: duplicated variants: %s", dup)
|
||
}
|
||
s := e.state(a)
|
||
if s.Status != "ready" {
|
||
return
|
||
}
|
||
if s.Version < 1 {
|
||
e.t.Fatalf("I2: ready without published version")
|
||
}
|
||
rows, err := e.pool.Query(e.ctx, `SELECT object_key, size_bytes, sha256 FROM attachment_variants
|
||
WHERE attachment_id = $1 AND pipeline_version = $2`, a.ID, s.Version)
|
||
if err != nil {
|
||
e.t.Fatal(err)
|
||
}
|
||
n := 0
|
||
for rows.Next() {
|
||
var key, sum string
|
||
var size int64
|
||
_ = rows.Scan(&key, &size, &sum)
|
||
b, ok := e.store.get(key)
|
||
got := sha256.Sum256(b)
|
||
if !ok || int64(len(b)) != size || hex.EncodeToString(got[:]) != sum {
|
||
e.t.Fatalf("I2: variant object %s does not match metadata", key)
|
||
}
|
||
n++
|
||
}
|
||
rows.Close()
|
||
if n != 3 {
|
||
e.t.Fatalf("I2: ready with %d variants of version %d", n, s.Version)
|
||
}
|
||
key := e.scalar(`SELECT object_key FROM attachments WHERE id = $1`, a.ID)
|
||
b, ok := e.store.get(key)
|
||
got := sha256.Sum256(b)
|
||
if !strings.HasPrefix(key, "o/") || !ok || hex.EncodeToString(got[:]) != a.SHA256 {
|
||
e.t.Fatalf("I3: published original %s missing or changed", key)
|
||
}
|
||
}
|
||
|
||
func (e *env) events(a *domain.Attachment) string {
|
||
rows, _ := e.pool.Query(e.ctx, `SELECT event FROM job_events e JOIN jobs j USING (job_id)
|
||
WHERE j.attachment_id = $1 ORDER BY e.id`, a.ID)
|
||
defer rows.Close()
|
||
var ev []string
|
||
for rows.Next() {
|
||
var s string
|
||
_ = rows.Scan(&s)
|
||
ev = append(ev, s)
|
||
}
|
||
return strings.Join(ev, ",")
|
||
}
|
||
|
||
func expect(t *testing.T, cond bool, format string, args ...any) {
|
||
t.Helper()
|
||
if !cond {
|
||
t.Fatalf(format, args...)
|
||
}
|
||
}
|
||
|
||
// ---------- сценарии ----------
|
||
|
||
func TestPipelineHappyPathAndDuplicate(t *testing.T) {
|
||
e := setup(t)
|
||
a := e.upload(pngBytes(320, 200, 1))
|
||
msgs := e.relay(nil)
|
||
expect(t, len(msgs) == 1, "outbox messages: %d", len(msgs))
|
||
w := e.worker(5 * time.Second)
|
||
|
||
d, _ := handle(w, msgs[0])
|
||
expect(t, d == worker.Ack, "decision %v", d)
|
||
s := e.state(a)
|
||
expect(t, s.Status == "ready" && s.Version == 1 && s.Variants == 3 && s.JobState == "done", "state %+v", s)
|
||
e.checkInvariants(a)
|
||
_, staged := e.store.get("s/" + e.chat + "/" + a.ID)
|
||
expect(t, !staged, "staging must be removed after publish")
|
||
|
||
puts := len(e.store.puts)
|
||
// Дубль после done (повторная доставка того же сообщения).
|
||
d, _ = handle(w, msgs[0])
|
||
expect(t, d == worker.Ack && len(e.store.puts) == puts, "duplicate must be a no-op ack")
|
||
expect(t, e.state(a).Attempt == 1, "duplicate must not start a new attempt")
|
||
e.checkInvariants(a)
|
||
}
|
||
|
||
func TestPipelineConcurrentDuplicateIsBusy(t *testing.T) {
|
||
e := setup(t)
|
||
a := e.upload(pngBytes(64, 64, 2))
|
||
msg := e.relay(nil)[0]
|
||
// Первый worker захватил lease и «работает» (имитация: claim без завершения).
|
||
out, _, _, err := e.repo.Claim(e.ctx, e.scalar(`SELECT job_id::text FROM jobs WHERE attachment_id=$1`, a.ID), "slow-worker", time.Minute)
|
||
expect(t, err == nil && out == postgres.Claimed, "claim %v %v", out, err)
|
||
d, _ := handle(e.worker(time.Minute), msg)
|
||
expect(t, d == worker.Ack, "duplicate during valid lease must be acked")
|
||
expect(t, e.state(a).Attempt == 1 && e.state(a).Variants == 0, "duplicate must not process: %+v", e.state(a))
|
||
}
|
||
|
||
func TestPipelineCrashAfterSideEffectBeforeCommit(t *testing.T) {
|
||
e := setup(t)
|
||
a := e.upload(pngBytes(300, 240, 3))
|
||
msg := e.relay(nil)[0]
|
||
_, crashed := handle(e.worker(time.Second, "worker.after_side_effect"), msg)
|
||
expect(t, crashed, "fault must crash the worker")
|
||
s := e.state(a)
|
||
expect(t, s.Status == "processing" && s.Variants == 0, "after crash: %+v", s)
|
||
puts := len(e.store.puts)
|
||
expect(t, puts >= 4, "side effects (3 variants + original copy) written before crash: %d", puts)
|
||
|
||
// Перезапущенный broker доставит то же сообщение снова, но lease ещё жив → ack дубля;
|
||
// восстановление делает sweeper после истечения lease.
|
||
d, _ := handle(e.worker(time.Second), msg)
|
||
expect(t, d == worker.Ack && e.state(a).Variants == 0, "redelivery during lease: %+v", e.state(a))
|
||
time.Sleep(1100 * time.Millisecond)
|
||
n, err := e.repo.SweepExpiredLeases(e.ctx)
|
||
expect(t, err == nil && n == 1, "sweep %d %v", n, err)
|
||
msgs := e.relay(nil)
|
||
expect(t, len(msgs) == 1, "requeued message: %d", len(msgs))
|
||
d, _ = handle(e.worker(time.Second), msgs[0])
|
||
s = e.state(a)
|
||
expect(t, d == worker.Ack && s.Status == "ready" && s.Attempt == 2 && s.Variants == 3, "after recovery: %+v", s)
|
||
// Повтор перезаписал те же ключи, не создал новых объектов.
|
||
keys := 0
|
||
for k := range e.store.puts {
|
||
if strings.HasPrefix(k, "v/") {
|
||
keys++
|
||
}
|
||
}
|
||
expect(t, keys == 3, "variant object keys: %d (must stay 3)", keys)
|
||
e.checkInvariants(a)
|
||
expect(t, strings.Contains(e.events(a), "lease_expired_requeued"), "events: %s", e.events(a))
|
||
}
|
||
|
||
func TestPipelineCrashAfterCommitBeforeAck(t *testing.T) {
|
||
e := setup(t)
|
||
a := e.upload(pngBytes(100, 100, 4))
|
||
msg := e.relay(nil)[0]
|
||
_, crashed := handle(e.worker(5*time.Second, "worker.after_commit"), msg)
|
||
expect(t, crashed && e.state(a).Status == "ready", "commit before crash: %+v", e.state(a))
|
||
puts := len(e.store.puts)
|
||
d, _ := handle(e.worker(5*time.Second), msg) // broker доставляет снова
|
||
expect(t, d == worker.Ack && len(e.store.puts) == puts && e.state(a).Attempt == 1, "redelivery after commit must be no-op")
|
||
e.checkInvariants(a)
|
||
}
|
||
|
||
func TestPipelineRelayCrashAfterPublishDuplicates(t *testing.T) {
|
||
e := setup(t)
|
||
a := e.upload(pngBytes(80, 60, 5))
|
||
var first [][]byte
|
||
func() {
|
||
defer func() { _ = recover() }()
|
||
first = e.relay(func() { panic(crash{137}) }) // confirm получен, COMMIT нет
|
||
}()
|
||
again := e.relay(nil) // outbox не помечен → публикуется ещё раз
|
||
expect(t, len(again) == 1, "message must be republished after relay crash")
|
||
w := e.worker(5 * time.Second)
|
||
handle(w, again[0])
|
||
handle(w, again[0]) // и ещё дубль
|
||
s := e.state(a)
|
||
expect(t, s.Status == "ready" && s.Attempt == 1 && s.Variants == 3, "duplicate publish: %+v", s)
|
||
e.checkInvariants(a)
|
||
_ = first
|
||
}
|
||
|
||
func TestPipelineVersionReorderAndStaleJob(t *testing.T) {
|
||
e := setup(t)
|
||
a := e.upload(pngBytes(320, 200, 6))
|
||
v1msg := e.relay(nil)[0]
|
||
|
||
// Reorder: v2 поставлена и обработана РАНЬШЕ, чем дошла v1.
|
||
_, err := e.repo.Reprocess(e.ctx, a.ID, 2, 3)
|
||
expect(t, err == nil, "reprocess: %v", err)
|
||
v2msg := e.relay(nil)[0]
|
||
handle(e.worker(5*time.Second), v2msg)
|
||
s := e.state(a)
|
||
expect(t, s.Status == "ready" && s.Version == 2, "v2 published: %+v", s)
|
||
thumbV2 := e.scalar(`SELECT sha256 FROM attachment_variants WHERE attachment_id=$1 AND pipeline_version=2 AND name='thumbnail'`, a.ID)
|
||
|
||
// Устаревшая v1 приходит позже: не должна перезаписать v2.
|
||
d, _ := handle(e.worker(5*time.Second), v1msg)
|
||
s = e.state(a)
|
||
expect(t, d == worker.Ack && s.Version == 2, "stale v1 overwrote v2: %+v", s)
|
||
expect(t, e.scalar(`SELECT count(*) FROM attachment_variants WHERE attachment_id=$1 AND pipeline_version=1`, a.ID) == "0",
|
||
"stale v1 must be a no-op when v2 is already published")
|
||
e.checkInvariants(a)
|
||
|
||
// Гонка: v1 УЖЕ обрабатывалась (lease), v3 успела опубликоваться, v1 фиксируется позже.
|
||
b := e.upload(pngBytes(200, 100, 7))
|
||
b1 := e.relay(nil)[0]
|
||
jobV1 := e.scalar(`SELECT job_id::text FROM jobs WHERE attachment_id=$1 AND pipeline_version=1`, b.ID)
|
||
out, j1, _, _ := e.repo.Claim(e.ctx, jobV1, "slow", time.Minute)
|
||
expect(t, out == postgres.Claimed, "claim v1")
|
||
_, _ = e.repo.Reprocess(e.ctx, b.ID, 3, 3)
|
||
handle(e.worker(5*time.Second), e.relay(nil)[0])
|
||
expect(t, e.state(b).Version == 3, "v3 published")
|
||
res, err := e.repo.Complete(e.ctx, jobV1, "slow", j1.Attempt, "o/"+e.chat+"/"+b.ID, []domain.Variant{
|
||
{Name: "thumbnail", ContentType: "image/jpeg", SizeBytes: 1, Width: 1, Height: 1, SHA256: strings.Repeat("a", 64), ObjectKey: "v/x/p1/thumbnail.jpg"}})
|
||
expect(t, err == nil && res.Applied && !res.Published && res.NewerPublished == 3, "late v1 commit: %+v %v", res, err)
|
||
expect(t, e.state(b).Version == 3, "late v1 commit downgraded version")
|
||
_ = b1
|
||
expect(t, e.scalar(`SELECT sha256 FROM attachment_variants WHERE attachment_id=$1 AND pipeline_version=2 AND name='thumbnail'`, a.ID) == thumbV2,
|
||
"v2 result changed")
|
||
}
|
||
|
||
func TestPipelineLostLeaseFencing(t *testing.T) {
|
||
e := setup(t)
|
||
a := e.upload(pngBytes(90, 90, 8))
|
||
jobID := e.scalar(`SELECT job_id::text FROM jobs WHERE attachment_id=$1`, a.ID)
|
||
e.relay(nil)
|
||
_, ja, _, _ := e.repo.Claim(e.ctx, jobID, "A", 300*time.Millisecond) // A «завис»
|
||
time.Sleep(400 * time.Millisecond)
|
||
_, _ = e.repo.SweepExpiredLeases(e.ctx)
|
||
handle(e.worker(5*time.Second), e.relay(nil)[0]) // B завершает попытку 2
|
||
expect(t, e.state(a).Status == "ready" && e.state(a).Attempt == 2, "B finished: %+v", e.state(a))
|
||
res, err := e.repo.Complete(e.ctx, jobID, "A", ja.Attempt, "o/zz", nil) // A «проснулся»
|
||
expect(t, err == nil && res.LostLease, "zombie A must be fenced: %+v %v", res, err)
|
||
e.checkInvariants(a)
|
||
}
|
||
|
||
func TestPipelineTransientRetryWithBackoff(t *testing.T) {
|
||
e := setup(t)
|
||
a := e.upload(pngBytes(120, 80, 9))
|
||
msg := e.relay(nil)[0]
|
||
e.store.failPut = 1
|
||
d, _ := handle(e.worker(5*time.Second), msg)
|
||
s := e.state(a)
|
||
expect(t, d == worker.Ack && s.Status == "queued" && s.JobState == "queued", "retry scheduled: %+v", s)
|
||
expect(t, len(e.relay(nil)) == 0, "retry must wait for backoff")
|
||
d, _ = handle(e.worker(5*time.Second), msg) // ранняя повторная доставка
|
||
expect(t, d == worker.Ack && e.state(a).Attempt == 1, "early delivery must not run: %+v", e.state(a))
|
||
time.Sleep(500 * time.Millisecond)
|
||
msgs := e.relay(nil)
|
||
var job domain.Job
|
||
expect(t, len(msgs) == 1 && json.Unmarshal(msgs[0], &job) == nil && job.Attempt == 2, "retry message attempt 2: %s", msgs)
|
||
handle(e.worker(5*time.Second), msgs[0])
|
||
expect(t, e.state(a).Status == "ready" && e.state(a).Attempt == 2, "after retry: %+v", e.state(a))
|
||
e.checkInvariants(a)
|
||
}
|
||
|
||
func TestPipelinePermanentFailureAndReplay(t *testing.T) {
|
||
e := setup(t)
|
||
good := pngBytes(50, 40, 10)
|
||
a := e.upload(good)
|
||
// Порча staging-объекта: checksum не совпадёт → постоянная ошибка, без повторов.
|
||
key := "s/" + e.chat + "/" + a.ID
|
||
_ = e.store.Put(e.ctx, key, bytes.NewReader([]byte("garbage")), 7, "image/png")
|
||
handle(e.worker(5*time.Second), e.relay(nil)[0])
|
||
s := e.state(a)
|
||
expect(t, s.Status == "failed" && s.ErrorCode == "checksum_mismatch" && s.Attempt == 1, "permanent: %+v", s)
|
||
|
||
// Replay успешной работы запрещён; failed — разрешён (с аудитом).
|
||
_, err := e.repo.Replay(e.ctx, a.ID, 1, "tester")
|
||
expect(t, err == nil, "replay failed job: %v", err)
|
||
_ = e.store.Put(e.ctx, key, bytes.NewReader(good), int64(len(good)), "image/png") // исходник восстановлен
|
||
handle(e.worker(5*time.Second), e.relay(nil)[0])
|
||
expect(t, e.state(a).Status == "ready", "after replay: %+v", e.state(a))
|
||
_, err = e.repo.Replay(e.ctx, a.ID, 1, "tester")
|
||
expect(t, err != nil, "replay of done job must be refused")
|
||
expect(t, strings.Contains(e.events(a), "replay,") && strings.Contains(e.events(a), "replay_refused"), "audit: %s", e.events(a))
|
||
e.checkInvariants(a)
|
||
}
|
||
|
||
func TestPipelineRetryExhausted(t *testing.T) {
|
||
e := setup(t)
|
||
a := e.upload(pngBytes(30, 30, 11))
|
||
e.store.failPut = 100
|
||
msg := e.relay(nil)[0]
|
||
for i := 0; i < 3; i++ {
|
||
handle(e.worker(5*time.Second), msg)
|
||
time.Sleep(450 * time.Millisecond)
|
||
if m := e.relay(nil); len(m) > 0 {
|
||
msg = m[0]
|
||
}
|
||
}
|
||
s := e.state(a)
|
||
expect(t, s.Status == "failed" && s.ErrorCode == "retry_exhausted" && s.Attempt == 3, "exhausted: %+v", s)
|
||
jobs, _ := e.repo.FailedJobs(e.ctx, 10)
|
||
expect(t, len(jobs) == 1, "failed registry: %d", len(jobs))
|
||
}
|