forked from mtusi2026/mtusi2026lab5
498 lines
19 KiB
Go
498 lines
19 KiB
Go
//go:build worker
|
|
|
|
package studenttests_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"image/jpeg"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"os/exec"
|
|
"reflect"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/jackc/pgx/v5"
|
|
"golang.org/x/image/webp"
|
|
|
|
"mtusi2026-messenger/internal/dto"
|
|
)
|
|
|
|
func workerCommand(t *testing.T, args ...string) string {
|
|
t.Helper()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
cmd := exec.CommandContext(ctx, "docker", args...)
|
|
cmd.Dir = ".."
|
|
out, err := cmd.Output()
|
|
if err != nil {
|
|
t.Fatalf("Docker command failed: %v", err)
|
|
}
|
|
return strings.TrimSpace(string(out))
|
|
}
|
|
|
|
func workerSQL(t *testing.T, query string) string {
|
|
t.Helper()
|
|
return workerCommand(t, "compose", "exec", "-T", "postgres", "psql",
|
|
"-U", "messenger", "-d", "messenger", "-Atqc", query)
|
|
}
|
|
|
|
func workerWait(t *testing.T, description string, timeout time.Duration, condition func() bool) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(timeout)
|
|
for time.Now().Before(deadline) {
|
|
if condition() {
|
|
return
|
|
}
|
|
time.Sleep(300 * time.Millisecond)
|
|
}
|
|
t.Fatalf("timeout waiting for %s", description)
|
|
}
|
|
|
|
func workerQueue(t *testing.T) string {
|
|
t.Helper()
|
|
out := workerCommand(t, "compose", "exec", "-T", "rabbitmq", "rabbitmqctl",
|
|
"list_queues", "name", "messages_ready", "messages_unacknowledged", "--silent")
|
|
for _, line := range strings.Split(out, "\n") {
|
|
fields := strings.Fields(line)
|
|
if len(fields) == 3 && fields[0] == "attachments.process.v1" {
|
|
return fields[1] + ":" + fields[2]
|
|
}
|
|
}
|
|
t.Fatal("attachment queue missing")
|
|
return ""
|
|
}
|
|
|
|
func workerUpload(t *testing.T, user testUser, chatID uuid.UUID) (dto.AttachmentResponse, []byte) {
|
|
t.Helper()
|
|
source := makePNG(t, 320, 200)
|
|
resp := uploadAttachment(t, user, chatID, "worker.png", "image/png", source)
|
|
requireStatus(t, resp, http.StatusAccepted)
|
|
var att dto.AttachmentResponse
|
|
mustDecode(t, resp, &att)
|
|
if att.Status != "queued" {
|
|
t.Fatalf("expected queued, got %s", att.Status)
|
|
}
|
|
return att, source
|
|
}
|
|
|
|
func workerContent(t *testing.T, user testUser, id uuid.UUID, variant string) []byte {
|
|
t.Helper()
|
|
resp := doRequest(t, http.MethodGet, "/api/v1/attachments/"+id.String()+"/content?variant="+variant, user.Token, nil)
|
|
requireStatus(t, resp, http.StatusOK)
|
|
defer resp.Body.Close()
|
|
data, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return data
|
|
}
|
|
|
|
func TestWorkerPersistenceAndDuplicate(t *testing.T) {
|
|
user := createUser(t)
|
|
chat := createChat(t, user, user.ID)
|
|
stopService(t, "worker")
|
|
if workerQueue(t) != "0:0" {
|
|
t.Fatal("run on an idle stand with an empty attachment queue")
|
|
}
|
|
att, source := workerUpload(t, user, chat.ID)
|
|
where := "attachment_id='" + att.ID.String() + "'"
|
|
resp := postMessage(t, user, chat.ID, "", []uuid.UUID{att.ID})
|
|
requireStatus(t, resp, http.StatusCreated)
|
|
var message dto.MessageResponse
|
|
mustDecode(t, resp, &message)
|
|
resp = doRequest(t, http.MethodGet, "/api/v1/attachments/"+att.ID.String()+"/content", user.Token, nil)
|
|
requireStatus(t, resp, http.StatusConflict)
|
|
resp.Body.Close()
|
|
resp = sendMessage(t, user, chat.ID, "worker is stopped")
|
|
requireStatus(t, resp, http.StatusCreated)
|
|
resp.Body.Close()
|
|
workerWait(t, "confirmed publication", 15*time.Second, func() bool {
|
|
return workerSQL(t, "SELECT published_at IS NOT NULL FROM outbox WHERE "+where) == "t"
|
|
})
|
|
stopService(t, "api")
|
|
if workerQueue(t) != "1:0" {
|
|
t.Fatal("accepted job is not pending in broker")
|
|
}
|
|
before := workerCommand(t, "compose", "ps", "-q", "rabbitmq")
|
|
compose(t, "up", "-d", "--no-deps", "--force-recreate", "rabbitmq")
|
|
after := workerCommand(t, "compose", "ps", "-q", "rabbitmq")
|
|
if before == after {
|
|
t.Fatal("broker was not recreated")
|
|
}
|
|
workerWait(t, "broker health", 30*time.Second, func() bool {
|
|
return workerCommand(t, "inspect", "--format", "{{.State.Health.Status}}", after) == "healthy"
|
|
})
|
|
if workerQueue(t) != "1:0" {
|
|
t.Fatal("job lost after broker recreation with existing volume")
|
|
}
|
|
compose(t, "start", "api")
|
|
waitReady(t)
|
|
att = waitAttachment(t, user, att.ID, "queued", 15*time.Second)
|
|
resp = doRequest(t, http.MethodGet, "/api/v1/chats/"+chat.ID.String()+"/messages", user.Token, nil)
|
|
requireStatus(t, resp, http.StatusOK)
|
|
var page dto.MessagePageResponse
|
|
mustDecode(t, resp, &page)
|
|
found := false
|
|
for _, item := range page.Items {
|
|
if item.ID == message.ID && reflect.DeepEqual(item.AttachmentIDs, message.AttachmentIDs) {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatal("message with attachment lost after API restart")
|
|
}
|
|
compose(t, "start", "worker")
|
|
ready := waitAttachment(t, user, att.ID, "ready", 30*time.Second)
|
|
if !bytes.Equal(workerContent(t, user, att.ID, "original"), source) {
|
|
t.Fatal("original changed")
|
|
}
|
|
thumb := workerContent(t, user, att.ID, "thumbnail")
|
|
cfg, err := jpeg.DecodeConfig(bytes.NewReader(thumb))
|
|
if err != nil || cfg.Width != 256 || cfg.Height != 160 {
|
|
t.Fatal("invalid JPEG thumbnail")
|
|
}
|
|
crop := workerContent(t, user, att.ID, "crop")
|
|
cfg, err = jpeg.DecodeConfig(bytes.NewReader(crop))
|
|
if err != nil || cfg.Width != 128 || cfg.Height != 128 {
|
|
t.Fatal("invalid JPEG crop")
|
|
}
|
|
optimized := workerContent(t, user, att.ID, "optimized")
|
|
cfg, err = webp.DecodeConfig(bytes.NewReader(optimized))
|
|
if err != nil || cfg.Width != 320 || cfg.Height != 200 {
|
|
t.Fatal("invalid optimized WebP")
|
|
}
|
|
|
|
workerSQL(t, "UPDATE outbox SET published_at=NULL WHERE "+where)
|
|
workerWait(t, "duplicate publication", 15*time.Second, func() bool {
|
|
return workerSQL(t, "SELECT published_at IS NOT NULL FROM outbox WHERE "+where) == "t"
|
|
})
|
|
workerWait(t, "duplicate acknowledgement", 15*time.Second, func() bool { return workerQueue(t) == "0:0" })
|
|
duplicate := waitAttachment(t, user, att.ID, "ready", 30*time.Second)
|
|
if !reflect.DeepEqual(ready, duplicate) || workerSQL(t, "SELECT count(*) FROM attachment_variants WHERE "+where) != "3" {
|
|
t.Fatal("duplicate job changed metadata or duplicated variants")
|
|
}
|
|
if !bytes.Equal(workerContent(t, user, att.ID, "optimized"), optimized) ||
|
|
!bytes.Equal(workerContent(t, user, att.ID, "crop"), crop) ||
|
|
!bytes.Equal(workerContent(t, user, att.ID, "thumbnail"), thumb) ||
|
|
!bytes.Equal(workerContent(t, user, att.ID, "original"), source) {
|
|
t.Fatal("duplicate changed content")
|
|
}
|
|
t.Logf("PASS attachment=%s: queued survives API/broker restart; ready; duplicate acknowledged with unchanged result", att.ID)
|
|
}
|
|
|
|
func TestWorkerRetryExhausted(t *testing.T) {
|
|
user := createUser(t)
|
|
chat := createChat(t, user, user.ID)
|
|
stopService(t, "worker")
|
|
att, _ := workerUpload(t, user, chat.ID)
|
|
workerID := workerCommand(t, "compose", "ps", "-a", "-q", "worker")
|
|
var env []string
|
|
if err := json.Unmarshal([]byte(workerCommand(t, "inspect", "--format", "{{json .Config.Env}}", workerID)), &env); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
maxAttempts := 5
|
|
for _, entry := range env {
|
|
if value, ok := strings.CutPrefix(entry, "JOB_MAX_ATTEMPTS="); ok {
|
|
n, err := strconv.Atoi(value)
|
|
if err != nil || n < 1 {
|
|
t.Fatal("invalid JOB_MAX_ATTEMPTS")
|
|
}
|
|
maxAttempts = n
|
|
}
|
|
}
|
|
// A separate worker gets an unreachable S3 endpoint and a short attempt timeout.
|
|
// Keep the configured attempt count and normal backoff; leave the shared S3 running.
|
|
testWorker := "worker-retry-" + uuid.NewString()
|
|
t.Cleanup(func() {
|
|
workerCommand(t, "rm", "-f", testWorker)
|
|
})
|
|
workerCommand(t, "compose", "run", "-d", "--no-deps", "--name", testWorker,
|
|
"-e", "JOB_TIMEOUT_SECONDS=3", "-e", "S3_ENDPOINT=http://127.0.0.1:1", "worker")
|
|
failed := waitAttachment(t, user, att.ID, "failed", 90*time.Second)
|
|
if failed.ErrorCode == nil || *failed.ErrorCode != "retry_exhausted" {
|
|
t.Fatal("expected safe retry_exhausted error")
|
|
}
|
|
attempts := workerSQL(t, "SELECT attempts FROM attachments WHERE id='"+att.ID.String()+"'")
|
|
if attempts != strconv.Itoa(maxAttempts) {
|
|
t.Fatalf("expected %d attempts, got %s", maxAttempts, attempts)
|
|
}
|
|
resp := doRequest(t, http.MethodGet, "/api/v1/attachments/"+att.ID.String()+"/content", user.Token, nil)
|
|
requireStatus(t, resp, http.StatusConflict)
|
|
resp.Body.Close()
|
|
resp = postMessage(t, user, chat.ID, "", []uuid.UUID{att.ID})
|
|
requireStatus(t, resp, http.StatusConflict)
|
|
resp.Body.Close()
|
|
t.Logf("PASS attachment=%s: %d attempts, failed/retry_exhausted, content/link rejected", att.ID, maxAttempts)
|
|
}
|
|
|
|
func TestWorkerProcessingTimeout(t *testing.T) {
|
|
user := createUser(t)
|
|
chat := createChat(t, user, user.ID)
|
|
stopService(t, "worker")
|
|
if workerQueue(t) != "0:0" {
|
|
t.Fatal("run on an idle stand with an empty attachment queue")
|
|
}
|
|
|
|
// Decode and resize of 36 million pixels must be interrupted by the short deadline.
|
|
resp := uploadAttachment(t, user, chat.ID, "large.png", "image/png", makePNG(t, 6000, 6000))
|
|
requireStatus(t, resp, http.StatusAccepted)
|
|
var att dto.AttachmentResponse
|
|
mustDecode(t, resp, &att)
|
|
|
|
testWorker := "worker-timeout-" + uuid.NewString()
|
|
t.Cleanup(func() {
|
|
workerCommand(t, "rm", "-f", testWorker)
|
|
})
|
|
workerCommand(t, "compose", "run", "-d", "--no-deps", "--name", testWorker,
|
|
"-e", "JOB_TIMEOUT_SECONDS=1", "-e", "JOB_MAX_ATTEMPTS=1", "worker")
|
|
parentPID := workerCommand(t, "inspect", "--format", "{{.State.Pid}}", testWorker)
|
|
workerWait(t, "image child process", 5*time.Second, func() bool {
|
|
return strings.Contains(workerCommand(t, "top", testWorker, "-eo", "pid,args"), "-process-image")
|
|
})
|
|
failed := waitAttachment(t, user, att.ID, "failed", 5*time.Second)
|
|
if failed.ErrorCode == nil || *failed.ErrorCode != "retry_exhausted" {
|
|
t.Fatal("expected retry_exhausted after processing deadline")
|
|
}
|
|
if strings.Contains(workerCommand(t, "top", testWorker, "-eo", "pid,args"), "-process-image") {
|
|
t.Fatal("image process survived its deadline")
|
|
}
|
|
if workerSQL(t, "SELECT count(*) FROM attachment_variants WHERE attachment_id='"+att.ID.String()+"'") != "0" {
|
|
t.Fatal("timed-out processing published variants")
|
|
}
|
|
|
|
// The same parent process must remain usable after terminating the image child.
|
|
uploadPNG(t, user, chat.ID)
|
|
after := workerCommand(t, "inspect", "--format", "{{.State.Pid}}", testWorker)
|
|
if parentPID != after || parentPID == "0" {
|
|
t.Fatal("parent worker restarted after the image timeout")
|
|
}
|
|
}
|
|
|
|
func TestWorkerReplay(t *testing.T) {
|
|
user := createUser(t)
|
|
chat := createChat(t, user, user.ID)
|
|
stopService(t, "worker")
|
|
if workerQueue(t) != "0:0" {
|
|
t.Fatal("run on an idle stand with an empty attachment queue")
|
|
}
|
|
att, source := workerUpload(t, user, chat.ID)
|
|
id := att.ID.String()
|
|
where := "attachment_id='" + id + "'"
|
|
|
|
testWorker := "worker-replay-" + uuid.NewString()
|
|
t.Cleanup(func() {
|
|
if testWorker != "" {
|
|
workerCommand(t, "rm", "-f", testWorker)
|
|
}
|
|
})
|
|
workerCommand(t, "compose", "run", "-d", "--no-deps", "--name", testWorker,
|
|
"-e", "JOB_TIMEOUT_SECONDS=3", "-e", "JOB_MAX_ATTEMPTS=1",
|
|
"-e", "S3_ENDPOINT=http://127.0.0.1:1", "worker")
|
|
waitAttachment(t, user, att.ID, "failed", 15*time.Second)
|
|
workerCommand(t, "rm", "-f", testWorker)
|
|
testWorker = ""
|
|
|
|
runCLI := func(args ...string) (string, error) {
|
|
t.Helper()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
cmd := exec.CommandContext(ctx, "docker", append([]string{
|
|
"compose", "run", "--rm", "--no-deps", "worker",
|
|
}, args...)...)
|
|
cmd.Dir = ".."
|
|
out, err := cmd.CombinedOutput()
|
|
return string(out), err
|
|
}
|
|
replayArgs := []string{"-replay", id, "-operator", "synthetic-operator", "-reason", "S3 restored"}
|
|
if out, err := runCLI("-list-failed"); err != nil || !strings.Contains(out, id) {
|
|
t.Fatalf("failed registry did not list attachment: %v\n%s", err, out)
|
|
}
|
|
if out, err := runCLI("-replay", id); err == nil || !strings.Contains(out, "nonempty operator and reason") {
|
|
t.Fatalf("replay without audit details was accepted: %v\n%s", err, out)
|
|
}
|
|
|
|
// A stale DB key must not make a missing S3 source replayable.
|
|
key := workerSQL(t, "SELECT staging_key FROM attachments WHERE id='"+id+"'")
|
|
workerSQL(t, "UPDATE attachments SET staging_key='staging/"+uuid.NewString()+"' WHERE id='"+id+"'")
|
|
out, err := runCLI(replayArgs...)
|
|
workerSQL(t, "UPDATE attachments SET staging_key='"+key+"' WHERE id='"+id+"'")
|
|
if err == nil || !strings.Contains(out, "verify replay source") {
|
|
t.Fatalf("missing S3 source was accepted: %v\n%s", err, out)
|
|
}
|
|
if workerSQL(t, "SELECT status||':'||attempts||':'||retry_base FROM attachments WHERE id='"+id+"'") != "failed:1:0" ||
|
|
workerSQL(t, "SELECT count(*) FROM attachment_replays WHERE "+where) != "0" ||
|
|
workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where) != "1" {
|
|
t.Fatal("rejected replay changed attachment, audit or outbox")
|
|
}
|
|
|
|
if out, err := runCLI(replayArgs...); err != nil {
|
|
t.Fatalf("replay failed: %v\n%s", err, out)
|
|
}
|
|
if workerSQL(t, "SELECT status||':'||attempts||':'||retry_base FROM attachments WHERE id='"+id+"'") != "queued:1:1" ||
|
|
workerSQL(t, "SELECT attempt||':'||operator||':'||reason||':'||previous_error_code FROM attachment_replays WHERE "+where) !=
|
|
"2:synthetic-operator:S3 restored:retry_exhausted" ||
|
|
workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where) != "2" {
|
|
t.Fatal("replay did not atomically queue the next attempt with audit and outbox")
|
|
}
|
|
if out, err := runCLI(replayArgs...); err == nil || !strings.Contains(out, "not replayable") {
|
|
t.Fatalf("queued replay was accepted twice: %v\n%s", err, out)
|
|
}
|
|
|
|
compose(t, "start", "worker")
|
|
ready := waitAttachment(t, user, att.ID, "ready", 30*time.Second)
|
|
if len(ready.Variants) != 3 || !bytes.Equal(workerContent(t, user, att.ID, "original"), source) {
|
|
t.Fatal("replay did not complete the same attachment with unchanged original")
|
|
}
|
|
if out, err := runCLI(replayArgs...); err == nil || !strings.Contains(out, "not replayable") {
|
|
t.Fatalf("ready replay was accepted: %v\n%s", err, out)
|
|
}
|
|
if workerSQL(t, "SELECT count(*) FROM attachment_replays WHERE "+where) != "1" ||
|
|
workerSQL(t, "SELECT attempts FROM attachments WHERE id='"+id+"'") != "2" {
|
|
t.Fatal("duplicate replay changed audit or attempt counter")
|
|
}
|
|
}
|
|
|
|
func TestWorkerOutboxRecovery(t *testing.T) {
|
|
user := createUser(t)
|
|
chat := createChat(t, user, user.ID)
|
|
stopService(t, "worker")
|
|
if workerQueue(t) != "0:0" {
|
|
t.Fatal("run on an idle stand with an empty attachment queue")
|
|
}
|
|
|
|
stopService(t, "rabbitmq")
|
|
att, source := workerUpload(t, user, chat.ID)
|
|
id := att.ID.String()
|
|
where := "attachment_id='" + id + "'"
|
|
jobID := workerSQL(t, "SELECT job_id FROM attachments WHERE id='"+id+"'")
|
|
if workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where+" AND attempt=1 AND published_at IS NULL") != "1" {
|
|
t.Fatal("upload 202 did not persist an unpublished intent")
|
|
}
|
|
|
|
t.Cleanup(func() {
|
|
compose(t, "start", "api")
|
|
waitReady(t)
|
|
})
|
|
compose(t, "kill", "-s", "SIGKILL", "api")
|
|
if workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where+" AND attempt=1 AND published_at IS NULL") != "1" {
|
|
t.Fatal("committed intent lost after API crash")
|
|
}
|
|
|
|
compose(t, "start", "api")
|
|
waitReady(t)
|
|
waitAttachment(t, user, att.ID, "queued", 15*time.Second)
|
|
compose(t, "up", "-d", "--no-deps", "--wait", "--wait-timeout", "30", "rabbitmq")
|
|
workerWait(t, "outbox publication after API crash", 15*time.Second, func() bool {
|
|
return workerSQL(t, "SELECT published_at IS NOT NULL FROM outbox WHERE "+where) == "t"
|
|
})
|
|
if workerQueue(t) != "1:0" {
|
|
t.Fatal("recovered intent was not delivered to broker")
|
|
}
|
|
|
|
compose(t, "start", "worker")
|
|
ready := waitAttachment(t, user, att.ID, "ready", 30*time.Second)
|
|
if len(ready.Variants) != 3 || !bytes.Equal(workerContent(t, user, att.ID, "original"), source) {
|
|
t.Fatal("recovered attachment has incomplete variants or changed original")
|
|
}
|
|
workerWait(t, "recovered job acknowledgement", 15*time.Second, func() bool {
|
|
return workerQueue(t) == "0:0"
|
|
})
|
|
if workerSQL(t, "SELECT job_id FROM attachments WHERE id='"+id+"'") != jobID ||
|
|
workerSQL(t, "SELECT attempts FROM attachments WHERE id='"+id+"'") != "1" ||
|
|
workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where) != "1" {
|
|
t.Fatal("recovery replaced the job or created another attempt")
|
|
}
|
|
|
|
t.Logf("PASS attachment=%s: commit, API SIGKILL before publication, automatic outbox recovery, same job ready", id)
|
|
}
|
|
|
|
func TestWorkerLeaseRecovery(t *testing.T) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
|
defer cancel()
|
|
secret, err := os.ReadFile("../secrets/test_database_url.txt")
|
|
if err != nil {
|
|
t.Fatal("read test database URL")
|
|
}
|
|
conn, err := pgx.Connect(ctx, strings.TrimSpace(string(secret)))
|
|
if err != nil {
|
|
t.Fatal("connect to test database")
|
|
}
|
|
t.Cleanup(func() { conn.Close(context.Background()) })
|
|
|
|
user := createUser(t)
|
|
chat := createChat(t, user, user.ID)
|
|
stopService(t, "worker")
|
|
if workerQueue(t) != "0:0" {
|
|
t.Fatal("run on an idle stand with an empty attachment queue")
|
|
}
|
|
att, source := workerUpload(t, user, chat.ID)
|
|
id := att.ID.String()
|
|
where := "attachment_id='" + id + "'"
|
|
jobID := workerSQL(t, "SELECT job_id FROM attachments WHERE id='"+id+"'")
|
|
|
|
testWorker := "worker-lease-" + uuid.NewString()
|
|
t.Cleanup(func() { workerCommand(t, "rm", "-f", testWorker) })
|
|
tx, err := conn.Begin(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { tx.Rollback(context.Background()) })
|
|
// S3 writes finish before variant metadata is inserted. Block that INSERT
|
|
// to kill the worker before the result COMMIT and broker acknowledgement.
|
|
if _, err := tx.Exec(ctx, "LOCK TABLE attachment_variants IN SHARE MODE"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
workerCommand(t, "compose", "run", "-d", "--no-deps", "--name", testWorker,
|
|
"-e", "JOB_TIMEOUT_SECONDS=15", "worker")
|
|
workerWait(t, "variant INSERT blocked after S3 writes", 10*time.Second, func() bool {
|
|
var blocked bool
|
|
if _, err := conn.Exec(ctx, "SELECT pg_stat_clear_snapshot()"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
err := conn.QueryRow(ctx, `SELECT EXISTS (
|
|
SELECT 1 FROM pg_stat_activity
|
|
WHERE $1::int = ANY(pg_blocking_pids(pid))
|
|
AND query LIKE '%INSERT INTO attachment_variants%'
|
|
)`, int32(conn.PgConn().PID())).Scan(&blocked)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return blocked
|
|
})
|
|
workerCommand(t, "kill", "--signal", "SIGKILL", testWorker)
|
|
if err := tx.Rollback(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if workerSQL(t, "SELECT status||':'||attempts FROM attachments WHERE id='"+id+"'") != "processing:1" ||
|
|
workerSQL(t, "SELECT count(*) FROM attachment_variants WHERE "+where) != "0" {
|
|
t.Fatal("crashed worker committed a result")
|
|
}
|
|
workerWait(t, "unacknowledged job returned to broker", 10*time.Second, func() bool {
|
|
return workerQueue(t) == "1:0"
|
|
})
|
|
|
|
// Leave lease_until unchanged: recovery must wait for its natural expiry.
|
|
compose(t, "start", "worker")
|
|
ready := waitAttachment(t, user, att.ID, "ready", 40*time.Second)
|
|
if len(ready.Variants) != 3 || !bytes.Equal(workerContent(t, user, att.ID, "original"), source) {
|
|
t.Fatal("lease recovery lost variants or changed original")
|
|
}
|
|
workerWait(t, "recovered job acknowledgement", 15*time.Second, func() bool {
|
|
return workerQueue(t) == "0:0"
|
|
})
|
|
if workerSQL(t, "SELECT job_id FROM attachments WHERE id='"+id+"'") != jobID ||
|
|
workerSQL(t, "SELECT attempts FROM attachments WHERE id='"+id+"'") != "2" ||
|
|
workerSQL(t, "SELECT count(*) FROM attachment_variants WHERE "+where) != "3" ||
|
|
workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where+" AND attempt=2 AND published_at IS NOT NULL") != "1" {
|
|
t.Fatal("lease recovery replaced the job, duplicated variants or skipped retry publication")
|
|
}
|
|
t.Logf("PASS attachment=%s: SIGKILL after S3 writes, expired lease recovered, same job ready on attempt 2", id)
|
|
}
|