Files
mtusi2026lab5/student-tests/worker_test.go
T

498 lines
19 KiB
Go

//go:build worker
package studenttests_test
import (
"bytes"
"context"
"encoding/json"
"image/jpeg"
"io"
"net/http"
"os"
"os/exec"
"reflect"
"strconv"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"golang.org/x/image/webp"
"mtusi2026-messenger/internal/dto"
)
func workerCommand(t *testing.T, args ...string) string {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, "docker", args...)
cmd.Dir = ".."
out, err := cmd.Output()
if err != nil {
t.Fatalf("Docker command failed: %v", err)
}
return strings.TrimSpace(string(out))
}
func workerSQL(t *testing.T, query string) string {
t.Helper()
return workerCommand(t, "compose", "exec", "-T", "postgres", "psql",
"-U", "messenger", "-d", "messenger", "-Atqc", query)
}
func workerWait(t *testing.T, description string, timeout time.Duration, condition func() bool) {
t.Helper()
deadline := time.Now().Add(timeout)
for time.Now().Before(deadline) {
if condition() {
return
}
time.Sleep(300 * time.Millisecond)
}
t.Fatalf("timeout waiting for %s", description)
}
func workerQueue(t *testing.T) string {
t.Helper()
out := workerCommand(t, "compose", "exec", "-T", "rabbitmq", "rabbitmqctl",
"list_queues", "name", "messages_ready", "messages_unacknowledged", "--silent")
for _, line := range strings.Split(out, "\n") {
fields := strings.Fields(line)
if len(fields) == 3 && fields[0] == "attachments.process.v1" {
return fields[1] + ":" + fields[2]
}
}
t.Fatal("attachment queue missing")
return ""
}
func workerUpload(t *testing.T, user testUser, chatID uuid.UUID) (dto.AttachmentResponse, []byte) {
t.Helper()
source := makePNG(t, 320, 200)
resp := uploadAttachment(t, user, chatID, "worker.png", "image/png", source)
requireStatus(t, resp, http.StatusAccepted)
var att dto.AttachmentResponse
mustDecode(t, resp, &att)
if att.Status != "queued" {
t.Fatalf("expected queued, got %s", att.Status)
}
return att, source
}
func workerContent(t *testing.T, user testUser, id uuid.UUID, variant string) []byte {
t.Helper()
resp := doRequest(t, http.MethodGet, "/api/v1/attachments/"+id.String()+"/content?variant="+variant, user.Token, nil)
requireStatus(t, resp, http.StatusOK)
defer resp.Body.Close()
data, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
return data
}
func TestWorkerPersistenceAndDuplicate(t *testing.T) {
user := createUser(t)
chat := createChat(t, user, user.ID)
stopService(t, "worker")
if workerQueue(t) != "0:0" {
t.Fatal("run on an idle stand with an empty attachment queue")
}
att, source := workerUpload(t, user, chat.ID)
where := "attachment_id='" + att.ID.String() + "'"
resp := postMessage(t, user, chat.ID, "", []uuid.UUID{att.ID})
requireStatus(t, resp, http.StatusCreated)
var message dto.MessageResponse
mustDecode(t, resp, &message)
resp = doRequest(t, http.MethodGet, "/api/v1/attachments/"+att.ID.String()+"/content", user.Token, nil)
requireStatus(t, resp, http.StatusConflict)
resp.Body.Close()
resp = sendMessage(t, user, chat.ID, "worker is stopped")
requireStatus(t, resp, http.StatusCreated)
resp.Body.Close()
workerWait(t, "confirmed publication", 15*time.Second, func() bool {
return workerSQL(t, "SELECT published_at IS NOT NULL FROM outbox WHERE "+where) == "t"
})
stopService(t, "api")
if workerQueue(t) != "1:0" {
t.Fatal("accepted job is not pending in broker")
}
before := workerCommand(t, "compose", "ps", "-q", "rabbitmq")
compose(t, "up", "-d", "--no-deps", "--force-recreate", "rabbitmq")
after := workerCommand(t, "compose", "ps", "-q", "rabbitmq")
if before == after {
t.Fatal("broker was not recreated")
}
workerWait(t, "broker health", 30*time.Second, func() bool {
return workerCommand(t, "inspect", "--format", "{{.State.Health.Status}}", after) == "healthy"
})
if workerQueue(t) != "1:0" {
t.Fatal("job lost after broker recreation with existing volume")
}
compose(t, "start", "api")
waitReady(t)
att = waitAttachment(t, user, att.ID, "queued", 15*time.Second)
resp = doRequest(t, http.MethodGet, "/api/v1/chats/"+chat.ID.String()+"/messages", user.Token, nil)
requireStatus(t, resp, http.StatusOK)
var page dto.MessagePageResponse
mustDecode(t, resp, &page)
found := false
for _, item := range page.Items {
if item.ID == message.ID && reflect.DeepEqual(item.AttachmentIDs, message.AttachmentIDs) {
found = true
}
}
if !found {
t.Fatal("message with attachment lost after API restart")
}
compose(t, "start", "worker")
ready := waitAttachment(t, user, att.ID, "ready", 30*time.Second)
if !bytes.Equal(workerContent(t, user, att.ID, "original"), source) {
t.Fatal("original changed")
}
thumb := workerContent(t, user, att.ID, "thumbnail")
cfg, err := jpeg.DecodeConfig(bytes.NewReader(thumb))
if err != nil || cfg.Width != 256 || cfg.Height != 160 {
t.Fatal("invalid JPEG thumbnail")
}
crop := workerContent(t, user, att.ID, "crop")
cfg, err = jpeg.DecodeConfig(bytes.NewReader(crop))
if err != nil || cfg.Width != 128 || cfg.Height != 128 {
t.Fatal("invalid JPEG crop")
}
optimized := workerContent(t, user, att.ID, "optimized")
cfg, err = webp.DecodeConfig(bytes.NewReader(optimized))
if err != nil || cfg.Width != 320 || cfg.Height != 200 {
t.Fatal("invalid optimized WebP")
}
workerSQL(t, "UPDATE outbox SET published_at=NULL WHERE "+where)
workerWait(t, "duplicate publication", 15*time.Second, func() bool {
return workerSQL(t, "SELECT published_at IS NOT NULL FROM outbox WHERE "+where) == "t"
})
workerWait(t, "duplicate acknowledgement", 15*time.Second, func() bool { return workerQueue(t) == "0:0" })
duplicate := waitAttachment(t, user, att.ID, "ready", 30*time.Second)
if !reflect.DeepEqual(ready, duplicate) || workerSQL(t, "SELECT count(*) FROM attachment_variants WHERE "+where) != "3" {
t.Fatal("duplicate job changed metadata or duplicated variants")
}
if !bytes.Equal(workerContent(t, user, att.ID, "optimized"), optimized) ||
!bytes.Equal(workerContent(t, user, att.ID, "crop"), crop) ||
!bytes.Equal(workerContent(t, user, att.ID, "thumbnail"), thumb) ||
!bytes.Equal(workerContent(t, user, att.ID, "original"), source) {
t.Fatal("duplicate changed content")
}
t.Logf("PASS attachment=%s: queued survives API/broker restart; ready; duplicate acknowledged with unchanged result", att.ID)
}
func TestWorkerRetryExhausted(t *testing.T) {
user := createUser(t)
chat := createChat(t, user, user.ID)
stopService(t, "worker")
att, _ := workerUpload(t, user, chat.ID)
workerID := workerCommand(t, "compose", "ps", "-a", "-q", "worker")
var env []string
if err := json.Unmarshal([]byte(workerCommand(t, "inspect", "--format", "{{json .Config.Env}}", workerID)), &env); err != nil {
t.Fatal(err)
}
maxAttempts := 5
for _, entry := range env {
if value, ok := strings.CutPrefix(entry, "JOB_MAX_ATTEMPTS="); ok {
n, err := strconv.Atoi(value)
if err != nil || n < 1 {
t.Fatal("invalid JOB_MAX_ATTEMPTS")
}
maxAttempts = n
}
}
// A separate worker gets an unreachable S3 endpoint and a short attempt timeout.
// Keep the configured attempt count and normal backoff; leave the shared S3 running.
testWorker := "worker-retry-" + uuid.NewString()
t.Cleanup(func() {
workerCommand(t, "rm", "-f", testWorker)
})
workerCommand(t, "compose", "run", "-d", "--no-deps", "--name", testWorker,
"-e", "JOB_TIMEOUT_SECONDS=3", "-e", "S3_ENDPOINT=http://127.0.0.1:1", "worker")
failed := waitAttachment(t, user, att.ID, "failed", 90*time.Second)
if failed.ErrorCode == nil || *failed.ErrorCode != "retry_exhausted" {
t.Fatal("expected safe retry_exhausted error")
}
attempts := workerSQL(t, "SELECT attempts FROM attachments WHERE id='"+att.ID.String()+"'")
if attempts != strconv.Itoa(maxAttempts) {
t.Fatalf("expected %d attempts, got %s", maxAttempts, attempts)
}
resp := doRequest(t, http.MethodGet, "/api/v1/attachments/"+att.ID.String()+"/content", user.Token, nil)
requireStatus(t, resp, http.StatusConflict)
resp.Body.Close()
resp = postMessage(t, user, chat.ID, "", []uuid.UUID{att.ID})
requireStatus(t, resp, http.StatusConflict)
resp.Body.Close()
t.Logf("PASS attachment=%s: %d attempts, failed/retry_exhausted, content/link rejected", att.ID, maxAttempts)
}
func TestWorkerProcessingTimeout(t *testing.T) {
user := createUser(t)
chat := createChat(t, user, user.ID)
stopService(t, "worker")
if workerQueue(t) != "0:0" {
t.Fatal("run on an idle stand with an empty attachment queue")
}
// Decode and resize of 36 million pixels must be interrupted by the short deadline.
resp := uploadAttachment(t, user, chat.ID, "large.png", "image/png", makePNG(t, 6000, 6000))
requireStatus(t, resp, http.StatusAccepted)
var att dto.AttachmentResponse
mustDecode(t, resp, &att)
testWorker := "worker-timeout-" + uuid.NewString()
t.Cleanup(func() {
workerCommand(t, "rm", "-f", testWorker)
})
workerCommand(t, "compose", "run", "-d", "--no-deps", "--name", testWorker,
"-e", "JOB_TIMEOUT_SECONDS=1", "-e", "JOB_MAX_ATTEMPTS=1", "worker")
parentPID := workerCommand(t, "inspect", "--format", "{{.State.Pid}}", testWorker)
workerWait(t, "image child process", 5*time.Second, func() bool {
return strings.Contains(workerCommand(t, "top", testWorker, "-eo", "pid,args"), "-process-image")
})
failed := waitAttachment(t, user, att.ID, "failed", 5*time.Second)
if failed.ErrorCode == nil || *failed.ErrorCode != "retry_exhausted" {
t.Fatal("expected retry_exhausted after processing deadline")
}
if strings.Contains(workerCommand(t, "top", testWorker, "-eo", "pid,args"), "-process-image") {
t.Fatal("image process survived its deadline")
}
if workerSQL(t, "SELECT count(*) FROM attachment_variants WHERE attachment_id='"+att.ID.String()+"'") != "0" {
t.Fatal("timed-out processing published variants")
}
// The same parent process must remain usable after terminating the image child.
uploadPNG(t, user, chat.ID)
after := workerCommand(t, "inspect", "--format", "{{.State.Pid}}", testWorker)
if parentPID != after || parentPID == "0" {
t.Fatal("parent worker restarted after the image timeout")
}
}
func TestWorkerReplay(t *testing.T) {
user := createUser(t)
chat := createChat(t, user, user.ID)
stopService(t, "worker")
if workerQueue(t) != "0:0" {
t.Fatal("run on an idle stand with an empty attachment queue")
}
att, source := workerUpload(t, user, chat.ID)
id := att.ID.String()
where := "attachment_id='" + id + "'"
testWorker := "worker-replay-" + uuid.NewString()
t.Cleanup(func() {
if testWorker != "" {
workerCommand(t, "rm", "-f", testWorker)
}
})
workerCommand(t, "compose", "run", "-d", "--no-deps", "--name", testWorker,
"-e", "JOB_TIMEOUT_SECONDS=3", "-e", "JOB_MAX_ATTEMPTS=1",
"-e", "S3_ENDPOINT=http://127.0.0.1:1", "worker")
waitAttachment(t, user, att.ID, "failed", 15*time.Second)
workerCommand(t, "rm", "-f", testWorker)
testWorker = ""
runCLI := func(args ...string) (string, error) {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
cmd := exec.CommandContext(ctx, "docker", append([]string{
"compose", "run", "--rm", "--no-deps", "worker",
}, args...)...)
cmd.Dir = ".."
out, err := cmd.CombinedOutput()
return string(out), err
}
replayArgs := []string{"-replay", id, "-operator", "synthetic-operator", "-reason", "S3 restored"}
if out, err := runCLI("-list-failed"); err != nil || !strings.Contains(out, id) {
t.Fatalf("failed registry did not list attachment: %v\n%s", err, out)
}
if out, err := runCLI("-replay", id); err == nil || !strings.Contains(out, "nonempty operator and reason") {
t.Fatalf("replay without audit details was accepted: %v\n%s", err, out)
}
// A stale DB key must not make a missing S3 source replayable.
key := workerSQL(t, "SELECT staging_key FROM attachments WHERE id='"+id+"'")
workerSQL(t, "UPDATE attachments SET staging_key='staging/"+uuid.NewString()+"' WHERE id='"+id+"'")
out, err := runCLI(replayArgs...)
workerSQL(t, "UPDATE attachments SET staging_key='"+key+"' WHERE id='"+id+"'")
if err == nil || !strings.Contains(out, "verify replay source") {
t.Fatalf("missing S3 source was accepted: %v\n%s", err, out)
}
if workerSQL(t, "SELECT status||':'||attempts||':'||retry_base FROM attachments WHERE id='"+id+"'") != "failed:1:0" ||
workerSQL(t, "SELECT count(*) FROM attachment_replays WHERE "+where) != "0" ||
workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where) != "1" {
t.Fatal("rejected replay changed attachment, audit or outbox")
}
if out, err := runCLI(replayArgs...); err != nil {
t.Fatalf("replay failed: %v\n%s", err, out)
}
if workerSQL(t, "SELECT status||':'||attempts||':'||retry_base FROM attachments WHERE id='"+id+"'") != "queued:1:1" ||
workerSQL(t, "SELECT attempt||':'||operator||':'||reason||':'||previous_error_code FROM attachment_replays WHERE "+where) !=
"2:synthetic-operator:S3 restored:retry_exhausted" ||
workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where) != "2" {
t.Fatal("replay did not atomically queue the next attempt with audit and outbox")
}
if out, err := runCLI(replayArgs...); err == nil || !strings.Contains(out, "not replayable") {
t.Fatalf("queued replay was accepted twice: %v\n%s", err, out)
}
compose(t, "start", "worker")
ready := waitAttachment(t, user, att.ID, "ready", 30*time.Second)
if len(ready.Variants) != 3 || !bytes.Equal(workerContent(t, user, att.ID, "original"), source) {
t.Fatal("replay did not complete the same attachment with unchanged original")
}
if out, err := runCLI(replayArgs...); err == nil || !strings.Contains(out, "not replayable") {
t.Fatalf("ready replay was accepted: %v\n%s", err, out)
}
if workerSQL(t, "SELECT count(*) FROM attachment_replays WHERE "+where) != "1" ||
workerSQL(t, "SELECT attempts FROM attachments WHERE id='"+id+"'") != "2" {
t.Fatal("duplicate replay changed audit or attempt counter")
}
}
func TestWorkerOutboxRecovery(t *testing.T) {
user := createUser(t)
chat := createChat(t, user, user.ID)
stopService(t, "worker")
if workerQueue(t) != "0:0" {
t.Fatal("run on an idle stand with an empty attachment queue")
}
stopService(t, "rabbitmq")
att, source := workerUpload(t, user, chat.ID)
id := att.ID.String()
where := "attachment_id='" + id + "'"
jobID := workerSQL(t, "SELECT job_id FROM attachments WHERE id='"+id+"'")
if workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where+" AND attempt=1 AND published_at IS NULL") != "1" {
t.Fatal("upload 202 did not persist an unpublished intent")
}
t.Cleanup(func() {
compose(t, "start", "api")
waitReady(t)
})
compose(t, "kill", "-s", "SIGKILL", "api")
if workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where+" AND attempt=1 AND published_at IS NULL") != "1" {
t.Fatal("committed intent lost after API crash")
}
compose(t, "start", "api")
waitReady(t)
waitAttachment(t, user, att.ID, "queued", 15*time.Second)
compose(t, "up", "-d", "--no-deps", "--wait", "--wait-timeout", "30", "rabbitmq")
workerWait(t, "outbox publication after API crash", 15*time.Second, func() bool {
return workerSQL(t, "SELECT published_at IS NOT NULL FROM outbox WHERE "+where) == "t"
})
if workerQueue(t) != "1:0" {
t.Fatal("recovered intent was not delivered to broker")
}
compose(t, "start", "worker")
ready := waitAttachment(t, user, att.ID, "ready", 30*time.Second)
if len(ready.Variants) != 3 || !bytes.Equal(workerContent(t, user, att.ID, "original"), source) {
t.Fatal("recovered attachment has incomplete variants or changed original")
}
workerWait(t, "recovered job acknowledgement", 15*time.Second, func() bool {
return workerQueue(t) == "0:0"
})
if workerSQL(t, "SELECT job_id FROM attachments WHERE id='"+id+"'") != jobID ||
workerSQL(t, "SELECT attempts FROM attachments WHERE id='"+id+"'") != "1" ||
workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where) != "1" {
t.Fatal("recovery replaced the job or created another attempt")
}
t.Logf("PASS attachment=%s: commit, API SIGKILL before publication, automatic outbox recovery, same job ready", id)
}
func TestWorkerLeaseRecovery(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
secret, err := os.ReadFile("../secrets/test_database_url.txt")
if err != nil {
t.Fatal("read test database URL")
}
conn, err := pgx.Connect(ctx, strings.TrimSpace(string(secret)))
if err != nil {
t.Fatal("connect to test database")
}
t.Cleanup(func() { conn.Close(context.Background()) })
user := createUser(t)
chat := createChat(t, user, user.ID)
stopService(t, "worker")
if workerQueue(t) != "0:0" {
t.Fatal("run on an idle stand with an empty attachment queue")
}
att, source := workerUpload(t, user, chat.ID)
id := att.ID.String()
where := "attachment_id='" + id + "'"
jobID := workerSQL(t, "SELECT job_id FROM attachments WHERE id='"+id+"'")
testWorker := "worker-lease-" + uuid.NewString()
t.Cleanup(func() { workerCommand(t, "rm", "-f", testWorker) })
tx, err := conn.Begin(ctx)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { tx.Rollback(context.Background()) })
// S3 writes finish before variant metadata is inserted. Block that INSERT
// to kill the worker before the result COMMIT and broker acknowledgement.
if _, err := tx.Exec(ctx, "LOCK TABLE attachment_variants IN SHARE MODE"); err != nil {
t.Fatal(err)
}
workerCommand(t, "compose", "run", "-d", "--no-deps", "--name", testWorker,
"-e", "JOB_TIMEOUT_SECONDS=15", "worker")
workerWait(t, "variant INSERT blocked after S3 writes", 10*time.Second, func() bool {
var blocked bool
if _, err := conn.Exec(ctx, "SELECT pg_stat_clear_snapshot()"); err != nil {
t.Fatal(err)
}
err := conn.QueryRow(ctx, `SELECT EXISTS (
SELECT 1 FROM pg_stat_activity
WHERE $1::int = ANY(pg_blocking_pids(pid))
AND query LIKE '%INSERT INTO attachment_variants%'
)`, int32(conn.PgConn().PID())).Scan(&blocked)
if err != nil {
t.Fatal(err)
}
return blocked
})
workerCommand(t, "kill", "--signal", "SIGKILL", testWorker)
if err := tx.Rollback(ctx); err != nil {
t.Fatal(err)
}
if workerSQL(t, "SELECT status||':'||attempts FROM attachments WHERE id='"+id+"'") != "processing:1" ||
workerSQL(t, "SELECT count(*) FROM attachment_variants WHERE "+where) != "0" {
t.Fatal("crashed worker committed a result")
}
workerWait(t, "unacknowledged job returned to broker", 10*time.Second, func() bool {
return workerQueue(t) == "1:0"
})
// Leave lease_until unchanged: recovery must wait for its natural expiry.
compose(t, "start", "worker")
ready := waitAttachment(t, user, att.ID, "ready", 40*time.Second)
if len(ready.Variants) != 3 || !bytes.Equal(workerContent(t, user, att.ID, "original"), source) {
t.Fatal("lease recovery lost variants or changed original")
}
workerWait(t, "recovered job acknowledgement", 15*time.Second, func() bool {
return workerQueue(t) == "0:0"
})
if workerSQL(t, "SELECT job_id FROM attachments WHERE id='"+id+"'") != jobID ||
workerSQL(t, "SELECT attempts FROM attachments WHERE id='"+id+"'") != "2" ||
workerSQL(t, "SELECT count(*) FROM attachment_variants WHERE "+where) != "3" ||
workerSQL(t, "SELECT count(*) FROM outbox WHERE "+where+" AND attempt=2 AND published_at IS NOT NULL") != "1" {
t.Fatal("lease recovery replaced the job, duplicated variants or skipped retry publication")
}
t.Logf("PASS attachment=%s: SIGKILL after S3 writes, expired lease recovered, same job ready on attempt 2", id)
}