219 lines
7.7 KiB
Go
219 lines
7.7 KiB
Go
//go:build worker
|
|
|
|
package studenttests_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"image/jpeg"
|
|
"io"
|
|
"net/http"
|
|
"os/exec"
|
|
"reflect"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
|
|
"mtusi2026-messenger/internal/dto"
|
|
)
|
|
|
|
func workerCommand(t *testing.T, args ...string) string {
|
|
t.Helper()
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
cmd := exec.CommandContext(ctx, "docker", args...)
|
|
cmd.Dir = ".."
|
|
out, err := cmd.Output()
|
|
if err != nil {
|
|
t.Fatalf("Docker command failed: %v", err)
|
|
}
|
|
return strings.TrimSpace(string(out))
|
|
}
|
|
|
|
func workerSQL(t *testing.T, query string) string {
|
|
t.Helper()
|
|
return workerCommand(t, "compose", "exec", "-T", "postgres", "psql",
|
|
"-U", "messenger", "-d", "messenger", "-Atqc", query)
|
|
}
|
|
|
|
func workerWait(t *testing.T, description string, timeout time.Duration, condition func() bool) {
|
|
t.Helper()
|
|
deadline := time.Now().Add(timeout)
|
|
for time.Now().Before(deadline) {
|
|
if condition() {
|
|
return
|
|
}
|
|
time.Sleep(300 * time.Millisecond)
|
|
}
|
|
t.Fatalf("timeout waiting for %s", description)
|
|
}
|
|
|
|
func workerQueue(t *testing.T) string {
|
|
t.Helper()
|
|
out := workerCommand(t, "compose", "exec", "-T", "rabbitmq", "rabbitmqctl",
|
|
"list_queues", "name", "messages_ready", "messages_unacknowledged", "--silent")
|
|
for _, line := range strings.Split(out, "\n") {
|
|
fields := strings.Fields(line)
|
|
if len(fields) == 3 && fields[0] == "attachments.process.v1" {
|
|
return fields[1] + ":" + fields[2]
|
|
}
|
|
}
|
|
t.Fatal("attachment queue missing")
|
|
return ""
|
|
}
|
|
|
|
func workerUpload(t *testing.T, user testUser, chatID uuid.UUID) (dto.AttachmentResponse, []byte) {
|
|
t.Helper()
|
|
source := makePNG(t, 320, 200)
|
|
resp := uploadAttachment(t, user, chatID, "worker.png", "image/png", source)
|
|
requireStatus(t, resp, http.StatusAccepted)
|
|
var att dto.AttachmentResponse
|
|
mustDecode(t, resp, &att)
|
|
if att.Status != "queued" {
|
|
t.Fatalf("expected queued, got %s", att.Status)
|
|
}
|
|
return att, source
|
|
}
|
|
|
|
func workerContent(t *testing.T, user testUser, id uuid.UUID, variant string) []byte {
|
|
t.Helper()
|
|
resp := doRequest(t, http.MethodGet, "/api/v1/attachments/"+id.String()+"/content?variant="+variant, user.Token, nil)
|
|
requireStatus(t, resp, http.StatusOK)
|
|
defer resp.Body.Close()
|
|
data, err := io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return data
|
|
}
|
|
|
|
func TestWorkerPersistenceAndDuplicate(t *testing.T) {
|
|
user := createUser(t)
|
|
chat := createChat(t, user, user.ID)
|
|
stopService(t, "worker")
|
|
if workerQueue(t) != "0:0" {
|
|
t.Fatal("run on an idle stand with an empty attachment queue")
|
|
}
|
|
att, source := workerUpload(t, user, chat.ID)
|
|
where := "attachment_id='" + att.ID.String() + "'"
|
|
resp := postMessage(t, user, chat.ID, "", []uuid.UUID{att.ID})
|
|
requireStatus(t, resp, http.StatusCreated)
|
|
var message dto.MessageResponse
|
|
mustDecode(t, resp, &message)
|
|
resp = doRequest(t, http.MethodGet, "/api/v1/attachments/"+att.ID.String()+"/content", user.Token, nil)
|
|
requireStatus(t, resp, http.StatusConflict)
|
|
resp.Body.Close()
|
|
resp = sendMessage(t, user, chat.ID, "worker is stopped")
|
|
requireStatus(t, resp, http.StatusCreated)
|
|
resp.Body.Close()
|
|
workerWait(t, "confirmed publication", 15*time.Second, func() bool {
|
|
return workerSQL(t, "SELECT published_at IS NOT NULL FROM outbox WHERE "+where) == "t"
|
|
})
|
|
stopService(t, "api")
|
|
if workerQueue(t) != "1:0" {
|
|
t.Fatal("accepted job is not pending in broker")
|
|
}
|
|
before := workerCommand(t, "compose", "ps", "-q", "rabbitmq")
|
|
compose(t, "up", "-d", "--no-deps", "--force-recreate", "rabbitmq")
|
|
after := workerCommand(t, "compose", "ps", "-q", "rabbitmq")
|
|
if before == after {
|
|
t.Fatal("broker was not recreated")
|
|
}
|
|
workerWait(t, "broker health", 30*time.Second, func() bool {
|
|
return workerCommand(t, "inspect", "--format", "{{.State.Health.Status}}", after) == "healthy"
|
|
})
|
|
if workerQueue(t) != "1:0" {
|
|
t.Fatal("job lost after broker recreation with existing volume")
|
|
}
|
|
compose(t, "start", "api")
|
|
waitReady(t)
|
|
att = waitAttachment(t, user, att.ID, "queued", 15*time.Second)
|
|
resp = doRequest(t, http.MethodGet, "/api/v1/chats/"+chat.ID.String()+"/messages", user.Token, nil)
|
|
requireStatus(t, resp, http.StatusOK)
|
|
var page dto.MessagePageResponse
|
|
mustDecode(t, resp, &page)
|
|
found := false
|
|
for _, item := range page.Items {
|
|
if item.ID == message.ID && reflect.DeepEqual(item.AttachmentIDs, message.AttachmentIDs) {
|
|
found = true
|
|
}
|
|
}
|
|
if !found {
|
|
t.Fatal("message with attachment lost after API restart")
|
|
}
|
|
compose(t, "start", "worker")
|
|
ready := waitAttachment(t, user, att.ID, "ready", 30*time.Second)
|
|
if !bytes.Equal(workerContent(t, user, att.ID, "original"), source) {
|
|
t.Fatal("original changed")
|
|
}
|
|
thumb := workerContent(t, user, att.ID, "thumbnail")
|
|
cfg, err := jpeg.DecodeConfig(bytes.NewReader(thumb))
|
|
if err != nil || cfg.Width != 256 || cfg.Height != 160 {
|
|
t.Fatal("invalid JPEG thumbnail")
|
|
}
|
|
// Re-publish the same intent, modeling duplicate delivery after a lost publisher confirmation.
|
|
workerSQL(t, "UPDATE outbox SET published_at=NULL WHERE "+where)
|
|
workerWait(t, "duplicate publication", 15*time.Second, func() bool {
|
|
return workerSQL(t, "SELECT published_at IS NOT NULL FROM outbox WHERE "+where) == "t"
|
|
})
|
|
workerWait(t, "duplicate acknowledgement", 15*time.Second, func() bool { return workerQueue(t) == "0:0" })
|
|
duplicate := waitAttachment(t, user, att.ID, "ready", 30*time.Second)
|
|
if !reflect.DeepEqual(ready, duplicate) || workerSQL(t, "SELECT count(*) FROM attachment_variants WHERE "+where) != "1" {
|
|
t.Fatal("duplicate job changed metadata or duplicated variants")
|
|
}
|
|
if !bytes.Equal(workerContent(t, user, att.ID, "thumbnail"), thumb) ||
|
|
!bytes.Equal(workerContent(t, user, att.ID, "original"), source) {
|
|
t.Fatal("duplicate changed content")
|
|
}
|
|
t.Logf("PASS attachment=%s: queued survives API/broker restart; ready; duplicate acknowledged with unchanged result", att.ID)
|
|
}
|
|
|
|
func TestWorkerRetryExhausted(t *testing.T) {
|
|
user := createUser(t)
|
|
chat := createChat(t, user, user.ID)
|
|
stopService(t, "worker")
|
|
att, _ := workerUpload(t, user, chat.ID)
|
|
workerID := workerCommand(t, "compose", "ps", "-a", "-q", "worker")
|
|
var env []string
|
|
if err := json.Unmarshal([]byte(workerCommand(t, "inspect", "--format", "{{json .Config.Env}}", workerID)), &env); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
maxAttempts := 5
|
|
for _, entry := range env {
|
|
if value, ok := strings.CutPrefix(entry, "JOB_MAX_ATTEMPTS="); ok {
|
|
n, err := strconv.Atoi(value)
|
|
if err != nil || n < 1 {
|
|
t.Fatal("invalid JOB_MAX_ATTEMPTS")
|
|
}
|
|
maxAttempts = n
|
|
}
|
|
}
|
|
// A separate worker gets an unreachable S3 endpoint and a short attempt timeout.
|
|
// Keep the configured attempt count and normal backoff; leave the shared S3 running.
|
|
testWorker := "worker-retry-" + uuid.NewString()
|
|
t.Cleanup(func() {
|
|
workerCommand(t, "rm", "-f", testWorker)
|
|
})
|
|
workerCommand(t, "compose", "run", "-d", "--no-deps", "--name", testWorker,
|
|
"-e", "JOB_TIMEOUT_SECONDS=3", "-e", "S3_ENDPOINT=http://127.0.0.1:1", "worker")
|
|
failed := waitAttachment(t, user, att.ID, "failed", 90*time.Second)
|
|
if failed.ErrorCode == nil || *failed.ErrorCode != "retry_exhausted" {
|
|
t.Fatal("expected safe retry_exhausted error")
|
|
}
|
|
attempts := workerSQL(t, "SELECT attempts FROM attachments WHERE id='"+att.ID.String()+"'")
|
|
if attempts != strconv.Itoa(maxAttempts) {
|
|
t.Fatalf("expected %d attempts, got %s", maxAttempts, attempts)
|
|
}
|
|
resp := doRequest(t, http.MethodGet, "/api/v1/attachments/"+att.ID.String()+"/content", user.Token, nil)
|
|
requireStatus(t, resp, http.StatusConflict)
|
|
resp.Body.Close()
|
|
resp = postMessage(t, user, chat.ID, "", []uuid.UUID{att.ID})
|
|
requireStatus(t, resp, http.StatusConflict)
|
|
resp.Body.Close()
|
|
t.Logf("PASS attachment=%s: %d attempts, failed/retry_exhausted, content/link rejected", att.ID, maxAttempts)
|
|
}
|