forked from mtusi2026/mtusi2026lab5
255 lines
9.2 KiB
Go
255 lines
9.2 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"log/slog"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"highload/db/migrations"
|
|
"highload/internal/database/sqlc"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
"github.com/pressly/goose/v3"
|
|
)
|
|
|
|
func TestAsyncAttachmentsMigrationAndJobTransitions(t *testing.T) {
|
|
db, ctx := migrationPostgresFixture(t)
|
|
provider, err := goose.NewProvider(goose.DialectPostgres, db, migrations.Files,
|
|
goose.WithDisableGlobalRegistry(true), goose.WithSlog(slog.New(slog.NewTextHandler(io.Discard, nil))))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := provider.UpTo(ctx, 6); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
const original = "attachments/step-two-original"
|
|
_, err = db.ExecContext(ctx, `
|
|
INSERT INTO users (id, username, display_name, password_hash) VALUES
|
|
('00000000-0000-4000-8000-000000000001', 'step_two', 'Step two', 'synthetic');
|
|
INSERT INTO chats (id, title) VALUES ('00000000-0000-4000-8000-000000000002', 'Step two');
|
|
INSERT INTO chat_members VALUES
|
|
('00000000-0000-4000-8000-000000000002', '00000000-0000-4000-8000-000000000001');
|
|
INSERT INTO messages (id, chat_id, sender_id, text) VALUES
|
|
('00000000-0000-4000-8000-000000000003', '00000000-0000-4000-8000-000000000002',
|
|
'00000000-0000-4000-8000-000000000001', 'Old message');
|
|
INSERT INTO attachments (id, chat_id, owner_id, filename, object_key, content_type, size_bytes, sha256) VALUES
|
|
('00000000-0000-4000-8000-000000000004', '00000000-0000-4000-8000-000000000002',
|
|
'00000000-0000-4000-8000-000000000001', 'old.png', 'attachments/step-two-original',
|
|
'image/png', 123, repeat('a', 64));
|
|
INSERT INTO messages_attachments (message_id, attachment_id, chat_id, position) VALUES
|
|
('00000000-0000-4000-8000-000000000003', '00000000-0000-4000-8000-000000000004',
|
|
'00000000-0000-4000-8000-000000000002', 0);
|
|
`)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := provider.UpTo(ctx, 7); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := provider.Down(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var preservedKey string
|
|
if err := db.QueryRowContext(ctx, "SELECT object_key FROM attachments JOIN messages_attachments ON attachment_id = id").Scan(&preservedKey); err != nil || preservedKey != original {
|
|
t.Fatalf("rollback lost the legacy attachment reference: %v", err)
|
|
}
|
|
if _, err := provider.UpTo(ctx, 7); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := provider.UpTo(ctx, lab5MigrationCount); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
dsn, err := os.ReadFile(os.Getenv("DATABASE_URL_FILE"))
|
|
if err != nil {
|
|
t.Fatal("cannot read test connection configuration")
|
|
}
|
|
pool, err := pgxpool.New(ctx, string(dsn))
|
|
if err != nil {
|
|
t.Fatal("cannot configure test pool")
|
|
}
|
|
defer pool.Close()
|
|
q := sqlc.New(pool)
|
|
|
|
var jobID, attachmentID pgtype.UUID
|
|
var payload []byte
|
|
err = pool.QueryRow(ctx, `
|
|
SELECT j.id, a.id, o.payload
|
|
FROM messages_attachments AS m
|
|
JOIN attachments AS a ON a.id = m.attachment_id
|
|
JOIN attachment_jobs AS j ON j.attachment_id = a.id
|
|
JOIN outbox AS o ON o.job_id = j.id
|
|
WHERE a.status = 'queued' AND a.object_key = $1 AND a.staging_key = $1
|
|
AND o.dedup_key = 'initial/' || j.id
|
|
`, original).Scan(&jobID, &attachmentID, &payload)
|
|
if err != nil {
|
|
t.Fatalf("legacy attachment was not queued with its initial event: %v", err)
|
|
}
|
|
var event struct {
|
|
JobID string `json:"job_id"`
|
|
Attempt int `json:"attempt"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
}
|
|
if err := json.Unmarshal(payload, &event); err != nil || event.JobID != jobID.String() ||
|
|
event.Attempt != 1 || event.CreatedAt.IsZero() {
|
|
t.Fatalf("invalid backfill payload: %v", err)
|
|
}
|
|
|
|
start := make(chan struct{})
|
|
results := make(chan error, 2)
|
|
claim := sqlc.ClaimJobParams{
|
|
JobID: jobID, NewLeaseUntil: pgtype.Timestamptz{Time: time.Now().Add(time.Minute), Valid: true},
|
|
}
|
|
for range 2 {
|
|
go func() {
|
|
<-start
|
|
_, err := q.ClaimJob(ctx, claim)
|
|
results <- err
|
|
}()
|
|
}
|
|
close(start)
|
|
winners := 0
|
|
for range 2 {
|
|
switch err := <-results; {
|
|
case err == nil:
|
|
winners++
|
|
case errors.Is(err, pgx.ErrNoRows):
|
|
default:
|
|
t.Fatalf("concurrent claim failed: %v", err)
|
|
}
|
|
}
|
|
if winners != 1 {
|
|
t.Fatalf("gave the lease to %d workers", winners)
|
|
}
|
|
|
|
exec := func(query string, args ...any) {
|
|
t.Helper()
|
|
if _, err := pool.Exec(ctx, query, args...); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
exec("UPDATE attachment_jobs SET lease_until = clock_timestamp() - interval '1 second' WHERE id = $1", jobID)
|
|
for _, want := range []int{1, 0} {
|
|
recovery, err := q.EnqueueExpiredLeases(ctx, 10)
|
|
if err != nil || len(recovery) != want {
|
|
t.Fatalf("recovery events = %d, want %d: %v", len(recovery), want, err)
|
|
}
|
|
if want == 1 {
|
|
exec("UPDATE outbox SET sent_at = now() WHERE id = $1", recovery[0].ID)
|
|
}
|
|
}
|
|
row, err := q.ClaimJob(ctx, claim)
|
|
if err != nil || row.LeaseGeneration != 2 || row.AttemptsStarted != 2 {
|
|
t.Fatalf("expired lease was not replaced correctly: %v", err)
|
|
}
|
|
fail := sqlc.FailJobParams{JobID: jobID, LeaseGeneration: 1, ErrorCode: "invalid_image"}
|
|
if n, err := q.FailJob(ctx, fail); err != nil || n != 0 {
|
|
t.Fatalf("stale worker could fail attachment: rows=%d, err=%v", n, err)
|
|
}
|
|
retry := sqlc.ScheduleRetryParams{
|
|
JobID: jobID, LeaseGeneration: 2, ErrorCode: "s3_unavailable",
|
|
AvailableAt: pgtype.Timestamptz{Time: time.Now().Add(time.Minute), Valid: true},
|
|
}
|
|
if n, err := q.ScheduleRetry(ctx, retry); err != nil || n != 1 {
|
|
t.Fatalf("retry could not be scheduled: rows=%d, err=%v", n, err)
|
|
}
|
|
if _, err := q.ClaimJob(ctx, claim); !errors.Is(err, pgx.ErrNoRows) {
|
|
t.Fatalf("worker bypassed the retry delay: %v", err)
|
|
}
|
|
exec("UPDATE attachment_jobs SET next_attempt_at = now(), max_attempts = 3 WHERE id = $1", jobID)
|
|
row, err = q.ClaimJob(ctx, claim)
|
|
if err != nil || row.LeaseGeneration != 3 {
|
|
t.Fatalf("due retry could not claim job: %v", err)
|
|
}
|
|
exec("UPDATE attachment_jobs SET lease_until = clock_timestamp() - interval '1 second' WHERE id = $1", jobID)
|
|
if _, err := q.ClaimJob(ctx, claim); !errors.Is(err, pgx.ErrNoRows) {
|
|
t.Fatalf("worker bypassed the attempt limit: %v", err)
|
|
}
|
|
fail.LeaseGeneration = 3
|
|
fail.ErrorCode = "retry_exhausted"
|
|
if n, err := q.FailJob(ctx, fail); err != nil || n != 1 {
|
|
t.Fatalf("exhausted job could not be failed: rows=%d, err=%v", n, err)
|
|
}
|
|
var requestID pgtype.UUID
|
|
if err := requestID.Scan("00000000-0000-4000-8000-000000000005"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
replay := sqlc.ReplayFailedParams{
|
|
JobID: jobID, RequestID: requestID, OperatorID: "operator", Reason: "test replay", ExtraAttempts: 1,
|
|
}
|
|
for _, want := range []int64{1, 0} {
|
|
if n, err := q.ReplayFailed(ctx, replay); err != nil || n != want {
|
|
t.Fatalf("replay changed %d rows, want %d: %v", n, want, err)
|
|
}
|
|
}
|
|
row, err = q.ClaimJob(ctx, claim)
|
|
if err != nil || row.AttemptsStarted != 4 || row.LeaseGeneration != 4 {
|
|
t.Fatalf("replay did not preserve attempt and lease counters: %v", err)
|
|
}
|
|
|
|
tx, err := pool.Begin(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
locked := q.WithTx(tx)
|
|
if _, err := locked.LockActiveAttachmentJob(ctx, sqlc.LockActiveAttachmentJobParams{JobID: jobID, LeaseGeneration: 4}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
complete := sqlc.CompleteJobParams{JobID: jobID, LeaseGeneration: 4, ObjectKey: "results/original"}
|
|
if n, err := locked.CompleteJob(ctx, complete); err != nil || n != 0 {
|
|
t.Fatalf("published result without variants: rows=%d, err=%v", n, err)
|
|
}
|
|
variant := sqlc.CreateAttachmentVariantParams{
|
|
AttachmentID: attachmentID, PipelineVersion: 1, Name: "thumbnail", ObjectKey: "results/thumb",
|
|
ContentType: "image/jpeg", SizeBytes: 123, Width: 16, Height: 10, Sha256: strings.Repeat("a", 64),
|
|
}
|
|
if err := locked.CreateAttachmentVariant(ctx, variant); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n, err := locked.CompleteJob(ctx, complete); err != nil || n != 0 {
|
|
t.Fatalf("published result without crop and optimized: rows=%d, err=%v", n, err)
|
|
}
|
|
variant.Name, variant.ObjectKey = "crop", "results/crop"
|
|
if err := locked.CreateAttachmentVariant(ctx, variant); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
variant.Name, variant.ObjectKey, variant.ContentType = "optimized", "results/webp", "image/webp"
|
|
if err := locked.CreateAttachmentVariant(ctx, variant); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n, err := locked.CompleteJob(ctx, complete); err != nil || n != 1 {
|
|
t.Fatalf("current owner could not publish result: rows=%d, err=%v", n, err)
|
|
}
|
|
if err := tx.Commit(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := q.ClaimJob(ctx, claim); !errors.Is(err, pgx.ErrNoRows) {
|
|
t.Fatalf("worker claimed a ready job: %v", err)
|
|
}
|
|
published, err := q.GetPublishedOriginalKey(ctx, sqlc.GetPublishedOriginalKeyParams{
|
|
AttachmentID: attachmentID, PipelineVersion: 1,
|
|
})
|
|
if err != nil || published.String != complete.ObjectKey {
|
|
t.Fatalf("published original not recorded for its version: %v", err)
|
|
}
|
|
|
|
var attempts, events, replays int
|
|
err = pool.QueryRow(ctx, `
|
|
SELECT (SELECT count(*) FROM attachment_attempts WHERE job_id = $1 AND finished_at IS NOT NULL),
|
|
(SELECT count(*) FROM outbox WHERE job_id = $1),
|
|
(SELECT count(*) FROM attachment_replays WHERE job_id = $1)
|
|
`, jobID).Scan(&attempts, &events, &replays)
|
|
if err != nil || attempts != 4 || events != 4 || replays != 1 {
|
|
t.Fatalf("unexpected history: attempts=%d, events=%d, replays=%d: %v", attempts, events, replays, err)
|
|
}
|
|
}
|