test(reconcile): add consistency checks and report evidence
This commit is contained in:
+1
-1
@@ -134,7 +134,7 @@ services:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
memory: 512M
|
||||
restart: unless-stopped
|
||||
|
||||
load_balancer:
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
> make reconcile-test-go
|
||||
docker compose --env-file .env build reconcile
|
||||
[+] Building 13.8s (18/18) FINISHED
|
||||
=> [internal] load local bake definitions 0.0s
|
||||
=> => reading from stdin 600B 0.0s
|
||||
=> [internal] load build definition from Dockerfile.reconcile 0.0s
|
||||
=> => transferring dockerfile: 342B 0.0s
|
||||
=> [internal] load metadata for gcr.io/distroless/static-debian12:nonroot 1.4s
|
||||
=> [internal] load metadata for docker.io/library/golang:1.27.1 1.3s
|
||||
=> [auth] library/golang:pull token for registry-1.docker.io 0.0s
|
||||
=> [internal] load .dockerignore 0.0s
|
||||
=> => transferring context: 203B 0.0s
|
||||
=> [build 1/6] FROM docker.io/library/golang:1.27.1@sha256:162be5298a40ed317005c8339c6de4d10d3eef336d66dc8e9259b03ab9d3a6d2 0.0s
|
||||
=> [internal] load build context 0.0s
|
||||
=> => transferring context: 11.25kB 0.0s
|
||||
=> [stage-1 1/3] FROM gcr.io/distroless/static-debian12:nonroot@sha256:afa5c872c891853ca7fcf1f12c3edb23f7eeef36189728842dd51042ff57f7ab 0.0s
|
||||
=> CACHED [build 2/6] WORKDIR /src 0.0s
|
||||
=> CACHED [build 3/6] COPY go.mod go.sum ./ 0.0s
|
||||
=> CACHED [build 4/6] RUN go mod download 0.0s
|
||||
=> [build 5/6] COPY . . 0.1s
|
||||
=> [build 6/6] RUN CGO_ENABLED=0 GOOS=linux go build -o /out/reconcile ./cmd/reconcile 11.9s
|
||||
=> CACHED [stage-1 2/3] WORKDIR /app 0.0s
|
||||
=> CACHED [stage-1 3/3] COPY --from=build /out/reconcile . 0.0s
|
||||
=> exporting to image 0.0s
|
||||
=> => exporting layers 0.0s
|
||||
=> => writing image sha256:0a06bc3c0229755a92090161a0913af4d053556c4934087af6539b95c1d9b1cd 0.0s
|
||||
=> => naming to docker.io/library/mtusi-messenger-reconcile:lab4 0.0s
|
||||
=> resolving provenance for metadata file 0.0s
|
||||
[+] build 1/1
|
||||
✔ Image mtusi-messenger-reconcile:lab4 Built 13.9s
|
||||
go test -count=1 -tags=reconcile ./student-tests/... -v -args -base-url=http://localhost:8080
|
||||
=== RUN TestReconcileRepair
|
||||
reconcile_test.go:85: dry_run=true objects=14606 rows=14612 list_pages=15 duration_ms=1813 orphan=1 missing=1 stale=0 skipped_fresh=0 skipped_referenced=0 deleted_objects=0 deleted_rows=0 marked_failed=0 errors=[]
|
||||
reconcile_test.go:99: dry_run=false objects=14606 rows=14612 list_pages=15 duration_ms=1557 orphan=1 missing=1 stale=0 skipped_fresh=0 skipped_referenced=0 deleted_objects=1 deleted_rows=0 marked_failed=1 errors=[]
|
||||
reconcile_test.go:109: dry_run=false objects=14605 rows=14612 list_pages=15 duration_ms=1836 orphan=0 missing=0 stale=0 skipped_fresh=0 skipped_referenced=0 deleted_objects=0 deleted_rows=0 marked_failed=0 errors=[]
|
||||
reconcile_test.go:116: PASS: dry-run preserved data; orphan deleted; missing marked failed; repeat made no changes
|
||||
--- PASS: TestReconcileRepair (8.81s)
|
||||
=== RUN TestReconcileSafety
|
||||
reconcile_test.go:137: dry_run=true objects=14607 rows=14614 list_pages=15 duration_ms=1630 orphan=0 missing=0 stale=0 skipped_fresh=1 skipped_referenced=1 deleted_objects=0 deleted_rows=0 marked_failed=0 errors=[]
|
||||
reconcile_test.go:137: dry_run=false objects=14607 rows=14614 list_pages=15 duration_ms=1560 orphan=0 missing=0 stale=0 skipped_fresh=1 skipped_referenced=1 deleted_objects=0 deleted_rows=0 marked_failed=0 errors=[]
|
||||
reconcile_test.go:161: PASS: fresh and old referenced uploads preserved, originals unchanged
|
||||
--- PASS: TestReconcileSafety (5.03s)
|
||||
=== RUN TestReconcileCost
|
||||
reconcile_test.go:166: scan 1/3
|
||||
reconcile_test.go:167: dry_run=true objects=14607 rows=14614 list_pages=15 duration_ms=1609 orphan=0 missing=0 stale=0 skipped_fresh=0 skipped_referenced=0 deleted_objects=0 deleted_rows=0 marked_failed=0 errors=[]
|
||||
reconcile_test.go:166: scan 2/3
|
||||
reconcile_test.go:167: dry_run=true objects=14607 rows=14614 list_pages=15 duration_ms=1721 orphan=0 missing=0 stale=0 skipped_fresh=0 skipped_referenced=0 deleted_objects=0 deleted_rows=0 marked_failed=0 errors=[]
|
||||
reconcile_test.go:166: scan 3/3
|
||||
reconcile_test.go:167: dry_run=true objects=14607 rows=14614 list_pages=15 duration_ms=1587 orphan=0 missing=0 stale=0 skipped_fresh=0 skipped_referenced=0 deleted_objects=0 deleted_rows=0 marked_failed=0 errors=[]
|
||||
--- PASS: TestReconcileCost (6.92s)
|
||||
PASS
|
||||
ok mtusi2026-messenger/student-tests 20.784s
|
||||
@@ -0,0 +1,106 @@
|
||||
//go:build contract || reconcile
|
||||
|
||||
package studenttests_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"mtusi2026-messenger/internal/dto"
|
||||
)
|
||||
|
||||
const maxUploadBytes = 10 * 1024 * 1024
|
||||
|
||||
func uploadAttachment(
|
||||
t *testing.T,
|
||||
actor testUser,
|
||||
chatID uuid.UUID,
|
||||
filename string,
|
||||
contentType string,
|
||||
body []byte,
|
||||
) *http.Response {
|
||||
t.Helper()
|
||||
|
||||
req, err := http.NewRequest(
|
||||
http.MethodPost,
|
||||
*baseURL+"/api/v1/chats/"+chatID.String()+"/attachments?filename="+url.QueryEscape(filename),
|
||||
bytes.NewReader(body),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("build upload request: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
req.Header.Set("Authorization", "Bearer "+actor.Token)
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("upload request failed: %v", err)
|
||||
}
|
||||
|
||||
return resp
|
||||
}
|
||||
|
||||
func uploadPNG(t *testing.T, actor testUser, chatID uuid.UUID) dto.AttachmentResponse {
|
||||
t.Helper()
|
||||
|
||||
resp := uploadAttachment(t, actor, chatID, "sample.png", "image/png", makePNG(t, 32, 32))
|
||||
requireStatus(t, resp, http.StatusCreated)
|
||||
|
||||
var att dto.AttachmentResponse
|
||||
mustDecode(t, resp, &att)
|
||||
|
||||
return att
|
||||
}
|
||||
|
||||
func postMessage(
|
||||
t *testing.T,
|
||||
actor testUser,
|
||||
chatID uuid.UUID,
|
||||
text string,
|
||||
attachmentIDs []uuid.UUID,
|
||||
) *http.Response {
|
||||
t.Helper()
|
||||
|
||||
body, err := json.Marshal(map[string]any{
|
||||
"text": text,
|
||||
"attachment_ids": attachmentIDs,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal message: %v", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(
|
||||
http.MethodPost,
|
||||
*baseURL+"/api/v1/chats/"+chatID.String()+"/messages",
|
||||
bytes.NewReader(body),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("build message request: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+actor.Token)
|
||||
req.Header.Set("Idempotency-Key", uuid.NewString())
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("message request failed: %v", err)
|
||||
}
|
||||
|
||||
return resp
|
||||
}
|
||||
|
||||
func requireErrorCode(t *testing.T, resp *http.Response, code dto.ErrorCode) {
|
||||
t.Helper()
|
||||
|
||||
var apiErr dto.ErrorResponse
|
||||
mustDecode(t, resp, &apiErr)
|
||||
|
||||
if apiErr.Error.Code != code {
|
||||
t.Errorf("expected error code %q, got %q", code, apiErr.Error.Code)
|
||||
}
|
||||
}
|
||||
@@ -4,12 +4,10 @@ package studenttests_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"testing"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -17,97 +15,6 @@ import (
|
||||
"mtusi2026-messenger/internal/dto"
|
||||
)
|
||||
|
||||
const maxUploadBytes = 10 * 1024 * 1024
|
||||
|
||||
func uploadAttachment(
|
||||
t *testing.T,
|
||||
actor testUser,
|
||||
chatID uuid.UUID,
|
||||
filename string,
|
||||
contentType string,
|
||||
body []byte,
|
||||
) *http.Response {
|
||||
t.Helper()
|
||||
|
||||
req, err := http.NewRequest(
|
||||
http.MethodPost,
|
||||
*baseURL+"/api/v1/chats/"+chatID.String()+"/attachments?filename="+url.QueryEscape(filename),
|
||||
bytes.NewReader(body),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("build upload request: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", contentType)
|
||||
req.Header.Set("Authorization", "Bearer "+actor.Token)
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("upload request failed: %v", err)
|
||||
}
|
||||
|
||||
return resp
|
||||
}
|
||||
|
||||
func uploadPNG(t *testing.T, actor testUser, chatID uuid.UUID) dto.AttachmentResponse {
|
||||
t.Helper()
|
||||
|
||||
resp := uploadAttachment(t, actor, chatID, "sample.png", "image/png", makePNG(t, 32, 32))
|
||||
requireStatus(t, resp, http.StatusCreated)
|
||||
|
||||
var att dto.AttachmentResponse
|
||||
mustDecode(t, resp, &att)
|
||||
|
||||
return att
|
||||
}
|
||||
|
||||
func postMessage(
|
||||
t *testing.T,
|
||||
actor testUser,
|
||||
chatID uuid.UUID,
|
||||
text string,
|
||||
attachmentIDs []uuid.UUID,
|
||||
) *http.Response {
|
||||
t.Helper()
|
||||
|
||||
body, err := json.Marshal(map[string]any{
|
||||
"text": text,
|
||||
"attachment_ids": attachmentIDs,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal message: %v", err)
|
||||
}
|
||||
|
||||
req, err := http.NewRequest(
|
||||
http.MethodPost,
|
||||
*baseURL+"/api/v1/chats/"+chatID.String()+"/messages",
|
||||
bytes.NewReader(body),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("build message request: %v", err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+actor.Token)
|
||||
req.Header.Set("Idempotency-Key", uuid.NewString())
|
||||
|
||||
resp, err := httpClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("message request failed: %v", err)
|
||||
}
|
||||
|
||||
return resp
|
||||
}
|
||||
|
||||
func requireErrorCode(t *testing.T, resp *http.Response, code dto.ErrorCode) {
|
||||
t.Helper()
|
||||
|
||||
var apiErr dto.ErrorResponse
|
||||
mustDecode(t, resp, &apiErr)
|
||||
|
||||
if apiErr.Error.Code != code {
|
||||
t.Errorf("expected error code %q, got %q", code, apiErr.Error.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAttachmentJPEG(t *testing.T) {
|
||||
user := createUser(t)
|
||||
chat := createChat(t, user, user.ID)
|
||||
|
||||
@@ -0,0 +1,169 @@
|
||||
//go:build reconcile
|
||||
|
||||
package studenttests_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
|
||||
"mtusi2026-messenger/internal/reconcile"
|
||||
)
|
||||
|
||||
func reconcileDB(t *testing.T) *pgx.Conn {
|
||||
t.Helper()
|
||||
secret, err := os.ReadFile("../secrets/test_database_url.txt")
|
||||
if err != nil {
|
||||
t.Fatalf("read test database URL: %v", err)
|
||||
}
|
||||
conn, err := pgx.Connect(context.Background(), strings.TrimSpace(string(secret)))
|
||||
if err != nil {
|
||||
t.Fatalf("connect to test database: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { conn.Close(context.Background()) })
|
||||
return conn
|
||||
}
|
||||
|
||||
func reconcileSQL(t *testing.T, conn *pgx.Conn, query string, args ...any) {
|
||||
t.Helper()
|
||||
if _, err := conn.Exec(context.Background(), query, args...); err != nil {
|
||||
t.Fatalf("prepare/check reconcile fixture: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func runReconcile(t *testing.T, dryRun bool, grace string) reconcile.Report {
|
||||
t.Helper()
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
defer cancel()
|
||||
dry := "false"
|
||||
if dryRun {
|
||||
dry = "true"
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "docker", "compose", "--env-file", ".env", "run", "--rm", "--no-deps", "reconcile", "-json", "-dry-run="+dry, "-grace="+grace)
|
||||
cmd.Dir = ".."
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
t.Fatalf("run reconciler: %v\n%s", err, stderr.String())
|
||||
}
|
||||
var report reconcile.Report
|
||||
if err := json.Unmarshal(output, &report); err != nil {
|
||||
t.Fatalf("decode reconcile report: %v\n%s", err, output)
|
||||
}
|
||||
t.Logf("dry_run=%t objects=%d rows=%d list_pages=%d duration_ms=%d orphan=%d missing=%d stale=%d skipped_fresh=%d skipped_referenced=%d deleted_objects=%d deleted_rows=%d marked_failed=%d errors=%v", report.DryRun, report.ScannedObjects, report.ScannedRows, report.ListPages, report.DurationMS, len(report.OrphanObjects), len(report.MissingAttachments), len(report.StaleUploads), report.SkippedFresh, report.SkippedReferenced, report.DeletedObjects, report.DeletedRows, report.MarkedFailed, report.Errors)
|
||||
if len(report.Errors) != 0 {
|
||||
t.Fatal("reconciler reported errors")
|
||||
}
|
||||
return report
|
||||
}
|
||||
|
||||
func TestReconcileRepair(t *testing.T) {
|
||||
conn := reconcileDB(t)
|
||||
user := createUser(t)
|
||||
chat := createChat(t, user, user.ID)
|
||||
att := uploadPNG(t, user, chat.ID)
|
||||
var originalKey string
|
||||
if err := conn.QueryRow(context.Background(), "SELECT s3_key FROM attachments WHERE id=$1", att.ID).Scan(&originalKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Simulate lost metadata for the original object and metadata pointing at a missing object.
|
||||
reconcileSQL(t, conn, "UPDATE attachments SET s3_key=$2 WHERE id=$1", att.ID, "chats/reconcile-missing/"+uuid.NewString())
|
||||
// Only this experiment uses a shortened grace period, to avoid waiting an hour.
|
||||
time.Sleep(1100 * time.Millisecond)
|
||||
dry := runReconcile(t, true, "1s")
|
||||
if !slices.Contains(dry.OrphanObjects, originalKey) || !slices.Contains(dry.MissingAttachments, att.ID) {
|
||||
t.Fatal("dry-run did not detect both injected inconsistencies")
|
||||
}
|
||||
if dry.DeletedObjects != 0 || dry.DeletedRows != 0 || dry.MarkedFailed != 0 {
|
||||
t.Fatal("dry-run changed data")
|
||||
}
|
||||
var status, code string
|
||||
if err := conn.QueryRow(context.Background(), "SELECT status::text FROM attachments WHERE id=$1", att.ID).Scan(&status); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status != "ready" {
|
||||
t.Fatalf("dry-run changed status: %s", status)
|
||||
}
|
||||
repair := runReconcile(t, false, "1s")
|
||||
if repair.DeletedObjects < 1 || repair.MarkedFailed < 1 {
|
||||
t.Fatal("repair did not apply fixes")
|
||||
}
|
||||
if err := conn.QueryRow(context.Background(), "SELECT status::text, error_code FROM attachments WHERE id=$1", att.ID).Scan(&status, &code); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status != "failed" || code != "object_missing" {
|
||||
t.Fatalf("unexpected repaired state: %s %s", status, code)
|
||||
}
|
||||
again := runReconcile(t, false, "1s")
|
||||
if slices.Contains(again.OrphanObjects, originalKey) || slices.Contains(again.MissingAttachments, att.ID) {
|
||||
t.Fatal("fixture still requires repair on repeat")
|
||||
}
|
||||
if again.DeletedObjects != 0 || again.DeletedRows != 0 || again.MarkedFailed != 0 {
|
||||
t.Fatal("repeat repair changed data; run on an idle test stand")
|
||||
}
|
||||
t.Log("PASS: dry-run preserved data; orphan deleted; missing marked failed; repeat made no changes")
|
||||
}
|
||||
|
||||
func TestReconcileSafety(t *testing.T) {
|
||||
conn := reconcileDB(t)
|
||||
user := createUser(t)
|
||||
chat := createChat(t, user, user.ID)
|
||||
fresh := uploadPNG(t, user, chat.ID)
|
||||
used := uploadPNG(t, user, chat.ID)
|
||||
resp := postMessage(t, user, chat.ID, "", []uuid.UUID{used.ID})
|
||||
requireStatus(t, resp, http.StatusCreated)
|
||||
resp.Body.Close()
|
||||
// Exercise both grace-period protection and protection of referenced queued uploads.
|
||||
reconcileSQL(t, conn, "UPDATE attachments SET status='queued' WHERE id=$1", fresh.ID)
|
||||
reconcileSQL(t, conn, "UPDATE attachments SET status='queued', created_at=now()-interval '2 hours' WHERE id=$1", used.ID)
|
||||
t.Cleanup(func() {
|
||||
if _, err := conn.Exec(context.Background(), "UPDATE attachments SET status='ready' WHERE id=ANY($1::uuid[])", []uuid.UUID{fresh.ID, used.ID}); err != nil {
|
||||
t.Errorf("restore fixture status: %v", err)
|
||||
}
|
||||
})
|
||||
for _, dryRun := range []bool{true, false} {
|
||||
report := runReconcile(t, dryRun, "1h")
|
||||
if report.SkippedFresh < 1 || report.SkippedReferenced < 1 {
|
||||
t.Fatal("fresh/referenced protection was not exercised")
|
||||
}
|
||||
for _, att := range []uuid.UUID{fresh.ID, used.ID} {
|
||||
var status string
|
||||
if err := conn.QueryRow(context.Background(), "SELECT status::text FROM attachments WHERE id=$1", att).Scan(&status); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if status != "queued" {
|
||||
t.Fatalf("protected fixture changed status: %s", status)
|
||||
}
|
||||
resp := doRequest(t, http.MethodGet, "/api/v1/attachments/"+att.String()+"/content", user.Token, nil)
|
||||
requireStatus(t, resp, http.StatusOK)
|
||||
data, err := io.ReadAll(resp.Body)
|
||||
resp.Body.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(data, makePNG(t, 32, 32)) {
|
||||
t.Fatal("protected original changed")
|
||||
}
|
||||
}
|
||||
}
|
||||
t.Log("PASS: fresh and old referenced uploads preserved, originals unchanged")
|
||||
}
|
||||
|
||||
func TestReconcileCost(t *testing.T) {
|
||||
for i := 1; i <= 3; i++ {
|
||||
t.Logf("scan %d/3", i)
|
||||
runReconcile(t, true, "1h")
|
||||
}
|
||||
}
|
||||
+5
-1
@@ -8,7 +8,7 @@ ACTION ?= stop-one.sh
|
||||
ACTION_AT ?= 10s
|
||||
DRY_RUN ?= true
|
||||
|
||||
.PHONY: help-go contract-go race-go load-go failure-go load-action-go reconcile
|
||||
.PHONY: help-go contract-go race-go load-go failure-go load-action-go reconcile reconcile-test-go
|
||||
|
||||
help-go:
|
||||
@echo "contract-go: contract tests"
|
||||
@@ -38,3 +38,7 @@ load-action-go:
|
||||
|
||||
reconcile:
|
||||
docker compose --env-file .env run --rm --no-deps reconcile -dry-run=$(DRY_RUN)
|
||||
|
||||
reconcile-test-go:
|
||||
docker compose --env-file .env build reconcile
|
||||
go test -count=1 -tags=reconcile ./student-tests/... -v -args -base-url=$(BASE_URL)
|
||||
|
||||
Reference in New Issue
Block a user