test(reconcile): add consistency checks and report evidence

This commit is contained in:
kk10
2026-11-04 22:33:54 +03:00
parent 7148499e63
commit fd56de4895
6 changed files with 333 additions and 95 deletions
+1 -1
View File
@@ -134,7 +134,7 @@ services:
resources:
limits:
cpus: "0.5"
memory: 256M
memory: 512M
restart: unless-stopped
load_balancer:
+52
View File
@@ -0,0 +1,52 @@
> make reconcile-test-go
docker compose --env-file .env build reconcile
[+] Building 13.8s (18/18) FINISHED
=> [internal] load local bake definitions 0.0s
=> => reading from stdin 600B 0.0s
=> [internal] load build definition from Dockerfile.reconcile 0.0s
=> => transferring dockerfile: 342B 0.0s
=> [internal] load metadata for gcr.io/distroless/static-debian12:nonroot 1.4s
=> [internal] load metadata for docker.io/library/golang:1.27.1 1.3s
=> [auth] library/golang:pull token for registry-1.docker.io 0.0s
=> [internal] load .dockerignore 0.0s
=> => transferring context: 203B 0.0s
=> [build 1/6] FROM docker.io/library/golang:1.27.1@sha256:162be5298a40ed317005c8339c6de4d10d3eef336d66dc8e9259b03ab9d3a6d2 0.0s
=> [internal] load build context 0.0s
=> => transferring context: 11.25kB 0.0s
=> [stage-1 1/3] FROM gcr.io/distroless/static-debian12:nonroot@sha256:afa5c872c891853ca7fcf1f12c3edb23f7eeef36189728842dd51042ff57f7ab 0.0s
=> CACHED [build 2/6] WORKDIR /src 0.0s
=> CACHED [build 3/6] COPY go.mod go.sum ./ 0.0s
=> CACHED [build 4/6] RUN go mod download 0.0s
=> [build 5/6] COPY . . 0.1s
=> [build 6/6] RUN CGO_ENABLED=0 GOOS=linux go build -o /out/reconcile ./cmd/reconcile 11.9s
=> CACHED [stage-1 2/3] WORKDIR /app 0.0s
=> CACHED [stage-1 3/3] COPY --from=build /out/reconcile . 0.0s
=> exporting to image 0.0s
=> => exporting layers 0.0s
=> => writing image sha256:0a06bc3c0229755a92090161a0913af4d053556c4934087af6539b95c1d9b1cd 0.0s
=> => naming to docker.io/library/mtusi-messenger-reconcile:lab4 0.0s
=> resolving provenance for metadata file 0.0s
[+] build 1/1
✔ Image mtusi-messenger-reconcile:lab4 Built 13.9s
go test -count=1 -tags=reconcile ./student-tests/... -v -args -base-url=http://localhost:8080
=== RUN TestReconcileRepair
reconcile_test.go:85: dry_run=true objects=14606 rows=14612 list_pages=15 duration_ms=1813 orphan=1 missing=1 stale=0 skipped_fresh=0 skipped_referenced=0 deleted_objects=0 deleted_rows=0 marked_failed=0 errors=[]
reconcile_test.go:99: dry_run=false objects=14606 rows=14612 list_pages=15 duration_ms=1557 orphan=1 missing=1 stale=0 skipped_fresh=0 skipped_referenced=0 deleted_objects=1 deleted_rows=0 marked_failed=1 errors=[]
reconcile_test.go:109: dry_run=false objects=14605 rows=14612 list_pages=15 duration_ms=1836 orphan=0 missing=0 stale=0 skipped_fresh=0 skipped_referenced=0 deleted_objects=0 deleted_rows=0 marked_failed=0 errors=[]
reconcile_test.go:116: PASS: dry-run preserved data; orphan deleted; missing marked failed; repeat made no changes
--- PASS: TestReconcileRepair (8.81s)
=== RUN TestReconcileSafety
reconcile_test.go:137: dry_run=true objects=14607 rows=14614 list_pages=15 duration_ms=1630 orphan=0 missing=0 stale=0 skipped_fresh=1 skipped_referenced=1 deleted_objects=0 deleted_rows=0 marked_failed=0 errors=[]
reconcile_test.go:137: dry_run=false objects=14607 rows=14614 list_pages=15 duration_ms=1560 orphan=0 missing=0 stale=0 skipped_fresh=1 skipped_referenced=1 deleted_objects=0 deleted_rows=0 marked_failed=0 errors=[]
reconcile_test.go:161: PASS: fresh and old referenced uploads preserved, originals unchanged
--- PASS: TestReconcileSafety (5.03s)
=== RUN TestReconcileCost
reconcile_test.go:166: scan 1/3
reconcile_test.go:167: dry_run=true objects=14607 rows=14614 list_pages=15 duration_ms=1609 orphan=0 missing=0 stale=0 skipped_fresh=0 skipped_referenced=0 deleted_objects=0 deleted_rows=0 marked_failed=0 errors=[]
reconcile_test.go:166: scan 2/3
reconcile_test.go:167: dry_run=true objects=14607 rows=14614 list_pages=15 duration_ms=1721 orphan=0 missing=0 stale=0 skipped_fresh=0 skipped_referenced=0 deleted_objects=0 deleted_rows=0 marked_failed=0 errors=[]
reconcile_test.go:166: scan 3/3
reconcile_test.go:167: dry_run=true objects=14607 rows=14614 list_pages=15 duration_ms=1587 orphan=0 missing=0 stale=0 skipped_fresh=0 skipped_referenced=0 deleted_objects=0 deleted_rows=0 marked_failed=0 errors=[]
--- PASS: TestReconcileCost (6.92s)
PASS
ok mtusi2026-messenger/student-tests 20.784s
+106
View File
@@ -0,0 +1,106 @@
//go:build contract || reconcile
package studenttests_test
import (
"bytes"
"encoding/json"
"net/http"
"net/url"
"testing"
"github.com/google/uuid"
"mtusi2026-messenger/internal/dto"
)
const maxUploadBytes = 10 * 1024 * 1024
func uploadAttachment(
t *testing.T,
actor testUser,
chatID uuid.UUID,
filename string,
contentType string,
body []byte,
) *http.Response {
t.Helper()
req, err := http.NewRequest(
http.MethodPost,
*baseURL+"/api/v1/chats/"+chatID.String()+"/attachments?filename="+url.QueryEscape(filename),
bytes.NewReader(body),
)
if err != nil {
t.Fatalf("build upload request: %v", err)
}
req.Header.Set("Content-Type", contentType)
req.Header.Set("Authorization", "Bearer "+actor.Token)
resp, err := httpClient.Do(req)
if err != nil {
t.Fatalf("upload request failed: %v", err)
}
return resp
}
func uploadPNG(t *testing.T, actor testUser, chatID uuid.UUID) dto.AttachmentResponse {
t.Helper()
resp := uploadAttachment(t, actor, chatID, "sample.png", "image/png", makePNG(t, 32, 32))
requireStatus(t, resp, http.StatusCreated)
var att dto.AttachmentResponse
mustDecode(t, resp, &att)
return att
}
func postMessage(
t *testing.T,
actor testUser,
chatID uuid.UUID,
text string,
attachmentIDs []uuid.UUID,
) *http.Response {
t.Helper()
body, err := json.Marshal(map[string]any{
"text": text,
"attachment_ids": attachmentIDs,
})
if err != nil {
t.Fatalf("marshal message: %v", err)
}
req, err := http.NewRequest(
http.MethodPost,
*baseURL+"/api/v1/chats/"+chatID.String()+"/messages",
bytes.NewReader(body),
)
if err != nil {
t.Fatalf("build message request: %v", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+actor.Token)
req.Header.Set("Idempotency-Key", uuid.NewString())
resp, err := httpClient.Do(req)
if err != nil {
t.Fatalf("message request failed: %v", err)
}
return resp
}
func requireErrorCode(t *testing.T, resp *http.Response, code dto.ErrorCode) {
t.Helper()
var apiErr dto.ErrorResponse
mustDecode(t, resp, &apiErr)
if apiErr.Error.Code != code {
t.Errorf("expected error code %q, got %q", code, apiErr.Error.Code)
}
}
-93
View File
@@ -4,12 +4,10 @@ package studenttests_test
import (
"bytes"
"encoding/json"
"image/jpeg"
"image/png"
"io"
"net/http"
"net/url"
"testing"
"github.com/google/uuid"
@@ -17,97 +15,6 @@ import (
"mtusi2026-messenger/internal/dto"
)
const maxUploadBytes = 10 * 1024 * 1024
func uploadAttachment(
t *testing.T,
actor testUser,
chatID uuid.UUID,
filename string,
contentType string,
body []byte,
) *http.Response {
t.Helper()
req, err := http.NewRequest(
http.MethodPost,
*baseURL+"/api/v1/chats/"+chatID.String()+"/attachments?filename="+url.QueryEscape(filename),
bytes.NewReader(body),
)
if err != nil {
t.Fatalf("build upload request: %v", err)
}
req.Header.Set("Content-Type", contentType)
req.Header.Set("Authorization", "Bearer "+actor.Token)
resp, err := httpClient.Do(req)
if err != nil {
t.Fatalf("upload request failed: %v", err)
}
return resp
}
func uploadPNG(t *testing.T, actor testUser, chatID uuid.UUID) dto.AttachmentResponse {
t.Helper()
resp := uploadAttachment(t, actor, chatID, "sample.png", "image/png", makePNG(t, 32, 32))
requireStatus(t, resp, http.StatusCreated)
var att dto.AttachmentResponse
mustDecode(t, resp, &att)
return att
}
func postMessage(
t *testing.T,
actor testUser,
chatID uuid.UUID,
text string,
attachmentIDs []uuid.UUID,
) *http.Response {
t.Helper()
body, err := json.Marshal(map[string]any{
"text": text,
"attachment_ids": attachmentIDs,
})
if err != nil {
t.Fatalf("marshal message: %v", err)
}
req, err := http.NewRequest(
http.MethodPost,
*baseURL+"/api/v1/chats/"+chatID.String()+"/messages",
bytes.NewReader(body),
)
if err != nil {
t.Fatalf("build message request: %v", err)
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", "Bearer "+actor.Token)
req.Header.Set("Idempotency-Key", uuid.NewString())
resp, err := httpClient.Do(req)
if err != nil {
t.Fatalf("message request failed: %v", err)
}
return resp
}
func requireErrorCode(t *testing.T, resp *http.Response, code dto.ErrorCode) {
t.Helper()
var apiErr dto.ErrorResponse
mustDecode(t, resp, &apiErr)
if apiErr.Error.Code != code {
t.Errorf("expected error code %q, got %q", code, apiErr.Error.Code)
}
}
func TestAttachmentJPEG(t *testing.T) {
user := createUser(t)
chat := createChat(t, user, user.ID)
+169
View File
@@ -0,0 +1,169 @@
//go:build reconcile
package studenttests_test
import (
"bytes"
"context"
"encoding/json"
"io"
"net/http"
"os"
"os/exec"
"slices"
"strings"
"testing"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"mtusi2026-messenger/internal/reconcile"
)
func reconcileDB(t *testing.T) *pgx.Conn {
t.Helper()
secret, err := os.ReadFile("../secrets/test_database_url.txt")
if err != nil {
t.Fatalf("read test database URL: %v", err)
}
conn, err := pgx.Connect(context.Background(), strings.TrimSpace(string(secret)))
if err != nil {
t.Fatalf("connect to test database: %v", err)
}
t.Cleanup(func() { conn.Close(context.Background()) })
return conn
}
func reconcileSQL(t *testing.T, conn *pgx.Conn, query string, args ...any) {
t.Helper()
if _, err := conn.Exec(context.Background(), query, args...); err != nil {
t.Fatalf("prepare/check reconcile fixture: %v", err)
}
}
func runReconcile(t *testing.T, dryRun bool, grace string) reconcile.Report {
t.Helper()
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
defer cancel()
dry := "false"
if dryRun {
dry = "true"
}
cmd := exec.CommandContext(ctx, "docker", "compose", "--env-file", ".env", "run", "--rm", "--no-deps", "reconcile", "-json", "-dry-run="+dry, "-grace="+grace)
cmd.Dir = ".."
var stderr bytes.Buffer
cmd.Stderr = &stderr
output, err := cmd.Output()
if err != nil {
t.Fatalf("run reconciler: %v\n%s", err, stderr.String())
}
var report reconcile.Report
if err := json.Unmarshal(output, &report); err != nil {
t.Fatalf("decode reconcile report: %v\n%s", err, output)
}
t.Logf("dry_run=%t objects=%d rows=%d list_pages=%d duration_ms=%d orphan=%d missing=%d stale=%d skipped_fresh=%d skipped_referenced=%d deleted_objects=%d deleted_rows=%d marked_failed=%d errors=%v", report.DryRun, report.ScannedObjects, report.ScannedRows, report.ListPages, report.DurationMS, len(report.OrphanObjects), len(report.MissingAttachments), len(report.StaleUploads), report.SkippedFresh, report.SkippedReferenced, report.DeletedObjects, report.DeletedRows, report.MarkedFailed, report.Errors)
if len(report.Errors) != 0 {
t.Fatal("reconciler reported errors")
}
return report
}
func TestReconcileRepair(t *testing.T) {
conn := reconcileDB(t)
user := createUser(t)
chat := createChat(t, user, user.ID)
att := uploadPNG(t, user, chat.ID)
var originalKey string
if err := conn.QueryRow(context.Background(), "SELECT s3_key FROM attachments WHERE id=$1", att.ID).Scan(&originalKey); err != nil {
t.Fatal(err)
}
// Simulate lost metadata for the original object and metadata pointing at a missing object.
reconcileSQL(t, conn, "UPDATE attachments SET s3_key=$2 WHERE id=$1", att.ID, "chats/reconcile-missing/"+uuid.NewString())
// Only this experiment uses a shortened grace period, to avoid waiting an hour.
time.Sleep(1100 * time.Millisecond)
dry := runReconcile(t, true, "1s")
if !slices.Contains(dry.OrphanObjects, originalKey) || !slices.Contains(dry.MissingAttachments, att.ID) {
t.Fatal("dry-run did not detect both injected inconsistencies")
}
if dry.DeletedObjects != 0 || dry.DeletedRows != 0 || dry.MarkedFailed != 0 {
t.Fatal("dry-run changed data")
}
var status, code string
if err := conn.QueryRow(context.Background(), "SELECT status::text FROM attachments WHERE id=$1", att.ID).Scan(&status); err != nil {
t.Fatal(err)
}
if status != "ready" {
t.Fatalf("dry-run changed status: %s", status)
}
repair := runReconcile(t, false, "1s")
if repair.DeletedObjects < 1 || repair.MarkedFailed < 1 {
t.Fatal("repair did not apply fixes")
}
if err := conn.QueryRow(context.Background(), "SELECT status::text, error_code FROM attachments WHERE id=$1", att.ID).Scan(&status, &code); err != nil {
t.Fatal(err)
}
if status != "failed" || code != "object_missing" {
t.Fatalf("unexpected repaired state: %s %s", status, code)
}
again := runReconcile(t, false, "1s")
if slices.Contains(again.OrphanObjects, originalKey) || slices.Contains(again.MissingAttachments, att.ID) {
t.Fatal("fixture still requires repair on repeat")
}
if again.DeletedObjects != 0 || again.DeletedRows != 0 || again.MarkedFailed != 0 {
t.Fatal("repeat repair changed data; run on an idle test stand")
}
t.Log("PASS: dry-run preserved data; orphan deleted; missing marked failed; repeat made no changes")
}
func TestReconcileSafety(t *testing.T) {
conn := reconcileDB(t)
user := createUser(t)
chat := createChat(t, user, user.ID)
fresh := uploadPNG(t, user, chat.ID)
used := uploadPNG(t, user, chat.ID)
resp := postMessage(t, user, chat.ID, "", []uuid.UUID{used.ID})
requireStatus(t, resp, http.StatusCreated)
resp.Body.Close()
// Exercise both grace-period protection and protection of referenced queued uploads.
reconcileSQL(t, conn, "UPDATE attachments SET status='queued' WHERE id=$1", fresh.ID)
reconcileSQL(t, conn, "UPDATE attachments SET status='queued', created_at=now()-interval '2 hours' WHERE id=$1", used.ID)
t.Cleanup(func() {
if _, err := conn.Exec(context.Background(), "UPDATE attachments SET status='ready' WHERE id=ANY($1::uuid[])", []uuid.UUID{fresh.ID, used.ID}); err != nil {
t.Errorf("restore fixture status: %v", err)
}
})
for _, dryRun := range []bool{true, false} {
report := runReconcile(t, dryRun, "1h")
if report.SkippedFresh < 1 || report.SkippedReferenced < 1 {
t.Fatal("fresh/referenced protection was not exercised")
}
for _, att := range []uuid.UUID{fresh.ID, used.ID} {
var status string
if err := conn.QueryRow(context.Background(), "SELECT status::text FROM attachments WHERE id=$1", att).Scan(&status); err != nil {
t.Fatal(err)
}
if status != "queued" {
t.Fatalf("protected fixture changed status: %s", status)
}
resp := doRequest(t, http.MethodGet, "/api/v1/attachments/"+att.String()+"/content", user.Token, nil)
requireStatus(t, resp, http.StatusOK)
data, err := io.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(data, makePNG(t, 32, 32)) {
t.Fatal("protected original changed")
}
}
}
t.Log("PASS: fresh and old referenced uploads preserved, originals unchanged")
}
func TestReconcileCost(t *testing.T) {
for i := 1; i <= 3; i++ {
t.Logf("scan %d/3", i)
runReconcile(t, true, "1h")
}
}
+5 -1
View File
@@ -8,7 +8,7 @@ ACTION ?= stop-one.sh
ACTION_AT ?= 10s
DRY_RUN ?= true
.PHONY: help-go contract-go race-go load-go failure-go load-action-go reconcile
.PHONY: help-go contract-go race-go load-go failure-go load-action-go reconcile reconcile-test-go
help-go:
@echo "contract-go: contract tests"
@@ -38,3 +38,7 @@ load-action-go:
reconcile:
docker compose --env-file .env run --rm --no-deps reconcile -dry-run=$(DRY_RUN)
reconcile-test-go:
docker compose --env-file .env build reconcile
go test -count=1 -tags=reconcile ./student-tests/... -v -args -base-url=$(BASE_URL)