diff --git a/Makefile b/Makefile index 2fe43da..7b08dd6 100644 --- a/Makefile +++ b/Makefile @@ -8,11 +8,11 @@ TEST_DATABASE_URL ?= $(shell sed 's/@postgres:5432/@127.0.0.1:55432/' secrets/da TEST_REDIS_URL ?= redis://127.0.0.1:56379/0 TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help up down check validate test load test-persistence test-failover test-idempotency test-attachments test-memory test-go +.PHONY: help up down check validate test load test-persistence test-failover test-idempotency test-attachments test-contract test-memory test-cost test-reconcile test-go help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" - @echo "up/down: start/stop the stand, volumes kept; test-idempotency: one key in parallel via LB; test-attachments: attachment contract checks; test-go: Go tests against the running stand" + @echo "up/down: start/stop the stand, volumes kept; test-idempotency: one key in parallel via LB; test-attachments: attachment contract checks; test-memory: concurrent upload memory; test-cost: reconciler traversal cost; test-reconcile: reconciler dry-run; test-go: Go tests against the running stand" up: $(COMPOSE) up -d --build @@ -46,9 +46,18 @@ test-idempotency: test-attachments: BASE_URL="$(BASE_URL)" $(PYTHON) scripts/attachments-check.py +test-contract: + BASE_URL="$(BASE_URL)" $(PYTHON) scripts/contract-audit.py + test-memory: BASE_URL="$(BASE_URL)" $(PYTHON) scripts/upload-load.py --concurrency 10 --output evidence/upload-memory.json +test-cost: + $(PYTHON) scripts/track2-cost.py + +test-reconcile: + scripts/reconcile.sh -grace 0s + test-go: @TEST_DATABASE_URL="$(TEST_DATABASE_URL)" TEST_REDIS_URL="$(TEST_REDIS_URL)" go test -count=1 -v ./... diff --git a/cmd/reconcile/main.go b/cmd/reconcile/main.go index 3802957..5ad1d1d 100644 --- a/cmd/reconcile/main.go +++ b/cmd/reconcile/main.go @@ -5,8 +5,11 @@ import ( "encoding/json" "flag" "fmt" + "net/http" "os" + "sync/atomic" "time" + "uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" @@ -17,20 +20,31 @@ import ( "messenger/internal/repository/s3" ) +type countingTransport struct { + base http.RoundTripper + count atomic.Int64 +} + +func (t *countingTransport) RoundTrip(request *http.Request) (*http.Response, error) { + t.count.Add(1) + return t.base.RoundTrip(request) +} + func main() { - dryRun := flag.Bool("dry-run", true, "report only, do not delete orphan objects") + dryRun := flag.Bool("dry-run", true, "report only, do not delete or repair anything") grace := flag.Duration("grace", time.Hour, "minimum object age before it can be deleted") prefix := flag.String("prefix", "chats/", "object key prefix to scan") limit := flag.Int("limit", 10000, "maximum number of objects to scan") + repairMissing := flag.Bool("repair-missing", true, "delete metadata rows whose object is gone and that no message references") flag.Parse() - if err := run(*dryRun, *grace, *prefix, *limit); err != nil { + if err := run(*dryRun, *grace, *prefix, *limit, *repairMissing); err != nil { fmt.Fprintln(os.Stderr, "reconcile failed:", err) os.Exit(1) } } -func run(dryRun bool, grace time.Duration, prefix string, limit int) error { +func run(dryRun bool, grace time.Duration, prefix string, limit int, repairMissing bool) error { if grace < 0 { return fmt.Errorf("grace must not be negative") } @@ -55,10 +69,11 @@ func run(dryRun bool, grace time.Duration, prefix string, limit int) error { return err } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute) defer cancel() - store, err := s3.New(endpoint, region, accessKeyID, secretAccessKey, bucket) + transport := &countingTransport{base: http.DefaultTransport} + store, err := s3.NewWithTransport(endpoint, region, accessKeyID, secretAccessKey, bucket, transport) if err != nil { return err } @@ -77,8 +92,12 @@ func run(dryRun bool, grace time.Duration, prefix string, limit int) error { if err != nil { return err } + referenced, err := loadReferences(ctx, pool) + if err != nil { + return err + } - orphans, missing := reconcile.Plan(toObjects(scanned), rows, prefix, time.Now(), grace) + orphans, missing := reconcile.Plan(toObjects(scanned), rows, referenced, prefix, time.Now(), grace) if truncated { missing = nil } @@ -93,16 +112,50 @@ func run(dryRun bool, grace time.Duration, prefix string, limit int) error { Orphans: orphans, Missing: missing, } + dbQueries := 2 + if !dryRun { for _, orphan := range orphans { if err := store.Delete(ctx, orphan.Key); err != nil { report.DeleteFailures++ continue } - report.Deleted++ + report.DeletedObjects++ + } + if repairMissing { + for _, item := range missing { + if item.Referenced { + report.RetainedMissing++ + continue + } + id, err := uuid.Parse(item.ID) + if err != nil { + report.DeleteFailures++ + continue + } + tag, err := pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1`, id) + dbQueries++ + if err != nil { + report.DeleteFailures++ + continue + } + report.DeletedMetadata += int(tag.RowsAffected()) + } + } else { + for _, item := range missing { + if item.Referenced { + report.RetainedMissing++ + } + } } } + + report.S3Requests = transport.count.Load() + report.DBQueries = dbQueries report.DurationMS = time.Since(started).Milliseconds() + if report.DurationMS > 0 { + report.ObjectsPerSecond = float64(report.ScannedObjects) / (float64(report.DurationMS) / 1000) + } encoder := json.NewEncoder(os.Stdout) encoder.SetIndent("", " ") @@ -110,7 +163,7 @@ func run(dryRun bool, grace time.Duration, prefix string, limit int) error { return err } if report.DeleteFailures > 0 { - return fmt.Errorf("%d orphan deletions failed", report.DeleteFailures) + return fmt.Errorf("%d repair operations failed", report.DeleteFailures) } return nil } @@ -127,6 +180,26 @@ func loadMetadata(ctx context.Context, pool *pgxpool.Pool) ([]reconcile.Metadata }) } +func loadReferences(ctx context.Context, pool *pgxpool.Pool) (map[uuid.UUID]bool, error) { + rows, err := pool.Query(ctx, `SELECT DISTINCT attachment_id FROM message_attachments`) + if err != nil { + return nil, err + } + ids, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (uuid.UUID, error) { + var id uuid.UUID + err := row.Scan(&id) + return id, err + }) + if err != nil { + return nil, err + } + referenced := make(map[uuid.UUID]bool, len(ids)) + for _, id := range ids { + referenced[id] = true + } + return referenced, nil +} + func toObjects(scanned []s3.ObjectInfo) []reconcile.Object { objects := make([]reconcile.Object, len(scanned)) for i, info := range scanned { diff --git a/cmd/s3tool/main.go b/cmd/s3tool/main.go index 0a41e04..55eb134 100644 --- a/cmd/s3tool/main.go +++ b/cmd/s3tool/main.go @@ -6,6 +6,8 @@ import ( "fmt" "io" "os" + "strconv" + "sync" "time" "messenger/internal/config" @@ -79,6 +81,22 @@ func run(command string, args []string) error { return err } fmt.Printf("removed %s\n", args[0]) + case "seed": + if len(args) != 3 { + usage() + } + count, err := strconv.Atoi(args[1]) + if err != nil || count <= 0 { + return fmt.Errorf("count must be a positive integer") + } + size, err := strconv.Atoi(args[2]) + if err != nil || size <= 0 { + return fmt.Errorf("size must be a positive integer") + } + if err := seed(ctx, store, args[0], count, size); err != nil { + return err + } + fmt.Printf("seeded %d objects under %s\n", count, args[0]) default: usage() } @@ -105,8 +123,36 @@ func newStore() (*s3.Store, error) { accessKeyID, secretAccessKey, getenv("S3_BUCKET", "messenger-attachments")) } +func seed(ctx context.Context, store *s3.Store, prefix string, count, size int) error { + content := make([]byte, size) + workers := 8 + work := make(chan int) + errors := make(chan error, workers) + var group sync.WaitGroup + for i := 0; i < workers; i++ { + group.Add(1) + go func() { + defer group.Done() + for index := range work { + key := fmt.Sprintf("%s/%06d", prefix, index) + if err := store.Put(ctx, key, bytes.NewReader(content), int64(size), "application/octet-stream"); err != nil { + errors <- err + return + } + } + }() + } + for index := 0; index < count; index++ { + work <- index + } + close(work) + group.Wait() + close(errors) + return <-errors +} + func usage() { - fmt.Fprintln(os.Stderr, "usage: s3tool put | delete | list [prefix] | buckets | remove-bucket ") + fmt.Fprintln(os.Stderr, "usage: s3tool put | delete | list [prefix] | buckets | remove-bucket | seed ") os.Exit(2) } diff --git a/evidence/check.txt b/evidence/check.txt index fe07159..5a9f1da 100644 --- a/evidence/check.txt +++ b/evidence/check.txt @@ -1,4 +1,4 @@ -2026-11-20T04:20:26+03:00 +2026-11-20T04:38:30+03:00 make check python3 scripts/check_contract.py diff --git a/evidence/compensation.txt b/evidence/compensation.txt index 3c29d6c..014bd69 100644 --- a/evidence/compensation.txt +++ b/evidence/compensation.txt @@ -1,4 +1,4 @@ -2026-11-20T04:14:17+03:00 +2026-11-20T04:41:27+03:00 failure between S3 write and database commit: an object must not stay unaccounted $ scripts/reconcile.sh # baseline @@ -6,21 +6,19 @@ $ scripts/reconcile.sh # baseline "prefix": "chats/", "grace_seconds": 3600, "dry_run": true, - "scanned_objects": 15, - "scanned_rows": 16, + "scanned_objects": 17, + "scanned_rows": 17, "truncated": false, "orphans": [], - "missing": [ - { - "id": "97121cfe-6b29-4227-867f-3188e72a725c", - "object_key": "chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c", - "chat_id": "9452049e-b86c-4d9a-8060-d98f6966b956", - "created_at": "2026-11-20T01:11:14Z" - } - ], + "missing": [], "deleted_objects": 0, + "deleted_metadata": 0, + "retained_missing": 0, "delete_failures": 0, - "duration_ms": 8 + "s3_requests": 1, + "db_queries": 2, + "duration_ms": 7, + "objects_per_second": 2428.5714285714284 } $ podman exec -i postgres psql < scripts/sql/inject-db-failure.sql @@ -37,21 +35,19 @@ $ scripts/reconcile.sh # same object count: compensating DELETE removed the ob "prefix": "chats/", "grace_seconds": 3600, "dry_run": true, - "scanned_objects": 15, - "scanned_rows": 16, + "scanned_objects": 17, + "scanned_rows": 17, "truncated": false, "orphans": [], - "missing": [ - { - "id": "97121cfe-6b29-4227-867f-3188e72a725c", - "object_key": "chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c", - "chat_id": "9452049e-b86c-4d9a-8060-d98f6966b956", - "created_at": "2026-11-20T01:11:14Z" - } - ], + "missing": [], "deleted_objects": 0, + "deleted_metadata": 0, + "retained_missing": 0, "delete_failures": 0, - "duration_ms": 7 + "s3_requests": 1, + "db_queries": 2, + "duration_ms": 7, + "objects_per_second": 2428.5714285714284 } $ podman exec -i postgres psql < scripts/sql/reset-db-failure.sql @@ -60,26 +56,24 @@ DROP FUNCTION $ scripts/upload-once.py --expect 201 # service healthy again status=201 -attachment=a6844b91-5b7b-4694-93fc-87d808b687fb size=653 +attachment=7dd89013-32f1-4732-a2fd-529fe2081bb8 size=653 $ scripts/reconcile.sh # one more accounted object { "prefix": "chats/", "grace_seconds": 3600, "dry_run": true, - "scanned_objects": 16, - "scanned_rows": 17, + "scanned_objects": 18, + "scanned_rows": 18, "truncated": false, "orphans": [], - "missing": [ - { - "id": "97121cfe-6b29-4227-867f-3188e72a725c", - "object_key": "chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c", - "chat_id": "9452049e-b86c-4d9a-8060-d98f6966b956", - "created_at": "2026-11-20T01:11:14Z" - } - ], + "missing": [], "deleted_objects": 0, + "deleted_metadata": 0, + "retained_missing": 0, "delete_failures": 0, - "duration_ms": 7 + "s3_requests": 1, + "db_queries": 2, + "duration_ms": 7, + "objects_per_second": 2571.4285714285716 } diff --git a/evidence/contract-audit.txt b/evidence/contract-audit.txt new file mode 100644 index 0000000..7005e11 --- /dev/null +++ b/evidence/contract-audit.txt @@ -0,0 +1,35 @@ +2026-11-20T04:53:36+03:00 +make test-contract + +BASE_URL="http://localhost:8080" python3 scripts/contract-audit.py +ok: explicit attachment_ids=null is 400 +ok: empty variant is 400 +ok: whitespace-only filename is 400 +ok: Content-Type parameters are accepted +ok: 41,000,000 pixels is 413 +ok: text length bounds +ok: attachment count bounds +ok: duplicate attachment_ids is 400 +ok: idempotent replay with the same set +ok: absent and empty attachment_ids agree +ok: attachment from another chat is 404 +ok: unknown attachment id is 404 +ok: outsider access is 404 +ok: upload to unknown chat is 404 +ok: media errors are 415 +ok: oversize is 413 +ok: filename errors are 400 +ok: chat validation errors are 400 +ok: pagination errors are 400 +ok: idempotency errors +ok: null request fields are 400 +ok: more than 100 members is 400 +ok: user field validation +ok: missing Content-Type is 415 +ok: variant is case sensitive +ok: filename control characters are 400 +ok: chat title bounds +ok: cursor is bound to the endpoint +ok: 204 responses carry headers and no body +ok: download Content-Length matches the bytes +PASS: 30 checks diff --git a/evidence/credentials.txt b/evidence/credentials.txt index fe15a87..ea758b3 100644 --- a/evidence/credentials.txt +++ b/evidence/credentials.txt @@ -1,4 +1,4 @@ -2026-11-20T04:17:09+03:00 +2026-11-20T04:43:58+03:00 separate restricted S3 credentials for the API and idempotent bucket bootstrap $ podman-compose run --rm s3-init (first run) diff --git a/evidence/dataset.txt b/evidence/dataset.txt index 49c5f8b..570dffc 100644 --- a/evidence/dataset.txt +++ b/evidence/dataset.txt @@ -1,7 +1,7 @@ -2026-11-20T04:18:05+03:00 +2026-11-20T04:56:32+03:00 users | chats | messages | attachments | links | db_size -------+-------+----------+-------------+-------+--------- - 58 | 17 | 303 | 30 | 2 | 8534 kB + 125 | 43 | 634 | 101 | 56 | 8958 kB (1 row) diff --git a/evidence/env.txt b/evidence/env.txt index e39418c..828b647 100644 --- a/evidence/env.txt +++ b/evidence/env.txt @@ -1,10 +1,10 @@ -2026-11-20T04:18:05+03:00 +2026-11-20T04:56:17+03:00 Linux archlinux 7.3.5-arch1-1 #1 SMP PREEMPT_DYNAMIC Tue, 17 Nov 2026 22:28:33 +0000 x86_64 GNU/Linux 12 total used free shared buff/cache available -Mem: 15Gi 6.2Gi 2.7Gi 476Mi 7.1Gi 9.2Gi -Swap: 8.0Gi 57Mi 7.9Gi +Mem: 15Gi 6.5Gi 3.6Gi 440Mi 5.8Gi 8.8Gi +Swap: 8.0Gi 187Mi 7.8Gi podman version 6.2.0 podman version 6.2.0 podman-compose version 1.6.0 @@ -16,4 +16,4 @@ docker.io/library/postgres:18.6-alpine3.24 57b23a0326ee docker.io/library/redis:8.10.3-alpine3.23 9722accf3e8e docker.io/migrate/migrate:v4.20.1 719a6f5d0ade docker.io/rustfs/rustfs:1.0.6 7cd1c573d76d -localhost/messenger-api:lab4 e2582dea4a72 +localhost/messenger-api:lab4 4c57d3f41dfb diff --git a/evidence/failover-1.txt b/evidence/failover-1.txt index 3d4e2ef..e7d810a 100644 --- a/evidence/failover-1.txt +++ b/evidence/failover-1.txt @@ -1,7 +1,7 @@ -2026-11-20T04:14:49+03:00 +2026-11-20T04:55:04+03:00 make test-failover ACTION_SCRIPT=scripts/stop-one.sh python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 101, "failed_attempts": 0, "first_recovery_seconds": 3.304} +{"observed_instances_before": 2, "successful_writes": 88, "failed_attempts": 0, "first_recovery_seconds": 3.256} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-2.txt b/evidence/failover-2.txt index 6346306..ecec6f6 100644 --- a/evidence/failover-2.txt +++ b/evidence/failover-2.txt @@ -1,7 +1,7 @@ -2026-11-20T04:15:25+03:00 +2026-11-20T04:55:38+03:00 make test-failover ACTION_SCRIPT=scripts/stop-one.sh python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 86, "failed_attempts": 0, "first_recovery_seconds": 3.228} +{"observed_instances_before": 2, "successful_writes": 99, "failed_attempts": 0, "first_recovery_seconds": 3.236} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/idempotency-race.txt b/evidence/idempotency-race.txt index 7cd908e..c9f24b2 100644 --- a/evidence/idempotency-race.txt +++ b/evidence/idempotency-race.txt @@ -1,4 +1,4 @@ -2026-11-20T04:20:36+03:00 +2026-11-20T04:53:46+03:00 make test-idempotency BASE_URL="http://localhost:8080" python3 scripts/idempotency-race.py @@ -7,5 +7,5 @@ successful responses (201): 20 failed requests: 0 [] distinct message ids in responses: 1 messages stored in history: 1 -instances that served requests: ['8535d7918d12', 'ad1700491457'] +instances that served requests: ['72c9d7b6cc57', 'e6d2e0d15c36'] OK: exactly one message created diff --git a/evidence/instances.txt b/evidence/instances.txt index 603ef19..1e21226 100644 --- a/evidence/instances.txt +++ b/evidence/instances.txt @@ -1,8 +1,8 @@ -2026-11-20T04:21:39+03:00 +2026-11-20T04:56:16+03:00 API replicas and single entrypoint -messenger-lab4_api_1 Up 26 seconds (healthy) -messenger-lab4_api_2 Up 26 seconds (healthy) +messenger-lab4_api_1 Up 53 seconds (healthy) +messenger-lab4_api_2 Up 21 seconds (healthy) messenger-lab4_lb_1 Up About a minute messenger-lab4_postgres_1 Up About a minute (healthy) messenger-lab4_redis_1 Up About a minute (healthy) @@ -19,5 +19,5 @@ messenger-lab4_s3_1 9000-9001/tcp X-Instance-Id distribution over 20 requests through LB - 10 x-instance-id: 33839df0eea0 - 10 x-instance-id: 4e3d4c4f1194 + 10 x-instance-id: a089fe9cc382 + 10 x-instance-id: f309c2ef4e46 diff --git a/evidence/limits.txt b/evidence/limits.txt index 7b66ee3..358d425 100644 --- a/evidence/limits.txt +++ b/evidence/limits.txt @@ -1,4 +1,4 @@ -2026-11-20T04:17:58+03:00 +2026-11-20T04:56:16+03:00 per-container limits, networks, secrets messenger-lab4_api_1 nano_cpus=1000000000 memory=536870912 networks=messenger-lab4_backend messenger-lab4_edge diff --git a/evidence/memory.txt b/evidence/memory.txt index b38b149..78ea566 100644 --- a/evidence/memory.txt +++ b/evidence/memory.txt @@ -1,5 +1,5 @@ -2026-11-20T04:21:28+03:00 -10 concurrent uploads of a 8000x5000 PNG (40,000,000 pixels) per replica; API memory limit 512 MiB; MAX_CONCURRENT_UPLOADS=8, MAX_CONCURRENT_DECODES=1 +2026-11-20T04:54:15+03:00 +10 concurrent uploads of a 8000x5000 PNG (40,000,000 pixels) per replica; API limit 512 MiB; MAX_CONCURRENT_UPLOADS=8, MAX_CONCURRENT_DECODES=1 BASE_URL="http://localhost:8080" python3 scripts/upload-load.py --concurrency 10 --output evidence/upload-memory.json { @@ -7,30 +7,30 @@ BASE_URL="http://localhost:8080" python3 scripts/upload-load.py --concurrency 10 "uploads": 10, "fixture": "large.png", "fixture_bytes": 126609, - "elapsed_seconds": 1.235, + "elapsed_seconds": 1.009, "successful": 10, "failed": 0, "failures": [], "latency_ms": { - "p50": 663.7, - "p95": 1224.7, - "p99": 1224.7 + "p50": 660.9, + "p95": 999.3, + "p99": 999.3 }, "api_containers": [ "messenger-lab4_api_1", "messenger-lab4_api_2" ], "peak_bytes_before": { - "messenger-lab4_api_1": 5050368, - "messenger-lab4_api_2": 4820992 + "messenger-lab4_api_1": 5951488, + "messenger-lab4_api_2": 4993024 }, "peak_bytes_after": { - "messenger-lab4_api_1": 368119808, - "messenger-lab4_api_2": 368185344 + "messenger-lab4_api_1": 348913664, + "messenger-lab4_api_2": 348721152 }, "peak_delta_bytes": { - "messenger-lab4_api_1": 363069440, - "messenger-lab4_api_2": 363364352 + "messenger-lab4_api_1": 342962176, + "messenger-lab4_api_2": 343728128 }, - "peak_max_mb": 351.13 + "peak_max_mb": 332.75 } diff --git a/evidence/migrate.txt b/evidence/migrate.txt index c151ff5..6cd4b2c 100644 --- a/evidence/migrate.txt +++ b/evidence/migrate.txt @@ -1,13 +1,9 @@ -2026-11-20T04:18:05+03:00 +2026-11-20T04:56:17+03:00 migrate container logs and schema version -1/u init (17.596326ms) -2/u message_idempotency (23.558514ms) -3/u attachments (35.669736ms) -no change -no change -no change -no change +1/u init (18.672238ms) +2/u message_idempotency (27.931065ms) +3/u attachments (37.481418ms) no change no change diff --git a/evidence/negative.txt b/evidence/negative.txt index d6231ce..2b4f163 100644 --- a/evidence/negative.txt +++ b/evidence/negative.txt @@ -1,4 +1,4 @@ -2026-11-20T04:20:27+03:00 +2026-11-20T04:53:36+03:00 make test-attachments BASE_URL="http://localhost:8080" python3 scripts/attachments-check.py diff --git a/evidence/persistence.txt b/evidence/persistence.txt index 08e52e5..a9ff82b 100644 --- a/evidence/persistence.txt +++ b/evidence/persistence.txt @@ -1,4 +1,4 @@ -2026-11-20T04:19:38+03:00 +2026-11-20T04:54:22+03:00 make test-persistence ACTION_SCRIPT=scripts/restart.sh python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" @@ -7,12 +7,12 @@ PASS: persistence preserved confirmed data and existing session. Hook scope need --- hook scope: containers and volumes before/after scripts/restart.sh --- -messenger-lab4_api_1 8535d7918d12 Up 1 second (starting) -messenger-lab4_api_2 ad1700491457 Up 1 second (starting) -messenger-lab4_lb_1 0bed57f994fa Up 1 second -messenger-lab4_postgres_1 53c091ca7f0a Up 1 second (starting) -messenger-lab4_redis_1 e0cfe44337f5 Up 1 second (healthy) -messenger-lab4_s3_1 8548fb0864af Up 1 second (healthy) +messenger-lab4_api_1 a089fe9cc382 Up 5 seconds (healthy) +messenger-lab4_api_2 f309c2ef4e46 Up 5 seconds (healthy) +messenger-lab4_lb_1 efc7da123526 Up 5 seconds +messenger-lab4_postgres_1 c64e954cf92a Up 5 seconds (healthy) +messenger-lab4_redis_1 90ac9f642b09 Up 5 seconds (healthy) +messenger-lab4_s3_1 6ae98c5aef4f Up 5 seconds (healthy) messenger-lab4_pgdata messenger-lab4_redisdata messenger-lab4_s3data @@ -20,12 +20,12 @@ messenger-lab4_s3data $ scripts/restart.sh -messenger-lab4_api_1 8535d7918d12 Up 15 seconds (healthy) -messenger-lab4_api_2 ad1700491457 Up 16 seconds (healthy) -messenger-lab4_lb_1 0bed57f994fa Up 16 seconds -messenger-lab4_postgres_1 53c091ca7f0a Up 15 seconds (healthy) -messenger-lab4_redis_1 e0cfe44337f5 Up 15 seconds (healthy) -messenger-lab4_s3_1 8548fb0864af Up 16 seconds (healthy) +messenger-lab4_api_1 a089fe9cc382 Up 15 seconds (healthy) +messenger-lab4_api_2 f309c2ef4e46 Up 15 seconds (healthy) +messenger-lab4_lb_1 efc7da123526 Up 15 seconds +messenger-lab4_postgres_1 c64e954cf92a Up 15 seconds (healthy) +messenger-lab4_redis_1 90ac9f642b09 Up 15 seconds (healthy) +messenger-lab4_s3_1 6ae98c5aef4f Up 15 seconds (healthy) messenger-lab4_pgdata messenger-lab4_redisdata messenger-lab4_s3data diff --git a/evidence/private-bucket.txt b/evidence/private-bucket.txt index 5bcedb2..9416439 100644 --- a/evidence/private-bucket.txt +++ b/evidence/private-bucket.txt @@ -1,20 +1,22 @@ -2026-11-20T04:15:49+03:00 +2026-11-20T04:44:19+03:00 bucket privacy and disabled console (checks outside the application API) -$ wget http://s3:9000/messenger-attachments/chats/.../... (anonymous, from backend network) -Connecting to s3:9000 (10.89.5.36:9000) +$ anonymous GET of an existing object from the backend network +Connecting to s3:9000 (10.89.5.22:9000) HTTP/1.1 403 Forbidden wget: server returned error: HTTP/1.1 403 Forbidden $ anonymous bucket listing -Connecting to s3:9000 (10.89.5.36:9000) +Connecting to s3:9000 (10.89.5.22:9000) HTTP/1.1 403 Forbidden wget: server returned error: HTTP/1.1 403 Forbidden -$ console port check inside s3 container +$ console port inside the s3 container console_http=000 console-closed $ published ports +messenger-lab4_api_1 8080/tcp messenger-lab4_api_2 8080/tcp +messenger-lab4_lb_1 0.0.0.0:8080->8080/tcp messenger-lab4_s3_1 9000-9001/tcp diff --git a/evidence/reconcile.txt b/evidence/reconcile.txt index cd84520..7f57a9f 100644 --- a/evidence/reconcile.txt +++ b/evidence/reconcile.txt @@ -1,4 +1,4 @@ -2026-11-20T04:13:27+03:00 +2026-11-20T04:41:03+03:00 reconciler drill: baseline, grace period, dry-run, apply, repeat $ scripts/reconcile.sh # dry-run, grace 1h, clean baseline @@ -6,54 +6,69 @@ $ scripts/reconcile.sh # dry-run, grace 1h, clean baseline "prefix": "chats/", "grace_seconds": 3600, "dry_run": true, - "scanned_objects": 15, - "scanned_rows": 15, + "scanned_objects": 18, + "scanned_rows": 18, "truncated": false, "orphans": [], "missing": [], "deleted_objects": 0, + "deleted_metadata": 0, + "retained_missing": 0, "delete_failures": 0, - "duration_ms": 7 + "s3_requests": 1, + "db_queries": 2, + "duration_ms": 7, + "objects_per_second": 2571.4285714285716 } $ scripts/s3tool.sh put chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000 - < scripts/fixtures/sample.jpg put chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000 (653 bytes) -$ scripts/reconcile.sh # fresh orphan stays: grace 1h +$ scripts/reconcile.sh # fresh orphan skipped by grace { "prefix": "chats/", "grace_seconds": 3600, "dry_run": true, - "scanned_objects": 16, - "scanned_rows": 15, + "scanned_objects": 19, + "scanned_rows": 18, "truncated": false, "orphans": [], "missing": [], "deleted_objects": 0, + "deleted_metadata": 0, + "retained_missing": 0, "delete_failures": 0, - "duration_ms": 7 + "s3_requests": 1, + "db_queries": 2, + "duration_ms": 8, + "objects_per_second": 2375 } -$ scripts/reconcile.sh -grace 0s # orphan detected, dry-run keeps it +$ scripts/reconcile.sh -grace 0s # detected, dry-run keeps it { "prefix": "chats/", "grace_seconds": 0, "dry_run": true, - "scanned_objects": 16, - "scanned_rows": 15, + "scanned_objects": 19, + "scanned_rows": 18, "truncated": false, "orphans": [ { "key": "chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000", "size_bytes": 653, - "last_modified": "2026-11-20T01:13:27Z", + "last_modified": "2026-11-20T01:41:03Z", "age_seconds": 0 } ], "missing": [], "deleted_objects": 0, + "deleted_metadata": 0, + "retained_missing": 0, "delete_failures": 0, - "duration_ms": 6 + "s3_requests": 1, + "db_queries": 2, + "duration_ms": 7, + "objects_per_second": 2714.285714285714 } $ scripts/reconcile.sh -grace 0s -dry-run=false # apply @@ -61,21 +76,26 @@ $ scripts/reconcile.sh -grace 0s -dry-run=false # apply "prefix": "chats/", "grace_seconds": 0, "dry_run": false, - "scanned_objects": 16, - "scanned_rows": 15, + "scanned_objects": 19, + "scanned_rows": 18, "truncated": false, "orphans": [ { "key": "chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000", "size_bytes": 653, - "last_modified": "2026-11-20T01:13:27Z", - "age_seconds": 3 + "last_modified": "2026-11-20T01:41:03Z", + "age_seconds": 4 } ], "missing": [], "deleted_objects": 1, + "deleted_metadata": 0, + "retained_missing": 0, "delete_failures": 0, - "duration_ms": 12 + "s3_requests": 2, + "db_queries": 2, + "duration_ms": 15, + "objects_per_second": 1266.6666666666667 } $ scripts/reconcile.sh -grace 0s -dry-run=false # repeat: nothing left @@ -83,41 +103,81 @@ $ scripts/reconcile.sh -grace 0s -dry-run=false # repeat: nothing left "prefix": "chats/", "grace_seconds": 0, "dry_run": false, - "scanned_objects": 15, - "scanned_rows": 15, + "scanned_objects": 18, + "scanned_rows": 18, "truncated": false, "orphans": [], "missing": [], "deleted_objects": 0, + "deleted_metadata": 0, + "retained_missing": 0, "delete_failures": 0, - "duration_ms": 7 + "s3_requests": 1, + "db_queries": 2, + "duration_ms": 8, + "objects_per_second": 2250 } missing drill: metadata row without object -$ scripts/s3tool.sh delete chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c -deleted chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c +$ scripts/s3tool.sh delete chats// of an existing attachment +$ scripts/s3tool.sh delete chats/bd1c9e17-0366-44b4-bd98-0a479a2a637c/1e9b9e12-eab2-4846-8952-e739a4257b37 +deleted chats/bd1c9e17-0366-44b4-bd98-0a479a2a637c/1e9b9e12-eab2-4846-8952-e739a4257b37 -$ scripts/reconcile.sh -grace 0s # reports missing, keeps metadata +$ scripts/reconcile.sh -grace 0s # missing reported, metadata kept in dry-run { "prefix": "chats/", "grace_seconds": 0, "dry_run": true, - "scanned_objects": 14, - "scanned_rows": 15, + "scanned_objects": 17, + "scanned_rows": 18, "truncated": false, "orphans": [], "missing": [ { - "id": "97121cfe-6b29-4227-867f-3188e72a725c", - "object_key": "chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c", - "chat_id": "9452049e-b86c-4d9a-8060-d98f6966b956", - "created_at": "2026-11-20T01:11:14Z" + "id": "1e9b9e12-eab2-4846-8952-e739a4257b37", + "object_key": "chats/bd1c9e17-0366-44b4-bd98-0a479a2a637c/1e9b9e12-eab2-4846-8952-e739a4257b37", + "chat_id": "bd1c9e17-0366-44b4-bd98-0a479a2a637c", + "created_at": "2026-11-20T01:40:28Z", + "referenced_by_message": false } ], "deleted_objects": 0, + "deleted_metadata": 0, + "retained_missing": 0, "delete_failures": 0, - "duration_ms": 6 + "s3_requests": 1, + "db_queries": 2, + "duration_ms": 9, + "objects_per_second": 1888.8888888888891 } metadata rows still present: 1 + +$ scripts/reconcile.sh -grace 0s -dry-run=false # safe repair: unreferenced dead row removed +{ + "prefix": "chats/", + "grace_seconds": 0, + "dry_run": false, + "scanned_objects": 17, + "scanned_rows": 18, + "truncated": false, + "orphans": [], + "missing": [ + { + "id": "1e9b9e12-eab2-4846-8952-e739a4257b37", + "object_key": "chats/bd1c9e17-0366-44b4-bd98-0a479a2a637c/1e9b9e12-eab2-4846-8952-e739a4257b37", + "chat_id": "bd1c9e17-0366-44b4-bd98-0a479a2a637c", + "created_at": "2026-11-20T01:40:28Z", + "referenced_by_message": false + } + ], + "deleted_objects": 0, + "deleted_metadata": 1, + "retained_missing": 0, + "delete_failures": 0, + "s3_requests": 1, + "db_queries": 3, + "duration_ms": 8, + "objects_per_second": 2125 +} diff --git a/evidence/s3-outage.txt b/evidence/s3-outage.txt new file mode 100644 index 0000000..7aef16e --- /dev/null +++ b/evidence/s3-outage.txt @@ -0,0 +1,17 @@ +2026-11-20T04:57:42+03:00 +S3 unavailable: readiness and client-visible behaviour + +$ GET /health/ready on a replica + HTTP/1.1 503 Service Unavailable +wget: server returned error: HTTP/1.1 503 Service Unavailable + +$ GET /health/live on a replica (process is alive) + HTTP/1.1 200 OK + Content-Type: application/json; charset=utf-8 + X-Instance-Id: a089fe9cc382 + +$ client request through the load balancer +status=503 + +$ readiness after S3 recovery +status=200 diff --git a/evidence/smoke.txt b/evidence/smoke.txt index 562cb53..01ae22b 100644 --- a/evidence/smoke.txt +++ b/evidence/smoke.txt @@ -1,4 +1,4 @@ -2026-11-20T04:20:26+03:00 +2026-11-20T04:53:35+03:00 make test python3 tests/smoke.py --base-url "http://localhost:8080" diff --git a/evidence/storage.txt b/evidence/storage.txt index 0bb5f52..e73446d 100644 --- a/evidence/storage.txt +++ b/evidence/storage.txt @@ -1,16 +1,16 @@ -2026-11-20T04:18:15+03:00 +2026-11-20T04:56:31+03:00 object bytes live in the RustFS volume, metadata in PostgreSQL id | object_key | size_bytes | sha256 --------------------------------------+---------------------------------------------------------------------------------+------------+------------------------------------------------------------------ - 3dbc5415-9791-4983-a253-bc71fdc79a24 | chats/3ec617cd-fddf-4ae1-ad9d-37a4bb319b76/3dbc5415-9791-4983-a253-bc71fdc79a24 | 126609 | 518fd54cb19f8ceaca99139652fb96ed3a5d502392450846c832924b1722d2bf + 82d5d4f3-e97a-4aea-9501-69ef2b74e138 | chats/e8618579-667a-4423-aa62-5402596e57d8/82d5d4f3-e97a-4aea-9501-69ef2b74e138 | 111817 | 99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75 (1 row) S3 volume listing for that object -total 128 --rw-r--r-- 1 rustfs rustfs 127190 Nov 20 01:17 xl.meta +total 112 +-rw-r--r-- 1 rustfs rustfs 112398 Nov 20 01:55 xl.meta attachments table columns (no binary content) diff --git a/evidence/test-go.txt b/evidence/test-go.txt index 66b924d..20ab219 100644 --- a/evidence/test-go.txt +++ b/evidence/test-go.txt @@ -1,4 +1,4 @@ -2026-11-20T04:20:28+03:00 +2026-11-20T04:53:38+03:00 make test-go ? messenger/cmd/reconcile [no test files] @@ -10,50 +10,52 @@ make test-go === RUN TestSessionExpiry --- PASS: TestSessionExpiry (3.09s) === RUN TestLogout ---- PASS: TestLogout (0.11s) +--- PASS: TestLogout (0.13s) === RUN TestWrongPassword --- PASS: TestWrongPassword (0.10s) === RUN TestRedisUnavailable -redis: 2026/11/20 04:20:32 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused -redis: 2026/11/20 04:20:32 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused -redis: 2026/11/20 04:20:33 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused -redis: 2026/11/20 04:20:33 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused -redis: 2026/11/20 04:20:34 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused -redis: 2026/11/20 04:20:34 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused ---- PASS: TestRedisUnavailable (2.83s) +redis: 2026/11/20 04:53:42 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused +redis: 2026/11/20 04:53:43 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused +redis: 2026/11/20 04:53:43 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused +redis: 2026/11/20 04:53:44 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused +redis: 2026/11/20 04:53:44 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused +redis: 2026/11/20 04:53:44 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused +--- PASS: TestRedisUnavailable (2.95s) === RUN TestDatabaseUnavailable --- PASS: TestDatabaseUnavailable (0.17s) === RUN TestInvalidJSON --- PASS: TestInvalidJSON (0.01s) === RUN TestForeignChat ---- PASS: TestForeignChat (0.16s) +--- PASS: TestForeignChat (0.18s) === RUN TestEmptyMessage -redis: 2026/11/20 04:20:35 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused +redis: 2026/11/20 04:53:45 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused --- PASS: TestEmptyMessage (0.08s) === RUN TestDuplicateUsername --- PASS: TestDuplicateUsername (0.10s) === RUN TestPaginationMultiplePages ---- PASS: TestPaginationMultiplePages (0.08s) +--- PASS: TestPaginationMultiplePages (0.09s) === RUN TestConcurrentMessages ---- PASS: TestConcurrentMessages (0.59s) +--- PASS: TestConcurrentMessages (0.57s) PASS -ok messenger/internal/handler 7.325s +ok messenger/internal/handler 7.474s === RUN TestPlanDetectsOrphansAndMissing --- PASS: TestPlanDetectsOrphansAndMissing (0.00s) === RUN TestPlanHonoursGraceAndPrefix --- PASS: TestPlanHonoursGraceAndPrefix (0.00s) === RUN TestPlanNeverDeletesReferencedObjects --- PASS: TestPlanNeverDeletesReferencedObjects (0.00s) +=== RUN TestPlanOnlyReportsRowsInsidePrefix +--- PASS: TestPlanOnlyReportsRowsInsidePrefix (0.00s) === RUN TestPlanOutputIsSorted --- PASS: TestPlanOutputIsSorted (0.00s) PASS ok messenger/internal/reconcile 0.002s === RUN TestRegisterUsernameRace ---- PASS: TestRegisterUsernameRace (0.05s) +--- PASS: TestRegisterUsernameRace (0.02s) === RUN TestCreateChatRollback --- PASS: TestCreateChatRollback (0.02s) PASS -ok messenger/internal/repository/postgres 0.071s +ok messenger/internal/repository/postgres 0.040s ? messenger/internal/repository/redis [no test files] ? messenger/internal/repository/s3 [no test files] ? messenger/internal/router [no test files] diff --git a/evidence/track2-cost.txt b/evidence/track2-cost.txt new file mode 100644 index 0000000..6745ea0 --- /dev/null +++ b/evidence/track2-cost.txt @@ -0,0 +1,41 @@ +2026-11-20T04:42:15+03:00 +traversal cost of the reconciler (synthetic objects under chats/bench/, 1 KiB each); scan uses LIST pages, repair deletes one by one + +{ + "prefix": "chats/bench", + "measurements": [ + { + "objects": 100, + "scan_ms": 9, + "scan_s3_requests": 1, + "scan_db_queries": 2, + "scan_objects_per_second": 11111.1, + "scan_ms_per_object": 0.09, + "repair_ms": 1324, + "repair_s3_requests": 101, + "repair_deleted": 100 + }, + { + "objects": 1000, + "scan_ms": 34, + "scan_s3_requests": 1, + "scan_db_queries": 2, + "scan_objects_per_second": 29411.8, + "scan_ms_per_object": 0.034, + "repair_ms": 12232, + "repair_s3_requests": 1001, + "repair_deleted": 1000 + }, + { + "objects": 5000, + "scan_ms": 166, + "scan_s3_requests": 5, + "scan_db_queries": 2, + "scan_objects_per_second": 30120.5, + "scan_ms_per_object": 0.0332, + "repair_ms": 69929, + "repair_s3_requests": 5005, + "repair_deleted": 5000 + } + ] +} diff --git a/evidence/track2-injection.txt b/evidence/track2-injection.txt new file mode 100644 index 0000000..f9dbbf5 --- /dev/null +++ b/evidence/track2-injection.txt @@ -0,0 +1,193 @@ +2026-11-20T04:41:49+03:00 +two-store consistency: injections on both sides, dry-run, safe repair, repeat, protection of fresh and used data + +injections: + S3 has an object that has no metadata row (fresh orphan) + metadata exists but the object is gone: one attachment used by a message, one unused + +$ scripts/s3tool.sh put chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000 - < scripts/fixtures/sample.jpg +put chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000 (653 bytes) +$ scripts/s3tool.sh delete chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1 # used +deleted chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1 +$ scripts/s3tool.sh delete chats/d03b2c6b-67fd-4b5d-bc17-9e926440ff8d/c49ab2de-258a-4851-8ad2-f7f585fd3488 # unused +deleted chats/d03b2c6b-67fd-4b5d-bc17-9e926440ff8d/c49ab2de-258a-4851-8ad2-f7f585fd3488 + +$ scripts/reconcile.sh -grace 1h # dry-run +{ + "prefix": "chats/", + "grace_seconds": 3600, + "dry_run": true, + "scanned_objects": 19, + "scanned_rows": 20, + "truncated": false, + "orphans": [], + "missing": [ + { + "id": "6a9f0506-2126-497a-b8b4-89ac728fe5a1", + "object_key": "chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1", + "chat_id": "47f62016-7525-4910-9823-0b21f2664b10", + "created_at": "2026-11-20T01:41:41Z", + "referenced_by_message": true + }, + { + "id": "c49ab2de-258a-4851-8ad2-f7f585fd3488", + "object_key": "chats/d03b2c6b-67fd-4b5d-bc17-9e926440ff8d/c49ab2de-258a-4851-8ad2-f7f585fd3488", + "chat_id": "d03b2c6b-67fd-4b5d-bc17-9e926440ff8d", + "created_at": "2026-11-20T01:41:41Z", + "referenced_by_message": false + } + ], + "deleted_objects": 0, + "deleted_metadata": 0, + "retained_missing": 0, + "delete_failures": 0, + "s3_requests": 1, + "db_queries": 2, + "duration_ms": 8, + "objects_per_second": 2375 +} + +$ scripts/reconcile.sh -grace 1h -dry-run=false # apply: fresh object survives, safe metadata repair +{ + "prefix": "chats/", + "grace_seconds": 3600, + "dry_run": false, + "scanned_objects": 19, + "scanned_rows": 20, + "truncated": false, + "orphans": [], + "missing": [ + { + "id": "6a9f0506-2126-497a-b8b4-89ac728fe5a1", + "object_key": "chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1", + "chat_id": "47f62016-7525-4910-9823-0b21f2664b10", + "created_at": "2026-11-20T01:41:41Z", + "referenced_by_message": true + }, + { + "id": "c49ab2de-258a-4851-8ad2-f7f585fd3488", + "object_key": "chats/d03b2c6b-67fd-4b5d-bc17-9e926440ff8d/c49ab2de-258a-4851-8ad2-f7f585fd3488", + "chat_id": "d03b2c6b-67fd-4b5d-bc17-9e926440ff8d", + "created_at": "2026-11-20T01:41:41Z", + "referenced_by_message": false + } + ], + "deleted_objects": 0, + "deleted_metadata": 1, + "retained_missing": 1, + "delete_failures": 0, + "s3_requests": 1, + "db_queries": 3, + "duration_ms": 12, + "objects_per_second": 1583.3333333333333 +} + +$ scripts/s3tool.sh list chats/aaaaaaaa-0000-4000-8000-000000000000 # fresh orphan still present +chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000 653 2026-11-20T01:41:49Z + +$ scripts/reconcile.sh -grace 0s # now eligible for deletion +{ + "prefix": "chats/", + "grace_seconds": 0, + "dry_run": true, + "scanned_objects": 19, + "scanned_rows": 19, + "truncated": false, + "orphans": [ + { + "key": "chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000", + "size_bytes": 653, + "last_modified": "2026-11-20T01:41:49Z", + "age_seconds": 7 + } + ], + "missing": [ + { + "id": "6a9f0506-2126-497a-b8b4-89ac728fe5a1", + "object_key": "chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1", + "chat_id": "47f62016-7525-4910-9823-0b21f2664b10", + "created_at": "2026-11-20T01:41:41Z", + "referenced_by_message": true + } + ], + "deleted_objects": 0, + "deleted_metadata": 0, + "retained_missing": 0, + "delete_failures": 0, + "s3_requests": 1, + "db_queries": 2, + "duration_ms": 8, + "objects_per_second": 2375 +} + +$ scripts/reconcile.sh -grace 0s -dry-run=false # apply +{ + "prefix": "chats/", + "grace_seconds": 0, + "dry_run": false, + "scanned_objects": 19, + "scanned_rows": 19, + "truncated": false, + "orphans": [ + { + "key": "chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000", + "size_bytes": 653, + "last_modified": "2026-11-20T01:41:49Z", + "age_seconds": 13 + } + ], + "missing": [ + { + "id": "6a9f0506-2126-497a-b8b4-89ac728fe5a1", + "object_key": "chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1", + "chat_id": "47f62016-7525-4910-9823-0b21f2664b10", + "created_at": "2026-11-20T01:41:41Z", + "referenced_by_message": true + } + ], + "deleted_objects": 1, + "deleted_metadata": 0, + "retained_missing": 1, + "delete_failures": 0, + "s3_requests": 2, + "db_queries": 2, + "duration_ms": 15, + "objects_per_second": 1266.6666666666667 +} + +$ scripts/reconcile.sh -grace 0s -dry-run=false # repeat: idempotent +{ + "prefix": "chats/", + "grace_seconds": 0, + "dry_run": false, + "scanned_objects": 18, + "scanned_rows": 19, + "truncated": false, + "orphans": [], + "missing": [ + { + "id": "6a9f0506-2126-497a-b8b4-89ac728fe5a1", + "object_key": "chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1", + "chat_id": "47f62016-7525-4910-9823-0b21f2664b10", + "created_at": "2026-11-20T01:41:41Z", + "referenced_by_message": true + } + ], + "deleted_objects": 0, + "deleted_metadata": 0, + "retained_missing": 1, + "delete_failures": 0, + "s3_requests": 1, + "db_queries": 2, + "duration_ms": 9, + "objects_per_second": 2000.0000000000002 +} + +used attachments remain untouched (rows and objects) + used_attachments | ready +------------------+------- + 3 | 3 +(1 row) + +retained referenced metadata row for the attachment whose object was deleted: 1 +object of that referenced attachment is still absent: 0 diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json index 620ad3d..36f1ada 100644 --- a/evidence/upload-memory.json +++ b/evidence/upload-memory.json @@ -3,30 +3,30 @@ "uploads": 10, "fixture": "large.png", "fixture_bytes": 126609, - "elapsed_seconds": 1.235, + "elapsed_seconds": 1.009, "successful": 10, "failed": 0, "failures": [], "latency_ms": { - "p50": 663.7, - "p95": 1224.7, - "p99": 1224.7 + "p50": 660.9, + "p95": 999.3, + "p99": 999.3 }, "api_containers": [ "messenger-lab4_api_1", "messenger-lab4_api_2" ], "peak_bytes_before": { - "messenger-lab4_api_1": 5050368, - "messenger-lab4_api_2": 4820992 + "messenger-lab4_api_1": 5951488, + "messenger-lab4_api_2": 4993024 }, "peak_bytes_after": { - "messenger-lab4_api_1": 368119808, - "messenger-lab4_api_2": 368185344 + "messenger-lab4_api_1": 348913664, + "messenger-lab4_api_2": 348721152 }, "peak_delta_bytes": { - "messenger-lab4_api_1": 363069440, - "messenger-lab4_api_2": 363364352 + "messenger-lab4_api_1": 342962176, + "messenger-lab4_api_2": 343728128 }, - "peak_max_mb": 351.13 + "peak_max_mb": 332.75 } diff --git a/internal/handler/attachments.go b/internal/handler/attachments.go index cc263d3..3ab2b07 100644 --- a/internal/handler/attachments.go +++ b/internal/handler/attachments.go @@ -53,7 +53,13 @@ func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) { h.fail(w, r, err) return } - attachment, content, err := h.attachments.Open(r.Context(), actor(r), id, r.URL.Query().Get("variant")) + query := r.URL.Query() + variant := query.Get("variant") + if query.Has("variant") && variant == "" { + h.fail(w, r, badRequest("variant must be original")) + return + } + attachment, content, err := h.attachments.Open(r.Context(), actor(r), id, variant) if err != nil { h.fail(w, r, err) return diff --git a/internal/handler/dto.go b/internal/handler/dto.go index f882af8..ad3773c 100644 --- a/internal/handler/dto.go +++ b/internal/handler/dto.go @@ -1,6 +1,7 @@ package handler import ( + "encoding/json" "time" "uuid" @@ -19,8 +20,8 @@ type createChatRequest struct { } type createMessageRequest struct { - Text *string `json:"text"` - AttachmentIDs *[]string `json:"attachment_ids"` + Text *string `json:"text"` + AttachmentIDs json.RawMessage `json:"attachment_ids"` } type userResponse struct { diff --git a/internal/handler/messages.go b/internal/handler/messages.go index a4fe5f5..0cc4039 100644 --- a/internal/handler/messages.go +++ b/internal/handler/messages.go @@ -1,6 +1,8 @@ package handler import ( + "bytes" + "encoding/json" "net/http" "regexp" "uuid" @@ -31,10 +33,19 @@ func (h *Handler) SendMessage(w http.ResponseWriter, r *http.Request) { return } var attachmentIDs []uuid.UUID - if req.AttachmentIDs != nil { - attachmentIDs = make([]uuid.UUID, len(*req.AttachmentIDs)) - for i, raw := range *req.AttachmentIDs { - id, ok := parseUUID(raw) + if len(req.AttachmentIDs) > 0 { + if bytes.Equal(bytes.TrimSpace(req.AttachmentIDs), []byte("null")) { + h.fail(w, r, badRequest("attachment_ids must be an array of UUIDs")) + return + } + var raw []string + if err := json.Unmarshal(req.AttachmentIDs, &raw); err != nil { + h.fail(w, r, badRequest("attachment_ids must be an array of UUIDs")) + return + } + attachmentIDs = make([]uuid.UUID, len(raw)) + for i, value := range raw { + id, ok := parseUUID(value) if !ok { h.fail(w, r, badRequest("attachment_ids must contain UUIDs")) return diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go index 0566230..847f095 100644 --- a/internal/reconcile/reconcile.go +++ b/internal/reconcile/reconcile.go @@ -28,31 +28,37 @@ type Orphan struct { } type Missing struct { - ID string `json:"id"` - ObjectKey string `json:"object_key"` - ChatID string `json:"chat_id"` - CreatedAt string `json:"created_at"` + ID string `json:"id"` + ObjectKey string `json:"object_key"` + ChatID string `json:"chat_id"` + CreatedAt string `json:"created_at"` + Referenced bool `json:"referenced_by_message"` } type Report struct { - Prefix string `json:"prefix"` - GraceSeconds int64 `json:"grace_seconds"` - DryRun bool `json:"dry_run"` - ScannedObjects int `json:"scanned_objects"` - ScannedRows int `json:"scanned_rows"` - Truncated bool `json:"truncated"` - Orphans []Orphan `json:"orphans"` - Missing []Missing `json:"missing"` - Deleted int `json:"deleted_objects"` - DeleteFailures int `json:"delete_failures"` - DurationMS int64 `json:"duration_ms"` + Prefix string `json:"prefix"` + GraceSeconds int64 `json:"grace_seconds"` + DryRun bool `json:"dry_run"` + ScannedObjects int `json:"scanned_objects"` + ScannedRows int `json:"scanned_rows"` + Truncated bool `json:"truncated"` + Orphans []Orphan `json:"orphans"` + Missing []Missing `json:"missing"` + DeletedObjects int `json:"deleted_objects"` + DeletedMetadata int `json:"deleted_metadata"` + RetainedMissing int `json:"retained_missing"` + DeleteFailures int `json:"delete_failures"` + S3Requests int64 `json:"s3_requests"` + DBQueries int `json:"db_queries"` + DurationMS int64 `json:"duration_ms"` + ObjectsPerSecond float64 `json:"objects_per_second"` } -func Plan(objects []Object, rows []Metadata, prefix string, now time.Time, grace time.Duration) ([]Orphan, []Missing) { - referenced := make(map[string]struct{}, len(rows)) +func Plan(objects []Object, rows []Metadata, referenced map[uuid.UUID]bool, prefix string, now time.Time, grace time.Duration) ([]Orphan, []Missing) { + known := make(map[string]struct{}, len(rows)) existing := make(map[string]struct{}, len(objects)) for _, row := range rows { - referenced[row.ObjectKey] = struct{}{} + known[row.ObjectKey] = struct{}{} } for _, object := range objects { existing[object.Key] = struct{}{} @@ -63,7 +69,7 @@ func Plan(objects []Object, rows []Metadata, prefix string, now time.Time, grace if !strings.HasPrefix(object.Key, prefix) { continue } - if _, ok := referenced[object.Key]; ok { + if _, ok := known[object.Key]; ok { continue } age := now.Sub(object.LastModified) @@ -80,14 +86,18 @@ func Plan(objects []Object, rows []Metadata, prefix string, now time.Time, grace missing := make([]Missing, 0) for _, row := range rows { + if !strings.HasPrefix(row.ObjectKey, prefix) { + continue + } if _, ok := existing[row.ObjectKey]; ok { continue } missing = append(missing, Missing{ - ID: row.ID.String(), - ObjectKey: row.ObjectKey, - ChatID: row.ChatID.String(), - CreatedAt: row.CreatedAt.UTC().Format(time.RFC3339), + ID: row.ID.String(), + ObjectKey: row.ObjectKey, + ChatID: row.ChatID.String(), + CreatedAt: row.CreatedAt.UTC().Format(time.RFC3339), + Referenced: referenced[row.ID], }) } diff --git a/internal/reconcile/reconcile_test.go b/internal/reconcile/reconcile_test.go index 4ea3a9a..bbd101b 100644 --- a/internal/reconcile/reconcile_test.go +++ b/internal/reconcile/reconcile_test.go @@ -8,7 +8,7 @@ import ( func TestPlanDetectsOrphansAndMissing(t *testing.T) { now := time.Date(2026, 11, 20, 12, 0, 0, 0, time.UTC) - usedID, missingID := uuid.New(), uuid.New() + usedID, missingID, unreferencedID := uuid.New(), uuid.New(), uuid.New() objects := []Object{ {Key: "chats/a/used", Size: 10, LastModified: now.Add(-2 * time.Hour)}, {Key: "chats/a/orphan-old", Size: 20, LastModified: now.Add(-2 * time.Hour)}, @@ -17,16 +17,28 @@ func TestPlanDetectsOrphansAndMissing(t *testing.T) { } rows := []Metadata{ {ID: usedID, ObjectKey: "chats/a/used", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)}, - {ID: missingID, ObjectKey: "chats/a/missing", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)}, + {ID: missingID, ObjectKey: "chats/a/missing-referenced", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)}, + {ID: unreferencedID, ObjectKey: "chats/a/missing-orphan-row", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)}, } + referenced := map[uuid.UUID]bool{missingID: true} - orphans, missing := Plan(objects, rows, "chats/", now, time.Hour) + orphans, missing := Plan(objects, rows, referenced, "chats/", now, time.Hour) if len(orphans) != 1 || orphans[0].Key != "chats/a/orphan-old" { t.Fatalf("orphans = %+v, want only chats/a/orphan-old", orphans) } - if len(missing) != 1 || missing[0].ObjectKey != "chats/a/missing" { - t.Fatalf("missing = %+v, want only chats/a/missing", missing) + if len(missing) != 2 { + t.Fatalf("missing = %+v, want two rows", missing) + } + byKey := map[string]Missing{} + for _, item := range missing { + byKey[item.ObjectKey] = item + } + if !byKey["chats/a/missing-referenced"].Referenced { + t.Error("referenced missing row must be flagged") + } + if byKey["chats/a/missing-orphan-row"].Referenced { + t.Error("unreferenced missing row must not be flagged") } } @@ -36,12 +48,12 @@ func TestPlanHonoursGraceAndPrefix(t *testing.T) { {Key: "chats/a/fresh", Size: 1, LastModified: now.Add(-30 * time.Second)}, {Key: "outside/key", Size: 1, LastModified: now.Add(-48 * time.Hour)}, } - orphans, _ := Plan(objects, nil, "chats/", now, time.Hour) + orphans, _ := Plan(objects, nil, nil, "chats/", now, time.Hour) if len(orphans) != 0 { t.Fatalf("orphans = %+v, want none", orphans) } - orphans, _ = Plan(objects, nil, "chats/", now, 0) + orphans, _ = Plan(objects, nil, nil, "chats/", now, 0) if len(orphans) != 1 || orphans[0].Key != "chats/a/fresh" { t.Fatalf("orphans with zero grace = %+v, want chats/a/fresh", orphans) } @@ -52,7 +64,7 @@ func TestPlanNeverDeletesReferencedObjects(t *testing.T) { objects := []Object{{Key: "chats/a/used", Size: 1, LastModified: now.Add(-100 * time.Hour)}} rows := []Metadata{{ID: uuid.New(), ObjectKey: "chats/a/used", ChatID: uuid.New(), CreatedAt: now.Add(-100 * time.Hour)}} - orphans, missing := Plan(objects, rows, "chats/", now, 0) + orphans, missing := Plan(objects, rows, nil, "chats/", now, 0) if len(orphans) != 0 { t.Fatalf("orphans = %+v, want none for referenced object", orphans) } @@ -61,6 +73,18 @@ func TestPlanNeverDeletesReferencedObjects(t *testing.T) { } } +func TestPlanOnlyReportsRowsInsidePrefix(t *testing.T) { + now := time.Date(2026, 11, 20, 12, 0, 0, 0, time.UTC) + rows := []Metadata{ + {ID: uuid.New(), ObjectKey: "chats/a/inside", ChatID: uuid.New(), CreatedAt: now}, + {ID: uuid.New(), ObjectKey: "bench/outside", ChatID: uuid.New(), CreatedAt: now}, + } + _, missing := Plan(nil, rows, nil, "chats/", now, 0) + if len(missing) != 1 || missing[0].ObjectKey != "chats/a/inside" { + t.Fatalf("missing = %+v, want only rows under the scanned prefix", missing) + } +} + func TestPlanOutputIsSorted(t *testing.T) { now := time.Date(2026, 11, 20, 12, 0, 0, 0, time.UTC) objects := []Object{ @@ -68,7 +92,7 @@ func TestPlanOutputIsSorted(t *testing.T) { {Key: "chats/a/b", LastModified: now.Add(-time.Hour)}, {Key: "chats/a/m", LastModified: now.Add(-time.Hour)}, } - orphans, _ := Plan(objects, nil, "chats/", now, 0) + orphans, _ := Plan(objects, nil, nil, "chats/", now, 0) if len(orphans) != 3 || orphans[0].Key != "chats/a/b" || orphans[2].Key != "chats/a/z" { t.Fatalf("orphans are not sorted: %+v", orphans) } diff --git a/internal/repository/s3/store.go b/internal/repository/s3/store.go index 455a520..08f1bc1 100644 --- a/internal/repository/s3/store.go +++ b/internal/repository/s3/store.go @@ -37,11 +37,16 @@ func EndpointHost(endpoint string) (string, bool) { } func New(endpoint, region, accessKeyID, secretAccessKey, bucket string) (*Store, error) { + return NewWithTransport(endpoint, region, accessKeyID, secretAccessKey, bucket, nil) +} + +func NewWithTransport(endpoint, region, accessKeyID, secretAccessKey, bucket string, transport http.RoundTripper) (*Store, error) { host, secure := EndpointHost(endpoint) client, err := minio.New(host, &minio.Options{ - Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), - Region: region, - Secure: secure, + Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""), + Region: region, + Secure: secure, + Transport: transport, }) if err != nil { return nil, err diff --git a/internal/service/attachments.go b/internal/service/attachments.go index 5fb0eb0..db694cb 100644 --- a/internal/service/attachments.go +++ b/internal/service/attachments.go @@ -11,6 +11,7 @@ import ( _ "image/png" "io" "net/http" + "strings" "time" "unicode" "unicode/utf8" @@ -159,7 +160,10 @@ func acquire(ctx context.Context, sem chan struct{}) error { func release(sem chan struct{}) { <-sem } func checkFilename(filename string) error { - if n := utf8.RuneCountInString(filename); n < 1 || n > maxFilenameRunes { + if strings.TrimSpace(filename) == "" { + return fmt.Errorf("%w: filename must not be blank", domain.ErrInvalidInput) + } + if n := utf8.RuneCountInString(filename); n > maxFilenameRunes { return fmt.Errorf("%w: filename must be 1 to 255 characters", domain.ErrInvalidInput) } for _, r := range filename { diff --git a/scripts/track2-cost.py b/scripts/track2-cost.py new file mode 100755 index 0000000..1e79a10 --- /dev/null +++ b/scripts/track2-cost.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +"""Reconciler traversal cost for several object counts; creates synthetic objects and removes them.""" +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +S3TOOL = ROOT / 'scripts/s3tool.sh' +RECONCILE = ROOT / 'scripts/reconcile.sh' +PREFIX = 'chats/bench' +SIZES = (100, 1000, 5000) + + +def call(arguments): + return subprocess.run(arguments, capture_output=True, text=True, check=False) + + +def reconcile(*arguments): + result = call([str(RECONCILE), *arguments]) + try: + return json.loads(result.stdout) + except json.JSONDecodeError: + print(result.stdout[-500:], file=sys.stderr) + print(result.stderr[-500:], file=sys.stderr) + raise SystemExit('reconcile did not return JSON') + + +def main(): + measurements = [] + for count in SIZES: + seeded = call([str(S3TOOL), 'seed', PREFIX, str(count), '1024']) + if seeded.returncode != 0: + raise SystemExit('seed failed: ' + seeded.stderr[-400:]) + + scan = reconcile('-prefix', PREFIX + '/', '-grace', '0s') + if scan['scanned_objects'] != count: + raise SystemExit(f'expected {count} objects, scanned {scan["scanned_objects"]}') + if len(scan['orphans']) != count: + raise SystemExit(f'expected {count} orphans, got {len(scan["orphans"])}') + + repair = reconcile('-prefix', PREFIX + '/', '-grace', '0s', '-dry-run=false') + if repair['deleted_objects'] != count: + raise SystemExit(f'expected {count} deletions, got {repair["deleted_objects"]}') + + measurements.append({ + 'objects': count, + 'scan_ms': scan['duration_ms'], + 'scan_s3_requests': scan['s3_requests'], + 'scan_db_queries': scan['db_queries'], + 'scan_objects_per_second': round(scan['objects_per_second'], 1), + 'scan_ms_per_object': round(scan['duration_ms'] / count, 4), + 'repair_ms': repair['duration_ms'], + 'repair_s3_requests': repair['s3_requests'], + 'repair_deleted': repair['deleted_objects'], + }) + + print(json.dumps({'prefix': PREFIX, 'measurements': measurements}, indent=2)) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/scripts/upload-once.py b/scripts/upload-once.py index 15a66e4..74be48d 100755 --- a/scripts/upload-once.py +++ b/scripts/upload-once.py @@ -14,6 +14,7 @@ def main(): p.add_argument('--expect', type=int, default=201) p.add_argument('--file', default=str(ROOT / 'scripts/fixtures/sample.jpg')) p.add_argument('--content-type', default='image/jpeg') + p.add_argument('--attach', action='store_true', help='also attach the upload to a new message') args = arguments('Single upload used by compensation drills; creates synthetic data', configure) source = Path(args.file) @@ -29,6 +30,9 @@ def main(): if response['status'] == 201: attachment = response['json'] print(f"attachment={attachment['id']} size={attachment['size_bytes']}") + if args.attach: + message = c.message(alice, chat, '', attachments=[attachment['id']])['json'] + print(f"message={message['id']} attachment_ids={message['attachment_ids']}") if __name__ == '__main__':