feat: two-store reconciler with safe repair and cost measurement (grade 5, track 2)

This commit is contained in:
2026-11-20 05:01:20 +03:00
parent 1c3011bf28
commit 60e8f4474b
36 changed files with 807 additions and 212 deletions
+11 -2
View File
@@ -8,11 +8,11 @@ TEST_DATABASE_URL ?= $(shell sed 's/@postgres:5432/@127.0.0.1:55432/' secrets/da
TEST_REDIS_URL ?= redis://127.0.0.1:56379/0
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help up down check validate test load test-persistence test-failover test-idempotency test-attachments test-memory test-go
.PHONY: help up down check validate test load test-persistence test-failover test-idempotency test-attachments test-contract test-memory test-cost test-reconcile test-go
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "up/down: start/stop the stand, volumes kept; test-idempotency: one key in parallel via LB; test-attachments: attachment contract checks; test-go: Go tests against the running stand"
@echo "up/down: start/stop the stand, volumes kept; test-idempotency: one key in parallel via LB; test-attachments: attachment contract checks; test-memory: concurrent upload memory; test-cost: reconciler traversal cost; test-reconcile: reconciler dry-run; test-go: Go tests against the running stand"
up:
$(COMPOSE) up -d --build
@@ -46,9 +46,18 @@ test-idempotency:
test-attachments:
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/attachments-check.py
test-contract:
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/contract-audit.py
test-memory:
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/upload-load.py --concurrency 10 --output evidence/upload-memory.json
test-cost:
$(PYTHON) scripts/track2-cost.py
test-reconcile:
scripts/reconcile.sh -grace 0s
test-go:
@TEST_DATABASE_URL="$(TEST_DATABASE_URL)" TEST_REDIS_URL="$(TEST_REDIS_URL)" go test -count=1 -v ./...
+81 -8
View File
@@ -5,8 +5,11 @@ import (
"encoding/json"
"flag"
"fmt"
"net/http"
"os"
"sync/atomic"
"time"
"uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
@@ -17,20 +20,31 @@ import (
"messenger/internal/repository/s3"
)
type countingTransport struct {
base http.RoundTripper
count atomic.Int64
}
func (t *countingTransport) RoundTrip(request *http.Request) (*http.Response, error) {
t.count.Add(1)
return t.base.RoundTrip(request)
}
func main() {
dryRun := flag.Bool("dry-run", true, "report only, do not delete orphan objects")
dryRun := flag.Bool("dry-run", true, "report only, do not delete or repair anything")
grace := flag.Duration("grace", time.Hour, "minimum object age before it can be deleted")
prefix := flag.String("prefix", "chats/", "object key prefix to scan")
limit := flag.Int("limit", 10000, "maximum number of objects to scan")
repairMissing := flag.Bool("repair-missing", true, "delete metadata rows whose object is gone and that no message references")
flag.Parse()
if err := run(*dryRun, *grace, *prefix, *limit); err != nil {
if err := run(*dryRun, *grace, *prefix, *limit, *repairMissing); err != nil {
fmt.Fprintln(os.Stderr, "reconcile failed:", err)
os.Exit(1)
}
}
func run(dryRun bool, grace time.Duration, prefix string, limit int) error {
func run(dryRun bool, grace time.Duration, prefix string, limit int, repairMissing bool) error {
if grace < 0 {
return fmt.Errorf("grace must not be negative")
}
@@ -55,10 +69,11 @@ func run(dryRun bool, grace time.Duration, prefix string, limit int) error {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
defer cancel()
store, err := s3.New(endpoint, region, accessKeyID, secretAccessKey, bucket)
transport := &countingTransport{base: http.DefaultTransport}
store, err := s3.NewWithTransport(endpoint, region, accessKeyID, secretAccessKey, bucket, transport)
if err != nil {
return err
}
@@ -77,8 +92,12 @@ func run(dryRun bool, grace time.Duration, prefix string, limit int) error {
if err != nil {
return err
}
referenced, err := loadReferences(ctx, pool)
if err != nil {
return err
}
orphans, missing := reconcile.Plan(toObjects(scanned), rows, prefix, time.Now(), grace)
orphans, missing := reconcile.Plan(toObjects(scanned), rows, referenced, prefix, time.Now(), grace)
if truncated {
missing = nil
}
@@ -93,16 +112,50 @@ func run(dryRun bool, grace time.Duration, prefix string, limit int) error {
Orphans: orphans,
Missing: missing,
}
dbQueries := 2
if !dryRun {
for _, orphan := range orphans {
if err := store.Delete(ctx, orphan.Key); err != nil {
report.DeleteFailures++
continue
}
report.Deleted++
report.DeletedObjects++
}
if repairMissing {
for _, item := range missing {
if item.Referenced {
report.RetainedMissing++
continue
}
id, err := uuid.Parse(item.ID)
if err != nil {
report.DeleteFailures++
continue
}
tag, err := pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1`, id)
dbQueries++
if err != nil {
report.DeleteFailures++
continue
}
report.DeletedMetadata += int(tag.RowsAffected())
}
} else {
for _, item := range missing {
if item.Referenced {
report.RetainedMissing++
}
}
}
}
report.S3Requests = transport.count.Load()
report.DBQueries = dbQueries
report.DurationMS = time.Since(started).Milliseconds()
if report.DurationMS > 0 {
report.ObjectsPerSecond = float64(report.ScannedObjects) / (float64(report.DurationMS) / 1000)
}
encoder := json.NewEncoder(os.Stdout)
encoder.SetIndent("", " ")
@@ -110,7 +163,7 @@ func run(dryRun bool, grace time.Duration, prefix string, limit int) error {
return err
}
if report.DeleteFailures > 0 {
return fmt.Errorf("%d orphan deletions failed", report.DeleteFailures)
return fmt.Errorf("%d repair operations failed", report.DeleteFailures)
}
return nil
}
@@ -127,6 +180,26 @@ func loadMetadata(ctx context.Context, pool *pgxpool.Pool) ([]reconcile.Metadata
})
}
func loadReferences(ctx context.Context, pool *pgxpool.Pool) (map[uuid.UUID]bool, error) {
rows, err := pool.Query(ctx, `SELECT DISTINCT attachment_id FROM message_attachments`)
if err != nil {
return nil, err
}
ids, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (uuid.UUID, error) {
var id uuid.UUID
err := row.Scan(&id)
return id, err
})
if err != nil {
return nil, err
}
referenced := make(map[uuid.UUID]bool, len(ids))
for _, id := range ids {
referenced[id] = true
}
return referenced, nil
}
func toObjects(scanned []s3.ObjectInfo) []reconcile.Object {
objects := make([]reconcile.Object, len(scanned))
for i, info := range scanned {
+47 -1
View File
@@ -6,6 +6,8 @@ import (
"fmt"
"io"
"os"
"strconv"
"sync"
"time"
"messenger/internal/config"
@@ -79,6 +81,22 @@ func run(command string, args []string) error {
return err
}
fmt.Printf("removed %s\n", args[0])
case "seed":
if len(args) != 3 {
usage()
}
count, err := strconv.Atoi(args[1])
if err != nil || count <= 0 {
return fmt.Errorf("count must be a positive integer")
}
size, err := strconv.Atoi(args[2])
if err != nil || size <= 0 {
return fmt.Errorf("size must be a positive integer")
}
if err := seed(ctx, store, args[0], count, size); err != nil {
return err
}
fmt.Printf("seeded %d objects under %s\n", count, args[0])
default:
usage()
}
@@ -105,8 +123,36 @@ func newStore() (*s3.Store, error) {
accessKeyID, secretAccessKey, getenv("S3_BUCKET", "messenger-attachments"))
}
func seed(ctx context.Context, store *s3.Store, prefix string, count, size int) error {
content := make([]byte, size)
workers := 8
work := make(chan int)
errors := make(chan error, workers)
var group sync.WaitGroup
for i := 0; i < workers; i++ {
group.Add(1)
go func() {
defer group.Done()
for index := range work {
key := fmt.Sprintf("%s/%06d", prefix, index)
if err := store.Put(ctx, key, bytes.NewReader(content), int64(size), "application/octet-stream"); err != nil {
errors <- err
return
}
}
}()
}
for index := 0; index < count; index++ {
work <- index
}
close(work)
group.Wait()
close(errors)
return <-errors
}
func usage() {
fmt.Fprintln(os.Stderr, "usage: s3tool put <key> <file> | delete <key> | list [prefix] | buckets | remove-bucket <name>")
fmt.Fprintln(os.Stderr, "usage: s3tool put <key> <file> | delete <key> | list [prefix] | buckets | remove-bucket <name> | seed <prefix> <count> <bytes>")
os.Exit(2)
}
+1 -1
View File
@@ -1,4 +1,4 @@
2026-11-20T04:20:26+03:00
2026-11-20T04:38:30+03:00
make check
python3 scripts/check_contract.py
+29 -35
View File
@@ -1,4 +1,4 @@
2026-11-20T04:14:17+03:00
2026-11-20T04:41:27+03:00
failure between S3 write and database commit: an object must not stay unaccounted
$ scripts/reconcile.sh # baseline
@@ -6,21 +6,19 @@ $ scripts/reconcile.sh # baseline
"prefix": "chats/",
"grace_seconds": 3600,
"dry_run": true,
"scanned_objects": 15,
"scanned_rows": 16,
"scanned_objects": 17,
"scanned_rows": 17,
"truncated": false,
"orphans": [],
"missing": [
{
"id": "97121cfe-6b29-4227-867f-3188e72a725c",
"object_key": "chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c",
"chat_id": "9452049e-b86c-4d9a-8060-d98f6966b956",
"created_at": "2026-11-20T01:11:14Z"
}
],
"missing": [],
"deleted_objects": 0,
"deleted_metadata": 0,
"retained_missing": 0,
"delete_failures": 0,
"duration_ms": 8
"s3_requests": 1,
"db_queries": 2,
"duration_ms": 7,
"objects_per_second": 2428.5714285714284
}
$ podman exec -i postgres psql < scripts/sql/inject-db-failure.sql
@@ -37,21 +35,19 @@ $ scripts/reconcile.sh # same object count: compensating DELETE removed the ob
"prefix": "chats/",
"grace_seconds": 3600,
"dry_run": true,
"scanned_objects": 15,
"scanned_rows": 16,
"scanned_objects": 17,
"scanned_rows": 17,
"truncated": false,
"orphans": [],
"missing": [
{
"id": "97121cfe-6b29-4227-867f-3188e72a725c",
"object_key": "chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c",
"chat_id": "9452049e-b86c-4d9a-8060-d98f6966b956",
"created_at": "2026-11-20T01:11:14Z"
}
],
"missing": [],
"deleted_objects": 0,
"deleted_metadata": 0,
"retained_missing": 0,
"delete_failures": 0,
"duration_ms": 7
"s3_requests": 1,
"db_queries": 2,
"duration_ms": 7,
"objects_per_second": 2428.5714285714284
}
$ podman exec -i postgres psql < scripts/sql/reset-db-failure.sql
@@ -60,26 +56,24 @@ DROP FUNCTION
$ scripts/upload-once.py --expect 201 # service healthy again
status=201
attachment=a6844b91-5b7b-4694-93fc-87d808b687fb size=653
attachment=7dd89013-32f1-4732-a2fd-529fe2081bb8 size=653
$ scripts/reconcile.sh # one more accounted object
{
"prefix": "chats/",
"grace_seconds": 3600,
"dry_run": true,
"scanned_objects": 16,
"scanned_rows": 17,
"scanned_objects": 18,
"scanned_rows": 18,
"truncated": false,
"orphans": [],
"missing": [
{
"id": "97121cfe-6b29-4227-867f-3188e72a725c",
"object_key": "chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c",
"chat_id": "9452049e-b86c-4d9a-8060-d98f6966b956",
"created_at": "2026-11-20T01:11:14Z"
}
],
"missing": [],
"deleted_objects": 0,
"deleted_metadata": 0,
"retained_missing": 0,
"delete_failures": 0,
"duration_ms": 7
"s3_requests": 1,
"db_queries": 2,
"duration_ms": 7,
"objects_per_second": 2571.4285714285716
}
+35
View File
@@ -0,0 +1,35 @@
2026-11-20T04:53:36+03:00
make test-contract
BASE_URL="http://localhost:8080" python3 scripts/contract-audit.py
ok: explicit attachment_ids=null is 400
ok: empty variant is 400
ok: whitespace-only filename is 400
ok: Content-Type parameters are accepted
ok: 41,000,000 pixels is 413
ok: text length bounds
ok: attachment count bounds
ok: duplicate attachment_ids is 400
ok: idempotent replay with the same set
ok: absent and empty attachment_ids agree
ok: attachment from another chat is 404
ok: unknown attachment id is 404
ok: outsider access is 404
ok: upload to unknown chat is 404
ok: media errors are 415
ok: oversize is 413
ok: filename errors are 400
ok: chat validation errors are 400
ok: pagination errors are 400
ok: idempotency errors
ok: null request fields are 400
ok: more than 100 members is 400
ok: user field validation
ok: missing Content-Type is 415
ok: variant is case sensitive
ok: filename control characters are 400
ok: chat title bounds
ok: cursor is bound to the endpoint
ok: 204 responses carry headers and no body
ok: download Content-Length matches the bytes
PASS: 30 checks
+1 -1
View File
@@ -1,4 +1,4 @@
2026-11-20T04:17:09+03:00
2026-11-20T04:43:58+03:00
separate restricted S3 credentials for the API and idempotent bucket bootstrap
$ podman-compose run --rm s3-init (first run)
+2 -2
View File
@@ -1,7 +1,7 @@
2026-11-20T04:18:05+03:00
2026-11-20T04:56:32+03:00
users | chats | messages | attachments | links | db_size
-------+-------+----------+-------------+-------+---------
58 | 17 | 303 | 30 | 2 | 8534 kB
125 | 43 | 634 | 101 | 56 | 8958 kB
(1 row)
+4 -4
View File
@@ -1,10 +1,10 @@
2026-11-20T04:18:05+03:00
2026-11-20T04:56:17+03:00
Linux archlinux 7.3.5-arch1-1 #1 SMP PREEMPT_DYNAMIC Tue, 17 Nov 2026 22:28:33 +0000 x86_64 GNU/Linux
12
total used free shared buff/cache available
Mem: 15Gi 6.2Gi 2.7Gi 476Mi 7.1Gi 9.2Gi
Swap: 8.0Gi 57Mi 7.9Gi
Mem: 15Gi 6.5Gi 3.6Gi 440Mi 5.8Gi 8.8Gi
Swap: 8.0Gi 187Mi 7.8Gi
podman version 6.2.0
podman version 6.2.0
podman-compose version 1.6.0
@@ -16,4 +16,4 @@ docker.io/library/postgres:18.6-alpine3.24 57b23a0326ee
docker.io/library/redis:8.10.3-alpine3.23 9722accf3e8e
docker.io/migrate/migrate:v4.20.1 719a6f5d0ade
docker.io/rustfs/rustfs:1.0.6 7cd1c573d76d
localhost/messenger-api:lab4 e2582dea4a72
localhost/messenger-api:lab4 4c57d3f41dfb
+2 -2
View File
@@ -1,7 +1,7 @@
2026-11-20T04:14:49+03:00
2026-11-20T04:55:04+03:00
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 101, "failed_attempts": 0, "first_recovery_seconds": 3.304}
{"observed_instances_before": 2, "successful_writes": 88, "failed_attempts": 0, "first_recovery_seconds": 3.256}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+2 -2
View File
@@ -1,7 +1,7 @@
2026-11-20T04:15:25+03:00
2026-11-20T04:55:38+03:00
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 86, "failed_attempts": 0, "first_recovery_seconds": 3.228}
{"observed_instances_before": 2, "successful_writes": 99, "failed_attempts": 0, "first_recovery_seconds": 3.236}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+2 -2
View File
@@ -1,4 +1,4 @@
2026-11-20T04:20:36+03:00
2026-11-20T04:53:46+03:00
make test-idempotency
BASE_URL="http://localhost:8080" python3 scripts/idempotency-race.py
@@ -7,5 +7,5 @@ successful responses (201): 20
failed requests: 0 []
distinct message ids in responses: 1
messages stored in history: 1
instances that served requests: ['8535d7918d12', 'ad1700491457']
instances that served requests: ['72c9d7b6cc57', 'e6d2e0d15c36']
OK: exactly one message created
+5 -5
View File
@@ -1,8 +1,8 @@
2026-11-20T04:21:39+03:00
2026-11-20T04:56:16+03:00
API replicas and single entrypoint
messenger-lab4_api_1 Up 26 seconds (healthy)
messenger-lab4_api_2 Up 26 seconds (healthy)
messenger-lab4_api_1 Up 53 seconds (healthy)
messenger-lab4_api_2 Up 21 seconds (healthy)
messenger-lab4_lb_1 Up About a minute
messenger-lab4_postgres_1 Up About a minute (healthy)
messenger-lab4_redis_1 Up About a minute (healthy)
@@ -19,5 +19,5 @@ messenger-lab4_s3_1 9000-9001/tcp
X-Instance-Id distribution over 20 requests through LB
10 x-instance-id: 33839df0eea0
10 x-instance-id: 4e3d4c4f1194
10 x-instance-id: a089fe9cc382
10 x-instance-id: f309c2ef4e46
+1 -1
View File
@@ -1,4 +1,4 @@
2026-11-20T04:17:58+03:00
2026-11-20T04:56:16+03:00
per-container limits, networks, secrets
messenger-lab4_api_1 nano_cpus=1000000000 memory=536870912 networks=messenger-lab4_backend messenger-lab4_edge
+13 -13
View File
@@ -1,5 +1,5 @@
2026-11-20T04:21:28+03:00
10 concurrent uploads of a 8000x5000 PNG (40,000,000 pixels) per replica; API memory limit 512 MiB; MAX_CONCURRENT_UPLOADS=8, MAX_CONCURRENT_DECODES=1
2026-11-20T04:54:15+03:00
10 concurrent uploads of a 8000x5000 PNG (40,000,000 pixels) per replica; API limit 512 MiB; MAX_CONCURRENT_UPLOADS=8, MAX_CONCURRENT_DECODES=1
BASE_URL="http://localhost:8080" python3 scripts/upload-load.py --concurrency 10 --output evidence/upload-memory.json
{
@@ -7,30 +7,30 @@ BASE_URL="http://localhost:8080" python3 scripts/upload-load.py --concurrency 10
"uploads": 10,
"fixture": "large.png",
"fixture_bytes": 126609,
"elapsed_seconds": 1.235,
"elapsed_seconds": 1.009,
"successful": 10,
"failed": 0,
"failures": [],
"latency_ms": {
"p50": 663.7,
"p95": 1224.7,
"p99": 1224.7
"p50": 660.9,
"p95": 999.3,
"p99": 999.3
},
"api_containers": [
"messenger-lab4_api_1",
"messenger-lab4_api_2"
],
"peak_bytes_before": {
"messenger-lab4_api_1": 5050368,
"messenger-lab4_api_2": 4820992
"messenger-lab4_api_1": 5951488,
"messenger-lab4_api_2": 4993024
},
"peak_bytes_after": {
"messenger-lab4_api_1": 368119808,
"messenger-lab4_api_2": 368185344
"messenger-lab4_api_1": 348913664,
"messenger-lab4_api_2": 348721152
},
"peak_delta_bytes": {
"messenger-lab4_api_1": 363069440,
"messenger-lab4_api_2": 363364352
"messenger-lab4_api_1": 342962176,
"messenger-lab4_api_2": 343728128
},
"peak_max_mb": 351.13
"peak_max_mb": 332.75
}
+4 -8
View File
@@ -1,13 +1,9 @@
2026-11-20T04:18:05+03:00
2026-11-20T04:56:17+03:00
migrate container logs and schema version
1/u init (17.596326ms)
2/u message_idempotency (23.558514ms)
3/u attachments (35.669736ms)
no change
no change
no change
no change
1/u init (18.672238ms)
2/u message_idempotency (27.931065ms)
3/u attachments (37.481418ms)
no change
no change
+1 -1
View File
@@ -1,4 +1,4 @@
2026-11-20T04:20:27+03:00
2026-11-20T04:53:36+03:00
make test-attachments
BASE_URL="http://localhost:8080" python3 scripts/attachments-check.py
+13 -13
View File
@@ -1,4 +1,4 @@
2026-11-20T04:19:38+03:00
2026-11-20T04:54:22+03:00
make test-persistence ACTION_SCRIPT=scripts/restart.sh
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
@@ -7,12 +7,12 @@ PASS: persistence preserved confirmed data and existing session. Hook scope need
--- hook scope: containers and volumes before/after scripts/restart.sh ---
messenger-lab4_api_1 8535d7918d12 Up 1 second (starting)
messenger-lab4_api_2 ad1700491457 Up 1 second (starting)
messenger-lab4_lb_1 0bed57f994fa Up 1 second
messenger-lab4_postgres_1 53c091ca7f0a Up 1 second (starting)
messenger-lab4_redis_1 e0cfe44337f5 Up 1 second (healthy)
messenger-lab4_s3_1 8548fb0864af Up 1 second (healthy)
messenger-lab4_api_1 a089fe9cc382 Up 5 seconds (healthy)
messenger-lab4_api_2 f309c2ef4e46 Up 5 seconds (healthy)
messenger-lab4_lb_1 efc7da123526 Up 5 seconds
messenger-lab4_postgres_1 c64e954cf92a Up 5 seconds (healthy)
messenger-lab4_redis_1 90ac9f642b09 Up 5 seconds (healthy)
messenger-lab4_s3_1 6ae98c5aef4f Up 5 seconds (healthy)
messenger-lab4_pgdata
messenger-lab4_redisdata
messenger-lab4_s3data
@@ -20,12 +20,12 @@ messenger-lab4_s3data
$ scripts/restart.sh
messenger-lab4_api_1 8535d7918d12 Up 15 seconds (healthy)
messenger-lab4_api_2 ad1700491457 Up 16 seconds (healthy)
messenger-lab4_lb_1 0bed57f994fa Up 16 seconds
messenger-lab4_postgres_1 53c091ca7f0a Up 15 seconds (healthy)
messenger-lab4_redis_1 e0cfe44337f5 Up 15 seconds (healthy)
messenger-lab4_s3_1 8548fb0864af Up 16 seconds (healthy)
messenger-lab4_api_1 a089fe9cc382 Up 15 seconds (healthy)
messenger-lab4_api_2 f309c2ef4e46 Up 15 seconds (healthy)
messenger-lab4_lb_1 efc7da123526 Up 15 seconds
messenger-lab4_postgres_1 c64e954cf92a Up 15 seconds (healthy)
messenger-lab4_redis_1 90ac9f642b09 Up 15 seconds (healthy)
messenger-lab4_s3_1 6ae98c5aef4f Up 15 seconds (healthy)
messenger-lab4_pgdata
messenger-lab4_redisdata
messenger-lab4_s3data
+7 -5
View File
@@ -1,20 +1,22 @@
2026-11-20T04:15:49+03:00
2026-11-20T04:44:19+03:00
bucket privacy and disabled console (checks outside the application API)
$ wget http://s3:9000/messenger-attachments/chats/.../... (anonymous, from backend network)
Connecting to s3:9000 (10.89.5.36:9000)
$ anonymous GET of an existing object from the backend network
Connecting to s3:9000 (10.89.5.22:9000)
HTTP/1.1 403 Forbidden
wget: server returned error: HTTP/1.1 403 Forbidden
$ anonymous bucket listing
Connecting to s3:9000 (10.89.5.36:9000)
Connecting to s3:9000 (10.89.5.22:9000)
HTTP/1.1 403 Forbidden
wget: server returned error: HTTP/1.1 403 Forbidden
$ console port check inside s3 container
$ console port inside the s3 container
console_http=000
console-closed
$ published ports
messenger-lab4_api_1 8080/tcp
messenger-lab4_api_2 8080/tcp
messenger-lab4_lb_1 0.0.0.0:8080->8080/tcp
messenger-lab4_s3_1 9000-9001/tcp
+91 -31
View File
@@ -1,4 +1,4 @@
2026-11-20T04:13:27+03:00
2026-11-20T04:41:03+03:00
reconciler drill: baseline, grace period, dry-run, apply, repeat
$ scripts/reconcile.sh # dry-run, grace 1h, clean baseline
@@ -6,54 +6,69 @@ $ scripts/reconcile.sh # dry-run, grace 1h, clean baseline
"prefix": "chats/",
"grace_seconds": 3600,
"dry_run": true,
"scanned_objects": 15,
"scanned_rows": 15,
"scanned_objects": 18,
"scanned_rows": 18,
"truncated": false,
"orphans": [],
"missing": [],
"deleted_objects": 0,
"deleted_metadata": 0,
"retained_missing": 0,
"delete_failures": 0,
"duration_ms": 7
"s3_requests": 1,
"db_queries": 2,
"duration_ms": 7,
"objects_per_second": 2571.4285714285716
}
$ scripts/s3tool.sh put chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000 - < scripts/fixtures/sample.jpg
put chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000 (653 bytes)
$ scripts/reconcile.sh # fresh orphan stays: grace 1h
$ scripts/reconcile.sh # fresh orphan skipped by grace
{
"prefix": "chats/",
"grace_seconds": 3600,
"dry_run": true,
"scanned_objects": 16,
"scanned_rows": 15,
"scanned_objects": 19,
"scanned_rows": 18,
"truncated": false,
"orphans": [],
"missing": [],
"deleted_objects": 0,
"deleted_metadata": 0,
"retained_missing": 0,
"delete_failures": 0,
"duration_ms": 7
"s3_requests": 1,
"db_queries": 2,
"duration_ms": 8,
"objects_per_second": 2375
}
$ scripts/reconcile.sh -grace 0s # orphan detected, dry-run keeps it
$ scripts/reconcile.sh -grace 0s # detected, dry-run keeps it
{
"prefix": "chats/",
"grace_seconds": 0,
"dry_run": true,
"scanned_objects": 16,
"scanned_rows": 15,
"scanned_objects": 19,
"scanned_rows": 18,
"truncated": false,
"orphans": [
{
"key": "chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000",
"size_bytes": 653,
"last_modified": "2026-11-20T01:13:27Z",
"last_modified": "2026-11-20T01:41:03Z",
"age_seconds": 0
}
],
"missing": [],
"deleted_objects": 0,
"deleted_metadata": 0,
"retained_missing": 0,
"delete_failures": 0,
"duration_ms": 6
"s3_requests": 1,
"db_queries": 2,
"duration_ms": 7,
"objects_per_second": 2714.285714285714
}
$ scripts/reconcile.sh -grace 0s -dry-run=false # apply
@@ -61,21 +76,26 @@ $ scripts/reconcile.sh -grace 0s -dry-run=false # apply
"prefix": "chats/",
"grace_seconds": 0,
"dry_run": false,
"scanned_objects": 16,
"scanned_rows": 15,
"scanned_objects": 19,
"scanned_rows": 18,
"truncated": false,
"orphans": [
{
"key": "chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000",
"size_bytes": 653,
"last_modified": "2026-11-20T01:13:27Z",
"age_seconds": 3
"last_modified": "2026-11-20T01:41:03Z",
"age_seconds": 4
}
],
"missing": [],
"deleted_objects": 1,
"deleted_metadata": 0,
"retained_missing": 0,
"delete_failures": 0,
"duration_ms": 12
"s3_requests": 2,
"db_queries": 2,
"duration_ms": 15,
"objects_per_second": 1266.6666666666667
}
$ scripts/reconcile.sh -grace 0s -dry-run=false # repeat: nothing left
@@ -83,41 +103,81 @@ $ scripts/reconcile.sh -grace 0s -dry-run=false # repeat: nothing left
"prefix": "chats/",
"grace_seconds": 0,
"dry_run": false,
"scanned_objects": 15,
"scanned_rows": 15,
"scanned_objects": 18,
"scanned_rows": 18,
"truncated": false,
"orphans": [],
"missing": [],
"deleted_objects": 0,
"deleted_metadata": 0,
"retained_missing": 0,
"delete_failures": 0,
"duration_ms": 7
"s3_requests": 1,
"db_queries": 2,
"duration_ms": 8,
"objects_per_second": 2250
}
missing drill: metadata row without object
$ scripts/s3tool.sh delete chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c
deleted chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c
$ scripts/s3tool.sh delete chats/<chat>/<attachment> of an existing attachment
$ scripts/s3tool.sh delete chats/bd1c9e17-0366-44b4-bd98-0a479a2a637c/1e9b9e12-eab2-4846-8952-e739a4257b37
deleted chats/bd1c9e17-0366-44b4-bd98-0a479a2a637c/1e9b9e12-eab2-4846-8952-e739a4257b37
$ scripts/reconcile.sh -grace 0s # reports missing, keeps metadata
$ scripts/reconcile.sh -grace 0s # missing reported, metadata kept in dry-run
{
"prefix": "chats/",
"grace_seconds": 0,
"dry_run": true,
"scanned_objects": 14,
"scanned_rows": 15,
"scanned_objects": 17,
"scanned_rows": 18,
"truncated": false,
"orphans": [],
"missing": [
{
"id": "97121cfe-6b29-4227-867f-3188e72a725c",
"object_key": "chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c",
"chat_id": "9452049e-b86c-4d9a-8060-d98f6966b956",
"created_at": "2026-11-20T01:11:14Z"
"id": "1e9b9e12-eab2-4846-8952-e739a4257b37",
"object_key": "chats/bd1c9e17-0366-44b4-bd98-0a479a2a637c/1e9b9e12-eab2-4846-8952-e739a4257b37",
"chat_id": "bd1c9e17-0366-44b4-bd98-0a479a2a637c",
"created_at": "2026-11-20T01:40:28Z",
"referenced_by_message": false
}
],
"deleted_objects": 0,
"deleted_metadata": 0,
"retained_missing": 0,
"delete_failures": 0,
"duration_ms": 6
"s3_requests": 1,
"db_queries": 2,
"duration_ms": 9,
"objects_per_second": 1888.8888888888891
}
metadata rows still present: 1
$ scripts/reconcile.sh -grace 0s -dry-run=false # safe repair: unreferenced dead row removed
{
"prefix": "chats/",
"grace_seconds": 0,
"dry_run": false,
"scanned_objects": 17,
"scanned_rows": 18,
"truncated": false,
"orphans": [],
"missing": [
{
"id": "1e9b9e12-eab2-4846-8952-e739a4257b37",
"object_key": "chats/bd1c9e17-0366-44b4-bd98-0a479a2a637c/1e9b9e12-eab2-4846-8952-e739a4257b37",
"chat_id": "bd1c9e17-0366-44b4-bd98-0a479a2a637c",
"created_at": "2026-11-20T01:40:28Z",
"referenced_by_message": false
}
],
"deleted_objects": 0,
"deleted_metadata": 1,
"retained_missing": 0,
"delete_failures": 0,
"s3_requests": 1,
"db_queries": 3,
"duration_ms": 8,
"objects_per_second": 2125
}
+17
View File
@@ -0,0 +1,17 @@
2026-11-20T04:57:42+03:00
S3 unavailable: readiness and client-visible behaviour
$ GET /health/ready on a replica
HTTP/1.1 503 Service Unavailable
wget: server returned error: HTTP/1.1 503 Service Unavailable
$ GET /health/live on a replica (process is alive)
HTTP/1.1 200 OK
Content-Type: application/json; charset=utf-8
X-Instance-Id: a089fe9cc382
$ client request through the load balancer
status=503
$ readiness after S3 recovery
status=200
+1 -1
View File
@@ -1,4 +1,4 @@
2026-11-20T04:20:26+03:00
2026-11-20T04:53:35+03:00
make test
python3 tests/smoke.py --base-url "http://localhost:8080"
+4 -4
View File
@@ -1,16 +1,16 @@
2026-11-20T04:18:15+03:00
2026-11-20T04:56:31+03:00
object bytes live in the RustFS volume, metadata in PostgreSQL
id | object_key | size_bytes | sha256
--------------------------------------+---------------------------------------------------------------------------------+------------+------------------------------------------------------------------
3dbc5415-9791-4983-a253-bc71fdc79a24 | chats/3ec617cd-fddf-4ae1-ad9d-37a4bb319b76/3dbc5415-9791-4983-a253-bc71fdc79a24 | 126609 | 518fd54cb19f8ceaca99139652fb96ed3a5d502392450846c832924b1722d2bf
82d5d4f3-e97a-4aea-9501-69ef2b74e138 | chats/e8618579-667a-4423-aa62-5402596e57d8/82d5d4f3-e97a-4aea-9501-69ef2b74e138 | 111817 | 99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75
(1 row)
S3 volume listing for that object
total 128
-rw-r--r-- 1 rustfs rustfs 127190 Nov 20 01:17 xl.meta
total 112
-rw-r--r-- 1 rustfs rustfs 112398 Nov 20 01:55 xl.meta
attachments table columns (no binary content)
+18 -16
View File
@@ -1,4 +1,4 @@
2026-11-20T04:20:28+03:00
2026-11-20T04:53:38+03:00
make test-go
? messenger/cmd/reconcile [no test files]
@@ -10,50 +10,52 @@ make test-go
=== RUN TestSessionExpiry
--- PASS: TestSessionExpiry (3.09s)
=== RUN TestLogout
--- PASS: TestLogout (0.11s)
--- PASS: TestLogout (0.13s)
=== RUN TestWrongPassword
--- PASS: TestWrongPassword (0.10s)
=== RUN TestRedisUnavailable
redis: 2026/11/20 04:20:32 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/20 04:20:32 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/20 04:20:33 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/20 04:20:33 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/20 04:20:34 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/20 04:20:34 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
--- PASS: TestRedisUnavailable (2.83s)
redis: 2026/11/20 04:53:42 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/20 04:53:43 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/20 04:53:43 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/20 04:53:44 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/20 04:53:44 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/20 04:53:44 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
--- PASS: TestRedisUnavailable (2.95s)
=== RUN TestDatabaseUnavailable
--- PASS: TestDatabaseUnavailable (0.17s)
=== RUN TestInvalidJSON
--- PASS: TestInvalidJSON (0.01s)
=== RUN TestForeignChat
--- PASS: TestForeignChat (0.16s)
--- PASS: TestForeignChat (0.18s)
=== RUN TestEmptyMessage
redis: 2026/11/20 04:20:35 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/20 04:53:45 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
--- PASS: TestEmptyMessage (0.08s)
=== RUN TestDuplicateUsername
--- PASS: TestDuplicateUsername (0.10s)
=== RUN TestPaginationMultiplePages
--- PASS: TestPaginationMultiplePages (0.08s)
--- PASS: TestPaginationMultiplePages (0.09s)
=== RUN TestConcurrentMessages
--- PASS: TestConcurrentMessages (0.59s)
--- PASS: TestConcurrentMessages (0.57s)
PASS
ok messenger/internal/handler 7.325s
ok messenger/internal/handler 7.474s
=== RUN TestPlanDetectsOrphansAndMissing
--- PASS: TestPlanDetectsOrphansAndMissing (0.00s)
=== RUN TestPlanHonoursGraceAndPrefix
--- PASS: TestPlanHonoursGraceAndPrefix (0.00s)
=== RUN TestPlanNeverDeletesReferencedObjects
--- PASS: TestPlanNeverDeletesReferencedObjects (0.00s)
=== RUN TestPlanOnlyReportsRowsInsidePrefix
--- PASS: TestPlanOnlyReportsRowsInsidePrefix (0.00s)
=== RUN TestPlanOutputIsSorted
--- PASS: TestPlanOutputIsSorted (0.00s)
PASS
ok messenger/internal/reconcile 0.002s
=== RUN TestRegisterUsernameRace
--- PASS: TestRegisterUsernameRace (0.05s)
--- PASS: TestRegisterUsernameRace (0.02s)
=== RUN TestCreateChatRollback
--- PASS: TestCreateChatRollback (0.02s)
PASS
ok messenger/internal/repository/postgres 0.071s
ok messenger/internal/repository/postgres 0.040s
? messenger/internal/repository/redis [no test files]
? messenger/internal/repository/s3 [no test files]
? messenger/internal/router [no test files]
+41
View File
@@ -0,0 +1,41 @@
2026-11-20T04:42:15+03:00
traversal cost of the reconciler (synthetic objects under chats/bench/, 1 KiB each); scan uses LIST pages, repair deletes one by one
{
"prefix": "chats/bench",
"measurements": [
{
"objects": 100,
"scan_ms": 9,
"scan_s3_requests": 1,
"scan_db_queries": 2,
"scan_objects_per_second": 11111.1,
"scan_ms_per_object": 0.09,
"repair_ms": 1324,
"repair_s3_requests": 101,
"repair_deleted": 100
},
{
"objects": 1000,
"scan_ms": 34,
"scan_s3_requests": 1,
"scan_db_queries": 2,
"scan_objects_per_second": 29411.8,
"scan_ms_per_object": 0.034,
"repair_ms": 12232,
"repair_s3_requests": 1001,
"repair_deleted": 1000
},
{
"objects": 5000,
"scan_ms": 166,
"scan_s3_requests": 5,
"scan_db_queries": 2,
"scan_objects_per_second": 30120.5,
"scan_ms_per_object": 0.0332,
"repair_ms": 69929,
"repair_s3_requests": 5005,
"repair_deleted": 5000
}
]
}
+193
View File
@@ -0,0 +1,193 @@
2026-11-20T04:41:49+03:00
two-store consistency: injections on both sides, dry-run, safe repair, repeat, protection of fresh and used data
injections:
S3 has an object that has no metadata row (fresh orphan)
metadata exists but the object is gone: one attachment used by a message, one unused
$ scripts/s3tool.sh put chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000 - < scripts/fixtures/sample.jpg
put chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000 (653 bytes)
$ scripts/s3tool.sh delete chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1 # used
deleted chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1
$ scripts/s3tool.sh delete chats/d03b2c6b-67fd-4b5d-bc17-9e926440ff8d/c49ab2de-258a-4851-8ad2-f7f585fd3488 # unused
deleted chats/d03b2c6b-67fd-4b5d-bc17-9e926440ff8d/c49ab2de-258a-4851-8ad2-f7f585fd3488
$ scripts/reconcile.sh -grace 1h # dry-run
{
"prefix": "chats/",
"grace_seconds": 3600,
"dry_run": true,
"scanned_objects": 19,
"scanned_rows": 20,
"truncated": false,
"orphans": [],
"missing": [
{
"id": "6a9f0506-2126-497a-b8b4-89ac728fe5a1",
"object_key": "chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1",
"chat_id": "47f62016-7525-4910-9823-0b21f2664b10",
"created_at": "2026-11-20T01:41:41Z",
"referenced_by_message": true
},
{
"id": "c49ab2de-258a-4851-8ad2-f7f585fd3488",
"object_key": "chats/d03b2c6b-67fd-4b5d-bc17-9e926440ff8d/c49ab2de-258a-4851-8ad2-f7f585fd3488",
"chat_id": "d03b2c6b-67fd-4b5d-bc17-9e926440ff8d",
"created_at": "2026-11-20T01:41:41Z",
"referenced_by_message": false
}
],
"deleted_objects": 0,
"deleted_metadata": 0,
"retained_missing": 0,
"delete_failures": 0,
"s3_requests": 1,
"db_queries": 2,
"duration_ms": 8,
"objects_per_second": 2375
}
$ scripts/reconcile.sh -grace 1h -dry-run=false # apply: fresh object survives, safe metadata repair
{
"prefix": "chats/",
"grace_seconds": 3600,
"dry_run": false,
"scanned_objects": 19,
"scanned_rows": 20,
"truncated": false,
"orphans": [],
"missing": [
{
"id": "6a9f0506-2126-497a-b8b4-89ac728fe5a1",
"object_key": "chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1",
"chat_id": "47f62016-7525-4910-9823-0b21f2664b10",
"created_at": "2026-11-20T01:41:41Z",
"referenced_by_message": true
},
{
"id": "c49ab2de-258a-4851-8ad2-f7f585fd3488",
"object_key": "chats/d03b2c6b-67fd-4b5d-bc17-9e926440ff8d/c49ab2de-258a-4851-8ad2-f7f585fd3488",
"chat_id": "d03b2c6b-67fd-4b5d-bc17-9e926440ff8d",
"created_at": "2026-11-20T01:41:41Z",
"referenced_by_message": false
}
],
"deleted_objects": 0,
"deleted_metadata": 1,
"retained_missing": 1,
"delete_failures": 0,
"s3_requests": 1,
"db_queries": 3,
"duration_ms": 12,
"objects_per_second": 1583.3333333333333
}
$ scripts/s3tool.sh list chats/aaaaaaaa-0000-4000-8000-000000000000 # fresh orphan still present
chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000 653 2026-11-20T01:41:49Z
$ scripts/reconcile.sh -grace 0s # now eligible for deletion
{
"prefix": "chats/",
"grace_seconds": 0,
"dry_run": true,
"scanned_objects": 19,
"scanned_rows": 19,
"truncated": false,
"orphans": [
{
"key": "chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000",
"size_bytes": 653,
"last_modified": "2026-11-20T01:41:49Z",
"age_seconds": 7
}
],
"missing": [
{
"id": "6a9f0506-2126-497a-b8b4-89ac728fe5a1",
"object_key": "chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1",
"chat_id": "47f62016-7525-4910-9823-0b21f2664b10",
"created_at": "2026-11-20T01:41:41Z",
"referenced_by_message": true
}
],
"deleted_objects": 0,
"deleted_metadata": 0,
"retained_missing": 0,
"delete_failures": 0,
"s3_requests": 1,
"db_queries": 2,
"duration_ms": 8,
"objects_per_second": 2375
}
$ scripts/reconcile.sh -grace 0s -dry-run=false # apply
{
"prefix": "chats/",
"grace_seconds": 0,
"dry_run": false,
"scanned_objects": 19,
"scanned_rows": 19,
"truncated": false,
"orphans": [
{
"key": "chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000",
"size_bytes": 653,
"last_modified": "2026-11-20T01:41:49Z",
"age_seconds": 13
}
],
"missing": [
{
"id": "6a9f0506-2126-497a-b8b4-89ac728fe5a1",
"object_key": "chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1",
"chat_id": "47f62016-7525-4910-9823-0b21f2664b10",
"created_at": "2026-11-20T01:41:41Z",
"referenced_by_message": true
}
],
"deleted_objects": 1,
"deleted_metadata": 0,
"retained_missing": 1,
"delete_failures": 0,
"s3_requests": 2,
"db_queries": 2,
"duration_ms": 15,
"objects_per_second": 1266.6666666666667
}
$ scripts/reconcile.sh -grace 0s -dry-run=false # repeat: idempotent
{
"prefix": "chats/",
"grace_seconds": 0,
"dry_run": false,
"scanned_objects": 18,
"scanned_rows": 19,
"truncated": false,
"orphans": [],
"missing": [
{
"id": "6a9f0506-2126-497a-b8b4-89ac728fe5a1",
"object_key": "chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1",
"chat_id": "47f62016-7525-4910-9823-0b21f2664b10",
"created_at": "2026-11-20T01:41:41Z",
"referenced_by_message": true
}
],
"deleted_objects": 0,
"deleted_metadata": 0,
"retained_missing": 1,
"delete_failures": 0,
"s3_requests": 1,
"db_queries": 2,
"duration_ms": 9,
"objects_per_second": 2000.0000000000002
}
used attachments remain untouched (rows and objects)
used_attachments | ready
------------------+-------
3 | 3
(1 row)
retained referenced metadata row for the attachment whose object was deleted: 1
object of that referenced attachment is still absent: 0
+11 -11
View File
@@ -3,30 +3,30 @@
"uploads": 10,
"fixture": "large.png",
"fixture_bytes": 126609,
"elapsed_seconds": 1.235,
"elapsed_seconds": 1.009,
"successful": 10,
"failed": 0,
"failures": [],
"latency_ms": {
"p50": 663.7,
"p95": 1224.7,
"p99": 1224.7
"p50": 660.9,
"p95": 999.3,
"p99": 999.3
},
"api_containers": [
"messenger-lab4_api_1",
"messenger-lab4_api_2"
],
"peak_bytes_before": {
"messenger-lab4_api_1": 5050368,
"messenger-lab4_api_2": 4820992
"messenger-lab4_api_1": 5951488,
"messenger-lab4_api_2": 4993024
},
"peak_bytes_after": {
"messenger-lab4_api_1": 368119808,
"messenger-lab4_api_2": 368185344
"messenger-lab4_api_1": 348913664,
"messenger-lab4_api_2": 348721152
},
"peak_delta_bytes": {
"messenger-lab4_api_1": 363069440,
"messenger-lab4_api_2": 363364352
"messenger-lab4_api_1": 342962176,
"messenger-lab4_api_2": 343728128
},
"peak_max_mb": 351.13
"peak_max_mb": 332.75
}
+7 -1
View File
@@ -53,7 +53,13 @@ func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) {
h.fail(w, r, err)
return
}
attachment, content, err := h.attachments.Open(r.Context(), actor(r), id, r.URL.Query().Get("variant"))
query := r.URL.Query()
variant := query.Get("variant")
if query.Has("variant") && variant == "" {
h.fail(w, r, badRequest("variant must be original"))
return
}
attachment, content, err := h.attachments.Open(r.Context(), actor(r), id, variant)
if err != nil {
h.fail(w, r, err)
return
+3 -2
View File
@@ -1,6 +1,7 @@
package handler
import (
"encoding/json"
"time"
"uuid"
@@ -19,8 +20,8 @@ type createChatRequest struct {
}
type createMessageRequest struct {
Text *string `json:"text"`
AttachmentIDs *[]string `json:"attachment_ids"`
Text *string `json:"text"`
AttachmentIDs json.RawMessage `json:"attachment_ids"`
}
type userResponse struct {
+15 -4
View File
@@ -1,6 +1,8 @@
package handler
import (
"bytes"
"encoding/json"
"net/http"
"regexp"
"uuid"
@@ -31,10 +33,19 @@ func (h *Handler) SendMessage(w http.ResponseWriter, r *http.Request) {
return
}
var attachmentIDs []uuid.UUID
if req.AttachmentIDs != nil {
attachmentIDs = make([]uuid.UUID, len(*req.AttachmentIDs))
for i, raw := range *req.AttachmentIDs {
id, ok := parseUUID(raw)
if len(req.AttachmentIDs) > 0 {
if bytes.Equal(bytes.TrimSpace(req.AttachmentIDs), []byte("null")) {
h.fail(w, r, badRequest("attachment_ids must be an array of UUIDs"))
return
}
var raw []string
if err := json.Unmarshal(req.AttachmentIDs, &raw); err != nil {
h.fail(w, r, badRequest("attachment_ids must be an array of UUIDs"))
return
}
attachmentIDs = make([]uuid.UUID, len(raw))
for i, value := range raw {
id, ok := parseUUID(value)
if !ok {
h.fail(w, r, badRequest("attachment_ids must contain UUIDs"))
return
+33 -23
View File
@@ -28,31 +28,37 @@ type Orphan struct {
}
type Missing struct {
ID string `json:"id"`
ObjectKey string `json:"object_key"`
ChatID string `json:"chat_id"`
CreatedAt string `json:"created_at"`
ID string `json:"id"`
ObjectKey string `json:"object_key"`
ChatID string `json:"chat_id"`
CreatedAt string `json:"created_at"`
Referenced bool `json:"referenced_by_message"`
}
type Report struct {
Prefix string `json:"prefix"`
GraceSeconds int64 `json:"grace_seconds"`
DryRun bool `json:"dry_run"`
ScannedObjects int `json:"scanned_objects"`
ScannedRows int `json:"scanned_rows"`
Truncated bool `json:"truncated"`
Orphans []Orphan `json:"orphans"`
Missing []Missing `json:"missing"`
Deleted int `json:"deleted_objects"`
DeleteFailures int `json:"delete_failures"`
DurationMS int64 `json:"duration_ms"`
Prefix string `json:"prefix"`
GraceSeconds int64 `json:"grace_seconds"`
DryRun bool `json:"dry_run"`
ScannedObjects int `json:"scanned_objects"`
ScannedRows int `json:"scanned_rows"`
Truncated bool `json:"truncated"`
Orphans []Orphan `json:"orphans"`
Missing []Missing `json:"missing"`
DeletedObjects int `json:"deleted_objects"`
DeletedMetadata int `json:"deleted_metadata"`
RetainedMissing int `json:"retained_missing"`
DeleteFailures int `json:"delete_failures"`
S3Requests int64 `json:"s3_requests"`
DBQueries int `json:"db_queries"`
DurationMS int64 `json:"duration_ms"`
ObjectsPerSecond float64 `json:"objects_per_second"`
}
func Plan(objects []Object, rows []Metadata, prefix string, now time.Time, grace time.Duration) ([]Orphan, []Missing) {
referenced := make(map[string]struct{}, len(rows))
func Plan(objects []Object, rows []Metadata, referenced map[uuid.UUID]bool, prefix string, now time.Time, grace time.Duration) ([]Orphan, []Missing) {
known := make(map[string]struct{}, len(rows))
existing := make(map[string]struct{}, len(objects))
for _, row := range rows {
referenced[row.ObjectKey] = struct{}{}
known[row.ObjectKey] = struct{}{}
}
for _, object := range objects {
existing[object.Key] = struct{}{}
@@ -63,7 +69,7 @@ func Plan(objects []Object, rows []Metadata, prefix string, now time.Time, grace
if !strings.HasPrefix(object.Key, prefix) {
continue
}
if _, ok := referenced[object.Key]; ok {
if _, ok := known[object.Key]; ok {
continue
}
age := now.Sub(object.LastModified)
@@ -80,14 +86,18 @@ func Plan(objects []Object, rows []Metadata, prefix string, now time.Time, grace
missing := make([]Missing, 0)
for _, row := range rows {
if !strings.HasPrefix(row.ObjectKey, prefix) {
continue
}
if _, ok := existing[row.ObjectKey]; ok {
continue
}
missing = append(missing, Missing{
ID: row.ID.String(),
ObjectKey: row.ObjectKey,
ChatID: row.ChatID.String(),
CreatedAt: row.CreatedAt.UTC().Format(time.RFC3339),
ID: row.ID.String(),
ObjectKey: row.ObjectKey,
ChatID: row.ChatID.String(),
CreatedAt: row.CreatedAt.UTC().Format(time.RFC3339),
Referenced: referenced[row.ID],
})
}
+33 -9
View File
@@ -8,7 +8,7 @@ import (
func TestPlanDetectsOrphansAndMissing(t *testing.T) {
now := time.Date(2026, 11, 20, 12, 0, 0, 0, time.UTC)
usedID, missingID := uuid.New(), uuid.New()
usedID, missingID, unreferencedID := uuid.New(), uuid.New(), uuid.New()
objects := []Object{
{Key: "chats/a/used", Size: 10, LastModified: now.Add(-2 * time.Hour)},
{Key: "chats/a/orphan-old", Size: 20, LastModified: now.Add(-2 * time.Hour)},
@@ -17,16 +17,28 @@ func TestPlanDetectsOrphansAndMissing(t *testing.T) {
}
rows := []Metadata{
{ID: usedID, ObjectKey: "chats/a/used", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)},
{ID: missingID, ObjectKey: "chats/a/missing", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)},
{ID: missingID, ObjectKey: "chats/a/missing-referenced", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)},
{ID: unreferencedID, ObjectKey: "chats/a/missing-orphan-row", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)},
}
referenced := map[uuid.UUID]bool{missingID: true}
orphans, missing := Plan(objects, rows, "chats/", now, time.Hour)
orphans, missing := Plan(objects, rows, referenced, "chats/", now, time.Hour)
if len(orphans) != 1 || orphans[0].Key != "chats/a/orphan-old" {
t.Fatalf("orphans = %+v, want only chats/a/orphan-old", orphans)
}
if len(missing) != 1 || missing[0].ObjectKey != "chats/a/missing" {
t.Fatalf("missing = %+v, want only chats/a/missing", missing)
if len(missing) != 2 {
t.Fatalf("missing = %+v, want two rows", missing)
}
byKey := map[string]Missing{}
for _, item := range missing {
byKey[item.ObjectKey] = item
}
if !byKey["chats/a/missing-referenced"].Referenced {
t.Error("referenced missing row must be flagged")
}
if byKey["chats/a/missing-orphan-row"].Referenced {
t.Error("unreferenced missing row must not be flagged")
}
}
@@ -36,12 +48,12 @@ func TestPlanHonoursGraceAndPrefix(t *testing.T) {
{Key: "chats/a/fresh", Size: 1, LastModified: now.Add(-30 * time.Second)},
{Key: "outside/key", Size: 1, LastModified: now.Add(-48 * time.Hour)},
}
orphans, _ := Plan(objects, nil, "chats/", now, time.Hour)
orphans, _ := Plan(objects, nil, nil, "chats/", now, time.Hour)
if len(orphans) != 0 {
t.Fatalf("orphans = %+v, want none", orphans)
}
orphans, _ = Plan(objects, nil, "chats/", now, 0)
orphans, _ = Plan(objects, nil, nil, "chats/", now, 0)
if len(orphans) != 1 || orphans[0].Key != "chats/a/fresh" {
t.Fatalf("orphans with zero grace = %+v, want chats/a/fresh", orphans)
}
@@ -52,7 +64,7 @@ func TestPlanNeverDeletesReferencedObjects(t *testing.T) {
objects := []Object{{Key: "chats/a/used", Size: 1, LastModified: now.Add(-100 * time.Hour)}}
rows := []Metadata{{ID: uuid.New(), ObjectKey: "chats/a/used", ChatID: uuid.New(), CreatedAt: now.Add(-100 * time.Hour)}}
orphans, missing := Plan(objects, rows, "chats/", now, 0)
orphans, missing := Plan(objects, rows, nil, "chats/", now, 0)
if len(orphans) != 0 {
t.Fatalf("orphans = %+v, want none for referenced object", orphans)
}
@@ -61,6 +73,18 @@ func TestPlanNeverDeletesReferencedObjects(t *testing.T) {
}
}
func TestPlanOnlyReportsRowsInsidePrefix(t *testing.T) {
now := time.Date(2026, 11, 20, 12, 0, 0, 0, time.UTC)
rows := []Metadata{
{ID: uuid.New(), ObjectKey: "chats/a/inside", ChatID: uuid.New(), CreatedAt: now},
{ID: uuid.New(), ObjectKey: "bench/outside", ChatID: uuid.New(), CreatedAt: now},
}
_, missing := Plan(nil, rows, nil, "chats/", now, 0)
if len(missing) != 1 || missing[0].ObjectKey != "chats/a/inside" {
t.Fatalf("missing = %+v, want only rows under the scanned prefix", missing)
}
}
func TestPlanOutputIsSorted(t *testing.T) {
now := time.Date(2026, 11, 20, 12, 0, 0, 0, time.UTC)
objects := []Object{
@@ -68,7 +92,7 @@ func TestPlanOutputIsSorted(t *testing.T) {
{Key: "chats/a/b", LastModified: now.Add(-time.Hour)},
{Key: "chats/a/m", LastModified: now.Add(-time.Hour)},
}
orphans, _ := Plan(objects, nil, "chats/", now, 0)
orphans, _ := Plan(objects, nil, nil, "chats/", now, 0)
if len(orphans) != 3 || orphans[0].Key != "chats/a/b" || orphans[2].Key != "chats/a/z" {
t.Fatalf("orphans are not sorted: %+v", orphans)
}
+8 -3
View File
@@ -37,11 +37,16 @@ func EndpointHost(endpoint string) (string, bool) {
}
func New(endpoint, region, accessKeyID, secretAccessKey, bucket string) (*Store, error) {
return NewWithTransport(endpoint, region, accessKeyID, secretAccessKey, bucket, nil)
}
func NewWithTransport(endpoint, region, accessKeyID, secretAccessKey, bucket string, transport http.RoundTripper) (*Store, error) {
host, secure := EndpointHost(endpoint)
client, err := minio.New(host, &minio.Options{
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
Region: region,
Secure: secure,
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
Region: region,
Secure: secure,
Transport: transport,
})
if err != nil {
return nil, err
+5 -1
View File
@@ -11,6 +11,7 @@ import (
_ "image/png"
"io"
"net/http"
"strings"
"time"
"unicode"
"unicode/utf8"
@@ -159,7 +160,10 @@ func acquire(ctx context.Context, sem chan struct{}) error {
func release(sem chan struct{}) { <-sem }
func checkFilename(filename string) error {
if n := utf8.RuneCountInString(filename); n < 1 || n > maxFilenameRunes {
if strings.TrimSpace(filename) == "" {
return fmt.Errorf("%w: filename must not be blank", domain.ErrInvalidInput)
}
if n := utf8.RuneCountInString(filename); n > maxFilenameRunes {
return fmt.Errorf("%w: filename must be 1 to 255 characters", domain.ErrInvalidInput)
}
for _, r := range filename {
+62
View File
@@ -0,0 +1,62 @@
#!/usr/bin/env python3
"""Reconciler traversal cost for several object counts; creates synthetic objects and removes them."""
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
S3TOOL = ROOT / 'scripts/s3tool.sh'
RECONCILE = ROOT / 'scripts/reconcile.sh'
PREFIX = 'chats/bench'
SIZES = (100, 1000, 5000)
def call(arguments):
return subprocess.run(arguments, capture_output=True, text=True, check=False)
def reconcile(*arguments):
result = call([str(RECONCILE), *arguments])
try:
return json.loads(result.stdout)
except json.JSONDecodeError:
print(result.stdout[-500:], file=sys.stderr)
print(result.stderr[-500:], file=sys.stderr)
raise SystemExit('reconcile did not return JSON')
def main():
measurements = []
for count in SIZES:
seeded = call([str(S3TOOL), 'seed', PREFIX, str(count), '1024'])
if seeded.returncode != 0:
raise SystemExit('seed failed: ' + seeded.stderr[-400:])
scan = reconcile('-prefix', PREFIX + '/', '-grace', '0s')
if scan['scanned_objects'] != count:
raise SystemExit(f'expected {count} objects, scanned {scan["scanned_objects"]}')
if len(scan['orphans']) != count:
raise SystemExit(f'expected {count} orphans, got {len(scan["orphans"])}')
repair = reconcile('-prefix', PREFIX + '/', '-grace', '0s', '-dry-run=false')
if repair['deleted_objects'] != count:
raise SystemExit(f'expected {count} deletions, got {repair["deleted_objects"]}')
measurements.append({
'objects': count,
'scan_ms': scan['duration_ms'],
'scan_s3_requests': scan['s3_requests'],
'scan_db_queries': scan['db_queries'],
'scan_objects_per_second': round(scan['objects_per_second'], 1),
'scan_ms_per_object': round(scan['duration_ms'] / count, 4),
'repair_ms': repair['duration_ms'],
'repair_s3_requests': repair['s3_requests'],
'repair_deleted': repair['deleted_objects'],
})
print(json.dumps({'prefix': PREFIX, 'measurements': measurements}, indent=2))
if __name__ == '__main__':
main()
+4
View File
@@ -14,6 +14,7 @@ def main():
p.add_argument('--expect', type=int, default=201)
p.add_argument('--file', default=str(ROOT / 'scripts/fixtures/sample.jpg'))
p.add_argument('--content-type', default='image/jpeg')
p.add_argument('--attach', action='store_true', help='also attach the upload to a new message')
args = arguments('Single upload used by compensation drills; creates synthetic data', configure)
source = Path(args.file)
@@ -29,6 +30,9 @@ def main():
if response['status'] == 201:
attachment = response['json']
print(f"attachment={attachment['id']} size={attachment['size_bytes']}")
if args.attach:
message = c.message(alice, chat, '', attachments=[attachment['id']])['json']
print(f"message={message['id']} attachment_ids={message['attachment_ids']}")
if __name__ == '__main__':