forked from mtusi2026/mtusi2026lab4
feat: two-store reconciler with safe repair and cost measurement (grade 5, track 2)
This commit is contained in:
@@ -8,11 +8,11 @@ TEST_DATABASE_URL ?= $(shell sed 's/@postgres:5432/@127.0.0.1:55432/' secrets/da
|
||||
TEST_REDIS_URL ?= redis://127.0.0.1:56379/0
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help up down check validate test load test-persistence test-failover test-idempotency test-attachments test-memory test-go
|
||||
.PHONY: help up down check validate test load test-persistence test-failover test-idempotency test-attachments test-contract test-memory test-cost test-reconcile test-go
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@echo "up/down: start/stop the stand, volumes kept; test-idempotency: one key in parallel via LB; test-attachments: attachment contract checks; test-go: Go tests against the running stand"
|
||||
@echo "up/down: start/stop the stand, volumes kept; test-idempotency: one key in parallel via LB; test-attachments: attachment contract checks; test-memory: concurrent upload memory; test-cost: reconciler traversal cost; test-reconcile: reconciler dry-run; test-go: Go tests against the running stand"
|
||||
|
||||
up:
|
||||
$(COMPOSE) up -d --build
|
||||
@@ -46,9 +46,18 @@ test-idempotency:
|
||||
test-attachments:
|
||||
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/attachments-check.py
|
||||
|
||||
test-contract:
|
||||
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/contract-audit.py
|
||||
|
||||
test-memory:
|
||||
BASE_URL="$(BASE_URL)" $(PYTHON) scripts/upload-load.py --concurrency 10 --output evidence/upload-memory.json
|
||||
|
||||
test-cost:
|
||||
$(PYTHON) scripts/track2-cost.py
|
||||
|
||||
test-reconcile:
|
||||
scripts/reconcile.sh -grace 0s
|
||||
|
||||
test-go:
|
||||
@TEST_DATABASE_URL="$(TEST_DATABASE_URL)" TEST_REDIS_URL="$(TEST_REDIS_URL)" go test -count=1 -v ./...
|
||||
|
||||
|
||||
+81
-8
@@ -5,8 +5,11 @@ import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
"uuid"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
@@ -17,20 +20,31 @@ import (
|
||||
"messenger/internal/repository/s3"
|
||||
)
|
||||
|
||||
type countingTransport struct {
|
||||
base http.RoundTripper
|
||||
count atomic.Int64
|
||||
}
|
||||
|
||||
func (t *countingTransport) RoundTrip(request *http.Request) (*http.Response, error) {
|
||||
t.count.Add(1)
|
||||
return t.base.RoundTrip(request)
|
||||
}
|
||||
|
||||
func main() {
|
||||
dryRun := flag.Bool("dry-run", true, "report only, do not delete orphan objects")
|
||||
dryRun := flag.Bool("dry-run", true, "report only, do not delete or repair anything")
|
||||
grace := flag.Duration("grace", time.Hour, "minimum object age before it can be deleted")
|
||||
prefix := flag.String("prefix", "chats/", "object key prefix to scan")
|
||||
limit := flag.Int("limit", 10000, "maximum number of objects to scan")
|
||||
repairMissing := flag.Bool("repair-missing", true, "delete metadata rows whose object is gone and that no message references")
|
||||
flag.Parse()
|
||||
|
||||
if err := run(*dryRun, *grace, *prefix, *limit); err != nil {
|
||||
if err := run(*dryRun, *grace, *prefix, *limit, *repairMissing); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "reconcile failed:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run(dryRun bool, grace time.Duration, prefix string, limit int) error {
|
||||
func run(dryRun bool, grace time.Duration, prefix string, limit int, repairMissing bool) error {
|
||||
if grace < 0 {
|
||||
return fmt.Errorf("grace must not be negative")
|
||||
}
|
||||
@@ -55,10 +69,11 @@ func run(dryRun bool, grace time.Duration, prefix string, limit int) error {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
store, err := s3.New(endpoint, region, accessKeyID, secretAccessKey, bucket)
|
||||
transport := &countingTransport{base: http.DefaultTransport}
|
||||
store, err := s3.NewWithTransport(endpoint, region, accessKeyID, secretAccessKey, bucket, transport)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -77,8 +92,12 @@ func run(dryRun bool, grace time.Duration, prefix string, limit int) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
referenced, err := loadReferences(ctx, pool)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
orphans, missing := reconcile.Plan(toObjects(scanned), rows, prefix, time.Now(), grace)
|
||||
orphans, missing := reconcile.Plan(toObjects(scanned), rows, referenced, prefix, time.Now(), grace)
|
||||
if truncated {
|
||||
missing = nil
|
||||
}
|
||||
@@ -93,16 +112,50 @@ func run(dryRun bool, grace time.Duration, prefix string, limit int) error {
|
||||
Orphans: orphans,
|
||||
Missing: missing,
|
||||
}
|
||||
dbQueries := 2
|
||||
|
||||
if !dryRun {
|
||||
for _, orphan := range orphans {
|
||||
if err := store.Delete(ctx, orphan.Key); err != nil {
|
||||
report.DeleteFailures++
|
||||
continue
|
||||
}
|
||||
report.Deleted++
|
||||
report.DeletedObjects++
|
||||
}
|
||||
if repairMissing {
|
||||
for _, item := range missing {
|
||||
if item.Referenced {
|
||||
report.RetainedMissing++
|
||||
continue
|
||||
}
|
||||
id, err := uuid.Parse(item.ID)
|
||||
if err != nil {
|
||||
report.DeleteFailures++
|
||||
continue
|
||||
}
|
||||
tag, err := pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1`, id)
|
||||
dbQueries++
|
||||
if err != nil {
|
||||
report.DeleteFailures++
|
||||
continue
|
||||
}
|
||||
report.DeletedMetadata += int(tag.RowsAffected())
|
||||
}
|
||||
} else {
|
||||
for _, item := range missing {
|
||||
if item.Referenced {
|
||||
report.RetainedMissing++
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
report.S3Requests = transport.count.Load()
|
||||
report.DBQueries = dbQueries
|
||||
report.DurationMS = time.Since(started).Milliseconds()
|
||||
if report.DurationMS > 0 {
|
||||
report.ObjectsPerSecond = float64(report.ScannedObjects) / (float64(report.DurationMS) / 1000)
|
||||
}
|
||||
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
@@ -110,7 +163,7 @@ func run(dryRun bool, grace time.Duration, prefix string, limit int) error {
|
||||
return err
|
||||
}
|
||||
if report.DeleteFailures > 0 {
|
||||
return fmt.Errorf("%d orphan deletions failed", report.DeleteFailures)
|
||||
return fmt.Errorf("%d repair operations failed", report.DeleteFailures)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -127,6 +180,26 @@ func loadMetadata(ctx context.Context, pool *pgxpool.Pool) ([]reconcile.Metadata
|
||||
})
|
||||
}
|
||||
|
||||
func loadReferences(ctx context.Context, pool *pgxpool.Pool) (map[uuid.UUID]bool, error) {
|
||||
rows, err := pool.Query(ctx, `SELECT DISTINCT attachment_id FROM message_attachments`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (uuid.UUID, error) {
|
||||
var id uuid.UUID
|
||||
err := row.Scan(&id)
|
||||
return id, err
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
referenced := make(map[uuid.UUID]bool, len(ids))
|
||||
for _, id := range ids {
|
||||
referenced[id] = true
|
||||
}
|
||||
return referenced, nil
|
||||
}
|
||||
|
||||
func toObjects(scanned []s3.ObjectInfo) []reconcile.Object {
|
||||
objects := make([]reconcile.Object, len(scanned))
|
||||
for i, info := range scanned {
|
||||
|
||||
+47
-1
@@ -6,6 +6,8 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"messenger/internal/config"
|
||||
@@ -79,6 +81,22 @@ func run(command string, args []string) error {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("removed %s\n", args[0])
|
||||
case "seed":
|
||||
if len(args) != 3 {
|
||||
usage()
|
||||
}
|
||||
count, err := strconv.Atoi(args[1])
|
||||
if err != nil || count <= 0 {
|
||||
return fmt.Errorf("count must be a positive integer")
|
||||
}
|
||||
size, err := strconv.Atoi(args[2])
|
||||
if err != nil || size <= 0 {
|
||||
return fmt.Errorf("size must be a positive integer")
|
||||
}
|
||||
if err := seed(ctx, store, args[0], count, size); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("seeded %d objects under %s\n", count, args[0])
|
||||
default:
|
||||
usage()
|
||||
}
|
||||
@@ -105,8 +123,36 @@ func newStore() (*s3.Store, error) {
|
||||
accessKeyID, secretAccessKey, getenv("S3_BUCKET", "messenger-attachments"))
|
||||
}
|
||||
|
||||
func seed(ctx context.Context, store *s3.Store, prefix string, count, size int) error {
|
||||
content := make([]byte, size)
|
||||
workers := 8
|
||||
work := make(chan int)
|
||||
errors := make(chan error, workers)
|
||||
var group sync.WaitGroup
|
||||
for i := 0; i < workers; i++ {
|
||||
group.Add(1)
|
||||
go func() {
|
||||
defer group.Done()
|
||||
for index := range work {
|
||||
key := fmt.Sprintf("%s/%06d", prefix, index)
|
||||
if err := store.Put(ctx, key, bytes.NewReader(content), int64(size), "application/octet-stream"); err != nil {
|
||||
errors <- err
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
for index := 0; index < count; index++ {
|
||||
work <- index
|
||||
}
|
||||
close(work)
|
||||
group.Wait()
|
||||
close(errors)
|
||||
return <-errors
|
||||
}
|
||||
|
||||
func usage() {
|
||||
fmt.Fprintln(os.Stderr, "usage: s3tool put <key> <file> | delete <key> | list [prefix] | buckets | remove-bucket <name>")
|
||||
fmt.Fprintln(os.Stderr, "usage: s3tool put <key> <file> | delete <key> | list [prefix] | buckets | remove-bucket <name> | seed <prefix> <count> <bytes>")
|
||||
os.Exit(2)
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
2026-11-20T04:20:26+03:00
|
||||
2026-11-20T04:38:30+03:00
|
||||
make check
|
||||
|
||||
python3 scripts/check_contract.py
|
||||
|
||||
+29
-35
@@ -1,4 +1,4 @@
|
||||
2026-11-20T04:14:17+03:00
|
||||
2026-11-20T04:41:27+03:00
|
||||
failure between S3 write and database commit: an object must not stay unaccounted
|
||||
|
||||
$ scripts/reconcile.sh # baseline
|
||||
@@ -6,21 +6,19 @@ $ scripts/reconcile.sh # baseline
|
||||
"prefix": "chats/",
|
||||
"grace_seconds": 3600,
|
||||
"dry_run": true,
|
||||
"scanned_objects": 15,
|
||||
"scanned_rows": 16,
|
||||
"scanned_objects": 17,
|
||||
"scanned_rows": 17,
|
||||
"truncated": false,
|
||||
"orphans": [],
|
||||
"missing": [
|
||||
{
|
||||
"id": "97121cfe-6b29-4227-867f-3188e72a725c",
|
||||
"object_key": "chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c",
|
||||
"chat_id": "9452049e-b86c-4d9a-8060-d98f6966b956",
|
||||
"created_at": "2026-11-20T01:11:14Z"
|
||||
}
|
||||
],
|
||||
"missing": [],
|
||||
"deleted_objects": 0,
|
||||
"deleted_metadata": 0,
|
||||
"retained_missing": 0,
|
||||
"delete_failures": 0,
|
||||
"duration_ms": 8
|
||||
"s3_requests": 1,
|
||||
"db_queries": 2,
|
||||
"duration_ms": 7,
|
||||
"objects_per_second": 2428.5714285714284
|
||||
}
|
||||
|
||||
$ podman exec -i postgres psql < scripts/sql/inject-db-failure.sql
|
||||
@@ -37,21 +35,19 @@ $ scripts/reconcile.sh # same object count: compensating DELETE removed the ob
|
||||
"prefix": "chats/",
|
||||
"grace_seconds": 3600,
|
||||
"dry_run": true,
|
||||
"scanned_objects": 15,
|
||||
"scanned_rows": 16,
|
||||
"scanned_objects": 17,
|
||||
"scanned_rows": 17,
|
||||
"truncated": false,
|
||||
"orphans": [],
|
||||
"missing": [
|
||||
{
|
||||
"id": "97121cfe-6b29-4227-867f-3188e72a725c",
|
||||
"object_key": "chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c",
|
||||
"chat_id": "9452049e-b86c-4d9a-8060-d98f6966b956",
|
||||
"created_at": "2026-11-20T01:11:14Z"
|
||||
}
|
||||
],
|
||||
"missing": [],
|
||||
"deleted_objects": 0,
|
||||
"deleted_metadata": 0,
|
||||
"retained_missing": 0,
|
||||
"delete_failures": 0,
|
||||
"duration_ms": 7
|
||||
"s3_requests": 1,
|
||||
"db_queries": 2,
|
||||
"duration_ms": 7,
|
||||
"objects_per_second": 2428.5714285714284
|
||||
}
|
||||
|
||||
$ podman exec -i postgres psql < scripts/sql/reset-db-failure.sql
|
||||
@@ -60,26 +56,24 @@ DROP FUNCTION
|
||||
|
||||
$ scripts/upload-once.py --expect 201 # service healthy again
|
||||
status=201
|
||||
attachment=a6844b91-5b7b-4694-93fc-87d808b687fb size=653
|
||||
attachment=7dd89013-32f1-4732-a2fd-529fe2081bb8 size=653
|
||||
|
||||
$ scripts/reconcile.sh # one more accounted object
|
||||
{
|
||||
"prefix": "chats/",
|
||||
"grace_seconds": 3600,
|
||||
"dry_run": true,
|
||||
"scanned_objects": 16,
|
||||
"scanned_rows": 17,
|
||||
"scanned_objects": 18,
|
||||
"scanned_rows": 18,
|
||||
"truncated": false,
|
||||
"orphans": [],
|
||||
"missing": [
|
||||
{
|
||||
"id": "97121cfe-6b29-4227-867f-3188e72a725c",
|
||||
"object_key": "chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c",
|
||||
"chat_id": "9452049e-b86c-4d9a-8060-d98f6966b956",
|
||||
"created_at": "2026-11-20T01:11:14Z"
|
||||
}
|
||||
],
|
||||
"missing": [],
|
||||
"deleted_objects": 0,
|
||||
"deleted_metadata": 0,
|
||||
"retained_missing": 0,
|
||||
"delete_failures": 0,
|
||||
"duration_ms": 7
|
||||
"s3_requests": 1,
|
||||
"db_queries": 2,
|
||||
"duration_ms": 7,
|
||||
"objects_per_second": 2571.4285714285716
|
||||
}
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
2026-11-20T04:53:36+03:00
|
||||
make test-contract
|
||||
|
||||
BASE_URL="http://localhost:8080" python3 scripts/contract-audit.py
|
||||
ok: explicit attachment_ids=null is 400
|
||||
ok: empty variant is 400
|
||||
ok: whitespace-only filename is 400
|
||||
ok: Content-Type parameters are accepted
|
||||
ok: 41,000,000 pixels is 413
|
||||
ok: text length bounds
|
||||
ok: attachment count bounds
|
||||
ok: duplicate attachment_ids is 400
|
||||
ok: idempotent replay with the same set
|
||||
ok: absent and empty attachment_ids agree
|
||||
ok: attachment from another chat is 404
|
||||
ok: unknown attachment id is 404
|
||||
ok: outsider access is 404
|
||||
ok: upload to unknown chat is 404
|
||||
ok: media errors are 415
|
||||
ok: oversize is 413
|
||||
ok: filename errors are 400
|
||||
ok: chat validation errors are 400
|
||||
ok: pagination errors are 400
|
||||
ok: idempotency errors
|
||||
ok: null request fields are 400
|
||||
ok: more than 100 members is 400
|
||||
ok: user field validation
|
||||
ok: missing Content-Type is 415
|
||||
ok: variant is case sensitive
|
||||
ok: filename control characters are 400
|
||||
ok: chat title bounds
|
||||
ok: cursor is bound to the endpoint
|
||||
ok: 204 responses carry headers and no body
|
||||
ok: download Content-Length matches the bytes
|
||||
PASS: 30 checks
|
||||
@@ -1,4 +1,4 @@
|
||||
2026-11-20T04:17:09+03:00
|
||||
2026-11-20T04:43:58+03:00
|
||||
separate restricted S3 credentials for the API and idempotent bucket bootstrap
|
||||
|
||||
$ podman-compose run --rm s3-init (first run)
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
2026-11-20T04:18:05+03:00
|
||||
2026-11-20T04:56:32+03:00
|
||||
|
||||
users | chats | messages | attachments | links | db_size
|
||||
-------+-------+----------+-------------+-------+---------
|
||||
58 | 17 | 303 | 30 | 2 | 8534 kB
|
||||
125 | 43 | 634 | 101 | 56 | 8958 kB
|
||||
(1 row)
|
||||
|
||||
|
||||
+4
-4
@@ -1,10 +1,10 @@
|
||||
2026-11-20T04:18:05+03:00
|
||||
2026-11-20T04:56:17+03:00
|
||||
|
||||
Linux archlinux 7.3.5-arch1-1 #1 SMP PREEMPT_DYNAMIC Tue, 17 Nov 2026 22:28:33 +0000 x86_64 GNU/Linux
|
||||
12
|
||||
total used free shared buff/cache available
|
||||
Mem: 15Gi 6.2Gi 2.7Gi 476Mi 7.1Gi 9.2Gi
|
||||
Swap: 8.0Gi 57Mi 7.9Gi
|
||||
Mem: 15Gi 6.5Gi 3.6Gi 440Mi 5.8Gi 8.8Gi
|
||||
Swap: 8.0Gi 187Mi 7.8Gi
|
||||
podman version 6.2.0
|
||||
podman version 6.2.0
|
||||
podman-compose version 1.6.0
|
||||
@@ -16,4 +16,4 @@ docker.io/library/postgres:18.6-alpine3.24 57b23a0326ee
|
||||
docker.io/library/redis:8.10.3-alpine3.23 9722accf3e8e
|
||||
docker.io/migrate/migrate:v4.20.1 719a6f5d0ade
|
||||
docker.io/rustfs/rustfs:1.0.6 7cd1c573d76d
|
||||
localhost/messenger-api:lab4 e2582dea4a72
|
||||
localhost/messenger-api:lab4 4c57d3f41dfb
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
2026-11-20T04:14:49+03:00
|
||||
2026-11-20T04:55:04+03:00
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 101, "failed_attempts": 0, "first_recovery_seconds": 3.304}
|
||||
{"observed_instances_before": 2, "successful_writes": 88, "failed_attempts": 0, "first_recovery_seconds": 3.256}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
2026-11-20T04:15:25+03:00
|
||||
2026-11-20T04:55:38+03:00
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 86, "failed_attempts": 0, "first_recovery_seconds": 3.228}
|
||||
{"observed_instances_before": 2, "successful_writes": 99, "failed_attempts": 0, "first_recovery_seconds": 3.236}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
2026-11-20T04:20:36+03:00
|
||||
2026-11-20T04:53:46+03:00
|
||||
make test-idempotency
|
||||
|
||||
BASE_URL="http://localhost:8080" python3 scripts/idempotency-race.py
|
||||
@@ -7,5 +7,5 @@ successful responses (201): 20
|
||||
failed requests: 0 []
|
||||
distinct message ids in responses: 1
|
||||
messages stored in history: 1
|
||||
instances that served requests: ['8535d7918d12', 'ad1700491457']
|
||||
instances that served requests: ['72c9d7b6cc57', 'e6d2e0d15c36']
|
||||
OK: exactly one message created
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
2026-11-20T04:21:39+03:00
|
||||
2026-11-20T04:56:16+03:00
|
||||
API replicas and single entrypoint
|
||||
|
||||
messenger-lab4_api_1 Up 26 seconds (healthy)
|
||||
messenger-lab4_api_2 Up 26 seconds (healthy)
|
||||
messenger-lab4_api_1 Up 53 seconds (healthy)
|
||||
messenger-lab4_api_2 Up 21 seconds (healthy)
|
||||
messenger-lab4_lb_1 Up About a minute
|
||||
messenger-lab4_postgres_1 Up About a minute (healthy)
|
||||
messenger-lab4_redis_1 Up About a minute (healthy)
|
||||
@@ -19,5 +19,5 @@ messenger-lab4_s3_1 9000-9001/tcp
|
||||
|
||||
X-Instance-Id distribution over 20 requests through LB
|
||||
|
||||
10 x-instance-id: 33839df0eea0
|
||||
10 x-instance-id: 4e3d4c4f1194
|
||||
10 x-instance-id: a089fe9cc382
|
||||
10 x-instance-id: f309c2ef4e46
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
2026-11-20T04:17:58+03:00
|
||||
2026-11-20T04:56:16+03:00
|
||||
per-container limits, networks, secrets
|
||||
|
||||
messenger-lab4_api_1 nano_cpus=1000000000 memory=536870912 networks=messenger-lab4_backend messenger-lab4_edge
|
||||
|
||||
+13
-13
@@ -1,5 +1,5 @@
|
||||
2026-11-20T04:21:28+03:00
|
||||
10 concurrent uploads of a 8000x5000 PNG (40,000,000 pixels) per replica; API memory limit 512 MiB; MAX_CONCURRENT_UPLOADS=8, MAX_CONCURRENT_DECODES=1
|
||||
2026-11-20T04:54:15+03:00
|
||||
10 concurrent uploads of a 8000x5000 PNG (40,000,000 pixels) per replica; API limit 512 MiB; MAX_CONCURRENT_UPLOADS=8, MAX_CONCURRENT_DECODES=1
|
||||
|
||||
BASE_URL="http://localhost:8080" python3 scripts/upload-load.py --concurrency 10 --output evidence/upload-memory.json
|
||||
{
|
||||
@@ -7,30 +7,30 @@ BASE_URL="http://localhost:8080" python3 scripts/upload-load.py --concurrency 10
|
||||
"uploads": 10,
|
||||
"fixture": "large.png",
|
||||
"fixture_bytes": 126609,
|
||||
"elapsed_seconds": 1.235,
|
||||
"elapsed_seconds": 1.009,
|
||||
"successful": 10,
|
||||
"failed": 0,
|
||||
"failures": [],
|
||||
"latency_ms": {
|
||||
"p50": 663.7,
|
||||
"p95": 1224.7,
|
||||
"p99": 1224.7
|
||||
"p50": 660.9,
|
||||
"p95": 999.3,
|
||||
"p99": 999.3
|
||||
},
|
||||
"api_containers": [
|
||||
"messenger-lab4_api_1",
|
||||
"messenger-lab4_api_2"
|
||||
],
|
||||
"peak_bytes_before": {
|
||||
"messenger-lab4_api_1": 5050368,
|
||||
"messenger-lab4_api_2": 4820992
|
||||
"messenger-lab4_api_1": 5951488,
|
||||
"messenger-lab4_api_2": 4993024
|
||||
},
|
||||
"peak_bytes_after": {
|
||||
"messenger-lab4_api_1": 368119808,
|
||||
"messenger-lab4_api_2": 368185344
|
||||
"messenger-lab4_api_1": 348913664,
|
||||
"messenger-lab4_api_2": 348721152
|
||||
},
|
||||
"peak_delta_bytes": {
|
||||
"messenger-lab4_api_1": 363069440,
|
||||
"messenger-lab4_api_2": 363364352
|
||||
"messenger-lab4_api_1": 342962176,
|
||||
"messenger-lab4_api_2": 343728128
|
||||
},
|
||||
"peak_max_mb": 351.13
|
||||
"peak_max_mb": 332.75
|
||||
}
|
||||
|
||||
@@ -1,13 +1,9 @@
|
||||
2026-11-20T04:18:05+03:00
|
||||
2026-11-20T04:56:17+03:00
|
||||
migrate container logs and schema version
|
||||
|
||||
1/u init (17.596326ms)
|
||||
2/u message_idempotency (23.558514ms)
|
||||
3/u attachments (35.669736ms)
|
||||
no change
|
||||
no change
|
||||
no change
|
||||
no change
|
||||
1/u init (18.672238ms)
|
||||
2/u message_idempotency (27.931065ms)
|
||||
3/u attachments (37.481418ms)
|
||||
no change
|
||||
no change
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
2026-11-20T04:20:27+03:00
|
||||
2026-11-20T04:53:36+03:00
|
||||
make test-attachments
|
||||
|
||||
BASE_URL="http://localhost:8080" python3 scripts/attachments-check.py
|
||||
|
||||
+13
-13
@@ -1,4 +1,4 @@
|
||||
2026-11-20T04:19:38+03:00
|
||||
2026-11-20T04:54:22+03:00
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
|
||||
@@ -7,12 +7,12 @@ PASS: persistence preserved confirmed data and existing session. Hook scope need
|
||||
|
||||
--- hook scope: containers and volumes before/after scripts/restart.sh ---
|
||||
|
||||
messenger-lab4_api_1 8535d7918d12 Up 1 second (starting)
|
||||
messenger-lab4_api_2 ad1700491457 Up 1 second (starting)
|
||||
messenger-lab4_lb_1 0bed57f994fa Up 1 second
|
||||
messenger-lab4_postgres_1 53c091ca7f0a Up 1 second (starting)
|
||||
messenger-lab4_redis_1 e0cfe44337f5 Up 1 second (healthy)
|
||||
messenger-lab4_s3_1 8548fb0864af Up 1 second (healthy)
|
||||
messenger-lab4_api_1 a089fe9cc382 Up 5 seconds (healthy)
|
||||
messenger-lab4_api_2 f309c2ef4e46 Up 5 seconds (healthy)
|
||||
messenger-lab4_lb_1 efc7da123526 Up 5 seconds
|
||||
messenger-lab4_postgres_1 c64e954cf92a Up 5 seconds (healthy)
|
||||
messenger-lab4_redis_1 90ac9f642b09 Up 5 seconds (healthy)
|
||||
messenger-lab4_s3_1 6ae98c5aef4f Up 5 seconds (healthy)
|
||||
messenger-lab4_pgdata
|
||||
messenger-lab4_redisdata
|
||||
messenger-lab4_s3data
|
||||
@@ -20,12 +20,12 @@ messenger-lab4_s3data
|
||||
$ scripts/restart.sh
|
||||
|
||||
|
||||
messenger-lab4_api_1 8535d7918d12 Up 15 seconds (healthy)
|
||||
messenger-lab4_api_2 ad1700491457 Up 16 seconds (healthy)
|
||||
messenger-lab4_lb_1 0bed57f994fa Up 16 seconds
|
||||
messenger-lab4_postgres_1 53c091ca7f0a Up 15 seconds (healthy)
|
||||
messenger-lab4_redis_1 e0cfe44337f5 Up 15 seconds (healthy)
|
||||
messenger-lab4_s3_1 8548fb0864af Up 16 seconds (healthy)
|
||||
messenger-lab4_api_1 a089fe9cc382 Up 15 seconds (healthy)
|
||||
messenger-lab4_api_2 f309c2ef4e46 Up 15 seconds (healthy)
|
||||
messenger-lab4_lb_1 efc7da123526 Up 15 seconds
|
||||
messenger-lab4_postgres_1 c64e954cf92a Up 15 seconds (healthy)
|
||||
messenger-lab4_redis_1 90ac9f642b09 Up 15 seconds (healthy)
|
||||
messenger-lab4_s3_1 6ae98c5aef4f Up 15 seconds (healthy)
|
||||
messenger-lab4_pgdata
|
||||
messenger-lab4_redisdata
|
||||
messenger-lab4_s3data
|
||||
|
||||
@@ -1,20 +1,22 @@
|
||||
2026-11-20T04:15:49+03:00
|
||||
2026-11-20T04:44:19+03:00
|
||||
bucket privacy and disabled console (checks outside the application API)
|
||||
|
||||
$ wget http://s3:9000/messenger-attachments/chats/.../... (anonymous, from backend network)
|
||||
Connecting to s3:9000 (10.89.5.36:9000)
|
||||
$ anonymous GET of an existing object from the backend network
|
||||
Connecting to s3:9000 (10.89.5.22:9000)
|
||||
HTTP/1.1 403 Forbidden
|
||||
wget: server returned error: HTTP/1.1 403 Forbidden
|
||||
|
||||
$ anonymous bucket listing
|
||||
Connecting to s3:9000 (10.89.5.36:9000)
|
||||
Connecting to s3:9000 (10.89.5.22:9000)
|
||||
HTTP/1.1 403 Forbidden
|
||||
wget: server returned error: HTTP/1.1 403 Forbidden
|
||||
|
||||
$ console port check inside s3 container
|
||||
$ console port inside the s3 container
|
||||
console_http=000
|
||||
console-closed
|
||||
|
||||
$ published ports
|
||||
messenger-lab4_api_1 8080/tcp
|
||||
messenger-lab4_api_2 8080/tcp
|
||||
messenger-lab4_lb_1 0.0.0.0:8080->8080/tcp
|
||||
messenger-lab4_s3_1 9000-9001/tcp
|
||||
|
||||
+91
-31
@@ -1,4 +1,4 @@
|
||||
2026-11-20T04:13:27+03:00
|
||||
2026-11-20T04:41:03+03:00
|
||||
reconciler drill: baseline, grace period, dry-run, apply, repeat
|
||||
|
||||
$ scripts/reconcile.sh # dry-run, grace 1h, clean baseline
|
||||
@@ -6,54 +6,69 @@ $ scripts/reconcile.sh # dry-run, grace 1h, clean baseline
|
||||
"prefix": "chats/",
|
||||
"grace_seconds": 3600,
|
||||
"dry_run": true,
|
||||
"scanned_objects": 15,
|
||||
"scanned_rows": 15,
|
||||
"scanned_objects": 18,
|
||||
"scanned_rows": 18,
|
||||
"truncated": false,
|
||||
"orphans": [],
|
||||
"missing": [],
|
||||
"deleted_objects": 0,
|
||||
"deleted_metadata": 0,
|
||||
"retained_missing": 0,
|
||||
"delete_failures": 0,
|
||||
"duration_ms": 7
|
||||
"s3_requests": 1,
|
||||
"db_queries": 2,
|
||||
"duration_ms": 7,
|
||||
"objects_per_second": 2571.4285714285716
|
||||
}
|
||||
|
||||
$ scripts/s3tool.sh put chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000 - < scripts/fixtures/sample.jpg
|
||||
put chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000 (653 bytes)
|
||||
|
||||
$ scripts/reconcile.sh # fresh orphan stays: grace 1h
|
||||
$ scripts/reconcile.sh # fresh orphan skipped by grace
|
||||
{
|
||||
"prefix": "chats/",
|
||||
"grace_seconds": 3600,
|
||||
"dry_run": true,
|
||||
"scanned_objects": 16,
|
||||
"scanned_rows": 15,
|
||||
"scanned_objects": 19,
|
||||
"scanned_rows": 18,
|
||||
"truncated": false,
|
||||
"orphans": [],
|
||||
"missing": [],
|
||||
"deleted_objects": 0,
|
||||
"deleted_metadata": 0,
|
||||
"retained_missing": 0,
|
||||
"delete_failures": 0,
|
||||
"duration_ms": 7
|
||||
"s3_requests": 1,
|
||||
"db_queries": 2,
|
||||
"duration_ms": 8,
|
||||
"objects_per_second": 2375
|
||||
}
|
||||
|
||||
$ scripts/reconcile.sh -grace 0s # orphan detected, dry-run keeps it
|
||||
$ scripts/reconcile.sh -grace 0s # detected, dry-run keeps it
|
||||
{
|
||||
"prefix": "chats/",
|
||||
"grace_seconds": 0,
|
||||
"dry_run": true,
|
||||
"scanned_objects": 16,
|
||||
"scanned_rows": 15,
|
||||
"scanned_objects": 19,
|
||||
"scanned_rows": 18,
|
||||
"truncated": false,
|
||||
"orphans": [
|
||||
{
|
||||
"key": "chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000",
|
||||
"size_bytes": 653,
|
||||
"last_modified": "2026-11-20T01:13:27Z",
|
||||
"last_modified": "2026-11-20T01:41:03Z",
|
||||
"age_seconds": 0
|
||||
}
|
||||
],
|
||||
"missing": [],
|
||||
"deleted_objects": 0,
|
||||
"deleted_metadata": 0,
|
||||
"retained_missing": 0,
|
||||
"delete_failures": 0,
|
||||
"duration_ms": 6
|
||||
"s3_requests": 1,
|
||||
"db_queries": 2,
|
||||
"duration_ms": 7,
|
||||
"objects_per_second": 2714.285714285714
|
||||
}
|
||||
|
||||
$ scripts/reconcile.sh -grace 0s -dry-run=false # apply
|
||||
@@ -61,21 +76,26 @@ $ scripts/reconcile.sh -grace 0s -dry-run=false # apply
|
||||
"prefix": "chats/",
|
||||
"grace_seconds": 0,
|
||||
"dry_run": false,
|
||||
"scanned_objects": 16,
|
||||
"scanned_rows": 15,
|
||||
"scanned_objects": 19,
|
||||
"scanned_rows": 18,
|
||||
"truncated": false,
|
||||
"orphans": [
|
||||
{
|
||||
"key": "chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000",
|
||||
"size_bytes": 653,
|
||||
"last_modified": "2026-11-20T01:13:27Z",
|
||||
"age_seconds": 3
|
||||
"last_modified": "2026-11-20T01:41:03Z",
|
||||
"age_seconds": 4
|
||||
}
|
||||
],
|
||||
"missing": [],
|
||||
"deleted_objects": 1,
|
||||
"deleted_metadata": 0,
|
||||
"retained_missing": 0,
|
||||
"delete_failures": 0,
|
||||
"duration_ms": 12
|
||||
"s3_requests": 2,
|
||||
"db_queries": 2,
|
||||
"duration_ms": 15,
|
||||
"objects_per_second": 1266.6666666666667
|
||||
}
|
||||
|
||||
$ scripts/reconcile.sh -grace 0s -dry-run=false # repeat: nothing left
|
||||
@@ -83,41 +103,81 @@ $ scripts/reconcile.sh -grace 0s -dry-run=false # repeat: nothing left
|
||||
"prefix": "chats/",
|
||||
"grace_seconds": 0,
|
||||
"dry_run": false,
|
||||
"scanned_objects": 15,
|
||||
"scanned_rows": 15,
|
||||
"scanned_objects": 18,
|
||||
"scanned_rows": 18,
|
||||
"truncated": false,
|
||||
"orphans": [],
|
||||
"missing": [],
|
||||
"deleted_objects": 0,
|
||||
"deleted_metadata": 0,
|
||||
"retained_missing": 0,
|
||||
"delete_failures": 0,
|
||||
"duration_ms": 7
|
||||
"s3_requests": 1,
|
||||
"db_queries": 2,
|
||||
"duration_ms": 8,
|
||||
"objects_per_second": 2250
|
||||
}
|
||||
|
||||
missing drill: metadata row without object
|
||||
|
||||
$ scripts/s3tool.sh delete chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c
|
||||
deleted chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c
|
||||
$ scripts/s3tool.sh delete chats/<chat>/<attachment> of an existing attachment
|
||||
$ scripts/s3tool.sh delete chats/bd1c9e17-0366-44b4-bd98-0a479a2a637c/1e9b9e12-eab2-4846-8952-e739a4257b37
|
||||
deleted chats/bd1c9e17-0366-44b4-bd98-0a479a2a637c/1e9b9e12-eab2-4846-8952-e739a4257b37
|
||||
|
||||
$ scripts/reconcile.sh -grace 0s # reports missing, keeps metadata
|
||||
$ scripts/reconcile.sh -grace 0s # missing reported, metadata kept in dry-run
|
||||
{
|
||||
"prefix": "chats/",
|
||||
"grace_seconds": 0,
|
||||
"dry_run": true,
|
||||
"scanned_objects": 14,
|
||||
"scanned_rows": 15,
|
||||
"scanned_objects": 17,
|
||||
"scanned_rows": 18,
|
||||
"truncated": false,
|
||||
"orphans": [],
|
||||
"missing": [
|
||||
{
|
||||
"id": "97121cfe-6b29-4227-867f-3188e72a725c",
|
||||
"object_key": "chats/9452049e-b86c-4d9a-8060-d98f6966b956/97121cfe-6b29-4227-867f-3188e72a725c",
|
||||
"chat_id": "9452049e-b86c-4d9a-8060-d98f6966b956",
|
||||
"created_at": "2026-11-20T01:11:14Z"
|
||||
"id": "1e9b9e12-eab2-4846-8952-e739a4257b37",
|
||||
"object_key": "chats/bd1c9e17-0366-44b4-bd98-0a479a2a637c/1e9b9e12-eab2-4846-8952-e739a4257b37",
|
||||
"chat_id": "bd1c9e17-0366-44b4-bd98-0a479a2a637c",
|
||||
"created_at": "2026-11-20T01:40:28Z",
|
||||
"referenced_by_message": false
|
||||
}
|
||||
],
|
||||
"deleted_objects": 0,
|
||||
"deleted_metadata": 0,
|
||||
"retained_missing": 0,
|
||||
"delete_failures": 0,
|
||||
"duration_ms": 6
|
||||
"s3_requests": 1,
|
||||
"db_queries": 2,
|
||||
"duration_ms": 9,
|
||||
"objects_per_second": 1888.8888888888891
|
||||
}
|
||||
|
||||
metadata rows still present: 1
|
||||
|
||||
$ scripts/reconcile.sh -grace 0s -dry-run=false # safe repair: unreferenced dead row removed
|
||||
{
|
||||
"prefix": "chats/",
|
||||
"grace_seconds": 0,
|
||||
"dry_run": false,
|
||||
"scanned_objects": 17,
|
||||
"scanned_rows": 18,
|
||||
"truncated": false,
|
||||
"orphans": [],
|
||||
"missing": [
|
||||
{
|
||||
"id": "1e9b9e12-eab2-4846-8952-e739a4257b37",
|
||||
"object_key": "chats/bd1c9e17-0366-44b4-bd98-0a479a2a637c/1e9b9e12-eab2-4846-8952-e739a4257b37",
|
||||
"chat_id": "bd1c9e17-0366-44b4-bd98-0a479a2a637c",
|
||||
"created_at": "2026-11-20T01:40:28Z",
|
||||
"referenced_by_message": false
|
||||
}
|
||||
],
|
||||
"deleted_objects": 0,
|
||||
"deleted_metadata": 1,
|
||||
"retained_missing": 0,
|
||||
"delete_failures": 0,
|
||||
"s3_requests": 1,
|
||||
"db_queries": 3,
|
||||
"duration_ms": 8,
|
||||
"objects_per_second": 2125
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
2026-11-20T04:57:42+03:00
|
||||
S3 unavailable: readiness and client-visible behaviour
|
||||
|
||||
$ GET /health/ready on a replica
|
||||
HTTP/1.1 503 Service Unavailable
|
||||
wget: server returned error: HTTP/1.1 503 Service Unavailable
|
||||
|
||||
$ GET /health/live on a replica (process is alive)
|
||||
HTTP/1.1 200 OK
|
||||
Content-Type: application/json; charset=utf-8
|
||||
X-Instance-Id: a089fe9cc382
|
||||
|
||||
$ client request through the load balancer
|
||||
status=503
|
||||
|
||||
$ readiness after S3 recovery
|
||||
status=200
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
2026-11-20T04:20:26+03:00
|
||||
2026-11-20T04:53:35+03:00
|
||||
make test
|
||||
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
2026-11-20T04:18:15+03:00
|
||||
2026-11-20T04:56:31+03:00
|
||||
object bytes live in the RustFS volume, metadata in PostgreSQL
|
||||
|
||||
id | object_key | size_bytes | sha256
|
||||
--------------------------------------+---------------------------------------------------------------------------------+------------+------------------------------------------------------------------
|
||||
3dbc5415-9791-4983-a253-bc71fdc79a24 | chats/3ec617cd-fddf-4ae1-ad9d-37a4bb319b76/3dbc5415-9791-4983-a253-bc71fdc79a24 | 126609 | 518fd54cb19f8ceaca99139652fb96ed3a5d502392450846c832924b1722d2bf
|
||||
82d5d4f3-e97a-4aea-9501-69ef2b74e138 | chats/e8618579-667a-4423-aa62-5402596e57d8/82d5d4f3-e97a-4aea-9501-69ef2b74e138 | 111817 | 99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75
|
||||
(1 row)
|
||||
|
||||
|
||||
S3 volume listing for that object
|
||||
|
||||
total 128
|
||||
-rw-r--r-- 1 rustfs rustfs 127190 Nov 20 01:17 xl.meta
|
||||
total 112
|
||||
-rw-r--r-- 1 rustfs rustfs 112398 Nov 20 01:55 xl.meta
|
||||
|
||||
attachments table columns (no binary content)
|
||||
|
||||
|
||||
+18
-16
@@ -1,4 +1,4 @@
|
||||
2026-11-20T04:20:28+03:00
|
||||
2026-11-20T04:53:38+03:00
|
||||
make test-go
|
||||
|
||||
? messenger/cmd/reconcile [no test files]
|
||||
@@ -10,50 +10,52 @@ make test-go
|
||||
=== RUN TestSessionExpiry
|
||||
--- PASS: TestSessionExpiry (3.09s)
|
||||
=== RUN TestLogout
|
||||
--- PASS: TestLogout (0.11s)
|
||||
--- PASS: TestLogout (0.13s)
|
||||
=== RUN TestWrongPassword
|
||||
--- PASS: TestWrongPassword (0.10s)
|
||||
=== RUN TestRedisUnavailable
|
||||
redis: 2026/11/20 04:20:32 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/20 04:20:32 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/20 04:20:33 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/20 04:20:33 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/20 04:20:34 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/20 04:20:34 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
--- PASS: TestRedisUnavailable (2.83s)
|
||||
redis: 2026/11/20 04:53:42 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/20 04:53:43 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/20 04:53:43 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/20 04:53:44 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/20 04:53:44 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/20 04:53:44 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
--- PASS: TestRedisUnavailable (2.95s)
|
||||
=== RUN TestDatabaseUnavailable
|
||||
--- PASS: TestDatabaseUnavailable (0.17s)
|
||||
=== RUN TestInvalidJSON
|
||||
--- PASS: TestInvalidJSON (0.01s)
|
||||
=== RUN TestForeignChat
|
||||
--- PASS: TestForeignChat (0.16s)
|
||||
--- PASS: TestForeignChat (0.18s)
|
||||
=== RUN TestEmptyMessage
|
||||
redis: 2026/11/20 04:20:35 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/20 04:53:45 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
--- PASS: TestEmptyMessage (0.08s)
|
||||
=== RUN TestDuplicateUsername
|
||||
--- PASS: TestDuplicateUsername (0.10s)
|
||||
=== RUN TestPaginationMultiplePages
|
||||
--- PASS: TestPaginationMultiplePages (0.08s)
|
||||
--- PASS: TestPaginationMultiplePages (0.09s)
|
||||
=== RUN TestConcurrentMessages
|
||||
--- PASS: TestConcurrentMessages (0.59s)
|
||||
--- PASS: TestConcurrentMessages (0.57s)
|
||||
PASS
|
||||
ok messenger/internal/handler 7.325s
|
||||
ok messenger/internal/handler 7.474s
|
||||
=== RUN TestPlanDetectsOrphansAndMissing
|
||||
--- PASS: TestPlanDetectsOrphansAndMissing (0.00s)
|
||||
=== RUN TestPlanHonoursGraceAndPrefix
|
||||
--- PASS: TestPlanHonoursGraceAndPrefix (0.00s)
|
||||
=== RUN TestPlanNeverDeletesReferencedObjects
|
||||
--- PASS: TestPlanNeverDeletesReferencedObjects (0.00s)
|
||||
=== RUN TestPlanOnlyReportsRowsInsidePrefix
|
||||
--- PASS: TestPlanOnlyReportsRowsInsidePrefix (0.00s)
|
||||
=== RUN TestPlanOutputIsSorted
|
||||
--- PASS: TestPlanOutputIsSorted (0.00s)
|
||||
PASS
|
||||
ok messenger/internal/reconcile 0.002s
|
||||
=== RUN TestRegisterUsernameRace
|
||||
--- PASS: TestRegisterUsernameRace (0.05s)
|
||||
--- PASS: TestRegisterUsernameRace (0.02s)
|
||||
=== RUN TestCreateChatRollback
|
||||
--- PASS: TestCreateChatRollback (0.02s)
|
||||
PASS
|
||||
ok messenger/internal/repository/postgres 0.071s
|
||||
ok messenger/internal/repository/postgres 0.040s
|
||||
? messenger/internal/repository/redis [no test files]
|
||||
? messenger/internal/repository/s3 [no test files]
|
||||
? messenger/internal/router [no test files]
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
2026-11-20T04:42:15+03:00
|
||||
traversal cost of the reconciler (synthetic objects under chats/bench/, 1 KiB each); scan uses LIST pages, repair deletes one by one
|
||||
|
||||
{
|
||||
"prefix": "chats/bench",
|
||||
"measurements": [
|
||||
{
|
||||
"objects": 100,
|
||||
"scan_ms": 9,
|
||||
"scan_s3_requests": 1,
|
||||
"scan_db_queries": 2,
|
||||
"scan_objects_per_second": 11111.1,
|
||||
"scan_ms_per_object": 0.09,
|
||||
"repair_ms": 1324,
|
||||
"repair_s3_requests": 101,
|
||||
"repair_deleted": 100
|
||||
},
|
||||
{
|
||||
"objects": 1000,
|
||||
"scan_ms": 34,
|
||||
"scan_s3_requests": 1,
|
||||
"scan_db_queries": 2,
|
||||
"scan_objects_per_second": 29411.8,
|
||||
"scan_ms_per_object": 0.034,
|
||||
"repair_ms": 12232,
|
||||
"repair_s3_requests": 1001,
|
||||
"repair_deleted": 1000
|
||||
},
|
||||
{
|
||||
"objects": 5000,
|
||||
"scan_ms": 166,
|
||||
"scan_s3_requests": 5,
|
||||
"scan_db_queries": 2,
|
||||
"scan_objects_per_second": 30120.5,
|
||||
"scan_ms_per_object": 0.0332,
|
||||
"repair_ms": 69929,
|
||||
"repair_s3_requests": 5005,
|
||||
"repair_deleted": 5000
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
2026-11-20T04:41:49+03:00
|
||||
two-store consistency: injections on both sides, dry-run, safe repair, repeat, protection of fresh and used data
|
||||
|
||||
injections:
|
||||
S3 has an object that has no metadata row (fresh orphan)
|
||||
metadata exists but the object is gone: one attachment used by a message, one unused
|
||||
|
||||
$ scripts/s3tool.sh put chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000 - < scripts/fixtures/sample.jpg
|
||||
put chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000 (653 bytes)
|
||||
$ scripts/s3tool.sh delete chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1 # used
|
||||
deleted chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1
|
||||
$ scripts/s3tool.sh delete chats/d03b2c6b-67fd-4b5d-bc17-9e926440ff8d/c49ab2de-258a-4851-8ad2-f7f585fd3488 # unused
|
||||
deleted chats/d03b2c6b-67fd-4b5d-bc17-9e926440ff8d/c49ab2de-258a-4851-8ad2-f7f585fd3488
|
||||
|
||||
$ scripts/reconcile.sh -grace 1h # dry-run
|
||||
{
|
||||
"prefix": "chats/",
|
||||
"grace_seconds": 3600,
|
||||
"dry_run": true,
|
||||
"scanned_objects": 19,
|
||||
"scanned_rows": 20,
|
||||
"truncated": false,
|
||||
"orphans": [],
|
||||
"missing": [
|
||||
{
|
||||
"id": "6a9f0506-2126-497a-b8b4-89ac728fe5a1",
|
||||
"object_key": "chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1",
|
||||
"chat_id": "47f62016-7525-4910-9823-0b21f2664b10",
|
||||
"created_at": "2026-11-20T01:41:41Z",
|
||||
"referenced_by_message": true
|
||||
},
|
||||
{
|
||||
"id": "c49ab2de-258a-4851-8ad2-f7f585fd3488",
|
||||
"object_key": "chats/d03b2c6b-67fd-4b5d-bc17-9e926440ff8d/c49ab2de-258a-4851-8ad2-f7f585fd3488",
|
||||
"chat_id": "d03b2c6b-67fd-4b5d-bc17-9e926440ff8d",
|
||||
"created_at": "2026-11-20T01:41:41Z",
|
||||
"referenced_by_message": false
|
||||
}
|
||||
],
|
||||
"deleted_objects": 0,
|
||||
"deleted_metadata": 0,
|
||||
"retained_missing": 0,
|
||||
"delete_failures": 0,
|
||||
"s3_requests": 1,
|
||||
"db_queries": 2,
|
||||
"duration_ms": 8,
|
||||
"objects_per_second": 2375
|
||||
}
|
||||
|
||||
$ scripts/reconcile.sh -grace 1h -dry-run=false # apply: fresh object survives, safe metadata repair
|
||||
{
|
||||
"prefix": "chats/",
|
||||
"grace_seconds": 3600,
|
||||
"dry_run": false,
|
||||
"scanned_objects": 19,
|
||||
"scanned_rows": 20,
|
||||
"truncated": false,
|
||||
"orphans": [],
|
||||
"missing": [
|
||||
{
|
||||
"id": "6a9f0506-2126-497a-b8b4-89ac728fe5a1",
|
||||
"object_key": "chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1",
|
||||
"chat_id": "47f62016-7525-4910-9823-0b21f2664b10",
|
||||
"created_at": "2026-11-20T01:41:41Z",
|
||||
"referenced_by_message": true
|
||||
},
|
||||
{
|
||||
"id": "c49ab2de-258a-4851-8ad2-f7f585fd3488",
|
||||
"object_key": "chats/d03b2c6b-67fd-4b5d-bc17-9e926440ff8d/c49ab2de-258a-4851-8ad2-f7f585fd3488",
|
||||
"chat_id": "d03b2c6b-67fd-4b5d-bc17-9e926440ff8d",
|
||||
"created_at": "2026-11-20T01:41:41Z",
|
||||
"referenced_by_message": false
|
||||
}
|
||||
],
|
||||
"deleted_objects": 0,
|
||||
"deleted_metadata": 1,
|
||||
"retained_missing": 1,
|
||||
"delete_failures": 0,
|
||||
"s3_requests": 1,
|
||||
"db_queries": 3,
|
||||
"duration_ms": 12,
|
||||
"objects_per_second": 1583.3333333333333
|
||||
}
|
||||
|
||||
$ scripts/s3tool.sh list chats/aaaaaaaa-0000-4000-8000-000000000000 # fresh orphan still present
|
||||
chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000 653 2026-11-20T01:41:49Z
|
||||
|
||||
$ scripts/reconcile.sh -grace 0s # now eligible for deletion
|
||||
{
|
||||
"prefix": "chats/",
|
||||
"grace_seconds": 0,
|
||||
"dry_run": true,
|
||||
"scanned_objects": 19,
|
||||
"scanned_rows": 19,
|
||||
"truncated": false,
|
||||
"orphans": [
|
||||
{
|
||||
"key": "chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000",
|
||||
"size_bytes": 653,
|
||||
"last_modified": "2026-11-20T01:41:49Z",
|
||||
"age_seconds": 7
|
||||
}
|
||||
],
|
||||
"missing": [
|
||||
{
|
||||
"id": "6a9f0506-2126-497a-b8b4-89ac728fe5a1",
|
||||
"object_key": "chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1",
|
||||
"chat_id": "47f62016-7525-4910-9823-0b21f2664b10",
|
||||
"created_at": "2026-11-20T01:41:41Z",
|
||||
"referenced_by_message": true
|
||||
}
|
||||
],
|
||||
"deleted_objects": 0,
|
||||
"deleted_metadata": 0,
|
||||
"retained_missing": 0,
|
||||
"delete_failures": 0,
|
||||
"s3_requests": 1,
|
||||
"db_queries": 2,
|
||||
"duration_ms": 8,
|
||||
"objects_per_second": 2375
|
||||
}
|
||||
|
||||
$ scripts/reconcile.sh -grace 0s -dry-run=false # apply
|
||||
{
|
||||
"prefix": "chats/",
|
||||
"grace_seconds": 0,
|
||||
"dry_run": false,
|
||||
"scanned_objects": 19,
|
||||
"scanned_rows": 19,
|
||||
"truncated": false,
|
||||
"orphans": [
|
||||
{
|
||||
"key": "chats/aaaaaaaa-0000-4000-8000-000000000000/bbbbbbbb-0000-4000-8000-000000000000",
|
||||
"size_bytes": 653,
|
||||
"last_modified": "2026-11-20T01:41:49Z",
|
||||
"age_seconds": 13
|
||||
}
|
||||
],
|
||||
"missing": [
|
||||
{
|
||||
"id": "6a9f0506-2126-497a-b8b4-89ac728fe5a1",
|
||||
"object_key": "chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1",
|
||||
"chat_id": "47f62016-7525-4910-9823-0b21f2664b10",
|
||||
"created_at": "2026-11-20T01:41:41Z",
|
||||
"referenced_by_message": true
|
||||
}
|
||||
],
|
||||
"deleted_objects": 1,
|
||||
"deleted_metadata": 0,
|
||||
"retained_missing": 1,
|
||||
"delete_failures": 0,
|
||||
"s3_requests": 2,
|
||||
"db_queries": 2,
|
||||
"duration_ms": 15,
|
||||
"objects_per_second": 1266.6666666666667
|
||||
}
|
||||
|
||||
$ scripts/reconcile.sh -grace 0s -dry-run=false # repeat: idempotent
|
||||
{
|
||||
"prefix": "chats/",
|
||||
"grace_seconds": 0,
|
||||
"dry_run": false,
|
||||
"scanned_objects": 18,
|
||||
"scanned_rows": 19,
|
||||
"truncated": false,
|
||||
"orphans": [],
|
||||
"missing": [
|
||||
{
|
||||
"id": "6a9f0506-2126-497a-b8b4-89ac728fe5a1",
|
||||
"object_key": "chats/47f62016-7525-4910-9823-0b21f2664b10/6a9f0506-2126-497a-b8b4-89ac728fe5a1",
|
||||
"chat_id": "47f62016-7525-4910-9823-0b21f2664b10",
|
||||
"created_at": "2026-11-20T01:41:41Z",
|
||||
"referenced_by_message": true
|
||||
}
|
||||
],
|
||||
"deleted_objects": 0,
|
||||
"deleted_metadata": 0,
|
||||
"retained_missing": 1,
|
||||
"delete_failures": 0,
|
||||
"s3_requests": 1,
|
||||
"db_queries": 2,
|
||||
"duration_ms": 9,
|
||||
"objects_per_second": 2000.0000000000002
|
||||
}
|
||||
|
||||
used attachments remain untouched (rows and objects)
|
||||
used_attachments | ready
|
||||
------------------+-------
|
||||
3 | 3
|
||||
(1 row)
|
||||
|
||||
retained referenced metadata row for the attachment whose object was deleted: 1
|
||||
object of that referenced attachment is still absent: 0
|
||||
+11
-11
@@ -3,30 +3,30 @@
|
||||
"uploads": 10,
|
||||
"fixture": "large.png",
|
||||
"fixture_bytes": 126609,
|
||||
"elapsed_seconds": 1.235,
|
||||
"elapsed_seconds": 1.009,
|
||||
"successful": 10,
|
||||
"failed": 0,
|
||||
"failures": [],
|
||||
"latency_ms": {
|
||||
"p50": 663.7,
|
||||
"p95": 1224.7,
|
||||
"p99": 1224.7
|
||||
"p50": 660.9,
|
||||
"p95": 999.3,
|
||||
"p99": 999.3
|
||||
},
|
||||
"api_containers": [
|
||||
"messenger-lab4_api_1",
|
||||
"messenger-lab4_api_2"
|
||||
],
|
||||
"peak_bytes_before": {
|
||||
"messenger-lab4_api_1": 5050368,
|
||||
"messenger-lab4_api_2": 4820992
|
||||
"messenger-lab4_api_1": 5951488,
|
||||
"messenger-lab4_api_2": 4993024
|
||||
},
|
||||
"peak_bytes_after": {
|
||||
"messenger-lab4_api_1": 368119808,
|
||||
"messenger-lab4_api_2": 368185344
|
||||
"messenger-lab4_api_1": 348913664,
|
||||
"messenger-lab4_api_2": 348721152
|
||||
},
|
||||
"peak_delta_bytes": {
|
||||
"messenger-lab4_api_1": 363069440,
|
||||
"messenger-lab4_api_2": 363364352
|
||||
"messenger-lab4_api_1": 342962176,
|
||||
"messenger-lab4_api_2": 343728128
|
||||
},
|
||||
"peak_max_mb": 351.13
|
||||
"peak_max_mb": 332.75
|
||||
}
|
||||
|
||||
@@ -53,7 +53,13 @@ func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
h.fail(w, r, err)
|
||||
return
|
||||
}
|
||||
attachment, content, err := h.attachments.Open(r.Context(), actor(r), id, r.URL.Query().Get("variant"))
|
||||
query := r.URL.Query()
|
||||
variant := query.Get("variant")
|
||||
if query.Has("variant") && variant == "" {
|
||||
h.fail(w, r, badRequest("variant must be original"))
|
||||
return
|
||||
}
|
||||
attachment, content, err := h.attachments.Open(r.Context(), actor(r), id, variant)
|
||||
if err != nil {
|
||||
h.fail(w, r, err)
|
||||
return
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
"uuid"
|
||||
|
||||
@@ -19,8 +20,8 @@ type createChatRequest struct {
|
||||
}
|
||||
|
||||
type createMessageRequest struct {
|
||||
Text *string `json:"text"`
|
||||
AttachmentIDs *[]string `json:"attachment_ids"`
|
||||
Text *string `json:"text"`
|
||||
AttachmentIDs json.RawMessage `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
type userResponse struct {
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"uuid"
|
||||
@@ -31,10 +33,19 @@ func (h *Handler) SendMessage(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
var attachmentIDs []uuid.UUID
|
||||
if req.AttachmentIDs != nil {
|
||||
attachmentIDs = make([]uuid.UUID, len(*req.AttachmentIDs))
|
||||
for i, raw := range *req.AttachmentIDs {
|
||||
id, ok := parseUUID(raw)
|
||||
if len(req.AttachmentIDs) > 0 {
|
||||
if bytes.Equal(bytes.TrimSpace(req.AttachmentIDs), []byte("null")) {
|
||||
h.fail(w, r, badRequest("attachment_ids must be an array of UUIDs"))
|
||||
return
|
||||
}
|
||||
var raw []string
|
||||
if err := json.Unmarshal(req.AttachmentIDs, &raw); err != nil {
|
||||
h.fail(w, r, badRequest("attachment_ids must be an array of UUIDs"))
|
||||
return
|
||||
}
|
||||
attachmentIDs = make([]uuid.UUID, len(raw))
|
||||
for i, value := range raw {
|
||||
id, ok := parseUUID(value)
|
||||
if !ok {
|
||||
h.fail(w, r, badRequest("attachment_ids must contain UUIDs"))
|
||||
return
|
||||
|
||||
@@ -28,31 +28,37 @@ type Orphan struct {
|
||||
}
|
||||
|
||||
type Missing struct {
|
||||
ID string `json:"id"`
|
||||
ObjectKey string `json:"object_key"`
|
||||
ChatID string `json:"chat_id"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
ID string `json:"id"`
|
||||
ObjectKey string `json:"object_key"`
|
||||
ChatID string `json:"chat_id"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
Referenced bool `json:"referenced_by_message"`
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
Prefix string `json:"prefix"`
|
||||
GraceSeconds int64 `json:"grace_seconds"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
ScannedObjects int `json:"scanned_objects"`
|
||||
ScannedRows int `json:"scanned_rows"`
|
||||
Truncated bool `json:"truncated"`
|
||||
Orphans []Orphan `json:"orphans"`
|
||||
Missing []Missing `json:"missing"`
|
||||
Deleted int `json:"deleted_objects"`
|
||||
DeleteFailures int `json:"delete_failures"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Prefix string `json:"prefix"`
|
||||
GraceSeconds int64 `json:"grace_seconds"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
ScannedObjects int `json:"scanned_objects"`
|
||||
ScannedRows int `json:"scanned_rows"`
|
||||
Truncated bool `json:"truncated"`
|
||||
Orphans []Orphan `json:"orphans"`
|
||||
Missing []Missing `json:"missing"`
|
||||
DeletedObjects int `json:"deleted_objects"`
|
||||
DeletedMetadata int `json:"deleted_metadata"`
|
||||
RetainedMissing int `json:"retained_missing"`
|
||||
DeleteFailures int `json:"delete_failures"`
|
||||
S3Requests int64 `json:"s3_requests"`
|
||||
DBQueries int `json:"db_queries"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
ObjectsPerSecond float64 `json:"objects_per_second"`
|
||||
}
|
||||
|
||||
func Plan(objects []Object, rows []Metadata, prefix string, now time.Time, grace time.Duration) ([]Orphan, []Missing) {
|
||||
referenced := make(map[string]struct{}, len(rows))
|
||||
func Plan(objects []Object, rows []Metadata, referenced map[uuid.UUID]bool, prefix string, now time.Time, grace time.Duration) ([]Orphan, []Missing) {
|
||||
known := make(map[string]struct{}, len(rows))
|
||||
existing := make(map[string]struct{}, len(objects))
|
||||
for _, row := range rows {
|
||||
referenced[row.ObjectKey] = struct{}{}
|
||||
known[row.ObjectKey] = struct{}{}
|
||||
}
|
||||
for _, object := range objects {
|
||||
existing[object.Key] = struct{}{}
|
||||
@@ -63,7 +69,7 @@ func Plan(objects []Object, rows []Metadata, prefix string, now time.Time, grace
|
||||
if !strings.HasPrefix(object.Key, prefix) {
|
||||
continue
|
||||
}
|
||||
if _, ok := referenced[object.Key]; ok {
|
||||
if _, ok := known[object.Key]; ok {
|
||||
continue
|
||||
}
|
||||
age := now.Sub(object.LastModified)
|
||||
@@ -80,14 +86,18 @@ func Plan(objects []Object, rows []Metadata, prefix string, now time.Time, grace
|
||||
|
||||
missing := make([]Missing, 0)
|
||||
for _, row := range rows {
|
||||
if !strings.HasPrefix(row.ObjectKey, prefix) {
|
||||
continue
|
||||
}
|
||||
if _, ok := existing[row.ObjectKey]; ok {
|
||||
continue
|
||||
}
|
||||
missing = append(missing, Missing{
|
||||
ID: row.ID.String(),
|
||||
ObjectKey: row.ObjectKey,
|
||||
ChatID: row.ChatID.String(),
|
||||
CreatedAt: row.CreatedAt.UTC().Format(time.RFC3339),
|
||||
ID: row.ID.String(),
|
||||
ObjectKey: row.ObjectKey,
|
||||
ChatID: row.ChatID.String(),
|
||||
CreatedAt: row.CreatedAt.UTC().Format(time.RFC3339),
|
||||
Referenced: referenced[row.ID],
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
func TestPlanDetectsOrphansAndMissing(t *testing.T) {
|
||||
now := time.Date(2026, 11, 20, 12, 0, 0, 0, time.UTC)
|
||||
usedID, missingID := uuid.New(), uuid.New()
|
||||
usedID, missingID, unreferencedID := uuid.New(), uuid.New(), uuid.New()
|
||||
objects := []Object{
|
||||
{Key: "chats/a/used", Size: 10, LastModified: now.Add(-2 * time.Hour)},
|
||||
{Key: "chats/a/orphan-old", Size: 20, LastModified: now.Add(-2 * time.Hour)},
|
||||
@@ -17,16 +17,28 @@ func TestPlanDetectsOrphansAndMissing(t *testing.T) {
|
||||
}
|
||||
rows := []Metadata{
|
||||
{ID: usedID, ObjectKey: "chats/a/used", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)},
|
||||
{ID: missingID, ObjectKey: "chats/a/missing", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)},
|
||||
{ID: missingID, ObjectKey: "chats/a/missing-referenced", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)},
|
||||
{ID: unreferencedID, ObjectKey: "chats/a/missing-orphan-row", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)},
|
||||
}
|
||||
referenced := map[uuid.UUID]bool{missingID: true}
|
||||
|
||||
orphans, missing := Plan(objects, rows, "chats/", now, time.Hour)
|
||||
orphans, missing := Plan(objects, rows, referenced, "chats/", now, time.Hour)
|
||||
|
||||
if len(orphans) != 1 || orphans[0].Key != "chats/a/orphan-old" {
|
||||
t.Fatalf("orphans = %+v, want only chats/a/orphan-old", orphans)
|
||||
}
|
||||
if len(missing) != 1 || missing[0].ObjectKey != "chats/a/missing" {
|
||||
t.Fatalf("missing = %+v, want only chats/a/missing", missing)
|
||||
if len(missing) != 2 {
|
||||
t.Fatalf("missing = %+v, want two rows", missing)
|
||||
}
|
||||
byKey := map[string]Missing{}
|
||||
for _, item := range missing {
|
||||
byKey[item.ObjectKey] = item
|
||||
}
|
||||
if !byKey["chats/a/missing-referenced"].Referenced {
|
||||
t.Error("referenced missing row must be flagged")
|
||||
}
|
||||
if byKey["chats/a/missing-orphan-row"].Referenced {
|
||||
t.Error("unreferenced missing row must not be flagged")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,12 +48,12 @@ func TestPlanHonoursGraceAndPrefix(t *testing.T) {
|
||||
{Key: "chats/a/fresh", Size: 1, LastModified: now.Add(-30 * time.Second)},
|
||||
{Key: "outside/key", Size: 1, LastModified: now.Add(-48 * time.Hour)},
|
||||
}
|
||||
orphans, _ := Plan(objects, nil, "chats/", now, time.Hour)
|
||||
orphans, _ := Plan(objects, nil, nil, "chats/", now, time.Hour)
|
||||
if len(orphans) != 0 {
|
||||
t.Fatalf("orphans = %+v, want none", orphans)
|
||||
}
|
||||
|
||||
orphans, _ = Plan(objects, nil, "chats/", now, 0)
|
||||
orphans, _ = Plan(objects, nil, nil, "chats/", now, 0)
|
||||
if len(orphans) != 1 || orphans[0].Key != "chats/a/fresh" {
|
||||
t.Fatalf("orphans with zero grace = %+v, want chats/a/fresh", orphans)
|
||||
}
|
||||
@@ -52,7 +64,7 @@ func TestPlanNeverDeletesReferencedObjects(t *testing.T) {
|
||||
objects := []Object{{Key: "chats/a/used", Size: 1, LastModified: now.Add(-100 * time.Hour)}}
|
||||
rows := []Metadata{{ID: uuid.New(), ObjectKey: "chats/a/used", ChatID: uuid.New(), CreatedAt: now.Add(-100 * time.Hour)}}
|
||||
|
||||
orphans, missing := Plan(objects, rows, "chats/", now, 0)
|
||||
orphans, missing := Plan(objects, rows, nil, "chats/", now, 0)
|
||||
if len(orphans) != 0 {
|
||||
t.Fatalf("orphans = %+v, want none for referenced object", orphans)
|
||||
}
|
||||
@@ -61,6 +73,18 @@ func TestPlanNeverDeletesReferencedObjects(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanOnlyReportsRowsInsidePrefix(t *testing.T) {
|
||||
now := time.Date(2026, 11, 20, 12, 0, 0, 0, time.UTC)
|
||||
rows := []Metadata{
|
||||
{ID: uuid.New(), ObjectKey: "chats/a/inside", ChatID: uuid.New(), CreatedAt: now},
|
||||
{ID: uuid.New(), ObjectKey: "bench/outside", ChatID: uuid.New(), CreatedAt: now},
|
||||
}
|
||||
_, missing := Plan(nil, rows, nil, "chats/", now, 0)
|
||||
if len(missing) != 1 || missing[0].ObjectKey != "chats/a/inside" {
|
||||
t.Fatalf("missing = %+v, want only rows under the scanned prefix", missing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanOutputIsSorted(t *testing.T) {
|
||||
now := time.Date(2026, 11, 20, 12, 0, 0, 0, time.UTC)
|
||||
objects := []Object{
|
||||
@@ -68,7 +92,7 @@ func TestPlanOutputIsSorted(t *testing.T) {
|
||||
{Key: "chats/a/b", LastModified: now.Add(-time.Hour)},
|
||||
{Key: "chats/a/m", LastModified: now.Add(-time.Hour)},
|
||||
}
|
||||
orphans, _ := Plan(objects, nil, "chats/", now, 0)
|
||||
orphans, _ := Plan(objects, nil, nil, "chats/", now, 0)
|
||||
if len(orphans) != 3 || orphans[0].Key != "chats/a/b" || orphans[2].Key != "chats/a/z" {
|
||||
t.Fatalf("orphans are not sorted: %+v", orphans)
|
||||
}
|
||||
|
||||
@@ -37,11 +37,16 @@ func EndpointHost(endpoint string) (string, bool) {
|
||||
}
|
||||
|
||||
func New(endpoint, region, accessKeyID, secretAccessKey, bucket string) (*Store, error) {
|
||||
return NewWithTransport(endpoint, region, accessKeyID, secretAccessKey, bucket, nil)
|
||||
}
|
||||
|
||||
func NewWithTransport(endpoint, region, accessKeyID, secretAccessKey, bucket string, transport http.RoundTripper) (*Store, error) {
|
||||
host, secure := EndpointHost(endpoint)
|
||||
client, err := minio.New(host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Region: region,
|
||||
Secure: secure,
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Region: region,
|
||||
Secure: secure,
|
||||
Transport: transport,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
_ "image/png"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
@@ -159,7 +160,10 @@ func acquire(ctx context.Context, sem chan struct{}) error {
|
||||
func release(sem chan struct{}) { <-sem }
|
||||
|
||||
func checkFilename(filename string) error {
|
||||
if n := utf8.RuneCountInString(filename); n < 1 || n > maxFilenameRunes {
|
||||
if strings.TrimSpace(filename) == "" {
|
||||
return fmt.Errorf("%w: filename must not be blank", domain.ErrInvalidInput)
|
||||
}
|
||||
if n := utf8.RuneCountInString(filename); n > maxFilenameRunes {
|
||||
return fmt.Errorf("%w: filename must be 1 to 255 characters", domain.ErrInvalidInput)
|
||||
}
|
||||
for _, r := range filename {
|
||||
|
||||
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reconciler traversal cost for several object counts; creates synthetic objects and removes them."""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
S3TOOL = ROOT / 'scripts/s3tool.sh'
|
||||
RECONCILE = ROOT / 'scripts/reconcile.sh'
|
||||
PREFIX = 'chats/bench'
|
||||
SIZES = (100, 1000, 5000)
|
||||
|
||||
|
||||
def call(arguments):
|
||||
return subprocess.run(arguments, capture_output=True, text=True, check=False)
|
||||
|
||||
|
||||
def reconcile(*arguments):
|
||||
result = call([str(RECONCILE), *arguments])
|
||||
try:
|
||||
return json.loads(result.stdout)
|
||||
except json.JSONDecodeError:
|
||||
print(result.stdout[-500:], file=sys.stderr)
|
||||
print(result.stderr[-500:], file=sys.stderr)
|
||||
raise SystemExit('reconcile did not return JSON')
|
||||
|
||||
|
||||
def main():
|
||||
measurements = []
|
||||
for count in SIZES:
|
||||
seeded = call([str(S3TOOL), 'seed', PREFIX, str(count), '1024'])
|
||||
if seeded.returncode != 0:
|
||||
raise SystemExit('seed failed: ' + seeded.stderr[-400:])
|
||||
|
||||
scan = reconcile('-prefix', PREFIX + '/', '-grace', '0s')
|
||||
if scan['scanned_objects'] != count:
|
||||
raise SystemExit(f'expected {count} objects, scanned {scan["scanned_objects"]}')
|
||||
if len(scan['orphans']) != count:
|
||||
raise SystemExit(f'expected {count} orphans, got {len(scan["orphans"])}')
|
||||
|
||||
repair = reconcile('-prefix', PREFIX + '/', '-grace', '0s', '-dry-run=false')
|
||||
if repair['deleted_objects'] != count:
|
||||
raise SystemExit(f'expected {count} deletions, got {repair["deleted_objects"]}')
|
||||
|
||||
measurements.append({
|
||||
'objects': count,
|
||||
'scan_ms': scan['duration_ms'],
|
||||
'scan_s3_requests': scan['s3_requests'],
|
||||
'scan_db_queries': scan['db_queries'],
|
||||
'scan_objects_per_second': round(scan['objects_per_second'], 1),
|
||||
'scan_ms_per_object': round(scan['duration_ms'] / count, 4),
|
||||
'repair_ms': repair['duration_ms'],
|
||||
'repair_s3_requests': repair['s3_requests'],
|
||||
'repair_deleted': repair['deleted_objects'],
|
||||
})
|
||||
|
||||
print(json.dumps({'prefix': PREFIX, 'measurements': measurements}, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -14,6 +14,7 @@ def main():
|
||||
p.add_argument('--expect', type=int, default=201)
|
||||
p.add_argument('--file', default=str(ROOT / 'scripts/fixtures/sample.jpg'))
|
||||
p.add_argument('--content-type', default='image/jpeg')
|
||||
p.add_argument('--attach', action='store_true', help='also attach the upload to a new message')
|
||||
|
||||
args = arguments('Single upload used by compensation drills; creates synthetic data', configure)
|
||||
source = Path(args.file)
|
||||
@@ -29,6 +30,9 @@ def main():
|
||||
if response['status'] == 201:
|
||||
attachment = response['json']
|
||||
print(f"attachment={attachment['id']} size={attachment['size_bytes']}")
|
||||
if args.attach:
|
||||
message = c.message(alice, chat, '', attachments=[attachment['id']])['json']
|
||||
print(f"message={message['id']} attachment_ids={message['attachment_ids']}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
Reference in New Issue
Block a user