feat: two-store reconciler with safe repair and cost measurement (grade 5, track 2)

This commit is contained in:
2026-11-20 05:01:20 +03:00
parent 1c3011bf28
commit 60e8f4474b
36 changed files with 807 additions and 212 deletions
+7 -1
View File
@@ -53,7 +53,13 @@ func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) {
h.fail(w, r, err)
return
}
attachment, content, err := h.attachments.Open(r.Context(), actor(r), id, r.URL.Query().Get("variant"))
query := r.URL.Query()
variant := query.Get("variant")
if query.Has("variant") && variant == "" {
h.fail(w, r, badRequest("variant must be original"))
return
}
attachment, content, err := h.attachments.Open(r.Context(), actor(r), id, variant)
if err != nil {
h.fail(w, r, err)
return
+3 -2
View File
@@ -1,6 +1,7 @@
package handler
import (
"encoding/json"
"time"
"uuid"
@@ -19,8 +20,8 @@ type createChatRequest struct {
}
type createMessageRequest struct {
Text *string `json:"text"`
AttachmentIDs *[]string `json:"attachment_ids"`
Text *string `json:"text"`
AttachmentIDs json.RawMessage `json:"attachment_ids"`
}
type userResponse struct {
+15 -4
View File
@@ -1,6 +1,8 @@
package handler
import (
"bytes"
"encoding/json"
"net/http"
"regexp"
"uuid"
@@ -31,10 +33,19 @@ func (h *Handler) SendMessage(w http.ResponseWriter, r *http.Request) {
return
}
var attachmentIDs []uuid.UUID
if req.AttachmentIDs != nil {
attachmentIDs = make([]uuid.UUID, len(*req.AttachmentIDs))
for i, raw := range *req.AttachmentIDs {
id, ok := parseUUID(raw)
if len(req.AttachmentIDs) > 0 {
if bytes.Equal(bytes.TrimSpace(req.AttachmentIDs), []byte("null")) {
h.fail(w, r, badRequest("attachment_ids must be an array of UUIDs"))
return
}
var raw []string
if err := json.Unmarshal(req.AttachmentIDs, &raw); err != nil {
h.fail(w, r, badRequest("attachment_ids must be an array of UUIDs"))
return
}
attachmentIDs = make([]uuid.UUID, len(raw))
for i, value := range raw {
id, ok := parseUUID(value)
if !ok {
h.fail(w, r, badRequest("attachment_ids must contain UUIDs"))
return
+33 -23
View File
@@ -28,31 +28,37 @@ type Orphan struct {
}
type Missing struct {
ID string `json:"id"`
ObjectKey string `json:"object_key"`
ChatID string `json:"chat_id"`
CreatedAt string `json:"created_at"`
ID string `json:"id"`
ObjectKey string `json:"object_key"`
ChatID string `json:"chat_id"`
CreatedAt string `json:"created_at"`
Referenced bool `json:"referenced_by_message"`
}
type Report struct {
Prefix string `json:"prefix"`
GraceSeconds int64 `json:"grace_seconds"`
DryRun bool `json:"dry_run"`
ScannedObjects int `json:"scanned_objects"`
ScannedRows int `json:"scanned_rows"`
Truncated bool `json:"truncated"`
Orphans []Orphan `json:"orphans"`
Missing []Missing `json:"missing"`
Deleted int `json:"deleted_objects"`
DeleteFailures int `json:"delete_failures"`
DurationMS int64 `json:"duration_ms"`
Prefix string `json:"prefix"`
GraceSeconds int64 `json:"grace_seconds"`
DryRun bool `json:"dry_run"`
ScannedObjects int `json:"scanned_objects"`
ScannedRows int `json:"scanned_rows"`
Truncated bool `json:"truncated"`
Orphans []Orphan `json:"orphans"`
Missing []Missing `json:"missing"`
DeletedObjects int `json:"deleted_objects"`
DeletedMetadata int `json:"deleted_metadata"`
RetainedMissing int `json:"retained_missing"`
DeleteFailures int `json:"delete_failures"`
S3Requests int64 `json:"s3_requests"`
DBQueries int `json:"db_queries"`
DurationMS int64 `json:"duration_ms"`
ObjectsPerSecond float64 `json:"objects_per_second"`
}
func Plan(objects []Object, rows []Metadata, prefix string, now time.Time, grace time.Duration) ([]Orphan, []Missing) {
referenced := make(map[string]struct{}, len(rows))
func Plan(objects []Object, rows []Metadata, referenced map[uuid.UUID]bool, prefix string, now time.Time, grace time.Duration) ([]Orphan, []Missing) {
known := make(map[string]struct{}, len(rows))
existing := make(map[string]struct{}, len(objects))
for _, row := range rows {
referenced[row.ObjectKey] = struct{}{}
known[row.ObjectKey] = struct{}{}
}
for _, object := range objects {
existing[object.Key] = struct{}{}
@@ -63,7 +69,7 @@ func Plan(objects []Object, rows []Metadata, prefix string, now time.Time, grace
if !strings.HasPrefix(object.Key, prefix) {
continue
}
if _, ok := referenced[object.Key]; ok {
if _, ok := known[object.Key]; ok {
continue
}
age := now.Sub(object.LastModified)
@@ -80,14 +86,18 @@ func Plan(objects []Object, rows []Metadata, prefix string, now time.Time, grace
missing := make([]Missing, 0)
for _, row := range rows {
if !strings.HasPrefix(row.ObjectKey, prefix) {
continue
}
if _, ok := existing[row.ObjectKey]; ok {
continue
}
missing = append(missing, Missing{
ID: row.ID.String(),
ObjectKey: row.ObjectKey,
ChatID: row.ChatID.String(),
CreatedAt: row.CreatedAt.UTC().Format(time.RFC3339),
ID: row.ID.String(),
ObjectKey: row.ObjectKey,
ChatID: row.ChatID.String(),
CreatedAt: row.CreatedAt.UTC().Format(time.RFC3339),
Referenced: referenced[row.ID],
})
}
+33 -9
View File
@@ -8,7 +8,7 @@ import (
func TestPlanDetectsOrphansAndMissing(t *testing.T) {
now := time.Date(2026, 11, 20, 12, 0, 0, 0, time.UTC)
usedID, missingID := uuid.New(), uuid.New()
usedID, missingID, unreferencedID := uuid.New(), uuid.New(), uuid.New()
objects := []Object{
{Key: "chats/a/used", Size: 10, LastModified: now.Add(-2 * time.Hour)},
{Key: "chats/a/orphan-old", Size: 20, LastModified: now.Add(-2 * time.Hour)},
@@ -17,16 +17,28 @@ func TestPlanDetectsOrphansAndMissing(t *testing.T) {
}
rows := []Metadata{
{ID: usedID, ObjectKey: "chats/a/used", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)},
{ID: missingID, ObjectKey: "chats/a/missing", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)},
{ID: missingID, ObjectKey: "chats/a/missing-referenced", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)},
{ID: unreferencedID, ObjectKey: "chats/a/missing-orphan-row", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)},
}
referenced := map[uuid.UUID]bool{missingID: true}
orphans, missing := Plan(objects, rows, "chats/", now, time.Hour)
orphans, missing := Plan(objects, rows, referenced, "chats/", now, time.Hour)
if len(orphans) != 1 || orphans[0].Key != "chats/a/orphan-old" {
t.Fatalf("orphans = %+v, want only chats/a/orphan-old", orphans)
}
if len(missing) != 1 || missing[0].ObjectKey != "chats/a/missing" {
t.Fatalf("missing = %+v, want only chats/a/missing", missing)
if len(missing) != 2 {
t.Fatalf("missing = %+v, want two rows", missing)
}
byKey := map[string]Missing{}
for _, item := range missing {
byKey[item.ObjectKey] = item
}
if !byKey["chats/a/missing-referenced"].Referenced {
t.Error("referenced missing row must be flagged")
}
if byKey["chats/a/missing-orphan-row"].Referenced {
t.Error("unreferenced missing row must not be flagged")
}
}
@@ -36,12 +48,12 @@ func TestPlanHonoursGraceAndPrefix(t *testing.T) {
{Key: "chats/a/fresh", Size: 1, LastModified: now.Add(-30 * time.Second)},
{Key: "outside/key", Size: 1, LastModified: now.Add(-48 * time.Hour)},
}
orphans, _ := Plan(objects, nil, "chats/", now, time.Hour)
orphans, _ := Plan(objects, nil, nil, "chats/", now, time.Hour)
if len(orphans) != 0 {
t.Fatalf("orphans = %+v, want none", orphans)
}
orphans, _ = Plan(objects, nil, "chats/", now, 0)
orphans, _ = Plan(objects, nil, nil, "chats/", now, 0)
if len(orphans) != 1 || orphans[0].Key != "chats/a/fresh" {
t.Fatalf("orphans with zero grace = %+v, want chats/a/fresh", orphans)
}
@@ -52,7 +64,7 @@ func TestPlanNeverDeletesReferencedObjects(t *testing.T) {
objects := []Object{{Key: "chats/a/used", Size: 1, LastModified: now.Add(-100 * time.Hour)}}
rows := []Metadata{{ID: uuid.New(), ObjectKey: "chats/a/used", ChatID: uuid.New(), CreatedAt: now.Add(-100 * time.Hour)}}
orphans, missing := Plan(objects, rows, "chats/", now, 0)
orphans, missing := Plan(objects, rows, nil, "chats/", now, 0)
if len(orphans) != 0 {
t.Fatalf("orphans = %+v, want none for referenced object", orphans)
}
@@ -61,6 +73,18 @@ func TestPlanNeverDeletesReferencedObjects(t *testing.T) {
}
}
func TestPlanOnlyReportsRowsInsidePrefix(t *testing.T) {
now := time.Date(2026, 11, 20, 12, 0, 0, 0, time.UTC)
rows := []Metadata{
{ID: uuid.New(), ObjectKey: "chats/a/inside", ChatID: uuid.New(), CreatedAt: now},
{ID: uuid.New(), ObjectKey: "bench/outside", ChatID: uuid.New(), CreatedAt: now},
}
_, missing := Plan(nil, rows, nil, "chats/", now, 0)
if len(missing) != 1 || missing[0].ObjectKey != "chats/a/inside" {
t.Fatalf("missing = %+v, want only rows under the scanned prefix", missing)
}
}
func TestPlanOutputIsSorted(t *testing.T) {
now := time.Date(2026, 11, 20, 12, 0, 0, 0, time.UTC)
objects := []Object{
@@ -68,7 +92,7 @@ func TestPlanOutputIsSorted(t *testing.T) {
{Key: "chats/a/b", LastModified: now.Add(-time.Hour)},
{Key: "chats/a/m", LastModified: now.Add(-time.Hour)},
}
orphans, _ := Plan(objects, nil, "chats/", now, 0)
orphans, _ := Plan(objects, nil, nil, "chats/", now, 0)
if len(orphans) != 3 || orphans[0].Key != "chats/a/b" || orphans[2].Key != "chats/a/z" {
t.Fatalf("orphans are not sorted: %+v", orphans)
}
+8 -3
View File
@@ -37,11 +37,16 @@ func EndpointHost(endpoint string) (string, bool) {
}
func New(endpoint, region, accessKeyID, secretAccessKey, bucket string) (*Store, error) {
return NewWithTransport(endpoint, region, accessKeyID, secretAccessKey, bucket, nil)
}
func NewWithTransport(endpoint, region, accessKeyID, secretAccessKey, bucket string, transport http.RoundTripper) (*Store, error) {
host, secure := EndpointHost(endpoint)
client, err := minio.New(host, &minio.Options{
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
Region: region,
Secure: secure,
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
Region: region,
Secure: secure,
Transport: transport,
})
if err != nil {
return nil, err
+5 -1
View File
@@ -11,6 +11,7 @@ import (
_ "image/png"
"io"
"net/http"
"strings"
"time"
"unicode"
"unicode/utf8"
@@ -159,7 +160,10 @@ func acquire(ctx context.Context, sem chan struct{}) error {
func release(sem chan struct{}) { <-sem }
func checkFilename(filename string) error {
if n := utf8.RuneCountInString(filename); n < 1 || n > maxFilenameRunes {
if strings.TrimSpace(filename) == "" {
return fmt.Errorf("%w: filename must not be blank", domain.ErrInvalidInput)
}
if n := utf8.RuneCountInString(filename); n > maxFilenameRunes {
return fmt.Errorf("%w: filename must be 1 to 255 characters", domain.ErrInvalidInput)
}
for _, r := range filename {