forked from mtusi2026/mtusi2026lab4
feat: two-store reconciler with safe repair and cost measurement (grade 5, track 2)
This commit is contained in:
@@ -53,7 +53,13 @@ func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
h.fail(w, r, err)
|
||||
return
|
||||
}
|
||||
attachment, content, err := h.attachments.Open(r.Context(), actor(r), id, r.URL.Query().Get("variant"))
|
||||
query := r.URL.Query()
|
||||
variant := query.Get("variant")
|
||||
if query.Has("variant") && variant == "" {
|
||||
h.fail(w, r, badRequest("variant must be original"))
|
||||
return
|
||||
}
|
||||
attachment, content, err := h.attachments.Open(r.Context(), actor(r), id, variant)
|
||||
if err != nil {
|
||||
h.fail(w, r, err)
|
||||
return
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
"uuid"
|
||||
|
||||
@@ -19,8 +20,8 @@ type createChatRequest struct {
|
||||
}
|
||||
|
||||
type createMessageRequest struct {
|
||||
Text *string `json:"text"`
|
||||
AttachmentIDs *[]string `json:"attachment_ids"`
|
||||
Text *string `json:"text"`
|
||||
AttachmentIDs json.RawMessage `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
type userResponse struct {
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"uuid"
|
||||
@@ -31,10 +33,19 @@ func (h *Handler) SendMessage(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
var attachmentIDs []uuid.UUID
|
||||
if req.AttachmentIDs != nil {
|
||||
attachmentIDs = make([]uuid.UUID, len(*req.AttachmentIDs))
|
||||
for i, raw := range *req.AttachmentIDs {
|
||||
id, ok := parseUUID(raw)
|
||||
if len(req.AttachmentIDs) > 0 {
|
||||
if bytes.Equal(bytes.TrimSpace(req.AttachmentIDs), []byte("null")) {
|
||||
h.fail(w, r, badRequest("attachment_ids must be an array of UUIDs"))
|
||||
return
|
||||
}
|
||||
var raw []string
|
||||
if err := json.Unmarshal(req.AttachmentIDs, &raw); err != nil {
|
||||
h.fail(w, r, badRequest("attachment_ids must be an array of UUIDs"))
|
||||
return
|
||||
}
|
||||
attachmentIDs = make([]uuid.UUID, len(raw))
|
||||
for i, value := range raw {
|
||||
id, ok := parseUUID(value)
|
||||
if !ok {
|
||||
h.fail(w, r, badRequest("attachment_ids must contain UUIDs"))
|
||||
return
|
||||
|
||||
@@ -28,31 +28,37 @@ type Orphan struct {
|
||||
}
|
||||
|
||||
type Missing struct {
|
||||
ID string `json:"id"`
|
||||
ObjectKey string `json:"object_key"`
|
||||
ChatID string `json:"chat_id"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
ID string `json:"id"`
|
||||
ObjectKey string `json:"object_key"`
|
||||
ChatID string `json:"chat_id"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
Referenced bool `json:"referenced_by_message"`
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
Prefix string `json:"prefix"`
|
||||
GraceSeconds int64 `json:"grace_seconds"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
ScannedObjects int `json:"scanned_objects"`
|
||||
ScannedRows int `json:"scanned_rows"`
|
||||
Truncated bool `json:"truncated"`
|
||||
Orphans []Orphan `json:"orphans"`
|
||||
Missing []Missing `json:"missing"`
|
||||
Deleted int `json:"deleted_objects"`
|
||||
DeleteFailures int `json:"delete_failures"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
Prefix string `json:"prefix"`
|
||||
GraceSeconds int64 `json:"grace_seconds"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
ScannedObjects int `json:"scanned_objects"`
|
||||
ScannedRows int `json:"scanned_rows"`
|
||||
Truncated bool `json:"truncated"`
|
||||
Orphans []Orphan `json:"orphans"`
|
||||
Missing []Missing `json:"missing"`
|
||||
DeletedObjects int `json:"deleted_objects"`
|
||||
DeletedMetadata int `json:"deleted_metadata"`
|
||||
RetainedMissing int `json:"retained_missing"`
|
||||
DeleteFailures int `json:"delete_failures"`
|
||||
S3Requests int64 `json:"s3_requests"`
|
||||
DBQueries int `json:"db_queries"`
|
||||
DurationMS int64 `json:"duration_ms"`
|
||||
ObjectsPerSecond float64 `json:"objects_per_second"`
|
||||
}
|
||||
|
||||
func Plan(objects []Object, rows []Metadata, prefix string, now time.Time, grace time.Duration) ([]Orphan, []Missing) {
|
||||
referenced := make(map[string]struct{}, len(rows))
|
||||
func Plan(objects []Object, rows []Metadata, referenced map[uuid.UUID]bool, prefix string, now time.Time, grace time.Duration) ([]Orphan, []Missing) {
|
||||
known := make(map[string]struct{}, len(rows))
|
||||
existing := make(map[string]struct{}, len(objects))
|
||||
for _, row := range rows {
|
||||
referenced[row.ObjectKey] = struct{}{}
|
||||
known[row.ObjectKey] = struct{}{}
|
||||
}
|
||||
for _, object := range objects {
|
||||
existing[object.Key] = struct{}{}
|
||||
@@ -63,7 +69,7 @@ func Plan(objects []Object, rows []Metadata, prefix string, now time.Time, grace
|
||||
if !strings.HasPrefix(object.Key, prefix) {
|
||||
continue
|
||||
}
|
||||
if _, ok := referenced[object.Key]; ok {
|
||||
if _, ok := known[object.Key]; ok {
|
||||
continue
|
||||
}
|
||||
age := now.Sub(object.LastModified)
|
||||
@@ -80,14 +86,18 @@ func Plan(objects []Object, rows []Metadata, prefix string, now time.Time, grace
|
||||
|
||||
missing := make([]Missing, 0)
|
||||
for _, row := range rows {
|
||||
if !strings.HasPrefix(row.ObjectKey, prefix) {
|
||||
continue
|
||||
}
|
||||
if _, ok := existing[row.ObjectKey]; ok {
|
||||
continue
|
||||
}
|
||||
missing = append(missing, Missing{
|
||||
ID: row.ID.String(),
|
||||
ObjectKey: row.ObjectKey,
|
||||
ChatID: row.ChatID.String(),
|
||||
CreatedAt: row.CreatedAt.UTC().Format(time.RFC3339),
|
||||
ID: row.ID.String(),
|
||||
ObjectKey: row.ObjectKey,
|
||||
ChatID: row.ChatID.String(),
|
||||
CreatedAt: row.CreatedAt.UTC().Format(time.RFC3339),
|
||||
Referenced: referenced[row.ID],
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
|
||||
func TestPlanDetectsOrphansAndMissing(t *testing.T) {
|
||||
now := time.Date(2026, 11, 20, 12, 0, 0, 0, time.UTC)
|
||||
usedID, missingID := uuid.New(), uuid.New()
|
||||
usedID, missingID, unreferencedID := uuid.New(), uuid.New(), uuid.New()
|
||||
objects := []Object{
|
||||
{Key: "chats/a/used", Size: 10, LastModified: now.Add(-2 * time.Hour)},
|
||||
{Key: "chats/a/orphan-old", Size: 20, LastModified: now.Add(-2 * time.Hour)},
|
||||
@@ -17,16 +17,28 @@ func TestPlanDetectsOrphansAndMissing(t *testing.T) {
|
||||
}
|
||||
rows := []Metadata{
|
||||
{ID: usedID, ObjectKey: "chats/a/used", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)},
|
||||
{ID: missingID, ObjectKey: "chats/a/missing", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)},
|
||||
{ID: missingID, ObjectKey: "chats/a/missing-referenced", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)},
|
||||
{ID: unreferencedID, ObjectKey: "chats/a/missing-orphan-row", ChatID: uuid.New(), CreatedAt: now.Add(-2 * time.Hour)},
|
||||
}
|
||||
referenced := map[uuid.UUID]bool{missingID: true}
|
||||
|
||||
orphans, missing := Plan(objects, rows, "chats/", now, time.Hour)
|
||||
orphans, missing := Plan(objects, rows, referenced, "chats/", now, time.Hour)
|
||||
|
||||
if len(orphans) != 1 || orphans[0].Key != "chats/a/orphan-old" {
|
||||
t.Fatalf("orphans = %+v, want only chats/a/orphan-old", orphans)
|
||||
}
|
||||
if len(missing) != 1 || missing[0].ObjectKey != "chats/a/missing" {
|
||||
t.Fatalf("missing = %+v, want only chats/a/missing", missing)
|
||||
if len(missing) != 2 {
|
||||
t.Fatalf("missing = %+v, want two rows", missing)
|
||||
}
|
||||
byKey := map[string]Missing{}
|
||||
for _, item := range missing {
|
||||
byKey[item.ObjectKey] = item
|
||||
}
|
||||
if !byKey["chats/a/missing-referenced"].Referenced {
|
||||
t.Error("referenced missing row must be flagged")
|
||||
}
|
||||
if byKey["chats/a/missing-orphan-row"].Referenced {
|
||||
t.Error("unreferenced missing row must not be flagged")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,12 +48,12 @@ func TestPlanHonoursGraceAndPrefix(t *testing.T) {
|
||||
{Key: "chats/a/fresh", Size: 1, LastModified: now.Add(-30 * time.Second)},
|
||||
{Key: "outside/key", Size: 1, LastModified: now.Add(-48 * time.Hour)},
|
||||
}
|
||||
orphans, _ := Plan(objects, nil, "chats/", now, time.Hour)
|
||||
orphans, _ := Plan(objects, nil, nil, "chats/", now, time.Hour)
|
||||
if len(orphans) != 0 {
|
||||
t.Fatalf("orphans = %+v, want none", orphans)
|
||||
}
|
||||
|
||||
orphans, _ = Plan(objects, nil, "chats/", now, 0)
|
||||
orphans, _ = Plan(objects, nil, nil, "chats/", now, 0)
|
||||
if len(orphans) != 1 || orphans[0].Key != "chats/a/fresh" {
|
||||
t.Fatalf("orphans with zero grace = %+v, want chats/a/fresh", orphans)
|
||||
}
|
||||
@@ -52,7 +64,7 @@ func TestPlanNeverDeletesReferencedObjects(t *testing.T) {
|
||||
objects := []Object{{Key: "chats/a/used", Size: 1, LastModified: now.Add(-100 * time.Hour)}}
|
||||
rows := []Metadata{{ID: uuid.New(), ObjectKey: "chats/a/used", ChatID: uuid.New(), CreatedAt: now.Add(-100 * time.Hour)}}
|
||||
|
||||
orphans, missing := Plan(objects, rows, "chats/", now, 0)
|
||||
orphans, missing := Plan(objects, rows, nil, "chats/", now, 0)
|
||||
if len(orphans) != 0 {
|
||||
t.Fatalf("orphans = %+v, want none for referenced object", orphans)
|
||||
}
|
||||
@@ -61,6 +73,18 @@ func TestPlanNeverDeletesReferencedObjects(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanOnlyReportsRowsInsidePrefix(t *testing.T) {
|
||||
now := time.Date(2026, 11, 20, 12, 0, 0, 0, time.UTC)
|
||||
rows := []Metadata{
|
||||
{ID: uuid.New(), ObjectKey: "chats/a/inside", ChatID: uuid.New(), CreatedAt: now},
|
||||
{ID: uuid.New(), ObjectKey: "bench/outside", ChatID: uuid.New(), CreatedAt: now},
|
||||
}
|
||||
_, missing := Plan(nil, rows, nil, "chats/", now, 0)
|
||||
if len(missing) != 1 || missing[0].ObjectKey != "chats/a/inside" {
|
||||
t.Fatalf("missing = %+v, want only rows under the scanned prefix", missing)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlanOutputIsSorted(t *testing.T) {
|
||||
now := time.Date(2026, 11, 20, 12, 0, 0, 0, time.UTC)
|
||||
objects := []Object{
|
||||
@@ -68,7 +92,7 @@ func TestPlanOutputIsSorted(t *testing.T) {
|
||||
{Key: "chats/a/b", LastModified: now.Add(-time.Hour)},
|
||||
{Key: "chats/a/m", LastModified: now.Add(-time.Hour)},
|
||||
}
|
||||
orphans, _ := Plan(objects, nil, "chats/", now, 0)
|
||||
orphans, _ := Plan(objects, nil, nil, "chats/", now, 0)
|
||||
if len(orphans) != 3 || orphans[0].Key != "chats/a/b" || orphans[2].Key != "chats/a/z" {
|
||||
t.Fatalf("orphans are not sorted: %+v", orphans)
|
||||
}
|
||||
|
||||
@@ -37,11 +37,16 @@ func EndpointHost(endpoint string) (string, bool) {
|
||||
}
|
||||
|
||||
func New(endpoint, region, accessKeyID, secretAccessKey, bucket string) (*Store, error) {
|
||||
return NewWithTransport(endpoint, region, accessKeyID, secretAccessKey, bucket, nil)
|
||||
}
|
||||
|
||||
func NewWithTransport(endpoint, region, accessKeyID, secretAccessKey, bucket string, transport http.RoundTripper) (*Store, error) {
|
||||
host, secure := EndpointHost(endpoint)
|
||||
client, err := minio.New(host, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Region: region,
|
||||
Secure: secure,
|
||||
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
|
||||
Region: region,
|
||||
Secure: secure,
|
||||
Transport: transport,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
_ "image/png"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
@@ -159,7 +160,10 @@ func acquire(ctx context.Context, sem chan struct{}) error {
|
||||
func release(sem chan struct{}) { <-sem }
|
||||
|
||||
func checkFilename(filename string) error {
|
||||
if n := utf8.RuneCountInString(filename); n < 1 || n > maxFilenameRunes {
|
||||
if strings.TrimSpace(filename) == "" {
|
||||
return fmt.Errorf("%w: filename must not be blank", domain.ErrInvalidInput)
|
||||
}
|
||||
if n := utf8.RuneCountInString(filename); n > maxFilenameRunes {
|
||||
return fmt.Errorf("%w: filename must be 1 to 255 characters", domain.ErrInvalidInput)
|
||||
}
|
||||
for _, r := range filename {
|
||||
|
||||
Reference in New Issue
Block a user