feat(app) implemented lab 4

This commit is contained in:
imadev
2026-11-06 19:06:21 +03:00
parent 68968c4a3a
commit b4239f8cf0
63 changed files with 3944 additions and 137 deletions
+105
View File
@@ -0,0 +1,105 @@
// Command reconciler periodically removes abandoned attachment objects and
// pending metadata rows. It is safe to run concurrently with the API and safe
// to re-run; see internal/reconcile for the grace-period rules.
package main
import (
"context"
"flag"
"log"
"log/slog"
"os/signal"
"syscall"
"time"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/config"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/database"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/reconcile"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
)
func main() {
once := flag.Bool("once", false, "run one reconciliation pass and exit")
dryRun := flag.Bool("dry-run", false, "log what would be removed without changing either store")
grace := flag.Duration("grace", 0, "override RECONCILE_GRACE_SECONDS for this run")
interval := flag.Duration("interval", 0, "override RECONCILE_INTERVAL_SECONDS for this run")
flag.Parse()
cfg, err := config.Load()
if err != nil {
log.Fatalf("configuration: %v", err)
}
if *grace > 0 {
cfg.ReconcileGrace = *grace
}
if *interval > 0 {
cfg.ReconcileInterval = *interval
}
rootCtx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
pool, err := database.Connect(rootCtx, cfg.DatabaseURL, cfg.DBMaxConns, cfg.StartupWait)
if err != nil {
log.Fatalf("database: %v", err)
}
defer pool.Close()
objects, err := objectstore.New(objectstore.Config{
Endpoint: cfg.S3Endpoint,
Region: cfg.S3Region,
Bucket: cfg.S3Bucket,
AccessKey: cfg.S3AccessKey,
SecretKey: cfg.S3SecretKey,
})
if err != nil {
log.Fatalf("object storage: %v", err)
}
worker := reconcile.New(
store.New(pool, cfg.DBTimeout, cfg.IdempotencyTTL),
objects,
cfg.ReconcileGrace,
slog.Default(),
)
if *once {
stats, runErr := worker.RunOnce(rootCtx, *dryRun)
logStats(stats, *dryRun)
if runErr != nil {
log.Fatalf("reconcile: %v", runErr)
}
return
}
for {
stats, runErr := worker.RunOnce(rootCtx, *dryRun)
if runErr != nil {
slog.Error("reconcile pass failed", "err", runErr)
} else {
logStats(stats, *dryRun)
}
// A timer avoids the ticker behaviour where missed ticks accumulate:
// the next pass starts one full interval after this one finished.
timer := time.NewTimer(cfg.ReconcileInterval)
select {
case <-rootCtx.Done():
timer.Stop()
return
case <-timer.C:
}
}
}
func logStats(stats reconcile.Stats, dryRun bool) {
slog.Info("reconcile pass complete",
"dry_run", dryRun,
"stale_pending", stats.StalePending,
"orphan_objects", stats.OrphanObjects,
"deleted_rows", stats.DeletedRows,
"deleted_objects", stats.DeletedObjects,
"skipped", stats.Skipped,
)
}
+7 -1
View File
@@ -11,6 +11,7 @@ import (
"time"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/attachments"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/config"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/database"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/health"
@@ -49,7 +50,12 @@ func main() {
}
defer sessions.Close()
app := internal.New(cfg, pool, sessions)
attachments.CleanupStaleUploads(cfg.UploadTempDir, time.Hour)
app, err := internal.New(cfg, pool, sessions)
if err != nil {
log.Fatalf("application: %v", err)
}
addr := net.JoinHostPort(cfg.Host, cfg.Port)
listenErr := make(chan error, 1)
+19 -1
View File
@@ -3,15 +3,34 @@ module gitflic.ru/project/bulla-max/mtusi2026lab
go 1.27.1
require (
github.com/aws/aws-sdk-go-v2 v1.47.5
github.com/aws/aws-sdk-go-v2/config v1.33.13
github.com/aws/aws-sdk-go-v2/credentials v1.20.13
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6
github.com/aws/smithy-go v1.28.7
github.com/gofiber/fiber/v3 v3.5.0
github.com/golang-migrate/migrate/v4 v4.20.1
github.com/google/uuid v1.6.0
github.com/jackc/pgx/v5 v5.11.0
github.com/redis/go-redis/v9 v9.23.0
golang.org/x/crypto v0.57.0
golang.org/x/sync v0.24.0
)
require (
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.39.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/gofiber/schema v1.8.8 // indirect
github.com/gofiber/utils/v2 v2.6.1 // indirect
@@ -29,7 +48,6 @@ require (
github.com/valyala/fasthttp v1.74.0 // indirect
go.uber.org/atomic v1.12.0 // indirect
golang.org/x/net v0.59.0 // indirect
golang.org/x/sync v0.23.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
golang.org/x/tools v0.50.0 // indirect
+38 -2
View File
@@ -2,6 +2,42 @@ github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEK
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4=
github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 h1:MW0Iql1SOTgQPj8mCxrU/eH/K/AIDWi08ItOsXaWRLU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25/go.mod h1:+ucVYmHYB3JqVueY0E1RHahdXAnKQbIHqybSbvOUbOM=
github.com/aws/aws-sdk-go-v2/config v1.33.13 h1:u52x6ISQOx172tpwM0VRxhJzEsTJYpMcEYUGM2A1wrw=
github.com/aws/aws-sdk-go-v2/config v1.33.13/go.mod h1:/ixtz6XPHb42zekjQ0LqD9XjTpTysNWhIGwufY1OzI4=
github.com/aws/aws-sdk-go-v2/credentials v1.20.13 h1:j4z41OtspmQ3ftQycaDunJ26b69a9SHfaSVUPqgRv8k=
github.com/aws/aws-sdk-go-v2/credentials v1.20.13/go.mod h1:SgPA1oS3b+dlMxZb618jDmGNscpXSCaxz15igS52OlE=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo=
github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng=
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 h1:GACgECI422Xt5n4Pojr3O0aocPv8lfH4PJe2Aon5Wv8=
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9/go.mod h1:lPfnRkHpn1qWZJLbVEw10rIqSuCee1pNQTyGkUjAYh8=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs=
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 h1:RS1Wt+K0r4X0ci9Vf4JYTUfHzsQihYA2LgGJeEneKpk=
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8/go.mod h1:adMnTy5kyoUW86jhXnGSVc/mvMQI4GZd1qYWMeaS8M8=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 h1:to7+Vr3rLkpYRwVKJ5QRriI0eE306vrVN/6TnVHgxvY=
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6/go.mod h1:Jc89ASa2k7HMAskj3bSnt8bf+k+p2IApzkoXnEjkgUw=
github.com/aws/aws-sdk-go-v2/service/signin v1.12.1 h1:WZLC+k/CZx11kZGQDqu9mpA8yhiXGX2I//jhuJE1FDo=
github.com/aws/aws-sdk-go-v2/service/signin v1.12.1/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y=
github.com/aws/aws-sdk-go-v2/service/sso v1.39.1 h1:4VB/dnDo7pmXSLQMszSoBzWndENuc7mzwTpD6lQn77A=
github.com/aws/aws-sdk-go-v2/service/sso v1.39.1/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1 h1:w8yui2ZqdHmGwVBDGvazoIvKw/DnLENBLkYnN8q8iqo=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.1/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps=
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ=
github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw=
github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE=
github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
@@ -119,8 +155,8 @@ golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
+44 -12
View File
@@ -5,14 +5,15 @@ package internal
import (
"errors"
"log/slog"
"time"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/attachments"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/auth"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/chats"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/config"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/health"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/messages"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/session"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/users"
@@ -22,15 +23,33 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
)
func New(cfg config.Config, pool *pgxpool.Pool, sessions *session.Store) *fiber.App {
func New(cfg config.Config, pool *pgxpool.Pool, sessions *session.Store) (*fiber.App, error) {
st := store.New(pool, cfg.DBTimeout, cfg.IdempotencyTTL)
objects, err := objectstore.New(objectstore.Config{
Endpoint: cfg.S3Endpoint,
Region: cfg.S3Region,
Bucket: cfg.S3Bucket,
AccessKey: cfg.S3AccessKey,
SecretKey: cfg.S3SecretKey,
})
if err != nil {
return nil, err
}
app := fiber.New(fiber.Config{
ReadTimeout: 10 * time.Second,
WriteTimeout: 10 * time.Second,
IdleTimeout: 60 * time.Second,
BodyLimit: 1 * 1024 * 1024,
ErrorHandler: handleError,
ReadTimeout: cfg.ReadTimeout,
WriteTimeout: cfg.WriteTimeout,
IdleTimeout: cfg.IdleTimeout,
// BodyLimit is the coarse HTTP envelope. The attachment handler and
// BindJSON enforce their own smaller budgets.
BodyLimit: cfg.HTTPBodyLimit,
// Without StreamRequestBody fasthttp buffers the entire body
// before the handler runs. Streaming lets the upload path copy in
// 32 KiB chunks and stops a 10 MiB body from becoming a per-request
// allocation even when the request is later rejected.
StreamRequestBody: true,
ErrorHandler: handleError,
})
app.Use(recover.New())
@@ -40,15 +59,25 @@ func New(cfg config.Config, pool *pgxpool.Pool, sessions *session.Store) *fiber.
return c.Next()
})
healthHandler := health.NewHandler(st, sessions)
healthHandler := health.NewHandler(st, sessions, objects)
healthRoutes := app.Group("/health")
healthRoutes.Get("/live", healthHandler.Live)
healthRoutes.Get("/ready", healthHandler.Ready)
authHandler := auth.NewHandler(st, sessions)
usersHandler := users.NewHandler(st)
chatsHandler := chats.NewHandler(st)
messagesHandler := messages.NewHandler(st)
usersHandler := users.NewHandler(st, config.MaxJSONBodyBytes)
chatsHandler := chats.NewHandler(st, config.MaxJSONBodyBytes)
messagesHandler := messages.NewHandler(st, config.MaxJSONBodyBytes)
attachmentsHandler := attachments.NewHandler(
st,
objects,
attachments.NewValidator(cfg.MaxImagePixels, cfg.ImageDecodeBytes),
cfg.MaxUploadBytes,
cfg.UploadDrainLimit,
cfg.UploadTimeout,
cfg.DownloadTimeout,
cfg.UploadTempDir,
)
api := app.Group("/api/v1")
api.Post("/users", usersHandler.CreateUser)
@@ -62,8 +91,11 @@ func New(cfg config.Config, pool *pgxpool.Pool, sessions *session.Store) *fiber.
protected.Get("/chats/:chat_id", chatsHandler.GetChat)
protected.Get("/chats/:chat_id/messages", messagesHandler.ListMessages)
protected.Post("/chats/:chat_id/messages", messagesHandler.CreateMessage)
protected.Post("/chats/:chat_id/attachments", attachmentsHandler.Upload)
protected.Get("/attachments/:attachment_id", attachmentsHandler.Get)
protected.Get("/attachments/:attachment_id/content", attachmentsHandler.Download)
return app
return app, nil
}
// handleError is the only writer of error responses: handlers and the store
+329
View File
@@ -0,0 +1,329 @@
package attachments
import (
"bufio"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"log/slog"
"mime"
"os"
"strconv"
"time"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/model"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
"github.com/gofiber/fiber/v3"
"github.com/google/uuid"
)
const copyBufferSize = 32 * 1024
type Handler struct {
store *store.Store
objects *objectstore.Client
validator *Validator
maxUploadBytes int64
drainLimit int64
uploadTimeout time.Duration
downloadTimeout time.Duration
tempDir string
}
func NewHandler(st *store.Store, client *objectstore.Client, validator *Validator, maxUploadBytes, drainLimit int64, uploadTimeout, downloadTimeout time.Duration, tempDir string) *Handler {
return &Handler{
store: st,
objects: client,
validator: validator,
maxUploadBytes: maxUploadBytes,
drainLimit: drainLimit,
uploadTimeout: uploadTimeout,
downloadTimeout: downloadTimeout,
tempDir: tempDir,
}
}
// Upload stores raw PNG/JPEG bytes. The order is deliberate:
// 1. authorize before reading the body;
// 2. stream the body to a bounded temporary file while hashing;
// 3. validate pixels and decode;
// 4. commit a pending metadata row;
// 5. PUT the object;
// 6. mark the row ready.
//
// A 201 is therefore returned only after both the object and its metadata are
// durable. Every failure between 4 and 6 has a compensation path, and the
// reconciler covers crashes that prevent compensation from running.
func (h *Handler) Upload(c fiber.Ctx) error {
userID := common.MustUserID(c)
chatID, err := common.PathUUID(c, "chat_id")
if err != nil {
return err
}
// Authorization precedes any body I/O: an outsider cannot make the server
// read a large upload before learning that the chat is not theirs.
isMember, err := h.store.IsMember(c.Context(), userID, chatID)
if err != nil {
return err
}
if !isMember {
return common.NotFound("chat not found")
}
filename := c.Query("filename")
if err := common.ValidateFilename(filename); err != nil {
return err
}
contentType, _, err := mime.ParseMediaType(c.Get(fiber.HeaderContentType))
if err != nil || (contentType != "image/png" && contentType != "image/jpeg") {
return common.UnsupportedMedia("Content-Type must be image/png or image/jpeg")
}
// Reject an obviously oversized request before allocating a temp file or
// decoding anything. Chunked requests are caught by the streaming counter
// below.
//
// AGENT: the declared body is drained first, but only within the bounded
// HTTP envelope (upload limit plus the JSON body slack). If the server
// closed immediately, the client would often see a connection reset instead
// of the required 413 while it was still sending. Draining is bounded and
// uses a 32 KiB buffer, so it does not weaken the memory guarantee.
if rawLength := c.Get(fiber.HeaderContentLength); rawLength != "" {
declared, parseErr := strconv.ParseInt(rawLength, 10, 64)
if parseErr != nil || declared < 0 {
return common.BadRequest("invalid Content-Length")
}
if declared > h.maxUploadBytes {
if declared <= h.drainLimit {
_, _ = io.CopyBuffer(io.Discard, io.LimitReader(requestBody(c), declared), make([]byte, copyBufferSize))
}
return common.PayloadTooLarge("upload exceeds the byte limit")
}
}
if err = os.MkdirAll(h.tempDir, 0o700); err != nil {
slog.Error("create upload temp dir", "err", err)
return common.Internal()
}
file, err := os.CreateTemp(h.tempDir, "upload-*.part")
if err != nil {
slog.Error("create upload temp file", "err", err)
return common.Internal()
}
filePath := file.Name()
defer func() {
_ = file.Close()
_ = os.Remove(filePath)
}()
// AGENT: the 32 KiB copy buffer is the only request-sized allocation here.
// The body is never read with io.ReadAll, so ten simultaneous 10 MiB
// uploads do not require ten 10 MiB Go slices.
var reader io.Reader = requestBody(c)
hasher := sha256.New()
copyBuffer := make([]byte, copyBufferSize)
written, err := io.CopyBuffer(io.MultiWriter(file, hasher), io.LimitReader(reader, h.maxUploadBytes+1), copyBuffer)
if err != nil {
return common.BadRequest("incomplete upload body")
}
if written == 0 {
return common.UnsupportedMedia("upload body is empty")
}
if written > h.maxUploadBytes {
return common.PayloadTooLarge("upload exceeds the byte limit")
}
ctx, cancel := context.WithTimeout(c.Context(), h.uploadTimeout)
defer cancel()
info, err := h.validator.Validate(ctx, file, contentType)
if err != nil {
return err
}
attachmentID := uuid.New()
attachment := model.Attachment{
ID: attachmentID,
ChatID: chatID,
OwnerID: userID,
Filename: filename,
ContentType: info.ContentType,
SizeBytes: written,
SHA256: hex.EncodeToString(hasher.Sum(nil)),
Status: "pending",
Variants: []model.Variant{},
ErrorCode: nil,
CreatedAt: common.NowUTC(),
UpdatedAt: common.NowUTC(),
// AGENT: the key is server-generated from UUIDs and never contains the
// client filename. That keeps traversal and collisions out of the
// object store even before application authorization is considered.
ObjectKey: objectKey(chatID, attachmentID),
}
if err = h.store.CreatePendingAttachment(ctx, attachment); err != nil {
return err
}
if _, err = file.Seek(0, 0); err != nil {
h.compensate(attachment.ID, attachment.ObjectKey)
return common.BadRequest("cannot read upload body")
}
if err = h.objects.Put(ctx, attachment.ObjectKey, file, written, attachment.ContentType); err != nil {
slog.Error("store attachment object", "err", err, "attachment_id", attachment.ID)
h.compensate(attachment.ID, attachment.ObjectKey)
return common.Unavailable("object storage unavailable")
}
updatedAt, err := h.store.MarkAttachmentReady(ctx, attachment.ID)
if err != nil {
slog.Error("mark attachment ready", "err", err, "attachment_id", attachment.ID)
h.compensate(attachment.ID, attachment.ObjectKey)
return common.Unavailable("attachment metadata unavailable")
}
attachment.Status = "ready"
attachment.UpdatedAt = common.RFCFormTime(updatedAt)
return c.Status(fiber.StatusCreated).JSON(attachment)
}
// compensate is best effort by design. If either delete fails, the pending row
// or the object remains visible to the reconciler, which retries after the
// configured grace period.
func (h *Handler) compensate(attachmentID uuid.UUID, key string) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := h.objects.Delete(ctx, key); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
slog.Error("compensate: delete object", "err", err, "attachment_id", attachmentID)
}
if err := h.store.DeletePendingAttachment(ctx, attachmentID); err != nil {
slog.Error("compensate: delete pending row", "err", err, "attachment_id", attachmentID)
}
}
func (h *Handler) Get(c fiber.Ctx) error {
userID := common.MustUserID(c)
attachmentID, err := common.PathUUID(c, "attachment_id")
if err != nil {
return err
}
attachment, err := h.store.GetAttachmentForUser(c.Context(), userID, attachmentID)
if err != nil {
return err
}
if attachment.Status != "ready" {
return common.Conflict("attachment is not ready")
}
return c.JSON(attachment)
}
func (h *Handler) Download(c fiber.Ctx) error {
userID := common.MustUserID(c)
attachmentID, err := common.PathUUID(c, "attachment_id")
if err != nil {
return err
}
variant := c.Query("variant", "original")
if variant == "" {
variant = "original"
}
if variant != "original" {
return common.BadRequest("unsupported variant")
}
attachment, err := h.store.GetAttachmentForUser(c.Context(), userID, attachmentID)
if err != nil {
return err
}
if attachment.Status != "ready" {
return common.Conflict("attachment is not ready")
}
ctx, cancel := context.WithTimeout(c.Context(), h.downloadTimeout)
body, size, err := h.objects.Get(ctx, attachment.ObjectKey)
if err != nil {
cancel()
if errors.Is(err, objectstore.ErrNotFound) {
slog.Error("attachment object missing", "attachment_id", attachment.ID)
return common.NotFound("attachment content not found")
}
slog.Error("read attachment object", "err", err, "attachment_id", attachment.ID)
return common.Unavailable("object storage unavailable")
}
if size != attachment.SizeBytes {
_ = body.Close()
cancel()
slog.Error("attachment object size mismatch",
"attachment_id", attachment.ID, "metadata_size", attachment.SizeBytes, "object_size", size)
return common.Unavailable("attachment content is inconsistent")
}
c.Set(fiber.HeaderContentType, attachment.ContentType)
c.Set(fiber.HeaderContentLength, strconv.FormatInt(attachment.SizeBytes, 10))
c.Set(fiber.HeaderContentDisposition, mime.FormatMediaType("inline", map[string]string{"filename": attachment.Filename}))
c.Set(fiber.HeaderCacheControl, "private, no-store")
c.Set("X-Content-Type-Options", "nosniff")
// AGENT: SendStreamWriter copies the S3 body straight to the response. The
// API never materializes the whole object in RAM and never redirects the
// client to a bucket URL, so authorization is enforced on every byte.
//
// The context and body are released inside the callback, not by a defer in
// this handler: the handler returns before fasthttp copies the stream, and
// canceling the S3 request context earlier truncates the response.
return c.SendStreamWriter(func(writer *bufio.Writer) {
defer cancel()
defer func() { _ = body.Close() }()
if _, copyErr := io.Copy(writer, body); copyErr != nil {
slog.Error("stream attachment to client", "err", copyErr, "attachment_id", attachment.ID)
}
})
}
func requestBody(c fiber.Ctx) io.Reader {
if reader := c.Request().BodyStream(); reader != nil {
return reader
}
return bytes.NewReader(c.Body())
}
func objectKey(chatID, attachmentID uuid.UUID) string {
return fmt.Sprintf("attachments/%s/%s", chatID, attachmentID)
}
// CleanupStaleUploads removes temp files abandoned by a previous crash. A
// running request removes its own file in a defer; this only handles the
// process-died-mid-upload case.
func CleanupStaleUploads(dir string, olderThan time.Duration) {
entries, err := os.ReadDir(dir)
if err != nil {
return
}
cutoff := time.Now().Add(-olderThan)
for _, entry := range entries {
if entry.IsDir() {
continue
}
info, infoErr := entry.Info()
if infoErr != nil || info.ModTime().After(cutoff) {
continue
}
_ = os.Remove(dir + string(os.PathSeparator) + entry.Name())
}
}
+129
View File
@@ -0,0 +1,129 @@
// Package attachments implements the synchronous upload, metadata lookup and
// authorized download path for chat attachments.
package attachments
import (
"context"
"errors"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"mime"
"net/http"
"os"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
"golang.org/x/sync/semaphore"
)
type Validator struct {
maxPixels int64
decodeBudget *semaphore.Weighted
budgetBytes int64
}
type ImageInfo struct {
ContentType string
Width int
Height int
}
func NewValidator(maxPixels, decodeBudgetBytes int64) *Validator {
return &Validator{
maxPixels: maxPixels,
decodeBudget: semaphore.NewWeighted(decodeBudgetBytes),
budgetBytes: decodeBudgetBytes,
}
}
// Validate checks the declared media type against the real bytes and then
// decodes the image under a shared memory budget. The decode is what turns a
// structurally broken or truncated file into 415 instead of a saved object.
//
// AGENT: image.DecodeConfig first reads only the header. This is the important
// ordering: a decompression bomb is rejected from width×height before the
// decoder is allowed to allocate pixel memory.
func (v *Validator) Validate(ctx context.Context, file *os.File, declared string) (ImageInfo, error) {
mediaType, _, err := mime.ParseMediaType(declared)
if err != nil {
return ImageInfo{}, common.UnsupportedMedia("Content-Type must be image/png or image/jpeg")
}
if mediaType != "image/png" && mediaType != "image/jpeg" {
return ImageInfo{}, common.UnsupportedMedia("Content-Type must be image/png or image/jpeg")
}
if _, err = file.Seek(0, 0); err != nil {
return ImageInfo{}, common.BadRequest("cannot read upload body")
}
head := make([]byte, 512)
n, readErr := file.Read(head)
if readErr != nil && !errors.Is(readErr, io.EOF) {
return ImageInfo{}, common.BadRequest("cannot read upload body")
}
detected := http.DetectContentType(head[:n])
if detected != mediaType {
return ImageInfo{}, common.UnsupportedMedia("file content does not match Content-Type")
}
if _, err = file.Seek(0, 0); err != nil {
return ImageInfo{}, common.BadRequest("cannot read upload body")
}
config, format, err := image.DecodeConfig(file)
if err != nil {
return ImageInfo{}, common.UnsupportedMedia("invalid or corrupted image")
}
if format != "png" && format != "jpeg" {
return ImageInfo{}, common.UnsupportedMedia("only PNG and JPEG images are accepted")
}
if (format == "png" && mediaType != "image/png") || (format == "jpeg" && mediaType != "image/jpeg") {
return ImageInfo{}, common.UnsupportedMedia("file content does not match Content-Type")
}
if config.Width <= 0 || config.Height <= 0 {
return ImageInfo{}, common.UnsupportedMedia("invalid or corrupted image")
}
pixels := int64(config.Width) * int64(config.Height)
if pixels <= 0 || pixels > v.maxPixels {
return ImageInfo{}, common.PayloadTooLarge("image has too many pixels")
}
// Approximate decoded RGBA size. The semaphore caps the sum across
// concurrent requests; a single 40 Mp image needs ~160 MiB, so a lower
// configured budget is a deliberate operator choice and becomes a 413.
weight := pixels * 4
if weight > v.budgetBytes {
return ImageInfo{}, common.PayloadTooLarge("image is too large to validate")
}
if err = v.decodeBudget.Acquire(ctx, weight); err != nil {
return ImageInfo{}, common.Unavailable("image validation is busy")
}
defer v.decodeBudget.Release(weight)
if _, err = file.Seek(0, 0); err != nil {
return ImageInfo{}, common.BadRequest("cannot read upload body")
}
info, decodeErr := safeDecode(file)
if decodeErr != nil {
return ImageInfo{}, common.UnsupportedMedia("invalid or corrupted image")
}
bounds := info.Bounds()
if bounds.Dx() != config.Width || bounds.Dy() != config.Height {
return ImageInfo{}, common.UnsupportedMedia("invalid or corrupted image")
}
return ImageInfo{ContentType: mediaType, Width: config.Width, Height: config.Height}, nil
}
// safeDecode converts a decoder panic into an ordinary error. A malformed
// client file must never crash the process or become a 500.
func safeDecode(file *os.File) (img image.Image, err error) {
defer func() {
if recovered := recover(); recovered != nil {
img = nil
err = errors.New("image decoder panic")
}
}()
img, _, err = image.Decode(file)
return img, err
}
+134
View File
@@ -0,0 +1,134 @@
package attachments
import (
"bytes"
"context"
"image"
"image/color"
"image/jpeg"
"image/png"
"os"
"path/filepath"
"testing"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
)
// writeTemp stores raw bytes in a temporary file and returns its path. The
// validator works on *os.File so it can seek, sniff and decode the same body
// without loading it into memory twice.
func writeTemp(t *testing.T, name string, data []byte) *os.File {
t.Helper()
path := filepath.Join(t.TempDir(), name)
if err := os.WriteFile(path, data, 0o600); err != nil {
t.Fatalf("write %s: %v", path, err)
}
file, err := os.Open(path)
if err != nil {
t.Fatalf("open %s: %v", path, err)
}
t.Cleanup(func() { _ = file.Close() })
return file
}
func encodePNG(t *testing.T, width, height int) []byte {
t.Helper()
img := image.NewRGBA(image.Rect(0, 0, width, height))
for x := 0; x < width; x++ {
for y := 0; y < height; y++ {
img.Set(x, y, color.RGBA{R: uint8(x), G: uint8(y), B: 0x40, A: 0xff})
}
}
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
t.Fatalf("encode png: %v", err)
}
return buf.Bytes()
}
func TestValidateAcceptsPNGAndJPEG(t *testing.T) {
validator := NewValidator(40_000_000, 256<<20)
ctx := context.Background()
pngFile := writeTemp(t, "image.png", encodePNG(t, 8, 6))
info, err := validator.Validate(ctx, pngFile, "image/png")
if err != nil {
t.Fatalf("valid png rejected: %v", err)
}
if info.ContentType != "image/png" || info.Width != 8 || info.Height != 6 {
t.Fatalf("unexpected png info: %+v", info)
}
img := image.NewRGBA(image.Rect(0, 0, 4, 3))
var buf bytes.Buffer
if err = jpeg.Encode(&buf, img, nil); err != nil {
t.Fatalf("encode jpeg: %v", err)
}
jpegFile := writeTemp(t, "image.jpg", buf.Bytes())
info, err = validator.Validate(ctx, jpegFile, "image/jpeg; charset=binary")
if err != nil {
t.Fatalf("valid jpeg rejected: %v", err)
}
if info.ContentType != "image/jpeg" || info.Width != 4 || info.Height != 3 {
t.Fatalf("unexpected jpeg info: %+v", info)
}
}
func TestValidateRejectsMismatchedOrBrokenContent(t *testing.T) {
validator := NewValidator(40_000_000, 256<<20)
ctx := context.Background()
pngBytes := encodePNG(t, 8, 6)
cases := []struct {
name string
data []byte
declared string
code string
}{
{"png declared as jpeg", pngBytes, "image/jpeg", common.ErrorCodeUnsupportedMedia},
{"text declared as png", []byte("this is not an image at all"), "image/png", common.ErrorCodeUnsupportedMedia},
{"unsupported declared type", pngBytes, "image/gif", common.ErrorCodeUnsupportedMedia},
{"garbage declared type", pngBytes, "not a media type", common.ErrorCodeUnsupportedMedia},
{"truncated png", pngBytes[:len(pngBytes)/2], "image/png", common.ErrorCodeUnsupportedMedia},
{"empty body", nil, "image/png", common.ErrorCodeUnsupportedMedia},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
file := writeTemp(t, "upload", tc.data)
if _, err := validator.Validate(ctx, file, tc.declared); !common.IsCode(err, tc.code) {
t.Fatalf("Validate(%q) = %v, want code %s", tc.name, err, tc.code)
}
})
}
}
func TestValidateRejectsTooManyPixelsBeforeDecoding(t *testing.T) {
validator := NewValidator(100, 256<<20)
file := writeTemp(t, "big.png", encodePNG(t, 20, 20))
_, err := validator.Validate(context.Background(), file, "image/png")
if !common.IsCode(err, common.ErrorCodePayloadTooLarge) {
t.Fatalf("Validate = %v, want %s", err, common.ErrorCodePayloadTooLarge)
}
}
func TestValidateRejectsImageAboveDecodeBudget(t *testing.T) {
// A 100-pixel image needs ~400 bytes of RGBA memory, more than this budget.
validator := NewValidator(40_000_000, 100)
file := writeTemp(t, "budget.png", encodePNG(t, 10, 10))
_, err := validator.Validate(context.Background(), file, "image/png")
if !common.IsCode(err, common.ErrorCodePayloadTooLarge) {
t.Fatalf("Validate = %v, want %s", err, common.ErrorCodePayloadTooLarge)
}
}
func TestSafeDecodeConvertsPanicToError(t *testing.T) {
// image.Decode does not panic on arbitrary bytes, so this only guards the
// contract that safeDecode always returns an error instead of crashing.
file := writeTemp(t, "junk", []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0xff})
if _, err := safeDecode(file); err == nil {
t.Fatal("safeDecode accepted junk bytes")
}
}
+5 -4
View File
@@ -9,11 +9,12 @@ import (
)
type Handler struct {
store *store.Store
store *store.Store
jsonBodyLimit int64
}
func NewHandler(st *store.Store) *Handler {
return &Handler{store: st}
func NewHandler(st *store.Store, jsonBodyLimit int64) *Handler {
return &Handler{store: st, jsonBodyLimit: jsonBodyLimit}
}
type CreateChatRequest struct {
@@ -25,7 +26,7 @@ func (h *Handler) CreateChat(c fiber.Ctx) error {
userID := common.MustUserID(c)
var req CreateChatRequest
if err := common.BindJSON(c, &req); err != nil {
if err := common.BindJSON(c, &req, h.jsonBodyLimit); err != nil {
return err
}
@@ -0,0 +1,64 @@
package common
import (
"strings"
"testing"
"github.com/google/uuid"
)
func TestValidateFilename(t *testing.T) {
valid := []string{"photo.png", "фото 1.jpeg", "a", strings.Repeat("a", 255)}
for _, name := range valid {
if err := ValidateFilename(name); err != nil {
t.Errorf("ValidateFilename(%q) = %v, want nil", name, err)
}
}
invalid := []string{
"", " ", ".", "..", "dir/photo.png", `dir\photo.png`,
"nul\x00byte", "line\nbreak", strings.Repeat("a", 256),
}
for _, name := range invalid {
if err := ValidateFilename(name); err == nil {
t.Errorf("ValidateFilename(%q) = nil, want error", name)
}
}
}
func TestValidateAttachmentIDs(t *testing.T) {
ids := make([]uuid.UUID, 10)
for index := range ids {
ids[index] = uuid.New()
}
if err := ValidateAttachmentIDs(ids); err != nil {
t.Fatalf("ten unique ids rejected: %v", err)
}
if err := ValidateAttachmentIDs(append(ids, uuid.New())); err == nil {
t.Fatal("eleven ids accepted")
}
if err := ValidateAttachmentIDs([]uuid.UUID{ids[0], ids[0]}); err == nil {
t.Fatal("duplicate ids accepted")
}
if err := ValidateAttachmentIDs(nil); err != nil {
t.Fatalf("empty list rejected: %v", err)
}
}
func TestValidateMessageText(t *testing.T) {
if err := ValidateMessageText("", 1); err != nil {
t.Fatalf("empty text with attachment rejected: %v", err)
}
if err := ValidateMessageText(" ", 1); err != nil {
t.Fatalf("blank text with attachment rejected: %v", err)
}
if err := ValidateMessageText("", 0); err == nil {
t.Fatal("empty text without attachment accepted")
}
if err := ValidateMessageText(strings.Repeat("ж", 4000), 0); err != nil {
t.Fatalf("4000 code points rejected: %v", err)
}
if err := ValidateMessageText(strings.Repeat("ж", 4001), 1); err == nil {
t.Fatal("4001 code points accepted")
}
}
+30 -2
View File
@@ -9,12 +9,40 @@ import (
"github.com/gofiber/fiber/v3"
)
func BindJSON(c fiber.Ctx, dst any) error {
// ReadBodyLimited reads at most limit bytes from the request body. The caller
// must use it instead of c.Body() when the endpoint has a smaller budget than
// the process-wide Fiber.BodyLimit. The limit itself belongs to the caller
// (config.MaxJSONBodyBytes for JSON endpoints).
//
// with StreamRequestBody enabled fasthttp exposes the network body as a
// stream, so io.ReadAll over a LimitReader allocates only the bytes the
// endpoint actually accepts instead of the whole upload limit.
func ReadBodyLimited(c fiber.Ctx, limit int64) ([]byte, error) {
var reader io.Reader = c.Request().BodyStream()
if reader == nil {
reader = bytes.NewReader(c.Body())
}
data, err := io.ReadAll(io.LimitReader(reader, limit+1))
if err != nil {
return nil, BadRequest("incomplete request body")
}
if int64(len(data)) > limit {
return nil, PayloadTooLarge("request body is too large")
}
return data, nil
}
func BindJSON(c fiber.Ctx, dst any, limit int64) error {
if contentType := c.Get(fiber.HeaderContentType); !strings.HasPrefix(contentType, "application/json") {
return UnsupportedMedia("invalid JSON body")
}
body := c.Body()
body, err := ReadBodyLimited(c, limit)
if err != nil {
return err
}
if len(body) == 0 {
return BadRequest("request body is empty")
}
+4
View File
@@ -79,6 +79,10 @@ func UnsupportedMedia(message string) *AppError {
return NewAppError(http.StatusUnsupportedMediaType, ErrorCodeUnsupportedMedia, message)
}
func PayloadTooLarge(message string) *AppError {
return NewAppError(http.StatusRequestEntityTooLarge, ErrorCodePayloadTooLarge, message)
}
func Internal() *AppError {
return NewAppError(http.StatusInternalServerError, ErrorCodeInternal, "Internal server error")
}
+41 -4
View File
@@ -3,6 +3,7 @@ package common
import (
"regexp"
"strings"
"unicode"
"unicode/utf8"
"github.com/google/uuid"
@@ -62,13 +63,49 @@ func ValidateChatTitle(title string) error {
return nil
}
func ValidateMessageText(text string) error {
if blank(text) {
func ValidateMessageText(text string, attachmentCount int) error {
if attachmentCount == 0 && blank(text) {
return BadRequest("text must not be empty or only whitespace")
}
if !strLenBetween(text, 1, 4000) {
return BadRequest("text must be between 1 and 4000 Unicode code points")
if !strLenBetween(text, 0, 4000) {
return BadRequest("text must be between 0 and 4000 Unicode code points")
}
return nil
}
func ValidateAttachmentIDs(ids []uuid.UUID) error {
if len(ids) > 10 {
return BadRequest("attachment_ids must contain at most 10 items")
}
seen := make(map[uuid.UUID]bool, len(ids))
for _, id := range ids {
if seen[id] {
return BadRequest("attachment_ids must not contain duplicates")
}
seen[id] = true
}
return nil
}
func ValidateFilename(filename string) error {
if !strLenBetween(filename, 1, 255) {
return BadRequest("filename must be between 1 and 255 Unicode code points")
}
if blank(filename) {
return BadRequest("filename must not be empty or only whitespace")
}
if filename == "." || filename == ".." {
return BadRequest("filename is not allowed")
}
for _, r := range filename {
if r == '/' || r == '\\' || r == 0 || unicode.IsControl(r) {
return BadRequest("filename contains a forbidden character")
}
}
return nil
+130 -15
View File
@@ -5,11 +5,23 @@ package config
import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"time"
)
const (
// Configuration may make a limit stricter, never looser.
MaxUploadBytesHardCap = 10 * 1024 * 1024
MaxImagePixelsHardCap = 40_000_000
// MaxJSONBodyBytes is the body budget of every JSON endpoint. It is also
// the slack between the global HTTP body limit and the upload limit, so a
// smaller MAX_UPLOAD_BYTES cannot strangle JSON requests.
MaxJSONBodyBytes = 1 * 1024 * 1024
)
type Config struct {
Host string
Port string
@@ -24,6 +36,30 @@ type Config struct {
StartupWait time.Duration
RunMigrations bool
MigrateOnly bool
S3Endpoint string
S3Region string
S3Bucket string
S3AccessKey string
S3SecretKey string
UploadTimeout time.Duration
DownloadTimeout time.Duration
MaxUploadBytes int64
MaxImagePixels int64
ImageDecodeBytes int64
UploadTempDir string
// HTTP timeouts and body limits are derived once in Load so that app.go
// and the handlers never invent values of their own.
ReadTimeout time.Duration
WriteTimeout time.Duration
IdleTimeout time.Duration
UploadDrainLimit int64
HTTPBodyLimit int
ReconcileGrace time.Duration
ReconcileInterval time.Duration
}
func Load() (Config, error) {
@@ -45,17 +81,42 @@ func Load() (Config, error) {
return Config{}, err
}
cfg := Config{
Host: envOr("APP_HOST", "0.0.0.0"),
Port: envOr("APP_PORT", "8080"),
InstanceID: instance,
SessionTTL: time.Duration(sessionTTL) * time.Second,
IdempotencyTTL: time.Duration(idempotencyTTL) * time.Second,
DBMaxConns: int32(maxConns),
DBTimeout: 5 * time.Second,
RedisTimeout: 2 * time.Second,
StartupWait: 30 * time.Second,
maxUpload, err := envInt[int64]("MAX_UPLOAD_BYTES", MaxUploadBytesHardCap, 1, MaxUploadBytesHardCap)
if err != nil {
return Config{}, err
}
maxPixels, err := envInt[int64]("MAX_IMAGE_PIXELS", MaxImagePixelsHardCap, 1, MaxImagePixelsHardCap)
if err != nil {
return Config{}, err
}
decodeMB, err := envInt("IMAGE_DECODE_MEMORY_MB", 256, 32, 4096)
if err != nil {
return Config{}, err
}
cfg := Config{
Host: envOr("APP_HOST", "0.0.0.0"),
Port: envOr("APP_PORT", "8080"),
InstanceID: instance,
SessionTTL: time.Duration(sessionTTL) * time.Second,
IdempotencyTTL: time.Duration(idempotencyTTL) * time.Second,
DBMaxConns: int32(maxConns),
DBTimeout: 5 * time.Second,
RedisTimeout: 2 * time.Second,
StartupWait: 30 * time.Second,
S3Region: envOr("S3_REGION", "us-east-1"),
S3Bucket: envOr("S3_BUCKET", "messenger-attachments"),
ReadTimeout: 30 * time.Second,
WriteTimeout: 60 * time.Second,
IdleTimeout: 60 * time.Second,
UploadTimeout: 30 * time.Second,
DownloadTimeout: 60 * time.Second,
MaxUploadBytes: maxUpload,
MaxImagePixels: maxPixels,
ImageDecodeBytes: int64(decodeMB) * 1024 * 1024,
UploadTempDir: envOr("UPLOAD_TEMP_DIR", filepath.Join(os.TempDir(), "messenger-uploads")),
}
cfg.deriveHTTPLimits()
if cfg.RunMigrations, err = envBool("RUN_MIGRATIONS", true); err != nil {
return Config{}, err
@@ -63,6 +124,22 @@ func Load() (Config, error) {
if cfg.MigrateOnly, err = envBool("MIGRATE_ONLY", false); err != nil {
return Config{}, err
}
reconcileOnly, err := envBool("RECONCILE_ONLY", false)
if err != nil {
return Config{}, err
}
reconcileGrace, err := envInt("RECONCILE_GRACE_SECONDS", 900, 60, 0)
if err != nil {
return Config{}, err
}
cfg.ReconcileGrace = time.Duration(reconcileGrace) * time.Second
reconcileInterval, err := envInt("RECONCILE_INTERVAL_SECONDS", 300, 10, 0)
if err != nil {
return Config{}, err
}
cfg.ReconcileInterval = time.Duration(reconcileInterval) * time.Second
if cfg.DatabaseURL, err = secret("DATABASE_URL"); err != nil {
return Config{}, err
@@ -75,6 +152,29 @@ func Load() (Config, error) {
return cfg, nil
}
if cfg.S3Endpoint, err = secret("S3_ENDPOINT"); err != nil {
return Config{}, err
}
if cfg.S3Endpoint == "" {
return Config{}, fmt.Errorf("S3_ENDPOINT or S3_ENDPOINT_FILE is required")
}
if cfg.S3AccessKey, err = secret("S3_ACCESS_KEY_ID"); err != nil {
return Config{}, err
}
if cfg.S3AccessKey == "" {
return Config{}, fmt.Errorf("S3_ACCESS_KEY_ID or S3_ACCESS_KEY_ID_FILE is required")
}
if cfg.S3SecretKey, err = secret("S3_SECRET_ACCESS_KEY"); err != nil {
return Config{}, err
}
if cfg.S3SecretKey == "" {
return Config{}, fmt.Errorf("S3_SECRET_ACCESS_KEY or S3_SECRET_ACCESS_KEY_FILE is required")
}
if reconcileOnly {
return cfg, nil
}
if cfg.CacheURL, err = secret("CACHE_URL"); err != nil {
return Config{}, err
}
@@ -94,6 +194,15 @@ func Load() (Config, error) {
return cfg, nil
}
// deriveHTTPLimits is the single place that relates the three HTTP budgets:
// the transport gate must admit both a full upload and a full JSON body, and it
// must leave room for the handler to drain an oversized declared body so the
// client receives a 413 instead of a connection reset.
func (c *Config) deriveHTTPLimits() {
c.UploadDrainLimit = c.MaxUploadBytes + MaxJSONBodyBytes
c.HTTPBodyLimit = int(c.UploadDrainLimit)
}
func instanceID() (string, error) {
if value := strings.TrimSpace(os.Getenv("INSTANCE_ID")); value != "" {
return value, nil
@@ -120,21 +229,27 @@ func envBool(name string, fallback bool) (bool, error) {
return parsed, nil
}
// envInt parses an integer setting; max <= 0 means "no upper bound".
func envInt(name string, fallback, min, max int) (int, error) {
// envInt parses an integer setting into int or int64; max <= 0 means "no upper
// bound". Both widths are needed by Config, and one helper keeps validation
// identical for every numeric variable.
func envInt[T ~int | ~int64](name string, fallback, min, max T) (T, error) {
value := os.Getenv(name)
if value == "" {
return fallback, nil
}
parsed, err := strconv.Atoi(value)
if err != nil || parsed < min || (max > 0 && parsed > max) {
parsed, err := strconv.ParseInt(value, 10, 64)
if err != nil || parsed < int64(min) || (max > 0 && parsed > int64(max)) {
if max > 0 {
return 0, fmt.Errorf("%s must be an integer between %d and %d", name, min, max)
}
return 0, fmt.Errorf("%s must be an integer >= %d", name, min)
}
return parsed, nil
result := T(parsed)
if int64(result) != parsed {
return 0, fmt.Errorf("%s is out of range", name)
}
return result, nil
}
func envOr(name, fallback string) string {
+37
View File
@@ -25,3 +25,40 @@ func TestInstanceIDFallsBackToHostname(t *testing.T) {
t.Fatal("instanceID returned an empty value")
}
}
func TestDerivedHTTPLimits(t *testing.T) {
cfg := Config{MaxUploadBytes: 2 * 1024 * 1024}
cfg.deriveHTTPLimits()
if cfg.UploadDrainLimit != cfg.MaxUploadBytes+MaxJSONBodyBytes {
t.Fatalf("drain limit = %d, want upload + JSON body", cfg.UploadDrainLimit)
}
if cfg.HTTPBodyLimit != int(cfg.UploadDrainLimit) {
t.Fatalf("body limit = %d, want %d", cfg.HTTPBodyLimit, cfg.UploadDrainLimit)
}
if cfg.HTTPBodyLimit-int(cfg.MaxUploadBytes) != MaxJSONBodyBytes {
t.Fatal("transport gate would strangle a full JSON body")
}
}
func TestEnvIntHandlesBothWidths(t *testing.T) {
t.Setenv("TEST_ENV_INT", "42")
small, err := envInt("TEST_ENV_INT", 1, 1, 100)
if err != nil || small != 42 {
t.Fatalf("envInt = %d, %v; want 42, nil", small, err)
}
large, err := envInt[int64]("TEST_ENV_INT", 1, 1, 100)
if err != nil || large != 42 {
t.Fatalf("envInt[int64] = %d, %v; want 42, nil", large, err)
}
}
func TestEnvIntRejectsOutOfRange(t *testing.T) {
for _, value := range []string{"0", "101", "not-a-number"} {
t.Setenv("TEST_ENV_INT", value)
if _, err := envInt("TEST_ENV_INT", 1, 1, 100); err == nil {
t.Fatalf("envInt accepted %q", value)
}
}
}
@@ -0,0 +1,6 @@
DROP TABLE IF EXISTS message_attachments;
DROP TABLE IF EXISTS attachments;
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_len;
ALTER TABLE messages
ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000);
@@ -0,0 +1,46 @@
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_len;
ALTER TABLE messages
ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 0 AND 4000);
CREATE TABLE IF NOT EXISTS attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
filename text NOT NULL,
content_type text NOT NULL,
size_bytes bigint NOT NULL,
sha256 bytea NOT NULL,
object_key text NOT NULL,
status text NOT NULL,
error_code text,
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL,
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
CONSTRAINT attachments_sha256_len CHECK (octet_length(sha256) = 32),
CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready', 'failed')),
CONSTRAINT attachments_error_state CHECK (
(status = 'pending' AND error_code IS NULL) OR
(status = 'ready' AND error_code IS NULL) OR
(status = 'failed' AND error_code IS NOT NULL)
),
CONSTRAINT attachments_object_key_key UNIQUE (object_key)
);
CREATE INDEX IF NOT EXISTS attachments_chat_created_idx
ON attachments (chat_id, created_at, id);
CREATE INDEX IF NOT EXISTS attachments_status_created_idx
ON attachments (status, created_at);
CREATE TABLE IF NOT EXISTS message_attachments (
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT,
position integer NOT NULL,
PRIMARY KEY (message_id, attachment_id),
CONSTRAINT message_attachments_position CHECK (position >= 0),
CONSTRAINT message_attachments_position_key UNIQUE (message_id, position)
);
CREATE INDEX IF NOT EXISTS message_attachments_attachment_idx
ON message_attachments (attachment_id);
+56 -21
View File
@@ -3,27 +3,59 @@ package database
import (
"errors"
"io/fs"
"regexp"
"strconv"
"testing"
"github.com/golang-migrate/migrate/v4/source/iofs"
)
func TestEmbeddedMigrationsParse(t *testing.T) {
// latestMigrationVersion is the only value to bump when a migration is added.
// The rest of the test derives filenames and ordering from the directory.
const latestMigrationVersion = 3
var migrationFilePattern = regexp.MustCompile(`^(\d{4})_[a-z0-9_]+\.(up|down)\.sql$`)
func TestEmbeddedMigrationsAreCompleteAndContiguous(t *testing.T) {
entries, err := fs.ReadDir(migrationFiles, "migrations")
if err != nil {
t.Fatalf("read embedded migrations: %v", err)
}
names := make(map[string]bool, len(entries))
up := make(map[int]string, latestMigrationVersion)
down := make(map[int]string, latestMigrationVersion)
for _, entry := range entries {
names[entry.Name()] = true
if entry.IsDir() {
continue
}
name := entry.Name()
match := migrationFilePattern.FindStringSubmatch(name)
if match == nil {
t.Fatalf("migration %q does not match NNNN_name.{up,down}.sql", name)
}
version, parseErr := strconv.Atoi(match[1])
if parseErr != nil || version < 1 || version > latestMigrationVersion {
t.Fatalf("migration %q has version outside 1..%d", name, latestMigrationVersion)
}
files := up
if match[2] == "down" {
files = down
}
if previous, ok := files[version]; ok {
t.Fatalf("version %d has two %s files: %q and %q", version, match[2], previous, name)
}
files[version] = name
}
for _, want := range []string{
"0001_init.up.sql", "0001_init.down.sql",
"0002_message_idempotency.up.sql", "0002_message_idempotency.down.sql",
} {
if !names[want] {
t.Fatalf("migration %q is missing from the embedded filesystem", want)
for version := 1; version <= latestMigrationVersion; version++ {
if up[version] == "" {
t.Fatalf("version %d is missing an up migration", version)
}
if down[version] == "" {
t.Fatalf("version %d is missing a down migration", version)
}
}
@@ -37,17 +69,20 @@ func TestEmbeddedMigrationsParse(t *testing.T) {
if err != nil {
t.Fatalf("read first migration: %v", err)
}
if version != 1 {
t.Fatalf("first migration version = %d, want 1", version)
}
version, err = source.Next(version)
if err != nil {
t.Fatalf("read second migration: %v", err)
}
if version != 2 {
t.Fatalf("second migration version = %d, want 2", version)
}
if _, err := source.Next(version); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("expected 0002 to be the last migration, got %v", err)
for expected := 1; expected <= latestMigrationVersion; expected++ {
if version != uint(expected) {
t.Fatalf("migration order = %d, want %d", version, expected)
}
next, nextErr := source.Next(version)
if expected == latestMigrationVersion {
if !errors.Is(nextErr, fs.ErrNotExist) {
t.Fatalf("expected %d to be the last migration, got %v", version, nextErr)
}
break
}
if nextErr != nil {
t.Fatalf("read migration after %d: %v", version, nextErr)
}
version = next
}
}
+9 -2
View File
@@ -6,6 +6,7 @@ import (
"sync/atomic"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/session"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
"github.com/gofiber/fiber/v3"
@@ -14,10 +15,11 @@ import (
type Handler struct {
store *store.Store
sessions *session.Store
objects *objectstore.Client
}
func NewHandler(st *store.Store, sessions *session.Store) *Handler {
return &Handler{store: st, sessions: sessions}
func NewHandler(st *store.Store, sessions *session.Store, objects *objectstore.Client) *Handler {
return &Handler{store: st, sessions: sessions, objects: objects}
}
var shuttingDown atomic.Bool
@@ -50,5 +52,10 @@ func (h *Handler) Ready(c fiber.Ctx) error {
common.ErrorCodeUnavailable, "session store unavailable")
}
if err := h.objects.Ping(c.Context()); err != nil {
return common.SendError(c, fiber.StatusServiceUnavailable,
common.ErrorCodeUnavailable, "object storage unavailable")
}
return c.JSON(Status{Status: "ok"})
}
+13 -7
View File
@@ -6,18 +6,21 @@ import (
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
"github.com/gofiber/fiber/v3"
"github.com/google/uuid"
)
type Handler struct {
store *store.Store
store *store.Store
jsonBodyLimit int64
}
func NewHandler(st *store.Store) *Handler {
return &Handler{store: st}
func NewHandler(st *store.Store, jsonBodyLimit int64) *Handler {
return &Handler{store: st, jsonBodyLimit: jsonBodyLimit}
}
type CreateMessageRequest struct {
Text string `json:"text"`
Text string `json:"text"`
AttachmentIDs []uuid.UUID `json:"attachment_ids"`
}
func (h *Handler) CreateMessage(c fiber.Ctx) error {
@@ -34,14 +37,17 @@ func (h *Handler) CreateMessage(c fiber.Ctx) error {
}
var req CreateMessageRequest
if err := common.BindJSON(c, &req); err != nil {
if err := common.BindJSON(c, &req, h.jsonBodyLimit); err != nil {
return err
}
if err := common.ValidateMessageText(req.Text); err != nil {
if err := common.ValidateAttachmentIDs(req.AttachmentIDs); err != nil {
return err
}
if err := common.ValidateMessageText(req.Text, len(req.AttachmentIDs)); err != nil {
return err
}
message, err := h.store.CreateMessage(c.Context(), chatID, userID, req.Text, idempotencyKey)
message, err := h.store.CreateMessage(c.Context(), chatID, userID, req.Text, idempotencyKey, req.AttachmentIDs)
if err != nil {
return err
}
+36 -5
View File
@@ -31,16 +31,47 @@ func (c Chat) GetID() uuid.UUID { return c.ID }
func (c Chat) GetCreatedAt() time.Time { return time.Time(c.CreatedAt) }
type Message struct {
ID uuid.UUID `json:"id"`
ChatID uuid.UUID `json:"chat_id"`
SenderID uuid.UUID `json:"sender_id"`
Text string `json:"text"`
CreatedAt common.RFCFormTime `json:"created_at"`
ID uuid.UUID `json:"id"`
ChatID uuid.UUID `json:"chat_id"`
SenderID uuid.UUID `json:"sender_id"`
Text string `json:"text"`
CreatedAt common.RFCFormTime `json:"created_at"`
AttachmentIDs []uuid.UUID `json:"attachment_ids"`
}
func (m Message) GetID() uuid.UUID { return m.ID }
func (m Message) GetCreatedAt() time.Time { return time.Time(m.CreatedAt) }
type Attachment struct {
ID uuid.UUID `json:"id"`
ChatID uuid.UUID `json:"chat_id"`
OwnerID uuid.UUID `json:"owner_id"`
Filename string `json:"filename"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
SHA256 string `json:"sha256"`
Status string `json:"status"`
Variants []Variant `json:"variants"`
ErrorCode *string `json:"error_code"`
CreatedAt common.RFCFormTime `json:"created_at"`
UpdatedAt common.RFCFormTime `json:"updated_at"`
// ObjectKey is internal and never serialized.
ObjectKey string `json:"-"`
}
func (a Attachment) GetID() uuid.UUID { return a.ID }
func (a Attachment) GetCreatedAt() time.Time { return time.Time(a.CreatedAt) }
type Variant struct {
Name string `json:"name"`
ContentType string `json:"content_type"`
SizeBytes int64 `json:"size_bytes"`
Width int `json:"width"`
Height int `json:"height"`
SHA256 string `json:"sha256"`
}
type Session struct {
Token string `json:"token"`
TokenType string `json:"token_type"`
+176
View File
@@ -0,0 +1,176 @@
// Package objectstore is the only place that knows how to talk to the private S3
// bucket. PostgreSQL remains the source of truth for metadata; this package
// stores and returns opaque bytes only.
package objectstore
import (
"context"
"errors"
"io"
"net/http"
"strings"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
awsconfig "github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/smithy-go"
)
var ErrNotFound = errors.New("object not found")
type Config struct {
Endpoint string
Region string
Bucket string
AccessKey string
SecretKey string
}
type Client struct {
s3 *s3.Client
bucket string
}
// Object describes one key returned by List. It intentionally exposes only the
// fields the reconciler needs.
type Object struct {
Key string
Size int64
LastModified time.Time
}
func New(cfg Config) (*Client, error) {
awsCfg, err := awsconfig.LoadDefaultConfig(context.Background(),
awsconfig.WithRegion(cfg.Region),
awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
cfg.AccessKey, cfg.SecretKey, "")),
)
if err != nil {
return nil, err
}
// AGENT: a bounded retry count keeps a broken S3 dependency from turning an
// HTTP upload into an unbounded wait. Context deadlines in the handlers set
// the overall request budget.
awsCfg.RetryMaxAttempts = 2
transport := http.DefaultTransport.(*http.Transport).Clone()
transport.MaxIdleConnsPerHost = 20
transport.MaxIdleConns = 40
client := s3.NewFromConfig(awsCfg, func(options *s3.Options) {
// RustFS is addressed by its Compose service name, so virtual-host
// addressing cannot work without extra DNS entries. Path-style is the
// portable choice for every S3-compatible server used in this stand.
options.UsePathStyle = true
options.BaseEndpoint = aws.String(strings.TrimRight(cfg.Endpoint, "/"))
options.HTTPClient = &http.Client{Transport: transport}
})
return &Client{s3: client, bucket: cfg.Bucket}, nil
}
func (c *Client) Bucket() string { return c.bucket }
// Ping is used by /health/ready. HeadBucket is the cheapest authenticated call
// that proves both the endpoint and the scoped credentials work.
func (c *Client) Ping(ctx context.Context) error {
pingCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
_, err := c.s3.HeadBucket(pingCtx, &s3.HeadBucketInput{Bucket: aws.String(c.bucket)})
return mapError(err)
}
func (c *Client) Put(ctx context.Context, key string, body io.ReadSeeker, size int64, contentType string) error {
_, err := c.s3.PutObject(ctx, &s3.PutObjectInput{
Bucket: aws.String(c.bucket),
Key: aws.String(key),
Body: body,
ContentLength: aws.Int64(size),
ContentType: aws.String(contentType),
})
return mapError(err)
}
func (c *Client) Get(ctx context.Context, key string) (io.ReadCloser, int64, error) {
output, err := c.s3.GetObject(ctx, &s3.GetObjectInput{
Bucket: aws.String(c.bucket),
Key: aws.String(key),
})
if err != nil {
return nil, 0, mapError(err)
}
size := int64(0)
if output.ContentLength != nil {
size = *output.ContentLength
}
return output.Body, size, nil
}
func (c *Client) Delete(ctx context.Context, key string) error {
_, err := c.s3.DeleteObject(ctx, &s3.DeleteObjectInput{
Bucket: aws.String(c.bucket),
Key: aws.String(key),
})
return mapError(err)
}
func (c *Client) List(ctx context.Context, prefix string) ([]Object, error) {
var objects []Object
var continuation *string
for {
output, err := c.s3.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
Bucket: aws.String(c.bucket),
Prefix: aws.String(prefix),
ContinuationToken: continuation,
})
if err != nil {
return nil, mapError(err)
}
for _, item := range output.Contents {
if item.Key == nil {
continue
}
object := Object{Key: *item.Key}
if item.Size != nil {
object.Size = *item.Size
}
if item.LastModified != nil {
object.LastModified = *item.LastModified
}
objects = append(objects, object)
}
if output.IsTruncated == nil || !*output.IsTruncated {
return objects, nil
}
continuation = output.NextContinuationToken
if continuation == nil {
return objects, nil
}
}
}
// mapError turns a missing object into a sentinel the HTTP layer can map to
// 404, while every other failure stays a dependency error (503).
func mapError(err error) error {
if err == nil {
return nil
}
var apiErr smithy.APIError
if errors.As(err, &apiErr) {
switch apiErr.ErrorCode() {
case "NoSuchKey", "NoSuchBucket", "NotFound", "XMinioInvalidObjectName":
return ErrNotFound
}
}
return err
}
+115
View File
@@ -0,0 +1,115 @@
// Package reconcile repairs the two failure windows between PostgreSQL
// metadata and S3 bytes:
//
// - a pending metadata row whose upload never finished or whose 201 was
// never returned;
// - an S3 object whose metadata transaction never committed.
//
// Both repairs use a grace period so an upload that is still in flight is
// never touched.
package reconcile
import (
"context"
"errors"
"log/slog"
"time"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/objectstore"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/store"
)
const objectPrefix = "attachments/"
type Reconciler struct {
store *store.Store
objects *objectstore.Client
grace time.Duration
logger *slog.Logger
}
type Stats struct {
StalePending int `json:"stale_pending"`
OrphanObjects int `json:"orphan_objects"`
DeletedRows int `json:"deleted_rows"`
DeletedObjects int `json:"deleted_objects"`
Skipped int `json:"skipped"`
}
func New(st *store.Store, objects *objectstore.Client, grace time.Duration, logger *slog.Logger) *Reconciler {
if logger == nil {
logger = slog.Default()
}
return &Reconciler{store: st, objects: objects, grace: grace, logger: logger}
}
func (r *Reconciler) RunOnce(ctx context.Context, dryRun bool) (Stats, error) {
var stats Stats
cutoff := time.Now().UTC().Add(-r.grace)
// Phase 1: rows that were committed as pending but never reached ready.
stale, err := r.store.ListStalePendingAttachments(ctx, cutoff, 500)
if err != nil {
return stats, err
}
for _, attachment := range stale {
stats.StalePending++
if dryRun {
r.logger.Info("reconcile: would remove stale pending attachment",
"attachment_id", attachment.ID, "object_key", attachment.ObjectKey,
"created_at", attachment.CreatedAt)
continue
}
if err = r.objects.Delete(ctx, attachment.ObjectKey); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
// Keep the row for the next pass. Deleting it now would lose the
// only pointer to an object that might still exist.
r.logger.Error("reconcile: delete stale object",
"err", err, "attachment_id", attachment.ID)
stats.Skipped++
continue
}
if err = r.store.DeletePendingAttachment(ctx, attachment.ID); err != nil {
r.logger.Error("reconcile: delete stale row", "err", err, "attachment_id", attachment.ID)
stats.Skipped++
continue
}
stats.DeletedRows++
stats.DeletedObjects++
}
// Phase 2: objects with no metadata row at all. The pending row is written
// before the PUT in the normal path, so this only happens after a crash in
// a narrow window (or manual bucket tampering).
listed, err := r.objects.List(ctx, objectPrefix)
if err != nil {
return stats, err
}
known, err := r.store.ListAttachmentObjectKeys(ctx)
if err != nil {
return stats, err
}
for _, object := range listed {
if _, ok := known[object.Key]; ok {
continue
}
// A missing LastModified must not be treated as ancient: skipping is
// safer than deleting an object that may have just been created.
if object.LastModified.IsZero() || object.LastModified.After(cutoff) {
continue
}
stats.OrphanObjects++
if dryRun {
r.logger.Info("reconcile: would delete orphan object", "object_key", object.Key)
continue
}
if err = r.objects.Delete(ctx, object.Key); err != nil && !errors.Is(err, objectstore.ErrNotFound) {
r.logger.Error("reconcile: delete orphan object", "err", err, "object_key", object.Key)
stats.Skipped++
continue
}
stats.DeletedObjects++
}
return stats, nil
}
+170
View File
@@ -0,0 +1,170 @@
package store
import (
"context"
"encoding/hex"
"errors"
"time"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/model"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
// CreatePendingAttachment records the intent to store an object before the S3
// PUT happens. Committing this row first is what makes the reconciler able to
// distinguish "never uploaded" from "uploaded but process died".
func (s *Store) CreatePendingAttachment(ctx context.Context, att model.Attachment) error {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
hash, err := hex.DecodeString(att.SHA256)
if err != nil {
return common.BadRequest("invalid sha256")
}
_, err = s.pool.Exec(ctx, `
INSERT INTO attachments
(id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
object_key, status, error_code, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'pending', NULL, $9, $9)`,
att.ID, att.ChatID, att.OwnerID, att.Filename, att.ContentType, att.SizeBytes,
hash, att.ObjectKey, time.Time(att.CreatedAt))
return dbError(err)
}
// MarkAttachmentReady publishes metadata only after the object bytes are known
// to be in S3. A 201 is therefore a promise about both stores.
func (s *Store) MarkAttachmentReady(ctx context.Context, id uuid.UUID) (time.Time, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
updatedAt := time.Now().UTC()
tag, err := s.pool.Exec(ctx, `
UPDATE attachments
SET status = 'ready', updated_at = $2
WHERE id = $1 AND status = 'pending'`, id, updatedAt)
if err != nil {
return time.Time{}, dbError(err)
}
if tag.RowsAffected() == 0 {
return time.Time{}, common.Conflict("attachment is not pending")
}
return updatedAt, nil
}
// DeletePendingAttachment removes a row only while it is still pending. The
// status guard is the safety property: if an upload completed concurrently, the
// compensation or reconciler pass must not delete a ready attachment.
func (s *Store) DeletePendingAttachment(ctx context.Context, id uuid.UUID) error {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
_, err := s.pool.Exec(ctx,
`DELETE FROM attachments WHERE id = $1 AND status = 'pending'`, id)
return dbError(err)
}
func (s *Store) GetAttachmentForUser(ctx context.Context, userID, attachmentID uuid.UUID) (model.Attachment, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
att, err := scanAttachment(s.pool.QueryRow(ctx, `
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type,
a.size_bytes, a.sha256, a.status, a.error_code,
a.created_at, a.updated_at, a.object_key
FROM attachments a
JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = $2
WHERE a.id = $1`, attachmentID, userID))
if err != nil {
return model.Attachment{}, err
}
return att, nil
}
// ListStalePendingAttachments is the entry point for crash recovery. cutoff is
// an instant, not a duration: rows created before it are eligible. The caller
// computes it as now - grace so the boundary can be tuned and tested.
func (s *Store) ListStalePendingAttachments(ctx context.Context, cutoff time.Time, limit int) ([]model.Attachment, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
rows, err := s.pool.Query(ctx, `
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type,
a.size_bytes, a.sha256, a.status, a.error_code,
a.created_at, a.updated_at, a.object_key
FROM attachments a
WHERE a.status = 'pending' AND a.created_at < $1
ORDER BY a.created_at
LIMIT $2`, cutoff, limit)
if err != nil {
return nil, dbError(err)
}
defer rows.Close()
var result []model.Attachment
for rows.Next() {
att, scanErr := scanAttachment(rows)
if scanErr != nil {
return nil, dbError(scanErr)
}
result = append(result, att)
}
if err = rows.Err(); err != nil {
return nil, dbError(err)
}
return result, nil
}
func (s *Store) ListAttachmentObjectKeys(ctx context.Context) (map[string]struct{}, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
rows, err := s.pool.Query(ctx, `SELECT object_key FROM attachments`)
if err != nil {
return nil, dbError(err)
}
defer rows.Close()
keys := make(map[string]struct{})
for rows.Next() {
var key string
if err = rows.Scan(&key); err != nil {
return nil, dbError(err)
}
keys[key] = struct{}{}
}
if err = rows.Err(); err != nil {
return nil, dbError(err)
}
return keys, nil
}
func scanAttachment(row rowScanner) (model.Attachment, error) {
var (
att model.Attachment
hash []byte
errCode *string
createdAt time.Time
updatedAt time.Time
)
if err := row.Scan(
&att.ID, &att.ChatID, &att.OwnerID, &att.Filename, &att.ContentType,
&att.SizeBytes, &hash, &att.Status, &errCode,
&createdAt, &updatedAt, &att.ObjectKey,
); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return model.Attachment{}, common.NotFound("attachment not found")
}
return model.Attachment{}, dbError(err)
}
att.SHA256 = hex.EncodeToString(hash)
att.ErrorCode = errCode
att.Variants = []model.Variant{}
att.CreatedAt = common.RFCFormTime(createdAt.UTC())
att.UpdatedAt = common.RFCFormTime(updatedAt.UTC())
return att, nil
}
+128 -23
View File
@@ -5,6 +5,7 @@ import (
"context"
"crypto/sha256"
"errors"
"sort"
"time"
"gitflic.ru/project/bulla-max/mtusi2026lab/internal/common"
@@ -13,24 +14,29 @@ import (
"github.com/jackc/pgx/v5"
)
func (s *Store) CreateMessage(ctx context.Context, chatID, senderID uuid.UUID, text, idempotencyKey string) (model.Message, error) {
func (s *Store) CreateMessage(ctx context.Context, chatID, senderID uuid.UUID, text, idempotencyKey string, attachmentIDs []uuid.UUID) (model.Message, error) {
ctx, cancel := s.withTimeout(ctx)
defer cancel()
fingerprint := sha256.Sum256([]byte(text))
// Sorting makes the idempotency fingerprint independent of the order
// in which the client listed the same set of attachment IDs. The response
// uses the same deterministic order.
attachmentIDs = canonicalAttachmentIDs(attachmentIDs)
fingerprint := messageFingerprint(text, attachmentIDs)
if message, found, err := s.findIdempotentMessage(ctx, senderID, chatID, idempotencyKey, fingerprint[:]); err != nil {
if message, found, err := s.findIdempotentMessage(ctx, senderID, chatID, idempotencyKey, fingerprint); err != nil {
return model.Message{}, err
} else if found {
return message, nil
}
message := model.Message{
ID: uuid.New(),
ChatID: chatID,
SenderID: senderID,
Text: text,
CreatedAt: common.NowUTC(),
ID: uuid.New(),
ChatID: chatID,
SenderID: senderID,
Text: text,
CreatedAt: common.NowUTC(),
AttachmentIDs: attachmentIDs,
}
tx, err := s.pool.Begin(ctx)
@@ -39,20 +45,73 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID uuid.UUID, t
}
defer rollback(tx)
tag, err := tx.Exec(ctx, `
// Check membership and attachment ownership in the same transaction as the
// insert. A foreign chat or a foreign attachment is intentionally the same
// 404 as a missing object, so the API does not leak existence.
var member int
err = tx.QueryRow(ctx,
`SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2`,
chatID, senderID).Scan(&member)
if errors.Is(err, pgx.ErrNoRows) {
return model.Message{}, common.NotFound("chat not found")
}
if err != nil {
return model.Message{}, dbError(err)
}
if len(attachmentIDs) > 0 {
rows, queryErr := tx.Query(ctx, `
SELECT id, status FROM attachments
WHERE chat_id = $1 AND id = ANY($2::uuid[])`, chatID, attachmentIDs)
if queryErr != nil {
return model.Message{}, dbError(queryErr)
}
found := 0
for rows.Next() {
var (
id uuid.UUID
status string
)
if scanErr := rows.Scan(&id, &status); scanErr != nil {
rows.Close()
return model.Message{}, dbError(scanErr)
}
found++
if status != "ready" {
rows.Close()
return model.Message{}, common.Conflict("attachment is not ready")
}
}
if err = rows.Err(); err != nil {
rows.Close()
return model.Message{}, dbError(err)
}
rows.Close()
if found != len(attachmentIDs) {
return model.Message{}, common.NotFound("attachment not found")
}
}
_, err = tx.Exec(ctx, `
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT $1, $2, $3, $4, $5
WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = $2 AND user_id = $3)`,
VALUES ($1, $2, $3, $4, $5)`,
message.ID, message.ChatID, message.SenderID, message.Text, time.Time(message.CreatedAt))
if err != nil {
return model.Message{}, dbError(err)
}
if tag.RowsAffected() == 0 {
return model.Message{}, common.NotFound("chat not found")
for position, attachmentID := range attachmentIDs {
if _, err = tx.Exec(ctx, `
INSERT INTO message_attachments (message_id, attachment_id, position)
VALUES ($1, $2, $3)`, message.ID, attachmentID, position); err != nil {
return model.Message{}, dbError(err)
}
}
expiresAt := time.Time(message.CreatedAt).Add(s.idempotencyTTL)
tag, err = tx.Exec(ctx, `
tag, err := tx.Exec(ctx, `
INSERT INTO message_idempotency
(sender_id, chat_id, idempotency_key, request_hash, message_id, created_at, expires_at)
VALUES ($1, $2, $3, $4, $5, $6, $7)
@@ -62,7 +121,7 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID uuid.UUID, t
created_at = EXCLUDED.created_at,
expires_at = EXCLUDED.expires_at
WHERE message_idempotency.expires_at <= now()`,
senderID, chatID, idempotencyKey, fingerprint[:], message.ID,
senderID, chatID, idempotencyKey, fingerprint, message.ID,
time.Time(message.CreatedAt), expiresAt)
if err != nil {
return model.Message{}, dbError(err)
@@ -70,7 +129,7 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID uuid.UUID, t
if tag.RowsAffected() == 0 {
rollback(tx)
stored, found, lookupErr := s.findIdempotentMessage(ctx, senderID, chatID, idempotencyKey, fingerprint[:])
stored, found, lookupErr := s.findIdempotentMessage(ctx, senderID, chatID, idempotencyKey, fingerprint)
if lookupErr != nil {
return model.Message{}, lookupErr
}
@@ -87,13 +146,47 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID uuid.UUID, t
return message, nil
}
func canonicalAttachmentIDs(ids []uuid.UUID) []uuid.UUID {
result := append([]uuid.UUID(nil), ids...)
sort.Slice(result, func(i, j int) bool {
return bytes.Compare(result[i][:], result[j][:]) < 0
})
if result == nil {
return []uuid.UUID{}
}
return result
}
// messageFingerprint keeps the lab3 hash for messages without attachments, so
// an idempotency key stored before this migration still replays correctly. For
// messages with attachments the sorted IDs are part of the canonical body.
func messageFingerprint(text string, attachmentIDs []uuid.UUID) []byte {
if len(attachmentIDs) == 0 {
sum := sha256.Sum256([]byte(text))
return sum[:]
}
hash := sha256.New()
hash.Write([]byte(text))
hash.Write([]byte{0})
for _, id := range attachmentIDs {
hash.Write(id[:])
}
return hash.Sum(nil)
}
func (s *Store) findIdempotentMessage(ctx context.Context, senderID, chatID uuid.UUID, idempotencyKey string, expectedHash []byte) (model.Message, bool, error) {
var message model.Message
var createdAt time.Time
var storedHash []byte
err := s.pool.QueryRow(ctx, `
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, i.request_hash
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(SELECT ma.attachment_id
FROM message_attachments ma
WHERE ma.message_id = m.id
ORDER BY ma.position),
i.request_hash
FROM message_idempotency i
JOIN messages m ON m.id = i.message_id
JOIN chat_members cm ON cm.chat_id = i.chat_id AND cm.user_id = i.sender_id
@@ -102,7 +195,8 @@ func (s *Store) findIdempotentMessage(ctx context.Context, senderID, chatID uuid
AND i.idempotency_key = $3
AND i.expires_at > now()`,
senderID, chatID, idempotencyKey).Scan(
&message.ID, &message.ChatID, &message.SenderID, &message.Text, &createdAt, &storedHash)
&message.ID, &message.ChatID, &message.SenderID, &message.Text, &createdAt,
&message.AttachmentIDs, &storedHash)
if errors.Is(err, pgx.ErrNoRows) {
return model.Message{}, false, nil
}
@@ -112,6 +206,9 @@ func (s *Store) findIdempotentMessage(ctx context.Context, senderID, chatID uuid
if !bytes.Equal(storedHash, expectedHash) {
return model.Message{}, false, common.Conflict("idempotency key reused with a different message")
}
if message.AttachmentIDs == nil {
message.AttachmentIDs = []uuid.UUID{}
}
message.CreatedAt = common.RFCFormTime(createdAt.UTC())
return message, true, nil
@@ -136,10 +233,14 @@ func (s *Store) ListMessages(ctx context.Context, userID, chatID uuid.UUID, limi
cursorTime, cursorID := cursorValues(cursor)
rows, err := s.pool.Query(ctx, `
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = $1 AND (created_at, id) > ($2, $3)
ORDER BY created_at, id
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
ARRAY(SELECT ma.attachment_id
FROM message_attachments ma
WHERE ma.message_id = m.id
ORDER BY ma.position)
FROM messages m
WHERE m.chat_id = $1 AND (m.created_at, m.id) > ($2, $3)
ORDER BY m.created_at, m.id
LIMIT $4`, chatID, cursorTime, cursorID, limit+1)
if err != nil {
return nil, dbError(err)
@@ -172,9 +273,13 @@ func (s *Store) ListMessages(ctx context.Context, userID, chatID uuid.UUID, limi
func scanMessage(row rowScanner) (model.Message, error) {
var message model.Message
var createdAt time.Time
if err := row.Scan(&message.ID, &message.ChatID, &message.SenderID, &message.Text, &createdAt); err != nil {
if err := row.Scan(&message.ID, &message.ChatID, &message.SenderID, &message.Text,
&createdAt, &message.AttachmentIDs); err != nil {
return model.Message{}, err
}
if message.AttachmentIDs == nil {
message.AttachmentIDs = []uuid.UUID{}
}
message.CreatedAt = common.RFCFormTime(createdAt.UTC())
return message, nil
}
+80
View File
@@ -0,0 +1,80 @@
package store
import (
"bytes"
"crypto/sha256"
"testing"
"github.com/google/uuid"
)
func TestCanonicalAttachmentIDsSortsAndCopies(t *testing.T) {
first, second, third := uuid.MustParse("00000000-0000-0000-0000-000000000003"),
uuid.MustParse("00000000-0000-0000-0000-000000000001"),
uuid.MustParse("00000000-0000-0000-0000-000000000002")
input := []uuid.UUID{first, second, third}
got := canonicalAttachmentIDs(input)
want := []uuid.UUID{second, third, first}
if !equalIDs(got, want) {
t.Fatalf("canonicalAttachmentIDs = %v, want %v", got, want)
}
if !equalIDs(input, []uuid.UUID{first, second, third}) {
t.Fatalf("canonicalAttachmentIDs mutated its input: %v", input)
}
}
func TestCanonicalAttachmentIDsReturnsEmptySliceForNil(t *testing.T) {
got := canonicalAttachmentIDs(nil)
if got == nil {
t.Fatal("canonicalAttachmentIDs(nil) = nil, want empty slice")
}
if len(got) != 0 {
t.Fatalf("canonicalAttachmentIDs(nil) = %v, want empty", got)
}
}
// TestMessageFingerprintWithoutAttachmentsPreservesLab3Hash is the
// compatibility guard: idempotency rows written before attachments existed
// must still match on replay.
func TestMessageFingerprintWithoutAttachmentsPreservesLab3Hash(t *testing.T) {
expected := sha256.Sum256([]byte("hello"))
if got := messageFingerprint("hello", nil); !bytes.Equal(got, expected[:]) {
t.Fatalf("fingerprint = %x, want %x", got, expected)
}
if got := messageFingerprint("hello", []uuid.UUID{}); !bytes.Equal(got, expected[:]) {
t.Fatalf("empty slice fingerprint = %x, want %x", got, expected)
}
}
func TestMessageFingerprintDependsOnSortedAttachments(t *testing.T) {
first := uuid.MustParse("00000000-0000-0000-0000-000000000001")
second := uuid.MustParse("00000000-0000-0000-0000-000000000002")
// CreateMessage canonicalizes before hashing; the pair must therefore be
// order independent even though messageFingerprint itself takes the list
// as given.
one := messageFingerprint("hello", canonicalAttachmentIDs([]uuid.UUID{first, second}))
reordered := messageFingerprint("hello", canonicalAttachmentIDs([]uuid.UUID{second, first}))
if !bytes.Equal(one, reordered) {
t.Fatal("fingerprint depends on attachment order")
}
other := messageFingerprint("hello", canonicalAttachmentIDs([]uuid.UUID{first}))
if bytes.Equal(one, other) {
t.Fatal("fingerprint ignores attachment identity")
}
}
func equalIDs(a, b []uuid.UUID) bool {
if len(a) != len(b) {
return false
}
for index := range a {
if a[index] != b[index] {
return false
}
}
return true
}
+5 -4
View File
@@ -11,11 +11,12 @@ import (
)
type Handler struct {
store *store.Store
store *store.Store
jsonBodyLimit int64
}
func NewHandler(st *store.Store) *Handler {
return &Handler{store: st}
func NewHandler(st *store.Store, jsonBodyLimit int64) *Handler {
return &Handler{store: st, jsonBodyLimit: jsonBodyLimit}
}
type CreateUserRequest struct {
@@ -26,7 +27,7 @@ type CreateUserRequest struct {
func (h *Handler) CreateUser(c fiber.Ctx) error {
var req CreateUserRequest
if err := common.BindJSON(c, &req); err != nil {
if err := common.BindJSON(c, &req, h.jsonBodyLimit); err != nil {
return err
}