forked from mtusi2026/mtusi2026lab5
feat:level4
This commit is contained in:
+5
-1
@@ -9,7 +9,9 @@ RUN go mod download
|
||||
|
||||
COPY cmd ./cmd
|
||||
COPY src ./src
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/messenger
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/messenger ./cmd/messenger && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/outbox ./cmd/outbox && \
|
||||
CGO_ENABLED=0 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/worker ./cmd/worker
|
||||
|
||||
# --- runtime stage ---
|
||||
FROM alpine:3.20
|
||||
@@ -18,6 +20,8 @@ RUN apk add --no-cache ca-certificates postgresql-client && \
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=build /out/messenger /app/messenger
|
||||
COPY --from=build /out/worker /app/worker
|
||||
COPY --from=build /out/outbox /app/outbox
|
||||
COPY migrations /app/migrations
|
||||
COPY scripts/migrate.sh /app/migrate.sh
|
||||
RUN chmod +x /app/migrate.sh
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"mtusi-messenger/src/config"
|
||||
"mtusi-messenger/src/domain"
|
||||
)
|
||||
|
||||
const (
|
||||
publishInterval = 1 * time.Second
|
||||
reapInterval = 30 * time.Second
|
||||
publishBatch = 50
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg := config.Load()
|
||||
if cfg.DatabaseURL == "" {
|
||||
log.Fatal("DATABASE_URL is required (env DATABASE_URL or DATABASE_URL_FILE)")
|
||||
}
|
||||
if cfg.CacheURL == "" {
|
||||
log.Fatal("CACHE_URL is required (env CACHE_URL or CACHE_URL_FILE)")
|
||||
}
|
||||
|
||||
rootCtx, cancelRoot := context.WithCancel(context.Background())
|
||||
defer cancelRoot()
|
||||
|
||||
store, err := domain.NewStore(rootCtx, cfg.DatabaseURL)
|
||||
if err != nil {
|
||||
log.Fatalf("postgres: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
broker, err := domain.NewBroker(rootCtx, cfg.CacheURL)
|
||||
if err != nil {
|
||||
log.Fatalf("broker: %v", err)
|
||||
}
|
||||
|
||||
s3, err := domain.NewS3Store(rootCtx, cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket,
|
||||
cfg.S3AccessKeyID, cfg.S3SecretKey, cfg.MaxUploadBytes, cfg.MaxImagePixels)
|
||||
if err != nil {
|
||||
log.Fatalf("s3: %v", err)
|
||||
}
|
||||
|
||||
go gcLoop(rootCtx, store, s3, cfg.S3GCGrace)
|
||||
|
||||
// Reaper зависших processing крутится здесь, потому что outbox уже
|
||||
// держит соединение с БД и Redis и запускается в единственном экземпляре.
|
||||
go reaperLoop(rootCtx, store)
|
||||
|
||||
stop := make(chan os.Signal, 1)
|
||||
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
|
||||
|
||||
go func() {
|
||||
<-stop
|
||||
log.Println("outbox: shutting down")
|
||||
cancelRoot()
|
||||
}()
|
||||
|
||||
ticker := time.NewTicker(publishInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
log.Printf("outbox: running (interval=%s batch=%d)", publishInterval, publishBatch)
|
||||
for {
|
||||
select {
|
||||
case <-rootCtx.Done():
|
||||
log.Println("outbox: stopped")
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := publishBatchOnce(rootCtx, store, broker); err != nil {
|
||||
log.Printf("outbox: publish: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// publishBatchOnce: выбираем партию неопубликованных job-ов, публикуем
|
||||
// по одному, после XADD — MarkPublished. Падение между XADD и MarkPublished
|
||||
// приведёт к повторной публикации при следующем цикле; worker идемпотентен
|
||||
// (ключ эффекта — (attachment_id, pipeline_version, variant)), поэтому
|
||||
// дубль в Streams — норма, а не баг.
|
||||
func publishBatchOnce(ctx context.Context, store *domain.Store, broker *domain.Broker) error {
|
||||
jobs, err := store.SelectUnpublishedJobs(ctx, publishBatch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(jobs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, j := range jobs {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
msg := domain.JobMessage{
|
||||
SchemaVersion: 1,
|
||||
JobID: j.JobID,
|
||||
Type: "attachment.process.v1",
|
||||
AttachmentID: j.AttachmentID,
|
||||
PipelineVersion: j.PipelineVersion,
|
||||
Attempt: j.Attempt,
|
||||
CreatedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
}
|
||||
if _, err := broker.Publish(ctx, msg); err != nil {
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return err
|
||||
}
|
||||
// Не фатально: попробуем в следующем цикле, зафиксировав причину.
|
||||
_ = store.BumpPublishAttempt(ctx, j.JobID, "publish_failed")
|
||||
log.Printf("outbox: publish job=%s attachment=%s: %v",
|
||||
j.JobID, j.AttachmentID, err)
|
||||
continue
|
||||
}
|
||||
if err := store.MarkPublished(ctx, j.JobID); err != nil {
|
||||
log.Printf("outbox: mark published job=%s: %v", j.JobID, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func reaperLoop(ctx context.Context, store *domain.Store) {
|
||||
t := time.NewTicker(reapInterval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
n, err := store.RequeueStuck(ctx)
|
||||
if err != nil {
|
||||
log.Printf("outbox: reaper: %v", err)
|
||||
continue
|
||||
}
|
||||
if n > 0 {
|
||||
log.Printf("outbox: reaper requeued %d stuck job(s)", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func gcLoop(ctx context.Context, store *domain.Store, s3 *domain.S3Store, grace time.Duration) {
|
||||
t := time.NewTicker(2 * time.Minute)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
if err := store.ReconcileOrphans(ctx, s3, grace); err != nil {
|
||||
log.Printf("outbox: gc: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"mtusi-messenger/src/config"
|
||||
"mtusi-messenger/src/domain"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
lease = 60 * time.Second
|
||||
maxAttempts = 5
|
||||
baseBackoff = 2 * time.Second
|
||||
)
|
||||
|
||||
func main() {
|
||||
cfg := config.Load()
|
||||
if cfg.CacheURL == "" {
|
||||
log.Fatal("CACHE_URL (redis) is required for broker")
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
store, err := domain.NewStore(ctx, cfg.DatabaseURL)
|
||||
if err != nil {
|
||||
log.Fatalf("db: %v", err)
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
s3, err := domain.NewS3Store(ctx, cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket,
|
||||
cfg.S3AccessKeyID, cfg.S3SecretKey, cfg.MaxUploadBytes, cfg.MaxImagePixels)
|
||||
if err != nil {
|
||||
log.Fatalf("s3: %v", err)
|
||||
}
|
||||
|
||||
broker, err := domain.NewBroker(ctx, cfg.CacheURL)
|
||||
if err != nil {
|
||||
log.Fatalf("broker: %v", err)
|
||||
}
|
||||
|
||||
consumer := cfg.InstanceID + "-worker"
|
||||
|
||||
go reaperLoop(ctx, store)
|
||||
|
||||
stop := make(chan os.Signal, 1)
|
||||
signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT)
|
||||
go func() {
|
||||
<-stop
|
||||
log.Println("worker shutting down")
|
||||
cancel()
|
||||
}()
|
||||
|
||||
for ctx.Err() == nil {
|
||||
msgs, err := broker.Consume(ctx, consumer)
|
||||
if err != nil {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
log.Printf("consume: %v", err)
|
||||
time.Sleep(time.Second)
|
||||
continue
|
||||
}
|
||||
for _, m := range msgs {
|
||||
handleOne(ctx, store, s3, broker, m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func reaperLoop(ctx context.Context, store *domain.Store) {
|
||||
t := time.NewTicker(30 * time.Second)
|
||||
defer t.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
if n, err := store.RequeueStuck(ctx); err != nil {
|
||||
log.Printf("reaper: %v", err)
|
||||
} else if n > 0 {
|
||||
log.Printf("reaper: requeued %d stuck job(s)", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handleOne(ctx context.Context, store *domain.Store, s3 *domain.S3Store,
|
||||
broker *domain.Broker, m redis.XMessage) {
|
||||
|
||||
var job domain.JobMessage
|
||||
if raw, ok := m.Values["payload"].(string); ok {
|
||||
_ = json.Unmarshal([]byte(raw), &job)
|
||||
} else if raw, ok := m.Values["payload"].([]byte); ok {
|
||||
_ = json.Unmarshal(raw, &job)
|
||||
} else {
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
|
||||
// Валидация схемы (обязательные поля).
|
||||
if job.SchemaVersion != 1 || job.Type != "attachment.process.v1" ||
|
||||
job.JobID == "" || job.AttachmentID == "" {
|
||||
log.Printf("poison message %s, acking", m.ID)
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
|
||||
// No-op для уже готового результата (redelivery после готовности).
|
||||
if done, _ := store.VariantsExist(ctx, job.AttachmentID, job.PipelineVersion); done {
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
|
||||
j, err := store.ClaimJobByID(ctx, job.JobID, workerID(), lease)
|
||||
if errors.Is(err, domain.ErrNotFound) {
|
||||
// Уже done/failed или lease держит кто-то другой — ack, не дублируем.
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
log.Printf("claim: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
stagingKey, _, err := store.GetStagingKey(ctx, j.AttachmentID)
|
||||
if err != nil {
|
||||
log.Printf("staging key: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
raw, ct, err := s3.FetchStagingForProcessing(ctx, stagingKey)
|
||||
if err != nil {
|
||||
code, permanent := classify(err)
|
||||
if permanent {
|
||||
// постоянная ошибка — failed без retry
|
||||
_ = store.MarkFailedPermanent(ctx, j.JobID, j.AttachmentID, code)
|
||||
} else {
|
||||
_, _ = store.RetryJob(ctx, j.JobID, code, maxAttempts, baseBackoff)
|
||||
}
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
|
||||
variants, err := processImage(raw, ct)
|
||||
if err != nil {
|
||||
// decode error — постоянная
|
||||
_ = store.MarkFailedPermanent(ctx, j.JobID, j.AttachmentID, "invalid_image")
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
|
||||
finalOriginalKey := "final/" + j.AttachmentID + "/original"
|
||||
var vrows []domain.Variant
|
||||
for _, v := range variants {
|
||||
if v.Name == "original" {
|
||||
// оригинал перекладываем из staging server-side.
|
||||
if err := s3.MoveToFinal(ctx, stagingKey, finalOriginalKey); err != nil {
|
||||
_, _ = store.RetryJob(ctx, j.JobID, "storage_error", maxAttempts, baseBackoff)
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
sum, err := s3.PutBytes(ctx, finalOriginalKey, v.ContentType, v.Data)
|
||||
if err != nil {
|
||||
_, _ = store.RetryJob(ctx, j.JobID, "storage_error", maxAttempts, baseBackoff)
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
v.SHA256 = sum
|
||||
}
|
||||
vrows = append(vrows, domain.Variant{
|
||||
Name: v.Name, ContentType: v.ContentType, SizeBytes: int64(len(v.Data)),
|
||||
Width: v.Width, Height: v.Height, SHA256: v.SHA256,
|
||||
})
|
||||
}
|
||||
|
||||
if err := store.MarkReady(ctx, j.JobID, j.AttachmentID, j.PipelineVersion, finalOriginalKey, vrows); err != nil {
|
||||
// Уже done — не страшно; иначе retry.
|
||||
if !errors.Is(err, domain.ErrConflict) {
|
||||
log.Printf("mark ready: %v", err)
|
||||
}
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
return
|
||||
}
|
||||
_ = broker.Ack(ctx, m.ID)
|
||||
}
|
||||
|
||||
func workerID() string {
|
||||
h, _ := os.Hostname()
|
||||
return h + "-" + time.Now().Format("150405")
|
||||
}
|
||||
|
||||
func classify(err error) (string, bool) {
|
||||
switch {
|
||||
case errors.Is(err, domain.ErrTooLarge):
|
||||
return "payload_too_large", true
|
||||
case errors.Is(err, domain.ErrUnsupported):
|
||||
return "unsupported_media_type", true
|
||||
case errors.Is(err, domain.ErrNotImage):
|
||||
return "invalid_image", true
|
||||
default:
|
||||
return "storage_error", false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"image"
|
||||
"image/jpeg"
|
||||
|
||||
"golang.org/x/image/draw"
|
||||
)
|
||||
|
||||
type VariantSpec struct {
|
||||
Name string
|
||||
ContentType string
|
||||
Width int
|
||||
Height int
|
||||
Data []byte
|
||||
SHA256 string
|
||||
}
|
||||
|
||||
func encodeOptimizedJPEG(src image.Image, quality int) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, src, &jpeg.Options{Quality: quality}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// processImage возвращает 4 варианта. Оригинал — byte-identical с входом.
|
||||
// thumbnail 256x256 (fit, без искажения), crop 128x128 (center crop),
|
||||
// optimized — WebP q=80 без метаданных.
|
||||
func processImage(raw []byte, contentType string) ([]VariantSpec, error) {
|
||||
src, _, err := image.Decode(bytes.NewReader(raw))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := make([]VariantSpec, 0, 4)
|
||||
|
||||
// original — ровно те байты, что пришли в staging.
|
||||
sum := sha256.Sum256(raw)
|
||||
out = append(out, VariantSpec{
|
||||
Name: "original", ContentType: contentType,
|
||||
Width: src.Bounds().Dx(), Height: src.Bounds().Dy(),
|
||||
Data: raw, SHA256: hex.EncodeToString(sum[:]),
|
||||
})
|
||||
|
||||
// thumbnail 256x256 (fit).
|
||||
th := resizeFit(src, 256, 256)
|
||||
if err := appendEncoded(&out, "thumbnail", "image/jpeg", th); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// crop 128x128 (center crop).
|
||||
cr := centerCrop(src, 128, 128)
|
||||
if err := appendEncoded(&out, "crop", "image/jpeg", cr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// optimized
|
||||
optBuf, err := encodeOptimizedJPEG(src, 70)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
optSum := sha256.Sum256(optBuf)
|
||||
out = append(out, VariantSpec{
|
||||
Name: "optimized", ContentType: "image/jpeg",
|
||||
Width: src.Bounds().Dx(), Height: src.Bounds().Dy(),
|
||||
Data: optBuf, SHA256: hex.EncodeToString(optSum[:]),
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func appendEncoded(out *[]VariantSpec, name, ct string, img image.Image) error {
|
||||
var buf bytes.Buffer
|
||||
if err := jpeg.Encode(&buf, img, &jpeg.Options{Quality: 85}); err != nil {
|
||||
return err
|
||||
}
|
||||
sum := sha256.Sum256(buf.Bytes())
|
||||
*out = append(*out, VariantSpec{
|
||||
Name: name, ContentType: ct,
|
||||
Width: img.Bounds().Dx(), Height: img.Bounds().Dy(),
|
||||
Data: buf.Bytes(), SHA256: hex.EncodeToString(sum[:]),
|
||||
})
|
||||
return nil
|
||||
}
|
||||
|
||||
func resizeFit(src image.Image, maxW, maxH int) image.Image {
|
||||
b := src.Bounds()
|
||||
w, h := b.Dx(), b.Dy()
|
||||
scale := float64(maxW) / float64(w)
|
||||
if float64(maxH)/float64(h) < scale {
|
||||
scale = float64(maxH) / float64(h)
|
||||
}
|
||||
if scale > 1 {
|
||||
scale = 1
|
||||
}
|
||||
nw, nh := int(float64(w)*scale), int(float64(h)*scale)
|
||||
dst := image.NewRGBA(image.Rect(0, 0, nw, nh))
|
||||
draw.CatmullRom.Scale(dst, dst.Bounds(), src, b, draw.Over, nil)
|
||||
return dst
|
||||
}
|
||||
|
||||
func centerCrop(src image.Image, w, h int) image.Image {
|
||||
b := src.Bounds()
|
||||
sw, sh := b.Dx(), b.Dy()
|
||||
cw, ch := w, h
|
||||
if sw < cw {
|
||||
cw = sw
|
||||
}
|
||||
if sh < ch {
|
||||
ch = sh
|
||||
}
|
||||
x0 := (sw - cw) / 2
|
||||
y0 := (sh - ch) / 2
|
||||
dst := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
// масштабируем crop-область в целевой размер (иначе мелкие картинки не влезут).
|
||||
draw.CatmullRom.Scale(dst, dst.Bounds(),
|
||||
src, image.Rect(b.Min.X+x0, b.Min.Y+y0, b.Min.X+x0+cw, b.Min.Y+y0+ch),
|
||||
draw.Over, nil)
|
||||
return dst
|
||||
}
|
||||
@@ -91,6 +91,29 @@ services:
|
||||
limits:
|
||||
cpus: "${API_CPU_PER_REPLICA:-0.5}"
|
||||
memory: "${API_MEM_PER_REPLICA:-128M}"
|
||||
|
||||
outbox:
|
||||
<<: *api
|
||||
entrypoint: ["/app/outbox"]
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
deploy:
|
||||
replicas: 1
|
||||
resources:
|
||||
limits: { cpus: "0.2", memory: "64M" }
|
||||
|
||||
worker:
|
||||
<<: *api
|
||||
entrypoint: ["/app/worker"]
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
s3: { condition: service_healthy }
|
||||
deploy:
|
||||
replicas: 2
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: "256M" }
|
||||
|
||||
migrate:
|
||||
<<: *api
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
module mtusi-messenger
|
||||
|
||||
go 1.22
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
github.com/google/uuid v1.6.0
|
||||
@@ -8,6 +8,7 @@ require (
|
||||
github.com/minio/minio-go/v7 v7.0.72
|
||||
github.com/redis/go-redis/v9 v9.5.1
|
||||
golang.org/x/crypto v0.23.0
|
||||
golang.org/x/image v0.48.0
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -23,8 +24,8 @@ require (
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/rs/xid v1.5.0 // indirect
|
||||
golang.org/x/net v0.23.0 // indirect
|
||||
golang.org/x/sync v0.1.0 // indirect
|
||||
golang.org/x/sys v0.20.0 // indirect
|
||||
golang.org/x/text v0.15.0 // indirect
|
||||
golang.org/x/sync v0.25.0 // indirect
|
||||
golang.org/x/sys v0.50.0 // indirect
|
||||
golang.org/x/text v0.44.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
)
|
||||
|
||||
@@ -45,15 +45,17 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/image v0.48.0 h1:1AvuPcuoOTJpA7s4VZ5umrb90T3So0rkB/bSGmxkxik=
|
||||
golang.org/x/image v0.48.0/go.mod h1:mmtg64iqJKGqEgy+r13dfybl85tmxUd2lMnmaiSdWYY=
|
||||
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
|
||||
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
|
||||
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs=
|
||||
golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs=
|
||||
golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo=
|
||||
golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
-- staging → final prefix; job-очередь и lease
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN IF NOT EXISTS pipeline_version integer NOT NULL DEFAULT 1,
|
||||
ADD COLUMN IF NOT EXISTS staging_key text,
|
||||
ADD COLUMN IF NOT EXISTS ready_at timestamptz,
|
||||
ADD COLUMN IF NOT EXISTS failed_at timestamptz,
|
||||
ADD COLUMN IF NOT EXISTS error_code text;
|
||||
|
||||
-- idempotency эффекта: (attachment_id, pipeline_version, variant)
|
||||
CREATE TABLE IF NOT EXISTS attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL,
|
||||
variant text NOT NULL,
|
||||
object_key text NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
size_bytes bigint NOT NULL,
|
||||
width int NOT NULL,
|
||||
height int NOT NULL,
|
||||
sha256 char(64) NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (attachment_id, pipeline_version, variant)
|
||||
);
|
||||
|
||||
-- outbox + очередь job-ов
|
||||
CREATE TABLE IF NOT EXISTS attachment_jobs (
|
||||
job_id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE CASCADE,
|
||||
pipeline_version integer NOT NULL,
|
||||
attempt int NOT NULL DEFAULT 1,
|
||||
status text NOT NULL DEFAULT 'queued', -- queued|processing|done|failed
|
||||
-- outbox
|
||||
published_at timestamptz,
|
||||
publish_attempts int NOT NULL DEFAULT 0,
|
||||
-- lease
|
||||
lease_owner text,
|
||||
lease_expires_at timestamptz,
|
||||
-- retry
|
||||
next_attempt_at timestamptz NOT NULL DEFAULT now(),
|
||||
last_error_code text,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (attachment_id, pipeline_version) -- одна логическая работа на версию
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_attachment_jobs_unpublished
|
||||
ON attachment_jobs (next_attempt_at)
|
||||
WHERE published_at IS NULL AND status = 'queued';
|
||||
CREATE INDEX IF NOT EXISTS idx_attachment_jobs_lease
|
||||
ON attachment_jobs (lease_expires_at)
|
||||
WHERE status = 'processing';
|
||||
+70
-11
@@ -419,6 +419,19 @@ func safeFilename(name string) (string, bool) {
|
||||
return name, true
|
||||
}
|
||||
|
||||
func errorCodeFor(err error) string {
|
||||
switch {
|
||||
case errors.Is(err, domain.ErrTooLarge):
|
||||
return "payload_too_large"
|
||||
case errors.Is(err, domain.ErrUnsupported):
|
||||
return "unsupported_media_type"
|
||||
case errors.Is(err, domain.ErrNotImage):
|
||||
return "invalid_image"
|
||||
default:
|
||||
return "storage_error"
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
me, _ := userFrom(r)
|
||||
@@ -442,17 +455,18 @@ func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
objectKey := "attachments/" + chatID + "/" + uuid.NewString()
|
||||
pendingID, err := h.Store.CreateAttachmentPending(r.Context(), chatID, me.ID, objectKey, filename)
|
||||
stagingKey := domain.StagingPrefix + chatID + "/" + uuid.NewString()
|
||||
pendingID, err := h.Store.CreateAttachmentPending(
|
||||
r.Context(), chatID, me.ID, stagingKey, filename)
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
|
||||
res, err := h.S3.PutValidatedImage(r.Context(), objectKey, r.Body)
|
||||
res, err := h.S3.PutValidatedImage(r.Context(), stagingKey, r.Body)
|
||||
if err != nil {
|
||||
_ = h.Store.MarkAttachmentFailed(r.Context(), pendingID, err.Error())
|
||||
_ = h.Store.ScheduleObjectGC(r.Context(), objectKey, "upload_failed")
|
||||
_ = h.Store.MarkAttachmentFailed(r.Context(), pendingID, errorCodeFor(err))
|
||||
_ = h.Store.ScheduleObjectGC(r.Context(), stagingKey, "upload_failed")
|
||||
switch {
|
||||
case errors.Is(err, domain.ErrTooLarge):
|
||||
writeError(w, r, http.StatusRequestEntityTooLarge, "payload_too_large", "file too large")
|
||||
@@ -466,21 +480,24 @@ func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
att, err := h.Store.FinalizeAttachment(r.Context(), pendingID, res.ContentType, res.SizeBytes, res.SHA256, res.Width, res.Height)
|
||||
// Атомарно: attachment queued + outbox-строка. 202 только после COMMIT.
|
||||
att, err := h.Store.EnqueueAttachment(
|
||||
r.Context(), pendingID, res.ContentType, res.SizeBytes, res.SHA256,
|
||||
res.Width, res.Height, domain.PipelineVersion)
|
||||
if err != nil {
|
||||
// S3 PUT прошёл, COMMIT — нет: объект в GC.
|
||||
_ = h.Store.ScheduleObjectGC(r.Context(), objectKey, "finalize_failed")
|
||||
_ = h.Store.ScheduleObjectGC(r.Context(), stagingKey, "enqueue_failed")
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, att)
|
||||
writeJSON(w, http.StatusAccepted, att)
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
func (h *Handler) GetAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
me, _ := userFrom(r)
|
||||
att, err := h.Store.GetAttachmentForUser(r.Context(), r.PathValue("attachment_id"), me.ID)
|
||||
id := r.PathValue("attachment_id")
|
||||
att, err := h.Store.GetAttachmentForUser(r.Context(), id, me.ID)
|
||||
switch {
|
||||
case err == nil:
|
||||
writeJSON(w, http.StatusOK, att)
|
||||
@@ -505,7 +522,27 @@ func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
key, err := h.Store.GetAttachmentObjectKey(r.Context(), att.ID)
|
||||
switch att.Status {
|
||||
case domain.AttachmentReady:
|
||||
// ok
|
||||
case domain.AttachmentFailed:
|
||||
writeError(w, r, http.StatusConflict, "conflict", "attachment failed")
|
||||
return
|
||||
default:
|
||||
writeError(w, r, http.StatusConflict, "conflict", "attachment not ready")
|
||||
return
|
||||
}
|
||||
|
||||
variant := r.URL.Query().Get("variant")
|
||||
if variant == "" {
|
||||
variant = "original"
|
||||
}
|
||||
var key string
|
||||
if variant == "original" {
|
||||
key, err = h.Store.GetAttachmentObjectKey(r.Context(), att.ID)
|
||||
} else {
|
||||
key, _, err = h.Store.GetVariantKey(r.Context(), att.ID, att.PipelineVersion, variant)
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "attachment not found")
|
||||
return
|
||||
@@ -535,3 +572,25 @@ func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
func (h *Handler) ReplayAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Только owner или участник чата; упрощённо — участник чата.
|
||||
me, _ := userFrom(r)
|
||||
id := r.PathValue("attachment_id")
|
||||
if _, err := h.Store.GetAttachmentForUser(r.Context(), id, me.ID); err != nil {
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "attachment not found")
|
||||
return
|
||||
}
|
||||
err := h.Store.ReplayFailed(r.Context(), id)
|
||||
if errors.Is(err, domain.ErrConflict) {
|
||||
writeError(w, r, http.StatusConflict, "conflict", "not in failed state")
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ func NewRouter(h *Handler, instanceID string) http.Handler {
|
||||
mux.HandleFunc("POST /api/v1/chats/{chat_id}/attachments", h.UploadAttachment)
|
||||
mux.HandleFunc("GET /api/v1/attachments/{attachment_id}", h.GetAttachment)
|
||||
mux.HandleFunc("GET /api/v1/attachments/{attachment_id}/content", h.DownloadAttachment)
|
||||
mux.HandleFunc("POST /api/v1/attachments/{attachment_id}/replay", h.ReplayAttachment)
|
||||
|
||||
// Порядок: instanceID снаружи, requestID внутри.
|
||||
return withInstanceID(instanceID)(withRequestID(mux))
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
package domain
|
||||
|
||||
const (
|
||||
AttachmentQueued = "queued"
|
||||
AttachmentProcessing = "processing"
|
||||
AttachmentReady = "ready"
|
||||
AttachmentFailed = "failed"
|
||||
)
|
||||
|
||||
const PipelineVersion = 1
|
||||
|
||||
var RequiredVariants = []string{"original", "thumbnail", "crop", "optimized"}
|
||||
+450
-43
@@ -3,6 +3,7 @@ package domain
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
@@ -11,57 +12,25 @@ import (
|
||||
|
||||
// CreateAttachmentPending резервирует строку ДО записи в S3.
|
||||
// Это даёт точку компенсации: если S3 PUT упал, строка снимается/чистится.
|
||||
func (s *Store) CreateAttachmentPending(ctx context.Context, chatID, ownerID, objectKey, filename string) (string, error) {
|
||||
func (s *Store) CreateAttachmentPending(ctx context.Context, chatID, ownerID, stagingKey, filename string) (string, error) {
|
||||
id := uuid.NewString()
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`INSERT INTO attachments (id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256, width, height, status)
|
||||
VALUES ($1,$2,$3,$4,$5,'image/png',1,repeat('0',64),1,1,'pending')`,
|
||||
id, chatID, ownerID, objectKey, filename)
|
||||
`INSERT INTO attachments (id, chat_id, owner_id, object_key, staging_key, filename,
|
||||
content_type, size_bytes, sha256, width, height,
|
||||
pipeline_version, status)
|
||||
VALUES ($1,$2,$3,$4,$4,$5,'application/octet-stream',0,repeat('0',64),0,0,1,'queued')`,
|
||||
id, chatID, ownerID, stagingKey, filename)
|
||||
if err != nil {
|
||||
return "", classifyDBError(err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// FinalizeAttachment атомарно переводит pending→ready с реальными метаданными.
|
||||
func (s *Store) FinalizeAttachment(ctx context.Context, id, contentType string, size int64, sha string, w, h int) (*Attachment, error) {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var a Attachment
|
||||
var errCode *string
|
||||
err = tx.QueryRow(ctx,
|
||||
`UPDATE attachments
|
||||
SET content_type=$2, size_bytes=$3, sha256=$4, width=$5, height=$6,
|
||||
status='ready', error_code=NULL, updated_at=now()
|
||||
WHERE id=$1 AND status='pending'
|
||||
RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
status, error_code, created_at, updated_at`,
|
||||
id, contentType, size, sha, w, h,
|
||||
).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256,
|
||||
&a.Status, &errCode, &a.CreatedAt, &a.UpdatedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
a.ErrorCode = errCode
|
||||
a.Variants = []Variant{}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
// MarkAttachmentFailed помечает pending как failed (компенсация).
|
||||
func (s *Store) MarkAttachmentFailed(ctx context.Context, id, code string) error {
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`UPDATE attachments SET status='failed', error_code=$2, updated_at=now()
|
||||
WHERE id=$1 AND status='pending'`, id, code)
|
||||
WHERE id=$1 AND status='queued'`, id, code)
|
||||
return classifyDBError(err)
|
||||
}
|
||||
|
||||
@@ -79,10 +48,10 @@ func (s *Store) GetAttachmentForUser(ctx context.Context, id, userID string) (*A
|
||||
var errCode *string
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256,
|
||||
a.status, a.error_code, a.created_at, a.updated_at
|
||||
a.status, a.error_code, a.created_at, a.updated_at, a.pipeline_version
|
||||
FROM attachments a
|
||||
JOIN chat_members m ON m.chat_id = a.chat_id
|
||||
WHERE a.id=$1 AND m.user_id=$2 AND a.status='ready'`, id, userID,
|
||||
WHERE a.id=$1 AND m.user_id=$2`, id, userID,
|
||||
).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256,
|
||||
&a.Status, &errCode, &a.CreatedAt, &a.UpdatedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
@@ -93,6 +62,27 @@ func (s *Store) GetAttachmentForUser(ctx context.Context, id, userID string) (*A
|
||||
}
|
||||
a.ErrorCode = errCode
|
||||
a.Variants = []Variant{}
|
||||
rows, err := s.pool.Query(ctx,
|
||||
`SELECT variant, content_type, size_bytes, width, height, sha256, object_key
|
||||
FROM attachment_variants
|
||||
WHERE attachment_id=$1 AND pipeline_version=$2
|
||||
ORDER BY variant`,
|
||||
a.ID, a.PipelineVersion)
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var v Variant
|
||||
if err := rows.Scan(&v.Name, &v.ContentType, &v.SizeBytes,
|
||||
&v.Width, &v.Height, &v.SHA256, &v.ObjectKey); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
a.Variants = append(a.Variants, v)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
@@ -164,8 +154,10 @@ func (s *Store) ReconcileOrphans(ctx context.Context, s3 *S3Store, grace time.Du
|
||||
|
||||
// 2) pending-записи старше grace: объект мог попасть в S3, а commit — нет.
|
||||
prow, err := s.pool.Query(ctx,
|
||||
`SELECT id, object_key FROM attachments
|
||||
WHERE status='pending' AND created_at < now() - $1::interval LIMIT 100`,
|
||||
`SELECT a.id, a.staging_key FROM attachments a
|
||||
LEFT JOIN attachment_jobs j ON j.attachment_id = a.id
|
||||
WHERE a.status='queued' AND j.job_id IS NULL
|
||||
AND a.created_at < now() - $1::interval LIMIT 100`,
|
||||
grace.String())
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
@@ -187,3 +179,418 @@ func (s *Store) ReconcileOrphans(ctx context.Context, s3 *S3Store, grace time.Du
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// EnqueueAttachment: атомарно фиксируем метаданные приёма и кладём outbox-строку.
|
||||
// 202 отвечаем только после успешного COMMIT этой транзакции.
|
||||
func (s *Store) EnqueueAttachment(
|
||||
ctx context.Context, attachmentID, contentType string,
|
||||
size int64, sha string, w, h int, pipelineVersion int,
|
||||
) (*Attachment, error) {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted})
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var a Attachment
|
||||
err = tx.QueryRow(ctx, `
|
||||
UPDATE attachments
|
||||
SET content_type=$2, size_bytes=$3, sha256=$4, width=$5, height=$6,
|
||||
pipeline_version=$7, status='queued', updated_at=now()
|
||||
WHERE id=$1 AND status='queued'
|
||||
RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
status, error_code, created_at, updated_at`,
|
||||
attachmentID, contentType, size, sha, w, h, pipelineVersion,
|
||||
).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes,
|
||||
&a.SHA256, &a.Status, &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO attachment_jobs
|
||||
(job_id, attachment_id, pipeline_version, status, next_attempt_at)
|
||||
VALUES ($1,$2,$3,'queued', now())`,
|
||||
uuid.NewString(), attachmentID, pipelineVersion,
|
||||
); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
a.Variants = []Variant{}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
// ClaimJob — берём одну доступную задачу под lease.
|
||||
// Работает и для queued, и для протухшего processing (reaper тем же запросом).
|
||||
func (s *Store) ClaimJob(ctx context.Context, workerID string, lease time.Duration) (*Job, error) {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted})
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var j Job
|
||||
err = tx.QueryRow(ctx, `
|
||||
WITH picked AS (
|
||||
SELECT job_id FROM attachment_jobs
|
||||
WHERE status IN ('queued','processing')
|
||||
AND next_attempt_at <= now()
|
||||
AND (status='queued' OR lease_expires_at < now())
|
||||
ORDER BY next_attempt_at
|
||||
LIMIT 1
|
||||
FOR UPDATE SKIP LOCKED
|
||||
)
|
||||
UPDATE attachment_jobs aj
|
||||
SET status='processing',
|
||||
lease_owner=$1,
|
||||
lease_expires_at = now() + $2::interval,
|
||||
updated_at = now()
|
||||
FROM picked
|
||||
WHERE aj.job_id = picked.job_id
|
||||
RETURNING aj.job_id, aj.attachment_id, aj.pipeline_version,
|
||||
aj.attempt, aj.status, aj.next_attempt_at, aj.last_error_code`,
|
||||
workerID, lease.String(),
|
||||
).Scan(&j.JobID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt, &j.Status,
|
||||
&j.NextAttemptAt, &j.LastError)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
return &j, nil
|
||||
}
|
||||
|
||||
// MarkReady: атомарно пишем все варианты (идемпотентно по ключу) и завершаем job.
|
||||
// Конкурентная попытка не перезапишет существующий вариант — ON CONFLICT DO NOTHING.
|
||||
func (s *Store) MarkReady(
|
||||
ctx context.Context, jobID, attachmentID string, pipelineVersion int,
|
||||
finalOriginalKey string, variants []Variant,
|
||||
) error {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.Serializable})
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
// Если job уже done — no-op (redelivery после ack).
|
||||
var status string
|
||||
err = tx.QueryRow(ctx,
|
||||
`SELECT status FROM attachment_jobs WHERE job_id=$1 FOR UPDATE`, jobID,
|
||||
).Scan(&status)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
if status == "done" {
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
if status == "failed" {
|
||||
return ErrConflict // нельзя откатывать failed→ready; см. replay
|
||||
}
|
||||
|
||||
// Записать оригинал в final prefix.
|
||||
if _, err := tx.Exec(ctx,
|
||||
`UPDATE attachments
|
||||
SET object_key=$2, status='ready', ready_at=now(), updated_at=now()
|
||||
WHERE id=$1 AND pipeline_version=$3`,
|
||||
attachmentID, finalOriginalKey, pipelineVersion,
|
||||
); err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
|
||||
for _, v := range variants {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO attachment_variants
|
||||
(attachment_id, pipeline_version, variant, object_key,
|
||||
content_type, size_bytes, width, height, sha256)
|
||||
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)
|
||||
ON CONFLICT (attachment_id, pipeline_version, variant) DO NOTHING`,
|
||||
attachmentID, pipelineVersion, v.Name, "final/"+attachmentID+"/"+v.Name,
|
||||
v.ContentType, v.SizeBytes, v.Width, v.Height, v.SHA256,
|
||||
); err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx,
|
||||
`UPDATE attachment_jobs
|
||||
SET status='done', lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
|
||||
WHERE job_id=$1`, jobID,
|
||||
); err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// RetryJob: временная ошибка — вернуть в очередь с backoff+jitter.
|
||||
// exhausted=true — если попытки кончились.
|
||||
func (s *Store) RetryJob(
|
||||
ctx context.Context, jobID string, errorCode string,
|
||||
maxAttempts int, baseBackoff time.Duration,
|
||||
) (exhausted bool, err error) {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return false, classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var attempt int
|
||||
var attachmentID string
|
||||
err = tx.QueryRow(ctx,
|
||||
`SELECT attempt, attachment_id FROM attachment_jobs
|
||||
WHERE job_id=$1 FOR UPDATE`, jobID,
|
||||
).Scan(&attempt, &attachmentID)
|
||||
if err != nil {
|
||||
return false, classifyDBError(err)
|
||||
}
|
||||
|
||||
if attempt >= maxAttempts {
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET status='failed', last_error_code=$2,
|
||||
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
|
||||
WHERE job_id=$1`, jobID, errorCode); err != nil {
|
||||
return false, classifyDBError(err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE attachments SET status='failed', failed_at=now(),
|
||||
error_code=$2, updated_at=now()
|
||||
WHERE id=$1`, attachmentID, errorCode); err != nil {
|
||||
return false, classifyDBError(err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, classifyDBError(err)
|
||||
}
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// backoff: base * 2^(attempt-1) + jitter
|
||||
backoff := baseBackoff * time.Duration(1<<(attempt-1))
|
||||
jitter := time.Duration(rand.Int63n(int64(backoff / 4)))
|
||||
next := time.Now().Add(backoff + jitter)
|
||||
|
||||
if _, err := tx.Exec(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET attempt = attempt + 1,
|
||||
status = 'queued',
|
||||
last_error_code=$2,
|
||||
next_attempt_at=$3,
|
||||
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
|
||||
WHERE job_id=$1`, jobID, errorCode, next); err != nil {
|
||||
return false, classifyDBError(err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, classifyDBError(err)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// RequeueStuck — reaper: обрабатывается отдельным тикером в worker/outbox.
|
||||
func (s *Store) RequeueStuck(ctx context.Context) (int64, error) {
|
||||
tag, err := s.pool.Exec(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET status='queued', lease_owner=NULL, lease_expires_at=NULL,
|
||||
next_attempt_at = now(), updated_at = now()
|
||||
WHERE status='processing' AND lease_expires_at < now()`)
|
||||
if err != nil {
|
||||
return 0, classifyDBError(err)
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
// ReplayFailed — операторский replay (уровень 4, CLI/endpoint).
|
||||
func (s *Store) ReplayFailed(ctx context.Context, attachmentID string) error {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
tag, err := tx.Exec(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET status='queued', attempt=1, next_attempt_at=now(),
|
||||
last_error_code=NULL, lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
|
||||
WHERE attachment_id=$1 AND status='failed'`, attachmentID)
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return ErrConflict
|
||||
}
|
||||
|
||||
if _, err := tx.Exec(ctx,
|
||||
`UPDATE attachments SET status='queued', error_code=NULL, failed_at=NULL,
|
||||
updated_at=now()
|
||||
WHERE id=$1 AND status='failed'`, attachmentID); err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// ClaimJobByID — берёт конкретную job под lease. Нужен для redelivery:
|
||||
// если worker упал после publish, повторно доставленное сообщение должно
|
||||
// либо взять ту же работу (lease протух), либо быть no-op (status=done).
|
||||
func (s *Store) ClaimJobByID(ctx context.Context, jobID, workerID string, lease time.Duration) (*Job, error) {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{IsoLevel: pgx.ReadCommitted})
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var j Job
|
||||
err = tx.QueryRow(ctx, `
|
||||
UPDATE attachment_jobs
|
||||
SET status='processing',
|
||||
lease_owner=$2,
|
||||
lease_expires_at = now() + $3::interval,
|
||||
updated_at = now()
|
||||
WHERE job_id=$1
|
||||
AND status IN ('queued','processing')
|
||||
AND (status='queued' OR lease_expires_at < now())
|
||||
RETURNING job_id, attachment_id, pipeline_version, attempt,
|
||||
status, next_attempt_at, last_error_code`,
|
||||
jobID, workerID, lease.String(),
|
||||
).Scan(&j.JobID, &j.AttachmentID, &j.PipelineVersion, &j.Attempt,
|
||||
&j.Status, &j.NextAttemptAt, &j.LastError)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
return &j, nil
|
||||
}
|
||||
|
||||
// GetStagingKey — доверенные метаданные для worker’а (не из сообщения брокера).
|
||||
func (s *Store) GetStagingKey(ctx context.Context, attachmentID string) (string, string, error) {
|
||||
var key, ct string
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT staging_key, content_type FROM attachments WHERE id=$1`, attachmentID,
|
||||
).Scan(&key, &ct)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", "", ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return "", "", classifyDBError(err)
|
||||
}
|
||||
return key, ct, nil
|
||||
}
|
||||
|
||||
// VariantsExist — no-op-проверка на повторную доставку уже готовой работы.
|
||||
func (s *Store) VariantsExist(ctx context.Context, attachmentID string, pv int) (bool, error) {
|
||||
var n int
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT count(*) FROM attachment_variants
|
||||
WHERE attachment_id=$1 AND pipeline_version=$2`,
|
||||
attachmentID, pv,
|
||||
).Scan(&n)
|
||||
if err != nil {
|
||||
return false, classifyDBError(err)
|
||||
}
|
||||
return n >= len(RequiredVariants), nil
|
||||
}
|
||||
|
||||
// MarkFailedPermanent — терминальный failed без retry (постоянная ошибка).
|
||||
func (s *Store) MarkFailedPermanent(ctx context.Context, jobID, attachmentID, code string) error {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
if _, err := tx.Exec(ctx,
|
||||
`UPDATE attachment_jobs SET status='failed', last_error_code=$2,
|
||||
lease_owner=NULL, lease_expires_at=NULL, updated_at=now()
|
||||
WHERE job_id=$1 AND status='processing'`, jobID, code); err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx,
|
||||
`UPDATE attachments SET status='failed', error_code=$2,
|
||||
failed_at=now(), updated_at=now()
|
||||
WHERE id=$1`, attachmentID, code); err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
type UnpublishedJob struct {
|
||||
JobID string
|
||||
AttachmentID string
|
||||
PipelineVersion int
|
||||
Attempt int
|
||||
}
|
||||
|
||||
// SelectUnpublishedJobs — выборка outbox’ом. Транзакция не нужна:
|
||||
// published_at ставим отдельным UPDATE после XADD; повторная публикация
|
||||
// допустима (worker идемпотентен).
|
||||
func (s *Store) SelectUnpublishedJobs(ctx context.Context, limit int) ([]UnpublishedJob, error) {
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT job_id, attachment_id, pipeline_version, attempt
|
||||
FROM attachment_jobs
|
||||
WHERE published_at IS NULL AND status='queued'
|
||||
AND next_attempt_at <= now()
|
||||
ORDER BY next_attempt_at
|
||||
LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []UnpublishedJob
|
||||
for rows.Next() {
|
||||
var u UnpublishedJob
|
||||
if err := rows.Scan(&u.JobID, &u.AttachmentID, &u.PipelineVersion, &u.Attempt); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
out = append(out, u)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// MarkPublished — фиксируем факт XADD (publisher confirm).
|
||||
func (s *Store) MarkPublished(ctx context.Context, jobID string) error {
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`UPDATE attachment_jobs SET published_at=now(),
|
||||
publish_attempts=publish_attempts+1, updated_at=now()
|
||||
WHERE job_id=$1`, jobID)
|
||||
return classifyDBError(err)
|
||||
}
|
||||
|
||||
// BumpPublishAttempt — XADD упал; фиксируем причину, повторим в следующем цикле.
|
||||
func (s *Store) BumpPublishAttempt(ctx context.Context, jobID, code string) error {
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`UPDATE attachment_jobs SET publish_attempts=publish_attempts+1,
|
||||
last_error_code=$2, updated_at=now()
|
||||
WHERE job_id=$1`, jobID, code)
|
||||
return classifyDBError(err)
|
||||
}
|
||||
|
||||
func (s *Store) GetVariantKey(ctx context.Context, attachmentID string, pv int, variant string) (string, string, error) {
|
||||
var key, ct string
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT object_key, content_type FROM attachment_variants
|
||||
WHERE attachment_id=$1 AND pipeline_version=$2 AND variant=$3`,
|
||||
attachmentID, pv, variant).Scan(&key, &ct)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", "", ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return "", "", classifyDBError(err)
|
||||
}
|
||||
return key, ct, nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
StreamName = "attachment.jobs"
|
||||
ConsumerGroup = "workers"
|
||||
)
|
||||
|
||||
type JobMessage struct {
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
JobID string `json:"job_id"`
|
||||
Type string `json:"type"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
PipelineVersion int `json:"pipeline_version"`
|
||||
Attempt int `json:"attempt"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
Traceparent string `json:"traceparent,omitempty"`
|
||||
}
|
||||
|
||||
type Broker struct{ rdb *redis.Client }
|
||||
|
||||
func NewBroker(ctx context.Context, url string) (*Broker, error) {
|
||||
opt, err := redis.ParseURL(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rdb := redis.NewClient(opt)
|
||||
if err := rdb.Ping(ctx).Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Группа создаётся идемпотентно.
|
||||
err = rdb.XGroupCreateMkStream(ctx, StreamName, ConsumerGroup, "0").Err()
|
||||
if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") {
|
||||
return nil, err
|
||||
}
|
||||
return &Broker{rdb: rdb}, nil
|
||||
}
|
||||
|
||||
func (b *Broker) Publish(ctx context.Context, m JobMessage) (string, error) {
|
||||
buf, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return b.rdb.XAdd(ctx, &redis.XAddArgs{
|
||||
Stream: StreamName,
|
||||
Values: map[string]any{"payload": buf},
|
||||
}).Result()
|
||||
}
|
||||
|
||||
// Consume — блокирующее чтение из consumer group; ack делает вызывающий.
|
||||
func (b *Broker) Consume(ctx context.Context, consumer string) ([]redis.XMessage, error) {
|
||||
res, err := b.rdb.XReadGroup(ctx, &redis.XReadGroupArgs{
|
||||
Group: ConsumerGroup, Consumer: consumer,
|
||||
Streams: []string{StreamName, ">"},
|
||||
Count: 1, Block: 5 * time.Second,
|
||||
}).Result()
|
||||
if errors.Is(err, redis.Nil) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(res) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return res[0].Messages, nil
|
||||
}
|
||||
|
||||
func (b *Broker) Ack(ctx context.Context, id string) error {
|
||||
return b.rdb.XAck(ctx, StreamName, ConsumerGroup, id).Err()
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
package domain
|
||||
|
||||
import "time"
|
||||
|
||||
type Job struct {
|
||||
JobID string
|
||||
AttachmentID string
|
||||
PipelineVersion int
|
||||
Attempt int
|
||||
Status string
|
||||
NextAttemptAt time.Time
|
||||
LastError *string
|
||||
}
|
||||
+14
-12
@@ -62,21 +62,23 @@ type Variant struct {
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
SHA256 string `json:"sha256"`
|
||||
ObjectKey string `json:"-"`
|
||||
}
|
||||
|
||||
type Attachment struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []Variant `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
PipelineVersion int `json:"-"`
|
||||
Variants []Variant `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
var (
|
||||
|
||||
@@ -158,6 +158,51 @@ func (s *S3Store) PutValidatedImage(ctx context.Context, objectKey string, body
|
||||
}, nil
|
||||
}
|
||||
|
||||
const (
|
||||
StagingPrefix = "staging/"
|
||||
FinalPrefix = "final/"
|
||||
)
|
||||
|
||||
// MoveToFinal: копирование staging→final и удаление staging.
|
||||
// Используем CopyObject, чтобы не тянуть байты в память. Если MinIO не умеет
|
||||
// server-side copy — используйте compose? MinIO умеет S3 CopyObject.
|
||||
func (s *S3Store) MoveToFinal(ctx context.Context, stagingKey, finalKey string) error {
|
||||
src := minio.CopySrcOptions{Bucket: s.bucket, Object: stagingKey}
|
||||
dst := minio.CopyDestOptions{Bucket: s.bucket, Object: finalKey}
|
||||
if _, err := s.client.CopyObject(ctx, dst, src); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.DeleteObject(ctx, stagingKey)
|
||||
}
|
||||
|
||||
func (s *S3Store) PutBytes(ctx context.Context, key, contentType string, data []byte) (string, error) {
|
||||
h := sha256.Sum256(data)
|
||||
_, err := s.client.PutObject(ctx, s.bucket, key, bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: contentType})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(h[:]), nil
|
||||
}
|
||||
|
||||
// FetchStagingForProcessing — читает staging в память с жёстким лимитом.
|
||||
// maxBytes гарантирует, что worker не съест RAM на «злом» файле.
|
||||
func (s *S3Store) FetchStagingForProcessing(ctx context.Context, key string) ([]byte, string, error) {
|
||||
rc, ct, size, err := s.GetObject(ctx, key)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
defer rc.Close()
|
||||
if size > s.maxBytes {
|
||||
return nil, "", ErrTooLarge
|
||||
}
|
||||
buf, err := io.ReadAll(rc)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return buf, ct, nil
|
||||
}
|
||||
|
||||
func (s *S3Store) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, string, int64, error) {
|
||||
// Общий дедлайн на все попытки — 6 секунд.
|
||||
overallCtx, overallCancel := context.WithTimeout(ctx, 6*time.Second)
|
||||
|
||||
Reference in New Issue
Block a user