feat(worker): process attachments with thumbnails and bounded retries
This commit is contained in:
@@ -0,0 +1,13 @@
|
||||
FROM golang:1.27.1 AS build
|
||||
WORKDIR /src
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
RUN go mod download
|
||||
|
||||
COPY . .
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -o /out/worker ./cmd/worker
|
||||
|
||||
FROM gcr.io/distroless/static-debian12:nonroot
|
||||
WORKDIR /app
|
||||
COPY --from=build /out/worker .
|
||||
ENTRYPOINT ["/app/worker"]
|
||||
@@ -0,0 +1,68 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
|
||||
"github.com/joho/godotenv"
|
||||
|
||||
"mtusi2026-messenger/internal/config"
|
||||
"mtusi2026-messenger/internal/logger"
|
||||
"mtusi2026-messenger/internal/postgres"
|
||||
s3store "mtusi2026-messenger/internal/s3"
|
||||
"mtusi2026-messenger/internal/worker"
|
||||
)
|
||||
|
||||
func main() {
|
||||
if err := run(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "worker:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
if err := godotenv.Load(); err != nil && !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg := config.Load()
|
||||
log := logger.New(cfg.Logging.Level)
|
||||
databaseURL, err := cfg.Database.URL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
brokerURL, err := cfg.Broker.URL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
|
||||
defer stop()
|
||||
|
||||
pool, err := postgres.New(ctx, databaseURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer pool.Close()
|
||||
|
||||
storage, err := s3store.New(ctx, cfg.S3, log)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
worker.New(
|
||||
pool,
|
||||
storage,
|
||||
brokerURL,
|
||||
config.LoadWorker(),
|
||||
cfg.S3.MaxUploadBytes,
|
||||
cfg.S3.MaxImagePixels,
|
||||
log,
|
||||
).Run(ctx)
|
||||
|
||||
return nil
|
||||
}
|
||||
+29
-1
@@ -3,7 +3,7 @@ services:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: mtusi-messenger:lab4
|
||||
image: mtusi-messenger:lab5
|
||||
env_file:
|
||||
- .env
|
||||
secrets:
|
||||
@@ -27,6 +27,34 @@ services:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
worker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.worker
|
||||
image: mtusi-messenger-worker:lab5
|
||||
env_file:
|
||||
- .env
|
||||
secrets:
|
||||
- database_url
|
||||
- broker_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
- s3_bootstrap_access_key_id
|
||||
- s3_bootstrap_secret_access_key
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 1G
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3:
|
||||
condition: service_healthy
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
postgres:
|
||||
image: postgres:18.6
|
||||
ports:
|
||||
|
||||
@@ -121,8 +121,6 @@ UPDATE attachments
|
||||
SET status = 'ready',
|
||||
error_code = NULL,
|
||||
lease_until = NULL,
|
||||
width = sqlc.arg(width)::int,
|
||||
height = sqlc.arg(height)::int,
|
||||
updated_at = now()
|
||||
WHERE id = sqlc.arg(id)::uuid
|
||||
AND status = 'processing'
|
||||
|
||||
@@ -8,6 +8,7 @@ require (
|
||||
github.com/aws/aws-sdk-go-v2/credentials v1.20.13
|
||||
github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6
|
||||
github.com/aws/smithy-go v1.28.7
|
||||
github.com/disintegration/imaging v1.6.2
|
||||
github.com/go-chi/chi/v5 v5.5.0
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.11.0
|
||||
@@ -36,7 +37,8 @@ require (
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
go.uber.org/atomic v1.12.0 // indirect
|
||||
golang.org/x/sync v0.24.0 // indirect
|
||||
golang.org/x/sys v0.49.0 // indirect
|
||||
golang.org/x/text v0.43.0 // indirect
|
||||
golang.org/x/image v0.48.0 // indirect
|
||||
golang.org/x/sync v0.25.0 // indirect
|
||||
golang.org/x/sys v0.50.0 // indirect
|
||||
golang.org/x/text v0.44.0 // indirect
|
||||
)
|
||||
|
||||
@@ -41,6 +41,8 @@ github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
|
||||
github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
|
||||
github.com/go-chi/chi/v5 v5.5.0 h1:xdKQ1dJmVJsmgahrXSKhUx8NyVPzpwdlyrrQFpMPJgg=
|
||||
github.com/go-chi/chi/v5 v5.5.0/go.mod h1:d1IG7GtKPSjPyBP365PH8sAGVHk3qfB/IhjFNRH1wWM=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
@@ -77,11 +79,15 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54=
|
||||
golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w=
|
||||
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
|
||||
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc=
|
||||
golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s=
|
||||
golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||
golang.org/x/image v0.48.0 h1:1AvuPcuoOTJpA7s4VZ5umrb90T3So0rkB/bSGmxkxik=
|
||||
golang.org/x/image v0.48.0/go.mod h1:mmtg64iqJKGqEgy+r13dfybl85tmxUd2lMnmaiSdWYY=
|
||||
golang.org/x/sync v0.25.0 h1:dJnifr0nMV8/SBnBvdo4eSGGnrc3O4FFx6IqvtLcmRs=
|
||||
golang.org/x/sync v0.25.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.50.0 h1:dkKZEFTEKaLr1sjhs0WiKqoi8Jj5nHbS3Njmx1UoMJs=
|
||||
golang.org/x/sys v0.50.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.44.0 h1:YItdp/mQNGnsmrgpcdA5yQ3bnRxCfo/Tj/iByGW9GGo=
|
||||
golang.org/x/text v0.44.0/go.mod h1:S4FR1waQ9LbjQTls97SyRjDaUUVkaM+cXwwuG/G+MwY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -141,6 +141,20 @@ func LoadS3() S3Config {
|
||||
}
|
||||
}
|
||||
|
||||
type WorkerConfig struct {
|
||||
Concurrency int
|
||||
MaxAttempts int
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
func LoadWorker() WorkerConfig {
|
||||
return WorkerConfig{
|
||||
Concurrency: getEnvPositiveInt("WORKER_CONCURRENCY", 2),
|
||||
MaxAttempts: getEnvPositiveInt("JOB_MAX_ATTEMPTS", 5),
|
||||
Timeout: time.Duration(getEnvPositiveInt("JOB_TIMEOUT_SECONDS", 60)) * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func getEnv(key, def string) string {
|
||||
value := os.Getenv(key)
|
||||
if value == "" {
|
||||
|
||||
@@ -72,29 +72,20 @@ UPDATE attachments
|
||||
SET status = 'ready',
|
||||
error_code = NULL,
|
||||
lease_until = NULL,
|
||||
width = $1::int,
|
||||
height = $2::int,
|
||||
updated_at = now()
|
||||
WHERE id = $3::uuid
|
||||
WHERE id = $1::uuid
|
||||
AND status = 'processing'
|
||||
AND attempts = $4::int
|
||||
AND attempts = $2::int
|
||||
AND lease_until > now()
|
||||
`
|
||||
|
||||
type CompleteAttachmentJobParams struct {
|
||||
Width int
|
||||
Height int
|
||||
ID uuid.UUID
|
||||
Attempt int
|
||||
}
|
||||
|
||||
func (q *Queries) CompleteAttachmentJob(ctx context.Context, arg CompleteAttachmentJobParams) (int64, error) {
|
||||
result, err := q.db.Exec(ctx, completeAttachmentJob,
|
||||
arg.Width,
|
||||
arg.Height,
|
||||
arg.ID,
|
||||
arg.Attempt,
|
||||
)
|
||||
result, err := q.db.Exec(ctx, completeAttachmentJob, arg.ID, arg.Attempt)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
+41
-22
@@ -19,27 +19,27 @@ type GetAttachmentContentRequest struct {
|
||||
}
|
||||
|
||||
type VariantResponse struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int `json:"size_bytes"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
Sha256 string `json:"sha256"`
|
||||
Name model.AttachmentVariantName `json:"name"`
|
||||
ContentType model.ContentType `json:"content_type"`
|
||||
SizeBytes int `json:"size_bytes"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
Sha256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
type AttachmentResponse struct {
|
||||
ID uuid.UUID `json:"id"`
|
||||
ChatID uuid.UUID `json:"chat_id"`
|
||||
OwnerID uuid.UUID `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int `json:"size_bytes"`
|
||||
Sha256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []VariantResponse `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
ID uuid.UUID `json:"id"`
|
||||
ChatID uuid.UUID `json:"chat_id"`
|
||||
OwnerID uuid.UUID `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType model.ContentType `json:"content_type"`
|
||||
SizeBytes int `json:"size_bytes"`
|
||||
Sha256 string `json:"sha256"`
|
||||
Status model.AttachmentStatus `json:"status"`
|
||||
Variants []VariantResponse `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
func (r CreateAttachmentRequest) Validate() bool {
|
||||
@@ -57,20 +57,39 @@ func (r CreateAttachmentRequest) Validate() bool {
|
||||
}
|
||||
|
||||
func (r GetAttachmentContentRequest) Validate() bool {
|
||||
return r.Variant == "" || r.Variant == "original"
|
||||
switch model.AttachmentVariantName(r.Variant) {
|
||||
case "",
|
||||
model.AttachmentContentOriginal,
|
||||
model.AttachmentVariantNameThumbnail,
|
||||
model.AttachmentVariantNameCrop,
|
||||
model.AttachmentVariantNameOptimized:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func ToAttachmentResponse(a model.Attachment) AttachmentResponse {
|
||||
variants := make([]VariantResponse, 0)
|
||||
variants := make([]VariantResponse, 0, len(a.Variants))
|
||||
for _, v := range a.Variants {
|
||||
variants = append(variants, VariantResponse{
|
||||
Name: v.Name,
|
||||
ContentType: v.ContentType,
|
||||
SizeBytes: v.SizeBytes,
|
||||
Width: v.Width,
|
||||
Height: v.Height,
|
||||
Sha256: v.Sha256,
|
||||
})
|
||||
}
|
||||
return AttachmentResponse{
|
||||
ID: a.ID,
|
||||
ChatID: a.ChatID,
|
||||
OwnerID: a.OwnerID,
|
||||
Filename: a.Filename,
|
||||
ContentType: string(a.ContentType),
|
||||
ContentType: a.ContentType,
|
||||
SizeBytes: a.SizeBytes,
|
||||
Sha256: a.Sha256,
|
||||
Status: string(a.Status),
|
||||
Status: a.Status,
|
||||
Variants: variants,
|
||||
ErrorCode: a.ErrorCode,
|
||||
CreatedAt: a.CreatedAt,
|
||||
|
||||
@@ -121,7 +121,7 @@ func (h *AttachmentHandler) GetContent(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
data, contentType, err := h.service.GetContent(r.Context(), userID, attachmentID)
|
||||
data, contentType, err := h.service.GetContent(r.Context(), userID, attachmentID, req.Variant)
|
||||
switch {
|
||||
case errors.Is(err, service.ErrAttachmentNotReady):
|
||||
httpx.WriteError(w, http.StatusConflict, dto.ErrorCodeConflict, "attachment not ready", requestID)
|
||||
|
||||
@@ -11,15 +11,39 @@ type ContentType string
|
||||
const (
|
||||
ContentTypePNG ContentType = "image/png"
|
||||
ContentTypeJPEG ContentType = "image/jpeg"
|
||||
ContentTypeWebP ContentType = "image/webp"
|
||||
)
|
||||
|
||||
type AttachmentStatus string
|
||||
|
||||
const (
|
||||
AttachmentStatusReady AttachmentStatus = "ready"
|
||||
AttachmentStatusQueued AttachmentStatus = "queued"
|
||||
AttachmentStatusProcessing AttachmentStatus = "processing"
|
||||
AttachmentStatusReady AttachmentStatus = "ready"
|
||||
AttachmentStatusFailed AttachmentStatus = "failed"
|
||||
)
|
||||
|
||||
const AttachmentContentOriginal = "original"
|
||||
|
||||
type AttachmentVariantName string
|
||||
|
||||
const (
|
||||
AttachmentVariantNameThumbnail AttachmentVariantName = "thumbnail"
|
||||
AttachmentVariantNameCrop AttachmentVariantName = "crop"
|
||||
AttachmentVariantNameOptimized AttachmentVariantName = "optimized"
|
||||
)
|
||||
|
||||
type AttachmentVariant struct {
|
||||
Name AttachmentVariantName
|
||||
ContentType ContentType
|
||||
SizeBytes int
|
||||
Width int
|
||||
Height int
|
||||
Sha256 string
|
||||
}
|
||||
|
||||
type Attachment struct {
|
||||
Variants []AttachmentVariant
|
||||
ID uuid.UUID
|
||||
ChatID uuid.UUID
|
||||
OwnerID uuid.UUID
|
||||
@@ -28,6 +52,8 @@ type Attachment struct {
|
||||
SizeBytes int
|
||||
Sha256 string
|
||||
Status AttachmentStatus
|
||||
Width *int32
|
||||
Height *int32
|
||||
ErrorCode *string
|
||||
CreatedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
|
||||
@@ -35,6 +35,7 @@ type attachmentQueries interface {
|
||||
CreateAttachment(ctx context.Context, arg db.CreateAttachmentParams) (db.Attachment, error)
|
||||
CreateOutboxEntry(ctx context.Context, arg db.CreateOutboxEntryParams) (db.Outbox, error)
|
||||
GetAttachmentByID(ctx context.Context, id uuid.UUID) (db.Attachment, error)
|
||||
ListAttachmentVariants(ctx context.Context, arg db.ListAttachmentVariantsParams) ([]db.AttachmentVariant, error)
|
||||
IsChatMember(ctx context.Context, arg db.IsChatMemberParams) (bool, error)
|
||||
}
|
||||
|
||||
@@ -183,6 +184,8 @@ func (s *AttachmentService) Upload(
|
||||
SizeBytes: row.SizeBytes,
|
||||
Sha256: row.Sha256,
|
||||
Status: model.AttachmentStatus(row.Status),
|
||||
Width: row.Width,
|
||||
Height: row.Height,
|
||||
ErrorCode: row.ErrorCode,
|
||||
CreatedAt: row.CreatedAt,
|
||||
UpdatedAt: row.UpdatedAt,
|
||||
@@ -205,7 +208,24 @@ func (s *AttachmentService) Get(
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
|
||||
rows, err := s.q.ListAttachmentVariants(ctx, db.ListAttachmentVariantsParams{AttachmentID: row.ID, PipelineVersion: row.PipelineVersion})
|
||||
if err != nil {
|
||||
return model.Attachment{}, err
|
||||
}
|
||||
variants := make([]model.AttachmentVariant, 0, len(rows))
|
||||
for _, v := range rows {
|
||||
variants = append(variants, model.AttachmentVariant{
|
||||
Name: model.AttachmentVariantName(v.Variant),
|
||||
ContentType: model.ContentType(v.ContentType),
|
||||
SizeBytes: v.SizeBytes,
|
||||
Width: v.Width,
|
||||
Height: v.Height,
|
||||
Sha256: v.Sha256,
|
||||
})
|
||||
}
|
||||
|
||||
return model.Attachment{
|
||||
Variants: variants,
|
||||
ID: row.ID,
|
||||
ChatID: row.ChatID,
|
||||
OwnerID: row.OwnerID,
|
||||
@@ -214,6 +234,8 @@ func (s *AttachmentService) Get(
|
||||
SizeBytes: row.SizeBytes,
|
||||
Sha256: row.Sha256,
|
||||
Status: model.AttachmentStatus(row.Status),
|
||||
Width: row.Width,
|
||||
Height: row.Height,
|
||||
ErrorCode: row.ErrorCode,
|
||||
CreatedAt: row.CreatedAt,
|
||||
UpdatedAt: row.UpdatedAt,
|
||||
@@ -223,6 +245,7 @@ func (s *AttachmentService) Get(
|
||||
func (s *AttachmentService) GetContent(
|
||||
ctx context.Context,
|
||||
userID, attachmentID uuid.UUID,
|
||||
variant string,
|
||||
) ([]byte, model.ContentType, error) {
|
||||
row, err := s.q.GetAttachmentByID(ctx, attachmentID)
|
||||
if postgres.IsNotFound(err) {
|
||||
@@ -240,7 +263,24 @@ func (s *AttachmentService) GetContent(
|
||||
return nil, "", ErrAttachmentNotReady
|
||||
}
|
||||
|
||||
out, err := s.s3.Get(ctx, row.S3Key)
|
||||
key, contentType := row.S3Key, model.ContentType(row.ContentType)
|
||||
if variant != "" && variant != model.AttachmentContentOriginal {
|
||||
rows, err := s.q.ListAttachmentVariants(ctx, db.ListAttachmentVariantsParams{AttachmentID: row.ID, PipelineVersion: row.PipelineVersion})
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
found := false
|
||||
for _, v := range rows {
|
||||
if string(v.Variant) == variant {
|
||||
key, contentType, found = v.S3Key, model.ContentType(v.ContentType), true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
return nil, "", ErrAttachmentNotFound
|
||||
}
|
||||
}
|
||||
out, err := s.s3.Get(ctx, key)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
@@ -251,7 +291,7 @@ func (s *AttachmentService) GetContent(
|
||||
return nil, "", err
|
||||
}
|
||||
|
||||
return data, model.ContentType(row.ContentType), nil
|
||||
return data, contentType, nil
|
||||
}
|
||||
|
||||
func (s *AttachmentService) checkImage(ctx context.Context, r io.ReadSeeker, contentType string) (model.ContentType, int, int, error) {
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/draw"
|
||||
"image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/disintegration/imaging"
|
||||
|
||||
"mtusi2026-messenger/internal/db"
|
||||
)
|
||||
|
||||
var errInvalidSource = errors.New("invalid source image")
|
||||
|
||||
func thumbnail(r io.ReadSeeker, maxPixels int) ([]byte, int, int, error) {
|
||||
cfg, format, err := image.DecodeConfig(r)
|
||||
if err != nil ||
|
||||
(format != "png" && format != "jpeg") ||
|
||||
cfg.Width <= 0 ||
|
||||
cfg.Height <= 0 ||
|
||||
cfg.Width > maxPixels/cfg.Height {
|
||||
return nil, 0, 0, errInvalidSource
|
||||
}
|
||||
|
||||
if _, err := r.Seek(0, io.SeekStart); err != nil {
|
||||
return nil, 0, 0, err
|
||||
}
|
||||
|
||||
src, err := imaging.Decode(r, imaging.AutoOrientation(true))
|
||||
if err != nil {
|
||||
return nil, 0, 0, errInvalidSource
|
||||
}
|
||||
|
||||
width, height := src.Bounds().Dx(), src.Bounds().Dy()
|
||||
if width > 256 || height > 256 {
|
||||
if width >= height {
|
||||
height = max(1, height*256/width)
|
||||
width = 256
|
||||
} else {
|
||||
width = max(1, width*256/height)
|
||||
height = 256
|
||||
}
|
||||
|
||||
src = imaging.Resize(src, width, height, imaging.Lanczos)
|
||||
}
|
||||
|
||||
dst := image.NewRGBA(image.Rect(0, 0, width, height))
|
||||
draw.Draw(dst, dst.Bounds(), image.NewUniform(color.White), image.Point{}, draw.Src)
|
||||
draw.Draw(dst, dst.Bounds(), src, src.Bounds().Min, draw.Over)
|
||||
|
||||
var out bytes.Buffer
|
||||
if err := jpeg.Encode(&out, dst, &jpeg.Options{Quality: 85}); err != nil {
|
||||
return nil, 0, 0, err
|
||||
}
|
||||
|
||||
return out.Bytes(), width, height, nil
|
||||
}
|
||||
|
||||
func (w *Worker) process(ctx context.Context, row db.Attachment) (db.CreateAttachmentVariantParams, error) {
|
||||
if row.StagingKey == nil {
|
||||
return db.CreateAttachmentVariantParams{}, errInvalidSource
|
||||
}
|
||||
|
||||
source, err := w.s3.Get(ctx, *row.StagingKey)
|
||||
if err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
}
|
||||
|
||||
defer source.Body.Close()
|
||||
|
||||
tmp, err := os.CreateTemp("", "attachment-job-*")
|
||||
if err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = tmp.Close()
|
||||
_ = os.Remove(tmp.Name())
|
||||
}()
|
||||
|
||||
hash := sha256.New()
|
||||
size, err := io.Copy(io.MultiWriter(tmp, hash), io.LimitReader(source.Body, int64(w.maxBytes)+1))
|
||||
if err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
}
|
||||
|
||||
if size != int64(row.SizeBytes) ||
|
||||
hex.EncodeToString(hash.Sum(nil)) != row.Sha256 {
|
||||
return db.CreateAttachmentVariantParams{}, errInvalidSource
|
||||
}
|
||||
|
||||
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
}
|
||||
|
||||
select {
|
||||
case w.decodeSlot <- struct{}{}:
|
||||
case <-ctx.Done():
|
||||
return db.CreateAttachmentVariantParams{}, ctx.Err()
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
<-w.decodeSlot
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
}
|
||||
|
||||
data, width, height, err := thumbnail(tmp, w.maxPixels)
|
||||
<-w.decodeSlot
|
||||
if err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
}
|
||||
|
||||
if err := ctx.Err(); err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
}
|
||||
|
||||
if _, err := tmp.Seek(0, io.SeekStart); err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
}
|
||||
|
||||
if err := w.s3.Put(ctx, row.S3Key, tmp, size, string(row.ContentType)); err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
}
|
||||
|
||||
key := "variants/" + row.ID.String() + "/v" + strconv.Itoa(row.PipelineVersion) + "/thumbnail"
|
||||
if err := w.s3.Put(ctx, key, bytes.NewReader(data), int64(len(data)), "image/jpeg"); err != nil {
|
||||
return db.CreateAttachmentVariantParams{}, err
|
||||
}
|
||||
|
||||
sum := sha256.Sum256(data)
|
||||
return db.CreateAttachmentVariantParams{
|
||||
AttachmentID: row.ID,
|
||||
PipelineVersion: row.PipelineVersion,
|
||||
Variant: db.AttachmentVariantNameThumbnail,
|
||||
S3Key: key,
|
||||
ContentType: "image/jpeg",
|
||||
SizeBytes: len(data),
|
||||
Width: width,
|
||||
Height: height,
|
||||
Sha256: hex.EncodeToString(sum[:]),
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestThumbnail(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
width, height, wantWidth, wantHeight int
|
||||
}{
|
||||
{"landscape", 320, 200, 256, 160}, {"portrait", 201, 321, 160, 256},
|
||||
{"small", 12, 7, 12, 7}, {"thin", 1, 1000, 1, 256},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
var source bytes.Buffer
|
||||
if err := png.Encode(&source, image.NewNRGBA(image.Rect(0, 0, tc.width, tc.height))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
data, width, height, err := thumbnail(bytes.NewReader(source.Bytes()), 40000000)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if width != tc.wantWidth || height != tc.wantHeight {
|
||||
t.Fatalf("got %dx%d", width, height)
|
||||
}
|
||||
|
||||
decoded, err := jpeg.Decode(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if decoded.Bounds().Dx() != width || decoded.Bounds().Dy() != height {
|
||||
t.Fatal("JPEG dimensions disagree")
|
||||
}
|
||||
|
||||
r, g, b, _ := decoded.At(0, 0).RGBA()
|
||||
if r < 65000 || g < 65000 || b < 65000 {
|
||||
t.Fatal("transparent source was not composited on white")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailEXIF(t *testing.T) {
|
||||
src := image.NewRGBA(image.Rect(0, 0, 40, 20))
|
||||
|
||||
for y := 0; y < 20; y++ {
|
||||
for x := 0; x < 40; x++ {
|
||||
c := color.RGBA{255, 0, 0, 255}
|
||||
if x >= 20 {
|
||||
c = color.RGBA{0, 0, 255, 255}
|
||||
}
|
||||
|
||||
src.Set(x, y, c)
|
||||
}
|
||||
}
|
||||
|
||||
var original bytes.Buffer
|
||||
if err := jpeg.Encode(&original, src, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Little-endian TIFF, orientation=6 (90 degrees clockwise).
|
||||
exif := []byte{
|
||||
'E', 'x', 'i', 'f', 0, 0, 'I', 'I',
|
||||
42, 0, 8, 0, 0, 0, 1, 0,
|
||||
0x12, 1, 3, 0, 1, 0, 0, 0,
|
||||
6, 0, 0, 0, 0, 0, 0, 0,
|
||||
}
|
||||
segment := []byte{0xff, 0xe1, 0, 0}
|
||||
binary.BigEndian.PutUint16(segment[2:], uint16(len(exif)+2))
|
||||
data := append([]byte{}, original.Bytes()[:2]...)
|
||||
data = append(data, segment...)
|
||||
data = append(data, exif...)
|
||||
data = append(data, original.Bytes()[2:]...)
|
||||
result, width, height, err := thumbnail(bytes.NewReader(data), 40000000)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if width != 20 || height != 40 {
|
||||
t.Fatalf("orientation ignored: %dx%d", width, height)
|
||||
}
|
||||
|
||||
decoded, err := jpeg.Decode(bytes.NewReader(result))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
r, _, b, _ := decoded.At(10, 5).RGBA()
|
||||
if r <= b {
|
||||
t.Fatal("incorrect EXIF rotation")
|
||||
}
|
||||
|
||||
if bytes.Contains(result, []byte("Exif")) {
|
||||
t.Fatal("EXIF retained")
|
||||
}
|
||||
}
|
||||
|
||||
func TestThumbnailRejectsInvalidSource(t *testing.T) {
|
||||
var source bytes.Buffer
|
||||
if err := png.Encode(&source, image.NewNRGBA(image.Rect(0, 0, 100, 100))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, _, _, err := thumbnail(bytes.NewReader(source.Bytes()), 9999); err == nil {
|
||||
t.Fatal("pixel limit ignored")
|
||||
}
|
||||
|
||||
if _, _, _, err := thumbnail(bytes.NewReader(source.Bytes()[:20]), 40000000); err == nil {
|
||||
t.Fatal("truncated image accepted")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,465 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math/rand/v2"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
|
||||
"mtusi2026-messenger/internal/config"
|
||||
"mtusi2026-messenger/internal/db"
|
||||
"mtusi2026-messenger/internal/model"
|
||||
"mtusi2026-messenger/internal/rabbitmq"
|
||||
s3store "mtusi2026-messenger/internal/s3"
|
||||
)
|
||||
|
||||
var (
|
||||
errLeaseExpired = errors.New("attachment lease expired")
|
||||
errMissingThumbnail = errors.New("ready attachment has no thumbnail")
|
||||
errAttemptProcessing = errors.New("attachment attempt is still processing")
|
||||
errAttemptUnavailable = errors.New("attachment attempt is not yet available")
|
||||
errEffectChecksumMismatch = errors.New("attachment effect checksum mismatch")
|
||||
)
|
||||
|
||||
type Worker struct {
|
||||
pool *pgxpool.Pool
|
||||
q *db.Queries
|
||||
s3 *s3store.Client
|
||||
brokerURL string
|
||||
cfg config.WorkerConfig
|
||||
maxBytes, maxPixels int
|
||||
decodeSlot chan struct{}
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
func New(
|
||||
pool *pgxpool.Pool,
|
||||
s3 *s3store.Client,
|
||||
brokerURL string,
|
||||
cfg config.WorkerConfig,
|
||||
maxBytes, maxPixels int,
|
||||
log *slog.Logger,
|
||||
) *Worker {
|
||||
return &Worker{
|
||||
pool: pool,
|
||||
q: db.New(pool),
|
||||
s3: s3,
|
||||
brokerURL: brokerURL,
|
||||
cfg: cfg,
|
||||
maxBytes: maxBytes,
|
||||
maxPixels: maxPixels,
|
||||
decodeSlot: make(chan struct{}, 1),
|
||||
log: log,
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) Run(ctx context.Context) {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
w.recoverLoop(ctx)
|
||||
}()
|
||||
|
||||
for range w.cfg.Concurrency {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
w.consumeLoop(ctx)
|
||||
}()
|
||||
}
|
||||
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func (w *Worker) consumeLoop(ctx context.Context) {
|
||||
for ctx.Err() == nil {
|
||||
client, err := rabbitmq.New(w.brokerURL)
|
||||
if err == nil {
|
||||
err = w.consume(ctx, client)
|
||||
_ = client.Close()
|
||||
}
|
||||
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
w.log.Warn("worker consumer reconnect", "error", err)
|
||||
if !wait(ctx, 2*time.Second) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) consume(ctx context.Context, client *rabbitmq.Client) error {
|
||||
stop := context.AfterFunc(ctx, func() {
|
||||
_ = client.Close()
|
||||
})
|
||||
defer stop()
|
||||
|
||||
deliveries, err := client.Consume(ctx, 1)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case delivery, ok := <-deliveries:
|
||||
if !ok {
|
||||
return amqp.ErrClosed
|
||||
}
|
||||
|
||||
var job model.AttachmentJob
|
||||
if json.Unmarshal(delivery.Body, &job) != nil ||
|
||||
job.SchemaVersion != 1 ||
|
||||
job.Type != "attachment.process.v1" ||
|
||||
job.JobID == uuid.Nil ||
|
||||
job.AttachmentID == uuid.Nil ||
|
||||
job.PipelineVersion != 1 ||
|
||||
job.Attempt < 1 ||
|
||||
job.CreatedAt.IsZero() {
|
||||
if err := delivery.Reject(false); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.log.Warn("invalid attachment job rejected")
|
||||
continue
|
||||
}
|
||||
|
||||
err := w.handle(ctx, job)
|
||||
if err == nil {
|
||||
if err := delivery.Ack(false); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
w.log.Warn(
|
||||
"attachment job not acknowledged",
|
||||
"job_id",
|
||||
job.JobID,
|
||||
"attempt",
|
||||
job.Attempt,
|
||||
"error",
|
||||
err,
|
||||
)
|
||||
if !wait(ctx, time.Second) {
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
if err := delivery.Nack(false, true); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) handle(ctx context.Context, job model.AttachmentJob) error {
|
||||
jobCtx, cancel := context.WithTimeout(ctx, w.cfg.Timeout)
|
||||
defer cancel()
|
||||
|
||||
row, err := w.q.ClaimAttachmentJob(jobCtx, db.ClaimAttachmentJobParams{
|
||||
ID: job.AttachmentID,
|
||||
JobID: job.JobID,
|
||||
PipelineVersion: job.PipelineVersion,
|
||||
Attempt: job.Attempt,
|
||||
LeaseUntil: time.Now().Add(w.cfg.Timeout),
|
||||
})
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return w.duplicate(jobCtx, job)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.log.Info("attachment processing", "job_id", job.JobID, "attachment_id", row.ID, "attempt", job.Attempt)
|
||||
variant, processErr := w.process(jobCtx, row)
|
||||
if processErr != nil {
|
||||
resultCtx, resultCancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer resultCancel()
|
||||
|
||||
return w.finishFailure(resultCtx, job, processErr)
|
||||
}
|
||||
|
||||
tx, err := w.pool.Begin(jobCtx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
q := w.q.WithTx(tx)
|
||||
_, err = q.LockAttachmentJob(jobCtx, db.LockAttachmentJobParams{
|
||||
ID: job.AttachmentID,
|
||||
JobID: job.JobID,
|
||||
PipelineVersion: job.PipelineVersion,
|
||||
Attempt: job.Attempt,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := q.CreateAttachmentVariant(jobCtx, variant); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n, err := q.CompleteAttachmentJob(jobCtx, db.CompleteAttachmentJobParams{
|
||||
ID: row.ID,
|
||||
Attempt: job.Attempt,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n != 1 {
|
||||
return errLeaseExpired
|
||||
}
|
||||
|
||||
if err := tx.Commit(jobCtx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.log.Info("attachment ready", "job_id", job.JobID, "attachment_id", row.ID, "attempt", job.Attempt)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) duplicate(ctx context.Context, job model.AttachmentJob) error {
|
||||
row, err := w.q.GetAttachmentByID(ctx, job.AttachmentID)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if row.JobID == nil || *row.JobID != job.JobID || row.PipelineVersion != job.PipelineVersion {
|
||||
return nil
|
||||
}
|
||||
|
||||
if row.Status == db.AttachmentStatusFailed {
|
||||
return nil
|
||||
}
|
||||
|
||||
if row.Status == db.AttachmentStatusReady {
|
||||
variants, err := w.q.ListAttachmentVariants(ctx, db.ListAttachmentVariantsParams{
|
||||
AttachmentID: row.ID,
|
||||
PipelineVersion: row.PipelineVersion,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := w.verify(ctx, row.S3Key, row.SizeBytes, row.Sha256); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, variant := range variants {
|
||||
if variant.Variant == db.AttachmentVariantNameThumbnail {
|
||||
return w.verify(ctx, variant.S3Key, variant.SizeBytes, variant.Sha256)
|
||||
}
|
||||
}
|
||||
|
||||
return errMissingThumbnail
|
||||
}
|
||||
|
||||
if row.Status == db.AttachmentStatusProcessing && row.Attempts == job.Attempt {
|
||||
return errAttemptProcessing
|
||||
}
|
||||
|
||||
if row.Attempts >= job.Attempt {
|
||||
return nil
|
||||
}
|
||||
|
||||
return errAttemptUnavailable
|
||||
}
|
||||
|
||||
func (w *Worker) verify(ctx context.Context, key string, size int, checksum string) error {
|
||||
object, err := w.s3.Get(ctx, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer object.Body.Close()
|
||||
|
||||
hash := sha256.New()
|
||||
n, err := io.Copy(hash, io.LimitReader(object.Body, int64(size)+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n != int64(size) || hex.EncodeToString(hash.Sum(nil)) != checksum {
|
||||
return errEffectChecksumMismatch
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) finishFailure(ctx context.Context, job model.AttachmentJob, cause error) error {
|
||||
tx, err := w.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
q := w.q.WithTx(tx)
|
||||
row, err := q.LockAttachmentJob(ctx, db.LockAttachmentJobParams{
|
||||
ID: job.AttachmentID,
|
||||
JobID: job.JobID,
|
||||
PipelineVersion: job.PipelineVersion,
|
||||
Attempt: job.Attempt,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
code := "processing_failed"
|
||||
permanent := errors.Is(cause, errInvalidSource) || errors.Is(cause, s3store.ErrObjectNotFound)
|
||||
if permanent {
|
||||
code = "invalid_source"
|
||||
} else if errors.Is(cause, context.DeadlineExceeded) {
|
||||
code = "job_timeout"
|
||||
}
|
||||
|
||||
var n int64
|
||||
if permanent || row.Attempts >= w.cfg.MaxAttempts {
|
||||
if !permanent {
|
||||
code = "retry_exhausted"
|
||||
}
|
||||
|
||||
n, err = q.FailAttachmentJob(ctx, db.FailAttachmentJobParams{
|
||||
ID: row.ID,
|
||||
Attempt: job.Attempt,
|
||||
ErrorCode: code,
|
||||
})
|
||||
} else {
|
||||
n, err = q.RetryAttachmentJob(ctx, db.RetryAttachmentJobParams{
|
||||
ID: row.ID,
|
||||
Attempt: job.Attempt,
|
||||
ErrorCode: code,
|
||||
})
|
||||
if err == nil && n == 1 {
|
||||
_, err = q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{
|
||||
AttachmentID: row.ID,
|
||||
Attempt: row.Attempts + 1,
|
||||
AvailableAt: time.Now().Add(retryDelay(row.Attempts)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if n != 1 {
|
||||
return errLeaseExpired
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
w.log.Warn(
|
||||
"attachment processing failed",
|
||||
"job_id",
|
||||
job.JobID,
|
||||
"attempt",
|
||||
job.Attempt,
|
||||
"error_code",
|
||||
code,
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *Worker) recoverLoop(ctx context.Context) {
|
||||
for ctx.Err() == nil {
|
||||
recovered, err := w.recoverOne(ctx)
|
||||
if err != nil && ctx.Err() == nil {
|
||||
w.log.Warn("recover attachment lease", "error", err)
|
||||
}
|
||||
|
||||
if (err != nil || !recovered) && !wait(ctx, time.Second) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *Worker) recoverOne(ctx context.Context) (bool, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
tx, err := w.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
defer tx.Rollback(context.Background())
|
||||
|
||||
q := w.q.WithTx(tx)
|
||||
rows, err := q.LockExpiredAttachmentJobs(ctx, 1)
|
||||
if err != nil || len(rows) == 0 {
|
||||
return false, err
|
||||
}
|
||||
|
||||
row := rows[0]
|
||||
if row.Attempts >= w.cfg.MaxAttempts {
|
||||
_, err = q.FailExpiredAttachmentJob(ctx, row.ID)
|
||||
} else {
|
||||
_, err = q.RecoverExpiredAttachmentJob(ctx, row.ID)
|
||||
if err == nil {
|
||||
_, err = q.CreateOutboxEntry(ctx, db.CreateOutboxEntryParams{
|
||||
AttachmentID: row.ID,
|
||||
Attempt: row.Attempts + 1,
|
||||
AvailableAt: time.Now().Add(retryDelay(row.Attempts)),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
w.log.Info("attachment lease recovered", "attachment_id", row.ID, "attempt", row.Attempts)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func wait(ctx context.Context, duration time.Duration) bool {
|
||||
timer := time.NewTimer(duration)
|
||||
defer timer.Stop()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-timer.C:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
func retryDelay(attempt int) time.Duration {
|
||||
ceiling := time.Second * time.Duration(1<<min(max(attempt, 1), 6))
|
||||
return ceiling/2 + time.Duration(rand.Int64N(int64(ceiling/2)))
|
||||
}
|
||||
Reference in New Issue
Block a user