413 lines
16 KiB
Go
413 lines
16 KiB
Go
// Package worker обрабатывает задания attachment.process.v1.
|
|
//
|
|
// Порядок обработки одного сообщения (at-least-once):
|
|
//
|
|
// 1. Claim: lease в PostgreSQL (fencing: owner + attempt). Дубль при живом lease,
|
|
// уже опубликованный результат, ранняя доставка до backoff → ack без работы.
|
|
// 2. Чтение оригинала из S3 по ключу из БД (не из сообщения), сверка SHA-256.
|
|
// 3. Варианты → PUT по детерминированным ключам v/{chat}/{id}/p{ver}/{name}
|
|
// (повтор перезаписывает те же байты), оригинал staging → o/ (CopyObject).
|
|
// 4. Complete: одна транзакция — варианты (ON CONFLICT по ключу эффекта),
|
|
// attachment → ready этой версии (только если новее опубликованной), job → done.
|
|
// 5. ack. Затем удаление staging (результат уже безопасно зафиксирован).
|
|
//
|
|
// Падение на любом шаге до ack приводит к повторной доставке; повтор либо
|
|
// доделывает тот же эффект (шаги 2–4 идемпотентны), либо видит done и ack-ает.
|
|
package worker
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"image"
|
|
"io"
|
|
"log/slog"
|
|
"math/rand/v2"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.opentelemetry.io/otel/codes"
|
|
"go.opentelemetry.io/otel/metric"
|
|
"go.opentelemetry.io/otel/trace"
|
|
"golang.org/x/sync/semaphore"
|
|
|
|
"github.com/charawein/mtusi2026lab6/internal/domain"
|
|
"github.com/charawein/mtusi2026lab6/internal/imaging"
|
|
"github.com/charawein/mtusi2026lab6/internal/repository/postgres"
|
|
"github.com/charawein/mtusi2026lab6/internal/storage"
|
|
"github.com/charawein/mtusi2026lab6/internal/telemetry"
|
|
)
|
|
|
|
// Jobs — операции над заданиями (реализует postgres.PipelineRepository).
|
|
type Jobs interface {
|
|
Claim(ctx context.Context, jobID, owner string, lease time.Duration) (postgres.ClaimOutcome, *postgres.JobRow, *domain.Attachment, error)
|
|
ExtendLease(ctx context.Context, jobID, owner string, attempt int, lease time.Duration) (bool, error)
|
|
Complete(ctx context.Context, jobID, owner string, attempt int, finalKey string, variants []domain.Variant) (postgres.CompleteResult, error)
|
|
RetryOrFail(ctx context.Context, jobID, owner string, attempt int, code, detail string, permanent bool, backoff time.Duration) (bool, error)
|
|
ClearStaging(ctx context.Context, attachmentID, key string) error
|
|
}
|
|
|
|
type Config struct {
|
|
Owner string
|
|
Lease time.Duration // = JOB_TIMEOUT: дольше работа не может идти без продления
|
|
JobTimeout time.Duration
|
|
MaxBytes int64
|
|
MaxPixels int64
|
|
PixelBudget int64 // суммарно декодируемых пикселей на процесс (ограничение памяти)
|
|
WebPQuality int
|
|
RetryBase time.Duration
|
|
RetryMax time.Duration
|
|
FaultPoints map[string]bool
|
|
FaultDir string
|
|
DeleteStaging bool
|
|
}
|
|
|
|
type Worker struct {
|
|
jobs Jobs
|
|
store storage.Store
|
|
webp imaging.WebPEncoder
|
|
cfg Config
|
|
pixels *semaphore.Weighted
|
|
exit func(int)
|
|
m instruments
|
|
}
|
|
|
|
// instruments — метрики worker (контракт наблюдаемости, уровни 3–4).
|
|
type instruments struct {
|
|
processed metric.Int64Counter // messenger.jobs.processed{job.type, outcome}
|
|
duration metric.Float64Histogram // messenger.job.duration{job.type, outcome}, s
|
|
timeToReady metric.Float64Histogram // messenger.attachment.time_to_ready{job.type}, s
|
|
busy metric.Int64UpDownCounter // messenger.worker.busy — занятые слоты
|
|
}
|
|
|
|
func newInstruments() instruments {
|
|
m := telemetry.Meter()
|
|
var in instruments
|
|
in.processed, _ = m.Int64Counter("messenger.jobs.processed", metric.WithUnit("{job}"),
|
|
metric.WithDescription("Deliveries handled by worker, by outcome"))
|
|
in.duration, _ = m.Float64Histogram("messenger.job.duration", metric.WithUnit("s"),
|
|
metric.WithDescription("Processing time of a claimed job attempt"))
|
|
in.timeToReady, _ = m.Float64Histogram("messenger.attachment.time_to_ready", metric.WithUnit("s"),
|
|
metric.WithDescription("From durable acceptance (202, job created) to published ready"))
|
|
in.busy, _ = m.Int64UpDownCounter("messenger.worker.busy", metric.WithUnit("{job}"),
|
|
metric.WithDescription("Jobs being processed now (saturation = busy / capacity)"))
|
|
return in
|
|
}
|
|
|
|
func New(jobs Jobs, store storage.Store, webp imaging.WebPEncoder, cfg Config) *Worker {
|
|
return &Worker{jobs: jobs, store: store, webp: webp, cfg: cfg,
|
|
pixels: semaphore.NewWeighted(cfg.PixelBudget), exit: os.Exit, m: newInstruments()}
|
|
}
|
|
|
|
// Decision — что сделать с доставкой.
|
|
type Decision int
|
|
|
|
const (
|
|
Ack Decision = iota
|
|
Requeue // nack с возвратом в очередь (инфраструктура недоступна)
|
|
Reject // nack без возврата → DLX (невалидное сообщение)
|
|
)
|
|
|
|
func (d Decision) String() string { return [...]string{"ack", "requeue", "reject"}[d] }
|
|
|
|
// permanentError — повтор не поможет (битое изображение, нет исходника, checksum).
|
|
type permanentError struct {
|
|
code string
|
|
err error
|
|
}
|
|
|
|
func (e *permanentError) Error() string { return e.code + ": " + e.err.Error() }
|
|
|
|
func permanent(code string, err error) error { return &permanentError{code: code, err: err} }
|
|
|
|
// Fault — точка внедрения сбоя (трек 5): процесс завершается как при kill -9
|
|
// ровно один раз (маркер в FaultDir переживает перезапуск контейнера).
|
|
func Fault(points map[string]bool, dir, point string, exit func(int)) {
|
|
if !points[point] {
|
|
return
|
|
}
|
|
marker := filepath.Join(dir, "fault-"+point)
|
|
if _, err := os.Stat(marker); err == nil {
|
|
return
|
|
}
|
|
_ = os.WriteFile(marker, []byte(time.Now().UTC().Format(time.RFC3339)), 0o644)
|
|
slog.Warn("FAULT INJECTION: crashing", "point", point)
|
|
exit(137)
|
|
}
|
|
|
|
func (w *Worker) fault(point string) { Fault(w.cfg.FaultPoints, w.cfg.FaultDir, point, w.exit) }
|
|
|
|
// Backoff — экспоненциальная задержка с полным jitter в [d/2, d].
|
|
func Backoff(attempt int, base, maxDelay time.Duration) time.Duration {
|
|
d := base << min(attempt-1, 20)
|
|
if d <= 0 || d > maxDelay {
|
|
d = maxDelay
|
|
}
|
|
half := d / 2
|
|
return half + time.Duration(rand.Int64N(int64(half)+1))
|
|
}
|
|
|
|
// Handle обрабатывает одну доставку. ctx несёт consumer span (если он есть):
|
|
// исход, попытка и ошибка записываются в span, метрики и структурные логи.
|
|
func (w *Worker) Handle(ctx context.Context, body []byte) Decision {
|
|
start := time.Now()
|
|
span := trace.SpanFromContext(ctx)
|
|
outcome, claimed := "", false
|
|
defer func() {
|
|
attrs := metric.WithAttributes(attribute.String("job.type", domain.JobType), attribute.String("outcome", outcome))
|
|
w.m.processed.Add(ctx, 1, attrs)
|
|
if claimed {
|
|
w.m.duration.Record(ctx, time.Since(start).Seconds(), attrs)
|
|
}
|
|
span.SetAttributes(attribute.String("messenger.job.outcome", outcome))
|
|
if outcome == "failed" || outcome == "retry" || outcome == "requeue" {
|
|
span.SetStatus(codes.Error, outcome)
|
|
}
|
|
}()
|
|
|
|
var job domain.Job
|
|
if err := json.Unmarshal(body, &job); err != nil || job.SchemaVersion != 1 || job.Type != domain.JobType ||
|
|
job.JobID == "" || job.AttachmentID == "" || job.PipelineVersion < 1 {
|
|
slog.ErrorContext(ctx, "invalid job message, dead-lettering", "event", "job.rejected", "err", err)
|
|
outcome = "rejected"
|
|
return Reject
|
|
}
|
|
span.SetAttributes(attribute.String("messenger.job.id", job.JobID),
|
|
attribute.Int("messenger.pipeline.version", job.PipelineVersion))
|
|
log := slog.With("job_id", job.JobID, "attachment_id", job.AttachmentID, "version", job.PipelineVersion)
|
|
|
|
claim, j, a, err := w.jobs.Claim(ctx, job.JobID, w.cfg.Owner, w.cfg.Lease)
|
|
if err != nil {
|
|
log.WarnContext(ctx, "claim failed, requeue", "event", "job.claim_failed", "err", err)
|
|
outcome = "requeue"
|
|
return Requeue
|
|
}
|
|
switch claim {
|
|
case postgres.AlreadyDone:
|
|
log.InfoContext(ctx, "duplicate delivery of finished job: ack without work", "event", "job.duplicate")
|
|
outcome = "duplicate"
|
|
return Ack
|
|
case postgres.Busy:
|
|
log.InfoContext(ctx, "job leased by another worker: ack duplicate", "event", "job.duplicate", "owner", j.LeaseOwner)
|
|
outcome = "duplicate"
|
|
return Ack
|
|
case postgres.NotDue:
|
|
log.InfoContext(ctx, "early delivery before backoff: ack, outbox will redeliver", "event", "job.not_due")
|
|
outcome = "not_due"
|
|
return Ack
|
|
case postgres.Terminal, postgres.ExhaustedNow:
|
|
log.InfoContext(ctx, "job terminal: ack", "event", "job.terminal", "state", stateOf(j))
|
|
outcome = "terminal"
|
|
return Ack
|
|
}
|
|
claimed = true
|
|
span.SetAttributes(attribute.Int("messenger.job.attempt", j.Attempt))
|
|
w.m.busy.Add(ctx, 1)
|
|
defer w.m.busy.Add(ctx, -1)
|
|
log.InfoContext(ctx, "job claimed", "event", "job.claimed", "attempt", j.Attempt)
|
|
|
|
// Heartbeat: продлеваем lease, пока работаем; потеря lease прерывает работу.
|
|
pctx, cancel := context.WithTimeout(ctx, w.cfg.JobTimeout)
|
|
defer cancel()
|
|
hbDone := make(chan struct{})
|
|
go func() {
|
|
defer close(hbDone)
|
|
t := time.NewTicker(w.cfg.Lease / 3)
|
|
defer t.Stop()
|
|
for {
|
|
select {
|
|
case <-pctx.Done():
|
|
return
|
|
case <-t.C:
|
|
ok, err := w.jobs.ExtendLease(context.WithoutCancel(pctx), j.JobID, w.cfg.Owner, j.Attempt, w.cfg.Lease)
|
|
if err == nil && !ok {
|
|
log.WarnContext(ctx, "lease lost, aborting attempt", "event", "job.lease_lost", "attempt", j.Attempt)
|
|
cancel()
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}()
|
|
|
|
finalKey, variants, perr := w.process(pctx, a, j.PipelineVersion)
|
|
cancel()
|
|
<-hbDone
|
|
|
|
dctx, dcancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
|
|
defer dcancel()
|
|
if perr != nil {
|
|
var pe *permanentError
|
|
code, isPermanent := "processing_failed", false
|
|
if errors.As(perr, &pe) {
|
|
code, isPermanent = pe.code, true
|
|
} else if errors.Is(perr, context.DeadlineExceeded) {
|
|
code = "timeout"
|
|
}
|
|
backoff := Backoff(j.Attempt, w.cfg.RetryBase, w.cfg.RetryMax)
|
|
failed, err := w.jobs.RetryOrFail(dctx, j.JobID, w.cfg.Owner, j.Attempt, code, safeDetail(perr), isPermanent, backoff)
|
|
if err != nil {
|
|
// Записать исход не удалось: lease истечёт, sweeper вернёт работу в очередь.
|
|
log.WarnContext(ctx, "record failure failed; lease expiry will recover", "event", "job.record_failed", "err", err)
|
|
outcome = "requeue"
|
|
return Requeue
|
|
}
|
|
span.RecordError(perr)
|
|
outcome = "retry"
|
|
if failed {
|
|
outcome = "failed"
|
|
}
|
|
log.WarnContext(ctx, "attempt failed", "event", "job."+outcome, "attempt", j.Attempt, "code", code, "final", failed, "backoff", backoff)
|
|
return Ack
|
|
}
|
|
|
|
w.fault("worker.after_side_effect") // объекты записаны, результат ещё не зафиксирован
|
|
|
|
res, err := w.jobs.Complete(dctx, j.JobID, w.cfg.Owner, j.Attempt, finalKey, variants)
|
|
if err != nil {
|
|
log.WarnContext(ctx, "complete failed; requeue", "event", "job.complete_failed", "err", err)
|
|
outcome = "requeue"
|
|
return Requeue
|
|
}
|
|
if res.LostLease {
|
|
log.WarnContext(ctx, "result rejected: lease lost (another attempt owns the job)", "event", "job.lost_lease", "attempt", j.Attempt)
|
|
outcome = "lost_lease"
|
|
return Ack
|
|
}
|
|
|
|
w.fault("worker.after_commit") // результат зафиксирован, ack ещё не отправлен
|
|
|
|
if res.Published && w.cfg.DeleteStaging && a.StagingKey != "" && a.StagingKey != finalKey {
|
|
// Staging больше не нужен: оригинал опубликован в o/ и зафиксирован.
|
|
if err := w.store.Delete(dctx, a.StagingKey); err == nil {
|
|
_ = w.jobs.ClearStaging(dctx, a.ID, a.StagingKey)
|
|
}
|
|
}
|
|
outcome = "done"
|
|
if res.Published && !j.CreatedAt.IsZero() {
|
|
w.m.timeToReady.Record(ctx, time.Since(j.CreatedAt).Seconds(),
|
|
metric.WithAttributes(attribute.String("job.type", domain.JobType)))
|
|
}
|
|
log.InfoContext(ctx, "job done", "event", "job.done", "attempt", j.Attempt, "published", res.Published, "newer_version", res.NewerPublished)
|
|
return Ack
|
|
}
|
|
|
|
func stateOf(j *postgres.JobRow) string {
|
|
if j == nil {
|
|
return "missing"
|
|
}
|
|
return j.State
|
|
}
|
|
|
|
func safeDetail(err error) string {
|
|
s := err.Error()
|
|
if len(s) > 300 {
|
|
s = s[:300]
|
|
}
|
|
return s
|
|
}
|
|
|
|
func ext(ctype string) string {
|
|
switch ctype {
|
|
case "image/webp":
|
|
return "webp"
|
|
case "image/png":
|
|
return "png"
|
|
}
|
|
return "jpg"
|
|
}
|
|
|
|
// process — шаги 2–3: чтение, проверка, варианты, публикация оригинала.
|
|
func (w *Worker) process(ctx context.Context, a *domain.Attachment, version int) (string, []domain.Variant, error) {
|
|
rc, size, err := w.store.Get(ctx, a.ObjectKey)
|
|
if errors.Is(err, storage.ErrNotFound) {
|
|
return "", nil, permanent("source_missing", err)
|
|
}
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
if size > w.cfg.MaxBytes {
|
|
rc.Close()
|
|
return "", nil, permanent("too_large", fmt.Errorf("source %d bytes", size))
|
|
}
|
|
data, err := io.ReadAll(io.LimitReader(rc, w.cfg.MaxBytes+1))
|
|
rc.Close()
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
if sum := imaging.SHA256Hex(data); sum != a.SHA256 {
|
|
return "", nil, permanent("checksum_mismatch", fmt.Errorf("source sha256 %s != metadata", sum[:12]))
|
|
}
|
|
cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
|
|
if err != nil {
|
|
return "", nil, permanent("invalid_image", err)
|
|
}
|
|
px := int64(cfg.Width) * int64(cfg.Height)
|
|
if px > w.cfg.MaxPixels {
|
|
return "", nil, permanent("too_large", fmt.Errorf("%d pixels", px))
|
|
}
|
|
// Ограничение памяти: суммарно декодируемых пикселей в процессе ≤ бюджета
|
|
// (RGBA = 4 байта/пиксель; ресайз добавляет меньшие буферы).
|
|
weight := min(px, w.cfg.PixelBudget)
|
|
if err := w.pixels.Acquire(ctx, weight); err != nil {
|
|
return "", nil, err
|
|
}
|
|
defer w.pixels.Release(weight)
|
|
|
|
_, dspan := telemetry.Tracer().Start(ctx, "image.decode", trace.WithAttributes(attribute.Int64("image.pixels", px)))
|
|
img, err := imaging.Decode(data)
|
|
dspan.End()
|
|
if err != nil {
|
|
return "", nil, permanent("invalid_image", err)
|
|
}
|
|
p := imaging.ParamsFor(version, w.cfg.WebPQuality)
|
|
outs := make([]imaging.Output, 0, 3)
|
|
vctx, vspan := telemetry.Tracer().Start(ctx, "image.variants")
|
|
defer vspan.End()
|
|
th, err := imaging.Thumbnail(img, p)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
cr, err := imaging.Crop(img, p)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
op, err := imaging.Optimized(vctx, img, p, w.webp)
|
|
if err != nil {
|
|
return "", nil, err
|
|
}
|
|
outs = append(outs, th, cr, op)
|
|
img = nil
|
|
vspan.End()
|
|
|
|
base := fmt.Sprintf("v/%s/%s/p%d/", a.ChatID, a.ID, version)
|
|
variants := make([]domain.Variant, 0, len(outs))
|
|
for _, o := range outs {
|
|
key := base + o.Name + "." + ext(o.ContentType)
|
|
if err := w.store.Put(ctx, key, bytes.NewReader(o.Data), int64(len(o.Data)), o.ContentType); err != nil {
|
|
return "", nil, err
|
|
}
|
|
variants = append(variants, domain.Variant{Name: o.Name, ContentType: o.ContentType, SizeBytes: int64(len(o.Data)),
|
|
Width: o.Width, Height: o.Height, SHA256: o.SHA256, ObjectKey: key, PipelineVersion: version})
|
|
}
|
|
// Оригинал: из staging в конечный prefix (серверная копия, байты неизменны).
|
|
finalKey := a.ObjectKey
|
|
if strings.HasPrefix(a.ObjectKey, "s/") {
|
|
finalKey = "o/" + strings.TrimPrefix(a.ObjectKey, "s/")
|
|
if err := w.store.Copy(ctx, a.ObjectKey, finalKey); err != nil {
|
|
if errors.Is(err, storage.ErrNotFound) {
|
|
return "", nil, permanent("source_missing", err)
|
|
}
|
|
return "", nil, err
|
|
}
|
|
}
|
|
return finalKey, variants, nil
|
|
}
|
|
|
|
// SetExit подменяет завершение процесса (тесты внедрения сбоев).
|
|
func SetExit(w *Worker, exit func(int)) { w.exit = exit }
|