feat(worker): lab6 record job outcomes and end-to-end readiness latency

This commit is contained in:
Голиков Maximillian Антон
2026-11-16 21:55:01 +03:00
parent 6558fb5497
commit 44c755f00c
4 changed files with 105 additions and 20 deletions
+2 -3
View File
@@ -108,7 +108,6 @@ func (s *AttachmentService) Upload(
sha := sha256.Sum256(buf)
id := uuid.NewString()
stagingKey := "staging/" + id
now := time.Now().UTC()
a := &domain.Attachment{
ID: id,
@@ -122,8 +121,6 @@ func (s *AttachmentService) Upload(
Status: domain.AttachmentStatusQueued,
StagingKey: stagingKey,
PipelineVersion: 1,
CreatedAt: now,
UpdatedAt: now,
}
if err := s.storage.Put(ctx, stagingKey, bytes.NewReader(buf), int64(len(buf)), contentType); err != nil {
@@ -140,6 +137,8 @@ func (s *AttachmentService) Upload(
}
err = s.tx.WithTx(ctx, func(tx domain.Tx) error {
a.CreatedAt = time.Now().UTC()
a.UpdatedAt = a.CreatedAt
if err := s.attachments.CreateInTx(ctx, tx, a); err != nil {
return err
}
@@ -200,10 +200,10 @@ func (r *AttachmentRepo) ClaimForProcessing(ctx context.Context, id string, vers
UPDATE attachments
SET status = 'processing', locked_until = now() + $2, updated_at = now(), attempts = attempts + 1
WHERE id = $1::uuid AND status = 'queued' AND pipeline_version = $3
RETURNING id, chat_id, staging_key, pipeline_version, object_key, content_type, attempts
RETURNING id, chat_id, staging_key, pipeline_version, object_key, content_type, attempts, created_at
`
var a domain.Attachment
err := r.pool.QueryRow(ctx, q, id, lease, version).Scan(&a.ID, &a.ChatID, &a.StagingKey, &a.PipelineVersion, &a.ObjectKey, &a.ContentType, &a.Attempts)
err := r.pool.QueryRow(ctx, q, id, lease, version).Scan(&a.ID, &a.ChatID, &a.StagingKey, &a.PipelineVersion, &a.ObjectKey, &a.ContentType, &a.Attempts, &a.CreatedAt)
if err != nil {
if err == pgx.ErrNoRows {
return nil, nil
+65
View File
@@ -0,0 +1,65 @@
package worker
import (
"context"
"log/slog"
"time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/trace"
"high-loud-apps-labs/internal/domain"
"high-loud-apps-labs/internal/telemetry"
)
type jobMetrics struct {
processed metric.Int64Counter
duration metric.Float64Histogram
ready metric.Float64Histogram
}
func newJobMetrics() jobMetrics {
meter := telemetry.Meter("worker.pipeline")
processed, err := meter.Int64Counter("jobs_processed_total")
if err != nil {
panic(err)
}
unit := metric.WithUnit("s")
duration, err := meter.Float64Histogram("job_processing_duration_seconds", unit,
metric.WithExplicitBucketBoundaries(.01, .05, .1, .25, .5, 1, 2, 5, 10, 30, 60))
if err != nil {
panic(err)
}
ready, err := meter.Float64Histogram("end_to_end_ready_seconds", unit,
metric.WithExplicitBucketBoundaries(.1, .5, 1, 2, 5, 10, 30, 60, 120, 300, 900, 3600))
if err != nil {
panic(err)
}
return jobMetrics{processed: processed, duration: duration, ready: ready}
}
func (m jobMetrics) recordAttempt(ctx context.Context, start time.Time, outcome string) {
attrs := metric.WithAttributes(attribute.String("job.type", domain.OutboxEventProcess), attribute.String("outcome", outcome))
m.duration.Record(ctx, time.Since(start).Seconds(), attrs)
m.processed.Add(ctx, 1, attrs)
span := trace.SpanFromContext(ctx)
span.SetAttributes(attribute.String("job.outcome", outcome))
level := slog.LevelInfo
switch outcome {
case "failed", "error", "retry":
level = slog.LevelWarn
span.SetStatus(codes.Error, outcome)
}
slog.Log(ctx, level, "Попытка обработки завершена", "event", "job.processed", "outcome", outcome)
}
func (m jobMetrics) recordReady(ctx context.Context, createdAt time.Time) {
if createdAt.IsZero() {
return
}
m.ready.Record(ctx, max(0, time.Since(createdAt).Seconds()),
metric.WithAttributes(attribute.String("job.type", domain.OutboxEventProcess)))
}
+36 -15
View File
@@ -26,6 +26,7 @@ type Pipeline struct {
outbox domain.OutboxRepository
lease time.Duration
maxRetries int
metrics jobMetrics
}
func NewPipeline(
@@ -43,10 +44,15 @@ func NewPipeline(
outbox: outbox,
lease: 5 * time.Minute,
maxRetries: 5,
metrics: newJobMetrics(),
}
}
func (p *Pipeline) Process(ctx context.Context, attachmentID string, pipelineVersion int) error {
start := time.Now()
outcome := "error"
defer func() { p.metrics.recordAttempt(ctx, start, outcome) }()
log.Printf("worker: начало обработки %s (версия пайплайна %d)", attachmentID, pipelineVersion)
jobCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
@@ -61,23 +67,29 @@ func (p *Pipeline) Process(ctx context.Context, attachmentID string, pipelineVer
return fmt.Errorf("захват задачи: %w", err)
}
if a == nil {
outcome = "skipped"
log.Printf("worker: задача %s уже взята, выполнена или версия устарела", attachmentID)
return nil // no-op
}
fail := func(err error, permanent bool) error {
outcome = p.handleFailure(a, err, permanent)
return nil
}
// 2. Скачиваем оригинал из staging
getCtx, getSpan := tracer.Start(jobCtx, "job.s3.get")
rc, _, _, err := p.store.Get(getCtx, a.StagingKey)
if err != nil {
endPipelineSpan(getSpan, err)
return p.handleFailure(a, fmt.Errorf("скачивание staging: %w", err), false)
return fail(fmt.Errorf("скачивание staging: %w", err), false)
}
defer rc.Close()
buf, err := io.ReadAll(rc)
endPipelineSpan(getSpan, err)
if err != nil {
return p.handleFailure(a, fmt.Errorf("чтение staging: %w", err), false)
return fail(fmt.Errorf("чтение staging: %w", err), false)
}
// 3. Генерируем все варианты (Thumbnail, Crop, Optimized)
@@ -86,26 +98,26 @@ func (p *Pipeline) Process(ctx context.Context, attachmentID string, pipelineVer
thumbMeta, thumbData, err := imaging.Thumbnail(a.ID, buf)
if err != nil {
endPipelineSpan(imageSpan, err)
return p.handleFailure(a, fmt.Errorf("генерация thumbnail: %w", err), true)
return fail(fmt.Errorf("генерация thumbnail: %w", err), true)
}
cropMeta, cropData, err := imaging.Crop(a.ID, buf)
if err != nil {
endPipelineSpan(imageSpan, err)
return p.handleFailure(a, fmt.Errorf("генерация crop: %w", err), true)
return fail(fmt.Errorf("генерация crop: %w", err), true)
}
webpMeta, webpData, err := imaging.Optimized(a.ID, buf)
endPipelineSpan(imageSpan, err)
if err != nil {
return p.handleFailure(a, fmt.Errorf("генерация webp: %w", err), true)
return fail(fmt.Errorf("генерация webp: %w", err), true)
}
// 4. Сохраняем оригинал в финальный prefix (chats/...)
putCtx, putSpan := tracer.Start(jobCtx, "job.s3.put")
if err := p.store.Put(putCtx, a.ObjectKey, bytes.NewReader(buf), int64(len(buf)), a.ContentType); err != nil {
endPipelineSpan(putSpan, err)
return p.handleFailure(a, fmt.Errorf("загрузка оригинала: %w", err), false)
return fail(fmt.Errorf("загрузка оригинала: %w", err), false)
}
// 5. Загружаем все сгенерированные варианты в S3
@@ -127,7 +139,7 @@ func (p *Pipeline) Process(ctx context.Context, attachmentID string, pipelineVer
if err := p.store.Put(putCtx, key, bytes.NewReader(data), v.SizeBytes, v.ContentType); err != nil {
endPipelineSpan(putSpan, err)
return p.handleFailure(a, fmt.Errorf("загрузка варианта %s: %w", v.Name, err), false)
return fail(fmt.Errorf("загрузка варианта %s: %w", v.Name, err), false)
}
v.ObjectKey = key
}
@@ -139,13 +151,16 @@ func (p *Pipeline) Process(ctx context.Context, attachmentID string, pipelineVer
ok, err := p.jobs.CompleteJob(completeCtx, a.ID, a.PipelineVersion, variants)
endPipelineSpan(completeSpan, err)
if err != nil {
return p.handleFailure(a, fmt.Errorf("фиксация в БД: %w", err), false)
return fail(fmt.Errorf("фиксация в БД: %w", err), false)
}
if !ok {
outcome = "skipped"
log.Printf("worker: версия устарела или статус изменился, пропускаем %s", a.ID)
return nil
}
outcome = "ready"
p.metrics.recordReady(ctx, a.CreatedAt)
log.Printf("worker: успешно завершено %s", a.ID)
return nil
}
@@ -157,7 +172,7 @@ func endPipelineSpan(span trace.Span, err error) {
span.End()
}
func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bool) error {
func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bool) string {
// Если исчерпаны попытки или ошибка постоянная (например, битый файл)
if isPermanent || a.Attempts >= p.maxRetries {
reason := domain.JobErrorInternal
@@ -170,13 +185,16 @@ func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bo
log.Printf("worker: финальная ошибка для %s (попытка %d): %v", a.ID, a.Attempts, err)
// Используем context.Background(), чтобы запись в БД прошла даже при истекшем jobCtx
_ = p.jobs.FailJob(context.Background(), a.ID, reason)
_ = p.failedJobs.Record(context.Background(), &domain.FailedJob{
updateErr := p.jobs.FailJob(context.Background(), a.ID, reason)
recordErr := p.failedJobs.Record(context.Background(), &domain.FailedJob{
AttachmentID: a.ID,
ReasonCode: reason,
Attempts: a.Attempts,
})
return nil
if updateErr != nil || recordErr != nil {
return "error"
}
return "failed"
}
// Временная ошибка - делаем Retry с backoff
@@ -187,7 +205,7 @@ func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bo
jitter := time.Duration(rand.Intn(1000)) * time.Millisecond
nextAt := time.Now().UTC().Add(backoff + jitter)
_ = p.jobs.ScheduleRetry(context.Background(), a.ID, nextAt, "")
scheduleErr := p.jobs.ScheduleRetry(context.Background(), a.ID, nextAt, "")
// Кладем в Outbox отложенное сообщение
payload, _ := json.Marshal(domain.JobMessage{
@@ -195,7 +213,7 @@ func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bo
PipelineVersion: a.PipelineVersion,
Attempt: a.Attempts + 1,
})
_ = p.outbox.EnqueueDirect(context.Background(), &domain.OutboxEntry{
enqueueErr := p.outbox.EnqueueDirect(context.Background(), &domain.OutboxEntry{
AggregateType: domain.OutboxAggregateAttachment,
AggregateID: a.ID,
EventType: domain.OutboxEventProcess,
@@ -203,5 +221,8 @@ func (p *Pipeline) handleFailure(a *domain.Attachment, err error, isPermanent bo
CreatedAt: nextAt,
})
return nil
if scheduleErr != nil || enqueueErr != nil {
return "error"
}
return "retry"
}