ЛР 5: асинхронная обработка изображений и идемпотентность pipeline

This commit is contained in:
R0flyan
2026-11-06 17:56:33 +03:00
parent 276dbdad8a
commit 05641f0bf4
48 changed files with 3554 additions and 41 deletions
+263
View File
@@ -0,0 +1,263 @@
package main
import (
"context"
"encoding/json"
"errors"
"log/slog"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/redis/go-redis/v9"
)
const imageStream = "attachment-jobs"
const imageGroup = "image-workers"
var ackImage = redis.NewScript(`
if redis.call('XACK', KEYS[1], ARGV[1], ARGV[2]) == 1 then
redis.call('XDEL', KEYS[1], ARGV[2])
end
return 1
`)
func imageCommand(db *pgxpool.Pool, args []string) error {
a := &App{db: db}
retention := positive("STAGING_RETENTION_SECONDS", 86400)
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if args[0] == "jobs" {
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
if len(args) == 2 && args[1] == "failed" {
return a.listFailedImages(ctx)
}
if len(args) != 3 {
return errors.New("usage: jobs failed | jobs replay <job-id> | jobs upgrade <attachment-id>")
}
id, err := normalizeID(args[2])
if err != nil {
return errors.New("invalid job or attachment ID")
}
switch args[1] {
case "replay":
return a.replayImage(ctx, id, retention)
case "upgrade":
return a.upgradeImage(ctx, id)
default:
return errors.New("unknown jobs command")
}
}
if len(args) != 1 || (args[0] != "dispatcher" && args[0] != "worker") {
return errors.New("unknown command")
}
attempts := positive("JOB_MAX_ATTEMPTS", 5)
timeoutSeconds := positive("JOB_TIMEOUT_SECONDS", 30)
if attempts > 20 || timeoutSeconds > 120 || positive("WORKER_CONCURRENCY", 1) != 1 {
return errors.New("invalid image job limits")
}
timeout := time.Duration(timeoutSeconds) * time.Second
s3, bucket, err := s3Client()
if err != nil {
return errors.New("invalid image storage configuration")
}
a.s3, a.bucket = s3, bucket
brokerURL, err := secretFile("BROKER_URL_FILE")
if err != nil {
return err
}
opts, err := redis.ParseURL(brokerURL)
if err != nil {
return errors.New("invalid broker configuration")
}
opts.MaxRetries = -1
opts.PoolSize = 2
opts.DialTimeout = time.Second
opts.ReadTimeout = 2 * time.Second
opts.WriteTimeout = time.Second
opts.ContextTimeoutEnabled = true
broker := redis.NewClient(opts)
defer broker.Close()
if args[0] == "dispatcher" {
a.dispatchImages(ctx, broker, attempts, retention)
} else {
a.consumeImages(ctx, broker, attempts, timeout)
}
return nil
}
func injectImageFault(phase, attachment string) {
if os.Getenv("LAB5_FAULT_ATTACHMENT") == attachment && os.Getenv("LAB5_FAULT") == phase {
slog.Info("test crash", "phase", phase, "attachment_id", attachment)
os.Exit(86)
}
}
func (a *App) publishImage(ctx context.Context, broker *redis.Client) (bool, error) {
tx, err := a.db.Begin(ctx)
if err != nil {
return false, err
}
defer rollbackAttachment(tx)
j := imageJob{SchemaVersion: 1, Type: "attachment.process.v1"}
var created time.Time
err = tx.QueryRow(ctx, `SELECT id::text,attachment_id::text,version,attempt,created_at FROM image_jobs
WHERE state='queued' AND NOT published AND next_at<=clock_timestamp()
ORDER BY next_at,id LIMIT 1 FOR UPDATE SKIP LOCKED`).Scan(&j.JobID, &j.AttachmentID, &j.Version, &j.Attempt, &created)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, err
}
j.CreatedAt = stamp(created)
payload, err := json.Marshal(j)
if err != nil {
return false, err
}
injectImageFault("before_publish", j.AttachmentID)
// XADD replies after the broker's AOF durability boundary. A lost reply
// or crash before the SQL commit causes another delivery of the same job.
err = broker.XAdd(ctx, &redis.XAddArgs{Stream: imageStream, Values: map[string]any{"job": string(payload)}}).Err()
if err != nil {
return false, err
}
injectImageFault("after_publish", j.AttachmentID)
if _, err = tx.Exec(ctx, "UPDATE image_jobs SET published=true WHERE id=$1", j.JobID); err != nil {
return false, err
}
return true, tx.Commit(ctx)
}
func (a *App) dispatchImages(ctx context.Context, broker *redis.Client, attempts, retention int) {
cleanup := time.Now()
for ctx.Err() == nil {
step, cancel := context.WithTimeout(ctx, 5*time.Second)
recoverErr := a.recoverImage(step, attempts)
published, publishErr := a.publishImage(step, broker)
cancel()
if recoverErr != nil || publishErr != nil {
slog.Warn("image outbox unavailable", "recover_failed", recoverErr != nil, "publish_failed", publishErr != nil)
}
if time.Now().After(cleanup) {
step, cancel = context.WithTimeout(ctx, 5*time.Second)
if err := a.cleanStaging(step, retention); err != nil && ctx.Err() == nil {
slog.Warn("staging cleanup incomplete")
}
cancel()
cleanup = time.Now().Add(15 * time.Second)
}
if !published || recoverErr != nil {
waitImage(ctx, 300*time.Millisecond)
}
}
}
func waitImage(ctx context.Context, delay time.Duration) {
timer := time.NewTimer(delay)
defer timer.Stop()
select {
case <-ctx.Done():
case <-timer.C:
}
}
func (a *App) consumeImages(ctx context.Context, broker *redis.Client, attempts int, timeout time.Duration) {
cursor := "0-0"
groupReady := false
for ctx.Err() == nil {
if !groupReady {
err := broker.XGroupCreateMkStream(ctx, imageStream, imageGroup, "0").Err()
if err != nil && !strings.HasPrefix(err.Error(), "BUSYGROUP") {
waitImage(ctx, time.Second)
continue
}
groupReady = true
}
messages, next, err := broker.XAutoClaim(ctx, &redis.XAutoClaimArgs{
Stream: imageStream, Group: imageGroup, Consumer: "worker", MinIdle: timeout + 10*time.Second, Start: cursor, Count: 1,
}).Result()
if err == nil {
cursor = next
if len(messages) == 0 {
var streams []redis.XStream
streams, err = broker.XReadGroup(ctx, &redis.XReadGroupArgs{
Group: imageGroup, Consumer: "worker", Streams: []string{imageStream, ">"}, Count: 1, Block: time.Second,
}).Result()
if err == nil && len(streams) > 0 {
messages = streams[0].Messages
}
}
}
if err != nil && !errors.Is(err, redis.Nil) {
groupReady = false
waitImage(ctx, time.Second)
continue
}
for _, message := range messages {
if err = a.handleImage(ctx, message, attempts, timeout); err != nil {
slog.Warn("image delivery left pending", "delivery_id", message.ID)
continue
}
if err = ackImage.Run(ctx, broker, []string{imageStream}, imageGroup, message.ID).Err(); err != nil {
slog.Warn("image acknowledgement unavailable", "delivery_id", message.ID)
}
}
}
}
func (a *App) handleImage(ctx context.Context, message redis.XMessage, attempts int, timeout time.Duration) error {
var j imageJob
payload, ok := message.Values["job"].(string)
if !ok || len(payload) > 2048 || json.Unmarshal([]byte(payload), &j) != nil ||
j.SchemaVersion != 1 || j.Type != "attachment.process.v1" || j.Attempt < 1 || (j.Version != 1 && j.Version != 2) {
slog.Warn("invalid image delivery", "delivery_id", message.ID)
return nil
}
for _, id := range []string{j.JobID, j.AttachmentID} {
if normalized, err := normalizeID(id); err != nil || normalized != id {
slog.Warn("invalid image delivery ID", "delivery_id", message.ID)
return nil
}
}
claimCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
src, claimed, err := a.claimImage(claimCtx, j, timeout+5*time.Second)
cancel()
if err != nil || !claimed {
return err // Stale/duplicate attempts are no-ops. A processing row has a recoverable lease.
}
injectImageFault("before_effect", j.AttachmentID)
processCtx, cancel := context.WithTimeout(ctx, timeout)
var variants []Variant
switch {
case os.Getenv("LAB5_FAULT_ATTACHMENT") == j.AttachmentID && os.Getenv("LAB5_FAULT") == "invalid_image":
err = imageFailure("invalid_image")
case os.Getenv("LAB5_FAULT_ATTACHMENT") == j.AttachmentID && os.Getenv("LAB5_FAULT") == "temporary":
err = errors.New("test transient failure")
default:
variants, err = a.generateImage(processCtx, j, src)
}
cancel()
finishCtx, cancel := context.WithTimeout(ctx, 4*time.Second)
defer cancel()
if err != nil {
code := "processing_failed"
var permanent imageFailure
if errors.As(err, &permanent) {
code = string(permanent)
}
return a.failImage(finishCtx, j, code, permanent != "", false, attempts)
}
if err = a.completeImage(finishCtx, j, variants); err != nil {
return err
}
injectImageFault("after_commit", j.AttachmentID)
return nil
}