ЛР 5: асинхронная обработка изображений и идемпотентность pipeline
This commit is contained in:
+263
@@ -0,0 +1,263 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const imageStream = "attachment-jobs"
|
||||
const imageGroup = "image-workers"
|
||||
|
||||
var ackImage = redis.NewScript(`
|
||||
if redis.call('XACK', KEYS[1], ARGV[1], ARGV[2]) == 1 then
|
||||
redis.call('XDEL', KEYS[1], ARGV[2])
|
||||
end
|
||||
return 1
|
||||
`)
|
||||
|
||||
func imageCommand(db *pgxpool.Pool, args []string) error {
|
||||
a := &App{db: db}
|
||||
retention := positive("STAGING_RETENTION_SECONDS", 86400)
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
if args[0] == "jobs" {
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if len(args) == 2 && args[1] == "failed" {
|
||||
return a.listFailedImages(ctx)
|
||||
}
|
||||
if len(args) != 3 {
|
||||
return errors.New("usage: jobs failed | jobs replay <job-id> | jobs upgrade <attachment-id>")
|
||||
}
|
||||
id, err := normalizeID(args[2])
|
||||
if err != nil {
|
||||
return errors.New("invalid job or attachment ID")
|
||||
}
|
||||
switch args[1] {
|
||||
case "replay":
|
||||
return a.replayImage(ctx, id, retention)
|
||||
case "upgrade":
|
||||
return a.upgradeImage(ctx, id)
|
||||
default:
|
||||
return errors.New("unknown jobs command")
|
||||
}
|
||||
}
|
||||
|
||||
if len(args) != 1 || (args[0] != "dispatcher" && args[0] != "worker") {
|
||||
return errors.New("unknown command")
|
||||
}
|
||||
attempts := positive("JOB_MAX_ATTEMPTS", 5)
|
||||
timeoutSeconds := positive("JOB_TIMEOUT_SECONDS", 30)
|
||||
if attempts > 20 || timeoutSeconds > 120 || positive("WORKER_CONCURRENCY", 1) != 1 {
|
||||
return errors.New("invalid image job limits")
|
||||
}
|
||||
timeout := time.Duration(timeoutSeconds) * time.Second
|
||||
s3, bucket, err := s3Client()
|
||||
if err != nil {
|
||||
return errors.New("invalid image storage configuration")
|
||||
}
|
||||
a.s3, a.bucket = s3, bucket
|
||||
brokerURL, err := secretFile("BROKER_URL_FILE")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
opts, err := redis.ParseURL(brokerURL)
|
||||
if err != nil {
|
||||
return errors.New("invalid broker configuration")
|
||||
}
|
||||
opts.MaxRetries = -1
|
||||
opts.PoolSize = 2
|
||||
opts.DialTimeout = time.Second
|
||||
opts.ReadTimeout = 2 * time.Second
|
||||
opts.WriteTimeout = time.Second
|
||||
opts.ContextTimeoutEnabled = true
|
||||
broker := redis.NewClient(opts)
|
||||
defer broker.Close()
|
||||
|
||||
if args[0] == "dispatcher" {
|
||||
a.dispatchImages(ctx, broker, attempts, retention)
|
||||
} else {
|
||||
a.consumeImages(ctx, broker, attempts, timeout)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func injectImageFault(phase, attachment string) {
|
||||
if os.Getenv("LAB5_FAULT_ATTACHMENT") == attachment && os.Getenv("LAB5_FAULT") == phase {
|
||||
slog.Info("test crash", "phase", phase, "attachment_id", attachment)
|
||||
os.Exit(86)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) publishImage(ctx context.Context, broker *redis.Client) (bool, error) {
|
||||
tx, err := a.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer rollbackAttachment(tx)
|
||||
j := imageJob{SchemaVersion: 1, Type: "attachment.process.v1"}
|
||||
var created time.Time
|
||||
err = tx.QueryRow(ctx, `SELECT id::text,attachment_id::text,version,attempt,created_at FROM image_jobs
|
||||
WHERE state='queued' AND NOT published AND next_at<=clock_timestamp()
|
||||
ORDER BY next_at,id LIMIT 1 FOR UPDATE SKIP LOCKED`).Scan(&j.JobID, &j.AttachmentID, &j.Version, &j.Attempt, &created)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
j.CreatedAt = stamp(created)
|
||||
payload, err := json.Marshal(j)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
injectImageFault("before_publish", j.AttachmentID)
|
||||
// XADD replies after the broker's AOF durability boundary. A lost reply
|
||||
// or crash before the SQL commit causes another delivery of the same job.
|
||||
err = broker.XAdd(ctx, &redis.XAddArgs{Stream: imageStream, Values: map[string]any{"job": string(payload)}}).Err()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
injectImageFault("after_publish", j.AttachmentID)
|
||||
if _, err = tx.Exec(ctx, "UPDATE image_jobs SET published=true WHERE id=$1", j.JobID); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (a *App) dispatchImages(ctx context.Context, broker *redis.Client, attempts, retention int) {
|
||||
cleanup := time.Now()
|
||||
for ctx.Err() == nil {
|
||||
step, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
recoverErr := a.recoverImage(step, attempts)
|
||||
published, publishErr := a.publishImage(step, broker)
|
||||
cancel()
|
||||
if recoverErr != nil || publishErr != nil {
|
||||
slog.Warn("image outbox unavailable", "recover_failed", recoverErr != nil, "publish_failed", publishErr != nil)
|
||||
}
|
||||
if time.Now().After(cleanup) {
|
||||
step, cancel = context.WithTimeout(ctx, 5*time.Second)
|
||||
if err := a.cleanStaging(step, retention); err != nil && ctx.Err() == nil {
|
||||
slog.Warn("staging cleanup incomplete")
|
||||
}
|
||||
cancel()
|
||||
cleanup = time.Now().Add(15 * time.Second)
|
||||
}
|
||||
if !published || recoverErr != nil {
|
||||
waitImage(ctx, 300*time.Millisecond)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func waitImage(ctx context.Context, delay time.Duration) {
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-timer.C:
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) consumeImages(ctx context.Context, broker *redis.Client, attempts int, timeout time.Duration) {
|
||||
cursor := "0-0"
|
||||
groupReady := false
|
||||
for ctx.Err() == nil {
|
||||
if !groupReady {
|
||||
err := broker.XGroupCreateMkStream(ctx, imageStream, imageGroup, "0").Err()
|
||||
if err != nil && !strings.HasPrefix(err.Error(), "BUSYGROUP") {
|
||||
waitImage(ctx, time.Second)
|
||||
continue
|
||||
}
|
||||
groupReady = true
|
||||
}
|
||||
messages, next, err := broker.XAutoClaim(ctx, &redis.XAutoClaimArgs{
|
||||
Stream: imageStream, Group: imageGroup, Consumer: "worker", MinIdle: timeout + 10*time.Second, Start: cursor, Count: 1,
|
||||
}).Result()
|
||||
if err == nil {
|
||||
cursor = next
|
||||
if len(messages) == 0 {
|
||||
var streams []redis.XStream
|
||||
streams, err = broker.XReadGroup(ctx, &redis.XReadGroupArgs{
|
||||
Group: imageGroup, Consumer: "worker", Streams: []string{imageStream, ">"}, Count: 1, Block: time.Second,
|
||||
}).Result()
|
||||
if err == nil && len(streams) > 0 {
|
||||
messages = streams[0].Messages
|
||||
}
|
||||
}
|
||||
}
|
||||
if err != nil && !errors.Is(err, redis.Nil) {
|
||||
groupReady = false
|
||||
waitImage(ctx, time.Second)
|
||||
continue
|
||||
}
|
||||
for _, message := range messages {
|
||||
if err = a.handleImage(ctx, message, attempts, timeout); err != nil {
|
||||
slog.Warn("image delivery left pending", "delivery_id", message.ID)
|
||||
continue
|
||||
}
|
||||
if err = ackImage.Run(ctx, broker, []string{imageStream}, imageGroup, message.ID).Err(); err != nil {
|
||||
slog.Warn("image acknowledgement unavailable", "delivery_id", message.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) handleImage(ctx context.Context, message redis.XMessage, attempts int, timeout time.Duration) error {
|
||||
var j imageJob
|
||||
payload, ok := message.Values["job"].(string)
|
||||
if !ok || len(payload) > 2048 || json.Unmarshal([]byte(payload), &j) != nil ||
|
||||
j.SchemaVersion != 1 || j.Type != "attachment.process.v1" || j.Attempt < 1 || (j.Version != 1 && j.Version != 2) {
|
||||
slog.Warn("invalid image delivery", "delivery_id", message.ID)
|
||||
return nil
|
||||
}
|
||||
for _, id := range []string{j.JobID, j.AttachmentID} {
|
||||
if normalized, err := normalizeID(id); err != nil || normalized != id {
|
||||
slog.Warn("invalid image delivery ID", "delivery_id", message.ID)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
claimCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
src, claimed, err := a.claimImage(claimCtx, j, timeout+5*time.Second)
|
||||
cancel()
|
||||
if err != nil || !claimed {
|
||||
return err // Stale/duplicate attempts are no-ops. A processing row has a recoverable lease.
|
||||
}
|
||||
injectImageFault("before_effect", j.AttachmentID)
|
||||
processCtx, cancel := context.WithTimeout(ctx, timeout)
|
||||
var variants []Variant
|
||||
switch {
|
||||
case os.Getenv("LAB5_FAULT_ATTACHMENT") == j.AttachmentID && os.Getenv("LAB5_FAULT") == "invalid_image":
|
||||
err = imageFailure("invalid_image")
|
||||
case os.Getenv("LAB5_FAULT_ATTACHMENT") == j.AttachmentID && os.Getenv("LAB5_FAULT") == "temporary":
|
||||
err = errors.New("test transient failure")
|
||||
default:
|
||||
variants, err = a.generateImage(processCtx, j, src)
|
||||
}
|
||||
cancel()
|
||||
finishCtx, cancel := context.WithTimeout(ctx, 4*time.Second)
|
||||
defer cancel()
|
||||
if err != nil {
|
||||
code := "processing_failed"
|
||||
var permanent imageFailure
|
||||
if errors.As(err, &permanent) {
|
||||
code = string(permanent)
|
||||
}
|
||||
return a.failImage(finishCtx, j, code, permanent != "", false, attempts)
|
||||
}
|
||||
if err = a.completeImage(finishCtx, j, variants); err != nil {
|
||||
return err
|
||||
}
|
||||
injectImageFault("after_commit", j.AttachmentID)
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user