Files
mtusi2026lab5/app/cmd/worker/main.go
T

376 lines
11 KiB
Go

package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"log"
"math/rand/v2"
"os"
"os/exec"
"os/signal"
"strconv"
"sync"
"syscall"
"time"
"errors"
"gitflic.ru/project/relationskatie/mtusi2026lab1-ashchkl/internal/config"
"gitflic.ru/project/relationskatie/mtusi2026lab1-ashchkl/internal/domain"
"gitflic.ru/project/relationskatie/mtusi2026lab1-ashchkl/internal/objectstore"
"gitflic.ru/project/relationskatie/mtusi2026lab1-ashchkl/internal/pipeline"
"gitflic.ru/project/relationskatie/mtusi2026lab1-ashchkl/internal/store/db"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/redis/go-redis/v9"
)
func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
cfg, err := config.Load()
if err != nil {
log.Fatal(err)
}
pool, err := db.Connect(ctx, cfg.Database)
if err != nil {
log.Fatal(err)
}
defer pool.Close()
objects, err := objectstore.NewS3(cfg.S3)
if err != nil {
log.Fatal(err)
}
if len(os.Args) == 4 && os.Args[1] == "render" {
conn, e := pool.Acquire(ctx)
if e != nil {
os.Exit(1)
}
defer conn.Release()
attempt, _ := strconv.Atoi(os.Args[3])
e = render(ctx, conn, objects, os.Args[2], attempt)
if e != nil {
var pe permanentError
if errors.As(e, &pe) {
os.Exit(42)
}
os.Exit(1)
}
return
}
if len(os.Args) == 3 && os.Args[1] == "replay" {
if err := replay(ctx, pool, objects, os.Args[2]); err != nil {
log.Fatal(err)
}
log.Print("replay queued")
return
}
if len(os.Args) == 2 && os.Args[1] == "failed" {
rows, e := pool.Query(ctx, `SELECT id::text,error_code,attempts FROM attachments WHERE status='failed' ORDER BY updated_at DESC LIMIT 100`)
if e != nil {
log.Fatal(e)
}
defer rows.Close()
for rows.Next() {
var id, code string
var n int
if e = rows.Scan(&id, &code, &n); e != nil {
log.Fatal(e)
}
fmt.Println(id, code, n)
}
if rows.Err() != nil {
log.Fatal(rows.Err())
}
return
}
go pipeline.Dispatch(ctx, pool)
c := pipeline.Client()
defer c.Close()
maxAttempts := 5
if s := os.Getenv("JOB_MAX_ATTEMPTS"); s != "" {
n, e := strconv.Atoi(s)
if e != nil || n < 1 {
log.Fatal("invalid JOB_MAX_ATTEMPTS")
}
maxAttempts = n
}
// Один worker держит сессионную блокировку PostgreSQL. После его смерти новый
// worker возвращает неподтверждённые сообщения в очередь.
lock, err := pool.Acquire(ctx)
if err != nil {
log.Fatal(err)
}
defer lock.Release()
if _, err = lock.Exec(ctx, "SELECT pg_advisory_lock(505001)"); err != nil {
log.Fatal(err)
}
for ctx.Err() == nil {
_, err = c.RPopLPush(ctx, pipeline.Active, pipeline.Pending).Result()
if errors.Is(err, redis.Nil) {
break
}
if err != nil {
log.Fatal("queue recovery failed")
}
}
concurrency := positiveEnv("WORKER_CONCURRENCY", 1, 8)
prefetch := positiveEnv("WORKER_PREFETCH", concurrency, 64)
if prefetch < concurrency {
log.Fatal("WORKER_PREFETCH must be >= WORKER_CONCURRENCY")
}
if int(cfg.Database.MaxConns) < concurrency+2 {
log.Fatal("DB_MAX_CONNS must be >= WORKER_CONCURRENCY+2")
}
slots := make(chan struct{}, prefetch)
jobs := make(chan string, prefetch)
var workers sync.WaitGroup
for i := 0; i < concurrency; i++ {
workers.Add(1)
go func() {
defer workers.Done()
for raw := range jobs {
func() {
defer func() { <-slots }()
conn, e := pool.Acquire(ctx)
if e != nil {
return
} // Active entry is recovered on restart; outbox also retries.
defer conn.Release()
var job pipeline.Job
if json.Unmarshal([]byte(raw), &job) != nil || !validJob(job) {
log.Print("invalid job")
if err := c.Eval(ctx, `redis.call("LPUSH", KEYS[2], ARGV[1]); return redis.call("LREM", KEYS[1], 1, ARGV[1])`, []string{pipeline.Active, "attachments:invalid"}, raw).Err(); err != nil {
return
}
return
}
for ctx.Err() == nil {
done, err := process(ctx, conn, objects, job.AttachmentID, maxAttempts)
if err != nil {
log.Printf("job %s: %v", job.JobID, err)
if conn.Conn().IsClosed() {
log.Fatal("worker database session lost; restart required")
}
}
if done {
if os.Getenv("FAULT_BEFORE_ACK") == "1" {
faultOnce("/tmp/fault-before-ack")
}
if err = c.LRem(ctx, pipeline.Active, 1, raw).Err(); err != nil {
log.Fatal("queue ack failed; restart required")
}
break
}
select {
case <-ctx.Done():
return
case <-time.After(time.Second):
}
}
}()
}
}()
}
defer func() { close(jobs); workers.Wait() }()
for ctx.Err() == nil {
// The owner session alone holds the advisory lock and performs maintenance.
rows, e := lock.Query(ctx, `SELECT o.attachment_id::text FROM attachment_outbox o JOIN attachments a ON a.id=o.attachment_id WHERE a.status='ready' LIMIT 16`)
if e != nil && lock.Conn().IsClosed() {
log.Fatal("worker database session lost; restart required")
}
if e == nil {
var ids []string
for rows.Next() {
var id string
if rows.Scan(&id) == nil {
ids = append(ids, id)
}
}
rows.Close()
for _, id := range ids {
_ = cleanup(ctx, lock, objects, id)
}
}
select {
case slots <- struct{}{}:
case <-ctx.Done():
return
case <-time.After(200 * time.Millisecond):
continue
}
raw, err := c.BRPopLPush(ctx, pipeline.Pending, pipeline.Active, time.Second).Result()
if err != nil {
<-slots
continue
}
select {
case jobs <- raw:
case <-ctx.Done():
<-slots
return
}
}
}
func positiveEnv(name string, fallback, maximum int) int {
value := os.Getenv(name)
if value == "" {
return fallback
}
n, err := strconv.Atoi(value)
if err != nil || n < 1 || n > maximum {
log.Fatalf("invalid %s (1..%d)", name, maximum)
}
return n
}
// One owner with bounded concurrent render processes. A killed render subprocess cannot outlive its deadline.
func process(ctx context.Context, p *pgxpool.Conn, s objectstore.Store, id string, max int) (bool, error) {
var attempt int
err := p.QueryRow(ctx, `UPDATE attachments SET status='processing',attempts=attempts+1,
lease_until=now()+interval '60 seconds',processing_started_at=COALESCE(processing_started_at,now()),updated_at=now()
WHERE id=$1 AND attempts<$2 AND ((status='queued' AND next_attempt_at<=now()) OR
(status='processing' AND lease_until<now())) RETURNING attempts`, id, max).Scan(&attempt)
if errors.Is(err, pgx.ErrNoRows) {
_, err = p.Exec(ctx, `UPDATE attachments SET status='failed',error_code='retry_exhausted',lease_until=NULL,updated_at=now()
WHERE id=$1 AND attempts >= $2 AND (status='queued' OR (status='processing' AND lease_until<now()))`, id, max)
return err == nil, err
}
if err != nil {
return false, err
}
deadline, cancel := context.WithTimeout(ctx, 45*time.Second)
defer cancel()
executable, err := os.Executable()
if err != nil {
return false, err
}
cmd := exec.CommandContext(deadline, executable, "render", id, strconv.Itoa(attempt))
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Pdeathsig: syscall.SIGKILL}
cmd.Cancel = func() error { return syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) }
cmd.WaitDelay = time.Second
err = cmd.Run()
if err == nil {
return true, nil
}
var ee *exec.ExitError
if errors.As(err, &ee) && ee.ExitCode() == 86 {
os.Exit(86)
}
code := "processing_error"
status := "queued"
if errors.As(err, &ee) && ee.ExitCode() == 42 {
code = "invalid_image"
status = "failed"
}
if attempt >= max {
code = "retry_exhausted"
status = "failed"
}
if deadline.Err() != nil {
code = "processing_timeout"
}
delay := retryDelay(attempt)
_, e := p.Exec(ctx, `UPDATE attachments SET status=$3,error_code=$4,lease_until=NULL,
next_attempt_at=now()+($5 * interval '1 millisecond'),updated_at=now()
WHERE id=$1 AND attempts=$2 AND status='processing'`, id, attempt, status, code, delay.Milliseconds())
// The outbox is still present. Ack only after durable retry/failed state.
return e == nil, e
}
func retryDelay(attempt int) time.Duration {
capMs := int64(1000) << min(max(attempt-1, 0), 6)
return time.Duration(capMs/2+rand.Int64N(capMs/2+1)) * time.Millisecond
}
type permanentError struct{ error }
func faultOnce(path string) {
f, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
if err == nil {
f.Close()
os.Exit(86)
}
}
func render(ctx context.Context, p *pgxpool.Conn, s objectstore.Store, id string, attempt int) error {
var key, ct string
if err := p.QueryRow(ctx, `SELECT object_key,content_type FROM attachments WHERE id=$1 AND attempts=$2 AND status='processing'`, id, attempt).Scan(&key, &ct); err != nil {
return err
}
r, _, err := s.Get(ctx, key)
if err != nil {
return err
}
data, err := io.ReadAll(io.LimitReader(r, 10485761))
r.Close()
if err != nil {
return err
}
if len(data) > 10485760 {
return permanentError{fmt.Errorf("image too large")}
}
cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil {
return permanentError{err}
}
if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width)*int64(cfg.Height) > 40000000 {
return permanentError{fmt.Errorf("invalid image dimensions")}
}
im, _, err := image.Decode(bytes.NewReader(data))
if err != nil {
return permanentError{err}
}
im = orient(im, data)
outputs, err := variants(ctx, im)
if err != nil {
return err
}
original := "originals/" + id
if err = s.Put(ctx, original, ct, bytes.NewReader(data), int64(len(data))); err != nil {
return err
}
var metadata []domain.Variant
for _, o := range outputs {
if err = s.Put(ctx, o.meta.Name+"s/"+id, o.meta.ContentType, bytes.NewReader(o.data), int64(len(o.data))); err != nil {
return err
}
metadata = append(metadata, o.meta)
}
if os.Getenv("FAULT_AFTER_OBJECTS") == "1" {
faultOnce("/tmp/fault-after-objects")
}
meta, _ := json.Marshal(metadata)
thumb, _ := json.Marshal(metadata[0])
result, err := p.Exec(ctx, `UPDATE attachments SET status='ready',object_key=$2,thumbnail=$3,variants=$4,
error_code=NULL,lease_until=NULL,updated_at=now() WHERE id=$1 AND attempts=$5 AND status='processing'`, id, original, thumb, meta, attempt)
if err != nil {
return err
}
if result.RowsAffected() != 1 {
return fmt.Errorf("lost lease")
}
// Terminal state first; cleanup is retried by the worker maintenance loop.
return cleanup(ctx, p, s, id)
}
func cleanup(ctx context.Context, p *pgxpool.Conn, s objectstore.Store, id string) error {
if err := s.Delete(ctx, "staging/"+id); err != nil {
return err
}
_, err := p.Exec(ctx, `DELETE FROM attachment_outbox WHERE attachment_id=$1 AND EXISTS(SELECT 1 FROM attachments WHERE id=$1 AND status='ready')`, id)
return err
}
func validJob(j pipeline.Job) bool {
_, e1 := uuid.Parse(j.JobID)
_, e2 := uuid.Parse(j.AttachmentID)
return e1 == nil && e2 == nil && j.SchemaVersion == 1 && j.PipelineVersion == 1 && j.Type == "attachment.process.v1" && j.Attempt >= 1 && !j.CreatedAt.IsZero()
}