diff --git a/compose.yml b/compose.yml index a2d84dff..7e063538 100644 --- a/compose.yml +++ b/compose.yml @@ -114,6 +114,7 @@ services: BROKER_URL_FILE: /run/secrets/broker_url JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + WORKER_MAX_PIXELS: ${WORKER_MAX_PIXELS:-8000000} UPLOAD_TMP_DIR: /tmp/uploads UPLOAD_CLEANUP_GRACE_SECONDS: ${UPLOAD_CLEANUP_GRACE_SECONDS:-600} # Go 1.22 sees host CPUs; keep parallelism suitable for this 0.5 CPU limit. @@ -177,6 +178,8 @@ services: start_period: 20s retries: 30 worker: + tmpfs: + - /tmp:rw,noexec,nosuid,size=64m,mode=1777 build: ./src restart: unless-stopped cpus: 1.0 diff --git a/infra/compose.faults.yml b/infra/compose.faults.yml new file mode 100644 index 00000000..1bf122ba --- /dev/null +++ b/infra/compose.faults.yml @@ -0,0 +1,13 @@ +# Operator-only fault injection. No client API can create these markers. +services: + dispatcher: + environment: + PIPELINE_FAULT_DIR: /run/pipeline-faults + volumes: + - ./.memory/grade4-faults:/run/pipeline-faults + worker: + environment: + PIPELINE_FAULT_DIR: /run/pipeline-faults + JOB_TIMEOUT_SECONDS: "8" + volumes: + - ./.memory/grade4-faults:/run/pipeline-faults diff --git a/scripts/check_async_grade4.py b/scripts/check_async_grade4.py new file mode 100644 index 00000000..444c5af9 --- /dev/null +++ b/scripts/check_async_grade4.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Grade 4: real fault injection on synthetic jobs; restores the base stack.""" +import datetime as dt +import hashlib +import hmac +import json +import os +from pathlib import Path +import struct +import subprocess +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET +import zlib + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check +from image_probe import jpeg_size + +FAULTS = ROOT / '.memory/grade4-faults' +PREFIX = ['docker', 'compose', '--project-name', 'highload-lab5', '-f', 'compose.yml', '-f', 'infra/compose.faults.yml'] + +def compose(*args): + return subprocess.check_output(PREFIX + list(args), cwd=ROOT, text=True, stderr=subprocess.DEVNULL).strip() + +def sql(query): + return compose('exec', '-T', 'db', 'sh', '-c', 'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"', 'sh', query) + +def until(fn, seconds=120): + deadline = time.monotonic()+seconds + last = None + while time.monotonic()>14)&16383)+1) + if kind==b'VP8X': + dimensions=(int.from_bytes(body[4:7],'little')+1,int.from_bytes(body[7:10],'little')+1) + offset+=8+length+(length%2) + return dimensions + +def large_png(w,h): + def chunk(kind,data): + return struct.pack('>I',len(data))+kind+data+struct.pack('>I',zlib.crc32(kind+data)&0xffffffff) + encoder=zlib.compressobj();pieces=[];row=b'\0'+b'\x80\x80\x80\xff'*w + for _ in range(h): + pieces.append(encoder.compress(row)) + pieces.append(encoder.flush()) + return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',w,h,8,6,0,0,0))+chunk(b'IDAT',b''.join(pieces))+chunk(b'IEND',b'') + +def main(): + FAULTS.mkdir(parents=True, exist_ok=True);os.chmod(FAULTS,0o777) + compose('up','-d','--wait') + c=Client(sys.argv[1] if len(sys.argv)>1 else 'http://localhost:8085') + alice,bob,eve,chat=c.bootstrap() + results={} + def metadata(att): + return c.call('GET','/api/v1/attachments/'+att['id'],actor=alice,schema='Attachment')['json'] + def ready(att): + return until(lambda: (value:=metadata(att))['status']=='ready' and value) + def verify(att, original): + value=ready(att) + check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==original,'original changed') + check({v['name'] for v in value['variants']}=={'thumbnail','crop','optimized'},'variant set incomplete') + for variant in value['variants']: + path='/api/v1/attachments/'+att['id']+'/content?variant='+variant['name'] + response=c.call('GET',path,actor=bob) + check(response['headers'].get_content_type()==variant['content_type'],'wrong variant MIME') + check(hashlib.sha256(response['bytes']).hexdigest()==variant['sha256'] and len(response['bytes'])==variant['size_bytes'],'wrong variant checksum') + dimensions=webp_size(response['bytes']) if variant['name']=='optimized' else jpeg_size(response['bytes']) + expected={'thumbnail':(256,160),'crop':(128,128),'optimized':(320,200)}[variant['name']] + check(dimensions==expected==(variant['width'],variant['height']),'actual variant dimensions differ') + c.call('GET',path,404,actor=eve) + until(lambda: s3('HEAD','staging/jobs/'+att['id'])[0]==404) + code,body=s3('GET',query={'list-type':'2','prefix':'variants/'+att['id']+'/'}) + check(code==200 and len(ET.fromstring(body).findall('{*}Contents'))==3,'duplicate object set') + return value + try: + compose('stop','dispatcher') + att,original=c.attachment(alice,chat) + marker=arm('after_commit_before_publish',att) + check(sql("SELECT published_at IS NULL FROM attachment_jobs WHERE id='"+job_id(att)+"'")=='t','job already published') + compose('start','dispatcher');until(marker.exists) + verify(att,original) + results['commit_publish_crash']={'attachment_id':att['id'],'ready':True,'same_job_count':int(sql("SELECT count(*) FROM attachment_jobs WHERE attachment_id='"+att['id']+"'"))} + print('PASS: crash between COMMIT and publish recovers durable intent.',flush=True) + + for point in ('after_object_before_commit','after_commit_before_ack'): + compose('stop','worker');att,original=c.attachment(alice,chat);marker=arm(point,att) + compose('start','worker');until(marker.exists) + value=verify(att,original) + attempts=int(sql("SELECT attempts FROM attachment_jobs WHERE id='"+job_id(att)+"'")) + check(attempts==(2 if point=='after_object_before_commit' else 1),'unexpected attempt count after crash') + results[point]={'attachment_id':att['id'],'attempts':attempts,'variant_count':len(value['variants'])} + print('PASS: '+point+' recovers without duplicate effects.',flush=True) + + compose('stop','worker');att,original=c.attachment(alice,chat);marker=arm('transform_delay',att) + compose('start','worker') + restarts_before=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}}','highload-lab5-worker-1'],text=True) + started=time.monotonic();until(marker.exists);verify(att,original) + elapsed=time.monotonic()-started + check(subprocess.check_output(['docker','inspect','--format','{{.RestartCount}}','highload-lab5-worker-1'],text=True)==restarts_before,'worker restarted during child timeout') + check(elapsed<30,'child timeout did not bound processing') + check(compose('exec','-T','worker','sh','-c',"find /tmp -maxdepth 1 -name 'image-job-*' -print")=="",'child scratch leaked') + reasons=sql("SELECT error_code FROM attachment_job_attempts WHERE job_id='"+job_id(att)+"' AND error_code IS NOT NULL") + check('processing_timeout' in reasons,'child timeout was not recorded') + results['child_timeout']={'attachment_id':att['id'],'audit_reason':'processing_timeout','elapsed_seconds':round(elapsed,3),'worker_survived':True,'scratch_removed':True} + print('PASS: stuck image child is killed and retry succeeds.',flush=True) + + compose('stop','worker');att,original=c.attachment(alice,chat);job=job_id(att) + sql("UPDATE attachment_jobs SET staging_key='staging/jobs/missing-test-object' WHERE id='"+job+"'") + compose('start','worker') + until(lambda: metadata(att)['status']=='failed',180) + failures=admin('list-failed');check(any(item['job_id']==job and item['attempts']==5 for item in failures),'failed registry missing job') + timing=json.loads(sql("SELECT json_agg(json_build_object('attempt',attempt,'delay',EXTRACT(EPOCH FROM next_attempt_at-finished_at))) FROM attachment_job_attempts WHERE job_id='"+job+"' AND next_attempt_at IS NOT NULL")) + for item in timing: + ceiling=min(30,2**item['attempt']);check(ceiling/2-.15<=float(item['delay'])<=ceiling+.15,'backoff not in jitter window') + sql("UPDATE attachment_jobs SET staging_key='staging/jobs/"+att['id']+"' WHERE id='"+job+"'") + check(s3('HEAD','staging/jobs/'+att['id'])[0]==200,'failed staging was removed before retention') + check(admin('replay',job,'Synthetic storage failure resolved')['changed'],'failed replay not accepted') + verify(att,original) + check(not admin('replay',job,'Already successful job')['changed'],'ready replay reopened job') + check(sql("SELECT count(*) FROM attachment_job_replays WHERE job_id='"+job+"'")=='1','replay not audited exactly once') + results['retry_and_replay']={'attachment_id':att['id'],'attempts_before_replay':5,'retry_delays':timing,'audited_replays':1} + print('PASS: bounded exponential jitter, failed registry, audited safe replay.',flush=True) + + data=large_png(3000,3000) + att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=large.png',202,actor=alice,raw=data,headers={'Content-Type':'image/png'},schema='Attachment')['json'] + until(lambda: metadata(att)['status']=='failed') + check(metadata(att)['error_code']=='invalid_image','worker pixel limit not enforced') + check(sql("SELECT attempts FROM attachment_jobs WHERE id='"+job_id(att)+"'")=='1','permanent error retried') + check(s3('HEAD','staging/jobs/'+att['id'])[0]==200,'failed source removed early') + sql("UPDATE attachment_job_failures SET failed_at=now()-interval '25 hours' WHERE job_id='"+job_id(att)+"'") + admin('cleanup-staging',job_id(att)) + check(s3('HEAD','staging/jobs/'+att['id'])[0]==404,'expired failed source not removed') + results['permanent_error_retention']={'attachment_id':att['id'],'pixels':9000000,'attempts':1,'expired_staging_removed':True} + compose('stop','worker');queued,_=c.attachment(alice,chat) + check(not admin('cleanup-staging',job_id(queued))['changed'],'cleanup deleted queued work') + check(s3('HEAD','staging/jobs/'+queued['id'])[0]==200,'queued source missing') + compose('start','worker');ready(queued) + config=json.loads(subprocess.check_output(['docker','inspect','highload-lab5-worker-1'],text=True))[0] + check(config['HostConfig']['Memory']==512*1024*1024,'worker hard memory limit absent') + check('size=64m' in config['HostConfig']['Tmpfs']['/tmp'],'temporary disk not bounded') + results['resources']={'memory_limit_bytes':config['HostConfig']['Memory'],'prefetch':1,'image_timeout_seconds':8,'worker_pixel_limit':8000000,'queued_staging_protected':True} + (ROOT/'.memory/grade4-results.json').write_text(json.dumps(results,ensure_ascii=False,indent=2)+'\n') + print('PASS: grade 4 variants, fault windows, timeout, retry/replay, retention and resource limits.',flush=True) + finally: + # Remove test-only mounts and restore regular timeout/configuration. + subprocess.run(['docker','compose','--project-name','highload-lab5','up','-d','--wait'],cwd=ROOT,stdout=subprocess.DEVNULL,check=True) + +if __name__=='__main__': + main() diff --git a/src/Dockerfile b/src/Dockerfile index d047c6d5..088e1496 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -1,5 +1,7 @@ FROM golang:1.22.2-bookworm AS builder +RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/* + WORKDIR /src COPY go.mod go.sum ./ @@ -19,7 +21,8 @@ RUN CGO_ENABLED=0 \ FROM debian:12.15-slim -RUN groupadd --system app \ +RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/* \ + && groupadd --system app \ && useradd --system --gid app --no-create-home app WORKDIR /app diff --git a/src/backend/app/server.go b/src/backend/app/server.go new file mode 100644 index 00000000..57c478ff --- /dev/null +++ b/src/backend/app/server.go @@ -0,0 +1,140 @@ +package app + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "net" + "net/http" + "os" + "os/signal" + "runtime" + "syscall" + "time" + + "messenger/src/backend/handlers" + "messenger/src/backend/repositories" + "messenger/src/backend/services" +) + +func serverAddress() string { + host := os.Getenv("APP_HOST") + if host == "" { + host = "0.0.0.0" + } + + port := os.Getenv("APP_PORT") + if port == "" { + port = "8080" + } + + return net.JoinHostPort(host, port) +} + +func RunHTTP() error { + if _, err := repositories.UploadCleanupGrace(); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + db, err := repositories.OpenDatabase(ctx) + cancel() + if err != nil { + return fmt.Errorf("database initialization failed: %w", err) + } + defer db.Close() + // Opt-in experiment telemetry goes to container logs, never a public port. + if os.Getenv("BENCH_POOL_STATS") == "1" { + go func() { + for range time.Tick(time.Second) { + s := db.Stat() + data, _ := json.Marshal(map[string]any{ + "time": time.Now().UTC(), "acquired": s.AcquiredConns(), + "total": s.TotalConns(), "max": s.MaxConns(), + "gomaxprocs": runtime.GOMAXPROCS(0), "goroutines": runtime.NumGoroutine(), + "acquire_count": s.AcquireCount(), "acquire_duration_ns": s.AcquireDuration().Nanoseconds(), + "empty_acquire_count": s.EmptyAcquireCount(), "canceled_acquire_count": s.CanceledAcquireCount(), + }) + log.Printf("benchmark_pool %s", data) + } + }() + } + redisClient, err := repositories.RedisClientFromEnvironment() + if err != nil { + return err + } + defer redisClient.Close() + sessionStore := repositories.NewSessionStore(redisClient, "messenger:session:") + userStore := repositories.NewUserStore(db) + chatStore := repositories.NewChatStore(db) + idempotencyTTL, err := repositories.IdempotencyTTL() + if err != nil { + return err + } + messageStore := repositories.NewMessageStoreWithTTL(db, idempotencyTTL) + ttl, err := services.SessionTTL() + if err != nil { + return err + } + userService := services.NewUserService(userStore) + sessionService := services.NewSessionService(userService, sessionStore, ttl) + chatService := services.NewChatService(chatStore) + messageService := services.NewMessageService(messageStore) + objects, err := repositories.NewObjectStoreFromEnvironment() + if err != nil { + return err + } + attachmentService := services.NewAttachmentService(repositories.NewAttachmentStore(db), objects) + mux := handlers.NewRouter(services.NewHealthService(repositories.NewHealthStore(db), sessionStore, objects), userService, sessionService, chatService, messageService, attachmentService) + + server := &http.Server{ + Addr: serverAddress(), + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 60 * time.Second, + } + + shutdownContext, stop := signal.NotifyContext( + context.Background(), + os.Interrupt, + syscall.SIGTERM, + ) + defer stop() + + serverErrors := make(chan error, 1) + + go func() { + log.Printf("server is listening on %s", server.Addr) + + if err := server.ListenAndServe(); err != nil && + !errors.Is(err, http.ErrServerClosed) { + serverErrors <- err + } + }() + + select { + case err := <-serverErrors: + log.Printf("server error: %v", err) + + case <-shutdownContext.Done(): + log.Printf("shutdown signal received") + handlers.BeginShutdown() + + shutdownDeadline, cancel := context.WithTimeout( + context.Background(), + 10*time.Second, + ) + defer cancel() + + if err := server.Shutdown(shutdownDeadline); err != nil { + log.Printf("graceful shutdown failed: %v", err) + return nil + } + + log.Printf("server stopped gracefully") + } + return nil +} diff --git a/src/backend/cleanup.go b/src/backend/commands/cleanup.go similarity index 83% rename from src/backend/cleanup.go rename to src/backend/commands/cleanup.go index 95198904..08c5d5e4 100644 --- a/src/backend/cleanup.go +++ b/src/backend/commands/cleanup.go @@ -1,15 +1,16 @@ -package main +package commands import ( "context" "encoding/json" "flag" "log" - "messenger/src/backend/repositories" "os" "os/signal" "syscall" "time" + + "messenger/src/backend/repositories" ) func runUploadCleanup(args []string) error { @@ -37,17 +38,21 @@ func runUploadCleanup(args []string) error { pass, cancel := context.WithTimeout(ctx, 30*time.Second) result, err := store.CleanupUploads(pass, *dryRun, objects.Delete) direct, directErr := store.CleanupDirectUploads(pass, *dryRun, objects.Delete) + jobs, jobsErr := store.CleanupJobs(pass, 24*time.Hour, *dryRun, objects.Delete) cancel() - if encodeErr := json.NewEncoder(os.Stdout).Encode(map[string]any{"originals": result, "direct": direct}); encodeErr != nil { + if encodeErr := json.NewEncoder(os.Stdout).Encode(map[string]any{"originals": result, "direct": direct, "job_staging": jobs}); encodeErr != nil { return encodeErr } if !*loop { if err == nil { err = directErr + if err == nil { + err = jobsErr + } } return err } - if err != nil || directErr != nil { + if err != nil || directErr != nil || jobsErr != nil { log.Print("upload cleanup pass failed; journal retained for retry") } select { diff --git a/src/backend/commands/jobs.go b/src/backend/commands/jobs.go new file mode 100644 index 00000000..e102d858 --- /dev/null +++ b/src/backend/commands/jobs.go @@ -0,0 +1,63 @@ +package commands + +import ( + "context" + "encoding/json" + "errors" + "os" + "time" + + "github.com/google/uuid" + + "messenger/src/backend/repositories" +) + +func runJobAdmin(args []string) error { + if len(args) < 1 { + return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID") + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + db, err := repositories.OpenDatabase(ctx) + if err != nil { + return errors.New("database unavailable") + } + defer db.Close() + store := repositories.NewAttachmentStore(db) + if args[0] == "list-failed" && len(args) == 1 { + jobs, e := store.FailedJobs(ctx) + if e != nil { + return errors.New("cannot read failed registry") + } + return json.NewEncoder(os.Stdout).Encode(jobs) + } + if len(args) < 2 { + return errors.New("job id required") + } + if _, err = uuid.Parse(args[1]); err != nil { + return errors.New("invalid job id") + } + objects, err := repositories.NewObjectStoreFromEnvironment() + if err != nil { + return err + } + var changed bool + switch { + case args[0] == "replay" && len(args) == 3: + changed, err = store.ReplayJob(ctx, args[1], args[2], func(ctx context.Context, key string) error { + r, e := objects.Get(ctx, key) + if e == nil { + r.Close() + } + return e + }) + case args[0] == "cleanup-staging" && len(args) == 2: + changed, err = store.CleanupJobStaging(ctx, args[1], 24*time.Hour, false, objects.Delete) + default: + return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID") + } + if err != nil { + return err + } + return json.NewEncoder(os.Stdout).Encode(map[string]bool{"changed": changed}) +} diff --git a/src/backend/commands/run.go b/src/backend/commands/run.go new file mode 100644 index 00000000..fa4fe476 --- /dev/null +++ b/src/backend/commands/run.go @@ -0,0 +1,79 @@ +package commands + +import ( + "context" + "errors" + "log" + "net" + "net/http" + "os" + "time" + + "messenger/src/backend/app" + "messenger/src/backend/repositories" + "messenger/src/backend/workers" +) + +// Run dispatches the existing executable modes; the binary and CLI stay the same. +func Run(args []string) error { + if len(args) == 0 { + if environment := os.Getenv("APP_ENV"); environment != "" { + log.SetPrefix("[" + environment + "] ") + } + return app.RunHTTP() + } + switch args[0] { + case "image-process": + return workers.RunImageChild(args[1:]) + case "jobs": + return runJobAdmin(args[1:]) + case "worker", "dispatcher", "broker-publish": + return workers.RunPipeline(args[0], args[1:]) + } + if environment := os.Getenv("APP_ENV"); environment != "" { + log.SetPrefix("[" + environment + "] ") + } + switch { + case args[0] == "s3-bootstrap" && len(args) == 1: + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := repositories.BootstrapS3(ctx); err != nil { + return err + } + log.Print("private S3 bucket ready") + return nil + case args[0] == "attachments-cleanup": + return runUploadCleanup(args[1:]) + case args[0] == "healthcheck" && len(args) == 1: + return healthcheck() + case args[0] == "migrate" && len(args) == 2 && (args[1] == "up" || args[1] == "down"): + url, err := repositories.DatabaseURL() + if err != nil { + return err + } + if err = repositories.RunMigrations(url, args[1]); err != nil { + return err + } + log.Printf("migration %s complete", args[1]) + return nil + default: + return errors.New("usage: messenger migrate up|down") + } +} + +func healthcheck() error { + client := http.Client{Timeout: 2 * time.Second} + port := os.Getenv("APP_PORT") + if port == "" { + port = "8080" + } + response, err := client.Get("http://" + net.JoinHostPort("127.0.0.1", port) + "/health/ready") + if err != nil { + return err + } + response.Body.Close() + if response.StatusCode != http.StatusOK { + return errors.New("API is not ready") + } + return nil +} diff --git a/src/backend/handlers/attachment_http.go b/src/backend/handlers/attachment_http.go index a122a6ee..15c6cc82 100644 --- a/src/backend/handlers/attachment_http.go +++ b/src/backend/handlers/attachment_http.go @@ -116,12 +116,16 @@ func attachmentHandler(attachments *services.AttachmentService) http.HandlerFunc writeError(w, 400, "invalid_request", "invalid variant", requestID) return } - if variants[0] == "thumbnail" { + if variants[0] == "thumbnail" || variants[0] == "crop" || variants[0] == "optimized" { found := false for _, raw := range a.Variants { - if v, ok := raw.(map[string]any); ok && v["name"] == "thumbnail" { - a.ObjectKey = "variants/" + a.ID + "/v1/thumbnail.jpg" - a.ContentType = "image/jpeg" + if v, ok := raw.(map[string]any); ok && v["name"] == variants[0] { + extension := ".jpg" + if variants[0] == "optimized" { + extension = ".webp" + } + a.ObjectKey = "variants/" + a.ID + "/v1/" + variants[0] + extension + a.ContentType = v["content_type"].(string) a.SizeBytes = int64(v["size_bytes"].(float64)) found = true break diff --git a/src/backend/internal/faults/faults.go b/src/backend/internal/faults/faults.go new file mode 100644 index 00000000..74c7c7ef --- /dev/null +++ b/src/backend/internal/faults/faults.go @@ -0,0 +1,29 @@ +package faults + +import ( + "log" + "os" + "path/filepath" + + "github.com/google/uuid" +) + +// Only an operator-mounted directory enables faults; clients cannot set them. +// Rename makes each injected crash one-shot across automatic container restarts. +func Take(point, id string) bool { + directory := os.Getenv("PIPELINE_FAULT_DIR") + if directory == "" { + return false + } + if _, err := uuid.Parse(id); err != nil { + return false + } + name := filepath.Join(directory, point+"."+id) + return os.Rename(name, name+".hit") == nil +} +func Crash(point, id string) { + if Take(point, id) { + log.Printf("injected pipeline crash: %s, attachment %s", point, id) + os.Exit(86) + } +} diff --git a/src/backend/main.go b/src/backend/main.go index b9b9368e..a7f2d4bf 100644 --- a/src/backend/main.go +++ b/src/backend/main.go @@ -1,191 +1,17 @@ package main import ( - "context" - "encoding/json" - "errors" "log" - "messenger/src/backend/handlers" - "messenger/src/backend/repositories" - "messenger/src/backend/services" - "net" - "net/http" "os" - "os/signal" - "runtime" - "syscall" - "time" + + "messenger/src/backend/commands" ) -func serverAddress() string { - host := os.Getenv("APP_HOST") - if host == "" { - host = "0.0.0.0" - } - - port := os.Getenv("APP_PORT") - if port == "" { - port = "8080" - } - - return net.JoinHostPort(host, port) -} - func main() { - if len(os.Args) >= 2 && (os.Args[1] == "worker" || os.Args[1] == "dispatcher" || os.Args[1] == "broker-publish") { - if err := runPipeline(os.Args[1]); err != nil { - log.Fatal("pipeline initialization failed: ", err) + if err := commands.Run(os.Args[1:]); err != nil { + if len(os.Args) > 1 && (os.Args[1] == "image-process" || os.Args[1] == "healthcheck") { + os.Exit(1) } - return - } - if environment := os.Getenv("APP_ENV"); environment != "" { - log.SetPrefix("[" + environment + "] ") - } - if len(os.Args) == 2 && os.Args[1] == "s3-bootstrap" { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - if err := repositories.BootstrapS3(ctx); err != nil { - log.Fatal("S3 bootstrap failed: ", err) - } - log.Print("private S3 bucket ready") - return - } - if len(os.Args) > 1 && os.Args[1] == "attachments-cleanup" { - if err := runUploadCleanup(os.Args[2:]); err != nil { - log.Fatal("upload cleanup failed: ", err) - } - return - } - if len(os.Args) > 1 { - if len(os.Args) == 2 && os.Args[1] == "healthcheck" { - client := http.Client{Timeout: 2 * time.Second} - port := os.Getenv("APP_PORT") - if port == "" { - port = "8080" - } - response, err := client.Get("http://" + net.JoinHostPort("127.0.0.1", port) + "/health/ready") - if err != nil { - os.Exit(1) - } - response.Body.Close() - if response.StatusCode != http.StatusOK { - os.Exit(1) - } - return - } - if len(os.Args) != 3 || os.Args[1] != "migrate" || (os.Args[2] != "up" && os.Args[2] != "down") { - log.Fatal("usage: messenger migrate up|down") - } - url, err := repositories.DatabaseURL() - if err != nil { - log.Fatal(err) - } - if err = repositories.RunMigrations(url, os.Args[2]); err != nil { - log.Fatal(err) - } - log.Printf("migration %s complete", os.Args[2]) - return - } - if _, err := repositories.UploadCleanupGrace(); err != nil { log.Fatal(err) } - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - db, err := repositories.OpenDatabase(ctx) - cancel() - if err != nil { - log.Fatalf("database initialization failed: %v", err) - } - defer db.Close() - // Opt-in experiment telemetry goes to container logs, never a public port. - if os.Getenv("BENCH_POOL_STATS") == "1" { - go func() { - for range time.Tick(time.Second) { - s := db.Stat() - data, _ := json.Marshal(map[string]any{ - "time": time.Now().UTC(), "acquired": s.AcquiredConns(), - "total": s.TotalConns(), "max": s.MaxConns(), - "gomaxprocs": runtime.GOMAXPROCS(0), "goroutines": runtime.NumGoroutine(), - "acquire_count": s.AcquireCount(), "acquire_duration_ns": s.AcquireDuration().Nanoseconds(), - "empty_acquire_count": s.EmptyAcquireCount(), "canceled_acquire_count": s.CanceledAcquireCount(), - }) - log.Printf("benchmark_pool %s", data) - } - }() - } - redisClient, err := repositories.RedisClientFromEnvironment() - if err != nil { - log.Fatal(err) - } - defer redisClient.Close() - sessionStore := repositories.NewSessionStore(redisClient, "messenger:session:") - userStore := repositories.NewUserStore(db) - chatStore := repositories.NewChatStore(db) - idempotencyTTL, err := repositories.IdempotencyTTL() - if err != nil { - log.Fatal(err) - } - messageStore := repositories.NewMessageStoreWithTTL(db, idempotencyTTL) - ttl, err := services.SessionTTL() - if err != nil { - log.Fatal(err) - } - userService := services.NewUserService(userStore) - sessionService := services.NewSessionService(userService, sessionStore, ttl) - chatService := services.NewChatService(chatStore) - messageService := services.NewMessageService(messageStore) - objects, err := repositories.NewObjectStoreFromEnvironment() - if err != nil { - log.Fatal(err) - } - attachmentService := services.NewAttachmentService(repositories.NewAttachmentStore(db), objects) - mux := handlers.NewRouter(services.NewHealthService(repositories.NewHealthStore(db), sessionStore, objects), userService, sessionService, chatService, messageService, attachmentService) - - server := &http.Server{ - Addr: serverAddress(), - Handler: mux, - ReadHeaderTimeout: 5 * time.Second, - ReadTimeout: 30 * time.Second, - WriteTimeout: 30 * time.Second, - IdleTimeout: 60 * time.Second, - } - - shutdownContext, stop := signal.NotifyContext( - context.Background(), - os.Interrupt, - syscall.SIGTERM, - ) - defer stop() - - serverErrors := make(chan error, 1) - - go func() { - log.Printf("server is listening on %s", server.Addr) - - if err := server.ListenAndServe(); err != nil && - !errors.Is(err, http.ErrServerClosed) { - serverErrors <- err - } - }() - - select { - case err := <-serverErrors: - log.Printf("server error: %v", err) - - case <-shutdownContext.Done(): - log.Printf("shutdown signal received") - handlers.BeginShutdown() - - shutdownDeadline, cancel := context.WithTimeout( - context.Background(), - 10*time.Second, - ) - defer cancel() - - if err := server.Shutdown(shutdownDeadline); err != nil { - log.Printf("graceful shutdown failed: %v", err) - return - } - - log.Printf("server stopped gracefully") - } } diff --git a/src/backend/pipeline.go b/src/backend/pipeline.go deleted file mode 100644 index b0525607..00000000 --- a/src/backend/pipeline.go +++ /dev/null @@ -1,238 +0,0 @@ -package main - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "github.com/disintegration/imaging" - - "image" - "image/color" - "image/draw" - "image/jpeg" - "io" - "log" - "messenger/src/backend/repositories" - "messenger/src/backend/services" - "os" - "os/signal" - "strconv" - "strings" - "syscall" - "time" -) - -func positiveEnv(name string, fallback int) (int, error) { - s := os.Getenv(name) - if s == "" { - return fallback, nil - } - n, e := strconv.Atoi(s) - if e != nil || n < 1 || n > 3600 { - return 0, errors.New("invalid " + name) - } - return n, nil -} -func runPipeline(mode string) error { - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() - db, err := repositories.OpenDatabase(ctx) - if err != nil { - return err - } - defer db.Close() - store := repositories.NewAttachmentStore(db) - data, err := os.ReadFile(os.Getenv("BROKER_URL_FILE")) - if err != nil { - return errors.New("cannot read broker URL") - } - - address := strings.TrimSpace(string(data)) - if mode == "broker-publish" { - var job repositories.Job - if len(os.Args) != 3 || json.Unmarshal([]byte(os.Args[2]), &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" { - return errors.New("valid internal job JSON required") - } - broker, e := openJobBroker(ctx, address) - if e != nil { - return e - } - defer broker.Close() - publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - return broker.Publish(publishCtx, []byte(os.Args[2])) - } - maxAttempts, err := positiveEnv("JOB_MAX_ATTEMPTS", 5) - if err != nil { - return err - } - seconds, err := positiveEnv("JOB_TIMEOUT_SECONDS", 60) - if err != nil { - return err - } - timeout := time.Duration(seconds) * time.Second - objects, err := repositories.NewObjectStoreFromEnvironment() - if err != nil { - return err - } - for ctx.Err() == nil { - broker, e := openJobBroker(ctx, address) - if e == nil { - if mode == "dispatcher" { - e = dispatchJobs(ctx, store, broker) - } else { - e = consumeJobs(ctx, store, objects, broker, maxAttempts, timeout) - } - broker.Close() - } - if e != nil && ctx.Err() == nil { - log.Print("pipeline: broker session interrupted; reconnecting") - } - select { - case <-ctx.Done(): - case <-time.After(time.Second): - } - } - return nil -} - -func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, broker *jobBroker) error { - for ctx.Err() == nil { - jobs, err := store.QueuedJobs(ctx) - if err != nil { - return err - } - for _, job := range jobs { - payload, err := json.Marshal(job) - if err != nil { - return err - } - publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second) - err = broker.Publish(publishCtx, payload) - cancel() - if err != nil { - return err - } - // Only confirmed publications are marked. Uncertainty causes safe duplicates. - if err = store.MarkPublished(ctx, job.ID); err != nil { - return err - } - } - select { - case <-ctx.Done(): - case <-time.After(time.Second): - } - } - return nil -} - -func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objects services.AttachmentObjects, broker *jobBroker, maxAttempts int, timeout time.Duration) error { - deliveries, err := broker.channel.Consume(jobQueue, "", false, false, false, false, nil) - if err != nil { - return errors.New("RabbitMQ consumer unavailable") - } - for { - select { - case <-ctx.Done(): - return nil - case message, ok := <-deliveries: - if !ok { - return errors.New("RabbitMQ consumer disconnected") - } - var job repositories.Job - if json.Unmarshal(message.Body, &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" { - log.Print("worker: invalid internal job rejected") - if err = message.Reject(false); err != nil { - return err - } - continue - } - task, cancel := context.WithTimeout(ctx, timeout) - claimed, workErr := store.StartAttempt(task, job, timeout) - if workErr == nil && claimed { - workErr = store.ProcessJob(task, job, maxAttempts, func(a repositories.Attachment, staging string) ([]repositories.Variant, string, error) { - return processImage(task, objects, a, staging) - }) - } - cancel() - completed := claimed && workErr == nil - if workErr == nil && !claimed { - completed, workErr = store.JobTerminal(ctx, job.ID) - } - // Ack after COMMIT of ready/failed or durable retry intent, or terminal no-op. - // A closed connection automatically returns unacked deliveries to RabbitMQ. - if completed { - if err = message.Ack(false); err != nil { - return err - } - } else { - if err = message.Nack(false, true); err != nil { - return err - } - select { - case <-ctx.Done(): - case <-time.After(time.Second): - } - } - } - } -} - -func processImage(ctx context.Context, objects services.AttachmentObjects, a repositories.Attachment, staging string) ([]repositories.Variant, string, error) { - reader, err := objects.Get(ctx, staging) - if err != nil { - return nil, "", err - } - data, err := io.ReadAll(io.LimitReader(reader, services.MaxUploadBytes+1)) - reader.Close() - if err != nil { - return nil, "", err - } - sum := sha256.Sum256(data) - if int64(len(data)) != a.SizeBytes || hex.EncodeToString(sum[:]) != a.SHA256 { - return nil, "invalid_image", errors.New("staging checksum mismatch") - } - config, _, err := image.DecodeConfig(bytes.NewReader(data)) - if err != nil || config.Width < 1 || config.Height < 1 || int64(config.Width) > services.MaxImagePixels/int64(config.Height) { - return nil, "invalid_image", errors.New("invalid image dimensions") - } - source, err := imaging.Decode(bytes.NewReader(data), imaging.AutoOrientation(true)) - if err != nil { - return nil, "invalid_image", err - } - w, h := source.Bounds().Dx(), source.Bounds().Dy() - if w > 256 || h > 256 { - if w >= h { - h = max(1, h*256/w) - w = 256 - } else { - w = max(1, w*256/h) - h = 256 - } - } - small := imaging.Resize(source, w, h, imaging.Lanczos) - white := image.NewRGBA(image.Rect(0, 0, w, h)) - draw.Draw(white, white.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src) - draw.Draw(white, white.Bounds(), small, image.Point{}, draw.Over) - var thumbnail bytes.Buffer - if err = jpeg.Encode(&thumbnail, white, &jpeg.Options{Quality: 85}); err != nil { - return nil, "processing_failed", err - } - if err = ctx.Err(); err != nil { - return nil, "", err - } - // Deterministic keys make redelivery repeat the same effects. - if err = objects.Put(ctx, a.ObjectKey, bytes.NewReader(data), a.SizeBytes, a.ContentType); err != nil { - return nil, "", err - } - thumb := thumbnail.Bytes() - key := "variants/" + a.ID + "/v1/thumbnail.jpg" - if err = objects.Put(ctx, key, bytes.NewReader(thumb), int64(len(thumb)), "image/jpeg"); err != nil { - return nil, "", err - } - digest := sha256.Sum256(thumb) - return []repositories.Variant{{Name: "thumbnail", ContentType: "image/jpeg", SizeBytes: int64(len(thumb)), Width: w, Height: h, SHA256: hex.EncodeToString(digest[:])}}, "", nil -} diff --git a/src/backend/broker.go b/src/backend/repositories/broker.go similarity index 79% rename from src/backend/broker.go rename to src/backend/repositories/broker.go index c6135796..f16e6643 100644 --- a/src/backend/broker.go +++ b/src/backend/repositories/broker.go @@ -1,4 +1,4 @@ -package main +package repositories import ( "context" @@ -11,13 +11,13 @@ import ( const jobQueue = "attachments.v1" -type jobBroker struct { +type JobBroker struct { connection *amqp.Connection channel *amqp.Channel returned chan amqp.Return } -func openJobBroker(ctx context.Context, address string) (*jobBroker, error) { +func NewJobBroker(ctx context.Context, address string) (*JobBroker, error) { connection, err := amqp.DialConfig(address, amqp.Config{ Heartbeat: 10 * time.Second, Dial: func(network, address string) (net.Conn, error) { @@ -32,7 +32,7 @@ func openJobBroker(ctx context.Context, address string) (*jobBroker, error) { connection.Close() return nil, errors.New("RabbitMQ channel unavailable") } - b := &jobBroker{connection: connection, channel: channel, returned: make(chan amqp.Return, 1)} + b := &JobBroker{connection: connection, channel: channel, returned: make(chan amqp.Return, 1)} // Queue metadata and message bodies both survive a broker restart. _, err = channel.QueueDeclare(jobQueue, true, false, false, false, nil) if err == nil { @@ -49,9 +49,9 @@ func openJobBroker(ctx context.Context, address string) (*jobBroker, error) { return b, nil } -func (b *jobBroker) Close() { b.channel.Close(); b.connection.Close() } +func (b *JobBroker) Close() { b.channel.Close(); b.connection.Close() } -func (b *jobBroker) Publish(ctx context.Context, body []byte) error { +func (b *JobBroker) Publish(ctx context.Context, body []byte) error { // mandatory catches an unroutable message; confirms alone do not prove routing. confirmation, err := b.channel.PublishWithDeferredConfirmWithContext(ctx, "", jobQueue, true, false, amqp.Publishing{ DeliveryMode: amqp.Persistent, ContentType: "application/json", Body: body, @@ -71,3 +71,7 @@ func (b *jobBroker) Publish(ctx context.Context, body []byte) error { return nil } } + +func (b *JobBroker) Consume() (<-chan amqp.Delivery, error) { + return b.channel.Consume(jobQueue, "", false, false, false, false, nil) +} diff --git a/src/backend/repositories/job_admin.go b/src/backend/repositories/job_admin.go new file mode 100644 index 00000000..e2be2e79 --- /dev/null +++ b/src/backend/repositories/job_admin.go @@ -0,0 +1,142 @@ +package repositories + +import ( + "context" + "errors" + "github.com/jackc/pgx/v5" + "strings" + "time" + "unicode/utf8" +) + +type FailedJob struct { + ID string `json:"job_id"` + AttachmentID string `json:"attachment_id"` + Code string `json:"error_code"` + Attempts int `json:"attempts"` + FailedAt time.Time `json:"failed_at"` +} + +func (s *AttachmentStore) FailedJobs(ctx context.Context) ([]FailedJob, error) { + rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,f.error_code,f.attempts,f.failed_at FROM attachment_job_failures f JOIN attachment_jobs j ON j.id=f.job_id JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed' ORDER BY f.failed_at DESC LIMIT 100`) + if err != nil { + return nil, err + } + defer rows.Close() + result := []FailedJob{} + for rows.Next() { + var j FailedJob + if err = rows.Scan(&j.ID, &j.AttachmentID, &j.Code, &j.Attempts, &j.FailedAt); err != nil { + return nil, err + } + result = append(result, j) + } + return result, rows.Err() +} + +func (s *AttachmentStore) ReplayJob(ctx context.Context, id, reason string, exists func(context.Context, string) error) (bool, error) { + reason = strings.TrimSpace(reason) + if reason == "" || utf8.RuneCountInString(reason) > 200 { + return false, errors.New("replay reason must contain 1-200 characters") + } + tx, err := s.db.Begin(ctx) + if err != nil { + return false, err + } + defer tx.Rollback(context.Background()) + var status, key, attachment string + var deleted bool + err = tx.QueryRow(ctx, `SELECT a.status,j.staging_key,a.id,j.staging_deleted_at IS NOT NULL FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 FOR UPDATE OF j,a`, id).Scan(&status, &key, &attachment, &deleted) + if err != nil { + return false, err + } + if status != "failed" { + return false, nil + } + if deleted { + return false, errors.New("staging has expired; replay unavailable") + } + if key != "staging/jobs/"+attachment && key != "originals/"+attachment { + return false, errors.New("invalid staging key") + } + if err = exists(ctx, key); err != nil { + return false, errors.New("staging unavailable; replay not scheduled") + } + if _, err = tx.Exec(ctx, `INSERT INTO attachment_job_replays(job_id,reason) VALUES($1,$2)`, id, reason); err != nil { + return false, err + } + if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET attempts=0,generation=generation+1,next_attempt_at=now(),lease_until=NULL,lease_token=NULL,published_at=NULL WHERE id=$1`, id); err != nil { + return false, err + } + if _, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=$1`, attachment); err != nil { + return false, err + } + return true, tx.Commit(ctx) +} + +// Replay and cleanup lock the same rows. A ready source is removed only after +// final publication, failed sources expire after retention; queued work is safe. +func (s *AttachmentStore) CleanupJobStaging(ctx context.Context, id string, retention time.Duration, dryRun bool, remove func(context.Context, string) error) (bool, error) { + tx, err := s.db.Begin(ctx) + if err != nil { + return false, err + } + defer tx.Rollback(context.Background()) + var key, attachment, status string + var eligible bool + err = tx.QueryRow(ctx, `SELECT j.staging_key,a.id,a.status,j.staging_deleted_at IS NULL AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND COALESCE(f.failed_at,a.updated_at)=3 OR a.status='failed' AND COALESCE(f.failed_at,a.updated_at)= maxAttempts { + if err = failJob(ctx, tx, j, "retry_exhausted", attempts); err != nil { + return "", err + } + return "", tx.Commit(ctx) + } + token := uuid.NewString() + _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET attempts=attempts+1,lease_token=$2,lease_until=now()+$3*interval '1 second' WHERE id=$1`, j.ID, token, timeout.Seconds()+5) + if err != nil { + return "", err + } + if _, err = tx.Exec(ctx, `UPDATE attachments SET status='processing',updated_at=now() WHERE id=$1`, j.AttachmentID); err != nil { + return "", err + } + if _, err = tx.Exec(ctx, `INSERT INTO attachment_job_attempts(job_id,generation,attempt) VALUES($1,$2,$3)`, j.ID, generation, attempts+1); err != nil { + return "", err + } + return token, tx.Commit(ctx) +} + +var ErrProcessingTimeout = errors.New("processing timeout") + +// ProcessJob holds the row lock across deterministic object writes. The lock and +// token check prevent overlapping/stale attempts from publishing metadata. +func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, maxAttempts int, process func(Attachment, string) ([]Variant, string, error)) error { tx, err := s.db.Begin(ctx) if err != nil { return err } defer tx.Rollback(context.Background()) var staging string - var attempts, version int + var attempts, generation int var a Attachment - err = tx.QueryRow(ctx, `SELECT a.id,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,j.staging_key,j.attempts,j.pipeline_version FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 FOR UPDATE OF j,a SKIP LOCKED`, j.ID, j.AttachmentID).Scan(&a.ID, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, &staging, &attempts, &version) + err = tx.QueryRow(ctx, `SELECT a.id,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,j.staging_key,j.attempts,j.generation FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.lease_token=$4 AND j.lease_until>now() FOR UPDATE OF j,a`, j.ID, j.AttachmentID, j.PipelineVersion, token).Scan(&a.ID, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, &staging, &attempts, &generation) if errors.Is(err, pgx.ErrNoRows) { - return nil + return errors.New("stale job lease") } if err != nil { return err } - if a.Status == "ready" || a.Status == "failed" || version != j.PipelineVersion { - return nil + if a.Status != "processing" { + return errors.New("job is not processing") } - var due bool - if err = tx.QueryRow(ctx, `SELECT next_attempt_at<=now() FROM attachment_jobs WHERE id=$1`, j.ID).Scan(&due); err != nil { - return err - } - if !due { - return nil - } - // Attempts include interrupted executions: StartAttempt commits the counter. - if attempts > maxAttempts { - _, err = tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now() WHERE id=$1`, a.ID) + variants, code, workErr := process(a, staging) + var next *time.Time + var auditCode *string + if workErr == nil { + data, e := json.Marshal(variants) + if e != nil { + return e + } + _, err = tx.Exec(ctx, `UPDATE attachments SET status='ready',variants=$2,error_code=NULL,updated_at=now() WHERE id=$1`, a.ID, data) } else { - variants, code, workErr := process(a, staging) - if workErr == nil { - data, e := json.Marshal(variants) - if e != nil { - return e - } - _, err = tx.Exec(ctx, `UPDATE attachments SET status='ready',variants=$2,error_code=NULL,updated_at=now() WHERE id=$1`, a.ID, data) - } else if code != "" || attempts >= maxAttempts { + reason := "temporary_failure" + if errors.Is(workErr, ErrProcessingTimeout) { + reason = "processing_timeout" + } + if code != "" { + reason = code + } + auditCode = &reason + if code != "" || attempts >= maxAttempts { if code == "" { code = "retry_exhausted" } - _, err = tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code=$2,updated_at=now() WHERE id=$1`, a.ID, code) + err = failJob(ctx, tx, j, code, attempts) } else { + due := time.Now().UTC().Add(RetryDelay(attempts)) + next = &due _, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=$1`, a.ID) } } if err != nil { return err } - _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET lease_until=NULL,published_at=NULL,next_attempt_at=now()+interval '2 seconds' WHERE id=$1`, j.ID) + _, err = tx.Exec(ctx, `UPDATE attachment_job_attempts SET finished_at=clock_timestamp(),error_code=$4,next_attempt_at=$5 WHERE job_id=$1 AND generation=$2 AND attempt=$3`, j.ID, generation, attempts, auditCode, next) + if err != nil { + return err + } + _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET lease_until=NULL,lease_token=NULL,published_at=NULL,next_attempt_at=COALESCE($2,now()) WHERE id=$1`, j.ID, next) if err != nil { return err } return tx.Commit(ctx) } -// StartAttempt publishes processing before the long transaction starts; expired -// leases are recoverable from PostgreSQL even if the broker delivery is pending. -func (s *AttachmentStore) StartAttempt(ctx context.Context, j Job, timeout time.Duration) (bool, error) { - tx, err := s.db.Begin(ctx) +func failJob(ctx context.Context, tx pgx.Tx, j Job, code string, attempts int) error { + if _, err := tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code=$2,updated_at=now() WHERE id=$1`, j.AttachmentID, code); err != nil { + return err + } + if _, err := tx.Exec(ctx, `UPDATE attachment_jobs SET lease_token=NULL,lease_until=NULL WHERE id=$1`, j.ID); err != nil { + return err + } + _, err := tx.Exec(ctx, `INSERT INTO attachment_job_failures(job_id,error_code,attempts) VALUES($1,$2,$3) ON CONFLICT(job_id) DO UPDATE SET error_code=EXCLUDED.error_code,attempts=EXCLUDED.attempts,failed_at=clock_timestamp()`, j.ID, code, attempts) + return err +} + +// Equal jitter: exponential ceiling 2,4,8,...,30 seconds; delay in [ceiling/2,ceiling). +func RetryDelay(attempt int) time.Duration { + ceiling := 2 * time.Second + for i := 1; i < attempt && ceiling < 30*time.Second; i++ { + ceiling *= 2 + } + if ceiling > 30*time.Second { + ceiling = 30 * time.Second + } + n, err := rand.Int(rand.Reader, big.NewInt(int64(ceiling/2))) if err != nil { - return false, err + return ceiling } - defer tx.Rollback(context.Background()) - var id string - err = tx.QueryRow(ctx, `UPDATE attachment_jobs j SET attempts=attempts+1,lease_until=now()+$2*interval '1 second' FROM attachments a WHERE j.id=$1 AND a.id=j.attachment_id AND a.status IN ('queued','processing') AND (j.lease_until IS NULL OR j.lease_until bounds[1] { + t.Fatalf("attempt %d: %v", attempt, delay) + } + seen[delay] = true + } + if len(seen) < 2 { + t.Fatal("jitter produced no variation") + } + } +} diff --git a/src/backend/tests/test_fixture_test.go b/src/backend/tests/test_fixture_test.go index 57ca463e..51f3281a 100644 --- a/src/backend/tests/test_fixture_test.go +++ b/src/backend/tests/test_fixture_test.go @@ -114,6 +114,13 @@ func newTestFixture(t *testing.T) *testFixture { if _, err = db.Exec(ctx, string(ddl)); err != nil { t.Fatal(err) } + ddl, err = os.ReadFile("../repositories/migrations/000008_job_recovery.up.sql") + if err != nil { + t.Fatal(err) + } + if _, err = db.Exec(ctx, string(ddl)); err != nil { + t.Fatal(err) + } t.Cleanup(func() { _ = redisClient.Close() }) if err := redisClient.Ping(ctx).Err(); err != nil { t.Fatal(err) diff --git a/src/backend/workers/attachment.go b/src/backend/workers/attachment.go new file mode 100644 index 00000000..f0a21182 --- /dev/null +++ b/src/backend/workers/attachment.go @@ -0,0 +1,57 @@ +package workers + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "io" + + "messenger/src/backend/internal/faults" + "messenger/src/backend/repositories" + "messenger/src/backend/services" +) + +func processImage(ctx context.Context, objects services.AttachmentObjects, a repositories.Attachment, staging string) ([]repositories.Variant, string, error) { + return publishImage(ctx, objects, a, staging, func(ctx context.Context, data []byte) ([]encodedVariant, string, error) { + return renderInSubprocess(ctx, a.ID, data) + }) +} + +func publishImage(ctx context.Context, objects services.AttachmentObjects, a repositories.Attachment, staging string, render imageRenderer) ([]repositories.Variant, string, error) { + reader, err := objects.Get(ctx, staging) + if err != nil { + return nil, "", err + } + data, err := io.ReadAll(io.LimitReader(reader, services.MaxUploadBytes+1)) + reader.Close() + if err != nil { + return nil, "", err + } + digest := sha256.Sum256(data) + if len(data) > int(services.MaxUploadBytes) || int64(len(data)) != a.SizeBytes || hex.EncodeToString(digest[:]) != a.SHA256 { + return nil, "invalid_image", errors.New("invalid staging checksum or size") + } + variants, code, err := render(ctx, data) + if err != nil { + return nil, code, err + } + if err = objects.Put(ctx, a.ObjectKey, bytes.NewReader(data), a.SizeBytes, a.ContentType); err != nil { + return nil, "", err + } + metadata := []repositories.Variant{} + for _, variant := range variants { + extension := ".jpg" + if variant.Metadata.Name == "optimized" { + extension = ".webp" + } + key := "variants/" + a.ID + "/v1/" + variant.Metadata.Name + extension + if err = objects.Put(ctx, key, bytes.NewReader(variant.Data), int64(len(variant.Data)), variant.Metadata.ContentType); err != nil { + return nil, "", err + } + metadata = append(metadata, variant.Metadata) + } + faults.Crash("after_object_before_commit", a.ID) + return metadata, "", nil +} diff --git a/src/backend/pipeline_test.go b/src/backend/workers/attachment_test.go similarity index 55% rename from src/backend/pipeline_test.go rename to src/backend/workers/attachment_test.go index ff2eedb0..7b49a57a 100644 --- a/src/backend/pipeline_test.go +++ b/src/backend/workers/attachment_test.go @@ -1,4 +1,4 @@ -package main +package workers import ( "bytes" @@ -11,8 +11,12 @@ import ( "image/jpeg" "image/png" "io" - "messenger/src/backend/repositories" + "os/exec" "testing" + + "golang.org/x/image/webp" + + "messenger/src/backend/repositories" ) type imageObjects map[string][]byte @@ -26,6 +30,9 @@ func (o imageObjects) Put(_ context.Context, key string, r io.Reader, _ int64, _ return e } func TestThumbnailContract(t *testing.T) { + if _, err := exec.LookPath("cwebp"); err != nil { + t.Skip("cwebp: use Docker builder tests") + } for _, size := range [][4]int{{320, 200, 256, 160}, {200, 320, 160, 256}, {17, 9, 17, 9}, {1000, 1, 256, 1}} { t.Run(fmt.Sprintf("%dx%d", size[0], size[1]), func(t *testing.T) { var source bytes.Buffer @@ -35,7 +42,7 @@ func TestThumbnailContract(t *testing.T) { digest := sha256.Sum256(source.Bytes()) o := imageObjects{"staging": source.Bytes()} a := repositories.Attachment{ID: "sample", ObjectKey: "originals/sample", SizeBytes: int64(source.Len()), SHA256: hex.EncodeToString(digest[:]), ContentType: "image/png"} - variants, code, err := processImage(context.Background(), o, a, "staging") + variants, code, err := publishImage(context.Background(), o, a, "staging", renderImage) if err != nil { t.Fatalf("%s: %v", code, err) } @@ -43,6 +50,17 @@ func TestThumbnailContract(t *testing.T) { if err != nil { t.Fatal(err) } + if len(variants) != 3 { + t.Fatal("three variants required") + } + crop, err := jpeg.Decode(bytes.NewReader(o["variants/sample/v1/crop.jpg"])) + if err != nil || crop.Bounds().Dx() != 128 || crop.Bounds().Dy() != 128 { + t.Fatal("invalid crop", err) + } + optimized, err := webp.Decode(bytes.NewReader(o["variants/sample/v1/optimized.webp"])) + if err != nil || optimized.Bounds().Dx() != size[0] || optimized.Bounds().Dy() != size[1] { + t.Fatal("invalid optimized", err) + } wantW, wantH := size[2], size[3] if thumb.Bounds().Dx() != wantW || thumb.Bounds().Dy() != wantH || variants[0].Width != wantW || variants[0].Height != wantH { t.Fatalf("wrong dimensions: %v", thumb.Bounds()) @@ -55,10 +73,10 @@ func TestThumbnailContract(t *testing.T) { t.Fatal("original changed") } first := append([]byte{}, o["variants/sample/v1/thumbnail.jpg"]...) - if _, _, err = processImage(context.Background(), o, a, "staging"); err != nil { + if _, _, err = publishImage(context.Background(), o, a, "staging", renderImage); err != nil { t.Fatal(err) } - if len(o) != 3 || !bytes.Equal(first, o["variants/sample/v1/thumbnail.jpg"]) { + if len(o) != 5 || !bytes.Equal(first, o["variants/sample/v1/thumbnail.jpg"]) { t.Fatal("redelivery changed or multiplied effects") } }) @@ -66,6 +84,9 @@ func TestThumbnailContract(t *testing.T) { } func TestThumbnailAppliesEXIFOrientation(t *testing.T) { + if _, err := exec.LookPath("cwebp"); err != nil { + t.Skip("cwebp: use Docker builder tests") + } var source bytes.Buffer if err := jpeg.Encode(&source, image.NewRGBA(image.Rect(0, 0, 320, 200)), nil); err != nil { t.Fatal(err) @@ -79,7 +100,7 @@ func TestThumbnailAppliesEXIFOrientation(t *testing.T) { digest := sha256.Sum256(data) o := imageObjects{"staging": data} a := repositories.Attachment{ID: "rotated", ObjectKey: "originals/rotated", SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), ContentType: "image/jpeg"} - variants, _, err := processImage(context.Background(), o, a, "staging") + variants, _, err := publishImage(context.Background(), o, a, "staging", renderImage) if err != nil { t.Fatal(err) } @@ -90,3 +111,50 @@ func TestThumbnailAppliesEXIFOrientation(t *testing.T) { t.Fatal("original EXIF modified") } } + +func TestOptimizedBoundsAndCenteredCrop(t *testing.T) { + if _, err := exec.LookPath("cwebp"); err != nil { + t.Skip("cwebp: use Docker builder tests") + } + var original bytes.Buffer + if err := png.Encode(&original, image.NewNRGBA(image.Rect(0, 0, 1600, 1000))); err != nil { + t.Fatal(err) + } + variants, _, err := renderImage(context.Background(), original.Bytes()) + if err != nil { + t.Fatal(err) + } + optimized, err := webp.Decode(bytes.NewReader(variants[2].Data)) + if err != nil || optimized.Bounds().Dx() != 1280 || optimized.Bounds().Dy() != 800 { + t.Fatal("optimized must fit 1280x1280", err) + } + stripe := image.NewNRGBA(image.Rect(0, 0, 4, 2)) + for y := 0; y < 2; y++ { + for x := 0; x < 4; x++ { + c := color.NRGBA{G: 255, A: 255} + if x == 0 { + c = color.NRGBA{R: 255, A: 255} + } + if x == 3 { + c = color.NRGBA{B: 255, A: 255} + } + stripe.SetNRGBA(x, y, c) + } + } + original = bytes.Buffer{} + if err = png.Encode(&original, stripe); err != nil { + t.Fatal(err) + } + variants, _, err = renderImage(context.Background(), original.Bytes()) + if err != nil { + t.Fatal(err) + } + crop, err := jpeg.Decode(bytes.NewReader(variants[1].Data)) + if err != nil { + t.Fatal(err) + } + pixel := color.NRGBAModel.Convert(crop.At(64, 64)).(color.NRGBA) + if pixel.G < 240 || pixel.R > 15 || pixel.B > 15 { + t.Fatal("crop did not preserve the green center", pixel) + } +} diff --git a/src/backend/workers/config.go b/src/backend/workers/config.go new file mode 100644 index 00000000..d134e912 --- /dev/null +++ b/src/backend/workers/config.go @@ -0,0 +1,19 @@ +package workers + +import ( + "errors" + "os" + "strconv" +) + +func positiveEnv(name string, fallback int) (int, error) { + s := os.Getenv(name) + if s == "" { + return fallback, nil + } + n, e := strconv.Atoi(s) + if e != nil || n < 1 || n > 3600 { + return 0, errors.New("invalid " + name) + } + return n, nil +} diff --git a/src/backend/workers/image_process.go b/src/backend/workers/image_process.go new file mode 100644 index 00000000..ce5641c3 --- /dev/null +++ b/src/backend/workers/image_process.go @@ -0,0 +1,98 @@ +package workers + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "time" + + "messenger/src/backend/internal/faults" + "messenger/src/backend/repositories" + "messenger/src/backend/services" +) + +type boundedOutput struct{ bytes.Buffer } + +func (b *boundedOutput) Write(data []byte) (int, error) { + if b.Len()+len(data) > 8*1024*1024 { + return 0, errors.New("image result exceeds limit") + } + return b.Buffer.Write(data) +} + +func renderInSubprocess(ctx context.Context, id string, data []byte) ([]encodedVariant, string, error) { + seconds, err := positiveEnv("JOB_TIMEOUT_SECONDS", 60) + if err != nil { + return nil, "", err + } + task, cancel := context.WithTimeout(ctx, time.Duration(seconds)*time.Second) + defer cancel() + executable, err := os.Executable() + if err != nil { + return nil, "", err + } + // The parent owns scratch files, so killing a child cannot leak its PNG files. + directory, err := os.MkdirTemp("", "image-job-*") + if err != nil { + return nil, "", err + } + defer os.RemoveAll(directory) + command := exec.CommandContext(task, executable, "image-process", id) + // No database, broker or S3 credentials are needed by a decoder. + command.Env = []string{"PATH=/usr/bin:/bin", "GOMAXPROCS=1", "GOMEMLIMIT=256MiB", "WORKER_MAX_PIXELS=" + workerPixelLimit(), "PIPELINE_FAULT_DIR=" + os.Getenv("PIPELINE_FAULT_DIR")} + command.Env = append(command.Env, "TMPDIR="+directory) + isolateImageProcess(command) + command.Stdin = bytes.NewReader(data) + var output boundedOutput + command.Stdout = &output + err = command.Run() + if task.Err() != nil { + return nil, "", repositories.ErrProcessingTimeout + } + if err != nil { + return nil, "", errors.New("image child terminated") + } + var result imageResult + if err = json.Unmarshal(output.Bytes(), &result); err != nil { + return nil, "", errors.New("invalid image child output") + } + if result.ErrorCode != "" { + return nil, result.ErrorCode, errors.New("invalid image") + } + if len(result.Variants) != 3 { + return nil, "", errors.New("incomplete image child output") + } + return result.Variants, "", nil +} + +func workerPixelLimit() string { + if value := os.Getenv("WORKER_MAX_PIXELS"); value != "" { + return value + } + return "8000000" +} + +func RunImageChild(args []string) error { + if len(args) != 1 { + return errors.New("image id required") + } + if faults.Take("transform_delay", args[0]) { + time.Sleep(time.Hour) + } + data, err := io.ReadAll(io.LimitReader(os.Stdin, services.MaxUploadBytes+1)) + if err != nil { + return err + } + if len(data) > int(services.MaxUploadBytes) { + return json.NewEncoder(os.Stdout).Encode(imageResult{ErrorCode: "invalid_image"}) + } + variants, code, err := renderImage(context.Background(), data) + if err != nil && code == "" { + return err + } + return json.NewEncoder(os.Stdout).Encode(imageResult{Variants: variants, ErrorCode: code}) +} diff --git a/src/backend/workers/image_process_linux.go b/src/backend/workers/image_process_linux.go new file mode 100644 index 00000000..d318491a --- /dev/null +++ b/src/backend/workers/image_process_linux.go @@ -0,0 +1,14 @@ +package workers + +import ( + "os/exec" + "syscall" + "time" +) + +func isolateImageProcess(command *exec.Cmd) { + command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + // Kill the whole group, including cwebp, on timeout or shutdown. + command.Cancel = func() error { return syscall.Kill(-command.Process.Pid, syscall.SIGKILL) } + command.WaitDelay = time.Second +} diff --git a/src/backend/workers/image_render.go b/src/backend/workers/image_render.go new file mode 100644 index 00000000..7a7569d5 --- /dev/null +++ b/src/backend/workers/image_render.go @@ -0,0 +1,113 @@ +package workers + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + "image/color" + "image/draw" + "image/jpeg" + "image/png" + "os" + "os/exec" + "strconv" + + "github.com/disintegration/imaging" + + "messenger/src/backend/repositories" + "messenger/src/backend/services" +) + +type encodedVariant struct { + Metadata repositories.Variant `json:"metadata"` + Data []byte `json:"data"` +} +type imageResult struct { + Variants []encodedVariant `json:"variants"` + ErrorCode string `json:"error_code"` +} +type imageRenderer func(context.Context, []byte) ([]encodedVariant, string, error) + +func fitSize(w, h, limit int) (int, int) { + if w > limit || h > limit { + if w >= h { + h = max(1, h*limit/w) + w = limit + } else { + w = max(1, w*limit/h) + h = limit + } + } + return w, h +} +func opaque(source image.Image) image.Image { + result := image.NewRGBA(source.Bounds()) + draw.Draw(result, result.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src) + draw.Draw(result, result.Bounds(), source, source.Bounds().Min, draw.Over) + return result +} + +// renderImage runs only in the disposable image child in production. DecodeConfig +// bounds pixels before allocation; the container bounds total memory to 512 MiB. +func renderImage(ctx context.Context, data []byte) ([]encodedVariant, string, error) { + pixels, err := strconv.Atoi(workerPixelLimit()) + if err != nil || pixels < 1 || int64(pixels) > services.MaxImagePixels { + return nil, "", errors.New("invalid WORKER_MAX_PIXELS") + } + config, _, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || config.Width < 1 || config.Height < 1 || int64(config.Width) > int64(pixels)/int64(config.Height) { + return nil, "invalid_image", errors.New("image dimensions exceed worker limit") + } + source, err := imaging.Decode(bytes.NewReader(data), imaging.AutoOrientation(true)) + if err != nil { + return nil, "invalid_image", err + } + result := []encodedVariant{} + appendVariant := func(name, mime string, img image.Image, body []byte) { + digest := sha256.Sum256(body) + result = append(result, encodedVariant{repositories.Variant{Name: name, ContentType: mime, SizeBytes: int64(len(body)), Width: img.Bounds().Dx(), Height: img.Bounds().Dy(), SHA256: hex.EncodeToString(digest[:])}, body}) + } + w, h := fitSize(source.Bounds().Dx(), source.Bounds().Dy(), 256) + thumbnail := opaque(imaging.Resize(source, w, h, imaging.Lanczos)) + var body bytes.Buffer + if err = jpeg.Encode(&body, thumbnail, &jpeg.Options{Quality: 85}); err != nil { + return nil, "", err + } + appendVariant("thumbnail", "image/jpeg", thumbnail, body.Bytes()) + crop := opaque(imaging.Fill(source, 128, 128, imaging.Center, imaging.Lanczos)) + body = bytes.Buffer{} + if err = jpeg.Encode(&body, crop, &jpeg.Options{Quality: 85}); err != nil { + return nil, "", err + } + appendVariant("crop", "image/jpeg", crop, body.Bytes()) + w, h = fitSize(source.Bounds().Dx(), source.Bounds().Dy(), 1280) + optimized := imaging.Resize(source, w, h, imaging.Lanczos) + directory, err := os.MkdirTemp("", "image-process-*") + if err != nil { + return nil, "", err + } + defer os.RemoveAll(directory) + input := directory + "/input.png" + file, err := os.Create(input) + if err != nil { + return nil, "", err + } + err = png.Encode(file, optimized) + closeErr := file.Close() + if err != nil { + return nil, "", err + } + if closeErr != nil { + return nil, "", closeErr + } + command := exec.CommandContext(ctx, "cwebp", "-quiet", "-q", "80", "-m", "4", "-metadata", "none", input, "-o", "-") + webp, err := command.Output() + if err != nil { + return nil, "", errors.New("WebP encoding failed") + } + appendVariant("optimized", "image/webp", optimized, webp) + return result, "", nil +} diff --git a/src/backend/workers/pipeline.go b/src/backend/workers/pipeline.go new file mode 100644 index 00000000..e19bda97 --- /dev/null +++ b/src/backend/workers/pipeline.go @@ -0,0 +1,172 @@ +package workers + +import ( + "context" + "encoding/json" + "errors" + "log" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "messenger/src/backend/internal/faults" + "messenger/src/backend/repositories" + "messenger/src/backend/services" +) + +func RunPipeline(mode string, args []string) error { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + db, err := repositories.OpenDatabase(ctx) + if err != nil { + return err + } + defer db.Close() + store := repositories.NewAttachmentStore(db) + data, err := os.ReadFile(os.Getenv("BROKER_URL_FILE")) + if err != nil { + return errors.New("cannot read broker URL") + } + + address := strings.TrimSpace(string(data)) + if mode == "broker-publish" { + var job repositories.Job + if len(args) != 1 || json.Unmarshal([]byte(args[0]), &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" { + return errors.New("valid internal job JSON required") + } + broker, e := repositories.NewJobBroker(ctx, address) + if e != nil { + return e + } + defer broker.Close() + publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + return broker.Publish(publishCtx, []byte(args[0])) + } + maxAttempts, err := positiveEnv("JOB_MAX_ATTEMPTS", 5) + if err != nil { + return err + } + seconds, err := positiveEnv("JOB_TIMEOUT_SECONDS", 60) + if err != nil { + return err + } + timeout := time.Duration(seconds) * time.Second + objects, err := repositories.NewObjectStoreFromEnvironment() + if err != nil { + return err + } + for ctx.Err() == nil { + broker, e := repositories.NewJobBroker(ctx, address) + if e == nil { + if mode == "dispatcher" { + e = dispatchJobs(ctx, store, broker) + } else { + e = consumeJobs(ctx, store, objects, broker, maxAttempts, timeout) + } + broker.Close() + } + if e != nil && ctx.Err() == nil { + log.Print("pipeline: broker session interrupted; reconnecting") + } + select { + case <-ctx.Done(): + case <-time.After(time.Second): + } + } + return nil +} + +func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, broker *repositories.JobBroker) error { + for ctx.Err() == nil { + jobs, err := store.QueuedJobs(ctx) + if err != nil { + return err + } + for _, job := range jobs { + faults.Crash("after_commit_before_publish", job.AttachmentID) + payload, err := json.Marshal(job) + if err != nil { + return err + } + publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + err = broker.Publish(publishCtx, payload) + cancel() + if err != nil { + return err + } + // Only confirmed publications are marked. Uncertainty causes safe duplicates. + if err = store.MarkPublished(ctx, job.ID); err != nil { + return err + } + } + select { + case <-ctx.Done(): + case <-time.After(time.Second): + } + } + return nil +} + +func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objects services.AttachmentObjects, broker *repositories.JobBroker, maxAttempts int, timeout time.Duration) error { + deliveries, err := broker.Consume() + if err != nil { + return errors.New("RabbitMQ consumer unavailable") + } + for { + select { + case <-ctx.Done(): + return nil + case message, ok := <-deliveries: + if !ok { + return errors.New("RabbitMQ consumer disconnected") + } + var job repositories.Job + if json.Unmarshal(message.Body, &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" { + log.Print("worker: invalid internal job rejected") + if err = message.Reject(false); err != nil { + return err + } + continue + } + task, cancel := context.WithTimeout(ctx, timeout+15*time.Second) + token, workErr := store.StartAttempt(task, job, timeout+15*time.Second, maxAttempts) + if workErr == nil && token != "" { + workErr = store.ProcessJob(task, job, token, maxAttempts, func(a repositories.Attachment, staging string) ([]repositories.Variant, string, error) { + return processImage(task, objects, a, staging) + }) + } + cancel() + completed := token != "" && workErr == nil + if workErr == nil && token == "" { + completed, workErr = store.JobTerminal(ctx, job.ID) + } + // Ack after COMMIT of ready/failed or durable retry intent, or terminal no-op. + // A closed connection automatically returns unacked deliveries to RabbitMQ. + if completed { + faults.Crash("after_commit_before_ack", job.AttachmentID) + if objectStore, ok := objects.(*repositories.ObjectStore); ok { + cleanupCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + _, cleanupErr := store.CleanupJobStaging(cleanupCtx, job.ID, 24*time.Hour, false, objectStore.Delete) + cancel() + if cleanupErr != nil { + log.Print("worker: staging cleanup deferred") + } + } + if err = message.Ack(false); err != nil { + return err + } + } else { + if err = message.Nack(false, true); err != nil { + return err + } + select { + case <-ctx.Done(): + case <-time.After(time.Second): + } + } + } + } +}