Реализовать уровень 4 и упорядочить структуру backend

Добавить crop и WebP, изолированную обработку с таймаутом, lease token, backoff с jitter, реестр failed, безопасный replay и очистку staging. Добавить проверки crash-окон и восстановления. Разделить запуск API, CLI-команды, фоновые процессы и инфраструктуру по пакетам.
This commit is contained in:
Dima
2026-11-04 20:14:49 +03:00
parent 495ad58cb7
commit 4bee6a5d4a
26 changed files with 1478 additions and 484 deletions
+3
View File
@@ -114,6 +114,7 @@ services:
BROKER_URL_FILE: /run/secrets/broker_url
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
WORKER_MAX_PIXELS: ${WORKER_MAX_PIXELS:-8000000}
UPLOAD_TMP_DIR: /tmp/uploads
UPLOAD_CLEANUP_GRACE_SECONDS: ${UPLOAD_CLEANUP_GRACE_SECONDS:-600}
# Go 1.22 sees host CPUs; keep parallelism suitable for this 0.5 CPU limit.
@@ -177,6 +178,8 @@ services:
start_period: 20s
retries: 30
worker:
tmpfs:
- /tmp:rw,noexec,nosuid,size=64m,mode=1777
build: ./src
restart: unless-stopped
cpus: 1.0
+13
View File
@@ -0,0 +1,13 @@
# Operator-only fault injection. No client API can create these markers.
services:
dispatcher:
environment:
PIPELINE_FAULT_DIR: /run/pipeline-faults
volumes:
- ./.memory/grade4-faults:/run/pipeline-faults
worker:
environment:
PIPELINE_FAULT_DIR: /run/pipeline-faults
JOB_TIMEOUT_SECONDS: "8"
volumes:
- ./.memory/grade4-faults:/run/pipeline-faults
+213
View File
@@ -0,0 +1,213 @@
#!/usr/bin/env python3
"""Grade 4: real fault injection on synthetic jobs; restores the base stack."""
import datetime as dt
import hashlib
import hmac
import json
import os
from pathlib import Path
import struct
import subprocess
import sys
import time
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
import zlib
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, check
from image_probe import jpeg_size
FAULTS = ROOT / '.memory/grade4-faults'
PREFIX = ['docker', 'compose', '--project-name', 'highload-lab5', '-f', 'compose.yml', '-f', 'infra/compose.faults.yml']
def compose(*args):
return subprocess.check_output(PREFIX + list(args), cwd=ROOT, text=True, stderr=subprocess.DEVNULL).strip()
def sql(query):
return compose('exec', '-T', 'db', 'sh', '-c', 'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"', 'sh', query)
def until(fn, seconds=120):
deadline = time.monotonic()+seconds
last = None
while time.monotonic()<deadline:
try:
value = fn()
if value:
return value
except (Failure, OSError, subprocess.CalledProcessError) as exc:
last = type(exc).__name__
time.sleep(.3)
raise AssertionError('condition timed out; last exception: '+str(last))
def job_id(att):
return sql("SELECT id FROM attachment_jobs WHERE attachment_id='"+att['id']+"'")
def admin(*args):
return json.loads(compose('exec', '-T', 'dispatcher', './messenger', 'jobs', *args))
def arm(point, att):
marker = FAULTS/(point+'.'+att['id'])
marker.write_text('operator-only synthetic fault\n')
return Path(str(marker)+'.hit')
def s3(method, key='', query=None):
# Sign with the existing restricted API identity, never the MinIO root user.
values = dict(line.split('=',1) for line in (ROOT/'.env').read_text().splitlines() if '=' in line and not line.startswith('#'))
bucket = values.get('S3_BUCKET', 'messenger-attachments')
access = (ROOT/'secrets/s3_api_access_key_id').read_text().strip()
secret = (ROOT/'secrets/s3_api_secret_access_key').read_text().strip()
host = '127.0.0.1:9000'
uri = '/'+bucket+'/'+key
query_string = urllib.parse.urlencode(sorted((query or {}).items()), quote_via=urllib.parse.quote)
now = dt.datetime.now(dt.timezone.utc)
timestamp, day = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d')
payload = hashlib.sha256(b'').hexdigest()
headers = 'host:'+host+'\nx-amz-content-sha256:'+payload+'\nx-amz-date:'+timestamp+'\n'
signed = 'host;x-amz-content-sha256;x-amz-date'
canonical = '\n'.join([method,uri,query_string,headers,signed,payload])
scope = day+'/us-east-1/s3/aws4_request'
message = 'AWS4-HMAC-SHA256\n'+timestamp+'\n'+scope+'\n'+hashlib.sha256(canonical.encode()).hexdigest()
signing = ('AWS4'+secret).encode()
for part in (day,'us-east-1','s3','aws4_request'):
signing = hmac.new(signing,part.encode(),hashlib.sha256).digest()
signature = hmac.new(signing,message.encode(),hashlib.sha256).hexdigest()
request = urllib.request.Request('http://'+host+uri+('?' + query_string if query_string else ''),method=method,headers={
'x-amz-date':timestamp,'x-amz-content-sha256':payload,
'Authorization':'AWS4-HMAC-SHA256 Credential='+access+'/'+scope+', SignedHeaders='+signed+', Signature='+signature})
try:
response=urllib.request.urlopen(request,timeout=10)
except urllib.error.HTTPError as exc:
response=exc
with response:
return response.code, response.read()
def webp_size(data):
check(data[:4]==b'RIFF' and data[8:12]==b'WEBP','invalid WebP container')
offset=12
while offset+8<=len(data):
kind=data[offset:offset+4];length=int.from_bytes(data[offset+4:offset+8],'little');body=data[offset+8:offset+8+length]
check(kind not in (b'EXIF',b'XMP ',b'ICCP'),'optimized contains metadata')
if kind==b'VP8 ':
dimensions=(int.from_bytes(body[6:8],'little')&16383,int.from_bytes(body[8:10],'little')&16383)
if kind==b'VP8L':
bits=int.from_bytes(body[1:5],'little');dimensions=((bits&16383)+1,((bits>>14)&16383)+1)
if kind==b'VP8X':
dimensions=(int.from_bytes(body[4:7],'little')+1,int.from_bytes(body[7:10],'little')+1)
offset+=8+length+(length%2)
return dimensions
def large_png(w,h):
def chunk(kind,data):
return struct.pack('>I',len(data))+kind+data+struct.pack('>I',zlib.crc32(kind+data)&0xffffffff)
encoder=zlib.compressobj();pieces=[];row=b'\0'+b'\x80\x80\x80\xff'*w
for _ in range(h):
pieces.append(encoder.compress(row))
pieces.append(encoder.flush())
return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',w,h,8,6,0,0,0))+chunk(b'IDAT',b''.join(pieces))+chunk(b'IEND',b'')
def main():
FAULTS.mkdir(parents=True, exist_ok=True);os.chmod(FAULTS,0o777)
compose('up','-d','--wait')
c=Client(sys.argv[1] if len(sys.argv)>1 else 'http://localhost:8085')
alice,bob,eve,chat=c.bootstrap()
results={}
def metadata(att):
return c.call('GET','/api/v1/attachments/'+att['id'],actor=alice,schema='Attachment')['json']
def ready(att):
return until(lambda: (value:=metadata(att))['status']=='ready' and value)
def verify(att, original):
value=ready(att)
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==original,'original changed')
check({v['name'] for v in value['variants']}=={'thumbnail','crop','optimized'},'variant set incomplete')
for variant in value['variants']:
path='/api/v1/attachments/'+att['id']+'/content?variant='+variant['name']
response=c.call('GET',path,actor=bob)
check(response['headers'].get_content_type()==variant['content_type'],'wrong variant MIME')
check(hashlib.sha256(response['bytes']).hexdigest()==variant['sha256'] and len(response['bytes'])==variant['size_bytes'],'wrong variant checksum')
dimensions=webp_size(response['bytes']) if variant['name']=='optimized' else jpeg_size(response['bytes'])
expected={'thumbnail':(256,160),'crop':(128,128),'optimized':(320,200)}[variant['name']]
check(dimensions==expected==(variant['width'],variant['height']),'actual variant dimensions differ')
c.call('GET',path,404,actor=eve)
until(lambda: s3('HEAD','staging/jobs/'+att['id'])[0]==404)
code,body=s3('GET',query={'list-type':'2','prefix':'variants/'+att['id']+'/'})
check(code==200 and len(ET.fromstring(body).findall('{*}Contents'))==3,'duplicate object set')
return value
try:
compose('stop','dispatcher')
att,original=c.attachment(alice,chat)
marker=arm('after_commit_before_publish',att)
check(sql("SELECT published_at IS NULL FROM attachment_jobs WHERE id='"+job_id(att)+"'")=='t','job already published')
compose('start','dispatcher');until(marker.exists)
verify(att,original)
results['commit_publish_crash']={'attachment_id':att['id'],'ready':True,'same_job_count':int(sql("SELECT count(*) FROM attachment_jobs WHERE attachment_id='"+att['id']+"'"))}
print('PASS: crash between COMMIT and publish recovers durable intent.',flush=True)
for point in ('after_object_before_commit','after_commit_before_ack'):
compose('stop','worker');att,original=c.attachment(alice,chat);marker=arm(point,att)
compose('start','worker');until(marker.exists)
value=verify(att,original)
attempts=int(sql("SELECT attempts FROM attachment_jobs WHERE id='"+job_id(att)+"'"))
check(attempts==(2 if point=='after_object_before_commit' else 1),'unexpected attempt count after crash')
results[point]={'attachment_id':att['id'],'attempts':attempts,'variant_count':len(value['variants'])}
print('PASS: '+point+' recovers without duplicate effects.',flush=True)
compose('stop','worker');att,original=c.attachment(alice,chat);marker=arm('transform_delay',att)
compose('start','worker')
restarts_before=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}}','highload-lab5-worker-1'],text=True)
started=time.monotonic();until(marker.exists);verify(att,original)
elapsed=time.monotonic()-started
check(subprocess.check_output(['docker','inspect','--format','{{.RestartCount}}','highload-lab5-worker-1'],text=True)==restarts_before,'worker restarted during child timeout')
check(elapsed<30,'child timeout did not bound processing')
check(compose('exec','-T','worker','sh','-c',"find /tmp -maxdepth 1 -name 'image-job-*' -print")=="",'child scratch leaked')
reasons=sql("SELECT error_code FROM attachment_job_attempts WHERE job_id='"+job_id(att)+"' AND error_code IS NOT NULL")
check('processing_timeout' in reasons,'child timeout was not recorded')
results['child_timeout']={'attachment_id':att['id'],'audit_reason':'processing_timeout','elapsed_seconds':round(elapsed,3),'worker_survived':True,'scratch_removed':True}
print('PASS: stuck image child is killed and retry succeeds.',flush=True)
compose('stop','worker');att,original=c.attachment(alice,chat);job=job_id(att)
sql("UPDATE attachment_jobs SET staging_key='staging/jobs/missing-test-object' WHERE id='"+job+"'")
compose('start','worker')
until(lambda: metadata(att)['status']=='failed',180)
failures=admin('list-failed');check(any(item['job_id']==job and item['attempts']==5 for item in failures),'failed registry missing job')
timing=json.loads(sql("SELECT json_agg(json_build_object('attempt',attempt,'delay',EXTRACT(EPOCH FROM next_attempt_at-finished_at))) FROM attachment_job_attempts WHERE job_id='"+job+"' AND next_attempt_at IS NOT NULL"))
for item in timing:
ceiling=min(30,2**item['attempt']);check(ceiling/2-.15<=float(item['delay'])<=ceiling+.15,'backoff not in jitter window')
sql("UPDATE attachment_jobs SET staging_key='staging/jobs/"+att['id']+"' WHERE id='"+job+"'")
check(s3('HEAD','staging/jobs/'+att['id'])[0]==200,'failed staging was removed before retention')
check(admin('replay',job,'Synthetic storage failure resolved')['changed'],'failed replay not accepted')
verify(att,original)
check(not admin('replay',job,'Already successful job')['changed'],'ready replay reopened job')
check(sql("SELECT count(*) FROM attachment_job_replays WHERE job_id='"+job+"'")=='1','replay not audited exactly once')
results['retry_and_replay']={'attachment_id':att['id'],'attempts_before_replay':5,'retry_delays':timing,'audited_replays':1}
print('PASS: bounded exponential jitter, failed registry, audited safe replay.',flush=True)
data=large_png(3000,3000)
att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=large.png',202,actor=alice,raw=data,headers={'Content-Type':'image/png'},schema='Attachment')['json']
until(lambda: metadata(att)['status']=='failed')
check(metadata(att)['error_code']=='invalid_image','worker pixel limit not enforced')
check(sql("SELECT attempts FROM attachment_jobs WHERE id='"+job_id(att)+"'")=='1','permanent error retried')
check(s3('HEAD','staging/jobs/'+att['id'])[0]==200,'failed source removed early')
sql("UPDATE attachment_job_failures SET failed_at=now()-interval '25 hours' WHERE job_id='"+job_id(att)+"'")
admin('cleanup-staging',job_id(att))
check(s3('HEAD','staging/jobs/'+att['id'])[0]==404,'expired failed source not removed')
results['permanent_error_retention']={'attachment_id':att['id'],'pixels':9000000,'attempts':1,'expired_staging_removed':True}
compose('stop','worker');queued,_=c.attachment(alice,chat)
check(not admin('cleanup-staging',job_id(queued))['changed'],'cleanup deleted queued work')
check(s3('HEAD','staging/jobs/'+queued['id'])[0]==200,'queued source missing')
compose('start','worker');ready(queued)
config=json.loads(subprocess.check_output(['docker','inspect','highload-lab5-worker-1'],text=True))[0]
check(config['HostConfig']['Memory']==512*1024*1024,'worker hard memory limit absent')
check('size=64m' in config['HostConfig']['Tmpfs']['/tmp'],'temporary disk not bounded')
results['resources']={'memory_limit_bytes':config['HostConfig']['Memory'],'prefetch':1,'image_timeout_seconds':8,'worker_pixel_limit':8000000,'queued_staging_protected':True}
(ROOT/'.memory/grade4-results.json').write_text(json.dumps(results,ensure_ascii=False,indent=2)+'\n')
print('PASS: grade 4 variants, fault windows, timeout, retry/replay, retention and resource limits.',flush=True)
finally:
# Remove test-only mounts and restore regular timeout/configuration.
subprocess.run(['docker','compose','--project-name','highload-lab5','up','-d','--wait'],cwd=ROOT,stdout=subprocess.DEVNULL,check=True)
if __name__=='__main__':
main()
+4 -1
View File
@@ -1,5 +1,7 @@
FROM golang:1.22.2-bookworm AS builder
RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/*
WORKDIR /src
COPY go.mod go.sum ./
@@ -19,7 +21,8 @@ RUN CGO_ENABLED=0 \
FROM debian:12.15-slim
RUN groupadd --system app \
RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/* \
&& groupadd --system app \
&& useradd --system --gid app --no-create-home app
WORKDIR /app
+140
View File
@@ -0,0 +1,140 @@
package app
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"net"
"net/http"
"os"
"os/signal"
"runtime"
"syscall"
"time"
"messenger/src/backend/handlers"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
)
func serverAddress() string {
host := os.Getenv("APP_HOST")
if host == "" {
host = "0.0.0.0"
}
port := os.Getenv("APP_PORT")
if port == "" {
port = "8080"
}
return net.JoinHostPort(host, port)
}
func RunHTTP() error {
if _, err := repositories.UploadCleanupGrace(); err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
db, err := repositories.OpenDatabase(ctx)
cancel()
if err != nil {
return fmt.Errorf("database initialization failed: %w", err)
}
defer db.Close()
// Opt-in experiment telemetry goes to container logs, never a public port.
if os.Getenv("BENCH_POOL_STATS") == "1" {
go func() {
for range time.Tick(time.Second) {
s := db.Stat()
data, _ := json.Marshal(map[string]any{
"time": time.Now().UTC(), "acquired": s.AcquiredConns(),
"total": s.TotalConns(), "max": s.MaxConns(),
"gomaxprocs": runtime.GOMAXPROCS(0), "goroutines": runtime.NumGoroutine(),
"acquire_count": s.AcquireCount(), "acquire_duration_ns": s.AcquireDuration().Nanoseconds(),
"empty_acquire_count": s.EmptyAcquireCount(), "canceled_acquire_count": s.CanceledAcquireCount(),
})
log.Printf("benchmark_pool %s", data)
}
}()
}
redisClient, err := repositories.RedisClientFromEnvironment()
if err != nil {
return err
}
defer redisClient.Close()
sessionStore := repositories.NewSessionStore(redisClient, "messenger:session:")
userStore := repositories.NewUserStore(db)
chatStore := repositories.NewChatStore(db)
idempotencyTTL, err := repositories.IdempotencyTTL()
if err != nil {
return err
}
messageStore := repositories.NewMessageStoreWithTTL(db, idempotencyTTL)
ttl, err := services.SessionTTL()
if err != nil {
return err
}
userService := services.NewUserService(userStore)
sessionService := services.NewSessionService(userService, sessionStore, ttl)
chatService := services.NewChatService(chatStore)
messageService := services.NewMessageService(messageStore)
objects, err := repositories.NewObjectStoreFromEnvironment()
if err != nil {
return err
}
attachmentService := services.NewAttachmentService(repositories.NewAttachmentStore(db), objects)
mux := handlers.NewRouter(services.NewHealthService(repositories.NewHealthStore(db), sessionStore, objects), userService, sessionService, chatService, messageService, attachmentService)
server := &http.Server{
Addr: serverAddress(),
Handler: mux,
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
IdleTimeout: 60 * time.Second,
}
shutdownContext, stop := signal.NotifyContext(
context.Background(),
os.Interrupt,
syscall.SIGTERM,
)
defer stop()
serverErrors := make(chan error, 1)
go func() {
log.Printf("server is listening on %s", server.Addr)
if err := server.ListenAndServe(); err != nil &&
!errors.Is(err, http.ErrServerClosed) {
serverErrors <- err
}
}()
select {
case err := <-serverErrors:
log.Printf("server error: %v", err)
case <-shutdownContext.Done():
log.Printf("shutdown signal received")
handlers.BeginShutdown()
shutdownDeadline, cancel := context.WithTimeout(
context.Background(),
10*time.Second,
)
defer cancel()
if err := server.Shutdown(shutdownDeadline); err != nil {
log.Printf("graceful shutdown failed: %v", err)
return nil
}
log.Printf("server stopped gracefully")
}
return nil
}
@@ -1,15 +1,16 @@
package main
package commands
import (
"context"
"encoding/json"
"flag"
"log"
"messenger/src/backend/repositories"
"os"
"os/signal"
"syscall"
"time"
"messenger/src/backend/repositories"
)
func runUploadCleanup(args []string) error {
@@ -37,17 +38,21 @@ func runUploadCleanup(args []string) error {
pass, cancel := context.WithTimeout(ctx, 30*time.Second)
result, err := store.CleanupUploads(pass, *dryRun, objects.Delete)
direct, directErr := store.CleanupDirectUploads(pass, *dryRun, objects.Delete)
jobs, jobsErr := store.CleanupJobs(pass, 24*time.Hour, *dryRun, objects.Delete)
cancel()
if encodeErr := json.NewEncoder(os.Stdout).Encode(map[string]any{"originals": result, "direct": direct}); encodeErr != nil {
if encodeErr := json.NewEncoder(os.Stdout).Encode(map[string]any{"originals": result, "direct": direct, "job_staging": jobs}); encodeErr != nil {
return encodeErr
}
if !*loop {
if err == nil {
err = directErr
if err == nil {
err = jobsErr
}
}
return err
}
if err != nil || directErr != nil {
if err != nil || directErr != nil || jobsErr != nil {
log.Print("upload cleanup pass failed; journal retained for retry")
}
select {
+63
View File
@@ -0,0 +1,63 @@
package commands
import (
"context"
"encoding/json"
"errors"
"os"
"time"
"github.com/google/uuid"
"messenger/src/backend/repositories"
)
func runJobAdmin(args []string) error {
if len(args) < 1 {
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID")
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
db, err := repositories.OpenDatabase(ctx)
if err != nil {
return errors.New("database unavailable")
}
defer db.Close()
store := repositories.NewAttachmentStore(db)
if args[0] == "list-failed" && len(args) == 1 {
jobs, e := store.FailedJobs(ctx)
if e != nil {
return errors.New("cannot read failed registry")
}
return json.NewEncoder(os.Stdout).Encode(jobs)
}
if len(args) < 2 {
return errors.New("job id required")
}
if _, err = uuid.Parse(args[1]); err != nil {
return errors.New("invalid job id")
}
objects, err := repositories.NewObjectStoreFromEnvironment()
if err != nil {
return err
}
var changed bool
switch {
case args[0] == "replay" && len(args) == 3:
changed, err = store.ReplayJob(ctx, args[1], args[2], func(ctx context.Context, key string) error {
r, e := objects.Get(ctx, key)
if e == nil {
r.Close()
}
return e
})
case args[0] == "cleanup-staging" && len(args) == 2:
changed, err = store.CleanupJobStaging(ctx, args[1], 24*time.Hour, false, objects.Delete)
default:
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID")
}
if err != nil {
return err
}
return json.NewEncoder(os.Stdout).Encode(map[string]bool{"changed": changed})
}
+79
View File
@@ -0,0 +1,79 @@
package commands
import (
"context"
"errors"
"log"
"net"
"net/http"
"os"
"time"
"messenger/src/backend/app"
"messenger/src/backend/repositories"
"messenger/src/backend/workers"
)
// Run dispatches the existing executable modes; the binary and CLI stay the same.
func Run(args []string) error {
if len(args) == 0 {
if environment := os.Getenv("APP_ENV"); environment != "" {
log.SetPrefix("[" + environment + "] ")
}
return app.RunHTTP()
}
switch args[0] {
case "image-process":
return workers.RunImageChild(args[1:])
case "jobs":
return runJobAdmin(args[1:])
case "worker", "dispatcher", "broker-publish":
return workers.RunPipeline(args[0], args[1:])
}
if environment := os.Getenv("APP_ENV"); environment != "" {
log.SetPrefix("[" + environment + "] ")
}
switch {
case args[0] == "s3-bootstrap" && len(args) == 1:
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := repositories.BootstrapS3(ctx); err != nil {
return err
}
log.Print("private S3 bucket ready")
return nil
case args[0] == "attachments-cleanup":
return runUploadCleanup(args[1:])
case args[0] == "healthcheck" && len(args) == 1:
return healthcheck()
case args[0] == "migrate" && len(args) == 2 && (args[1] == "up" || args[1] == "down"):
url, err := repositories.DatabaseURL()
if err != nil {
return err
}
if err = repositories.RunMigrations(url, args[1]); err != nil {
return err
}
log.Printf("migration %s complete", args[1])
return nil
default:
return errors.New("usage: messenger migrate up|down")
}
}
func healthcheck() error {
client := http.Client{Timeout: 2 * time.Second}
port := os.Getenv("APP_PORT")
if port == "" {
port = "8080"
}
response, err := client.Get("http://" + net.JoinHostPort("127.0.0.1", port) + "/health/ready")
if err != nil {
return err
}
response.Body.Close()
if response.StatusCode != http.StatusOK {
return errors.New("API is not ready")
}
return nil
}
+8 -4
View File
@@ -116,12 +116,16 @@ func attachmentHandler(attachments *services.AttachmentService) http.HandlerFunc
writeError(w, 400, "invalid_request", "invalid variant", requestID)
return
}
if variants[0] == "thumbnail" {
if variants[0] == "thumbnail" || variants[0] == "crop" || variants[0] == "optimized" {
found := false
for _, raw := range a.Variants {
if v, ok := raw.(map[string]any); ok && v["name"] == "thumbnail" {
a.ObjectKey = "variants/" + a.ID + "/v1/thumbnail.jpg"
a.ContentType = "image/jpeg"
if v, ok := raw.(map[string]any); ok && v["name"] == variants[0] {
extension := ".jpg"
if variants[0] == "optimized" {
extension = ".webp"
}
a.ObjectKey = "variants/" + a.ID + "/v1/" + variants[0] + extension
a.ContentType = v["content_type"].(string)
a.SizeBytes = int64(v["size_bytes"].(float64))
found = true
break
+29
View File
@@ -0,0 +1,29 @@
package faults
import (
"log"
"os"
"path/filepath"
"github.com/google/uuid"
)
// Only an operator-mounted directory enables faults; clients cannot set them.
// Rename makes each injected crash one-shot across automatic container restarts.
func Take(point, id string) bool {
directory := os.Getenv("PIPELINE_FAULT_DIR")
if directory == "" {
return false
}
if _, err := uuid.Parse(id); err != nil {
return false
}
name := filepath.Join(directory, point+"."+id)
return os.Rename(name, name+".hit") == nil
}
func Crash(point, id string) {
if Take(point, id) {
log.Printf("injected pipeline crash: %s, attachment %s", point, id)
os.Exit(86)
}
}
+5 -179
View File
@@ -1,191 +1,17 @@
package main
import (
"context"
"encoding/json"
"errors"
"log"
"messenger/src/backend/handlers"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
"net"
"net/http"
"os"
"os/signal"
"runtime"
"syscall"
"time"
"messenger/src/backend/commands"
)
func serverAddress() string {
host := os.Getenv("APP_HOST")
if host == "" {
host = "0.0.0.0"
}
port := os.Getenv("APP_PORT")
if port == "" {
port = "8080"
}
return net.JoinHostPort(host, port)
}
func main() {
if len(os.Args) >= 2 && (os.Args[1] == "worker" || os.Args[1] == "dispatcher" || os.Args[1] == "broker-publish") {
if err := runPipeline(os.Args[1]); err != nil {
log.Fatal("pipeline initialization failed: ", err)
if err := commands.Run(os.Args[1:]); err != nil {
if len(os.Args) > 1 && (os.Args[1] == "image-process" || os.Args[1] == "healthcheck") {
os.Exit(1)
}
return
}
if environment := os.Getenv("APP_ENV"); environment != "" {
log.SetPrefix("[" + environment + "] ")
}
if len(os.Args) == 2 && os.Args[1] == "s3-bootstrap" {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if err := repositories.BootstrapS3(ctx); err != nil {
log.Fatal("S3 bootstrap failed: ", err)
}
log.Print("private S3 bucket ready")
return
}
if len(os.Args) > 1 && os.Args[1] == "attachments-cleanup" {
if err := runUploadCleanup(os.Args[2:]); err != nil {
log.Fatal("upload cleanup failed: ", err)
}
return
}
if len(os.Args) > 1 {
if len(os.Args) == 2 && os.Args[1] == "healthcheck" {
client := http.Client{Timeout: 2 * time.Second}
port := os.Getenv("APP_PORT")
if port == "" {
port = "8080"
}
response, err := client.Get("http://" + net.JoinHostPort("127.0.0.1", port) + "/health/ready")
if err != nil {
os.Exit(1)
}
response.Body.Close()
if response.StatusCode != http.StatusOK {
os.Exit(1)
}
return
}
if len(os.Args) != 3 || os.Args[1] != "migrate" || (os.Args[2] != "up" && os.Args[2] != "down") {
log.Fatal("usage: messenger migrate up|down")
}
url, err := repositories.DatabaseURL()
if err != nil {
log.Fatal(err)
}
if err = repositories.RunMigrations(url, os.Args[2]); err != nil {
log.Fatal(err)
}
log.Printf("migration %s complete", os.Args[2])
return
}
if _, err := repositories.UploadCleanupGrace(); err != nil {
log.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
db, err := repositories.OpenDatabase(ctx)
cancel()
if err != nil {
log.Fatalf("database initialization failed: %v", err)
}
defer db.Close()
// Opt-in experiment telemetry goes to container logs, never a public port.
if os.Getenv("BENCH_POOL_STATS") == "1" {
go func() {
for range time.Tick(time.Second) {
s := db.Stat()
data, _ := json.Marshal(map[string]any{
"time": time.Now().UTC(), "acquired": s.AcquiredConns(),
"total": s.TotalConns(), "max": s.MaxConns(),
"gomaxprocs": runtime.GOMAXPROCS(0), "goroutines": runtime.NumGoroutine(),
"acquire_count": s.AcquireCount(), "acquire_duration_ns": s.AcquireDuration().Nanoseconds(),
"empty_acquire_count": s.EmptyAcquireCount(), "canceled_acquire_count": s.CanceledAcquireCount(),
})
log.Printf("benchmark_pool %s", data)
}
}()
}
redisClient, err := repositories.RedisClientFromEnvironment()
if err != nil {
log.Fatal(err)
}
defer redisClient.Close()
sessionStore := repositories.NewSessionStore(redisClient, "messenger:session:")
userStore := repositories.NewUserStore(db)
chatStore := repositories.NewChatStore(db)
idempotencyTTL, err := repositories.IdempotencyTTL()
if err != nil {
log.Fatal(err)
}
messageStore := repositories.NewMessageStoreWithTTL(db, idempotencyTTL)
ttl, err := services.SessionTTL()
if err != nil {
log.Fatal(err)
}
userService := services.NewUserService(userStore)
sessionService := services.NewSessionService(userService, sessionStore, ttl)
chatService := services.NewChatService(chatStore)
messageService := services.NewMessageService(messageStore)
objects, err := repositories.NewObjectStoreFromEnvironment()
if err != nil {
log.Fatal(err)
}
attachmentService := services.NewAttachmentService(repositories.NewAttachmentStore(db), objects)
mux := handlers.NewRouter(services.NewHealthService(repositories.NewHealthStore(db), sessionStore, objects), userService, sessionService, chatService, messageService, attachmentService)
server := &http.Server{
Addr: serverAddress(),
Handler: mux,
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
IdleTimeout: 60 * time.Second,
}
shutdownContext, stop := signal.NotifyContext(
context.Background(),
os.Interrupt,
syscall.SIGTERM,
)
defer stop()
serverErrors := make(chan error, 1)
go func() {
log.Printf("server is listening on %s", server.Addr)
if err := server.ListenAndServe(); err != nil &&
!errors.Is(err, http.ErrServerClosed) {
serverErrors <- err
}
}()
select {
case err := <-serverErrors:
log.Printf("server error: %v", err)
case <-shutdownContext.Done():
log.Printf("shutdown signal received")
handlers.BeginShutdown()
shutdownDeadline, cancel := context.WithTimeout(
context.Background(),
10*time.Second,
)
defer cancel()
if err := server.Shutdown(shutdownDeadline); err != nil {
log.Printf("graceful shutdown failed: %v", err)
return
}
log.Printf("server stopped gracefully")
}
}
-238
View File
@@ -1,238 +0,0 @@
package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"github.com/disintegration/imaging"
"image"
"image/color"
"image/draw"
"image/jpeg"
"io"
"log"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"
)
func positiveEnv(name string, fallback int) (int, error) {
s := os.Getenv(name)
if s == "" {
return fallback, nil
}
n, e := strconv.Atoi(s)
if e != nil || n < 1 || n > 3600 {
return 0, errors.New("invalid " + name)
}
return n, nil
}
func runPipeline(mode string) error {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
db, err := repositories.OpenDatabase(ctx)
if err != nil {
return err
}
defer db.Close()
store := repositories.NewAttachmentStore(db)
data, err := os.ReadFile(os.Getenv("BROKER_URL_FILE"))
if err != nil {
return errors.New("cannot read broker URL")
}
address := strings.TrimSpace(string(data))
if mode == "broker-publish" {
var job repositories.Job
if len(os.Args) != 3 || json.Unmarshal([]byte(os.Args[2]), &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" {
return errors.New("valid internal job JSON required")
}
broker, e := openJobBroker(ctx, address)
if e != nil {
return e
}
defer broker.Close()
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
return broker.Publish(publishCtx, []byte(os.Args[2]))
}
maxAttempts, err := positiveEnv("JOB_MAX_ATTEMPTS", 5)
if err != nil {
return err
}
seconds, err := positiveEnv("JOB_TIMEOUT_SECONDS", 60)
if err != nil {
return err
}
timeout := time.Duration(seconds) * time.Second
objects, err := repositories.NewObjectStoreFromEnvironment()
if err != nil {
return err
}
for ctx.Err() == nil {
broker, e := openJobBroker(ctx, address)
if e == nil {
if mode == "dispatcher" {
e = dispatchJobs(ctx, store, broker)
} else {
e = consumeJobs(ctx, store, objects, broker, maxAttempts, timeout)
}
broker.Close()
}
if e != nil && ctx.Err() == nil {
log.Print("pipeline: broker session interrupted; reconnecting")
}
select {
case <-ctx.Done():
case <-time.After(time.Second):
}
}
return nil
}
func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, broker *jobBroker) error {
for ctx.Err() == nil {
jobs, err := store.QueuedJobs(ctx)
if err != nil {
return err
}
for _, job := range jobs {
payload, err := json.Marshal(job)
if err != nil {
return err
}
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
err = broker.Publish(publishCtx, payload)
cancel()
if err != nil {
return err
}
// Only confirmed publications are marked. Uncertainty causes safe duplicates.
if err = store.MarkPublished(ctx, job.ID); err != nil {
return err
}
}
select {
case <-ctx.Done():
case <-time.After(time.Second):
}
}
return nil
}
func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objects services.AttachmentObjects, broker *jobBroker, maxAttempts int, timeout time.Duration) error {
deliveries, err := broker.channel.Consume(jobQueue, "", false, false, false, false, nil)
if err != nil {
return errors.New("RabbitMQ consumer unavailable")
}
for {
select {
case <-ctx.Done():
return nil
case message, ok := <-deliveries:
if !ok {
return errors.New("RabbitMQ consumer disconnected")
}
var job repositories.Job
if json.Unmarshal(message.Body, &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" {
log.Print("worker: invalid internal job rejected")
if err = message.Reject(false); err != nil {
return err
}
continue
}
task, cancel := context.WithTimeout(ctx, timeout)
claimed, workErr := store.StartAttempt(task, job, timeout)
if workErr == nil && claimed {
workErr = store.ProcessJob(task, job, maxAttempts, func(a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
return processImage(task, objects, a, staging)
})
}
cancel()
completed := claimed && workErr == nil
if workErr == nil && !claimed {
completed, workErr = store.JobTerminal(ctx, job.ID)
}
// Ack after COMMIT of ready/failed or durable retry intent, or terminal no-op.
// A closed connection automatically returns unacked deliveries to RabbitMQ.
if completed {
if err = message.Ack(false); err != nil {
return err
}
} else {
if err = message.Nack(false, true); err != nil {
return err
}
select {
case <-ctx.Done():
case <-time.After(time.Second):
}
}
}
}
}
func processImage(ctx context.Context, objects services.AttachmentObjects, a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
reader, err := objects.Get(ctx, staging)
if err != nil {
return nil, "", err
}
data, err := io.ReadAll(io.LimitReader(reader, services.MaxUploadBytes+1))
reader.Close()
if err != nil {
return nil, "", err
}
sum := sha256.Sum256(data)
if int64(len(data)) != a.SizeBytes || hex.EncodeToString(sum[:]) != a.SHA256 {
return nil, "invalid_image", errors.New("staging checksum mismatch")
}
config, _, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil || config.Width < 1 || config.Height < 1 || int64(config.Width) > services.MaxImagePixels/int64(config.Height) {
return nil, "invalid_image", errors.New("invalid image dimensions")
}
source, err := imaging.Decode(bytes.NewReader(data), imaging.AutoOrientation(true))
if err != nil {
return nil, "invalid_image", err
}
w, h := source.Bounds().Dx(), source.Bounds().Dy()
if w > 256 || h > 256 {
if w >= h {
h = max(1, h*256/w)
w = 256
} else {
w = max(1, w*256/h)
h = 256
}
}
small := imaging.Resize(source, w, h, imaging.Lanczos)
white := image.NewRGBA(image.Rect(0, 0, w, h))
draw.Draw(white, white.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src)
draw.Draw(white, white.Bounds(), small, image.Point{}, draw.Over)
var thumbnail bytes.Buffer
if err = jpeg.Encode(&thumbnail, white, &jpeg.Options{Quality: 85}); err != nil {
return nil, "processing_failed", err
}
if err = ctx.Err(); err != nil {
return nil, "", err
}
// Deterministic keys make redelivery repeat the same effects.
if err = objects.Put(ctx, a.ObjectKey, bytes.NewReader(data), a.SizeBytes, a.ContentType); err != nil {
return nil, "", err
}
thumb := thumbnail.Bytes()
key := "variants/" + a.ID + "/v1/thumbnail.jpg"
if err = objects.Put(ctx, key, bytes.NewReader(thumb), int64(len(thumb)), "image/jpeg"); err != nil {
return nil, "", err
}
digest := sha256.Sum256(thumb)
return []repositories.Variant{{Name: "thumbnail", ContentType: "image/jpeg", SizeBytes: int64(len(thumb)), Width: w, Height: h, SHA256: hex.EncodeToString(digest[:])}}, "", nil
}
@@ -1,4 +1,4 @@
package main
package repositories
import (
"context"
@@ -11,13 +11,13 @@ import (
const jobQueue = "attachments.v1"
type jobBroker struct {
type JobBroker struct {
connection *amqp.Connection
channel *amqp.Channel
returned chan amqp.Return
}
func openJobBroker(ctx context.Context, address string) (*jobBroker, error) {
func NewJobBroker(ctx context.Context, address string) (*JobBroker, error) {
connection, err := amqp.DialConfig(address, amqp.Config{
Heartbeat: 10 * time.Second,
Dial: func(network, address string) (net.Conn, error) {
@@ -32,7 +32,7 @@ func openJobBroker(ctx context.Context, address string) (*jobBroker, error) {
connection.Close()
return nil, errors.New("RabbitMQ channel unavailable")
}
b := &jobBroker{connection: connection, channel: channel, returned: make(chan amqp.Return, 1)}
b := &JobBroker{connection: connection, channel: channel, returned: make(chan amqp.Return, 1)}
// Queue metadata and message bodies both survive a broker restart.
_, err = channel.QueueDeclare(jobQueue, true, false, false, false, nil)
if err == nil {
@@ -49,9 +49,9 @@ func openJobBroker(ctx context.Context, address string) (*jobBroker, error) {
return b, nil
}
func (b *jobBroker) Close() { b.channel.Close(); b.connection.Close() }
func (b *JobBroker) Close() { b.channel.Close(); b.connection.Close() }
func (b *jobBroker) Publish(ctx context.Context, body []byte) error {
func (b *JobBroker) Publish(ctx context.Context, body []byte) error {
// mandatory catches an unroutable message; confirms alone do not prove routing.
confirmation, err := b.channel.PublishWithDeferredConfirmWithContext(ctx, "", jobQueue, true, false, amqp.Publishing{
DeliveryMode: amqp.Persistent, ContentType: "application/json", Body: body,
@@ -71,3 +71,7 @@ func (b *jobBroker) Publish(ctx context.Context, body []byte) error {
return nil
}
}
func (b *JobBroker) Consume() (<-chan amqp.Delivery, error) {
return b.channel.Consume(jobQueue, "", false, false, false, false, nil)
}
+142
View File
@@ -0,0 +1,142 @@
package repositories
import (
"context"
"errors"
"github.com/jackc/pgx/v5"
"strings"
"time"
"unicode/utf8"
)
type FailedJob struct {
ID string `json:"job_id"`
AttachmentID string `json:"attachment_id"`
Code string `json:"error_code"`
Attempts int `json:"attempts"`
FailedAt time.Time `json:"failed_at"`
}
func (s *AttachmentStore) FailedJobs(ctx context.Context) ([]FailedJob, error) {
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,f.error_code,f.attempts,f.failed_at FROM attachment_job_failures f JOIN attachment_jobs j ON j.id=f.job_id JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed' ORDER BY f.failed_at DESC LIMIT 100`)
if err != nil {
return nil, err
}
defer rows.Close()
result := []FailedJob{}
for rows.Next() {
var j FailedJob
if err = rows.Scan(&j.ID, &j.AttachmentID, &j.Code, &j.Attempts, &j.FailedAt); err != nil {
return nil, err
}
result = append(result, j)
}
return result, rows.Err()
}
func (s *AttachmentStore) ReplayJob(ctx context.Context, id, reason string, exists func(context.Context, string) error) (bool, error) {
reason = strings.TrimSpace(reason)
if reason == "" || utf8.RuneCountInString(reason) > 200 {
return false, errors.New("replay reason must contain 1-200 characters")
}
tx, err := s.db.Begin(ctx)
if err != nil {
return false, err
}
defer tx.Rollback(context.Background())
var status, key, attachment string
var deleted bool
err = tx.QueryRow(ctx, `SELECT a.status,j.staging_key,a.id,j.staging_deleted_at IS NOT NULL FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 FOR UPDATE OF j,a`, id).Scan(&status, &key, &attachment, &deleted)
if err != nil {
return false, err
}
if status != "failed" {
return false, nil
}
if deleted {
return false, errors.New("staging has expired; replay unavailable")
}
if key != "staging/jobs/"+attachment && key != "originals/"+attachment {
return false, errors.New("invalid staging key")
}
if err = exists(ctx, key); err != nil {
return false, errors.New("staging unavailable; replay not scheduled")
}
if _, err = tx.Exec(ctx, `INSERT INTO attachment_job_replays(job_id,reason) VALUES($1,$2)`, id, reason); err != nil {
return false, err
}
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET attempts=0,generation=generation+1,next_attempt_at=now(),lease_until=NULL,lease_token=NULL,published_at=NULL WHERE id=$1`, id); err != nil {
return false, err
}
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=$1`, attachment); err != nil {
return false, err
}
return true, tx.Commit(ctx)
}
// Replay and cleanup lock the same rows. A ready source is removed only after
// final publication, failed sources expire after retention; queued work is safe.
func (s *AttachmentStore) CleanupJobStaging(ctx context.Context, id string, retention time.Duration, dryRun bool, remove func(context.Context, string) error) (bool, error) {
tx, err := s.db.Begin(ctx)
if err != nil {
return false, err
}
defer tx.Rollback(context.Background())
var key, attachment, status string
var eligible bool
err = tx.QueryRow(ctx, `SELECT j.staging_key,a.id,a.status,j.staging_deleted_at IS NULL AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND COALESCE(f.failed_at,a.updated_at)<now()-$2*interval '1 second') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.id=$1 FOR UPDATE OF j,a SKIP LOCKED`, id, retention.Seconds()).Scan(&key, &attachment, &status, &eligible)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, err
}
if !eligible || key == "originals/"+attachment {
return false, nil
}
if key != "staging/jobs/"+attachment {
return false, errors.New("invalid staging key")
}
if dryRun {
return true, nil
}
if err = remove(ctx, key); err != nil {
return false, err
}
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET staging_deleted_at=now() WHERE id=$1`, id); err != nil {
return false, err
}
return true, tx.Commit(ctx)
}
func (s *AttachmentStore) CleanupJobs(ctx context.Context, retention time.Duration, dryRun bool, remove func(context.Context, string) error) (int, error) {
rows, err := s.db.Query(ctx, `SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.staging_deleted_at IS NULL AND j.staging_key='staging/jobs/'||j.attachment_id::text AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND COALESCE(f.failed_at,a.updated_at)<now()-$1*interval '1 second') ORDER BY j.created_at LIMIT 100`, retention.Seconds())
if err != nil {
return 0, err
}
ids := []string{}
for rows.Next() {
var id string
if err = rows.Scan(&id); err != nil {
rows.Close()
return 0, err
}
ids = append(ids, id)
}
err = rows.Err()
rows.Close()
if err != nil {
return 0, err
}
count := 0
for _, id := range ids {
done, e := s.CleanupJobStaging(ctx, id, retention, dryRun, remove)
if e != nil {
return count, e
}
if done {
count++
}
}
return count, nil
}
+106 -46
View File
@@ -2,10 +2,12 @@ package repositories
import (
"context"
"crypto/rand"
"encoding/json"
"errors"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"math/big"
"time"
)
@@ -51,84 +53,142 @@ func (s *AttachmentStore) MarkPublished(ctx context.Context, id string) error {
return err
}
// Claim holds a PostgreSQL row lock across object effects. Concurrent deliveries
// skip the lock. A process crash releases it; the persisted lease bounds recovery.
func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, maxAttempts int, process func(Attachment, string) ([]Variant, string, error)) error {
// StartAttempt commits the lease and attempt before decoding. A token fences a
// consumer that was paused between claim and processing past its lease deadline.
func (s *AttachmentStore) StartAttempt(ctx context.Context, j Job, timeout time.Duration, maxAttempts int) (string, error) {
tx, err := s.db.Begin(ctx)
if err != nil {
return "", err
}
defer tx.Rollback(context.Background())
var status string
var attempts, generation int
var available bool
err = tx.QueryRow(ctx, `SELECT a.status,j.attempts,j.generation,j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 FOR UPDATE OF j,a SKIP LOCKED`, j.ID, j.AttachmentID, j.PipelineVersion).Scan(&status, &attempts, &generation, &available)
if errors.Is(err, pgx.ErrNoRows) {
return "", nil
}
if err != nil {
return "", err
}
if status == "ready" || status == "failed" || !available {
return "", nil
}
// A crashed attempt remains visible in the audit trail.
if _, err = tx.Exec(ctx, `UPDATE attachment_job_attempts SET finished_at=clock_timestamp(),error_code='processing_timeout' WHERE job_id=$1 AND finished_at IS NULL`, j.ID); err != nil {
return "", err
}
if attempts >= maxAttempts {
if err = failJob(ctx, tx, j, "retry_exhausted", attempts); err != nil {
return "", err
}
return "", tx.Commit(ctx)
}
token := uuid.NewString()
_, err = tx.Exec(ctx, `UPDATE attachment_jobs SET attempts=attempts+1,lease_token=$2,lease_until=now()+$3*interval '1 second' WHERE id=$1`, j.ID, token, timeout.Seconds()+5)
if err != nil {
return "", err
}
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='processing',updated_at=now() WHERE id=$1`, j.AttachmentID); err != nil {
return "", err
}
if _, err = tx.Exec(ctx, `INSERT INTO attachment_job_attempts(job_id,generation,attempt) VALUES($1,$2,$3)`, j.ID, generation, attempts+1); err != nil {
return "", err
}
return token, tx.Commit(ctx)
}
var ErrProcessingTimeout = errors.New("processing timeout")
// ProcessJob holds the row lock across deterministic object writes. The lock and
// token check prevent overlapping/stale attempts from publishing metadata.
func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, maxAttempts int, process func(Attachment, string) ([]Variant, string, error)) error {
tx, err := s.db.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(context.Background())
var staging string
var attempts, version int
var attempts, generation int
var a Attachment
err = tx.QueryRow(ctx, `SELECT a.id,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,j.staging_key,j.attempts,j.pipeline_version FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 FOR UPDATE OF j,a SKIP LOCKED`, j.ID, j.AttachmentID).Scan(&a.ID, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, &staging, &attempts, &version)
err = tx.QueryRow(ctx, `SELECT a.id,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,j.staging_key,j.attempts,j.generation FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.lease_token=$4 AND j.lease_until>now() FOR UPDATE OF j,a`, j.ID, j.AttachmentID, j.PipelineVersion, token).Scan(&a.ID, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, &staging, &attempts, &generation)
if errors.Is(err, pgx.ErrNoRows) {
return nil
return errors.New("stale job lease")
}
if err != nil {
return err
}
if a.Status == "ready" || a.Status == "failed" || version != j.PipelineVersion {
return nil
if a.Status != "processing" {
return errors.New("job is not processing")
}
var due bool
if err = tx.QueryRow(ctx, `SELECT next_attempt_at<=now() FROM attachment_jobs WHERE id=$1`, j.ID).Scan(&due); err != nil {
return err
}
if !due {
return nil
}
// Attempts include interrupted executions: StartAttempt commits the counter.
if attempts > maxAttempts {
_, err = tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now() WHERE id=$1`, a.ID)
variants, code, workErr := process(a, staging)
var next *time.Time
var auditCode *string
if workErr == nil {
data, e := json.Marshal(variants)
if e != nil {
return e
}
_, err = tx.Exec(ctx, `UPDATE attachments SET status='ready',variants=$2,error_code=NULL,updated_at=now() WHERE id=$1`, a.ID, data)
} else {
variants, code, workErr := process(a, staging)
if workErr == nil {
data, e := json.Marshal(variants)
if e != nil {
return e
}
_, err = tx.Exec(ctx, `UPDATE attachments SET status='ready',variants=$2,error_code=NULL,updated_at=now() WHERE id=$1`, a.ID, data)
} else if code != "" || attempts >= maxAttempts {
reason := "temporary_failure"
if errors.Is(workErr, ErrProcessingTimeout) {
reason = "processing_timeout"
}
if code != "" {
reason = code
}
auditCode = &reason
if code != "" || attempts >= maxAttempts {
if code == "" {
code = "retry_exhausted"
}
_, err = tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code=$2,updated_at=now() WHERE id=$1`, a.ID, code)
err = failJob(ctx, tx, j, code, attempts)
} else {
due := time.Now().UTC().Add(RetryDelay(attempts))
next = &due
_, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=$1`, a.ID)
}
}
if err != nil {
return err
}
_, err = tx.Exec(ctx, `UPDATE attachment_jobs SET lease_until=NULL,published_at=NULL,next_attempt_at=now()+interval '2 seconds' WHERE id=$1`, j.ID)
_, err = tx.Exec(ctx, `UPDATE attachment_job_attempts SET finished_at=clock_timestamp(),error_code=$4,next_attempt_at=$5 WHERE job_id=$1 AND generation=$2 AND attempt=$3`, j.ID, generation, attempts, auditCode, next)
if err != nil {
return err
}
_, err = tx.Exec(ctx, `UPDATE attachment_jobs SET lease_until=NULL,lease_token=NULL,published_at=NULL,next_attempt_at=COALESCE($2,now()) WHERE id=$1`, j.ID, next)
if err != nil {
return err
}
return tx.Commit(ctx)
}
// StartAttempt publishes processing before the long transaction starts; expired
// leases are recoverable from PostgreSQL even if the broker delivery is pending.
func (s *AttachmentStore) StartAttempt(ctx context.Context, j Job, timeout time.Duration) (bool, error) {
tx, err := s.db.Begin(ctx)
func failJob(ctx context.Context, tx pgx.Tx, j Job, code string, attempts int) error {
if _, err := tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code=$2,updated_at=now() WHERE id=$1`, j.AttachmentID, code); err != nil {
return err
}
if _, err := tx.Exec(ctx, `UPDATE attachment_jobs SET lease_token=NULL,lease_until=NULL WHERE id=$1`, j.ID); err != nil {
return err
}
_, err := tx.Exec(ctx, `INSERT INTO attachment_job_failures(job_id,error_code,attempts) VALUES($1,$2,$3) ON CONFLICT(job_id) DO UPDATE SET error_code=EXCLUDED.error_code,attempts=EXCLUDED.attempts,failed_at=clock_timestamp()`, j.ID, code, attempts)
return err
}
// Equal jitter: exponential ceiling 2,4,8,...,30 seconds; delay in [ceiling/2,ceiling).
func RetryDelay(attempt int) time.Duration {
ceiling := 2 * time.Second
for i := 1; i < attempt && ceiling < 30*time.Second; i++ {
ceiling *= 2
}
if ceiling > 30*time.Second {
ceiling = 30 * time.Second
}
n, err := rand.Int(rand.Reader, big.NewInt(int64(ceiling/2)))
if err != nil {
return false, err
return ceiling
}
defer tx.Rollback(context.Background())
var id string
err = tx.QueryRow(ctx, `UPDATE attachment_jobs j SET attempts=attempts+1,lease_until=now()+$2*interval '1 second' FROM attachments a WHERE j.id=$1 AND a.id=j.attachment_id AND a.status IN ('queued','processing') AND (j.lease_until IS NULL OR j.lease_until<now()) AND j.next_attempt_at<=now() RETURNING j.attachment_id`, j.ID, timeout.Seconds()+10).Scan(&id)
if errors.Is(err, pgx.ErrNoRows) {
return false, nil
}
if err != nil {
return false, err
}
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='processing',updated_at=now() WHERE id=$1`, id); err != nil {
return false, err
}
return true, tx.Commit(ctx)
return ceiling/2 + time.Duration(n.Int64())
}
func enqueueJob(ctx context.Context, tx pgx.Tx, a Attachment, staging string) error {
@@ -0,0 +1,3 @@
DROP INDEX job_dispatch_due_idx;
DROP TABLE attachment_job_replays,attachment_job_failures,attachment_job_attempts;
ALTER TABLE attachment_jobs DROP COLUMN lease_token,DROP COLUMN generation,DROP COLUMN staging_deleted_at;
@@ -0,0 +1,22 @@
ALTER TABLE attachment_jobs ADD COLUMN lease_token uuid;
ALTER TABLE attachment_jobs ADD COLUMN generation integer NOT NULL DEFAULT 1;
ALTER TABLE attachment_jobs ADD COLUMN staging_deleted_at timestamptz;
CREATE TABLE attachment_job_attempts (
job_id uuid NOT NULL REFERENCES attachment_jobs(id), generation integer NOT NULL,
attempt integer NOT NULL, started_at timestamptz NOT NULL DEFAULT now(),
finished_at timestamptz, error_code text, next_attempt_at timestamptz,
PRIMARY KEY(job_id,generation,attempt)
);
CREATE TABLE attachment_job_failures (
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id), error_code text NOT NULL,
attempts integer NOT NULL, failed_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE attachment_job_replays (
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id),
reason text NOT NULL CHECK(char_length(reason) BETWEEN 1 AND 200),
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX job_dispatch_due_idx ON attachment_jobs(next_attempt_at,published_at);
INSERT INTO attachment_job_failures(job_id,error_code,attempts,failed_at)
SELECT j.id,COALESCE(a.error_code,'processing_failed'),j.attempts,a.updated_at
FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed';
+75
View File
@@ -0,0 +1,75 @@
package tests
import (
"context"
"errors"
"github.com/google/uuid"
"messenger/src/backend/repositories"
"testing"
"time"
)
func TestJobExpiredLeaseCannotPublishOrExceedAttempts(t *testing.T) {
f := newTestFixture(t)
user, _ := createTestUser(t, f.users, "lease_user", "Lease")
chat := createTestChat(t, f.chats, "Lease", []string{user.ID})
ctx := context.Background()
id := uuid.NewString()
store := repositories.NewAttachmentStore(f.db)
if err := store.Create(ctx, repositories.Attachment{ID: id, ChatID: chat.ID, OwnerID: user.ID, Filename: "x.png", ObjectKey: "originals/" + id, ContentType: "image/png", SizeBytes: 1, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", CreatedAt: time.Now(), UpdatedAt: time.Now()}); err != nil {
t.Fatal(err)
}
job := repositories.Job{ID: uuid.NewString(), AttachmentID: id, PipelineVersion: 1}
if _, err := f.db.Exec(ctx, `UPDATE attachments SET status='queued' WHERE id=$1;`, id); err != nil {
t.Fatal(err)
}
if _, err := f.db.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,staging_key) VALUES($1,$2,$3)`, job.ID, id, "staging/jobs/"+id); err != nil {
t.Fatal(err)
}
first, err := store.StartAttempt(ctx, job, time.Second, 2)
if err != nil || first == "" {
t.Fatal("claim", err)
}
if _, err = f.db.Exec(ctx, `UPDATE attachment_jobs SET lease_until=now()-interval '1 second' WHERE id=$1`, job.ID); err != nil {
t.Fatal(err)
}
second, err := store.StartAttempt(ctx, job, time.Second, 2)
if err != nil || second == "" || second == first {
t.Fatal("reclaim", err)
}
effects := 0
callback := func(repositories.Attachment, string) ([]repositories.Variant, string, error) {
effects++
return nil, "", errors.New("injected storage failure")
}
if err = store.ProcessJob(ctx, job, first, 2, callback); err == nil || effects != 0 {
t.Fatal("stale worker performed side effect")
}
if err = store.ProcessJob(ctx, job, second, 2, callback); err != nil || effects != 1 {
t.Fatal("new worker failed", err)
}
if token, err := store.StartAttempt(ctx, job, time.Second, 2); err != nil || token != "" {
t.Fatal("failed job reclaimed", err)
}
var status string
var attempts int
if err = f.db.QueryRow(ctx, `SELECT a.status,j.attempts FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1`, job.ID).Scan(&status, &attempts); err != nil || status != "failed" || attempts != 2 {
t.Fatalf("%s/%d %v", status, attempts, err)
}
}
func TestRetryBackoffStaysInExponentialJitterWindow(t *testing.T) {
for attempt, bounds := range map[int][2]time.Duration{1: {time.Second, 2 * time.Second}, 2: {2 * time.Second, 4 * time.Second}, 3: {4 * time.Second, 8 * time.Second}, 8: {15 * time.Second, 30 * time.Second}} {
seen := map[time.Duration]bool{}
for i := 0; i < 20; i++ {
delay := repositories.RetryDelay(attempt)
if delay < bounds[0] || delay > bounds[1] {
t.Fatalf("attempt %d: %v", attempt, delay)
}
seen[delay] = true
}
if len(seen) < 2 {
t.Fatal("jitter produced no variation")
}
}
}
+7
View File
@@ -114,6 +114,13 @@ func newTestFixture(t *testing.T) *testFixture {
if _, err = db.Exec(ctx, string(ddl)); err != nil {
t.Fatal(err)
}
ddl, err = os.ReadFile("../repositories/migrations/000008_job_recovery.up.sql")
if err != nil {
t.Fatal(err)
}
if _, err = db.Exec(ctx, string(ddl)); err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = redisClient.Close() })
if err := redisClient.Ping(ctx).Err(); err != nil {
t.Fatal(err)
+57
View File
@@ -0,0 +1,57 @@
package workers
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"messenger/src/backend/internal/faults"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
)
func processImage(ctx context.Context, objects services.AttachmentObjects, a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
return publishImage(ctx, objects, a, staging, func(ctx context.Context, data []byte) ([]encodedVariant, string, error) {
return renderInSubprocess(ctx, a.ID, data)
})
}
func publishImage(ctx context.Context, objects services.AttachmentObjects, a repositories.Attachment, staging string, render imageRenderer) ([]repositories.Variant, string, error) {
reader, err := objects.Get(ctx, staging)
if err != nil {
return nil, "", err
}
data, err := io.ReadAll(io.LimitReader(reader, services.MaxUploadBytes+1))
reader.Close()
if err != nil {
return nil, "", err
}
digest := sha256.Sum256(data)
if len(data) > int(services.MaxUploadBytes) || int64(len(data)) != a.SizeBytes || hex.EncodeToString(digest[:]) != a.SHA256 {
return nil, "invalid_image", errors.New("invalid staging checksum or size")
}
variants, code, err := render(ctx, data)
if err != nil {
return nil, code, err
}
if err = objects.Put(ctx, a.ObjectKey, bytes.NewReader(data), a.SizeBytes, a.ContentType); err != nil {
return nil, "", err
}
metadata := []repositories.Variant{}
for _, variant := range variants {
extension := ".jpg"
if variant.Metadata.Name == "optimized" {
extension = ".webp"
}
key := "variants/" + a.ID + "/v1/" + variant.Metadata.Name + extension
if err = objects.Put(ctx, key, bytes.NewReader(variant.Data), int64(len(variant.Data)), variant.Metadata.ContentType); err != nil {
return nil, "", err
}
metadata = append(metadata, variant.Metadata)
}
faults.Crash("after_object_before_commit", a.ID)
return metadata, "", nil
}
@@ -1,4 +1,4 @@
package main
package workers
import (
"bytes"
@@ -11,8 +11,12 @@ import (
"image/jpeg"
"image/png"
"io"
"messenger/src/backend/repositories"
"os/exec"
"testing"
"golang.org/x/image/webp"
"messenger/src/backend/repositories"
)
type imageObjects map[string][]byte
@@ -26,6 +30,9 @@ func (o imageObjects) Put(_ context.Context, key string, r io.Reader, _ int64, _
return e
}
func TestThumbnailContract(t *testing.T) {
if _, err := exec.LookPath("cwebp"); err != nil {
t.Skip("cwebp: use Docker builder tests")
}
for _, size := range [][4]int{{320, 200, 256, 160}, {200, 320, 160, 256}, {17, 9, 17, 9}, {1000, 1, 256, 1}} {
t.Run(fmt.Sprintf("%dx%d", size[0], size[1]), func(t *testing.T) {
var source bytes.Buffer
@@ -35,7 +42,7 @@ func TestThumbnailContract(t *testing.T) {
digest := sha256.Sum256(source.Bytes())
o := imageObjects{"staging": source.Bytes()}
a := repositories.Attachment{ID: "sample", ObjectKey: "originals/sample", SizeBytes: int64(source.Len()), SHA256: hex.EncodeToString(digest[:]), ContentType: "image/png"}
variants, code, err := processImage(context.Background(), o, a, "staging")
variants, code, err := publishImage(context.Background(), o, a, "staging", renderImage)
if err != nil {
t.Fatalf("%s: %v", code, err)
}
@@ -43,6 +50,17 @@ func TestThumbnailContract(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if len(variants) != 3 {
t.Fatal("three variants required")
}
crop, err := jpeg.Decode(bytes.NewReader(o["variants/sample/v1/crop.jpg"]))
if err != nil || crop.Bounds().Dx() != 128 || crop.Bounds().Dy() != 128 {
t.Fatal("invalid crop", err)
}
optimized, err := webp.Decode(bytes.NewReader(o["variants/sample/v1/optimized.webp"]))
if err != nil || optimized.Bounds().Dx() != size[0] || optimized.Bounds().Dy() != size[1] {
t.Fatal("invalid optimized", err)
}
wantW, wantH := size[2], size[3]
if thumb.Bounds().Dx() != wantW || thumb.Bounds().Dy() != wantH || variants[0].Width != wantW || variants[0].Height != wantH {
t.Fatalf("wrong dimensions: %v", thumb.Bounds())
@@ -55,10 +73,10 @@ func TestThumbnailContract(t *testing.T) {
t.Fatal("original changed")
}
first := append([]byte{}, o["variants/sample/v1/thumbnail.jpg"]...)
if _, _, err = processImage(context.Background(), o, a, "staging"); err != nil {
if _, _, err = publishImage(context.Background(), o, a, "staging", renderImage); err != nil {
t.Fatal(err)
}
if len(o) != 3 || !bytes.Equal(first, o["variants/sample/v1/thumbnail.jpg"]) {
if len(o) != 5 || !bytes.Equal(first, o["variants/sample/v1/thumbnail.jpg"]) {
t.Fatal("redelivery changed or multiplied effects")
}
})
@@ -66,6 +84,9 @@ func TestThumbnailContract(t *testing.T) {
}
func TestThumbnailAppliesEXIFOrientation(t *testing.T) {
if _, err := exec.LookPath("cwebp"); err != nil {
t.Skip("cwebp: use Docker builder tests")
}
var source bytes.Buffer
if err := jpeg.Encode(&source, image.NewRGBA(image.Rect(0, 0, 320, 200)), nil); err != nil {
t.Fatal(err)
@@ -79,7 +100,7 @@ func TestThumbnailAppliesEXIFOrientation(t *testing.T) {
digest := sha256.Sum256(data)
o := imageObjects{"staging": data}
a := repositories.Attachment{ID: "rotated", ObjectKey: "originals/rotated", SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), ContentType: "image/jpeg"}
variants, _, err := processImage(context.Background(), o, a, "staging")
variants, _, err := publishImage(context.Background(), o, a, "staging", renderImage)
if err != nil {
t.Fatal(err)
}
@@ -90,3 +111,50 @@ func TestThumbnailAppliesEXIFOrientation(t *testing.T) {
t.Fatal("original EXIF modified")
}
}
func TestOptimizedBoundsAndCenteredCrop(t *testing.T) {
if _, err := exec.LookPath("cwebp"); err != nil {
t.Skip("cwebp: use Docker builder tests")
}
var original bytes.Buffer
if err := png.Encode(&original, image.NewNRGBA(image.Rect(0, 0, 1600, 1000))); err != nil {
t.Fatal(err)
}
variants, _, err := renderImage(context.Background(), original.Bytes())
if err != nil {
t.Fatal(err)
}
optimized, err := webp.Decode(bytes.NewReader(variants[2].Data))
if err != nil || optimized.Bounds().Dx() != 1280 || optimized.Bounds().Dy() != 800 {
t.Fatal("optimized must fit 1280x1280", err)
}
stripe := image.NewNRGBA(image.Rect(0, 0, 4, 2))
for y := 0; y < 2; y++ {
for x := 0; x < 4; x++ {
c := color.NRGBA{G: 255, A: 255}
if x == 0 {
c = color.NRGBA{R: 255, A: 255}
}
if x == 3 {
c = color.NRGBA{B: 255, A: 255}
}
stripe.SetNRGBA(x, y, c)
}
}
original = bytes.Buffer{}
if err = png.Encode(&original, stripe); err != nil {
t.Fatal(err)
}
variants, _, err = renderImage(context.Background(), original.Bytes())
if err != nil {
t.Fatal(err)
}
crop, err := jpeg.Decode(bytes.NewReader(variants[1].Data))
if err != nil {
t.Fatal(err)
}
pixel := color.NRGBAModel.Convert(crop.At(64, 64)).(color.NRGBA)
if pixel.G < 240 || pixel.R > 15 || pixel.B > 15 {
t.Fatal("crop did not preserve the green center", pixel)
}
}
+19
View File
@@ -0,0 +1,19 @@
package workers
import (
"errors"
"os"
"strconv"
)
func positiveEnv(name string, fallback int) (int, error) {
s := os.Getenv(name)
if s == "" {
return fallback, nil
}
n, e := strconv.Atoi(s)
if e != nil || n < 1 || n > 3600 {
return 0, errors.New("invalid " + name)
}
return n, nil
}
+98
View File
@@ -0,0 +1,98 @@
package workers
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"os"
"os/exec"
"time"
"messenger/src/backend/internal/faults"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
)
type boundedOutput struct{ bytes.Buffer }
func (b *boundedOutput) Write(data []byte) (int, error) {
if b.Len()+len(data) > 8*1024*1024 {
return 0, errors.New("image result exceeds limit")
}
return b.Buffer.Write(data)
}
func renderInSubprocess(ctx context.Context, id string, data []byte) ([]encodedVariant, string, error) {
seconds, err := positiveEnv("JOB_TIMEOUT_SECONDS", 60)
if err != nil {
return nil, "", err
}
task, cancel := context.WithTimeout(ctx, time.Duration(seconds)*time.Second)
defer cancel()
executable, err := os.Executable()
if err != nil {
return nil, "", err
}
// The parent owns scratch files, so killing a child cannot leak its PNG files.
directory, err := os.MkdirTemp("", "image-job-*")
if err != nil {
return nil, "", err
}
defer os.RemoveAll(directory)
command := exec.CommandContext(task, executable, "image-process", id)
// No database, broker or S3 credentials are needed by a decoder.
command.Env = []string{"PATH=/usr/bin:/bin", "GOMAXPROCS=1", "GOMEMLIMIT=256MiB", "WORKER_MAX_PIXELS=" + workerPixelLimit(), "PIPELINE_FAULT_DIR=" + os.Getenv("PIPELINE_FAULT_DIR")}
command.Env = append(command.Env, "TMPDIR="+directory)
isolateImageProcess(command)
command.Stdin = bytes.NewReader(data)
var output boundedOutput
command.Stdout = &output
err = command.Run()
if task.Err() != nil {
return nil, "", repositories.ErrProcessingTimeout
}
if err != nil {
return nil, "", errors.New("image child terminated")
}
var result imageResult
if err = json.Unmarshal(output.Bytes(), &result); err != nil {
return nil, "", errors.New("invalid image child output")
}
if result.ErrorCode != "" {
return nil, result.ErrorCode, errors.New("invalid image")
}
if len(result.Variants) != 3 {
return nil, "", errors.New("incomplete image child output")
}
return result.Variants, "", nil
}
func workerPixelLimit() string {
if value := os.Getenv("WORKER_MAX_PIXELS"); value != "" {
return value
}
return "8000000"
}
func RunImageChild(args []string) error {
if len(args) != 1 {
return errors.New("image id required")
}
if faults.Take("transform_delay", args[0]) {
time.Sleep(time.Hour)
}
data, err := io.ReadAll(io.LimitReader(os.Stdin, services.MaxUploadBytes+1))
if err != nil {
return err
}
if len(data) > int(services.MaxUploadBytes) {
return json.NewEncoder(os.Stdout).Encode(imageResult{ErrorCode: "invalid_image"})
}
variants, code, err := renderImage(context.Background(), data)
if err != nil && code == "" {
return err
}
return json.NewEncoder(os.Stdout).Encode(imageResult{Variants: variants, ErrorCode: code})
}
@@ -0,0 +1,14 @@
package workers
import (
"os/exec"
"syscall"
"time"
)
func isolateImageProcess(command *exec.Cmd) {
command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
// Kill the whole group, including cwebp, on timeout or shutdown.
command.Cancel = func() error { return syscall.Kill(-command.Process.Pid, syscall.SIGKILL) }
command.WaitDelay = time.Second
}
+113
View File
@@ -0,0 +1,113 @@
package workers
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"image"
"image/color"
"image/draw"
"image/jpeg"
"image/png"
"os"
"os/exec"
"strconv"
"github.com/disintegration/imaging"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
)
type encodedVariant struct {
Metadata repositories.Variant `json:"metadata"`
Data []byte `json:"data"`
}
type imageResult struct {
Variants []encodedVariant `json:"variants"`
ErrorCode string `json:"error_code"`
}
type imageRenderer func(context.Context, []byte) ([]encodedVariant, string, error)
func fitSize(w, h, limit int) (int, int) {
if w > limit || h > limit {
if w >= h {
h = max(1, h*limit/w)
w = limit
} else {
w = max(1, w*limit/h)
h = limit
}
}
return w, h
}
func opaque(source image.Image) image.Image {
result := image.NewRGBA(source.Bounds())
draw.Draw(result, result.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src)
draw.Draw(result, result.Bounds(), source, source.Bounds().Min, draw.Over)
return result
}
// renderImage runs only in the disposable image child in production. DecodeConfig
// bounds pixels before allocation; the container bounds total memory to 512 MiB.
func renderImage(ctx context.Context, data []byte) ([]encodedVariant, string, error) {
pixels, err := strconv.Atoi(workerPixelLimit())
if err != nil || pixels < 1 || int64(pixels) > services.MaxImagePixels {
return nil, "", errors.New("invalid WORKER_MAX_PIXELS")
}
config, _, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil || config.Width < 1 || config.Height < 1 || int64(config.Width) > int64(pixels)/int64(config.Height) {
return nil, "invalid_image", errors.New("image dimensions exceed worker limit")
}
source, err := imaging.Decode(bytes.NewReader(data), imaging.AutoOrientation(true))
if err != nil {
return nil, "invalid_image", err
}
result := []encodedVariant{}
appendVariant := func(name, mime string, img image.Image, body []byte) {
digest := sha256.Sum256(body)
result = append(result, encodedVariant{repositories.Variant{Name: name, ContentType: mime, SizeBytes: int64(len(body)), Width: img.Bounds().Dx(), Height: img.Bounds().Dy(), SHA256: hex.EncodeToString(digest[:])}, body})
}
w, h := fitSize(source.Bounds().Dx(), source.Bounds().Dy(), 256)
thumbnail := opaque(imaging.Resize(source, w, h, imaging.Lanczos))
var body bytes.Buffer
if err = jpeg.Encode(&body, thumbnail, &jpeg.Options{Quality: 85}); err != nil {
return nil, "", err
}
appendVariant("thumbnail", "image/jpeg", thumbnail, body.Bytes())
crop := opaque(imaging.Fill(source, 128, 128, imaging.Center, imaging.Lanczos))
body = bytes.Buffer{}
if err = jpeg.Encode(&body, crop, &jpeg.Options{Quality: 85}); err != nil {
return nil, "", err
}
appendVariant("crop", "image/jpeg", crop, body.Bytes())
w, h = fitSize(source.Bounds().Dx(), source.Bounds().Dy(), 1280)
optimized := imaging.Resize(source, w, h, imaging.Lanczos)
directory, err := os.MkdirTemp("", "image-process-*")
if err != nil {
return nil, "", err
}
defer os.RemoveAll(directory)
input := directory + "/input.png"
file, err := os.Create(input)
if err != nil {
return nil, "", err
}
err = png.Encode(file, optimized)
closeErr := file.Close()
if err != nil {
return nil, "", err
}
if closeErr != nil {
return nil, "", closeErr
}
command := exec.CommandContext(ctx, "cwebp", "-quiet", "-q", "80", "-m", "4", "-metadata", "none", input, "-o", "-")
webp, err := command.Output()
if err != nil {
return nil, "", errors.New("WebP encoding failed")
}
appendVariant("optimized", "image/webp", optimized, webp)
return result, "", nil
}
+172
View File
@@ -0,0 +1,172 @@
package workers
import (
"context"
"encoding/json"
"errors"
"log"
"os"
"os/signal"
"strings"
"syscall"
"time"
"messenger/src/backend/internal/faults"
"messenger/src/backend/repositories"
"messenger/src/backend/services"
)
func RunPipeline(mode string, args []string) error {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
db, err := repositories.OpenDatabase(ctx)
if err != nil {
return err
}
defer db.Close()
store := repositories.NewAttachmentStore(db)
data, err := os.ReadFile(os.Getenv("BROKER_URL_FILE"))
if err != nil {
return errors.New("cannot read broker URL")
}
address := strings.TrimSpace(string(data))
if mode == "broker-publish" {
var job repositories.Job
if len(args) != 1 || json.Unmarshal([]byte(args[0]), &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" {
return errors.New("valid internal job JSON required")
}
broker, e := repositories.NewJobBroker(ctx, address)
if e != nil {
return e
}
defer broker.Close()
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
defer cancel()
return broker.Publish(publishCtx, []byte(args[0]))
}
maxAttempts, err := positiveEnv("JOB_MAX_ATTEMPTS", 5)
if err != nil {
return err
}
seconds, err := positiveEnv("JOB_TIMEOUT_SECONDS", 60)
if err != nil {
return err
}
timeout := time.Duration(seconds) * time.Second
objects, err := repositories.NewObjectStoreFromEnvironment()
if err != nil {
return err
}
for ctx.Err() == nil {
broker, e := repositories.NewJobBroker(ctx, address)
if e == nil {
if mode == "dispatcher" {
e = dispatchJobs(ctx, store, broker)
} else {
e = consumeJobs(ctx, store, objects, broker, maxAttempts, timeout)
}
broker.Close()
}
if e != nil && ctx.Err() == nil {
log.Print("pipeline: broker session interrupted; reconnecting")
}
select {
case <-ctx.Done():
case <-time.After(time.Second):
}
}
return nil
}
func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, broker *repositories.JobBroker) error {
for ctx.Err() == nil {
jobs, err := store.QueuedJobs(ctx)
if err != nil {
return err
}
for _, job := range jobs {
faults.Crash("after_commit_before_publish", job.AttachmentID)
payload, err := json.Marshal(job)
if err != nil {
return err
}
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
err = broker.Publish(publishCtx, payload)
cancel()
if err != nil {
return err
}
// Only confirmed publications are marked. Uncertainty causes safe duplicates.
if err = store.MarkPublished(ctx, job.ID); err != nil {
return err
}
}
select {
case <-ctx.Done():
case <-time.After(time.Second):
}
}
return nil
}
func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objects services.AttachmentObjects, broker *repositories.JobBroker, maxAttempts int, timeout time.Duration) error {
deliveries, err := broker.Consume()
if err != nil {
return errors.New("RabbitMQ consumer unavailable")
}
for {
select {
case <-ctx.Done():
return nil
case message, ok := <-deliveries:
if !ok {
return errors.New("RabbitMQ consumer disconnected")
}
var job repositories.Job
if json.Unmarshal(message.Body, &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" {
log.Print("worker: invalid internal job rejected")
if err = message.Reject(false); err != nil {
return err
}
continue
}
task, cancel := context.WithTimeout(ctx, timeout+15*time.Second)
token, workErr := store.StartAttempt(task, job, timeout+15*time.Second, maxAttempts)
if workErr == nil && token != "" {
workErr = store.ProcessJob(task, job, token, maxAttempts, func(a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
return processImage(task, objects, a, staging)
})
}
cancel()
completed := token != "" && workErr == nil
if workErr == nil && token == "" {
completed, workErr = store.JobTerminal(ctx, job.ID)
}
// Ack after COMMIT of ready/failed or durable retry intent, or terminal no-op.
// A closed connection automatically returns unacked deliveries to RabbitMQ.
if completed {
faults.Crash("after_commit_before_ack", job.AttachmentID)
if objectStore, ok := objects.(*repositories.ObjectStore); ok {
cleanupCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
_, cleanupErr := store.CleanupJobStaging(cleanupCtx, job.ID, 24*time.Hour, false, objectStore.Delete)
cancel()
if cleanupErr != nil {
log.Print("worker: staging cleanup deferred")
}
}
if err = message.Ack(false); err != nil {
return err
}
} else {
if err = message.Nack(false, true); err != nil {
return err
}
select {
case <-ctx.Done():
case <-time.After(time.Second):
}
}
}
}
}