Реализовать уровень 4 и упорядочить структуру backend
Добавить crop и WebP, изолированную обработку с таймаутом, lease token, backoff с jitter, реестр failed, безопасный replay и очистку staging. Добавить проверки crash-окон и восстановления. Разделить запуск API, CLI-команды, фоновые процессы и инфраструктуру по пакетам.
This commit is contained in:
@@ -114,6 +114,7 @@ services:
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
WORKER_MAX_PIXELS: ${WORKER_MAX_PIXELS:-8000000}
|
||||
UPLOAD_TMP_DIR: /tmp/uploads
|
||||
UPLOAD_CLEANUP_GRACE_SECONDS: ${UPLOAD_CLEANUP_GRACE_SECONDS:-600}
|
||||
# Go 1.22 sees host CPUs; keep parallelism suitable for this 0.5 CPU limit.
|
||||
@@ -177,6 +178,8 @@ services:
|
||||
start_period: 20s
|
||||
retries: 30
|
||||
worker:
|
||||
tmpfs:
|
||||
- /tmp:rw,noexec,nosuid,size=64m,mode=1777
|
||||
build: ./src
|
||||
restart: unless-stopped
|
||||
cpus: 1.0
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# Operator-only fault injection. No client API can create these markers.
|
||||
services:
|
||||
dispatcher:
|
||||
environment:
|
||||
PIPELINE_FAULT_DIR: /run/pipeline-faults
|
||||
volumes:
|
||||
- ./.memory/grade4-faults:/run/pipeline-faults
|
||||
worker:
|
||||
environment:
|
||||
PIPELINE_FAULT_DIR: /run/pipeline-faults
|
||||
JOB_TIMEOUT_SECONDS: "8"
|
||||
volumes:
|
||||
- ./.memory/grade4-faults:/run/pipeline-faults
|
||||
@@ -0,0 +1,213 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade 4: real fault injection on synthetic jobs; restores the base stack."""
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import xml.etree.ElementTree as ET
|
||||
import zlib
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check
|
||||
from image_probe import jpeg_size
|
||||
|
||||
FAULTS = ROOT / '.memory/grade4-faults'
|
||||
PREFIX = ['docker', 'compose', '--project-name', 'highload-lab5', '-f', 'compose.yml', '-f', 'infra/compose.faults.yml']
|
||||
|
||||
def compose(*args):
|
||||
return subprocess.check_output(PREFIX + list(args), cwd=ROOT, text=True, stderr=subprocess.DEVNULL).strip()
|
||||
|
||||
def sql(query):
|
||||
return compose('exec', '-T', 'db', 'sh', '-c', 'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"', 'sh', query)
|
||||
|
||||
def until(fn, seconds=120):
|
||||
deadline = time.monotonic()+seconds
|
||||
last = None
|
||||
while time.monotonic()<deadline:
|
||||
try:
|
||||
value = fn()
|
||||
if value:
|
||||
return value
|
||||
except (Failure, OSError, subprocess.CalledProcessError) as exc:
|
||||
last = type(exc).__name__
|
||||
time.sleep(.3)
|
||||
raise AssertionError('condition timed out; last exception: '+str(last))
|
||||
|
||||
def job_id(att):
|
||||
return sql("SELECT id FROM attachment_jobs WHERE attachment_id='"+att['id']+"'")
|
||||
|
||||
def admin(*args):
|
||||
return json.loads(compose('exec', '-T', 'dispatcher', './messenger', 'jobs', *args))
|
||||
|
||||
def arm(point, att):
|
||||
marker = FAULTS/(point+'.'+att['id'])
|
||||
marker.write_text('operator-only synthetic fault\n')
|
||||
return Path(str(marker)+'.hit')
|
||||
|
||||
def s3(method, key='', query=None):
|
||||
# Sign with the existing restricted API identity, never the MinIO root user.
|
||||
values = dict(line.split('=',1) for line in (ROOT/'.env').read_text().splitlines() if '=' in line and not line.startswith('#'))
|
||||
bucket = values.get('S3_BUCKET', 'messenger-attachments')
|
||||
access = (ROOT/'secrets/s3_api_access_key_id').read_text().strip()
|
||||
secret = (ROOT/'secrets/s3_api_secret_access_key').read_text().strip()
|
||||
host = '127.0.0.1:9000'
|
||||
uri = '/'+bucket+'/'+key
|
||||
query_string = urllib.parse.urlencode(sorted((query or {}).items()), quote_via=urllib.parse.quote)
|
||||
now = dt.datetime.now(dt.timezone.utc)
|
||||
timestamp, day = now.strftime('%Y%m%dT%H%M%SZ'), now.strftime('%Y%m%d')
|
||||
payload = hashlib.sha256(b'').hexdigest()
|
||||
headers = 'host:'+host+'\nx-amz-content-sha256:'+payload+'\nx-amz-date:'+timestamp+'\n'
|
||||
signed = 'host;x-amz-content-sha256;x-amz-date'
|
||||
canonical = '\n'.join([method,uri,query_string,headers,signed,payload])
|
||||
scope = day+'/us-east-1/s3/aws4_request'
|
||||
message = 'AWS4-HMAC-SHA256\n'+timestamp+'\n'+scope+'\n'+hashlib.sha256(canonical.encode()).hexdigest()
|
||||
signing = ('AWS4'+secret).encode()
|
||||
for part in (day,'us-east-1','s3','aws4_request'):
|
||||
signing = hmac.new(signing,part.encode(),hashlib.sha256).digest()
|
||||
signature = hmac.new(signing,message.encode(),hashlib.sha256).hexdigest()
|
||||
request = urllib.request.Request('http://'+host+uri+('?' + query_string if query_string else ''),method=method,headers={
|
||||
'x-amz-date':timestamp,'x-amz-content-sha256':payload,
|
||||
'Authorization':'AWS4-HMAC-SHA256 Credential='+access+'/'+scope+', SignedHeaders='+signed+', Signature='+signature})
|
||||
try:
|
||||
response=urllib.request.urlopen(request,timeout=10)
|
||||
except urllib.error.HTTPError as exc:
|
||||
response=exc
|
||||
with response:
|
||||
return response.code, response.read()
|
||||
|
||||
def webp_size(data):
|
||||
check(data[:4]==b'RIFF' and data[8:12]==b'WEBP','invalid WebP container')
|
||||
offset=12
|
||||
while offset+8<=len(data):
|
||||
kind=data[offset:offset+4];length=int.from_bytes(data[offset+4:offset+8],'little');body=data[offset+8:offset+8+length]
|
||||
check(kind not in (b'EXIF',b'XMP ',b'ICCP'),'optimized contains metadata')
|
||||
if kind==b'VP8 ':
|
||||
dimensions=(int.from_bytes(body[6:8],'little')&16383,int.from_bytes(body[8:10],'little')&16383)
|
||||
if kind==b'VP8L':
|
||||
bits=int.from_bytes(body[1:5],'little');dimensions=((bits&16383)+1,((bits>>14)&16383)+1)
|
||||
if kind==b'VP8X':
|
||||
dimensions=(int.from_bytes(body[4:7],'little')+1,int.from_bytes(body[7:10],'little')+1)
|
||||
offset+=8+length+(length%2)
|
||||
return dimensions
|
||||
|
||||
def large_png(w,h):
|
||||
def chunk(kind,data):
|
||||
return struct.pack('>I',len(data))+kind+data+struct.pack('>I',zlib.crc32(kind+data)&0xffffffff)
|
||||
encoder=zlib.compressobj();pieces=[];row=b'\0'+b'\x80\x80\x80\xff'*w
|
||||
for _ in range(h):
|
||||
pieces.append(encoder.compress(row))
|
||||
pieces.append(encoder.flush())
|
||||
return b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',struct.pack('>IIBBBBB',w,h,8,6,0,0,0))+chunk(b'IDAT',b''.join(pieces))+chunk(b'IEND',b'')
|
||||
|
||||
def main():
|
||||
FAULTS.mkdir(parents=True, exist_ok=True);os.chmod(FAULTS,0o777)
|
||||
compose('up','-d','--wait')
|
||||
c=Client(sys.argv[1] if len(sys.argv)>1 else 'http://localhost:8085')
|
||||
alice,bob,eve,chat=c.bootstrap()
|
||||
results={}
|
||||
def metadata(att):
|
||||
return c.call('GET','/api/v1/attachments/'+att['id'],actor=alice,schema='Attachment')['json']
|
||||
def ready(att):
|
||||
return until(lambda: (value:=metadata(att))['status']=='ready' and value)
|
||||
def verify(att, original):
|
||||
value=ready(att)
|
||||
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==original,'original changed')
|
||||
check({v['name'] for v in value['variants']}=={'thumbnail','crop','optimized'},'variant set incomplete')
|
||||
for variant in value['variants']:
|
||||
path='/api/v1/attachments/'+att['id']+'/content?variant='+variant['name']
|
||||
response=c.call('GET',path,actor=bob)
|
||||
check(response['headers'].get_content_type()==variant['content_type'],'wrong variant MIME')
|
||||
check(hashlib.sha256(response['bytes']).hexdigest()==variant['sha256'] and len(response['bytes'])==variant['size_bytes'],'wrong variant checksum')
|
||||
dimensions=webp_size(response['bytes']) if variant['name']=='optimized' else jpeg_size(response['bytes'])
|
||||
expected={'thumbnail':(256,160),'crop':(128,128),'optimized':(320,200)}[variant['name']]
|
||||
check(dimensions==expected==(variant['width'],variant['height']),'actual variant dimensions differ')
|
||||
c.call('GET',path,404,actor=eve)
|
||||
until(lambda: s3('HEAD','staging/jobs/'+att['id'])[0]==404)
|
||||
code,body=s3('GET',query={'list-type':'2','prefix':'variants/'+att['id']+'/'})
|
||||
check(code==200 and len(ET.fromstring(body).findall('{*}Contents'))==3,'duplicate object set')
|
||||
return value
|
||||
try:
|
||||
compose('stop','dispatcher')
|
||||
att,original=c.attachment(alice,chat)
|
||||
marker=arm('after_commit_before_publish',att)
|
||||
check(sql("SELECT published_at IS NULL FROM attachment_jobs WHERE id='"+job_id(att)+"'")=='t','job already published')
|
||||
compose('start','dispatcher');until(marker.exists)
|
||||
verify(att,original)
|
||||
results['commit_publish_crash']={'attachment_id':att['id'],'ready':True,'same_job_count':int(sql("SELECT count(*) FROM attachment_jobs WHERE attachment_id='"+att['id']+"'"))}
|
||||
print('PASS: crash between COMMIT and publish recovers durable intent.',flush=True)
|
||||
|
||||
for point in ('after_object_before_commit','after_commit_before_ack'):
|
||||
compose('stop','worker');att,original=c.attachment(alice,chat);marker=arm(point,att)
|
||||
compose('start','worker');until(marker.exists)
|
||||
value=verify(att,original)
|
||||
attempts=int(sql("SELECT attempts FROM attachment_jobs WHERE id='"+job_id(att)+"'"))
|
||||
check(attempts==(2 if point=='after_object_before_commit' else 1),'unexpected attempt count after crash')
|
||||
results[point]={'attachment_id':att['id'],'attempts':attempts,'variant_count':len(value['variants'])}
|
||||
print('PASS: '+point+' recovers without duplicate effects.',flush=True)
|
||||
|
||||
compose('stop','worker');att,original=c.attachment(alice,chat);marker=arm('transform_delay',att)
|
||||
compose('start','worker')
|
||||
restarts_before=subprocess.check_output(['docker','inspect','--format','{{.RestartCount}}','highload-lab5-worker-1'],text=True)
|
||||
started=time.monotonic();until(marker.exists);verify(att,original)
|
||||
elapsed=time.monotonic()-started
|
||||
check(subprocess.check_output(['docker','inspect','--format','{{.RestartCount}}','highload-lab5-worker-1'],text=True)==restarts_before,'worker restarted during child timeout')
|
||||
check(elapsed<30,'child timeout did not bound processing')
|
||||
check(compose('exec','-T','worker','sh','-c',"find /tmp -maxdepth 1 -name 'image-job-*' -print")=="",'child scratch leaked')
|
||||
reasons=sql("SELECT error_code FROM attachment_job_attempts WHERE job_id='"+job_id(att)+"' AND error_code IS NOT NULL")
|
||||
check('processing_timeout' in reasons,'child timeout was not recorded')
|
||||
results['child_timeout']={'attachment_id':att['id'],'audit_reason':'processing_timeout','elapsed_seconds':round(elapsed,3),'worker_survived':True,'scratch_removed':True}
|
||||
print('PASS: stuck image child is killed and retry succeeds.',flush=True)
|
||||
|
||||
compose('stop','worker');att,original=c.attachment(alice,chat);job=job_id(att)
|
||||
sql("UPDATE attachment_jobs SET staging_key='staging/jobs/missing-test-object' WHERE id='"+job+"'")
|
||||
compose('start','worker')
|
||||
until(lambda: metadata(att)['status']=='failed',180)
|
||||
failures=admin('list-failed');check(any(item['job_id']==job and item['attempts']==5 for item in failures),'failed registry missing job')
|
||||
timing=json.loads(sql("SELECT json_agg(json_build_object('attempt',attempt,'delay',EXTRACT(EPOCH FROM next_attempt_at-finished_at))) FROM attachment_job_attempts WHERE job_id='"+job+"' AND next_attempt_at IS NOT NULL"))
|
||||
for item in timing:
|
||||
ceiling=min(30,2**item['attempt']);check(ceiling/2-.15<=float(item['delay'])<=ceiling+.15,'backoff not in jitter window')
|
||||
sql("UPDATE attachment_jobs SET staging_key='staging/jobs/"+att['id']+"' WHERE id='"+job+"'")
|
||||
check(s3('HEAD','staging/jobs/'+att['id'])[0]==200,'failed staging was removed before retention')
|
||||
check(admin('replay',job,'Synthetic storage failure resolved')['changed'],'failed replay not accepted')
|
||||
verify(att,original)
|
||||
check(not admin('replay',job,'Already successful job')['changed'],'ready replay reopened job')
|
||||
check(sql("SELECT count(*) FROM attachment_job_replays WHERE job_id='"+job+"'")=='1','replay not audited exactly once')
|
||||
results['retry_and_replay']={'attachment_id':att['id'],'attempts_before_replay':5,'retry_delays':timing,'audited_replays':1}
|
||||
print('PASS: bounded exponential jitter, failed registry, audited safe replay.',flush=True)
|
||||
|
||||
data=large_png(3000,3000)
|
||||
att=c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=large.png',202,actor=alice,raw=data,headers={'Content-Type':'image/png'},schema='Attachment')['json']
|
||||
until(lambda: metadata(att)['status']=='failed')
|
||||
check(metadata(att)['error_code']=='invalid_image','worker pixel limit not enforced')
|
||||
check(sql("SELECT attempts FROM attachment_jobs WHERE id='"+job_id(att)+"'")=='1','permanent error retried')
|
||||
check(s3('HEAD','staging/jobs/'+att['id'])[0]==200,'failed source removed early')
|
||||
sql("UPDATE attachment_job_failures SET failed_at=now()-interval '25 hours' WHERE job_id='"+job_id(att)+"'")
|
||||
admin('cleanup-staging',job_id(att))
|
||||
check(s3('HEAD','staging/jobs/'+att['id'])[0]==404,'expired failed source not removed')
|
||||
results['permanent_error_retention']={'attachment_id':att['id'],'pixels':9000000,'attempts':1,'expired_staging_removed':True}
|
||||
compose('stop','worker');queued,_=c.attachment(alice,chat)
|
||||
check(not admin('cleanup-staging',job_id(queued))['changed'],'cleanup deleted queued work')
|
||||
check(s3('HEAD','staging/jobs/'+queued['id'])[0]==200,'queued source missing')
|
||||
compose('start','worker');ready(queued)
|
||||
config=json.loads(subprocess.check_output(['docker','inspect','highload-lab5-worker-1'],text=True))[0]
|
||||
check(config['HostConfig']['Memory']==512*1024*1024,'worker hard memory limit absent')
|
||||
check('size=64m' in config['HostConfig']['Tmpfs']['/tmp'],'temporary disk not bounded')
|
||||
results['resources']={'memory_limit_bytes':config['HostConfig']['Memory'],'prefetch':1,'image_timeout_seconds':8,'worker_pixel_limit':8000000,'queued_staging_protected':True}
|
||||
(ROOT/'.memory/grade4-results.json').write_text(json.dumps(results,ensure_ascii=False,indent=2)+'\n')
|
||||
print('PASS: grade 4 variants, fault windows, timeout, retry/replay, retention and resource limits.',flush=True)
|
||||
finally:
|
||||
# Remove test-only mounts and restore regular timeout/configuration.
|
||||
subprocess.run(['docker','compose','--project-name','highload-lab5','up','-d','--wait'],cwd=ROOT,stdout=subprocess.DEVNULL,check=True)
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
+4
-1
@@ -1,5 +1,7 @@
|
||||
FROM golang:1.22.2-bookworm AS builder
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /src
|
||||
|
||||
COPY go.mod go.sum ./
|
||||
@@ -19,7 +21,8 @@ RUN CGO_ENABLED=0 \
|
||||
|
||||
FROM debian:12.15-slim
|
||||
|
||||
RUN groupadd --system app \
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends webp && rm -rf /var/lib/apt/lists/* \
|
||||
&& groupadd --system app \
|
||||
&& useradd --system --gid app --no-create-home app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/handlers"
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/services"
|
||||
)
|
||||
|
||||
func serverAddress() string {
|
||||
host := os.Getenv("APP_HOST")
|
||||
if host == "" {
|
||||
host = "0.0.0.0"
|
||||
}
|
||||
|
||||
port := os.Getenv("APP_PORT")
|
||||
if port == "" {
|
||||
port = "8080"
|
||||
}
|
||||
|
||||
return net.JoinHostPort(host, port)
|
||||
}
|
||||
|
||||
func RunHTTP() error {
|
||||
if _, err := repositories.UploadCleanupGrace(); err != nil {
|
||||
return err
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
db, err := repositories.OpenDatabase(ctx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return fmt.Errorf("database initialization failed: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
// Opt-in experiment telemetry goes to container logs, never a public port.
|
||||
if os.Getenv("BENCH_POOL_STATS") == "1" {
|
||||
go func() {
|
||||
for range time.Tick(time.Second) {
|
||||
s := db.Stat()
|
||||
data, _ := json.Marshal(map[string]any{
|
||||
"time": time.Now().UTC(), "acquired": s.AcquiredConns(),
|
||||
"total": s.TotalConns(), "max": s.MaxConns(),
|
||||
"gomaxprocs": runtime.GOMAXPROCS(0), "goroutines": runtime.NumGoroutine(),
|
||||
"acquire_count": s.AcquireCount(), "acquire_duration_ns": s.AcquireDuration().Nanoseconds(),
|
||||
"empty_acquire_count": s.EmptyAcquireCount(), "canceled_acquire_count": s.CanceledAcquireCount(),
|
||||
})
|
||||
log.Printf("benchmark_pool %s", data)
|
||||
}
|
||||
}()
|
||||
}
|
||||
redisClient, err := repositories.RedisClientFromEnvironment()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer redisClient.Close()
|
||||
sessionStore := repositories.NewSessionStore(redisClient, "messenger:session:")
|
||||
userStore := repositories.NewUserStore(db)
|
||||
chatStore := repositories.NewChatStore(db)
|
||||
idempotencyTTL, err := repositories.IdempotencyTTL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
messageStore := repositories.NewMessageStoreWithTTL(db, idempotencyTTL)
|
||||
ttl, err := services.SessionTTL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
userService := services.NewUserService(userStore)
|
||||
sessionService := services.NewSessionService(userService, sessionStore, ttl)
|
||||
chatService := services.NewChatService(chatStore)
|
||||
messageService := services.NewMessageService(messageStore)
|
||||
objects, err := repositories.NewObjectStoreFromEnvironment()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
attachmentService := services.NewAttachmentService(repositories.NewAttachmentStore(db), objects)
|
||||
mux := handlers.NewRouter(services.NewHealthService(repositories.NewHealthStore(db), sessionStore, objects), userService, sessionService, chatService, messageService, attachmentService)
|
||||
|
||||
server := &http.Server{
|
||||
Addr: serverAddress(),
|
||||
Handler: mux,
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
}
|
||||
|
||||
shutdownContext, stop := signal.NotifyContext(
|
||||
context.Background(),
|
||||
os.Interrupt,
|
||||
syscall.SIGTERM,
|
||||
)
|
||||
defer stop()
|
||||
|
||||
serverErrors := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
log.Printf("server is listening on %s", server.Addr)
|
||||
|
||||
if err := server.ListenAndServe(); err != nil &&
|
||||
!errors.Is(err, http.ErrServerClosed) {
|
||||
serverErrors <- err
|
||||
}
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-serverErrors:
|
||||
log.Printf("server error: %v", err)
|
||||
|
||||
case <-shutdownContext.Done():
|
||||
log.Printf("shutdown signal received")
|
||||
handlers.BeginShutdown()
|
||||
|
||||
shutdownDeadline, cancel := context.WithTimeout(
|
||||
context.Background(),
|
||||
10*time.Second,
|
||||
)
|
||||
defer cancel()
|
||||
|
||||
if err := server.Shutdown(shutdownDeadline); err != nil {
|
||||
log.Printf("graceful shutdown failed: %v", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Printf("server stopped gracefully")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,15 +1,16 @@
|
||||
package main
|
||||
package commands
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"log"
|
||||
"messenger/src/backend/repositories"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/repositories"
|
||||
)
|
||||
|
||||
func runUploadCleanup(args []string) error {
|
||||
@@ -37,17 +38,21 @@ func runUploadCleanup(args []string) error {
|
||||
pass, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
result, err := store.CleanupUploads(pass, *dryRun, objects.Delete)
|
||||
direct, directErr := store.CleanupDirectUploads(pass, *dryRun, objects.Delete)
|
||||
jobs, jobsErr := store.CleanupJobs(pass, 24*time.Hour, *dryRun, objects.Delete)
|
||||
cancel()
|
||||
if encodeErr := json.NewEncoder(os.Stdout).Encode(map[string]any{"originals": result, "direct": direct}); encodeErr != nil {
|
||||
if encodeErr := json.NewEncoder(os.Stdout).Encode(map[string]any{"originals": result, "direct": direct, "job_staging": jobs}); encodeErr != nil {
|
||||
return encodeErr
|
||||
}
|
||||
if !*loop {
|
||||
if err == nil {
|
||||
err = directErr
|
||||
if err == nil {
|
||||
err = jobsErr
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
if err != nil || directErr != nil {
|
||||
if err != nil || directErr != nil || jobsErr != nil {
|
||||
log.Print("upload cleanup pass failed; journal retained for retry")
|
||||
}
|
||||
select {
|
||||
@@ -0,0 +1,63 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
|
||||
"messenger/src/backend/repositories"
|
||||
)
|
||||
|
||||
func runJobAdmin(args []string) error {
|
||||
if len(args) < 1 {
|
||||
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
defer cancel()
|
||||
db, err := repositories.OpenDatabase(ctx)
|
||||
if err != nil {
|
||||
return errors.New("database unavailable")
|
||||
}
|
||||
defer db.Close()
|
||||
store := repositories.NewAttachmentStore(db)
|
||||
if args[0] == "list-failed" && len(args) == 1 {
|
||||
jobs, e := store.FailedJobs(ctx)
|
||||
if e != nil {
|
||||
return errors.New("cannot read failed registry")
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(jobs)
|
||||
}
|
||||
if len(args) < 2 {
|
||||
return errors.New("job id required")
|
||||
}
|
||||
if _, err = uuid.Parse(args[1]); err != nil {
|
||||
return errors.New("invalid job id")
|
||||
}
|
||||
objects, err := repositories.NewObjectStoreFromEnvironment()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var changed bool
|
||||
switch {
|
||||
case args[0] == "replay" && len(args) == 3:
|
||||
changed, err = store.ReplayJob(ctx, args[1], args[2], func(ctx context.Context, key string) error {
|
||||
r, e := objects.Get(ctx, key)
|
||||
if e == nil {
|
||||
r.Close()
|
||||
}
|
||||
return e
|
||||
})
|
||||
case args[0] == "cleanup-staging" && len(args) == 2:
|
||||
changed, err = store.CleanupJobStaging(ctx, args[1], 24*time.Hour, false, objects.Delete)
|
||||
default:
|
||||
return errors.New("usage: jobs list-failed | replay JOB_ID REASON | cleanup-staging JOB_ID")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(map[string]bool{"changed": changed})
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
package commands
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/app"
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/workers"
|
||||
)
|
||||
|
||||
// Run dispatches the existing executable modes; the binary and CLI stay the same.
|
||||
func Run(args []string) error {
|
||||
if len(args) == 0 {
|
||||
if environment := os.Getenv("APP_ENV"); environment != "" {
|
||||
log.SetPrefix("[" + environment + "] ")
|
||||
}
|
||||
return app.RunHTTP()
|
||||
}
|
||||
switch args[0] {
|
||||
case "image-process":
|
||||
return workers.RunImageChild(args[1:])
|
||||
case "jobs":
|
||||
return runJobAdmin(args[1:])
|
||||
case "worker", "dispatcher", "broker-publish":
|
||||
return workers.RunPipeline(args[0], args[1:])
|
||||
}
|
||||
if environment := os.Getenv("APP_ENV"); environment != "" {
|
||||
log.SetPrefix("[" + environment + "] ")
|
||||
}
|
||||
switch {
|
||||
case args[0] == "s3-bootstrap" && len(args) == 1:
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := repositories.BootstrapS3(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Print("private S3 bucket ready")
|
||||
return nil
|
||||
case args[0] == "attachments-cleanup":
|
||||
return runUploadCleanup(args[1:])
|
||||
case args[0] == "healthcheck" && len(args) == 1:
|
||||
return healthcheck()
|
||||
case args[0] == "migrate" && len(args) == 2 && (args[1] == "up" || args[1] == "down"):
|
||||
url, err := repositories.DatabaseURL()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = repositories.RunMigrations(url, args[1]); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("migration %s complete", args[1])
|
||||
return nil
|
||||
default:
|
||||
return errors.New("usage: messenger migrate up|down")
|
||||
}
|
||||
}
|
||||
|
||||
func healthcheck() error {
|
||||
client := http.Client{Timeout: 2 * time.Second}
|
||||
port := os.Getenv("APP_PORT")
|
||||
if port == "" {
|
||||
port = "8080"
|
||||
}
|
||||
response, err := client.Get("http://" + net.JoinHostPort("127.0.0.1", port) + "/health/ready")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return errors.New("API is not ready")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -116,12 +116,16 @@ func attachmentHandler(attachments *services.AttachmentService) http.HandlerFunc
|
||||
writeError(w, 400, "invalid_request", "invalid variant", requestID)
|
||||
return
|
||||
}
|
||||
if variants[0] == "thumbnail" {
|
||||
if variants[0] == "thumbnail" || variants[0] == "crop" || variants[0] == "optimized" {
|
||||
found := false
|
||||
for _, raw := range a.Variants {
|
||||
if v, ok := raw.(map[string]any); ok && v["name"] == "thumbnail" {
|
||||
a.ObjectKey = "variants/" + a.ID + "/v1/thumbnail.jpg"
|
||||
a.ContentType = "image/jpeg"
|
||||
if v, ok := raw.(map[string]any); ok && v["name"] == variants[0] {
|
||||
extension := ".jpg"
|
||||
if variants[0] == "optimized" {
|
||||
extension = ".webp"
|
||||
}
|
||||
a.ObjectKey = "variants/" + a.ID + "/v1/" + variants[0] + extension
|
||||
a.ContentType = v["content_type"].(string)
|
||||
a.SizeBytes = int64(v["size_bytes"].(float64))
|
||||
found = true
|
||||
break
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
package faults
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// Only an operator-mounted directory enables faults; clients cannot set them.
|
||||
// Rename makes each injected crash one-shot across automatic container restarts.
|
||||
func Take(point, id string) bool {
|
||||
directory := os.Getenv("PIPELINE_FAULT_DIR")
|
||||
if directory == "" {
|
||||
return false
|
||||
}
|
||||
if _, err := uuid.Parse(id); err != nil {
|
||||
return false
|
||||
}
|
||||
name := filepath.Join(directory, point+"."+id)
|
||||
return os.Rename(name, name+".hit") == nil
|
||||
}
|
||||
func Crash(point, id string) {
|
||||
if Take(point, id) {
|
||||
log.Printf("injected pipeline crash: %s, attachment %s", point, id)
|
||||
os.Exit(86)
|
||||
}
|
||||
}
|
||||
+5
-179
@@ -1,191 +1,17 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"messenger/src/backend/handlers"
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/services"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/commands"
|
||||
)
|
||||
|
||||
func serverAddress() string {
|
||||
host := os.Getenv("APP_HOST")
|
||||
if host == "" {
|
||||
host = "0.0.0.0"
|
||||
}
|
||||
|
||||
port := os.Getenv("APP_PORT")
|
||||
if port == "" {
|
||||
port = "8080"
|
||||
}
|
||||
|
||||
return net.JoinHostPort(host, port)
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) >= 2 && (os.Args[1] == "worker" || os.Args[1] == "dispatcher" || os.Args[1] == "broker-publish") {
|
||||
if err := runPipeline(os.Args[1]); err != nil {
|
||||
log.Fatal("pipeline initialization failed: ", err)
|
||||
if err := commands.Run(os.Args[1:]); err != nil {
|
||||
if len(os.Args) > 1 && (os.Args[1] == "image-process" || os.Args[1] == "healthcheck") {
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
if environment := os.Getenv("APP_ENV"); environment != "" {
|
||||
log.SetPrefix("[" + environment + "] ")
|
||||
}
|
||||
if len(os.Args) == 2 && os.Args[1] == "s3-bootstrap" {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
if err := repositories.BootstrapS3(ctx); err != nil {
|
||||
log.Fatal("S3 bootstrap failed: ", err)
|
||||
}
|
||||
log.Print("private S3 bucket ready")
|
||||
return
|
||||
}
|
||||
if len(os.Args) > 1 && os.Args[1] == "attachments-cleanup" {
|
||||
if err := runUploadCleanup(os.Args[2:]); err != nil {
|
||||
log.Fatal("upload cleanup failed: ", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(os.Args) > 1 {
|
||||
if len(os.Args) == 2 && os.Args[1] == "healthcheck" {
|
||||
client := http.Client{Timeout: 2 * time.Second}
|
||||
port := os.Getenv("APP_PORT")
|
||||
if port == "" {
|
||||
port = "8080"
|
||||
}
|
||||
response, err := client.Get("http://" + net.JoinHostPort("127.0.0.1", port) + "/health/ready")
|
||||
if err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(os.Args) != 3 || os.Args[1] != "migrate" || (os.Args[2] != "up" && os.Args[2] != "down") {
|
||||
log.Fatal("usage: messenger migrate up|down")
|
||||
}
|
||||
url, err := repositories.DatabaseURL()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
if err = repositories.RunMigrations(url, os.Args[2]); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Printf("migration %s complete", os.Args[2])
|
||||
return
|
||||
}
|
||||
if _, err := repositories.UploadCleanupGrace(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||
db, err := repositories.OpenDatabase(ctx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Fatalf("database initialization failed: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
// Opt-in experiment telemetry goes to container logs, never a public port.
|
||||
if os.Getenv("BENCH_POOL_STATS") == "1" {
|
||||
go func() {
|
||||
for range time.Tick(time.Second) {
|
||||
s := db.Stat()
|
||||
data, _ := json.Marshal(map[string]any{
|
||||
"time": time.Now().UTC(), "acquired": s.AcquiredConns(),
|
||||
"total": s.TotalConns(), "max": s.MaxConns(),
|
||||
"gomaxprocs": runtime.GOMAXPROCS(0), "goroutines": runtime.NumGoroutine(),
|
||||
"acquire_count": s.AcquireCount(), "acquire_duration_ns": s.AcquireDuration().Nanoseconds(),
|
||||
"empty_acquire_count": s.EmptyAcquireCount(), "canceled_acquire_count": s.CanceledAcquireCount(),
|
||||
})
|
||||
log.Printf("benchmark_pool %s", data)
|
||||
}
|
||||
}()
|
||||
}
|
||||
redisClient, err := repositories.RedisClientFromEnvironment()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer redisClient.Close()
|
||||
sessionStore := repositories.NewSessionStore(redisClient, "messenger:session:")
|
||||
userStore := repositories.NewUserStore(db)
|
||||
chatStore := repositories.NewChatStore(db)
|
||||
idempotencyTTL, err := repositories.IdempotencyTTL()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
messageStore := repositories.NewMessageStoreWithTTL(db, idempotencyTTL)
|
||||
ttl, err := services.SessionTTL()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
userService := services.NewUserService(userStore)
|
||||
sessionService := services.NewSessionService(userService, sessionStore, ttl)
|
||||
chatService := services.NewChatService(chatStore)
|
||||
messageService := services.NewMessageService(messageStore)
|
||||
objects, err := repositories.NewObjectStoreFromEnvironment()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
attachmentService := services.NewAttachmentService(repositories.NewAttachmentStore(db), objects)
|
||||
mux := handlers.NewRouter(services.NewHealthService(repositories.NewHealthStore(db), sessionStore, objects), userService, sessionService, chatService, messageService, attachmentService)
|
||||
|
||||
server := &http.Server{
|
||||
Addr: serverAddress(),
|
||||
Handler: mux,
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 30 * time.Second,
|
||||
IdleTimeout: 60 * time.Second,
|
||||
}
|
||||
|
||||
shutdownContext, stop := signal.NotifyContext(
|
||||
context.Background(),
|
||||
os.Interrupt,
|
||||
syscall.SIGTERM,
|
||||
)
|
||||
defer stop()
|
||||
|
||||
serverErrors := make(chan error, 1)
|
||||
|
||||
go func() {
|
||||
log.Printf("server is listening on %s", server.Addr)
|
||||
|
||||
if err := server.ListenAndServe(); err != nil &&
|
||||
!errors.Is(err, http.ErrServerClosed) {
|
||||
serverErrors <- err
|
||||
}
|
||||
}()
|
||||
|
||||
select {
|
||||
case err := <-serverErrors:
|
||||
log.Printf("server error: %v", err)
|
||||
|
||||
case <-shutdownContext.Done():
|
||||
log.Printf("shutdown signal received")
|
||||
handlers.BeginShutdown()
|
||||
|
||||
shutdownDeadline, cancel := context.WithTimeout(
|
||||
context.Background(),
|
||||
10*time.Second,
|
||||
)
|
||||
defer cancel()
|
||||
|
||||
if err := server.Shutdown(shutdownDeadline); err != nil {
|
||||
log.Printf("graceful shutdown failed: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf("server stopped gracefully")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,238 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"github.com/disintegration/imaging"
|
||||
|
||||
"image"
|
||||
"image/color"
|
||||
"image/draw"
|
||||
"image/jpeg"
|
||||
"io"
|
||||
"log"
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/services"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
func positiveEnv(name string, fallback int) (int, error) {
|
||||
s := os.Getenv(name)
|
||||
if s == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
n, e := strconv.Atoi(s)
|
||||
if e != nil || n < 1 || n > 3600 {
|
||||
return 0, errors.New("invalid " + name)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
func runPipeline(mode string) error {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
db, err := repositories.OpenDatabase(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
store := repositories.NewAttachmentStore(db)
|
||||
data, err := os.ReadFile(os.Getenv("BROKER_URL_FILE"))
|
||||
if err != nil {
|
||||
return errors.New("cannot read broker URL")
|
||||
}
|
||||
|
||||
address := strings.TrimSpace(string(data))
|
||||
if mode == "broker-publish" {
|
||||
var job repositories.Job
|
||||
if len(os.Args) != 3 || json.Unmarshal([]byte(os.Args[2]), &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" {
|
||||
return errors.New("valid internal job JSON required")
|
||||
}
|
||||
broker, e := openJobBroker(ctx, address)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
defer broker.Close()
|
||||
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
return broker.Publish(publishCtx, []byte(os.Args[2]))
|
||||
}
|
||||
maxAttempts, err := positiveEnv("JOB_MAX_ATTEMPTS", 5)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
seconds, err := positiveEnv("JOB_TIMEOUT_SECONDS", 60)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
timeout := time.Duration(seconds) * time.Second
|
||||
objects, err := repositories.NewObjectStoreFromEnvironment()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
broker, e := openJobBroker(ctx, address)
|
||||
if e == nil {
|
||||
if mode == "dispatcher" {
|
||||
e = dispatchJobs(ctx, store, broker)
|
||||
} else {
|
||||
e = consumeJobs(ctx, store, objects, broker, maxAttempts, timeout)
|
||||
}
|
||||
broker.Close()
|
||||
}
|
||||
if e != nil && ctx.Err() == nil {
|
||||
log.Print("pipeline: broker session interrupted; reconnecting")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, broker *jobBroker) error {
|
||||
for ctx.Err() == nil {
|
||||
jobs, err := store.QueuedJobs(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, job := range jobs {
|
||||
payload, err := json.Marshal(job)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
err = broker.Publish(publishCtx, payload)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Only confirmed publications are marked. Uncertainty causes safe duplicates.
|
||||
if err = store.MarkPublished(ctx, job.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objects services.AttachmentObjects, broker *jobBroker, maxAttempts int, timeout time.Duration) error {
|
||||
deliveries, err := broker.channel.Consume(jobQueue, "", false, false, false, false, nil)
|
||||
if err != nil {
|
||||
return errors.New("RabbitMQ consumer unavailable")
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case message, ok := <-deliveries:
|
||||
if !ok {
|
||||
return errors.New("RabbitMQ consumer disconnected")
|
||||
}
|
||||
var job repositories.Job
|
||||
if json.Unmarshal(message.Body, &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" {
|
||||
log.Print("worker: invalid internal job rejected")
|
||||
if err = message.Reject(false); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
task, cancel := context.WithTimeout(ctx, timeout)
|
||||
claimed, workErr := store.StartAttempt(task, job, timeout)
|
||||
if workErr == nil && claimed {
|
||||
workErr = store.ProcessJob(task, job, maxAttempts, func(a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
|
||||
return processImage(task, objects, a, staging)
|
||||
})
|
||||
}
|
||||
cancel()
|
||||
completed := claimed && workErr == nil
|
||||
if workErr == nil && !claimed {
|
||||
completed, workErr = store.JobTerminal(ctx, job.ID)
|
||||
}
|
||||
// Ack after COMMIT of ready/failed or durable retry intent, or terminal no-op.
|
||||
// A closed connection automatically returns unacked deliveries to RabbitMQ.
|
||||
if completed {
|
||||
if err = message.Ack(false); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err = message.Nack(false, true); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func processImage(ctx context.Context, objects services.AttachmentObjects, a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
|
||||
reader, err := objects.Get(ctx, staging)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(reader, services.MaxUploadBytes+1))
|
||||
reader.Close()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
if int64(len(data)) != a.SizeBytes || hex.EncodeToString(sum[:]) != a.SHA256 {
|
||||
return nil, "invalid_image", errors.New("staging checksum mismatch")
|
||||
}
|
||||
config, _, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
if err != nil || config.Width < 1 || config.Height < 1 || int64(config.Width) > services.MaxImagePixels/int64(config.Height) {
|
||||
return nil, "invalid_image", errors.New("invalid image dimensions")
|
||||
}
|
||||
source, err := imaging.Decode(bytes.NewReader(data), imaging.AutoOrientation(true))
|
||||
if err != nil {
|
||||
return nil, "invalid_image", err
|
||||
}
|
||||
w, h := source.Bounds().Dx(), source.Bounds().Dy()
|
||||
if w > 256 || h > 256 {
|
||||
if w >= h {
|
||||
h = max(1, h*256/w)
|
||||
w = 256
|
||||
} else {
|
||||
w = max(1, w*256/h)
|
||||
h = 256
|
||||
}
|
||||
}
|
||||
small := imaging.Resize(source, w, h, imaging.Lanczos)
|
||||
white := image.NewRGBA(image.Rect(0, 0, w, h))
|
||||
draw.Draw(white, white.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src)
|
||||
draw.Draw(white, white.Bounds(), small, image.Point{}, draw.Over)
|
||||
var thumbnail bytes.Buffer
|
||||
if err = jpeg.Encode(&thumbnail, white, &jpeg.Options{Quality: 85}); err != nil {
|
||||
return nil, "processing_failed", err
|
||||
}
|
||||
if err = ctx.Err(); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
// Deterministic keys make redelivery repeat the same effects.
|
||||
if err = objects.Put(ctx, a.ObjectKey, bytes.NewReader(data), a.SizeBytes, a.ContentType); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
thumb := thumbnail.Bytes()
|
||||
key := "variants/" + a.ID + "/v1/thumbnail.jpg"
|
||||
if err = objects.Put(ctx, key, bytes.NewReader(thumb), int64(len(thumb)), "image/jpeg"); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
digest := sha256.Sum256(thumb)
|
||||
return []repositories.Variant{{Name: "thumbnail", ContentType: "image/jpeg", SizeBytes: int64(len(thumb)), Width: w, Height: h, SHA256: hex.EncodeToString(digest[:])}}, "", nil
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package main
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
@@ -11,13 +11,13 @@ import (
|
||||
|
||||
const jobQueue = "attachments.v1"
|
||||
|
||||
type jobBroker struct {
|
||||
type JobBroker struct {
|
||||
connection *amqp.Connection
|
||||
channel *amqp.Channel
|
||||
returned chan amqp.Return
|
||||
}
|
||||
|
||||
func openJobBroker(ctx context.Context, address string) (*jobBroker, error) {
|
||||
func NewJobBroker(ctx context.Context, address string) (*JobBroker, error) {
|
||||
connection, err := amqp.DialConfig(address, amqp.Config{
|
||||
Heartbeat: 10 * time.Second,
|
||||
Dial: func(network, address string) (net.Conn, error) {
|
||||
@@ -32,7 +32,7 @@ func openJobBroker(ctx context.Context, address string) (*jobBroker, error) {
|
||||
connection.Close()
|
||||
return nil, errors.New("RabbitMQ channel unavailable")
|
||||
}
|
||||
b := &jobBroker{connection: connection, channel: channel, returned: make(chan amqp.Return, 1)}
|
||||
b := &JobBroker{connection: connection, channel: channel, returned: make(chan amqp.Return, 1)}
|
||||
// Queue metadata and message bodies both survive a broker restart.
|
||||
_, err = channel.QueueDeclare(jobQueue, true, false, false, false, nil)
|
||||
if err == nil {
|
||||
@@ -49,9 +49,9 @@ func openJobBroker(ctx context.Context, address string) (*jobBroker, error) {
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (b *jobBroker) Close() { b.channel.Close(); b.connection.Close() }
|
||||
func (b *JobBroker) Close() { b.channel.Close(); b.connection.Close() }
|
||||
|
||||
func (b *jobBroker) Publish(ctx context.Context, body []byte) error {
|
||||
func (b *JobBroker) Publish(ctx context.Context, body []byte) error {
|
||||
// mandatory catches an unroutable message; confirms alone do not prove routing.
|
||||
confirmation, err := b.channel.PublishWithDeferredConfirmWithContext(ctx, "", jobQueue, true, false, amqp.Publishing{
|
||||
DeliveryMode: amqp.Persistent, ContentType: "application/json", Body: body,
|
||||
@@ -71,3 +71,7 @@ func (b *jobBroker) Publish(ctx context.Context, body []byte) error {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (b *JobBroker) Consume() (<-chan amqp.Delivery, error) {
|
||||
return b.channel.Consume(jobQueue, "", false, false, false, false, nil)
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
type FailedJob struct {
|
||||
ID string `json:"job_id"`
|
||||
AttachmentID string `json:"attachment_id"`
|
||||
Code string `json:"error_code"`
|
||||
Attempts int `json:"attempts"`
|
||||
FailedAt time.Time `json:"failed_at"`
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) FailedJobs(ctx context.Context) ([]FailedJob, error) {
|
||||
rows, err := s.db.Query(ctx, `SELECT j.id,j.attachment_id,f.error_code,f.attempts,f.failed_at FROM attachment_job_failures f JOIN attachment_jobs j ON j.id=f.job_id JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed' ORDER BY f.failed_at DESC LIMIT 100`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
result := []FailedJob{}
|
||||
for rows.Next() {
|
||||
var j FailedJob
|
||||
if err = rows.Scan(&j.ID, &j.AttachmentID, &j.Code, &j.Attempts, &j.FailedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
result = append(result, j)
|
||||
}
|
||||
return result, rows.Err()
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) ReplayJob(ctx context.Context, id, reason string, exists func(context.Context, string) error) (bool, error) {
|
||||
reason = strings.TrimSpace(reason)
|
||||
if reason == "" || utf8.RuneCountInString(reason) > 200 {
|
||||
return false, errors.New("replay reason must contain 1-200 characters")
|
||||
}
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var status, key, attachment string
|
||||
var deleted bool
|
||||
err = tx.QueryRow(ctx, `SELECT a.status,j.staging_key,a.id,j.staging_deleted_at IS NOT NULL FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 FOR UPDATE OF j,a`, id).Scan(&status, &key, &attachment, &deleted)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if status != "failed" {
|
||||
return false, nil
|
||||
}
|
||||
if deleted {
|
||||
return false, errors.New("staging has expired; replay unavailable")
|
||||
}
|
||||
if key != "staging/jobs/"+attachment && key != "originals/"+attachment {
|
||||
return false, errors.New("invalid staging key")
|
||||
}
|
||||
if err = exists(ctx, key); err != nil {
|
||||
return false, errors.New("staging unavailable; replay not scheduled")
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO attachment_job_replays(job_id,reason) VALUES($1,$2)`, id, reason); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET attempts=0,generation=generation+1,next_attempt_at=now(),lease_until=NULL,lease_token=NULL,published_at=NULL WHERE id=$1`, id); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=$1`, attachment); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// Replay and cleanup lock the same rows. A ready source is removed only after
|
||||
// final publication, failed sources expire after retention; queued work is safe.
|
||||
func (s *AttachmentStore) CleanupJobStaging(ctx context.Context, id string, retention time.Duration, dryRun bool, remove func(context.Context, string) error) (bool, error) {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var key, attachment, status string
|
||||
var eligible bool
|
||||
err = tx.QueryRow(ctx, `SELECT j.staging_key,a.id,a.status,j.staging_deleted_at IS NULL AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND COALESCE(f.failed_at,a.updated_at)<now()-$2*interval '1 second') FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.id=$1 FOR UPDATE OF j,a SKIP LOCKED`, id, retention.Seconds()).Scan(&key, &attachment, &status, &eligible)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if !eligible || key == "originals/"+attachment {
|
||||
return false, nil
|
||||
}
|
||||
if key != "staging/jobs/"+attachment {
|
||||
return false, errors.New("invalid staging key")
|
||||
}
|
||||
if dryRun {
|
||||
return true, nil
|
||||
}
|
||||
if err = remove(ctx, key); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachment_jobs SET staging_deleted_at=now() WHERE id=$1`, id); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
func (s *AttachmentStore) CleanupJobs(ctx context.Context, retention time.Duration, dryRun bool, remove func(context.Context, string) error) (int, error) {
|
||||
rows, err := s.db.Query(ctx, `SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id LEFT JOIN attachment_job_failures f ON f.job_id=j.id WHERE j.staging_deleted_at IS NULL AND j.staging_key='staging/jobs/'||j.attachment_id::text AND (a.status='ready' AND jsonb_array_length(a.variants)>=3 OR a.status='failed' AND COALESCE(f.failed_at,a.updated_at)<now()-$1*interval '1 second') ORDER BY j.created_at LIMIT 100`, retention.Seconds())
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ids := []string{}
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err = rows.Scan(&id); err != nil {
|
||||
rows.Close()
|
||||
return 0, err
|
||||
}
|
||||
ids = append(ids, id)
|
||||
}
|
||||
err = rows.Err()
|
||||
rows.Close()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
count := 0
|
||||
for _, id := range ids {
|
||||
done, e := s.CleanupJobStaging(ctx, id, retention, dryRun, remove)
|
||||
if e != nil {
|
||||
return count, e
|
||||
}
|
||||
if done {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
@@ -2,10 +2,12 @@ package repositories
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"math/big"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -51,84 +53,142 @@ func (s *AttachmentStore) MarkPublished(ctx context.Context, id string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Claim holds a PostgreSQL row lock across object effects. Concurrent deliveries
|
||||
// skip the lock. A process crash releases it; the persisted lease bounds recovery.
|
||||
func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, maxAttempts int, process func(Attachment, string) ([]Variant, string, error)) error {
|
||||
// StartAttempt commits the lease and attempt before decoding. A token fences a
|
||||
// consumer that was paused between claim and processing past its lease deadline.
|
||||
func (s *AttachmentStore) StartAttempt(ctx context.Context, j Job, timeout time.Duration, maxAttempts int) (string, error) {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var status string
|
||||
var attempts, generation int
|
||||
var available bool
|
||||
err = tx.QueryRow(ctx, `SELECT a.status,j.attempts,j.generation,j.next_attempt_at<=now() AND (j.lease_until IS NULL OR j.lease_until<now()) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 FOR UPDATE OF j,a SKIP LOCKED`, j.ID, j.AttachmentID, j.PipelineVersion).Scan(&status, &attempts, &generation, &available)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", nil
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if status == "ready" || status == "failed" || !available {
|
||||
return "", nil
|
||||
}
|
||||
// A crashed attempt remains visible in the audit trail.
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachment_job_attempts SET finished_at=clock_timestamp(),error_code='processing_timeout' WHERE job_id=$1 AND finished_at IS NULL`, j.ID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if attempts >= maxAttempts {
|
||||
if err = failJob(ctx, tx, j, "retry_exhausted", attempts); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "", tx.Commit(ctx)
|
||||
}
|
||||
token := uuid.NewString()
|
||||
_, err = tx.Exec(ctx, `UPDATE attachment_jobs SET attempts=attempts+1,lease_token=$2,lease_until=now()+$3*interval '1 second' WHERE id=$1`, j.ID, token, timeout.Seconds()+5)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='processing',updated_at=now() WHERE id=$1`, j.AttachmentID); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `INSERT INTO attachment_job_attempts(job_id,generation,attempt) VALUES($1,$2,$3)`, j.ID, generation, attempts+1); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return token, tx.Commit(ctx)
|
||||
}
|
||||
|
||||
var ErrProcessingTimeout = errors.New("processing timeout")
|
||||
|
||||
// ProcessJob holds the row lock across deterministic object writes. The lock and
|
||||
// token check prevent overlapping/stale attempts from publishing metadata.
|
||||
func (s *AttachmentStore) ProcessJob(ctx context.Context, j Job, token string, maxAttempts int, process func(Attachment, string) ([]Variant, string, error)) error {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var staging string
|
||||
var attempts, version int
|
||||
var attempts, generation int
|
||||
var a Attachment
|
||||
err = tx.QueryRow(ctx, `SELECT a.id,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,j.staging_key,j.attempts,j.pipeline_version FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 FOR UPDATE OF j,a SKIP LOCKED`, j.ID, j.AttachmentID).Scan(&a.ID, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, &staging, &attempts, &version)
|
||||
err = tx.QueryRow(ctx, `SELECT a.id,a.object_key,a.content_type,a.size_bytes,a.sha256,a.status,j.staging_key,j.attempts,j.generation FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1 AND j.attachment_id=$2 AND j.pipeline_version=$3 AND j.lease_token=$4 AND j.lease_until>now() FOR UPDATE OF j,a`, j.ID, j.AttachmentID, j.PipelineVersion, token).Scan(&a.ID, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, &staging, &attempts, &generation)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil
|
||||
return errors.New("stale job lease")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if a.Status == "ready" || a.Status == "failed" || version != j.PipelineVersion {
|
||||
return nil
|
||||
if a.Status != "processing" {
|
||||
return errors.New("job is not processing")
|
||||
}
|
||||
var due bool
|
||||
if err = tx.QueryRow(ctx, `SELECT next_attempt_at<=now() FROM attachment_jobs WHERE id=$1`, j.ID).Scan(&due); err != nil {
|
||||
return err
|
||||
}
|
||||
if !due {
|
||||
return nil
|
||||
}
|
||||
// Attempts include interrupted executions: StartAttempt commits the counter.
|
||||
if attempts > maxAttempts {
|
||||
_, err = tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now() WHERE id=$1`, a.ID)
|
||||
variants, code, workErr := process(a, staging)
|
||||
var next *time.Time
|
||||
var auditCode *string
|
||||
if workErr == nil {
|
||||
data, e := json.Marshal(variants)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE attachments SET status='ready',variants=$2,error_code=NULL,updated_at=now() WHERE id=$1`, a.ID, data)
|
||||
} else {
|
||||
variants, code, workErr := process(a, staging)
|
||||
if workErr == nil {
|
||||
data, e := json.Marshal(variants)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE attachments SET status='ready',variants=$2,error_code=NULL,updated_at=now() WHERE id=$1`, a.ID, data)
|
||||
} else if code != "" || attempts >= maxAttempts {
|
||||
reason := "temporary_failure"
|
||||
if errors.Is(workErr, ErrProcessingTimeout) {
|
||||
reason = "processing_timeout"
|
||||
}
|
||||
if code != "" {
|
||||
reason = code
|
||||
}
|
||||
auditCode = &reason
|
||||
if code != "" || attempts >= maxAttempts {
|
||||
if code == "" {
|
||||
code = "retry_exhausted"
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code=$2,updated_at=now() WHERE id=$1`, a.ID, code)
|
||||
err = failJob(ctx, tx, j, code, attempts)
|
||||
} else {
|
||||
due := time.Now().UTC().Add(RetryDelay(attempts))
|
||||
next = &due
|
||||
_, err = tx.Exec(ctx, `UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=$1`, a.ID)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE attachment_jobs SET lease_until=NULL,published_at=NULL,next_attempt_at=now()+interval '2 seconds' WHERE id=$1`, j.ID)
|
||||
_, err = tx.Exec(ctx, `UPDATE attachment_job_attempts SET finished_at=clock_timestamp(),error_code=$4,next_attempt_at=$5 WHERE job_id=$1 AND generation=$2 AND attempt=$3`, j.ID, generation, attempts, auditCode, next)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.Exec(ctx, `UPDATE attachment_jobs SET lease_until=NULL,lease_token=NULL,published_at=NULL,next_attempt_at=COALESCE($2,now()) WHERE id=$1`, j.ID, next)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// StartAttempt publishes processing before the long transaction starts; expired
|
||||
// leases are recoverable from PostgreSQL even if the broker delivery is pending.
|
||||
func (s *AttachmentStore) StartAttempt(ctx context.Context, j Job, timeout time.Duration) (bool, error) {
|
||||
tx, err := s.db.Begin(ctx)
|
||||
func failJob(ctx context.Context, tx pgx.Tx, j Job, code string, attempts int) error {
|
||||
if _, err := tx.Exec(ctx, `UPDATE attachments SET status='failed',error_code=$2,updated_at=now() WHERE id=$1`, j.AttachmentID, code); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `UPDATE attachment_jobs SET lease_token=NULL,lease_until=NULL WHERE id=$1`, j.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tx.Exec(ctx, `INSERT INTO attachment_job_failures(job_id,error_code,attempts) VALUES($1,$2,$3) ON CONFLICT(job_id) DO UPDATE SET error_code=EXCLUDED.error_code,attempts=EXCLUDED.attempts,failed_at=clock_timestamp()`, j.ID, code, attempts)
|
||||
return err
|
||||
}
|
||||
|
||||
// Equal jitter: exponential ceiling 2,4,8,...,30 seconds; delay in [ceiling/2,ceiling).
|
||||
func RetryDelay(attempt int) time.Duration {
|
||||
ceiling := 2 * time.Second
|
||||
for i := 1; i < attempt && ceiling < 30*time.Second; i++ {
|
||||
ceiling *= 2
|
||||
}
|
||||
if ceiling > 30*time.Second {
|
||||
ceiling = 30 * time.Second
|
||||
}
|
||||
n, err := rand.Int(rand.Reader, big.NewInt(int64(ceiling/2)))
|
||||
if err != nil {
|
||||
return false, err
|
||||
return ceiling
|
||||
}
|
||||
defer tx.Rollback(context.Background())
|
||||
var id string
|
||||
err = tx.QueryRow(ctx, `UPDATE attachment_jobs j SET attempts=attempts+1,lease_until=now()+$2*interval '1 second' FROM attachments a WHERE j.id=$1 AND a.id=j.attachment_id AND a.status IN ('queued','processing') AND (j.lease_until IS NULL OR j.lease_until<now()) AND j.next_attempt_at<=now() RETURNING j.attachment_id`, j.ID, timeout.Seconds()+10).Scan(&id)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if _, err = tx.Exec(ctx, `UPDATE attachments SET status='processing',updated_at=now() WHERE id=$1`, id); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, tx.Commit(ctx)
|
||||
return ceiling/2 + time.Duration(n.Int64())
|
||||
}
|
||||
|
||||
func enqueueJob(ctx context.Context, tx pgx.Tx, a Attachment, staging string) error {
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
DROP INDEX job_dispatch_due_idx;
|
||||
DROP TABLE attachment_job_replays,attachment_job_failures,attachment_job_attempts;
|
||||
ALTER TABLE attachment_jobs DROP COLUMN lease_token,DROP COLUMN generation,DROP COLUMN staging_deleted_at;
|
||||
@@ -0,0 +1,22 @@
|
||||
ALTER TABLE attachment_jobs ADD COLUMN lease_token uuid;
|
||||
ALTER TABLE attachment_jobs ADD COLUMN generation integer NOT NULL DEFAULT 1;
|
||||
ALTER TABLE attachment_jobs ADD COLUMN staging_deleted_at timestamptz;
|
||||
CREATE TABLE attachment_job_attempts (
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs(id), generation integer NOT NULL,
|
||||
attempt integer NOT NULL, started_at timestamptz NOT NULL DEFAULT now(),
|
||||
finished_at timestamptz, error_code text, next_attempt_at timestamptz,
|
||||
PRIMARY KEY(job_id,generation,attempt)
|
||||
);
|
||||
CREATE TABLE attachment_job_failures (
|
||||
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id), error_code text NOT NULL,
|
||||
attempts integer NOT NULL, failed_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE TABLE attachment_job_replays (
|
||||
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id),
|
||||
reason text NOT NULL CHECK(char_length(reason) BETWEEN 1 AND 200),
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX job_dispatch_due_idx ON attachment_jobs(next_attempt_at,published_at);
|
||||
INSERT INTO attachment_job_failures(job_id,error_code,attempts,failed_at)
|
||||
SELECT j.id,COALESCE(a.error_code,'processing_failed'),j.attempts,a.updated_at
|
||||
FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE a.status='failed';
|
||||
@@ -0,0 +1,75 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/google/uuid"
|
||||
"messenger/src/backend/repositories"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestJobExpiredLeaseCannotPublishOrExceedAttempts(t *testing.T) {
|
||||
f := newTestFixture(t)
|
||||
user, _ := createTestUser(t, f.users, "lease_user", "Lease")
|
||||
chat := createTestChat(t, f.chats, "Lease", []string{user.ID})
|
||||
ctx := context.Background()
|
||||
id := uuid.NewString()
|
||||
store := repositories.NewAttachmentStore(f.db)
|
||||
if err := store.Create(ctx, repositories.Attachment{ID: id, ChatID: chat.ID, OwnerID: user.ID, Filename: "x.png", ObjectKey: "originals/" + id, ContentType: "image/png", SizeBytes: 1, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", CreatedAt: time.Now(), UpdatedAt: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
job := repositories.Job{ID: uuid.NewString(), AttachmentID: id, PipelineVersion: 1}
|
||||
if _, err := f.db.Exec(ctx, `UPDATE attachments SET status='queued' WHERE id=$1;`, id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := f.db.Exec(ctx, `INSERT INTO attachment_jobs(id,attachment_id,staging_key) VALUES($1,$2,$3)`, job.ID, id, "staging/jobs/"+id); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
first, err := store.StartAttempt(ctx, job, time.Second, 2)
|
||||
if err != nil || first == "" {
|
||||
t.Fatal("claim", err)
|
||||
}
|
||||
if _, err = f.db.Exec(ctx, `UPDATE attachment_jobs SET lease_until=now()-interval '1 second' WHERE id=$1`, job.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
second, err := store.StartAttempt(ctx, job, time.Second, 2)
|
||||
if err != nil || second == "" || second == first {
|
||||
t.Fatal("reclaim", err)
|
||||
}
|
||||
effects := 0
|
||||
callback := func(repositories.Attachment, string) ([]repositories.Variant, string, error) {
|
||||
effects++
|
||||
return nil, "", errors.New("injected storage failure")
|
||||
}
|
||||
if err = store.ProcessJob(ctx, job, first, 2, callback); err == nil || effects != 0 {
|
||||
t.Fatal("stale worker performed side effect")
|
||||
}
|
||||
if err = store.ProcessJob(ctx, job, second, 2, callback); err != nil || effects != 1 {
|
||||
t.Fatal("new worker failed", err)
|
||||
}
|
||||
if token, err := store.StartAttempt(ctx, job, time.Second, 2); err != nil || token != "" {
|
||||
t.Fatal("failed job reclaimed", err)
|
||||
}
|
||||
var status string
|
||||
var attempts int
|
||||
if err = f.db.QueryRow(ctx, `SELECT a.status,j.attempts FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=$1`, job.ID).Scan(&status, &attempts); err != nil || status != "failed" || attempts != 2 {
|
||||
t.Fatalf("%s/%d %v", status, attempts, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryBackoffStaysInExponentialJitterWindow(t *testing.T) {
|
||||
for attempt, bounds := range map[int][2]time.Duration{1: {time.Second, 2 * time.Second}, 2: {2 * time.Second, 4 * time.Second}, 3: {4 * time.Second, 8 * time.Second}, 8: {15 * time.Second, 30 * time.Second}} {
|
||||
seen := map[time.Duration]bool{}
|
||||
for i := 0; i < 20; i++ {
|
||||
delay := repositories.RetryDelay(attempt)
|
||||
if delay < bounds[0] || delay > bounds[1] {
|
||||
t.Fatalf("attempt %d: %v", attempt, delay)
|
||||
}
|
||||
seen[delay] = true
|
||||
}
|
||||
if len(seen) < 2 {
|
||||
t.Fatal("jitter produced no variation")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -114,6 +114,13 @@ func newTestFixture(t *testing.T) *testFixture {
|
||||
if _, err = db.Exec(ctx, string(ddl)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ddl, err = os.ReadFile("../repositories/migrations/000008_job_recovery.up.sql")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = db.Exec(ctx, string(ddl)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { _ = redisClient.Close() })
|
||||
if err := redisClient.Ping(ctx).Err(); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
package workers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"messenger/src/backend/internal/faults"
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/services"
|
||||
)
|
||||
|
||||
func processImage(ctx context.Context, objects services.AttachmentObjects, a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
|
||||
return publishImage(ctx, objects, a, staging, func(ctx context.Context, data []byte) ([]encodedVariant, string, error) {
|
||||
return renderInSubprocess(ctx, a.ID, data)
|
||||
})
|
||||
}
|
||||
|
||||
func publishImage(ctx context.Context, objects services.AttachmentObjects, a repositories.Attachment, staging string, render imageRenderer) ([]repositories.Variant, string, error) {
|
||||
reader, err := objects.Get(ctx, staging)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(reader, services.MaxUploadBytes+1))
|
||||
reader.Close()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
digest := sha256.Sum256(data)
|
||||
if len(data) > int(services.MaxUploadBytes) || int64(len(data)) != a.SizeBytes || hex.EncodeToString(digest[:]) != a.SHA256 {
|
||||
return nil, "invalid_image", errors.New("invalid staging checksum or size")
|
||||
}
|
||||
variants, code, err := render(ctx, data)
|
||||
if err != nil {
|
||||
return nil, code, err
|
||||
}
|
||||
if err = objects.Put(ctx, a.ObjectKey, bytes.NewReader(data), a.SizeBytes, a.ContentType); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
metadata := []repositories.Variant{}
|
||||
for _, variant := range variants {
|
||||
extension := ".jpg"
|
||||
if variant.Metadata.Name == "optimized" {
|
||||
extension = ".webp"
|
||||
}
|
||||
key := "variants/" + a.ID + "/v1/" + variant.Metadata.Name + extension
|
||||
if err = objects.Put(ctx, key, bytes.NewReader(variant.Data), int64(len(variant.Data)), variant.Metadata.ContentType); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
metadata = append(metadata, variant.Metadata)
|
||||
}
|
||||
faults.Crash("after_object_before_commit", a.ID)
|
||||
return metadata, "", nil
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package main
|
||||
package workers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -11,8 +11,12 @@ import (
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"io"
|
||||
"messenger/src/backend/repositories"
|
||||
"os/exec"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/image/webp"
|
||||
|
||||
"messenger/src/backend/repositories"
|
||||
)
|
||||
|
||||
type imageObjects map[string][]byte
|
||||
@@ -26,6 +30,9 @@ func (o imageObjects) Put(_ context.Context, key string, r io.Reader, _ int64, _
|
||||
return e
|
||||
}
|
||||
func TestThumbnailContract(t *testing.T) {
|
||||
if _, err := exec.LookPath("cwebp"); err != nil {
|
||||
t.Skip("cwebp: use Docker builder tests")
|
||||
}
|
||||
for _, size := range [][4]int{{320, 200, 256, 160}, {200, 320, 160, 256}, {17, 9, 17, 9}, {1000, 1, 256, 1}} {
|
||||
t.Run(fmt.Sprintf("%dx%d", size[0], size[1]), func(t *testing.T) {
|
||||
var source bytes.Buffer
|
||||
@@ -35,7 +42,7 @@ func TestThumbnailContract(t *testing.T) {
|
||||
digest := sha256.Sum256(source.Bytes())
|
||||
o := imageObjects{"staging": source.Bytes()}
|
||||
a := repositories.Attachment{ID: "sample", ObjectKey: "originals/sample", SizeBytes: int64(source.Len()), SHA256: hex.EncodeToString(digest[:]), ContentType: "image/png"}
|
||||
variants, code, err := processImage(context.Background(), o, a, "staging")
|
||||
variants, code, err := publishImage(context.Background(), o, a, "staging", renderImage)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", code, err)
|
||||
}
|
||||
@@ -43,6 +50,17 @@ func TestThumbnailContract(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(variants) != 3 {
|
||||
t.Fatal("three variants required")
|
||||
}
|
||||
crop, err := jpeg.Decode(bytes.NewReader(o["variants/sample/v1/crop.jpg"]))
|
||||
if err != nil || crop.Bounds().Dx() != 128 || crop.Bounds().Dy() != 128 {
|
||||
t.Fatal("invalid crop", err)
|
||||
}
|
||||
optimized, err := webp.Decode(bytes.NewReader(o["variants/sample/v1/optimized.webp"]))
|
||||
if err != nil || optimized.Bounds().Dx() != size[0] || optimized.Bounds().Dy() != size[1] {
|
||||
t.Fatal("invalid optimized", err)
|
||||
}
|
||||
wantW, wantH := size[2], size[3]
|
||||
if thumb.Bounds().Dx() != wantW || thumb.Bounds().Dy() != wantH || variants[0].Width != wantW || variants[0].Height != wantH {
|
||||
t.Fatalf("wrong dimensions: %v", thumb.Bounds())
|
||||
@@ -55,10 +73,10 @@ func TestThumbnailContract(t *testing.T) {
|
||||
t.Fatal("original changed")
|
||||
}
|
||||
first := append([]byte{}, o["variants/sample/v1/thumbnail.jpg"]...)
|
||||
if _, _, err = processImage(context.Background(), o, a, "staging"); err != nil {
|
||||
if _, _, err = publishImage(context.Background(), o, a, "staging", renderImage); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(o) != 3 || !bytes.Equal(first, o["variants/sample/v1/thumbnail.jpg"]) {
|
||||
if len(o) != 5 || !bytes.Equal(first, o["variants/sample/v1/thumbnail.jpg"]) {
|
||||
t.Fatal("redelivery changed or multiplied effects")
|
||||
}
|
||||
})
|
||||
@@ -66,6 +84,9 @@ func TestThumbnailContract(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestThumbnailAppliesEXIFOrientation(t *testing.T) {
|
||||
if _, err := exec.LookPath("cwebp"); err != nil {
|
||||
t.Skip("cwebp: use Docker builder tests")
|
||||
}
|
||||
var source bytes.Buffer
|
||||
if err := jpeg.Encode(&source, image.NewRGBA(image.Rect(0, 0, 320, 200)), nil); err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -79,7 +100,7 @@ func TestThumbnailAppliesEXIFOrientation(t *testing.T) {
|
||||
digest := sha256.Sum256(data)
|
||||
o := imageObjects{"staging": data}
|
||||
a := repositories.Attachment{ID: "rotated", ObjectKey: "originals/rotated", SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(digest[:]), ContentType: "image/jpeg"}
|
||||
variants, _, err := processImage(context.Background(), o, a, "staging")
|
||||
variants, _, err := publishImage(context.Background(), o, a, "staging", renderImage)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -90,3 +111,50 @@ func TestThumbnailAppliesEXIFOrientation(t *testing.T) {
|
||||
t.Fatal("original EXIF modified")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOptimizedBoundsAndCenteredCrop(t *testing.T) {
|
||||
if _, err := exec.LookPath("cwebp"); err != nil {
|
||||
t.Skip("cwebp: use Docker builder tests")
|
||||
}
|
||||
var original bytes.Buffer
|
||||
if err := png.Encode(&original, image.NewNRGBA(image.Rect(0, 0, 1600, 1000))); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
variants, _, err := renderImage(context.Background(), original.Bytes())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
optimized, err := webp.Decode(bytes.NewReader(variants[2].Data))
|
||||
if err != nil || optimized.Bounds().Dx() != 1280 || optimized.Bounds().Dy() != 800 {
|
||||
t.Fatal("optimized must fit 1280x1280", err)
|
||||
}
|
||||
stripe := image.NewNRGBA(image.Rect(0, 0, 4, 2))
|
||||
for y := 0; y < 2; y++ {
|
||||
for x := 0; x < 4; x++ {
|
||||
c := color.NRGBA{G: 255, A: 255}
|
||||
if x == 0 {
|
||||
c = color.NRGBA{R: 255, A: 255}
|
||||
}
|
||||
if x == 3 {
|
||||
c = color.NRGBA{B: 255, A: 255}
|
||||
}
|
||||
stripe.SetNRGBA(x, y, c)
|
||||
}
|
||||
}
|
||||
original = bytes.Buffer{}
|
||||
if err = png.Encode(&original, stripe); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
variants, _, err = renderImage(context.Background(), original.Bytes())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
crop, err := jpeg.Decode(bytes.NewReader(variants[1].Data))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pixel := color.NRGBAModel.Convert(crop.At(64, 64)).(color.NRGBA)
|
||||
if pixel.G < 240 || pixel.R > 15 || pixel.B > 15 {
|
||||
t.Fatal("crop did not preserve the green center", pixel)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
package workers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func positiveEnv(name string, fallback int) (int, error) {
|
||||
s := os.Getenv(name)
|
||||
if s == "" {
|
||||
return fallback, nil
|
||||
}
|
||||
n, e := strconv.Atoi(s)
|
||||
if e != nil || n < 1 || n > 3600 {
|
||||
return 0, errors.New("invalid " + name)
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
@@ -0,0 +1,98 @@
|
||||
package workers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/internal/faults"
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/services"
|
||||
)
|
||||
|
||||
type boundedOutput struct{ bytes.Buffer }
|
||||
|
||||
func (b *boundedOutput) Write(data []byte) (int, error) {
|
||||
if b.Len()+len(data) > 8*1024*1024 {
|
||||
return 0, errors.New("image result exceeds limit")
|
||||
}
|
||||
return b.Buffer.Write(data)
|
||||
}
|
||||
|
||||
func renderInSubprocess(ctx context.Context, id string, data []byte) ([]encodedVariant, string, error) {
|
||||
seconds, err := positiveEnv("JOB_TIMEOUT_SECONDS", 60)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
task, cancel := context.WithTimeout(ctx, time.Duration(seconds)*time.Second)
|
||||
defer cancel()
|
||||
executable, err := os.Executable()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
// The parent owns scratch files, so killing a child cannot leak its PNG files.
|
||||
directory, err := os.MkdirTemp("", "image-job-*")
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
defer os.RemoveAll(directory)
|
||||
command := exec.CommandContext(task, executable, "image-process", id)
|
||||
// No database, broker or S3 credentials are needed by a decoder.
|
||||
command.Env = []string{"PATH=/usr/bin:/bin", "GOMAXPROCS=1", "GOMEMLIMIT=256MiB", "WORKER_MAX_PIXELS=" + workerPixelLimit(), "PIPELINE_FAULT_DIR=" + os.Getenv("PIPELINE_FAULT_DIR")}
|
||||
command.Env = append(command.Env, "TMPDIR="+directory)
|
||||
isolateImageProcess(command)
|
||||
command.Stdin = bytes.NewReader(data)
|
||||
var output boundedOutput
|
||||
command.Stdout = &output
|
||||
err = command.Run()
|
||||
if task.Err() != nil {
|
||||
return nil, "", repositories.ErrProcessingTimeout
|
||||
}
|
||||
if err != nil {
|
||||
return nil, "", errors.New("image child terminated")
|
||||
}
|
||||
var result imageResult
|
||||
if err = json.Unmarshal(output.Bytes(), &result); err != nil {
|
||||
return nil, "", errors.New("invalid image child output")
|
||||
}
|
||||
if result.ErrorCode != "" {
|
||||
return nil, result.ErrorCode, errors.New("invalid image")
|
||||
}
|
||||
if len(result.Variants) != 3 {
|
||||
return nil, "", errors.New("incomplete image child output")
|
||||
}
|
||||
return result.Variants, "", nil
|
||||
}
|
||||
|
||||
func workerPixelLimit() string {
|
||||
if value := os.Getenv("WORKER_MAX_PIXELS"); value != "" {
|
||||
return value
|
||||
}
|
||||
return "8000000"
|
||||
}
|
||||
|
||||
func RunImageChild(args []string) error {
|
||||
if len(args) != 1 {
|
||||
return errors.New("image id required")
|
||||
}
|
||||
if faults.Take("transform_delay", args[0]) {
|
||||
time.Sleep(time.Hour)
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(os.Stdin, services.MaxUploadBytes+1))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(data) > int(services.MaxUploadBytes) {
|
||||
return json.NewEncoder(os.Stdout).Encode(imageResult{ErrorCode: "invalid_image"})
|
||||
}
|
||||
variants, code, err := renderImage(context.Background(), data)
|
||||
if err != nil && code == "" {
|
||||
return err
|
||||
}
|
||||
return json.NewEncoder(os.Stdout).Encode(imageResult{Variants: variants, ErrorCode: code})
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package workers
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
func isolateImageProcess(command *exec.Cmd) {
|
||||
command.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
|
||||
// Kill the whole group, including cwebp, on timeout or shutdown.
|
||||
command.Cancel = func() error { return syscall.Kill(-command.Process.Pid, syscall.SIGKILL) }
|
||||
command.WaitDelay = time.Second
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package workers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"image"
|
||||
"image/color"
|
||||
"image/draw"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
|
||||
"github.com/disintegration/imaging"
|
||||
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/services"
|
||||
)
|
||||
|
||||
type encodedVariant struct {
|
||||
Metadata repositories.Variant `json:"metadata"`
|
||||
Data []byte `json:"data"`
|
||||
}
|
||||
type imageResult struct {
|
||||
Variants []encodedVariant `json:"variants"`
|
||||
ErrorCode string `json:"error_code"`
|
||||
}
|
||||
type imageRenderer func(context.Context, []byte) ([]encodedVariant, string, error)
|
||||
|
||||
func fitSize(w, h, limit int) (int, int) {
|
||||
if w > limit || h > limit {
|
||||
if w >= h {
|
||||
h = max(1, h*limit/w)
|
||||
w = limit
|
||||
} else {
|
||||
w = max(1, w*limit/h)
|
||||
h = limit
|
||||
}
|
||||
}
|
||||
return w, h
|
||||
}
|
||||
func opaque(source image.Image) image.Image {
|
||||
result := image.NewRGBA(source.Bounds())
|
||||
draw.Draw(result, result.Bounds(), &image.Uniform{C: color.White}, image.Point{}, draw.Src)
|
||||
draw.Draw(result, result.Bounds(), source, source.Bounds().Min, draw.Over)
|
||||
return result
|
||||
}
|
||||
|
||||
// renderImage runs only in the disposable image child in production. DecodeConfig
|
||||
// bounds pixels before allocation; the container bounds total memory to 512 MiB.
|
||||
func renderImage(ctx context.Context, data []byte) ([]encodedVariant, string, error) {
|
||||
pixels, err := strconv.Atoi(workerPixelLimit())
|
||||
if err != nil || pixels < 1 || int64(pixels) > services.MaxImagePixels {
|
||||
return nil, "", errors.New("invalid WORKER_MAX_PIXELS")
|
||||
}
|
||||
config, _, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
if err != nil || config.Width < 1 || config.Height < 1 || int64(config.Width) > int64(pixels)/int64(config.Height) {
|
||||
return nil, "invalid_image", errors.New("image dimensions exceed worker limit")
|
||||
}
|
||||
source, err := imaging.Decode(bytes.NewReader(data), imaging.AutoOrientation(true))
|
||||
if err != nil {
|
||||
return nil, "invalid_image", err
|
||||
}
|
||||
result := []encodedVariant{}
|
||||
appendVariant := func(name, mime string, img image.Image, body []byte) {
|
||||
digest := sha256.Sum256(body)
|
||||
result = append(result, encodedVariant{repositories.Variant{Name: name, ContentType: mime, SizeBytes: int64(len(body)), Width: img.Bounds().Dx(), Height: img.Bounds().Dy(), SHA256: hex.EncodeToString(digest[:])}, body})
|
||||
}
|
||||
w, h := fitSize(source.Bounds().Dx(), source.Bounds().Dy(), 256)
|
||||
thumbnail := opaque(imaging.Resize(source, w, h, imaging.Lanczos))
|
||||
var body bytes.Buffer
|
||||
if err = jpeg.Encode(&body, thumbnail, &jpeg.Options{Quality: 85}); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
appendVariant("thumbnail", "image/jpeg", thumbnail, body.Bytes())
|
||||
crop := opaque(imaging.Fill(source, 128, 128, imaging.Center, imaging.Lanczos))
|
||||
body = bytes.Buffer{}
|
||||
if err = jpeg.Encode(&body, crop, &jpeg.Options{Quality: 85}); err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
appendVariant("crop", "image/jpeg", crop, body.Bytes())
|
||||
w, h = fitSize(source.Bounds().Dx(), source.Bounds().Dy(), 1280)
|
||||
optimized := imaging.Resize(source, w, h, imaging.Lanczos)
|
||||
directory, err := os.MkdirTemp("", "image-process-*")
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
defer os.RemoveAll(directory)
|
||||
input := directory + "/input.png"
|
||||
file, err := os.Create(input)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
err = png.Encode(file, optimized)
|
||||
closeErr := file.Close()
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return nil, "", closeErr
|
||||
}
|
||||
command := exec.CommandContext(ctx, "cwebp", "-quiet", "-q", "80", "-m", "4", "-metadata", "none", input, "-o", "-")
|
||||
webp, err := command.Output()
|
||||
if err != nil {
|
||||
return nil, "", errors.New("WebP encoding failed")
|
||||
}
|
||||
appendVariant("optimized", "image/webp", optimized, webp)
|
||||
return result, "", nil
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
package workers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"messenger/src/backend/internal/faults"
|
||||
"messenger/src/backend/repositories"
|
||||
"messenger/src/backend/services"
|
||||
)
|
||||
|
||||
func RunPipeline(mode string, args []string) error {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||
defer stop()
|
||||
db, err := repositories.OpenDatabase(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
store := repositories.NewAttachmentStore(db)
|
||||
data, err := os.ReadFile(os.Getenv("BROKER_URL_FILE"))
|
||||
if err != nil {
|
||||
return errors.New("cannot read broker URL")
|
||||
}
|
||||
|
||||
address := strings.TrimSpace(string(data))
|
||||
if mode == "broker-publish" {
|
||||
var job repositories.Job
|
||||
if len(args) != 1 || json.Unmarshal([]byte(args[0]), &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" {
|
||||
return errors.New("valid internal job JSON required")
|
||||
}
|
||||
broker, e := repositories.NewJobBroker(ctx, address)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
defer broker.Close()
|
||||
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
return broker.Publish(publishCtx, []byte(args[0]))
|
||||
}
|
||||
maxAttempts, err := positiveEnv("JOB_MAX_ATTEMPTS", 5)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
seconds, err := positiveEnv("JOB_TIMEOUT_SECONDS", 60)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
timeout := time.Duration(seconds) * time.Second
|
||||
objects, err := repositories.NewObjectStoreFromEnvironment()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for ctx.Err() == nil {
|
||||
broker, e := repositories.NewJobBroker(ctx, address)
|
||||
if e == nil {
|
||||
if mode == "dispatcher" {
|
||||
e = dispatchJobs(ctx, store, broker)
|
||||
} else {
|
||||
e = consumeJobs(ctx, store, objects, broker, maxAttempts, timeout)
|
||||
}
|
||||
broker.Close()
|
||||
}
|
||||
if e != nil && ctx.Err() == nil {
|
||||
log.Print("pipeline: broker session interrupted; reconnecting")
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func dispatchJobs(ctx context.Context, store *repositories.AttachmentStore, broker *repositories.JobBroker) error {
|
||||
for ctx.Err() == nil {
|
||||
jobs, err := store.QueuedJobs(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, job := range jobs {
|
||||
faults.Crash("after_commit_before_publish", job.AttachmentID)
|
||||
payload, err := json.Marshal(job)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
publishCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
err = broker.Publish(publishCtx, payload)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Only confirmed publications are marked. Uncertainty causes safe duplicates.
|
||||
if err = store.MarkPublished(ctx, job.ID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func consumeJobs(ctx context.Context, store *repositories.AttachmentStore, objects services.AttachmentObjects, broker *repositories.JobBroker, maxAttempts int, timeout time.Duration) error {
|
||||
deliveries, err := broker.Consume()
|
||||
if err != nil {
|
||||
return errors.New("RabbitMQ consumer unavailable")
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case message, ok := <-deliveries:
|
||||
if !ok {
|
||||
return errors.New("RabbitMQ consumer disconnected")
|
||||
}
|
||||
var job repositories.Job
|
||||
if json.Unmarshal(message.Body, &job) != nil || job.SchemaVersion != 1 || job.Type != "attachment.process.v1" {
|
||||
log.Print("worker: invalid internal job rejected")
|
||||
if err = message.Reject(false); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
task, cancel := context.WithTimeout(ctx, timeout+15*time.Second)
|
||||
token, workErr := store.StartAttempt(task, job, timeout+15*time.Second, maxAttempts)
|
||||
if workErr == nil && token != "" {
|
||||
workErr = store.ProcessJob(task, job, token, maxAttempts, func(a repositories.Attachment, staging string) ([]repositories.Variant, string, error) {
|
||||
return processImage(task, objects, a, staging)
|
||||
})
|
||||
}
|
||||
cancel()
|
||||
completed := token != "" && workErr == nil
|
||||
if workErr == nil && token == "" {
|
||||
completed, workErr = store.JobTerminal(ctx, job.ID)
|
||||
}
|
||||
// Ack after COMMIT of ready/failed or durable retry intent, or terminal no-op.
|
||||
// A closed connection automatically returns unacked deliveries to RabbitMQ.
|
||||
if completed {
|
||||
faults.Crash("after_commit_before_ack", job.AttachmentID)
|
||||
if objectStore, ok := objects.(*repositories.ObjectStore); ok {
|
||||
cleanupCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
_, cleanupErr := store.CleanupJobStaging(cleanupCtx, job.ID, 24*time.Hour, false, objectStore.Delete)
|
||||
cancel()
|
||||
if cleanupErr != nil {
|
||||
log.Print("worker: staging cleanup deferred")
|
||||
}
|
||||
}
|
||||
if err = message.Ack(false); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if err = message.Nack(false, true); err != nil {
|
||||
return err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-time.After(time.Second):
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user