ЛР 5: асинхронная обработка изображений и идемпотентность pipeline

This commit is contained in:
R0flyan
2026-11-06 17:56:33 +03:00
parent 276dbdad8a
commit 05641f0bf4
48 changed files with 3554 additions and 41 deletions
+132
View File
@@ -0,0 +1,132 @@
package main
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"io"
"os"
"os/exec"
"path/filepath"
"strconv"
"github.com/minio/minio-go/v7"
)
type imageFailure string
func (e imageFailure) Error() string { return string(e) }
func (a *App) generateImage(ctx context.Context, j imageJob, src imageSource) ([]Variant, error) {
dir, err := os.MkdirTemp("", "image-job-")
if err != nil {
return nil, err
}
defer os.RemoveAll(dir)
key := stagingKey(j.AttachmentID)
if j.Version == 2 {
key = resultKey(j.AttachmentID, 1, "original")
}
original := filepath.Join(dir, "input")
if err = a.fetchImage(ctx, src, key, original); err != nil {
return nil, err
}
command := exec.CommandContext(ctx, "/usr/local/bin/python3", "-I", "/pipeline.py", dir, src.contentType, strconv.Itoa(j.Version))
command.Env = []string{"LC_ALL=C.UTF-8"}
manifest, err := command.Output()
if err != nil {
var exit *exec.ExitError
if errors.As(err, &exit) && exit.ExitCode() == 2 {
return nil, imageFailure("invalid_image")
}
return nil, err
}
var variants []Variant
if err = json.Unmarshal(manifest, &variants); err != nil || len(variants) != 3 {
return nil, errors.New("invalid encoder manifest")
}
if err = a.putImage(ctx, resultKey(j.AttachmentID, j.Version, "original"), original, src); err != nil {
return nil, err
}
injectImageFault("after_effect", j.AttachmentID)
for i, name := range []string{"thumbnail", "crop", "optimized"} {
v := &variants[i]
v.Name = name
v.ContentType = "image/jpeg"
if name == "optimized" {
v.ContentType = "image/webp"
}
path := filepath.Join(dir, name)
v.SizeBytes, v.SHA256, err = fileDigest(path)
if err != nil {
return nil, err
}
if err = a.putImage(ctx, resultKey(j.AttachmentID, j.Version, name), path, imageSource{v.ContentType, v.SizeBytes, v.SHA256}); err != nil {
return nil, err
}
}
return variants, nil
}
func (a *App) fetchImage(ctx context.Context, src imageSource, key, path string) error {
object, err := a.s3.GetObject(ctx, a.bucket, key, minio.GetObjectOptions{})
if err != nil {
return err
}
defer object.Close()
file, err := os.Create(path)
if err != nil {
return err
}
digest := sha256.New()
size, err := io.Copy(io.MultiWriter(file, digest), io.LimitReader(object, maxUploadBytes+1))
closeErr := file.Close()
if err != nil {
return err
}
if closeErr != nil {
return closeErr
}
if size != src.size || hex.EncodeToString(digest.Sum(nil)) != src.sha256 {
return imageFailure("source_integrity")
}
return nil
}
func fileDigest(path string) (int64, string, error) {
file, err := os.Open(path)
if err != nil {
return 0, "", err
}
defer file.Close()
digest := sha256.New()
size, err := io.Copy(digest, file)
return size, hex.EncodeToString(digest.Sum(nil)), err
}
func (a *App) putImage(ctx context.Context, key, path string, src imageSource) error {
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
opts := minio.PutObjectOptions{ContentType: src.contentType, DisableMultipart: true,
UserMetadata: map[string]string{"sha256": src.sha256}}
opts.SetMatchETagExcept("*")
_, err = a.s3.PutObject(ctx, a.bucket, key, file, src.size, opts)
if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
return err
}
info, err := a.s3.StatObject(ctx, a.bucket, key, minio.StatObjectOptions{})
if err != nil {
return err
}
if info.Size != src.size || info.ContentType != src.contentType || info.Metadata.Get("X-Amz-Meta-Sha256") != src.sha256 {
return imageFailure("result_integrity")
}
return nil
}