ЛР 5: асинхронная обработка изображений и идемпотентность pipeline
This commit is contained in:
+132
@@ -0,0 +1,132 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
)
|
||||
|
||||
type imageFailure string
|
||||
|
||||
func (e imageFailure) Error() string { return string(e) }
|
||||
|
||||
func (a *App) generateImage(ctx context.Context, j imageJob, src imageSource) ([]Variant, error) {
|
||||
dir, err := os.MkdirTemp("", "image-job-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(dir)
|
||||
key := stagingKey(j.AttachmentID)
|
||||
if j.Version == 2 {
|
||||
key = resultKey(j.AttachmentID, 1, "original")
|
||||
}
|
||||
original := filepath.Join(dir, "input")
|
||||
if err = a.fetchImage(ctx, src, key, original); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
command := exec.CommandContext(ctx, "/usr/local/bin/python3", "-I", "/pipeline.py", dir, src.contentType, strconv.Itoa(j.Version))
|
||||
command.Env = []string{"LC_ALL=C.UTF-8"}
|
||||
manifest, err := command.Output()
|
||||
if err != nil {
|
||||
var exit *exec.ExitError
|
||||
if errors.As(err, &exit) && exit.ExitCode() == 2 {
|
||||
return nil, imageFailure("invalid_image")
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
var variants []Variant
|
||||
if err = json.Unmarshal(manifest, &variants); err != nil || len(variants) != 3 {
|
||||
return nil, errors.New("invalid encoder manifest")
|
||||
}
|
||||
|
||||
if err = a.putImage(ctx, resultKey(j.AttachmentID, j.Version, "original"), original, src); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
injectImageFault("after_effect", j.AttachmentID)
|
||||
for i, name := range []string{"thumbnail", "crop", "optimized"} {
|
||||
v := &variants[i]
|
||||
v.Name = name
|
||||
v.ContentType = "image/jpeg"
|
||||
if name == "optimized" {
|
||||
v.ContentType = "image/webp"
|
||||
}
|
||||
path := filepath.Join(dir, name)
|
||||
v.SizeBytes, v.SHA256, err = fileDigest(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = a.putImage(ctx, resultKey(j.AttachmentID, j.Version, name), path, imageSource{v.ContentType, v.SizeBytes, v.SHA256}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return variants, nil
|
||||
}
|
||||
|
||||
func (a *App) fetchImage(ctx context.Context, src imageSource, key, path string) error {
|
||||
object, err := a.s3.GetObject(ctx, a.bucket, key, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer object.Close()
|
||||
file, err := os.Create(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
digest := sha256.New()
|
||||
size, err := io.Copy(io.MultiWriter(file, digest), io.LimitReader(object, maxUploadBytes+1))
|
||||
closeErr := file.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if closeErr != nil {
|
||||
return closeErr
|
||||
}
|
||||
if size != src.size || hex.EncodeToString(digest.Sum(nil)) != src.sha256 {
|
||||
return imageFailure("source_integrity")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fileDigest(path string) (int64, string, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return 0, "", err
|
||||
}
|
||||
defer file.Close()
|
||||
digest := sha256.New()
|
||||
size, err := io.Copy(digest, file)
|
||||
return size, hex.EncodeToString(digest.Sum(nil)), err
|
||||
}
|
||||
|
||||
func (a *App) putImage(ctx context.Context, key, path string, src imageSource) error {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
opts := minio.PutObjectOptions{ContentType: src.contentType, DisableMultipart: true,
|
||||
UserMetadata: map[string]string{"sha256": src.sha256}}
|
||||
opts.SetMatchETagExcept("*")
|
||||
_, err = a.s3.PutObject(ctx, a.bucket, key, file, src.size, opts)
|
||||
if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
|
||||
return err
|
||||
}
|
||||
info, err := a.s3.StatObject(ctx, a.bucket, key, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.Size != src.size || info.ContentType != src.contentType || info.Metadata.Get("X-Amz-Meta-Sha256") != src.sha256 {
|
||||
return imageFailure("result_integrity")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user