forked from mtusi2026/mtusi2026lab5
323 lines
11 KiB
Go
323 lines
11 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"image"
|
|
_ "image/jpeg"
|
|
_ "image/png"
|
|
"io"
|
|
"mime"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/minio/minio-go/v7"
|
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
|
)
|
|
|
|
const maxUploadBytes int64 = 10 * 1024 * 1024
|
|
const maxImagePixels int64 = 40000000
|
|
|
|
type Variant struct {
|
|
Name string `json:"name"`
|
|
ContentType string `json:"content_type"`
|
|
SizeBytes int64 `json:"size_bytes"`
|
|
Width int `json:"width"`
|
|
Height int `json:"height"`
|
|
SHA256 string `json:"sha256"`
|
|
}
|
|
|
|
type Attachment struct {
|
|
ID string `json:"id"`
|
|
ChatID string `json:"chat_id"`
|
|
OwnerID string `json:"owner_id"`
|
|
Filename string `json:"filename"`
|
|
ContentType string `json:"content_type"`
|
|
SizeBytes int64 `json:"size_bytes"`
|
|
SHA256 string `json:"sha256"`
|
|
Status string `json:"status"`
|
|
Variants []Variant `json:"variants"`
|
|
ErrorCode *string `json:"error_code"`
|
|
CreatedAt string `json:"created_at"`
|
|
UpdatedAt string `json:"updated_at"`
|
|
resultVersion int
|
|
}
|
|
|
|
func s3Client() (*minio.Client, string, error) {
|
|
endpoint, err := url.Parse(env("S3_ENDPOINT", "http://s3:9000"))
|
|
if err != nil || (endpoint.Scheme != "http" && endpoint.Scheme != "https") || endpoint.Host == "" || endpoint.User != nil || endpoint.RawQuery != "" || (endpoint.Path != "" && endpoint.Path != "/") {
|
|
return nil, "", errors.New("invalid S3 endpoint")
|
|
}
|
|
access, err := secretFile("S3_ACCESS_KEY_ID_FILE")
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
secret, err := secretFile("S3_SECRET_ACCESS_KEY_FILE")
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
transport := &http.Transport{Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{Timeout: 2 * time.Second, KeepAlive: 30 * time.Second}).DialContext, MaxIdleConns: 16, MaxIdleConnsPerHost: 8, MaxConnsPerHost: 8, ResponseHeaderTimeout: 5 * time.Second, TLSHandshakeTimeout: 3 * time.Second, IdleConnTimeout: 30 * time.Second, DisableCompression: true}
|
|
client, err := minio.New(endpoint.Host, &minio.Options{Creds: credentials.NewStaticV4(access, secret, ""), Secure: endpoint.Scheme == "https", Region: env("S3_REGION", "us-east-1"), Transport: transport, MaxRetries: 1, BucketLookup: minio.BucketLookupPath})
|
|
return client, env("S3_BUCKET", "messenger-attachments"), err
|
|
}
|
|
|
|
func validFilename(s string) bool {
|
|
if !utf8.ValidString(s) || strings.TrimSpace(s) == "" || utf8.RuneCountInString(s) > 255 {
|
|
return false
|
|
}
|
|
for _, c := range s {
|
|
if c == '/' || c == '\\' || unicode.IsControl(c) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
func messageFields(body map[string]json.RawMessage) (string, []string, error) {
|
|
var text string
|
|
if string(body["text"]) == "null" || json.Unmarshal(body["text"], &text) != nil || utf8.RuneCountInString(text) > 4000 {
|
|
return "", nil, invalid()
|
|
}
|
|
ids := []string{}
|
|
if raw, ok := body["attachment_ids"]; ok {
|
|
if string(raw) == "null" || json.Unmarshal(raw, &ids) != nil || len(ids) > 10 {
|
|
return "", nil, invalid()
|
|
}
|
|
}
|
|
seen := map[string]bool{}
|
|
for i, id := range ids {
|
|
normalized, err := normalizeID(id)
|
|
if err != nil || seen[normalized] {
|
|
return "", nil, invalid()
|
|
}
|
|
ids[i] = normalized
|
|
seen[normalized] = true
|
|
}
|
|
if strings.TrimSpace(text) == "" && len(ids) == 0 {
|
|
return "", nil, invalid()
|
|
}
|
|
return text, ids, nil
|
|
}
|
|
|
|
func (a *App) s3Ready(ctx context.Context) bool {
|
|
check, cancel := context.WithTimeout(ctx, time.Second)
|
|
defer cancel()
|
|
ok, err := a.s3.BucketExists(check, a.bucket)
|
|
return err == nil && ok
|
|
}
|
|
func mediaError() error {
|
|
return apiError{415, "unsupported_media_type", "Expected a valid PNG or JPEG matching Content-Type"}
|
|
}
|
|
func tooLarge() error { return apiError{413, "payload_too_large", "Image exceeds byte or pixel limit"} }
|
|
|
|
// The encoded file lives only in a bounded tmpfs during validation/PUT. At most
|
|
// five uploads and one decoded image per process; no bytes enter PostgreSQL.
|
|
func (a *App) upload(ctx context.Context, w http.ResponseWriter, r *http.Request, chat, user string) (Attachment, error) {
|
|
result := Attachment{}
|
|
query, err := url.ParseQuery(r.URL.RawQuery)
|
|
if err != nil || len(query) != 1 || len(query["filename"]) != 1 || !validFilename(query.Get("filename")) {
|
|
return result, invalid()
|
|
}
|
|
contentType, params, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
|
if err != nil || len(params) != 0 || (contentType != "image/png" && contentType != "image/jpeg") {
|
|
return result, mediaError()
|
|
}
|
|
if r.ContentLength > maxUploadBytes {
|
|
return result, tooLarge()
|
|
}
|
|
select {
|
|
case a.uploadSlots <- struct{}{}:
|
|
defer func() { <-a.uploadSlots }()
|
|
default:
|
|
return result, unavailable()
|
|
}
|
|
f, err := os.CreateTemp("", "upload-*")
|
|
if err != nil {
|
|
return result, unavailable()
|
|
}
|
|
defer os.Remove(f.Name())
|
|
defer f.Close()
|
|
digest := sha256.New()
|
|
size, err := io.Copy(io.MultiWriter(f, digest), http.MaxBytesReader(w, r.Body, maxUploadBytes))
|
|
if err != nil {
|
|
var big *http.MaxBytesError
|
|
if errors.As(err, &big) {
|
|
return result, tooLarge()
|
|
}
|
|
return result, invalid()
|
|
}
|
|
if size == 0 {
|
|
return result, mediaError()
|
|
}
|
|
if _, err = f.Seek(0, io.SeekStart); err != nil {
|
|
return result, unavailable()
|
|
}
|
|
cfg, format, err := image.DecodeConfig(f)
|
|
if err != nil || (format != "png" && format != "jpeg") || "image/"+format != contentType {
|
|
return result, mediaError()
|
|
}
|
|
if cfg.Width <= 0 || cfg.Height <= 0 || int64(cfg.Width) > maxImagePixels/int64(cfg.Height) {
|
|
return result, tooLarge()
|
|
}
|
|
select {
|
|
case a.decodeSlot <- struct{}{}:
|
|
case <-ctx.Done():
|
|
return result, unavailable()
|
|
}
|
|
validation := func() error {
|
|
defer func() { <-a.decodeSlot }()
|
|
if _, e := f.Seek(0, io.SeekStart); e != nil {
|
|
return unavailable()
|
|
}
|
|
decoded, actual, e := image.Decode(f)
|
|
if e != nil || actual != format || decoded.Bounds().Dx() != cfg.Width || decoded.Bounds().Dy() != cfg.Height {
|
|
return mediaError()
|
|
}
|
|
return nil
|
|
}
|
|
if err = validation(); err != nil {
|
|
return result, err
|
|
}
|
|
if ctx.Err() != nil {
|
|
return result, unavailable()
|
|
}
|
|
if _, err = f.Seek(0, io.SeekStart); err != nil {
|
|
return result, unavailable()
|
|
}
|
|
result = Attachment{ID: uuid(), ChatID: chat, OwnerID: user, Filename: query.Get("filename"), ContentType: contentType, SizeBytes: size, SHA256: hex.EncodeToString(digest.Sum(nil)), Status: "queued", Variants: []Variant{}}
|
|
tx, err := a.db.Begin(ctx)
|
|
if err != nil {
|
|
return result, unavailable()
|
|
}
|
|
defer rollbackAttachment(tx)
|
|
if err = attachmentLock(ctx, tx, result.ID); err != nil {
|
|
return result, unavailable()
|
|
}
|
|
key := "original/" + result.ID
|
|
_, err = a.s3.PutObject(ctx, a.bucket, stagingKey(result.ID), f, size, minio.PutObjectOptions{ContentType: contentType, DisableMultipart: true, UserMetadata: map[string]string{"sha256": result.SHA256}})
|
|
if err != nil {
|
|
return result, unavailable()
|
|
}
|
|
var created, updated time.Time
|
|
err = tx.QueryRow(ctx, `INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,status,active_version)
|
|
VALUES($1,$2,$3,$4,$5,$6,$7,$8,'queued',1) RETURNING created_at,updated_at`, result.ID, chat, user, key, result.Filename, contentType, size, result.SHA256).Scan(&created, &updated)
|
|
if err != nil {
|
|
return result, unavailable()
|
|
}
|
|
if err = enqueueImage(ctx, tx, result.ID, 1); err != nil {
|
|
return result, unavailable()
|
|
}
|
|
// COMMIT may have succeeded even when its acknowledgement is lost. Never
|
|
// blindly delete the blob here: reconciler checks durable metadata + grace.
|
|
if err = tx.Commit(ctx); err != nil {
|
|
return result, unavailable()
|
|
}
|
|
result.CreatedAt = stamp(created)
|
|
result.UpdatedAt = stamp(updated)
|
|
return result, nil
|
|
}
|
|
|
|
func attachmentLock(ctx context.Context, tx pgx.Tx, id string) error {
|
|
_, err := tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1,0))", "attachment:"+id)
|
|
return err
|
|
}
|
|
|
|
func rollbackAttachment(tx pgx.Tx) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
_ = tx.Rollback(ctx)
|
|
}
|
|
|
|
func (a *App) attachment(ctx context.Context, id, user string) (Attachment, error) {
|
|
x := Attachment{}
|
|
var created, updated time.Time
|
|
var missingSince *time.Time
|
|
err := a.db.QueryRow(ctx, `SELECT a.id::text,a.chat_id::text,a.owner_id::text,a.filename,a.content_type,a.size_bytes,a.sha256,a.created_at,a.updated_at,a.missing_since,a.status,a.variants,a.error_code,a.result_version
|
|
FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id AND m.user_id=$2 WHERE a.id=$1`, id, user).Scan(&x.ID, &x.ChatID, &x.OwnerID, &x.Filename, &x.ContentType, &x.SizeBytes, &x.SHA256, &created, &updated, &missingSince, &x.Status, &x.Variants, &x.ErrorCode, &x.resultVersion)
|
|
if err != nil {
|
|
return x, dbError(err)
|
|
}
|
|
if missingSince != nil {
|
|
return x, unavailable()
|
|
}
|
|
x.CreatedAt = stamp(created)
|
|
x.UpdatedAt = stamp(updated)
|
|
return x, nil
|
|
}
|
|
|
|
type binaryContent struct {
|
|
attachment Attachment
|
|
object *minio.Object
|
|
}
|
|
|
|
func (a *App) content(ctx context.Context, r *http.Request, x Attachment) (binaryContent, error) {
|
|
q, err := url.ParseQuery(r.URL.RawQuery)
|
|
if err != nil || len(q) > 1 || len(q["variant"]) > 1 {
|
|
return binaryContent{}, invalid()
|
|
}
|
|
for k := range q {
|
|
if k != "variant" {
|
|
return binaryContent{}, invalid()
|
|
}
|
|
}
|
|
name := "original"
|
|
if values, ok := q["variant"]; ok {
|
|
name = values[0]
|
|
switch name {
|
|
case "original", "thumbnail", "crop", "optimized":
|
|
default:
|
|
return binaryContent{}, invalid()
|
|
}
|
|
}
|
|
if x.Status != "ready" {
|
|
return binaryContent{}, apiError{409, "conflict", "Attachment is not ready"}
|
|
}
|
|
if name != "original" {
|
|
found := false
|
|
for _, variant := range x.Variants {
|
|
if variant.Name == name {
|
|
x.ContentType, x.SizeBytes = variant.ContentType, variant.SizeBytes
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
return binaryContent{}, missing()
|
|
}
|
|
}
|
|
object, err := a.s3.GetObject(ctx, a.bucket, resultKey(x.ID, x.resultVersion, name), minio.GetObjectOptions{})
|
|
if err != nil {
|
|
return binaryContent{}, unavailable()
|
|
}
|
|
info, err := object.Stat()
|
|
if err != nil || info.Size != x.SizeBytes {
|
|
object.Close()
|
|
return binaryContent{}, unavailable()
|
|
}
|
|
return binaryContent{x, object}, nil
|
|
}
|
|
|
|
func writeContent(w http.ResponseWriter, b binaryContent) {
|
|
defer b.object.Close()
|
|
w.Header().Set("Content-Type", b.attachment.ContentType)
|
|
w.Header().Set("Content-Length", strconv.FormatInt(b.attachment.SizeBytes, 10))
|
|
w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": b.attachment.Filename}))
|
|
w.Header().Set("X-Content-Type-Options", "nosniff")
|
|
w.Header().Set("Cache-Control", "private, no-store")
|
|
w.WriteHeader(200)
|
|
if _, err := io.CopyN(w, b.object, b.attachment.SizeBytes); err != nil {
|
|
panic(http.ErrAbortHandler)
|
|
}
|
|
}
|