316 lines
9.4 KiB
Go
316 lines
9.4 KiB
Go
package messenger
|
|
|
|
import (
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"image"
|
|
_ "image/jpeg"
|
|
_ "image/png"
|
|
"io"
|
|
"mime"
|
|
"mtusi/messenger/internal/telemetry"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/minio/minio-go/v7"
|
|
"github.com/minio/minio-go/v7/pkg/credentials"
|
|
)
|
|
|
|
const maxUpload = 10 << 20
|
|
const maxPixels = 40000000
|
|
|
|
type ObjectStore struct {
|
|
Client *minio.Client
|
|
Bucket string
|
|
}
|
|
|
|
func NewObjectStore(endpoint, bucket, access, secret string) (*ObjectStore, error) {
|
|
u, e := url.Parse(endpoint)
|
|
if e != nil || u.Host == "" || (u.Scheme != "http" && u.Scheme != "https") {
|
|
return nil, fmt.Errorf("invalid S3 endpoint")
|
|
}
|
|
transport := http.DefaultTransport.(*http.Transport).Clone()
|
|
transport.MaxConnsPerHost = 8
|
|
transport.ResponseHeaderTimeout = 5 * time.Second
|
|
client, e := minio.New(u.Host, &minio.Options{Creds: credentials.NewStaticV4(access, secret, ""), Secure: u.Scheme == "https", Region: "us-east-1", Transport: telemetry.Transport(transport), MaxRetries: 1})
|
|
if e != nil {
|
|
return nil, fmt.Errorf("invalid S3 configuration")
|
|
}
|
|
return &ObjectStore{client, bucket}, nil
|
|
}
|
|
func (o *ObjectStore) Ready(ctx context.Context) bool {
|
|
ok, e := o.Client.BucketExists(ctx, o.Bucket)
|
|
return e == nil && ok
|
|
}
|
|
func (s *Server) SetObjectStore(o *ObjectStore) { s.objects = o; s.uploads = make(chan struct{}, 1) }
|
|
|
|
type Attachment struct {
|
|
ID string `json:"id"`
|
|
ChatID string `json:"chat_id"`
|
|
OwnerID string `json:"owner_id"`
|
|
Filename string `json:"filename"`
|
|
ContentType string `json:"content_type"`
|
|
Size int64 `json:"size_bytes"`
|
|
SHA256 string `json:"sha256"`
|
|
Status string `json:"status"`
|
|
Variants []Variant `json:"variants"`
|
|
ErrorCode *string `json:"error_code"`
|
|
CreatedAt string `json:"created_at"`
|
|
UpdatedAt string `json:"updated_at"`
|
|
}
|
|
|
|
const attachmentColumns = `a.id::text,a.chat_id::text,a.owner_id::text,a.filename,a.content_type,a.size_bytes,a.sha256,a.created_at,a.updated_at,a.object_key,a.status,a.error_code,a.variants`
|
|
|
|
func scanAttachment(row scanner) (Attachment, string, error) {
|
|
a := Attachment{Status: "ready", Variants: []Variant{}}
|
|
var c, u time.Time
|
|
var key string
|
|
var variants []byte
|
|
e := row.Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.Size, &a.SHA256, &c, &u, &key, &a.Status, &a.ErrorCode, &variants)
|
|
if e == nil {
|
|
e = json.Unmarshal(variants, &a.Variants)
|
|
}
|
|
a.CreatedAt = stamp(c)
|
|
a.UpdatedAt = stamp(u)
|
|
return a, key, e
|
|
}
|
|
func validFilename(name string) bool {
|
|
if !validText(name, 255) {
|
|
return false
|
|
}
|
|
for _, c := range name {
|
|
if c == '/' || c == '\\' || unicode.IsControl(c) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// A temporary capped spool avoids holding a compressed file in RAM; it is
|
|
// removed on every exit. DecodeConfig bounds pixels before full validation.
|
|
func (s *Server) uploadAttachment(w http.ResponseWriter, r *http.Request, uid, cid string) {
|
|
q, e := url.ParseQuery(r.URL.RawQuery)
|
|
if e != nil || len(q["filename"]) != 1 || !validFilename(q.Get("filename")) {
|
|
bad(w)
|
|
return
|
|
}
|
|
contentType, _, e := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
|
if e != nil || (contentType != "image/png" && contentType != "image/jpeg") {
|
|
fail(w, 415, "unsupported_media_type", "Expected PNG or JPEG")
|
|
return
|
|
}
|
|
if r.ContentLength > maxUpload {
|
|
_, _ = io.Copy(io.Discard, io.LimitReader(r.Body, maxUpload+1))
|
|
fail(w, 413, "payload_too_large", "Maximum upload is 10 MiB")
|
|
return
|
|
}
|
|
select {
|
|
case s.uploads <- struct{}{}:
|
|
defer func() { <-s.uploads }()
|
|
case <-r.Context().Done():
|
|
unavailable(w)
|
|
return
|
|
}
|
|
f, e := os.CreateTemp("", "messenger-upload-*")
|
|
if e != nil {
|
|
unavailable(w)
|
|
return
|
|
}
|
|
defer os.Remove(f.Name())
|
|
defer f.Close()
|
|
hash := sha256.New()
|
|
n, e := io.Copy(io.MultiWriter(f, hash), http.MaxBytesReader(w, r.Body, maxUpload))
|
|
if e != nil {
|
|
var over *http.MaxBytesError
|
|
if errors.As(e, &over) {
|
|
fail(w, 413, "payload_too_large", "Maximum upload is 10 MiB")
|
|
} else {
|
|
bad(w)
|
|
}
|
|
return
|
|
}
|
|
_, _ = f.Seek(0, io.SeekStart)
|
|
config, format, e := image.DecodeConfig(f)
|
|
if e != nil || "image/"+format != contentType {
|
|
fail(w, 415, "unsupported_media_type", "Invalid image or MIME mismatch")
|
|
return
|
|
}
|
|
if config.Width < 1 || config.Height < 1 || int64(config.Width)*int64(config.Height) > maxPixels {
|
|
fail(w, 413, "payload_too_large", "Maximum image is 40 million pixels")
|
|
return
|
|
}
|
|
_, _ = f.Seek(0, io.SeekStart)
|
|
if !validateImage(f) {
|
|
fail(w, 415, "unsupported_media_type", "Invalid image")
|
|
return
|
|
}
|
|
_, _ = f.Seek(0, io.SeekStart)
|
|
id := uuid()
|
|
key := "staging/" + id
|
|
ctx := r.Context()
|
|
// Journal survives a crash between PUT and metadata COMMIT.
|
|
if _, e = s.db.Exec(ctx, "INSERT INTO upload_intents(object_key) VALUES($1)", key); e != nil {
|
|
unavailable(w)
|
|
return
|
|
}
|
|
tx, e := s.db.Begin(ctx)
|
|
if e != nil {
|
|
unavailable(w)
|
|
return
|
|
}
|
|
defer tx.Rollback(context.Background())
|
|
if _, e = tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1,4))", key); e != nil {
|
|
unavailable(w)
|
|
return
|
|
}
|
|
// Disable multipart: the bounded spool is <=10MiB, hence one atomic object PUT.
|
|
_, e = s.objects.Client.PutObject(ctx, s.objects.Bucket, key, f, n, minio.PutObjectOptions{ContentType: contentType, DisableMultipart: true})
|
|
if e != nil {
|
|
unavailable(w)
|
|
return
|
|
}
|
|
a, _, e := scanAttachment(tx.QueryRow(ctx, `INSERT INTO attachments AS a(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,status,staging_key) VALUES($1,$2,$3,$4,$5,$6,$7,$8,'queued',$9) RETURNING `+attachmentColumns, id, cid, uid, q.Get("filename"), "originals/"+id+"/v1/original", contentType, n, hex.EncodeToString(hash.Sum(nil)), key))
|
|
if e != nil {
|
|
unavailable(w)
|
|
return
|
|
}
|
|
// Admission and intent share a transaction; no unbounded accepted backlog.
|
|
if _, e = tx.Exec(ctx, "SELECT pg_advisory_xact_lock(2026005)"); e != nil {
|
|
unavailable(w)
|
|
return
|
|
}
|
|
var pending int
|
|
if e = tx.QueryRow(ctx, "SELECT count(*) FROM pipeline_jobs WHERE finished_at IS NULL").Scan(&pending); e != nil {
|
|
unavailable(w)
|
|
return
|
|
}
|
|
if pending >= 1000 {
|
|
w.Header().Set("Retry-After", "5")
|
|
fail(w, 429, "rate_limited", "Processing backlog is full")
|
|
return
|
|
}
|
|
parent, state := telemetry.Inject(ctx)
|
|
if _, e = tx.Exec(ctx, `INSERT INTO pipeline_jobs(job_id,attachment_id,pipeline_version,traceparent,tracestate) VALUES($1,$2,1,$3,$4)`, uuid(), id, parent, state); e != nil {
|
|
unavailable(w)
|
|
return
|
|
}
|
|
if _, e = tx.Exec(ctx, "DELETE FROM upload_intents WHERE object_key=$1", key); e != nil {
|
|
unavailable(w)
|
|
return
|
|
}
|
|
if tx.Commit(ctx) != nil {
|
|
unavailable(w)
|
|
return
|
|
}
|
|
reply(w, 202, a)
|
|
}
|
|
func validateImage(r io.Reader) bool { _, _, e := image.Decode(r); return e == nil }
|
|
|
|
func (s *Server) serveAttachment(w http.ResponseWriter, r *http.Request, uid string, parts []string) {
|
|
if r.Method != "GET" || (len(parts) != 4 && len(parts) != 5) || (len(parts) == 5 && parts[4] != "content") {
|
|
missing(w)
|
|
return
|
|
}
|
|
id := strings.ToLower(parts[3])
|
|
if !uuidRE.MatchString(id) {
|
|
bad(w)
|
|
return
|
|
}
|
|
a, key, e := scanAttachment(s.db.QueryRow(r.Context(), `SELECT `+attachmentColumns+` FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id AND m.user_id=$2 WHERE a.id=$1`, id, uid))
|
|
if errors.Is(e, pgx.ErrNoRows) {
|
|
missing(w)
|
|
return
|
|
}
|
|
if e != nil {
|
|
unavailable(w)
|
|
return
|
|
}
|
|
if len(parts) == 4 {
|
|
reply(w, 200, a)
|
|
return
|
|
}
|
|
q, e := url.ParseQuery(r.URL.RawQuery)
|
|
if e != nil || len(q["variant"]) > 1 {
|
|
bad(w)
|
|
return
|
|
}
|
|
if a.Status != "ready" {
|
|
fail(w, 409, "conflict", "Attachment is not ready")
|
|
return
|
|
}
|
|
variant := q.Get("variant")
|
|
if variant != "" && variant != "original" {
|
|
found := false
|
|
for _, v := range a.Variants {
|
|
if v.Name == variant {
|
|
a.ContentType = v.ContentType
|
|
a.Size = v.Size
|
|
key = strings.TrimSuffix(key, "original") + variant
|
|
found = true
|
|
break
|
|
}
|
|
}
|
|
if !found {
|
|
missing(w)
|
|
return
|
|
}
|
|
}
|
|
object, e := s.objects.Client.GetObject(r.Context(), s.objects.Bucket, key, minio.GetObjectOptions{})
|
|
if e != nil {
|
|
unavailable(w)
|
|
return
|
|
}
|
|
defer object.Close()
|
|
info, e := object.Stat()
|
|
if e != nil || info.Size != a.Size {
|
|
unavailable(w)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", a.ContentType)
|
|
w.Header().Set("Content-Length", fmt.Sprint(a.Size))
|
|
w.Header().Set("X-Content-Type-Options", "nosniff")
|
|
w.Header().Set("Content-Disposition", mime.FormatMediaType("attachment", map[string]string{"filename": a.Filename}))
|
|
w.WriteHeader(200)
|
|
_, _ = io.Copy(w, object)
|
|
}
|
|
|
|
// Normalize optional attachments before hashing; JSON key ordering is irrelevant.
|
|
func parseMessage(w http.ResponseWriter, r *http.Request) (string, []string, bool) {
|
|
var in struct {
|
|
Text *string `json:"text"`
|
|
Attachments []string `json:"attachment_ids"`
|
|
}
|
|
// strict decoder accepts an explicitly optional field (suffix '?').
|
|
if !decode(w, r, &in, "text", "attachment_ids?") {
|
|
return "", nil, false
|
|
}
|
|
if in.Text == nil || !utf8.ValidString(*in.Text) || utf8.RuneCountInString(*in.Text) > 4000 || len(in.Attachments) > 10 || (strings.TrimSpace(*in.Text) == "" && len(in.Attachments) == 0) {
|
|
bad(w)
|
|
return "", nil, false
|
|
}
|
|
seen := map[string]bool{}
|
|
for i, id := range in.Attachments {
|
|
id = strings.ToLower(id)
|
|
if !uuidRE.MatchString(id) || seen[id] {
|
|
bad(w)
|
|
return "", nil, false
|
|
}
|
|
seen[id] = true
|
|
in.Attachments[i] = id
|
|
}
|
|
if in.Attachments == nil {
|
|
in.Attachments = []string{}
|
|
}
|
|
return *in.Text, in.Attachments, true
|
|
}
|