531 lines
14 KiB
Go
531 lines
14 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"hash"
|
|
"image"
|
|
_ "image/jpeg"
|
|
_ "image/png"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
const (
|
|
objectPrefix = "attachments/"
|
|
stagingPrefix = "staging/"
|
|
probeHead = 256 << 10
|
|
tailLen = 12
|
|
)
|
|
|
|
var (
|
|
pngMagic = []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}
|
|
pngTail = []byte{0, 0, 0, 0, 'I', 'E', 'N', 'D', 0xae, 0x42, 0x60, 0x82}
|
|
jpegMagic = []byte{0xff, 0xd8, 0xff}
|
|
jpegTail = []byte{0xff, 0xd9}
|
|
)
|
|
|
|
type limits struct {
|
|
maxBytes int64
|
|
maxPixels int64
|
|
uploadSlots int
|
|
maxAttempts int
|
|
}
|
|
|
|
type attachment struct {
|
|
ID, ChatID, OwnerID, Filename, ContentType, SHA256, Key, Status string
|
|
ErrorCode *string
|
|
Size int64
|
|
Created, Updated time.Time
|
|
Variants []variant
|
|
}
|
|
|
|
type variant struct {
|
|
Name string `json:"name"`
|
|
ContentType string `json:"content_type"`
|
|
Size int64 `json:"size_bytes"`
|
|
Width int `json:"width"`
|
|
Height int `json:"height"`
|
|
SHA256 string `json:"sha256"`
|
|
key string
|
|
}
|
|
|
|
func (x *attachment) json() map[string]any {
|
|
variants := x.Variants
|
|
if variants == nil {
|
|
variants = []variant{}
|
|
}
|
|
return map[string]any{
|
|
"id": x.ID,
|
|
"chat_id": x.ChatID,
|
|
"owner_id": x.OwnerID,
|
|
"filename": x.Filename,
|
|
"content_type": x.ContentType,
|
|
"size_bytes": x.Size,
|
|
"sha256": x.SHA256,
|
|
"status": x.Status,
|
|
"variants": variants,
|
|
"error_code": x.ErrorCode,
|
|
"created_at": formatTime(x.Created),
|
|
"updated_at": formatTime(x.Updated),
|
|
}
|
|
}
|
|
|
|
type probe struct {
|
|
sum hash.Hash
|
|
size int64
|
|
head []byte
|
|
tail []byte
|
|
}
|
|
|
|
func newProbe() *probe {
|
|
return &probe{sum: sha256.New()}
|
|
}
|
|
|
|
func (p *probe) Write(b []byte) (int, error) {
|
|
p.sum.Write(b)
|
|
p.size += int64(len(b))
|
|
if room := probeHead - len(p.head); room > 0 {
|
|
if room > len(b) {
|
|
room = len(b)
|
|
}
|
|
p.head = append(p.head, b[:room]...)
|
|
}
|
|
p.tail = append(p.tail, b...)
|
|
if len(p.tail) > tailLen {
|
|
p.tail = append(p.tail[:0], p.tail[len(p.tail)-tailLen:]...)
|
|
}
|
|
return len(b), nil
|
|
}
|
|
|
|
func (p *probe) digest() string {
|
|
return hex.EncodeToString(p.sum.Sum(nil))
|
|
}
|
|
|
|
var (
|
|
errNotImage = errors.New("not an image")
|
|
errTooBig = errors.New("too large")
|
|
)
|
|
|
|
func (p *probe) verify(contentType string, maxPixels int64) error {
|
|
cfg, format, err := image.DecodeConfig(bytes.NewReader(p.head))
|
|
if err != nil {
|
|
return errNotImage
|
|
}
|
|
var tail []byte
|
|
switch contentType {
|
|
case "image/png":
|
|
if format != "png" {
|
|
return errNotImage
|
|
}
|
|
tail = pngTail
|
|
case "image/jpeg":
|
|
if format != "jpeg" {
|
|
return errNotImage
|
|
}
|
|
tail = jpegTail
|
|
}
|
|
if !bytes.HasSuffix(p.tail, tail) {
|
|
return errNotImage
|
|
}
|
|
if int64(cfg.Width)*int64(cfg.Height) > maxPixels {
|
|
return errTooBig
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func magicMatches(contentType string, head []byte) bool {
|
|
switch contentType {
|
|
case "image/png":
|
|
return bytes.HasPrefix(head, pngMagic)
|
|
case "image/jpeg":
|
|
return bytes.HasPrefix(head, jpegMagic)
|
|
}
|
|
return false
|
|
}
|
|
|
|
func validFilename(name string) bool {
|
|
n := utf8.RuneCountInString(name)
|
|
if n < 1 || n > 255 || !utf8.ValidString(name) {
|
|
return false
|
|
}
|
|
for _, r := range name {
|
|
if r == '/' || r == '\\' || r == 0 || unicode.IsControl(r) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
type trackedReader struct {
|
|
r io.Reader
|
|
err error
|
|
}
|
|
|
|
func (t *trackedReader) Read(b []byte) (int, error) {
|
|
n, err := t.r.Read(b)
|
|
if err != nil && err != io.EOF && t.err == nil {
|
|
t.err = err
|
|
}
|
|
return n, err
|
|
}
|
|
|
|
func isTooLarge(err error) bool {
|
|
var mbe *http.MaxBytesError
|
|
return errors.As(err, &mbe)
|
|
}
|
|
|
|
func (a *App) dropObject(key string) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
if err := a.store.Remove(ctx, key); err != nil {
|
|
a.log.Warn("object cleanup failed", "key", key, "err", err.Error())
|
|
}
|
|
}
|
|
|
|
func (a *App) handleUploadAttachment(w http.ResponseWriter, r *http.Request) {
|
|
reqID := reqIDFrom(r)
|
|
u, _ := a.authUser(w, r)
|
|
if u == nil {
|
|
return
|
|
}
|
|
chatID := r.PathValue("chat_id")
|
|
if !isUUID(chatID) {
|
|
a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id")
|
|
return
|
|
}
|
|
ok, err := a.isMember(r.Context(), chatID, u.ID)
|
|
if err != nil {
|
|
a.writeErr(w, reqID, 503, "unavailable", "database unavailable")
|
|
return
|
|
}
|
|
if !ok {
|
|
a.writeErr(w, reqID, 404, "not_found", "chat not found")
|
|
return
|
|
}
|
|
select {
|
|
case a.uploads <- struct{}{}:
|
|
defer func() { <-a.uploads }()
|
|
default:
|
|
w.Header().Set("Retry-After", "1")
|
|
a.writeErr(w, reqID, 429, "rate_limited", "too many concurrent uploads")
|
|
return
|
|
}
|
|
filename := r.URL.Query().Get("filename")
|
|
if !validFilename(filename) {
|
|
a.writeErr(w, reqID, 400, "invalid_request", "invalid filename")
|
|
return
|
|
}
|
|
contentType := strings.ToLower(strings.TrimSpace(strings.Split(r.Header.Get("Content-Type"), ";")[0]))
|
|
if contentType != "image/png" && contentType != "image/jpeg" {
|
|
a.writeErr(w, reqID, 415, "unsupported_media_type", "Content-Type must be image/png or image/jpeg")
|
|
return
|
|
}
|
|
if r.ContentLength > a.limits.maxBytes {
|
|
a.writeErr(w, reqID, 413, "payload_too_large", "file too large")
|
|
return
|
|
}
|
|
body := &trackedReader{r: http.MaxBytesReader(w, r.Body, a.limits.maxBytes)}
|
|
head := make([]byte, 16)
|
|
n, _ := io.ReadFull(body, head)
|
|
if body.err != nil {
|
|
if isTooLarge(body.err) {
|
|
a.writeErr(w, reqID, 413, "payload_too_large", "file too large")
|
|
} else {
|
|
a.writeErr(w, reqID, 400, "invalid_request", "upload interrupted")
|
|
}
|
|
return
|
|
}
|
|
head = head[:n]
|
|
if !magicMatches(contentType, head) {
|
|
a.writeErr(w, reqID, 415, "unsupported_media_type", "content does not match image type")
|
|
return
|
|
}
|
|
|
|
id := newUUID()
|
|
key := stagingPrefix + id
|
|
pr := newProbe()
|
|
size := r.ContentLength
|
|
if size < 0 {
|
|
size = -1
|
|
}
|
|
src := io.TeeReader(io.MultiReader(bytes.NewReader(head), body), pr)
|
|
putCtx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
|
|
defer cancel()
|
|
if err := a.store.Put(putCtx, key, src, size, contentType); err != nil {
|
|
a.dropObject(key)
|
|
switch {
|
|
case isTooLarge(body.err):
|
|
a.writeErr(w, reqID, 413, "payload_too_large", "file too large")
|
|
case body.err != nil:
|
|
a.writeErr(w, reqID, 400, "invalid_request", "upload interrupted")
|
|
default:
|
|
a.log.Error("s3 put failed", "request_id", reqID, "err", err.Error())
|
|
a.writeErr(w, reqID, 503, "unavailable", "storage unavailable")
|
|
}
|
|
return
|
|
}
|
|
if err := pr.verify(contentType, a.limits.maxPixels); err != nil {
|
|
a.dropObject(key)
|
|
if err == errTooBig {
|
|
a.writeErr(w, reqID, 413, "payload_too_large", "image too large")
|
|
} else {
|
|
a.writeErr(w, reqID, 415, "unsupported_media_type", "invalid image")
|
|
}
|
|
return
|
|
}
|
|
|
|
now := nowUTC()
|
|
att := &attachment{
|
|
ID: id, ChatID: chatID, OwnerID: u.ID, Filename: filename, ContentType: contentType,
|
|
SHA256: pr.digest(), Size: pr.size, Status: "queued", Created: now, Updated: now,
|
|
}
|
|
if err := a.enqueue(r.Context(), att, key); err != nil {
|
|
a.dropObject(key)
|
|
a.log.Error("enqueue failed", "request_id", reqID, "err", err.Error())
|
|
a.writeErr(w, reqID, 503, "unavailable", "database unavailable")
|
|
return
|
|
}
|
|
if a.fault == "after_commit" {
|
|
a.log.Error("fault injected", "point", "after_commit", "attachment_id", att.ID)
|
|
os.Exit(3)
|
|
}
|
|
a.writeJSON(w, 202, att.json())
|
|
}
|
|
|
|
func (a *App) enqueue(ctx context.Context, att *attachment, key string) error {
|
|
ctx, cancel := dbCtx(ctx)
|
|
defer cancel()
|
|
tx, err := a.db.Begin(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer tx.Rollback(ctx)
|
|
if _, err := tx.Exec(ctx, `
|
|
insert into attachments(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key, status, created_at, updated_at)
|
|
values($1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, 'queued', $9, $9)`,
|
|
att.ID, att.ChatID, att.OwnerID, att.Filename, att.ContentType, att.Size, att.SHA256, key, att.Created); err != nil {
|
|
return err
|
|
}
|
|
jobID := newUUID()
|
|
if _, err := tx.Exec(ctx, `
|
|
insert into jobs(id, attachment_id, pipeline_version, status, attempt, max_attempts, next_attempt_at, created_at, updated_at)
|
|
values($1::uuid, $2::uuid, $3, 'queued', 1, $4, $5, $5, $5)`,
|
|
jobID, att.ID, pipelineVersion, a.limits.maxAttempts, att.Created); err != nil {
|
|
return err
|
|
}
|
|
if _, err := tx.Exec(ctx, `
|
|
insert into outbox(job_id, attempt, available_at, created_at) values($1::uuid, 1, $2, $2)`, jobID, att.Created); err != nil {
|
|
return err
|
|
}
|
|
return tx.Commit(ctx)
|
|
}
|
|
|
|
func (a *App) loadAttachment(ctx context.Context, id, userID string) (*attachment, error) {
|
|
var x attachment
|
|
err := a.db.QueryRow(ctx, `
|
|
select a.id::text, a.chat_id::text, a.owner_id::text, a.filename, a.content_type,
|
|
a.size_bytes, a.sha256, coalesce(a.object_key, ''), a.status, a.error_code, a.created_at, a.updated_at
|
|
from attachments a
|
|
join chat_members m on m.chat_id = a.chat_id and m.user_id = $2::uuid
|
|
where a.id = $1::uuid`, id, userID).
|
|
Scan(&x.ID, &x.ChatID, &x.OwnerID, &x.Filename, &x.ContentType, &x.Size, &x.SHA256, &x.Key, &x.Status, &x.ErrorCode, &x.Created, &x.Updated)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if x.Status != "ready" {
|
|
return &x, nil
|
|
}
|
|
rows, err := a.db.Query(ctx, `
|
|
select name, content_type, size_bytes, width, height, sha256, object_key
|
|
from attachment_variants where attachment_id = $1::uuid and pipeline_version = $2
|
|
order by array_position(array['thumbnail', 'crop', 'optimized'], name)`, id, pipelineVersion)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
x.Variants, err = pgx.CollectRows(rows, func(row pgx.CollectableRow) (variant, error) {
|
|
var v variant
|
|
return v, row.Scan(&v.Name, &v.ContentType, &v.Size, &v.Width, &v.Height, &v.SHA256, &v.key)
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &x, nil
|
|
}
|
|
|
|
func (a *App) attachmentFor(w http.ResponseWriter, r *http.Request) *attachment {
|
|
reqID := reqIDFrom(r)
|
|
u, _ := a.authUser(w, r)
|
|
if u == nil {
|
|
return nil
|
|
}
|
|
id := r.PathValue("attachment_id")
|
|
if !isUUID(id) {
|
|
a.writeErr(w, reqID, 400, "invalid_request", "invalid attachment_id")
|
|
return nil
|
|
}
|
|
ctx, cancel := dbCtx(r.Context())
|
|
defer cancel()
|
|
att, err := a.loadAttachment(ctx, id, u.ID)
|
|
if err != nil {
|
|
a.writeErr(w, reqID, 503, "unavailable", "database unavailable")
|
|
return nil
|
|
}
|
|
if att == nil {
|
|
a.writeErr(w, reqID, 404, "not_found", "attachment not found")
|
|
return nil
|
|
}
|
|
return att
|
|
}
|
|
|
|
func (a *App) handleGetAttachment(w http.ResponseWriter, r *http.Request) {
|
|
if att := a.attachmentFor(w, r); att != nil {
|
|
a.writeJSON(w, 200, att.json())
|
|
}
|
|
}
|
|
|
|
func (a *App) handleDownloadAttachment(w http.ResponseWriter, r *http.Request) {
|
|
reqID := reqIDFrom(r)
|
|
name := r.URL.Query().Get("variant")
|
|
if name == "" {
|
|
name = "original"
|
|
}
|
|
if name != "original" && name != "thumbnail" && name != "crop" && name != "optimized" {
|
|
a.writeErr(w, reqID, 400, "invalid_request", "invalid variant")
|
|
return
|
|
}
|
|
att := a.attachmentFor(w, r)
|
|
if att == nil {
|
|
return
|
|
}
|
|
if att.Status != "ready" {
|
|
a.writeErr(w, reqID, 409, "conflict", "attachment is not ready")
|
|
return
|
|
}
|
|
key, contentType := att.Key, att.ContentType
|
|
if name != "original" {
|
|
key = ""
|
|
for _, v := range att.Variants {
|
|
if v.Name == name {
|
|
key, contentType = v.key, v.ContentType
|
|
}
|
|
}
|
|
if key == "" {
|
|
a.writeErr(w, reqID, 404, "not_found", "variant not found")
|
|
return
|
|
}
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
|
|
defer cancel()
|
|
obj, info, err := a.store.Get(ctx, key)
|
|
if err != nil {
|
|
if isNoSuchKey(err) {
|
|
a.writeErr(w, reqID, 404, "not_found", "attachment not found")
|
|
return
|
|
}
|
|
a.log.Error("s3 get failed", "request_id", reqID, "err", err.Error())
|
|
a.writeErr(w, reqID, 503, "unavailable", "storage unavailable")
|
|
return
|
|
}
|
|
defer obj.Close()
|
|
first := make([]byte, 32<<10)
|
|
n, err := io.ReadFull(obj, first)
|
|
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
|
a.log.Error("s3 read failed", "request_id", reqID, "err", err.Error())
|
|
a.writeErr(w, reqID, 503, "unavailable", "storage unavailable")
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", contentType)
|
|
w.Header().Set("Content-Length", strconv.FormatInt(info.Size, 10))
|
|
w.Header().Set("X-Content-Type-Options", "nosniff")
|
|
w.WriteHeader(200)
|
|
_, _ = w.Write(first[:n])
|
|
_, _ = io.Copy(w, obj)
|
|
}
|
|
|
|
var keepQueries = map[string]string{
|
|
objectPrefix: `
|
|
select k from unnest($1::text[]) k
|
|
where exists(select 1 from attachments where object_key = k)
|
|
or exists(select 1 from attachment_variants where object_key = k)`,
|
|
stagingPrefix: `
|
|
select k from unnest($1::text[]) k
|
|
where exists(select 1 from attachments where staging_key = k and (status in ('queued', 'processing')
|
|
or (status = 'failed' and updated_at > now() - make_interval(secs => $2))))`,
|
|
}
|
|
|
|
func cleanupOrphans(ctx context.Context, db *pgxpool.Pool, store *Store, grace, retention time.Duration, log *slog.Logger) (int, error) {
|
|
cutoff := time.Now().Add(-grace)
|
|
removed := 0
|
|
flush := func(prefix string, keys []string) error {
|
|
if len(keys) == 0 {
|
|
return nil
|
|
}
|
|
args := []any{keys}
|
|
if prefix == stagingPrefix {
|
|
args = append(args, retention.Seconds())
|
|
}
|
|
rows, err := db.Query(ctx, keepQueries[prefix], args...)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
kept, err := pgx.CollectRows(rows, pgx.RowTo[string])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
known := map[string]bool{}
|
|
for _, k := range kept {
|
|
known[k] = true
|
|
}
|
|
for _, k := range keys {
|
|
if known[k] {
|
|
continue
|
|
}
|
|
if err := store.Remove(ctx, k); err != nil {
|
|
return err
|
|
}
|
|
log.Info("orphan object removed", "key", k)
|
|
removed++
|
|
}
|
|
return nil
|
|
}
|
|
for _, prefix := range []string{objectPrefix, stagingPrefix} {
|
|
var batch []string
|
|
for obj := range store.List(ctx, prefix) {
|
|
if obj.Err != nil {
|
|
return removed, obj.Err
|
|
}
|
|
if obj.LastModified.After(cutoff) {
|
|
continue
|
|
}
|
|
batch = append(batch, obj.Key)
|
|
if len(batch) == 500 {
|
|
if err := flush(prefix, batch); err != nil {
|
|
return removed, err
|
|
}
|
|
batch = batch[:0]
|
|
}
|
|
}
|
|
if err := flush(prefix, batch); err != nil {
|
|
return removed, err
|
|
}
|
|
}
|
|
return removed, nil
|
|
}
|