Files
mtusi2026lab5-fork/attachments.go
T

531 lines
14 KiB
Go

package main
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"hash"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"log/slog"
"net/http"
"os"
"strconv"
"strings"
"time"
"unicode"
"unicode/utf8"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
const (
objectPrefix = "attachments/"
stagingPrefix = "staging/"
probeHead = 256 << 10
tailLen = 12
)
var (
pngMagic = []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}
pngTail = []byte{0, 0, 0, 0, 'I', 'E', 'N', 'D', 0xae, 0x42, 0x60, 0x82}
jpegMagic = []byte{0xff, 0xd8, 0xff}
jpegTail = []byte{0xff, 0xd9}
)
type limits struct {
maxBytes int64
maxPixels int64
uploadSlots int
maxAttempts int
}
type attachment struct {
ID, ChatID, OwnerID, Filename, ContentType, SHA256, Key, Status string
ErrorCode *string
Size int64
Created, Updated time.Time
Variants []variant
}
type variant struct {
Name string `json:"name"`
ContentType string `json:"content_type"`
Size int64 `json:"size_bytes"`
Width int `json:"width"`
Height int `json:"height"`
SHA256 string `json:"sha256"`
key string
}
func (x *attachment) json() map[string]any {
variants := x.Variants
if variants == nil {
variants = []variant{}
}
return map[string]any{
"id": x.ID,
"chat_id": x.ChatID,
"owner_id": x.OwnerID,
"filename": x.Filename,
"content_type": x.ContentType,
"size_bytes": x.Size,
"sha256": x.SHA256,
"status": x.Status,
"variants": variants,
"error_code": x.ErrorCode,
"created_at": formatTime(x.Created),
"updated_at": formatTime(x.Updated),
}
}
type probe struct {
sum hash.Hash
size int64
head []byte
tail []byte
}
func newProbe() *probe {
return &probe{sum: sha256.New()}
}
func (p *probe) Write(b []byte) (int, error) {
p.sum.Write(b)
p.size += int64(len(b))
if room := probeHead - len(p.head); room > 0 {
if room > len(b) {
room = len(b)
}
p.head = append(p.head, b[:room]...)
}
p.tail = append(p.tail, b...)
if len(p.tail) > tailLen {
p.tail = append(p.tail[:0], p.tail[len(p.tail)-tailLen:]...)
}
return len(b), nil
}
func (p *probe) digest() string {
return hex.EncodeToString(p.sum.Sum(nil))
}
var (
errNotImage = errors.New("not an image")
errTooBig = errors.New("too large")
)
func (p *probe) verify(contentType string, maxPixels int64) error {
cfg, format, err := image.DecodeConfig(bytes.NewReader(p.head))
if err != nil {
return errNotImage
}
var tail []byte
switch contentType {
case "image/png":
if format != "png" {
return errNotImage
}
tail = pngTail
case "image/jpeg":
if format != "jpeg" {
return errNotImage
}
tail = jpegTail
}
if !bytes.HasSuffix(p.tail, tail) {
return errNotImage
}
if int64(cfg.Width)*int64(cfg.Height) > maxPixels {
return errTooBig
}
return nil
}
func magicMatches(contentType string, head []byte) bool {
switch contentType {
case "image/png":
return bytes.HasPrefix(head, pngMagic)
case "image/jpeg":
return bytes.HasPrefix(head, jpegMagic)
}
return false
}
func validFilename(name string) bool {
n := utf8.RuneCountInString(name)
if n < 1 || n > 255 || !utf8.ValidString(name) {
return false
}
for _, r := range name {
if r == '/' || r == '\\' || r == 0 || unicode.IsControl(r) {
return false
}
}
return true
}
type trackedReader struct {
r io.Reader
err error
}
func (t *trackedReader) Read(b []byte) (int, error) {
n, err := t.r.Read(b)
if err != nil && err != io.EOF && t.err == nil {
t.err = err
}
return n, err
}
func isTooLarge(err error) bool {
var mbe *http.MaxBytesError
return errors.As(err, &mbe)
}
func (a *App) dropObject(key string) {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := a.store.Remove(ctx, key); err != nil {
a.log.Warn("object cleanup failed", "key", key, "err", err.Error())
}
}
func (a *App) handleUploadAttachment(w http.ResponseWriter, r *http.Request) {
reqID := reqIDFrom(r)
u, _ := a.authUser(w, r)
if u == nil {
return
}
chatID := r.PathValue("chat_id")
if !isUUID(chatID) {
a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id")
return
}
ok, err := a.isMember(r.Context(), chatID, u.ID)
if err != nil {
a.writeErr(w, reqID, 503, "unavailable", "database unavailable")
return
}
if !ok {
a.writeErr(w, reqID, 404, "not_found", "chat not found")
return
}
select {
case a.uploads <- struct{}{}:
defer func() { <-a.uploads }()
default:
w.Header().Set("Retry-After", "1")
a.writeErr(w, reqID, 429, "rate_limited", "too many concurrent uploads")
return
}
filename := r.URL.Query().Get("filename")
if !validFilename(filename) {
a.writeErr(w, reqID, 400, "invalid_request", "invalid filename")
return
}
contentType := strings.ToLower(strings.TrimSpace(strings.Split(r.Header.Get("Content-Type"), ";")[0]))
if contentType != "image/png" && contentType != "image/jpeg" {
a.writeErr(w, reqID, 415, "unsupported_media_type", "Content-Type must be image/png or image/jpeg")
return
}
if r.ContentLength > a.limits.maxBytes {
a.writeErr(w, reqID, 413, "payload_too_large", "file too large")
return
}
body := &trackedReader{r: http.MaxBytesReader(w, r.Body, a.limits.maxBytes)}
head := make([]byte, 16)
n, _ := io.ReadFull(body, head)
if body.err != nil {
if isTooLarge(body.err) {
a.writeErr(w, reqID, 413, "payload_too_large", "file too large")
} else {
a.writeErr(w, reqID, 400, "invalid_request", "upload interrupted")
}
return
}
head = head[:n]
if !magicMatches(contentType, head) {
a.writeErr(w, reqID, 415, "unsupported_media_type", "content does not match image type")
return
}
id := newUUID()
key := stagingPrefix + id
pr := newProbe()
size := r.ContentLength
if size < 0 {
size = -1
}
src := io.TeeReader(io.MultiReader(bytes.NewReader(head), body), pr)
putCtx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
defer cancel()
if err := a.store.Put(putCtx, key, src, size, contentType); err != nil {
a.dropObject(key)
switch {
case isTooLarge(body.err):
a.writeErr(w, reqID, 413, "payload_too_large", "file too large")
case body.err != nil:
a.writeErr(w, reqID, 400, "invalid_request", "upload interrupted")
default:
a.log.Error("s3 put failed", "request_id", reqID, "err", err.Error())
a.writeErr(w, reqID, 503, "unavailable", "storage unavailable")
}
return
}
if err := pr.verify(contentType, a.limits.maxPixels); err != nil {
a.dropObject(key)
if err == errTooBig {
a.writeErr(w, reqID, 413, "payload_too_large", "image too large")
} else {
a.writeErr(w, reqID, 415, "unsupported_media_type", "invalid image")
}
return
}
now := nowUTC()
att := &attachment{
ID: id, ChatID: chatID, OwnerID: u.ID, Filename: filename, ContentType: contentType,
SHA256: pr.digest(), Size: pr.size, Status: "queued", Created: now, Updated: now,
}
if err := a.enqueue(r.Context(), att, key); err != nil {
a.dropObject(key)
a.log.Error("enqueue failed", "request_id", reqID, "err", err.Error())
a.writeErr(w, reqID, 503, "unavailable", "database unavailable")
return
}
if a.fault == "after_commit" {
a.log.Error("fault injected", "point", "after_commit", "attachment_id", att.ID)
os.Exit(3)
}
a.writeJSON(w, 202, att.json())
}
func (a *App) enqueue(ctx context.Context, att *attachment, key string) error {
ctx, cancel := dbCtx(ctx)
defer cancel()
tx, err := a.db.Begin(ctx)
if err != nil {
return err
}
defer tx.Rollback(ctx)
if _, err := tx.Exec(ctx, `
insert into attachments(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key, status, created_at, updated_at)
values($1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, 'queued', $9, $9)`,
att.ID, att.ChatID, att.OwnerID, att.Filename, att.ContentType, att.Size, att.SHA256, key, att.Created); err != nil {
return err
}
jobID := newUUID()
if _, err := tx.Exec(ctx, `
insert into jobs(id, attachment_id, pipeline_version, status, attempt, max_attempts, next_attempt_at, created_at, updated_at)
values($1::uuid, $2::uuid, $3, 'queued', 1, $4, $5, $5, $5)`,
jobID, att.ID, pipelineVersion, a.limits.maxAttempts, att.Created); err != nil {
return err
}
if _, err := tx.Exec(ctx, `
insert into outbox(job_id, attempt, available_at, created_at) values($1::uuid, 1, $2, $2)`, jobID, att.Created); err != nil {
return err
}
return tx.Commit(ctx)
}
func (a *App) loadAttachment(ctx context.Context, id, userID string) (*attachment, error) {
var x attachment
err := a.db.QueryRow(ctx, `
select a.id::text, a.chat_id::text, a.owner_id::text, a.filename, a.content_type,
a.size_bytes, a.sha256, coalesce(a.object_key, ''), a.status, a.error_code, a.created_at, a.updated_at
from attachments a
join chat_members m on m.chat_id = a.chat_id and m.user_id = $2::uuid
where a.id = $1::uuid`, id, userID).
Scan(&x.ID, &x.ChatID, &x.OwnerID, &x.Filename, &x.ContentType, &x.Size, &x.SHA256, &x.Key, &x.Status, &x.ErrorCode, &x.Created, &x.Updated)
if errors.Is(err, pgx.ErrNoRows) {
return nil, nil
}
if err != nil {
return nil, err
}
if x.Status != "ready" {
return &x, nil
}
rows, err := a.db.Query(ctx, `
select name, content_type, size_bytes, width, height, sha256, object_key
from attachment_variants where attachment_id = $1::uuid and pipeline_version = $2
order by array_position(array['thumbnail', 'crop', 'optimized'], name)`, id, pipelineVersion)
if err != nil {
return nil, err
}
x.Variants, err = pgx.CollectRows(rows, func(row pgx.CollectableRow) (variant, error) {
var v variant
return v, row.Scan(&v.Name, &v.ContentType, &v.Size, &v.Width, &v.Height, &v.SHA256, &v.key)
})
if err != nil {
return nil, err
}
return &x, nil
}
func (a *App) attachmentFor(w http.ResponseWriter, r *http.Request) *attachment {
reqID := reqIDFrom(r)
u, _ := a.authUser(w, r)
if u == nil {
return nil
}
id := r.PathValue("attachment_id")
if !isUUID(id) {
a.writeErr(w, reqID, 400, "invalid_request", "invalid attachment_id")
return nil
}
ctx, cancel := dbCtx(r.Context())
defer cancel()
att, err := a.loadAttachment(ctx, id, u.ID)
if err != nil {
a.writeErr(w, reqID, 503, "unavailable", "database unavailable")
return nil
}
if att == nil {
a.writeErr(w, reqID, 404, "not_found", "attachment not found")
return nil
}
return att
}
func (a *App) handleGetAttachment(w http.ResponseWriter, r *http.Request) {
if att := a.attachmentFor(w, r); att != nil {
a.writeJSON(w, 200, att.json())
}
}
func (a *App) handleDownloadAttachment(w http.ResponseWriter, r *http.Request) {
reqID := reqIDFrom(r)
name := r.URL.Query().Get("variant")
if name == "" {
name = "original"
}
if name != "original" && name != "thumbnail" && name != "crop" && name != "optimized" {
a.writeErr(w, reqID, 400, "invalid_request", "invalid variant")
return
}
att := a.attachmentFor(w, r)
if att == nil {
return
}
if att.Status != "ready" {
a.writeErr(w, reqID, 409, "conflict", "attachment is not ready")
return
}
key, contentType := att.Key, att.ContentType
if name != "original" {
key = ""
for _, v := range att.Variants {
if v.Name == name {
key, contentType = v.key, v.ContentType
}
}
if key == "" {
a.writeErr(w, reqID, 404, "not_found", "variant not found")
return
}
}
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
defer cancel()
obj, info, err := a.store.Get(ctx, key)
if err != nil {
if isNoSuchKey(err) {
a.writeErr(w, reqID, 404, "not_found", "attachment not found")
return
}
a.log.Error("s3 get failed", "request_id", reqID, "err", err.Error())
a.writeErr(w, reqID, 503, "unavailable", "storage unavailable")
return
}
defer obj.Close()
first := make([]byte, 32<<10)
n, err := io.ReadFull(obj, first)
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
a.log.Error("s3 read failed", "request_id", reqID, "err", err.Error())
a.writeErr(w, reqID, 503, "unavailable", "storage unavailable")
return
}
w.Header().Set("Content-Type", contentType)
w.Header().Set("Content-Length", strconv.FormatInt(info.Size, 10))
w.Header().Set("X-Content-Type-Options", "nosniff")
w.WriteHeader(200)
_, _ = w.Write(first[:n])
_, _ = io.Copy(w, obj)
}
var keepQueries = map[string]string{
objectPrefix: `
select k from unnest($1::text[]) k
where exists(select 1 from attachments where object_key = k)
or exists(select 1 from attachment_variants where object_key = k)`,
stagingPrefix: `
select k from unnest($1::text[]) k
where exists(select 1 from attachments where staging_key = k and (status in ('queued', 'processing')
or (status = 'failed' and updated_at > now() - make_interval(secs => $2))))`,
}
func cleanupOrphans(ctx context.Context, db *pgxpool.Pool, store *Store, grace, retention time.Duration, log *slog.Logger) (int, error) {
cutoff := time.Now().Add(-grace)
removed := 0
flush := func(prefix string, keys []string) error {
if len(keys) == 0 {
return nil
}
args := []any{keys}
if prefix == stagingPrefix {
args = append(args, retention.Seconds())
}
rows, err := db.Query(ctx, keepQueries[prefix], args...)
if err != nil {
return err
}
kept, err := pgx.CollectRows(rows, pgx.RowTo[string])
if err != nil {
return err
}
known := map[string]bool{}
for _, k := range kept {
known[k] = true
}
for _, k := range keys {
if known[k] {
continue
}
if err := store.Remove(ctx, k); err != nil {
return err
}
log.Info("orphan object removed", "key", k)
removed++
}
return nil
}
for _, prefix := range []string{objectPrefix, stagingPrefix} {
var batch []string
for obj := range store.List(ctx, prefix) {
if obj.Err != nil {
return removed, obj.Err
}
if obj.LastModified.After(cutoff) {
continue
}
batch = append(batch, obj.Key)
if len(batch) == 500 {
if err := flush(prefix, batch); err != nil {
return removed, err
}
batch = batch[:0]
}
}
if err := flush(prefix, batch); err != nil {
return removed, err
}
}
return removed, nil
}