forked from mtusi2026/mtusi2026lab4
431 lines
11 KiB
Go
431 lines
11 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"errors"
|
|
"hash"
|
|
"image"
|
|
_ "image/jpeg"
|
|
_ "image/png"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
"unicode"
|
|
"unicode/utf8"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
const (
|
|
objectPrefix = "attachments/"
|
|
probeHead = 256 << 10
|
|
tailLen = 12
|
|
)
|
|
|
|
var (
|
|
pngMagic = []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}
|
|
pngTail = []byte{0, 0, 0, 0, 'I', 'E', 'N', 'D', 0xae, 0x42, 0x60, 0x82}
|
|
jpegMagic = []byte{0xff, 0xd8, 0xff}
|
|
jpegTail = []byte{0xff, 0xd9}
|
|
)
|
|
|
|
type limits struct {
|
|
maxBytes int64
|
|
maxPixels int64
|
|
uploadSlots int
|
|
}
|
|
|
|
type attachment struct {
|
|
ID, ChatID, OwnerID, Filename, ContentType, SHA256, Key string
|
|
Size int64
|
|
Created time.Time
|
|
}
|
|
|
|
func (x *attachment) json() map[string]any {
|
|
return map[string]any{
|
|
"id": x.ID,
|
|
"chat_id": x.ChatID,
|
|
"owner_id": x.OwnerID,
|
|
"filename": x.Filename,
|
|
"content_type": x.ContentType,
|
|
"size_bytes": x.Size,
|
|
"sha256": x.SHA256,
|
|
"status": "ready",
|
|
"variants": []any{},
|
|
"error_code": nil,
|
|
"created_at": formatTime(x.Created),
|
|
"updated_at": formatTime(x.Created),
|
|
}
|
|
}
|
|
|
|
type probe struct {
|
|
sum hash.Hash
|
|
size int64
|
|
head []byte
|
|
tail []byte
|
|
}
|
|
|
|
func newProbe() *probe {
|
|
return &probe{sum: sha256.New()}
|
|
}
|
|
|
|
func (p *probe) Write(b []byte) (int, error) {
|
|
p.sum.Write(b)
|
|
p.size += int64(len(b))
|
|
if room := probeHead - len(p.head); room > 0 {
|
|
if room > len(b) {
|
|
room = len(b)
|
|
}
|
|
p.head = append(p.head, b[:room]...)
|
|
}
|
|
p.tail = append(p.tail, b...)
|
|
if len(p.tail) > tailLen {
|
|
p.tail = append(p.tail[:0], p.tail[len(p.tail)-tailLen:]...)
|
|
}
|
|
return len(b), nil
|
|
}
|
|
|
|
func (p *probe) digest() string {
|
|
return hex.EncodeToString(p.sum.Sum(nil))
|
|
}
|
|
|
|
var (
|
|
errNotImage = errors.New("not an image")
|
|
errTooBig = errors.New("too large")
|
|
)
|
|
|
|
func (p *probe) verify(contentType string, maxPixels int64) error {
|
|
cfg, format, err := image.DecodeConfig(bytes.NewReader(p.head))
|
|
if err != nil {
|
|
return errNotImage
|
|
}
|
|
var tail []byte
|
|
switch contentType {
|
|
case "image/png":
|
|
if format != "png" {
|
|
return errNotImage
|
|
}
|
|
tail = pngTail
|
|
case "image/jpeg":
|
|
if format != "jpeg" {
|
|
return errNotImage
|
|
}
|
|
tail = jpegTail
|
|
}
|
|
if !bytes.HasSuffix(p.tail, tail) {
|
|
return errNotImage
|
|
}
|
|
if int64(cfg.Width)*int64(cfg.Height) > maxPixels {
|
|
return errTooBig
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func magicMatches(contentType string, head []byte) bool {
|
|
switch contentType {
|
|
case "image/png":
|
|
return bytes.HasPrefix(head, pngMagic)
|
|
case "image/jpeg":
|
|
return bytes.HasPrefix(head, jpegMagic)
|
|
}
|
|
return false
|
|
}
|
|
|
|
func validFilename(name string) bool {
|
|
n := utf8.RuneCountInString(name)
|
|
if n < 1 || n > 255 || !utf8.ValidString(name) {
|
|
return false
|
|
}
|
|
for _, r := range name {
|
|
if r == '/' || r == '\\' || r == 0 || unicode.IsControl(r) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
type trackedReader struct {
|
|
r io.Reader
|
|
err error
|
|
}
|
|
|
|
func (t *trackedReader) Read(b []byte) (int, error) {
|
|
n, err := t.r.Read(b)
|
|
if err != nil && err != io.EOF && t.err == nil {
|
|
t.err = err
|
|
}
|
|
return n, err
|
|
}
|
|
|
|
func isTooLarge(err error) bool {
|
|
var mbe *http.MaxBytesError
|
|
return errors.As(err, &mbe)
|
|
}
|
|
|
|
func (a *App) dropObject(key string) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
if err := a.store.Remove(ctx, key); err != nil {
|
|
a.log.Warn("object cleanup failed", "key", key, "err", err.Error())
|
|
}
|
|
}
|
|
|
|
func (a *App) handleUploadAttachment(w http.ResponseWriter, r *http.Request) {
|
|
reqID := reqIDFrom(r)
|
|
u, _ := a.authUser(w, r)
|
|
if u == nil {
|
|
return
|
|
}
|
|
chatID := r.PathValue("chat_id")
|
|
if !isUUID(chatID) {
|
|
a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id")
|
|
return
|
|
}
|
|
ok, err := a.isMember(r.Context(), chatID, u.ID)
|
|
if err != nil {
|
|
a.writeErr(w, reqID, 503, "unavailable", "database unavailable")
|
|
return
|
|
}
|
|
if !ok {
|
|
a.writeErr(w, reqID, 404, "not_found", "chat not found")
|
|
return
|
|
}
|
|
select {
|
|
case a.uploads <- struct{}{}:
|
|
defer func() { <-a.uploads }()
|
|
default:
|
|
w.Header().Set("Retry-After", "1")
|
|
a.writeErr(w, reqID, 429, "rate_limited", "too many concurrent uploads")
|
|
return
|
|
}
|
|
filename := r.URL.Query().Get("filename")
|
|
if !validFilename(filename) {
|
|
a.writeErr(w, reqID, 400, "invalid_request", "invalid filename")
|
|
return
|
|
}
|
|
contentType := strings.ToLower(strings.TrimSpace(strings.Split(r.Header.Get("Content-Type"), ";")[0]))
|
|
if contentType != "image/png" && contentType != "image/jpeg" {
|
|
a.writeErr(w, reqID, 415, "unsupported_media_type", "Content-Type must be image/png or image/jpeg")
|
|
return
|
|
}
|
|
if r.ContentLength > a.limits.maxBytes {
|
|
a.writeErr(w, reqID, 413, "payload_too_large", "file too large")
|
|
return
|
|
}
|
|
body := &trackedReader{r: http.MaxBytesReader(w, r.Body, a.limits.maxBytes)}
|
|
head := make([]byte, 16)
|
|
n, _ := io.ReadFull(body, head)
|
|
if body.err != nil {
|
|
if isTooLarge(body.err) {
|
|
a.writeErr(w, reqID, 413, "payload_too_large", "file too large")
|
|
} else {
|
|
a.writeErr(w, reqID, 400, "invalid_request", "upload interrupted")
|
|
}
|
|
return
|
|
}
|
|
head = head[:n]
|
|
if !magicMatches(contentType, head) {
|
|
a.writeErr(w, reqID, 415, "unsupported_media_type", "content does not match image type")
|
|
return
|
|
}
|
|
|
|
id := newUUID()
|
|
key := objectPrefix + id
|
|
pr := newProbe()
|
|
size := r.ContentLength
|
|
if size < 0 {
|
|
size = -1
|
|
}
|
|
src := io.TeeReader(io.MultiReader(bytes.NewReader(head), body), pr)
|
|
putCtx, cancel := context.WithTimeout(r.Context(), 60*time.Second)
|
|
defer cancel()
|
|
if err := a.store.Put(putCtx, key, src, size, contentType); err != nil {
|
|
a.dropObject(key)
|
|
switch {
|
|
case isTooLarge(body.err):
|
|
a.writeErr(w, reqID, 413, "payload_too_large", "file too large")
|
|
case body.err != nil:
|
|
a.writeErr(w, reqID, 400, "invalid_request", "upload interrupted")
|
|
default:
|
|
a.log.Error("s3 put failed", "request_id", reqID, "err", err.Error())
|
|
a.writeErr(w, reqID, 503, "unavailable", "storage unavailable")
|
|
}
|
|
return
|
|
}
|
|
if err := pr.verify(contentType, a.limits.maxPixels); err != nil {
|
|
a.dropObject(key)
|
|
if err == errTooBig {
|
|
a.writeErr(w, reqID, 413, "payload_too_large", "image too large")
|
|
} else {
|
|
a.writeErr(w, reqID, 415, "unsupported_media_type", "invalid image")
|
|
}
|
|
return
|
|
}
|
|
|
|
att := &attachment{
|
|
ID: id, ChatID: chatID, OwnerID: u.ID, Filename: filename, ContentType: contentType,
|
|
SHA256: pr.digest(), Key: key, Size: pr.size, Created: nowUTC(),
|
|
}
|
|
ctx, cancelDB := dbCtx(r.Context())
|
|
defer cancelDB()
|
|
_, err = a.db.Exec(ctx, `
|
|
insert into attachments(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, created_at)
|
|
values($1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, $9)`,
|
|
att.ID, att.ChatID, att.OwnerID, att.Filename, att.ContentType, att.Size, att.SHA256, att.Key, att.Created)
|
|
if err != nil {
|
|
a.dropObject(key)
|
|
a.writeErr(w, reqID, 503, "unavailable", "database unavailable")
|
|
return
|
|
}
|
|
a.writeJSON(w, 201, att.json())
|
|
}
|
|
|
|
func (a *App) loadAttachment(ctx context.Context, id, userID string) (*attachment, error) {
|
|
var x attachment
|
|
err := a.db.QueryRow(ctx, `
|
|
select a.id::text, a.chat_id::text, a.owner_id::text, a.filename, a.content_type,
|
|
a.size_bytes, a.sha256, a.object_key, a.created_at
|
|
from attachments a
|
|
join chat_members m on m.chat_id = a.chat_id and m.user_id = $2::uuid
|
|
where a.id = $1::uuid`, id, userID).
|
|
Scan(&x.ID, &x.ChatID, &x.OwnerID, &x.Filename, &x.ContentType, &x.Size, &x.SHA256, &x.Key, &x.Created)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &x, nil
|
|
}
|
|
|
|
func (a *App) attachmentFor(w http.ResponseWriter, r *http.Request) *attachment {
|
|
reqID := reqIDFrom(r)
|
|
u, _ := a.authUser(w, r)
|
|
if u == nil {
|
|
return nil
|
|
}
|
|
id := r.PathValue("attachment_id")
|
|
if !isUUID(id) {
|
|
a.writeErr(w, reqID, 400, "invalid_request", "invalid attachment_id")
|
|
return nil
|
|
}
|
|
ctx, cancel := dbCtx(r.Context())
|
|
defer cancel()
|
|
att, err := a.loadAttachment(ctx, id, u.ID)
|
|
if err != nil {
|
|
a.writeErr(w, reqID, 503, "unavailable", "database unavailable")
|
|
return nil
|
|
}
|
|
if att == nil {
|
|
a.writeErr(w, reqID, 404, "not_found", "attachment not found")
|
|
return nil
|
|
}
|
|
return att
|
|
}
|
|
|
|
func (a *App) handleGetAttachment(w http.ResponseWriter, r *http.Request) {
|
|
if att := a.attachmentFor(w, r); att != nil {
|
|
a.writeJSON(w, 200, att.json())
|
|
}
|
|
}
|
|
|
|
func (a *App) handleDownloadAttachment(w http.ResponseWriter, r *http.Request) {
|
|
reqID := reqIDFrom(r)
|
|
if v := r.URL.Query().Get("variant"); v != "" && v != "original" {
|
|
a.writeErr(w, reqID, 400, "invalid_request", "invalid variant")
|
|
return
|
|
}
|
|
att := a.attachmentFor(w, r)
|
|
if att == nil {
|
|
return
|
|
}
|
|
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second)
|
|
defer cancel()
|
|
obj, info, err := a.store.Get(ctx, att.Key)
|
|
if err != nil {
|
|
if isNoSuchKey(err) {
|
|
a.writeErr(w, reqID, 404, "not_found", "attachment not found")
|
|
return
|
|
}
|
|
a.log.Error("s3 get failed", "request_id", reqID, "err", err.Error())
|
|
a.writeErr(w, reqID, 503, "unavailable", "storage unavailable")
|
|
return
|
|
}
|
|
defer obj.Close()
|
|
first := make([]byte, 32<<10)
|
|
n, err := io.ReadFull(obj, first)
|
|
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
|
a.log.Error("s3 read failed", "request_id", reqID, "err", err.Error())
|
|
a.writeErr(w, reqID, 503, "unavailable", "storage unavailable")
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", att.ContentType)
|
|
w.Header().Set("Content-Length", strconv.FormatInt(info.Size, 10))
|
|
w.Header().Set("X-Content-Type-Options", "nosniff")
|
|
w.WriteHeader(200)
|
|
_, _ = w.Write(first[:n])
|
|
_, _ = io.Copy(w, obj)
|
|
}
|
|
|
|
func cleanupOrphans(ctx context.Context, db *pgxpool.Pool, store *Store, grace time.Duration, log *slog.Logger) (int, error) {
|
|
cutoff := time.Now().Add(-grace)
|
|
removed := 0
|
|
flush := func(keys []string) error {
|
|
if len(keys) == 0 {
|
|
return nil
|
|
}
|
|
rows, err := db.Query(ctx, `select object_key from attachments where object_key = any($1)`, keys)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
known := map[string]bool{}
|
|
for rows.Next() {
|
|
var k string
|
|
if err := rows.Scan(&k); err != nil {
|
|
rows.Close()
|
|
return err
|
|
}
|
|
known[k] = true
|
|
}
|
|
rows.Close()
|
|
if err := rows.Err(); err != nil {
|
|
return err
|
|
}
|
|
for _, k := range keys {
|
|
if known[k] {
|
|
continue
|
|
}
|
|
if err := store.Remove(ctx, k); err != nil {
|
|
return err
|
|
}
|
|
log.Info("orphan object removed", "key", k)
|
|
removed++
|
|
}
|
|
return nil
|
|
}
|
|
var batch []string
|
|
for obj := range store.List(ctx) {
|
|
if obj.Err != nil {
|
|
return removed, obj.Err
|
|
}
|
|
if obj.LastModified.After(cutoff) {
|
|
continue
|
|
}
|
|
batch = append(batch, obj.Key)
|
|
if len(batch) == 500 {
|
|
if err := flush(batch); err != nil {
|
|
return removed, err
|
|
}
|
|
batch = batch[:0]
|
|
}
|
|
}
|
|
return removed, flush(batch)
|
|
}
|