From bf8da2b7d92f59ba9c755d39e117a2ac274acc8d Mon Sep 17 00:00:00 2001 From: Rakshasa2 Date: Thu, 5 Nov 2026 13:37:38 +0300 Subject: [PATCH] lab4 - fixed --- docker-compose.yml | 40 ++++++---- src/cmd/server/main.go | 6 -- src/internal/handler/handler.go | 48 +++++++++++- src/internal/store/postgres.go | 129 +++++++++++++++++++++++++++++++- 4 files changed, 197 insertions(+), 26 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 50daf4e8..33ba9830 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -112,31 +112,39 @@ services: s3: image: rustfs/rustfs@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2 - command: ["server", "--address", ":9000"] + command: [ "server", "--address", ":9000" ] environment: RUSTFS_ADDRESS: ":9000" RUSTFS_VOLUMES: "/data" RUSTFS_REGION: "${S3_REGION:-us-east-1}" - - # Root/admin credentials (используются Go-кодом для создания bucket) - RUSTFS_ROOT_USER: "${S3_ACCESS_KEY_ID}" - RUSTFS_ROOT_PASSWORD: "${S3_SECRET_ACCESS_KEY}" - - # Console отключена для безопасности + # RustFS принимает root-учётные данные при инициализации + RUSTFS_ROOT_USER: "${S3_ROOT_USER}" + RUSTFS_ROOT_PASSWORD: "${S3_ROOT_PASSWORD}" RUSTFS_CONSOLE_ENABLE: "false" - RUSTFS_OBS_LOGGER_LEVEL: "warn" - volumes: - s3_data:/data - - deploy: - resources: - limits: - cpus: "0.50" - memory: 256M - restart: unless-stopped + s3-init: + image: docker.io/minio/mc:latest # Или тот образ, который у тебя успешно тянется + depends_on: + s3: + condition: service_started + environment: + S3_ENDPOINT: "${S3_ENDPOINT:-http://s3:9000}" + S3_BUCKET: "${S3_BUCKET:-messenger-attachments}" + # Скрипт использует root-ключи для создания bucket и ограничения прав API-пользователя + S3_ROOT_USER: "${S3_ROOT_USER}" + S3_ROOT_PASSWORD: "${S3_ROOT_PASSWORD}" + S3_API_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID}" + S3_API_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY}" + volumes: + - ./infra/rustfs-bootstrap.sh:/bootstrap.sh:ro + entrypoint: [ "/bin/sh", "/bootstrap.sh" ] + restart: "no" + + + volumes: postgres_data: redis_data: diff --git a/src/cmd/server/main.go b/src/cmd/server/main.go index 2a806652..8a78c074 100644 --- a/src/cmd/server/main.go +++ b/src/cmd/server/main.go @@ -188,12 +188,6 @@ func main() { log.Printf("INFO: [%s] exited properly", instanceID) } -// runAttachmentReconciler чистит зависшие uploading-вложения. -// -// Это нужно для уровня 4: если процесс умер между S3 PUT и PostgreSQL UPDATE, -// в S3 может остаться объект, а в БД — строка status='uploading'. -// Reconciler спустя grace period помечает такие записи failed, удаляет объект -// из S3 и затем удаляет строку метаданных. func runAttachmentReconciler(ctx context.Context, st store.Store, cl *storage.Client, grace time.Duration) { ticker := time.NewTicker(30 * time.Second) defer ticker.Stop() diff --git a/src/internal/handler/handler.go b/src/internal/handler/handler.go index 06c475cc..cb87373d 100644 --- a/src/internal/handler/handler.go +++ b/src/internal/handler/handler.go @@ -694,21 +694,66 @@ func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) { respondError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type") return } + + reqContentType := r.Header.Get("Content-Type") + if reqContentType != "" && !strings.HasPrefix(reqContentType, contentType) { + respondError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type") + return + } + pixels := int64(imgCfg.Width) * int64(imgCfg.Height) if pixels > cfg.MaxPixels { respondError(w, r, http.StatusRequestEntityTooLarge, "payload_too_large") return } + if _, _, err := image.Decode(bytes.NewReader(data)); err != nil { + respondError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type") + return + } + sum := sha256.Sum256(data) sha := hex.EncodeToString(sum[:]) + + existingInChat, err := h.store.GetAttachmentBySHA256InChat(r.Context(), chatID, sha) + if err == nil && existingInChat != nil { + log.Printf("INFO: dedup hit (same chat): sha256=%s chat_id=%s attachment_id=%s [request_id=%s]", + sha[:16], chatID, existingInChat.ID, r.Context().Value(RequestIDKey)) + respondJSON(w, http.StatusOK, attachmentToMap(existingInChat)) + return + } + + existingAnywhere, err := h.store.GetReadyAttachmentBySHA256(r.Context(), sha) + if err == nil && existingAnywhere != nil { + log.Printf("INFO: dedup hit (reuse S3 object): sha256=%s from_chat=%s to_chat=%s [request_id=%s]", + sha[:16], existingAnywhere.ChatID, chatID, r.Context().Value(RequestIDKey)) + + newAtt, err := h.store.CreateAttachmentReady( + r.Context(), + chatID, userID, filename, + existingAnywhere.ObjectKey, + existingAnywhere.ContentType, + int64(len(data)), + sha, + ) + if err != nil { + log.Printf("ERROR: create attachment ready (dedup): %v [request_id=%s]", err, r.Context().Value(RequestIDKey)) + respondError(w, r, http.StatusInternalServerError, "internal_error") + return + } + respondJSON(w, http.StatusCreated, attachmentToMap(newAtt)) + return + } + ext := ".png" if contentType == "image/jpeg" { ext = ".jpeg" } objectKey := uuid.NewString() + ext - att, err := h.store.CreateAttachmentUploading(r.Context(), chatID, userID, filename, objectKey, contentType, int64(len(data)), sha) + att, err := h.store.CreateAttachmentUploading( + r.Context(), chatID, userID, filename, objectKey, contentType, int64(len(data)), sha, + ) if err != nil { switch { case errors.Is(err, store.ErrNotFound): @@ -741,6 +786,7 @@ func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) { respondError(w, r, http.StatusServiceUnavailable, "unavailable") return } + respondJSON(w, http.StatusCreated, attachmentToMap(ready)) } diff --git a/src/internal/store/postgres.go b/src/internal/store/postgres.go index 417e2407..6023de9d 100644 --- a/src/internal/store/postgres.go +++ b/src/internal/store/postgres.go @@ -54,6 +54,10 @@ type Store interface { GetAttachmentForUser(ctx context.Context, attachmentID, userID string) (*model.Attachment, error) ClaimStaleUploading(ctx context.Context, olderThan time.Time) ([]model.Attachment, error) DeleteFailedStaleAttachment(ctx context.Context, id string) error + + GetReadyAttachmentBySHA256(ctx context.Context, sha256Hex string) (*model.Attachment, error) + GetAttachmentBySHA256InChat(ctx context.Context, chatID, sha256Hex string) (*model.Attachment, error) + CreateAttachmentReady(ctx context.Context, chatID, ownerID, filename, objectKey, contentType string, sizeBytes int64, sha256Hex string) (*model.Attachment, error) } type PostgresStore struct { @@ -500,7 +504,6 @@ func (s *PostgresStore) GetMessages(ctx context.Context, chatID, userID, cursor query = `SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = $1 ORDER BY created_at, id LIMIT $2` args = []interface{}{chatID, fetchLimit} } else { - // Используем подзапрос, чтобы избежать потери точности времени при передаче через Go query = ` SELECT id, chat_id, sender_id, text, created_at FROM messages @@ -623,6 +626,32 @@ func (s *PostgresStore) CreateAttachmentUploading(ctx context.Context, chatID, o return &a, nil } +// GetAttachmentByID возвращает метаданные вложения по его ID. +// Используется воркером для получения object_key перед скачиванием из S3. +func (s *PostgresStore) GetAttachmentByID(ctx context.Context, id string) (*model.Attachment, error) { + var a model.Attachment + err := s.pool.QueryRow(ctx, ` + SELECT id::text, chat_id::text, owner_id::text, filename, object_key, + content_type, size_bytes, sha256, status, error_code, created_at, updated_at + FROM attachments + WHERE id = $1 + `, id).Scan( + &a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, + &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, + &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, + ) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("get attachment by id: %w", err) + } + a.CreatedAt = a.CreatedAt.UTC() + a.UpdatedAt = a.UpdatedAt.UTC() + a.Variants = []any{} + return &a, nil +} + func (s *PostgresStore) MarkAttachmentReady(ctx context.Context, id string) (*model.Attachment, error) { var a model.Attachment err := s.pool.QueryRow(ctx, `UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = NOW() WHERE id = $1 AND status = 'uploading' RETURNING id::text, chat_id::text, owner_id::text, filename, object_key, content_type, size_bytes, sha256, status, error_code, created_at, updated_at`, id). @@ -667,11 +696,17 @@ func (s *PostgresStore) GetAttachmentForUser(ctx context.Context, attachmentID, } func (s *PostgresStore) ClaimStaleUploading(ctx context.Context, olderThan time.Time) ([]model.Attachment, error) { - rows, err := s.pool.Query(ctx, `UPDATE attachments SET status = 'failed', error_code = 'stale_upload', updated_at = NOW() WHERE status = 'uploading' AND created_at < $1 RETURNING id::text, object_key`, olderThan) + rows, err := s.pool.Query(ctx, ` + UPDATE attachments + SET status = 'deleting', error_code = 'stale_cleanup', updated_at = NOW() + WHERE status = 'uploading' AND created_at < $1 + RETURNING id::text, object_key + `, olderThan) if err != nil { return nil, fmt.Errorf("claim stale uploading: %w", err) } defer rows.Close() + var out []model.Attachment for rows.Next() { var a model.Attachment @@ -684,13 +719,101 @@ func (s *PostgresStore) ClaimStaleUploading(ctx context.Context, olderThan time. } func (s *PostgresStore) DeleteFailedStaleAttachment(ctx context.Context, id string) error { - _, err := s.pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1 AND status = 'failed' AND error_code = 'stale_upload'`, id) + _, err := s.pool.Exec(ctx, ` + DELETE FROM attachments + WHERE id = $1 AND status = 'deleting' + `, id) if err != nil { return fmt.Errorf("delete failed stale attachment: %w", err) } return nil } +func (s *PostgresStore) GetReadyAttachmentBySHA256(ctx context.Context, sha256Hex string) (*model.Attachment, error) { + var a model.Attachment + err := s.pool.QueryRow(ctx, ` + SELECT id::text, chat_id::text, owner_id::text, filename, object_key, + content_type, size_bytes, sha256, status, error_code, + created_at, updated_at + FROM attachments + WHERE sha256 = $1 AND status = 'ready' + ORDER BY created_at + LIMIT 1 + `, sha256Hex).Scan( + &a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, + &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, + &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, + ) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("get ready attachment by sha256: %w", err) + } + a.CreatedAt = a.CreatedAt.UTC() + a.UpdatedAt = a.UpdatedAt.UTC() + a.Variants = []any{} + return &a, nil +} + +func (s *PostgresStore) GetAttachmentBySHA256InChat(ctx context.Context, chatID, sha256Hex string) (*model.Attachment, error) { + var a model.Attachment + err := s.pool.QueryRow(ctx, ` + SELECT id::text, chat_id::text, owner_id::text, filename, object_key, + content_type, size_bytes, sha256, status, error_code, + created_at, updated_at + FROM attachments + WHERE chat_id = $1 AND sha256 = $2 AND status = 'ready' + ORDER BY created_at + LIMIT 1 + `, chatID, sha256Hex).Scan( + &a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, + &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, + &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, + ) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("get attachment by sha256 in chat: %w", err) + } + a.CreatedAt = a.CreatedAt.UTC() + a.UpdatedAt = a.UpdatedAt.UTC() + a.Variants = []any{} + return &a, nil +} + +func (s *PostgresStore) CreateAttachmentReady(ctx context.Context, chatID, ownerID, filename, objectKey, contentType string, sizeBytes int64, sha256Hex string) (*model.Attachment, error) { + var a model.Attachment + err := s.pool.QueryRow(ctx, ` + INSERT INTO attachments ( + chat_id, owner_id, filename, object_key, + content_type, size_bytes, sha256, status + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, 'ready') + RETURNING id::text, chat_id::text, owner_id::text, filename, + object_key, content_type, size_bytes, sha256, + status, error_code, created_at, updated_at + `, chatID, ownerID, filename, objectKey, contentType, sizeBytes, sha256Hex).Scan( + &a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, + &a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status, + &a.ErrorCode, &a.CreatedAt, &a.UpdatedAt, + ) + if err != nil { + if isUniqueViolation(err) { + return nil, ErrConflict + } + if isForeignKeyViolation(err) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("insert attachment ready: %w", err) + } + a.CreatedAt = a.CreatedAt.UTC() + a.UpdatedAt = a.UpdatedAt.UTC() + a.Variants = []any{} + return &a, nil +} + func hashScope(s string) int64 { h := sha256.Sum256([]byte(s)) return int64(binary.BigEndian.Uint64(h[:8]))