forked from mtusi2026/mtusi2026lab4
lab4 - fixed
This commit is contained in:
+24
-16
@@ -112,31 +112,39 @@ services:
|
||||
|
||||
s3:
|
||||
image: rustfs/rustfs@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2
|
||||
command: ["server", "--address", ":9000"]
|
||||
command: [ "server", "--address", ":9000" ]
|
||||
environment:
|
||||
RUSTFS_ADDRESS: ":9000"
|
||||
RUSTFS_VOLUMES: "/data"
|
||||
RUSTFS_REGION: "${S3_REGION:-us-east-1}"
|
||||
|
||||
# Root/admin credentials (используются Go-кодом для создания bucket)
|
||||
RUSTFS_ROOT_USER: "${S3_ACCESS_KEY_ID}"
|
||||
RUSTFS_ROOT_PASSWORD: "${S3_SECRET_ACCESS_KEY}"
|
||||
|
||||
# Console отключена для безопасности
|
||||
# RustFS принимает root-учётные данные при инициализации
|
||||
RUSTFS_ROOT_USER: "${S3_ROOT_USER}"
|
||||
RUSTFS_ROOT_PASSWORD: "${S3_ROOT_PASSWORD}"
|
||||
RUSTFS_CONSOLE_ENABLE: "false"
|
||||
RUSTFS_OBS_LOGGER_LEVEL: "warn"
|
||||
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 256M
|
||||
|
||||
restart: unless-stopped
|
||||
|
||||
s3-init:
|
||||
image: docker.io/minio/mc:latest # Или тот образ, который у тебя успешно тянется
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_started
|
||||
environment:
|
||||
S3_ENDPOINT: "${S3_ENDPOINT:-http://s3:9000}"
|
||||
S3_BUCKET: "${S3_BUCKET:-messenger-attachments}"
|
||||
# Скрипт использует root-ключи для создания bucket и ограничения прав API-пользователя
|
||||
S3_ROOT_USER: "${S3_ROOT_USER}"
|
||||
S3_ROOT_PASSWORD: "${S3_ROOT_PASSWORD}"
|
||||
S3_API_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID}"
|
||||
S3_API_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY}"
|
||||
volumes:
|
||||
- ./infra/rustfs-bootstrap.sh:/bootstrap.sh:ro
|
||||
entrypoint: [ "/bin/sh", "/bootstrap.sh" ]
|
||||
restart: "no"
|
||||
|
||||
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
|
||||
@@ -188,12 +188,6 @@ func main() {
|
||||
log.Printf("INFO: [%s] exited properly", instanceID)
|
||||
}
|
||||
|
||||
// runAttachmentReconciler чистит зависшие uploading-вложения.
|
||||
//
|
||||
// Это нужно для уровня 4: если процесс умер между S3 PUT и PostgreSQL UPDATE,
|
||||
// в S3 может остаться объект, а в БД — строка status='uploading'.
|
||||
// Reconciler спустя grace period помечает такие записи failed, удаляет объект
|
||||
// из S3 и затем удаляет строку метаданных.
|
||||
func runAttachmentReconciler(ctx context.Context, st store.Store, cl *storage.Client, grace time.Duration) {
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
@@ -694,21 +694,66 @@ func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
respondError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type")
|
||||
return
|
||||
}
|
||||
|
||||
reqContentType := r.Header.Get("Content-Type")
|
||||
if reqContentType != "" && !strings.HasPrefix(reqContentType, contentType) {
|
||||
respondError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type")
|
||||
return
|
||||
}
|
||||
|
||||
pixels := int64(imgCfg.Width) * int64(imgCfg.Height)
|
||||
if pixels > cfg.MaxPixels {
|
||||
respondError(w, r, http.StatusRequestEntityTooLarge, "payload_too_large")
|
||||
return
|
||||
}
|
||||
|
||||
if _, _, err := image.Decode(bytes.NewReader(data)); err != nil {
|
||||
respondError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type")
|
||||
return
|
||||
}
|
||||
|
||||
sum := sha256.Sum256(data)
|
||||
sha := hex.EncodeToString(sum[:])
|
||||
|
||||
existingInChat, err := h.store.GetAttachmentBySHA256InChat(r.Context(), chatID, sha)
|
||||
if err == nil && existingInChat != nil {
|
||||
log.Printf("INFO: dedup hit (same chat): sha256=%s chat_id=%s attachment_id=%s [request_id=%s]",
|
||||
sha[:16], chatID, existingInChat.ID, r.Context().Value(RequestIDKey))
|
||||
respondJSON(w, http.StatusOK, attachmentToMap(existingInChat))
|
||||
return
|
||||
}
|
||||
|
||||
existingAnywhere, err := h.store.GetReadyAttachmentBySHA256(r.Context(), sha)
|
||||
if err == nil && existingAnywhere != nil {
|
||||
log.Printf("INFO: dedup hit (reuse S3 object): sha256=%s from_chat=%s to_chat=%s [request_id=%s]",
|
||||
sha[:16], existingAnywhere.ChatID, chatID, r.Context().Value(RequestIDKey))
|
||||
|
||||
newAtt, err := h.store.CreateAttachmentReady(
|
||||
r.Context(),
|
||||
chatID, userID, filename,
|
||||
existingAnywhere.ObjectKey,
|
||||
existingAnywhere.ContentType,
|
||||
int64(len(data)),
|
||||
sha,
|
||||
)
|
||||
if err != nil {
|
||||
log.Printf("ERROR: create attachment ready (dedup): %v [request_id=%s]", err, r.Context().Value(RequestIDKey))
|
||||
respondError(w, r, http.StatusInternalServerError, "internal_error")
|
||||
return
|
||||
}
|
||||
respondJSON(w, http.StatusCreated, attachmentToMap(newAtt))
|
||||
return
|
||||
}
|
||||
|
||||
ext := ".png"
|
||||
if contentType == "image/jpeg" {
|
||||
ext = ".jpeg"
|
||||
}
|
||||
objectKey := uuid.NewString() + ext
|
||||
|
||||
att, err := h.store.CreateAttachmentUploading(r.Context(), chatID, userID, filename, objectKey, contentType, int64(len(data)), sha)
|
||||
att, err := h.store.CreateAttachmentUploading(
|
||||
r.Context(), chatID, userID, filename, objectKey, contentType, int64(len(data)), sha,
|
||||
)
|
||||
if err != nil {
|
||||
switch {
|
||||
case errors.Is(err, store.ErrNotFound):
|
||||
@@ -741,6 +786,7 @@ func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
respondError(w, r, http.StatusServiceUnavailable, "unavailable")
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, http.StatusCreated, attachmentToMap(ready))
|
||||
}
|
||||
|
||||
|
||||
@@ -54,6 +54,10 @@ type Store interface {
|
||||
GetAttachmentForUser(ctx context.Context, attachmentID, userID string) (*model.Attachment, error)
|
||||
ClaimStaleUploading(ctx context.Context, olderThan time.Time) ([]model.Attachment, error)
|
||||
DeleteFailedStaleAttachment(ctx context.Context, id string) error
|
||||
|
||||
GetReadyAttachmentBySHA256(ctx context.Context, sha256Hex string) (*model.Attachment, error)
|
||||
GetAttachmentBySHA256InChat(ctx context.Context, chatID, sha256Hex string) (*model.Attachment, error)
|
||||
CreateAttachmentReady(ctx context.Context, chatID, ownerID, filename, objectKey, contentType string, sizeBytes int64, sha256Hex string) (*model.Attachment, error)
|
||||
}
|
||||
|
||||
type PostgresStore struct {
|
||||
@@ -500,7 +504,6 @@ func (s *PostgresStore) GetMessages(ctx context.Context, chatID, userID, cursor
|
||||
query = `SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = $1 ORDER BY created_at, id LIMIT $2`
|
||||
args = []interface{}{chatID, fetchLimit}
|
||||
} else {
|
||||
// Используем подзапрос, чтобы избежать потери точности времени при передаче через Go
|
||||
query = `
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
@@ -623,6 +626,32 @@ func (s *PostgresStore) CreateAttachmentUploading(ctx context.Context, chatID, o
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
// GetAttachmentByID возвращает метаданные вложения по его ID.
|
||||
// Используется воркером для получения object_key перед скачиванием из S3.
|
||||
func (s *PostgresStore) GetAttachmentByID(ctx context.Context, id string) (*model.Attachment, error) {
|
||||
var a model.Attachment
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT id::text, chat_id::text, owner_id::text, filename, object_key,
|
||||
content_type, size_bytes, sha256, status, error_code, created_at, updated_at
|
||||
FROM attachments
|
||||
WHERE id = $1
|
||||
`, id).Scan(
|
||||
&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey,
|
||||
&a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status,
|
||||
&a.ErrorCode, &a.CreatedAt, &a.UpdatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("get attachment by id: %w", err)
|
||||
}
|
||||
a.CreatedAt = a.CreatedAt.UTC()
|
||||
a.UpdatedAt = a.UpdatedAt.UTC()
|
||||
a.Variants = []any{}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
func (s *PostgresStore) MarkAttachmentReady(ctx context.Context, id string) (*model.Attachment, error) {
|
||||
var a model.Attachment
|
||||
err := s.pool.QueryRow(ctx, `UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = NOW() WHERE id = $1 AND status = 'uploading' RETURNING id::text, chat_id::text, owner_id::text, filename, object_key, content_type, size_bytes, sha256, status, error_code, created_at, updated_at`, id).
|
||||
@@ -667,11 +696,17 @@ func (s *PostgresStore) GetAttachmentForUser(ctx context.Context, attachmentID,
|
||||
}
|
||||
|
||||
func (s *PostgresStore) ClaimStaleUploading(ctx context.Context, olderThan time.Time) ([]model.Attachment, error) {
|
||||
rows, err := s.pool.Query(ctx, `UPDATE attachments SET status = 'failed', error_code = 'stale_upload', updated_at = NOW() WHERE status = 'uploading' AND created_at < $1 RETURNING id::text, object_key`, olderThan)
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
UPDATE attachments
|
||||
SET status = 'deleting', error_code = 'stale_cleanup', updated_at = NOW()
|
||||
WHERE status = 'uploading' AND created_at < $1
|
||||
RETURNING id::text, object_key
|
||||
`, olderThan)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("claim stale uploading: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var out []model.Attachment
|
||||
for rows.Next() {
|
||||
var a model.Attachment
|
||||
@@ -684,13 +719,101 @@ func (s *PostgresStore) ClaimStaleUploading(ctx context.Context, olderThan time.
|
||||
}
|
||||
|
||||
func (s *PostgresStore) DeleteFailedStaleAttachment(ctx context.Context, id string) error {
|
||||
_, err := s.pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1 AND status = 'failed' AND error_code = 'stale_upload'`, id)
|
||||
_, err := s.pool.Exec(ctx, `
|
||||
DELETE FROM attachments
|
||||
WHERE id = $1 AND status = 'deleting'
|
||||
`, id)
|
||||
if err != nil {
|
||||
return fmt.Errorf("delete failed stale attachment: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *PostgresStore) GetReadyAttachmentBySHA256(ctx context.Context, sha256Hex string) (*model.Attachment, error) {
|
||||
var a model.Attachment
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT id::text, chat_id::text, owner_id::text, filename, object_key,
|
||||
content_type, size_bytes, sha256, status, error_code,
|
||||
created_at, updated_at
|
||||
FROM attachments
|
||||
WHERE sha256 = $1 AND status = 'ready'
|
||||
ORDER BY created_at
|
||||
LIMIT 1
|
||||
`, sha256Hex).Scan(
|
||||
&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey,
|
||||
&a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status,
|
||||
&a.ErrorCode, &a.CreatedAt, &a.UpdatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("get ready attachment by sha256: %w", err)
|
||||
}
|
||||
a.CreatedAt = a.CreatedAt.UTC()
|
||||
a.UpdatedAt = a.UpdatedAt.UTC()
|
||||
a.Variants = []any{}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
func (s *PostgresStore) GetAttachmentBySHA256InChat(ctx context.Context, chatID, sha256Hex string) (*model.Attachment, error) {
|
||||
var a model.Attachment
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT id::text, chat_id::text, owner_id::text, filename, object_key,
|
||||
content_type, size_bytes, sha256, status, error_code,
|
||||
created_at, updated_at
|
||||
FROM attachments
|
||||
WHERE chat_id = $1 AND sha256 = $2 AND status = 'ready'
|
||||
ORDER BY created_at
|
||||
LIMIT 1
|
||||
`, chatID, sha256Hex).Scan(
|
||||
&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey,
|
||||
&a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status,
|
||||
&a.ErrorCode, &a.CreatedAt, &a.UpdatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("get attachment by sha256 in chat: %w", err)
|
||||
}
|
||||
a.CreatedAt = a.CreatedAt.UTC()
|
||||
a.UpdatedAt = a.UpdatedAt.UTC()
|
||||
a.Variants = []any{}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
func (s *PostgresStore) CreateAttachmentReady(ctx context.Context, chatID, ownerID, filename, objectKey, contentType string, sizeBytes int64, sha256Hex string) (*model.Attachment, error) {
|
||||
var a model.Attachment
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
INSERT INTO attachments (
|
||||
chat_id, owner_id, filename, object_key,
|
||||
content_type, size_bytes, sha256, status
|
||||
)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, 'ready')
|
||||
RETURNING id::text, chat_id::text, owner_id::text, filename,
|
||||
object_key, content_type, size_bytes, sha256,
|
||||
status, error_code, created_at, updated_at
|
||||
`, chatID, ownerID, filename, objectKey, contentType, sizeBytes, sha256Hex).Scan(
|
||||
&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey,
|
||||
&a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status,
|
||||
&a.ErrorCode, &a.CreatedAt, &a.UpdatedAt,
|
||||
)
|
||||
if err != nil {
|
||||
if isUniqueViolation(err) {
|
||||
return nil, ErrConflict
|
||||
}
|
||||
if isForeignKeyViolation(err) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("insert attachment ready: %w", err)
|
||||
}
|
||||
a.CreatedAt = a.CreatedAt.UTC()
|
||||
a.UpdatedAt = a.UpdatedAt.UTC()
|
||||
a.Variants = []any{}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
func hashScope(s string) int64 {
|
||||
h := sha256.Sum256([]byte(s))
|
||||
return int64(binary.BigEndian.Uint64(h[:8]))
|
||||
|
||||
Reference in New Issue
Block a user