lab4 - fixed

This commit is contained in:
Rakshasa2
2026-11-05 13:37:38 +03:00
parent 8c7375a6ea
commit bf8da2b7d9
4 changed files with 197 additions and 26 deletions
+24 -16
View File
@@ -112,31 +112,39 @@ services:
s3:
image: rustfs/rustfs@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2
command: ["server", "--address", ":9000"]
command: [ "server", "--address", ":9000" ]
environment:
RUSTFS_ADDRESS: ":9000"
RUSTFS_VOLUMES: "/data"
RUSTFS_REGION: "${S3_REGION:-us-east-1}"
# Root/admin credentials (используются Go-кодом для создания bucket)
RUSTFS_ROOT_USER: "${S3_ACCESS_KEY_ID}"
RUSTFS_ROOT_PASSWORD: "${S3_SECRET_ACCESS_KEY}"
# Console отключена для безопасности
# RustFS принимает root-учётные данные при инициализации
RUSTFS_ROOT_USER: "${S3_ROOT_USER}"
RUSTFS_ROOT_PASSWORD: "${S3_ROOT_PASSWORD}"
RUSTFS_CONSOLE_ENABLE: "false"
RUSTFS_OBS_LOGGER_LEVEL: "warn"
volumes:
- s3_data:/data
deploy:
resources:
limits:
cpus: "0.50"
memory: 256M
restart: unless-stopped
s3-init:
image: docker.io/minio/mc:latest # Или тот образ, который у тебя успешно тянется
depends_on:
s3:
condition: service_started
environment:
S3_ENDPOINT: "${S3_ENDPOINT:-http://s3:9000}"
S3_BUCKET: "${S3_BUCKET:-messenger-attachments}"
# Скрипт использует root-ключи для создания bucket и ограничения прав API-пользователя
S3_ROOT_USER: "${S3_ROOT_USER}"
S3_ROOT_PASSWORD: "${S3_ROOT_PASSWORD}"
S3_API_ACCESS_KEY_ID: "${S3_ACCESS_KEY_ID}"
S3_API_SECRET_ACCESS_KEY: "${S3_SECRET_ACCESS_KEY}"
volumes:
- ./infra/rustfs-bootstrap.sh:/bootstrap.sh:ro
entrypoint: [ "/bin/sh", "/bootstrap.sh" ]
restart: "no"
volumes:
postgres_data:
redis_data:
-6
View File
@@ -188,12 +188,6 @@ func main() {
log.Printf("INFO: [%s] exited properly", instanceID)
}
// runAttachmentReconciler чистит зависшие uploading-вложения.
//
// Это нужно для уровня 4: если процесс умер между S3 PUT и PostgreSQL UPDATE,
// в S3 может остаться объект, а в БД — строка status='uploading'.
// Reconciler спустя grace period помечает такие записи failed, удаляет объект
// из S3 и затем удаляет строку метаданных.
func runAttachmentReconciler(ctx context.Context, st store.Store, cl *storage.Client, grace time.Duration) {
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
+47 -1
View File
@@ -694,21 +694,66 @@ func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) {
respondError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type")
return
}
reqContentType := r.Header.Get("Content-Type")
if reqContentType != "" && !strings.HasPrefix(reqContentType, contentType) {
respondError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type")
return
}
pixels := int64(imgCfg.Width) * int64(imgCfg.Height)
if pixels > cfg.MaxPixels {
respondError(w, r, http.StatusRequestEntityTooLarge, "payload_too_large")
return
}
if _, _, err := image.Decode(bytes.NewReader(data)); err != nil {
respondError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type")
return
}
sum := sha256.Sum256(data)
sha := hex.EncodeToString(sum[:])
existingInChat, err := h.store.GetAttachmentBySHA256InChat(r.Context(), chatID, sha)
if err == nil && existingInChat != nil {
log.Printf("INFO: dedup hit (same chat): sha256=%s chat_id=%s attachment_id=%s [request_id=%s]",
sha[:16], chatID, existingInChat.ID, r.Context().Value(RequestIDKey))
respondJSON(w, http.StatusOK, attachmentToMap(existingInChat))
return
}
existingAnywhere, err := h.store.GetReadyAttachmentBySHA256(r.Context(), sha)
if err == nil && existingAnywhere != nil {
log.Printf("INFO: dedup hit (reuse S3 object): sha256=%s from_chat=%s to_chat=%s [request_id=%s]",
sha[:16], existingAnywhere.ChatID, chatID, r.Context().Value(RequestIDKey))
newAtt, err := h.store.CreateAttachmentReady(
r.Context(),
chatID, userID, filename,
existingAnywhere.ObjectKey,
existingAnywhere.ContentType,
int64(len(data)),
sha,
)
if err != nil {
log.Printf("ERROR: create attachment ready (dedup): %v [request_id=%s]", err, r.Context().Value(RequestIDKey))
respondError(w, r, http.StatusInternalServerError, "internal_error")
return
}
respondJSON(w, http.StatusCreated, attachmentToMap(newAtt))
return
}
ext := ".png"
if contentType == "image/jpeg" {
ext = ".jpeg"
}
objectKey := uuid.NewString() + ext
att, err := h.store.CreateAttachmentUploading(r.Context(), chatID, userID, filename, objectKey, contentType, int64(len(data)), sha)
att, err := h.store.CreateAttachmentUploading(
r.Context(), chatID, userID, filename, objectKey, contentType, int64(len(data)), sha,
)
if err != nil {
switch {
case errors.Is(err, store.ErrNotFound):
@@ -741,6 +786,7 @@ func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) {
respondError(w, r, http.StatusServiceUnavailable, "unavailable")
return
}
respondJSON(w, http.StatusCreated, attachmentToMap(ready))
}
+126 -3
View File
@@ -54,6 +54,10 @@ type Store interface {
GetAttachmentForUser(ctx context.Context, attachmentID, userID string) (*model.Attachment, error)
ClaimStaleUploading(ctx context.Context, olderThan time.Time) ([]model.Attachment, error)
DeleteFailedStaleAttachment(ctx context.Context, id string) error
GetReadyAttachmentBySHA256(ctx context.Context, sha256Hex string) (*model.Attachment, error)
GetAttachmentBySHA256InChat(ctx context.Context, chatID, sha256Hex string) (*model.Attachment, error)
CreateAttachmentReady(ctx context.Context, chatID, ownerID, filename, objectKey, contentType string, sizeBytes int64, sha256Hex string) (*model.Attachment, error)
}
type PostgresStore struct {
@@ -500,7 +504,6 @@ func (s *PostgresStore) GetMessages(ctx context.Context, chatID, userID, cursor
query = `SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = $1 ORDER BY created_at, id LIMIT $2`
args = []interface{}{chatID, fetchLimit}
} else {
// Используем подзапрос, чтобы избежать потери точности времени при передаче через Go
query = `
SELECT id, chat_id, sender_id, text, created_at
FROM messages
@@ -623,6 +626,32 @@ func (s *PostgresStore) CreateAttachmentUploading(ctx context.Context, chatID, o
return &a, nil
}
// GetAttachmentByID возвращает метаданные вложения по его ID.
// Используется воркером для получения object_key перед скачиванием из S3.
func (s *PostgresStore) GetAttachmentByID(ctx context.Context, id string) (*model.Attachment, error) {
var a model.Attachment
err := s.pool.QueryRow(ctx, `
SELECT id::text, chat_id::text, owner_id::text, filename, object_key,
content_type, size_bytes, sha256, status, error_code, created_at, updated_at
FROM attachments
WHERE id = $1
`, id).Scan(
&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey,
&a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status,
&a.ErrorCode, &a.CreatedAt, &a.UpdatedAt,
)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("get attachment by id: %w", err)
}
a.CreatedAt = a.CreatedAt.UTC()
a.UpdatedAt = a.UpdatedAt.UTC()
a.Variants = []any{}
return &a, nil
}
func (s *PostgresStore) MarkAttachmentReady(ctx context.Context, id string) (*model.Attachment, error) {
var a model.Attachment
err := s.pool.QueryRow(ctx, `UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = NOW() WHERE id = $1 AND status = 'uploading' RETURNING id::text, chat_id::text, owner_id::text, filename, object_key, content_type, size_bytes, sha256, status, error_code, created_at, updated_at`, id).
@@ -667,11 +696,17 @@ func (s *PostgresStore) GetAttachmentForUser(ctx context.Context, attachmentID,
}
func (s *PostgresStore) ClaimStaleUploading(ctx context.Context, olderThan time.Time) ([]model.Attachment, error) {
rows, err := s.pool.Query(ctx, `UPDATE attachments SET status = 'failed', error_code = 'stale_upload', updated_at = NOW() WHERE status = 'uploading' AND created_at < $1 RETURNING id::text, object_key`, olderThan)
rows, err := s.pool.Query(ctx, `
UPDATE attachments
SET status = 'deleting', error_code = 'stale_cleanup', updated_at = NOW()
WHERE status = 'uploading' AND created_at < $1
RETURNING id::text, object_key
`, olderThan)
if err != nil {
return nil, fmt.Errorf("claim stale uploading: %w", err)
}
defer rows.Close()
var out []model.Attachment
for rows.Next() {
var a model.Attachment
@@ -684,13 +719,101 @@ func (s *PostgresStore) ClaimStaleUploading(ctx context.Context, olderThan time.
}
func (s *PostgresStore) DeleteFailedStaleAttachment(ctx context.Context, id string) error {
_, err := s.pool.Exec(ctx, `DELETE FROM attachments WHERE id = $1 AND status = 'failed' AND error_code = 'stale_upload'`, id)
_, err := s.pool.Exec(ctx, `
DELETE FROM attachments
WHERE id = $1 AND status = 'deleting'
`, id)
if err != nil {
return fmt.Errorf("delete failed stale attachment: %w", err)
}
return nil
}
func (s *PostgresStore) GetReadyAttachmentBySHA256(ctx context.Context, sha256Hex string) (*model.Attachment, error) {
var a model.Attachment
err := s.pool.QueryRow(ctx, `
SELECT id::text, chat_id::text, owner_id::text, filename, object_key,
content_type, size_bytes, sha256, status, error_code,
created_at, updated_at
FROM attachments
WHERE sha256 = $1 AND status = 'ready'
ORDER BY created_at
LIMIT 1
`, sha256Hex).Scan(
&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey,
&a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status,
&a.ErrorCode, &a.CreatedAt, &a.UpdatedAt,
)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("get ready attachment by sha256: %w", err)
}
a.CreatedAt = a.CreatedAt.UTC()
a.UpdatedAt = a.UpdatedAt.UTC()
a.Variants = []any{}
return &a, nil
}
func (s *PostgresStore) GetAttachmentBySHA256InChat(ctx context.Context, chatID, sha256Hex string) (*model.Attachment, error) {
var a model.Attachment
err := s.pool.QueryRow(ctx, `
SELECT id::text, chat_id::text, owner_id::text, filename, object_key,
content_type, size_bytes, sha256, status, error_code,
created_at, updated_at
FROM attachments
WHERE chat_id = $1 AND sha256 = $2 AND status = 'ready'
ORDER BY created_at
LIMIT 1
`, chatID, sha256Hex).Scan(
&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey,
&a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status,
&a.ErrorCode, &a.CreatedAt, &a.UpdatedAt,
)
if err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("get attachment by sha256 in chat: %w", err)
}
a.CreatedAt = a.CreatedAt.UTC()
a.UpdatedAt = a.UpdatedAt.UTC()
a.Variants = []any{}
return &a, nil
}
func (s *PostgresStore) CreateAttachmentReady(ctx context.Context, chatID, ownerID, filename, objectKey, contentType string, sizeBytes int64, sha256Hex string) (*model.Attachment, error) {
var a model.Attachment
err := s.pool.QueryRow(ctx, `
INSERT INTO attachments (
chat_id, owner_id, filename, object_key,
content_type, size_bytes, sha256, status
)
VALUES ($1, $2, $3, $4, $5, $6, $7, 'ready')
RETURNING id::text, chat_id::text, owner_id::text, filename,
object_key, content_type, size_bytes, sha256,
status, error_code, created_at, updated_at
`, chatID, ownerID, filename, objectKey, contentType, sizeBytes, sha256Hex).Scan(
&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey,
&a.ContentType, &a.SizeBytes, &a.SHA256, &a.Status,
&a.ErrorCode, &a.CreatedAt, &a.UpdatedAt,
)
if err != nil {
if isUniqueViolation(err) {
return nil, ErrConflict
}
if isForeignKeyViolation(err) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("insert attachment ready: %w", err)
}
a.CreatedAt = a.CreatedAt.UTC()
a.UpdatedAt = a.UpdatedAt.UTC()
a.Variants = []any{}
return &a, nil
}
func hashScope(s string) int64 {
h := sha256.Sum256([]byte(s))
return int64(binary.BigEndian.Uint64(h[:8]))