forked from mtusi2026/mtusi2026lab4
feat:level4
This commit is contained in:
@@ -28,3 +28,6 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
my_tests:
|
||||
$(PYTHON) student-tests/negative.py
|
||||
+27
-1
@@ -31,6 +31,17 @@ func main() {
|
||||
}
|
||||
defer store.Close()
|
||||
|
||||
var s3store *domain.S3Store
|
||||
if cfg.S3Endpoint != "" {
|
||||
s3store, err = domain.NewS3Store(rootCtx, cfg.S3Endpoint, cfg.S3Region, cfg.S3Bucket,
|
||||
cfg.S3AccessKeyID, cfg.S3SecretKey, cfg.MaxUploadBytes, cfg.MaxImagePixels)
|
||||
if err != nil {
|
||||
log.Fatalf("s3: %v", err)
|
||||
}
|
||||
} else {
|
||||
log.Fatal("S3_ENDPOINT is required for lab4")
|
||||
}
|
||||
|
||||
// Кэш сессий в Redis/Valkey. Если CACHE_URL не задан — работаем без кэша.
|
||||
var cache *domain.SessionCache
|
||||
if cfg.CacheURL != "" {
|
||||
@@ -49,9 +60,24 @@ func main() {
|
||||
}
|
||||
|
||||
sessions := domain.NewSessionStore(store, cache, cfg.SessionTTL)
|
||||
handler := &api.Handler{Store: store, Sessions: sessions}
|
||||
handler := &api.Handler{Store: store, Sessions: sessions, S3: s3store, MaxUpload: cfg.MaxUploadBytes}
|
||||
router := api.NewRouter(handler, cfg.InstanceID)
|
||||
|
||||
go func() {
|
||||
ticker := time.NewTicker(2 * time.Minute)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-rootCtx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
if err := store.ReconcileOrphans(rootCtx, s3store, cfg.S3GCGrace); err != nil {
|
||||
log.Printf("reconcile: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: fmt.Sprintf("%s:%d", cfg.Host, cfg.Port),
|
||||
Handler: router,
|
||||
|
||||
+55
-14
@@ -2,20 +2,14 @@ x-api: &api
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
env_file: [.env]
|
||||
environment:
|
||||
APP_HOST: "0.0.0.0"
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: "lab"
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
SESSION_STORE: "postgres"
|
||||
SESSION_TTL_SECONDS: "3600"
|
||||
# INSTANCE_ID сгенерируется в config.go
|
||||
secrets: [database_url, cache_url]
|
||||
env_file:
|
||||
- .env
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
|
||||
depends_on:
|
||||
postgres: { condition: service_healthy }
|
||||
redis: { condition: service_healthy }
|
||||
s3: { condition: service_healthy }
|
||||
s3-init: { condition: service_completed_successfully }
|
||||
networks: [backend]
|
||||
|
||||
services:
|
||||
@@ -47,6 +41,48 @@ services:
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
networks: [backend]
|
||||
|
||||
s3:
|
||||
image: pgsty/silo:latest
|
||||
command: ["server", "/data", "--console-address", ":9001"]
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
|
||||
secrets: [minio_root_user, minio_root_password]
|
||||
volumes: [s3data:/data]
|
||||
healthcheck:
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
networks: [backend]
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "${S3_CPUS:-1.0}", memory: "${S3_MEM:-512M}" }
|
||||
|
||||
s3-init:
|
||||
image: pgsty/silo:latest
|
||||
depends_on:
|
||||
s3: { condition: service_healthy }
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
entrypoint: ["/bin/sh", "/bootstrap/bootstrap.sh"]
|
||||
secrets:
|
||||
- minio_root_user
|
||||
- minio_root_password
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
volumes:
|
||||
- ./infra/minio/bootstrap.sh:/bootstrap/bootstrap.sh:ro
|
||||
- /tmp/silo-init-data:/data
|
||||
networks: [backend]
|
||||
deploy:
|
||||
replicas: 1
|
||||
restart_policy: { condition: "no" }
|
||||
|
||||
api:
|
||||
<<: *api
|
||||
deploy:
|
||||
@@ -78,12 +114,17 @@ services:
|
||||
volumes:
|
||||
pgdata:
|
||||
redisdata:
|
||||
s3data:
|
||||
|
||||
networks:
|
||||
backend:
|
||||
driver: bridge
|
||||
|
||||
secrets:
|
||||
postgres_password: { file: ./secrets/postgres_password.txt }
|
||||
database_url: { file: ./secrets/database_url.txt }
|
||||
cache_url: { file: ./secrets/cache_url.txt }
|
||||
postgres_password: { file: ./secrets/postgres_password.txt }
|
||||
database_url: { file: ./secrets/database_url.txt }
|
||||
cache_url: { file: ./secrets/cache_url.txt }
|
||||
s3_access_key_id: { file: ./secrets/s3_access_key_id.txt }
|
||||
s3_secret_access_key: { file: ./secrets/s3_secret_access_key.txt }
|
||||
minio_root_user: { file: ./secrets/minio_root_user.txt }
|
||||
minio_root_password: { file: ./secrets/minio_root_password.txt }
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 87 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 14 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 61 KiB |
@@ -5,6 +5,7 @@ go 1.22
|
||||
require (
|
||||
github.com/google/uuid v1.6.0
|
||||
github.com/jackc/pgx/v5 v5.5.5
|
||||
github.com/minio/minio-go/v7 v7.0.72
|
||||
github.com/redis/go-redis/v9 v9.5.1
|
||||
golang.org/x/crypto v0.23.0
|
||||
)
|
||||
@@ -12,10 +13,18 @@ require (
|
||||
require (
|
||||
github.com/cespare/xxhash/v2 v2.2.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.1 // indirect
|
||||
github.com/klauspost/compress v1.17.6 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.6 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/rs/xid v1.5.0 // indirect
|
||||
golang.org/x/net v0.23.0 // indirect
|
||||
golang.org/x/sync v0.1.0 // indirect
|
||||
golang.org/x/sys v0.20.0 // indirect
|
||||
golang.org/x/text v0.15.0 // indirect
|
||||
gopkg.in/ini.v1 v1.67.0 // indirect
|
||||
)
|
||||
|
||||
@@ -9,6 +9,10 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
@@ -19,10 +23,21 @@ github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
|
||||
github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
|
||||
github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
|
||||
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
|
||||
github.com/klauspost/compress v1.17.6/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
|
||||
github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.72 h1:ZSbxs2BfJensLyHdVOgHv+pfmvxYraaUy07ER04dWnA=
|
||||
github.com/minio/minio-go/v7 v7.0.72/go.mod h1:4yBA8v80xGA30cfM3fz0DKYMXunWl/AV/6tWEs9ryzo=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8=
|
||||
github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
|
||||
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
|
||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
@@ -30,13 +45,18 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
|
||||
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
|
||||
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
|
||||
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
MINIO_ROOT_USER_FILE="${MINIO_ROOT_USER_FILE:-/run/secrets/minio_root_user}"
|
||||
MINIO_ROOT_PASSWORD_FILE="${MINIO_ROOT_PASSWORD_FILE:-/run/secrets/minio_root_password}"
|
||||
S3_ENDPOINT="${S3_ENDPOINT:-http://s3:9000}"
|
||||
S3_BUCKET="${S3_BUCKET:-messenger-attachments}"
|
||||
API_USER_FILE="${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}"
|
||||
API_PASS_FILE="${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}"
|
||||
|
||||
for f in "$MINIO_ROOT_USER_FILE" "$MINIO_ROOT_PASSWORD_FILE"; do
|
||||
[ -f "$f" ] || { echo "ERROR: missing $f" >&2; exit 1; }
|
||||
done
|
||||
|
||||
ROOT_USER="$(cat "$MINIO_ROOT_USER_FILE")"
|
||||
ROOT_PASS="$(cat "$MINIO_ROOT_PASSWORD_FILE")"
|
||||
|
||||
mc alias set local "$S3_ENDPOINT" "$ROOT_USER" "$ROOT_PASS" >/dev/null
|
||||
|
||||
# 1) bucket: приватный, создаётся один раз
|
||||
mc mb --ignore-existing "local/$S3_BUCKET" >/dev/null
|
||||
mc anonymous set none "local/$S3_BUCKET" >/dev/null
|
||||
|
||||
# 2) ограниченный пользователь API (если креды заданы заранее)
|
||||
if [ -s "$API_USER_FILE" ] && [ -s "$API_PASS_FILE" ]; then
|
||||
API_USER="$(cat "$API_USER_FILE")"
|
||||
API_PASS="$(cat "$API_PASS_FILE")"
|
||||
if ! mc admin user info local "$API_USER" >/dev/null 2>&1; then
|
||||
mc admin user add local "$API_USER" "$API_PASS" >/dev/null
|
||||
fi
|
||||
# Политика: только PutObject/GetObject/DeleteObject в bucket.
|
||||
cat >/tmp/api-policy.json <<EOF
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:PutObject","s3:GetObject","s3:DeleteObject","s3:AbortMultipartUpload","s3:ListMultipartUploadParts"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments/*"]
|
||||
},{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments"]
|
||||
}]
|
||||
}
|
||||
EOF
|
||||
mc admin policy create local api-attachments /tmp/api-policy.json >/dev/null 2>&1 || true
|
||||
mc admin policy attach local api-attachments --user "$API_USER" >/dev/null 2>&1 || true
|
||||
else
|
||||
echo "WARN: API S3 credentials not pre-created; generate and place them in secrets/" >&2
|
||||
fi
|
||||
|
||||
echo "s3 bootstrap done (bucket=$S3_BUCKET)"
|
||||
@@ -8,6 +8,7 @@ http {
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
client_max_body_size 12m;
|
||||
|
||||
location / {
|
||||
set $api_backend http://api:8080;
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
filename TEXT NOT NULL,
|
||||
content_type TEXT NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes BIGINT NOT NULL CHECK (size_bytes > 0 AND size_bytes <= 10485760),
|
||||
sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
width INTEGER NOT NULL CHECK (width > 0),
|
||||
height INTEGER NOT NULL CHECK (height > 0),
|
||||
status TEXT NOT NULL DEFAULT 'ready' CHECK (status IN ('pending','ready','failed')),
|
||||
error_code TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments (chat_id, created_at, id);
|
||||
CREATE INDEX IF NOT EXISTS idx_attachments_owner ON attachments (owner_id);
|
||||
-- Для reconciler/поиска «зависших» pending-записей:
|
||||
CREATE INDEX IF NOT EXISTS idx_attachments_pending ON attachments (created_at) WHERE status <> 'ready';
|
||||
|
||||
ALTER TABLE messages ADD COLUMN IF NOT EXISTS attachment_ids UUID[] NOT NULL DEFAULT '{}';
|
||||
|
||||
-- Журнал компенсации: объекты S3, которые могли остаться без метаданных.
|
||||
CREATE TABLE IF NOT EXISTS attachment_gc (
|
||||
object_key TEXT PRIMARY KEY,
|
||||
reason TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
+143
-5
@@ -11,14 +11,18 @@ import (
|
||||
"strings"
|
||||
|
||||
"mtusi-messenger/src/domain"
|
||||
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
var usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`)
|
||||
var idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
|
||||
|
||||
type Handler struct {
|
||||
Store *domain.Store
|
||||
Sessions *domain.SessionStore
|
||||
Store *domain.Store
|
||||
Sessions *domain.SessionStore
|
||||
S3 *domain.S3Store
|
||||
MaxUpload int64
|
||||
}
|
||||
|
||||
// --- Health ---
|
||||
@@ -237,7 +241,8 @@ func (h *Handler) GetChat(w http.ResponseWriter, r *http.Request) {
|
||||
// --- Messages ---
|
||||
|
||||
type createMessageReq struct {
|
||||
Text string `json:"text"`
|
||||
Text string `json:"text"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -285,7 +290,7 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "message too long")
|
||||
return
|
||||
}
|
||||
if strings.TrimSpace(req.Text) == "" {
|
||||
if strings.TrimSpace(req.Text) == "" && len(req.AttachmentIDs) == 0 {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "empty message")
|
||||
return
|
||||
}
|
||||
@@ -294,9 +299,16 @@ func (h *Handler) CreateMessage(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "malformed JSON")
|
||||
return
|
||||
}
|
||||
if len(req.Text) > 0 && len([]rune(req.Text)) == 0 {
|
||||
// всё пробелы, но вложения есть — допустимо
|
||||
}
|
||||
|
||||
attachmentIDs := req.AttachmentIDs
|
||||
if attachmentIDs == nil {
|
||||
attachmentIDs = []string{}
|
||||
}
|
||||
msg, rec, replay, err := h.Store.AddMessageIdempotent(
|
||||
r.Context(), chatID, me.ID, req.Text, key, bodyHash)
|
||||
r.Context(), chatID, me.ID, req.Text, attachmentIDs, key, bodyHash)
|
||||
switch {
|
||||
case err == nil:
|
||||
// ok
|
||||
@@ -386,3 +398,129 @@ func decodeSingleJSON(r io.Reader, v any) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- Attachments ---
|
||||
|
||||
// safeFilename отбрасывает пути и опасные имена; используется только для отображения.
|
||||
func safeFilename(name string) (string, bool) {
|
||||
name = strings.TrimSpace(name)
|
||||
if name == "" || len(name) > 255 {
|
||||
return "", false
|
||||
}
|
||||
if strings.ContainsAny(name, "/\\") || strings.Contains(name, "..") {
|
||||
return "", false
|
||||
}
|
||||
for _, r := range name {
|
||||
if r < 0x20 || r == 0x7f {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
return name, true
|
||||
}
|
||||
|
||||
func (h *Handler) UploadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
me, _ := userFrom(r)
|
||||
chatID := r.PathValue("chat_id")
|
||||
|
||||
if _, err := h.Store.GetChat(r.Context(), chatID, me.ID); err != nil {
|
||||
switch {
|
||||
case errors.Is(err, domain.ErrInvalid):
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid chat_id")
|
||||
case errors.Is(err, domain.ErrNotFound):
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "chat not found")
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
filename, ok := safeFilename(r.URL.Query().Get("filename"))
|
||||
if !ok {
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid filename")
|
||||
return
|
||||
}
|
||||
|
||||
objectKey := "attachments/" + chatID + "/" + uuid.NewString()
|
||||
pendingID, err := h.Store.CreateAttachmentPending(r.Context(), chatID, me.ID, objectKey, filename)
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
|
||||
res, err := h.S3.PutValidatedImage(r.Context(), objectKey, r.Body)
|
||||
if err != nil {
|
||||
_ = h.Store.MarkAttachmentFailed(r.Context(), pendingID, err.Error())
|
||||
_ = h.Store.ScheduleObjectGC(r.Context(), objectKey, "upload_failed")
|
||||
switch {
|
||||
case errors.Is(err, domain.ErrTooLarge):
|
||||
writeError(w, r, http.StatusRequestEntityTooLarge, "payload_too_large", "file too large")
|
||||
case errors.Is(err, domain.ErrUnsupported):
|
||||
writeError(w, r, http.StatusUnsupportedMediaType, "unsupported_media_type", "only PNG/JPEG")
|
||||
case errors.Is(err, domain.ErrNotImage):
|
||||
writeError(w, r, http.StatusBadRequest, "invalid_request", "invalid image")
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "storage unavailable")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
att, err := h.Store.FinalizeAttachment(r.Context(), pendingID, res.ContentType, res.SizeBytes, res.SHA256, res.Width, res.Height)
|
||||
if err != nil {
|
||||
// S3 PUT прошёл, COMMIT — нет: объект в GC.
|
||||
_ = h.Store.ScheduleObjectGC(r.Context(), objectKey, "finalize_failed")
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, att)
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
func (h *Handler) GetAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
me, _ := userFrom(r)
|
||||
att, err := h.Store.GetAttachmentForUser(r.Context(), r.PathValue("attachment_id"), me.ID)
|
||||
switch {
|
||||
case err == nil:
|
||||
writeJSON(w, http.StatusOK, att)
|
||||
case errors.Is(err, domain.ErrNotFound):
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "attachment not found")
|
||||
default:
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
}
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
func (h *Handler) DownloadAttachment(w http.ResponseWriter, r *http.Request) {
|
||||
h.bearerAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
me, _ := userFrom(r)
|
||||
id := r.PathValue("attachment_id")
|
||||
att, err := h.Store.GetAttachmentForUser(r.Context(), id, me.ID)
|
||||
if err != nil {
|
||||
if errors.Is(err, domain.ErrNotFound) {
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "attachment not found")
|
||||
return
|
||||
}
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "database unavailable")
|
||||
return
|
||||
}
|
||||
key, err := h.Store.GetAttachmentObjectKey(r.Context(), att.ID)
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "attachment not found")
|
||||
return
|
||||
}
|
||||
rc, contentType, size, err := h.S3.GetObject(r.Context(), key)
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusServiceUnavailable, "unavailable", "storage unavailable")
|
||||
return
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
w.Header().Set("Content-Type", contentType)
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
|
||||
w.Header().Set("Content-Disposition", `inline; filename="`+strings.ReplaceAll(att.Filename, `"`, "")+`"`)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
// Потоково, без буферизации всего файла в память.
|
||||
_, _ = io.Copy(w, rc)
|
||||
})(w, r)
|
||||
}
|
||||
|
||||
@@ -15,6 +15,9 @@ func NewRouter(h *Handler, instanceID string) http.Handler {
|
||||
mux.HandleFunc("GET /api/v1/chats/{chat_id}", h.GetChat)
|
||||
mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", h.CreateMessage)
|
||||
mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", h.ListMessages)
|
||||
mux.HandleFunc("POST /api/v1/chats/{chat_id}/attachments", h.UploadAttachment)
|
||||
mux.HandleFunc("GET /api/v1/attachments/{attachment_id}", h.GetAttachment)
|
||||
mux.HandleFunc("GET /api/v1/attachments/{attachment_id}/content", h.DownloadAttachment)
|
||||
|
||||
// Порядок: instanceID снаружи, requestID внутри.
|
||||
return withInstanceID(instanceID)(withRequestID(mux))
|
||||
|
||||
+61
-18
@@ -9,15 +9,23 @@ import (
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Host string
|
||||
Port int
|
||||
Env string
|
||||
DatabaseURL string
|
||||
CacheURL string
|
||||
SessionTTL time.Duration
|
||||
SessionStore string // "postgres" | "redis"
|
||||
RedisRequired bool
|
||||
InstanceID string
|
||||
Host string
|
||||
Port int
|
||||
Env string
|
||||
DatabaseURL string
|
||||
CacheURL string
|
||||
SessionTTL time.Duration
|
||||
SessionStore string
|
||||
RedisRequired bool
|
||||
InstanceID string
|
||||
S3Endpoint string
|
||||
S3Region string
|
||||
S3Bucket string
|
||||
S3AccessKeyID string
|
||||
S3SecretKey string
|
||||
MaxUploadBytes int64
|
||||
MaxImagePixels int64
|
||||
S3GCGrace time.Duration
|
||||
}
|
||||
|
||||
func getenv(key, def string) string {
|
||||
@@ -71,16 +79,51 @@ func Load() Config {
|
||||
}
|
||||
|
||||
store := getenv("SESSION_STORE", "postgres")
|
||||
|
||||
s3Access := os.Getenv("S3_ACCESS_KEY_ID")
|
||||
if s3Access == "" {
|
||||
s3Access = readSecretFromFile(os.Getenv("S3_ACCESS_KEY_ID_FILE"))
|
||||
}
|
||||
s3Secret := os.Getenv("S3_SECRET_ACCESS_KEY")
|
||||
if s3Secret == "" {
|
||||
s3Secret = readSecretFromFile(os.Getenv("S3_SECRET_ACCESS_KEY_FILE"))
|
||||
}
|
||||
maxUpload := int64(10485760)
|
||||
if v := os.Getenv("MAX_UPLOAD_BYTES"); v != "" {
|
||||
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
|
||||
maxUpload = n
|
||||
}
|
||||
}
|
||||
maxPixels := int64(40000000)
|
||||
if v := os.Getenv("MAX_IMAGE_PIXELS"); v != "" {
|
||||
if n, err := strconv.ParseInt(v, 10, 64); err == nil && n > 0 {
|
||||
maxPixels = n
|
||||
}
|
||||
}
|
||||
gcGrace := 600 * time.Second
|
||||
if v := os.Getenv("S3_GC_GRACE_SECONDS"); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
gcGrace = time.Duration(n) * time.Second
|
||||
}
|
||||
}
|
||||
return Config{
|
||||
Host: getenv("APP_HOST", "0.0.0.0"),
|
||||
Port: port,
|
||||
Env: getenv("APP_ENV", "lab"),
|
||||
DatabaseURL: dbURL,
|
||||
CacheURL: cacheURL,
|
||||
SessionTTL: time.Duration(ttl) * time.Second,
|
||||
SessionStore: store,
|
||||
RedisRequired: store == "redis",
|
||||
InstanceID: instanceID,
|
||||
Host: getenv("APP_HOST", "0.0.0.0"),
|
||||
Port: port,
|
||||
Env: getenv("APP_ENV", "lab"),
|
||||
DatabaseURL: dbURL,
|
||||
CacheURL: cacheURL,
|
||||
SessionTTL: time.Duration(ttl) * time.Second,
|
||||
SessionStore: store,
|
||||
RedisRequired: store == "redis",
|
||||
InstanceID: instanceID,
|
||||
S3Endpoint: getenv("S3_ENDPOINT", ""),
|
||||
S3Region: getenv("S3_REGION", "us-east-1"),
|
||||
S3Bucket: getenv("S3_BUCKET", "messenger-attachments"),
|
||||
S3AccessKeyID: s3Access,
|
||||
S3SecretKey: s3Secret,
|
||||
MaxUploadBytes: maxUpload,
|
||||
MaxImagePixels: maxPixels,
|
||||
S3GCGrace: gcGrace,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,189 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
// CreateAttachmentPending резервирует строку ДО записи в S3.
|
||||
// Это даёт точку компенсации: если S3 PUT упал, строка снимается/чистится.
|
||||
func (s *Store) CreateAttachmentPending(ctx context.Context, chatID, ownerID, objectKey, filename string) (string, error) {
|
||||
id := uuid.NewString()
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`INSERT INTO attachments (id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256, width, height, status)
|
||||
VALUES ($1,$2,$3,$4,$5,'image/png',1,repeat('0',64),1,1,'pending')`,
|
||||
id, chatID, ownerID, objectKey, filename)
|
||||
if err != nil {
|
||||
return "", classifyDBError(err)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// FinalizeAttachment атомарно переводит pending→ready с реальными метаданными.
|
||||
func (s *Store) FinalizeAttachment(ctx context.Context, id, contentType string, size int64, sha string, w, h int) (*Attachment, error) {
|
||||
tx, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
defer tx.Rollback(ctx)
|
||||
|
||||
var a Attachment
|
||||
var errCode *string
|
||||
err = tx.QueryRow(ctx,
|
||||
`UPDATE attachments
|
||||
SET content_type=$2, size_bytes=$3, sha256=$4, width=$5, height=$6,
|
||||
status='ready', error_code=NULL, updated_at=now()
|
||||
WHERE id=$1 AND status='pending'
|
||||
RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
status, error_code, created_at, updated_at`,
|
||||
id, contentType, size, sha, w, h,
|
||||
).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256,
|
||||
&a.Status, &errCode, &a.CreatedAt, &a.UpdatedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
a.ErrorCode = errCode
|
||||
a.Variants = []Variant{}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
// MarkAttachmentFailed помечает pending как failed (компенсация).
|
||||
func (s *Store) MarkAttachmentFailed(ctx context.Context, id, code string) error {
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`UPDATE attachments SET status='failed', error_code=$2, updated_at=now()
|
||||
WHERE id=$1 AND status='pending'`, id, code)
|
||||
return classifyDBError(err)
|
||||
}
|
||||
|
||||
// ScheduleObjectGC ставит object_key в очередь на удаление (grace period).
|
||||
func (s *Store) ScheduleObjectGC(ctx context.Context, objectKey, reason string) error {
|
||||
_, err := s.pool.Exec(ctx,
|
||||
`INSERT INTO attachment_gc (object_key, reason) VALUES ($1,$2)
|
||||
ON CONFLICT (object_key) DO NOTHING`, objectKey, reason)
|
||||
return classifyDBError(err)
|
||||
}
|
||||
|
||||
// GetAttachmentForUser возвращает вложение, только если пользователь — участник чата.
|
||||
func (s *Store) GetAttachmentForUser(ctx context.Context, id, userID string) (*Attachment, error) {
|
||||
var a Attachment
|
||||
var errCode *string
|
||||
err := s.pool.QueryRow(ctx, `
|
||||
SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256,
|
||||
a.status, a.error_code, a.created_at, a.updated_at
|
||||
FROM attachments a
|
||||
JOIN chat_members m ON m.chat_id = a.chat_id
|
||||
WHERE a.id=$1 AND m.user_id=$2 AND a.status='ready'`, id, userID,
|
||||
).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256,
|
||||
&a.Status, &errCode, &a.CreatedAt, &a.UpdatedAt)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
a.ErrorCode = errCode
|
||||
a.Variants = []Variant{}
|
||||
return &a, nil
|
||||
}
|
||||
|
||||
// GetAttachmentObjectKey — только для внутреннего download после проверки ACL.
|
||||
func (s *Store) GetAttachmentObjectKey(ctx context.Context, id string) (string, error) {
|
||||
var key string
|
||||
err := s.pool.QueryRow(ctx, `SELECT object_key FROM attachments WHERE id=$1`, id).Scan(&key)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return "", ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return "", classifyDBError(err)
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// ValidateAttachmentIDsForChat проверяет, что все вложения принадлежат чату,
|
||||
// имеют статус ready и лимит 10. Возвращает ErrInvalid/ErrNotFound.
|
||||
func (s *Store) ValidateAttachmentIDsForChat(ctx context.Context, chatID string, ids []string) error {
|
||||
if len(ids) > 10 {
|
||||
return ErrInvalid
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, id := range ids {
|
||||
if seen[id] {
|
||||
return ErrInvalid
|
||||
}
|
||||
seen[id] = true
|
||||
var ok bool
|
||||
err := s.pool.QueryRow(ctx,
|
||||
`SELECT true FROM attachments WHERE id=$1 AND chat_id=$2 AND status='ready'`,
|
||||
id, chatID).Scan(&ok)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReconcileOrphans: удаляет объекты из attachment_gc старше grace, которых нет в БД,
|
||||
// и «призрачные» объекты pending старше grace. Вызывается из фонового тикера.
|
||||
func (s *Store) ReconcileOrphans(ctx context.Context, s3 *S3Store, grace time.Duration) error {
|
||||
// 1) GC-очередь: удалить объекты в S3, затем строку.
|
||||
rows, err := s.pool.Query(ctx,
|
||||
`SELECT object_key FROM attachment_gc WHERE created_at < now() - $1::interval LIMIT 100`,
|
||||
grace.String())
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
var keys []string
|
||||
for rows.Next() {
|
||||
var k string
|
||||
if err := rows.Scan(&k); err != nil {
|
||||
rows.Close()
|
||||
return classifyDBError(err)
|
||||
}
|
||||
keys = append(keys, k)
|
||||
}
|
||||
rows.Close()
|
||||
for _, k := range keys {
|
||||
if err := s3.DeleteObject(ctx, k); err != nil {
|
||||
continue
|
||||
}
|
||||
_, _ = s.pool.Exec(ctx, `DELETE FROM attachment_gc WHERE object_key=$1`, k)
|
||||
}
|
||||
|
||||
// 2) pending-записи старше grace: объект мог попасть в S3, а commit — нет.
|
||||
prow, err := s.pool.Query(ctx,
|
||||
`SELECT id, object_key FROM attachments
|
||||
WHERE status='pending' AND created_at < now() - $1::interval LIMIT 100`,
|
||||
grace.String())
|
||||
if err != nil {
|
||||
return classifyDBError(err)
|
||||
}
|
||||
type pend struct{ id, key string }
|
||||
var pendings []pend
|
||||
for prow.Next() {
|
||||
var p pend
|
||||
if err := prow.Scan(&p.id, &p.key); err != nil {
|
||||
prow.Close()
|
||||
return classifyDBError(err)
|
||||
}
|
||||
pendings = append(pendings, p)
|
||||
}
|
||||
prow.Close()
|
||||
for _, p := range pendings {
|
||||
_ = s3.DeleteObject(ctx, p.key)
|
||||
_, _ = s.pool.Exec(ctx, `DELETE FROM attachments WHERE id=$1 AND status='pending'`, p.id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+33
-5
@@ -20,11 +20,12 @@ type Chat struct {
|
||||
}
|
||||
|
||||
type Message struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
SenderID string `json:"sender_id"`
|
||||
Text string `json:"text"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
AttachmentIDs []string `json:"attachment_ids"`
|
||||
}
|
||||
|
||||
type MessagePage struct {
|
||||
@@ -54,6 +55,30 @@ type ErrorDetail struct {
|
||||
RequestID string `json:"request_id"`
|
||||
}
|
||||
|
||||
type Variant struct {
|
||||
Name string `json:"name"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
type Attachment struct {
|
||||
ID string `json:"id"`
|
||||
ChatID string `json:"chat_id"`
|
||||
OwnerID string `json:"owner_id"`
|
||||
Filename string `json:"filename"`
|
||||
ContentType string `json:"content_type"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Status string `json:"status"`
|
||||
Variants []Variant `json:"variants"`
|
||||
ErrorCode *string `json:"error_code"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
}
|
||||
|
||||
var (
|
||||
ErrNotFound = errors.New("not found")
|
||||
ErrConflict = errors.New("conflict")
|
||||
@@ -61,4 +86,7 @@ var (
|
||||
ErrUnauthorized = errors.New("unauthorized")
|
||||
ErrTimeout = errors.New("timeout")
|
||||
ErrUnavailable = errors.New("unavailable")
|
||||
ErrTooLarge = errors.New("payload too large")
|
||||
ErrUnsupported = errors.New("unsupported media type")
|
||||
ErrNotImage = errors.New("not a valid image")
|
||||
)
|
||||
|
||||
+50
-14
@@ -373,18 +373,23 @@ func (s *Store) AddMessage(ctx context.Context, chatID, senderID, text string) (
|
||||
}
|
||||
|
||||
var m Message
|
||||
var attach []string
|
||||
err = tx.QueryRow(ctx,
|
||||
`INSERT INTO messages (id, chat_id, sender_id, text)
|
||||
VALUES ($1,$2,$3,$4)
|
||||
RETURNING id, chat_id, sender_id, text, created_at`,
|
||||
uuid.NewString(), chatID, senderID, text,
|
||||
).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt)
|
||||
`INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids)
|
||||
VALUES ($1,$2,$3,$4,$5)
|
||||
RETURNING id, chat_id, sender_id, text, created_at, attachment_ids`,
|
||||
uuid.NewString(), chatID, senderID, text, []string{},
|
||||
).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &attach)
|
||||
if err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, classifyDBError(err)
|
||||
}
|
||||
m.AttachmentIDs = attach
|
||||
if m.AttachmentIDs == nil {
|
||||
m.AttachmentIDs = []string{}
|
||||
}
|
||||
return &m, nil
|
||||
}
|
||||
|
||||
@@ -397,7 +402,7 @@ func (s *Store) ListMessages(ctx context.Context, chatID string, limit int, curs
|
||||
|
||||
if cursor == "" {
|
||||
rows, err = s.pool.Query(ctx,
|
||||
`SELECT id, chat_id, sender_id, text, created_at
|
||||
`SELECT id, chat_id, sender_id, text, created_at, attachment_ids
|
||||
FROM messages
|
||||
WHERE chat_id=$1
|
||||
ORDER BY created_at, id
|
||||
@@ -415,7 +420,7 @@ func (s *Store) ListMessages(ctx context.Context, chatID string, limit int, curs
|
||||
return nil, nil, classifyDBError(err)
|
||||
}
|
||||
rows, err = s.pool.Query(ctx,
|
||||
`SELECT id, chat_id, sender_id, text, created_at
|
||||
`SELECT id, chat_id, sender_id, text, created_at, attachment_ids
|
||||
FROM messages
|
||||
WHERE chat_id=$1
|
||||
AND (created_at, id) > ($2, $3)
|
||||
@@ -430,9 +435,12 @@ func (s *Store) ListMessages(ctx context.Context, chatID string, limit int, curs
|
||||
var items []Message
|
||||
for rows.Next() {
|
||||
var m Message
|
||||
if err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt); err != nil {
|
||||
if err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &m.AttachmentIDs); err != nil {
|
||||
return nil, nil, classifyDBError(err)
|
||||
}
|
||||
if m.AttachmentIDs == nil {
|
||||
m.AttachmentIDs = []string{}
|
||||
}
|
||||
items = append(items, m)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
@@ -484,7 +492,9 @@ func classifyDBError(err error) error {
|
||||
// Если ключ уже есть с тем же body — возвращает сохранённый ответ и replay=true.
|
||||
func (s *Store) AddMessageIdempotent(
|
||||
ctx context.Context,
|
||||
chatID, senderID, text, key, bodyHash string,
|
||||
chatID, senderID, text string,
|
||||
attachmentIDs []string,
|
||||
key, bodyHash string,
|
||||
) (msg *Message, rec *IdempotencyRecord, replay bool, err error) {
|
||||
path := "/api/v1/chats/" + chatID + "/messages"
|
||||
|
||||
@@ -506,6 +516,27 @@ func (s *Store) AddMessageIdempotent(
|
||||
return nil, nil, false, classifyDBError(err)
|
||||
}
|
||||
|
||||
// Вложения должны принадлежать этому чату и быть ready.
|
||||
if len(attachmentIDs) > 10 {
|
||||
return nil, nil, false, ErrInvalid
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, aid := range attachmentIDs {
|
||||
if seen[aid] {
|
||||
return nil, nil, false, ErrInvalid
|
||||
}
|
||||
seen[aid] = true
|
||||
var ok bool
|
||||
if err := tx.QueryRow(ctx,
|
||||
`SELECT true FROM attachments WHERE id=$1 AND chat_id=$2 AND status='ready'`,
|
||||
aid, chatID).Scan(&ok); err != nil {
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, nil, false, ErrNotFound
|
||||
}
|
||||
return nil, nil, false, classifyDBError(err)
|
||||
}
|
||||
}
|
||||
|
||||
// 2) Уже был такой ключ?
|
||||
var storedHash string
|
||||
var storedStatus int
|
||||
@@ -528,17 +559,22 @@ func (s *Store) AddMessageIdempotent(
|
||||
|
||||
// 3) Вставка сообщения.
|
||||
var m Message
|
||||
var attach []string
|
||||
err = tx.QueryRow(ctx,
|
||||
`INSERT INTO messages (id, chat_id, sender_id, text)
|
||||
VALUES ($1,$2,$3,$4)
|
||||
RETURNING id, chat_id, sender_id, text, created_at`,
|
||||
uuid.NewString(), chatID, senderID, text,
|
||||
).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt)
|
||||
`INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids)
|
||||
VALUES ($1,$2,$3,$4,$5)
|
||||
RETURNING id, chat_id, sender_id, text, created_at, attachment_ids`,
|
||||
uuid.NewString(), chatID, senderID, text, attachmentIDs,
|
||||
).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &attach)
|
||||
if err != nil {
|
||||
// Гонка на одном ключе в разных репликах: unique violation в idempotency_keys
|
||||
// маловероятен здесь, но serialization failure возможен.
|
||||
return nil, nil, false, classifyDBError(err)
|
||||
}
|
||||
m.AttachmentIDs = attach
|
||||
if m.AttachmentIDs == nil {
|
||||
m.AttachmentIDs = []string{}
|
||||
}
|
||||
|
||||
// 4) Сохранение ответа.
|
||||
respJSON, _ := json.Marshal(m)
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
_ "image/jpeg"
|
||||
_ "image/png"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
type S3Store struct {
|
||||
client *minio.Client
|
||||
bucket string
|
||||
maxBytes int64
|
||||
maxPixels int64
|
||||
}
|
||||
|
||||
func NewS3Store(ctx context.Context, endpoint, region, bucket, access, secret string, maxBytes, maxPixels int64) (*S3Store, error) {
|
||||
if endpoint == "" || access == "" || secret == "" || bucket == "" {
|
||||
return nil, errors.New("s3 configuration incomplete")
|
||||
}
|
||||
cli, err := minio.New(endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(access, secret, ""),
|
||||
Secure: strings.HasPrefix(endpoint, "https://"), // Лабораторный стенд использует HTTP внутри backend-сети, всегда будет false
|
||||
Region: region,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Короткая проверка доступности bucket (idempotent bootstrap делает s3-init).
|
||||
pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
|
||||
defer cancel()
|
||||
if _, err := cli.BucketExists(pingCtx, bucket); err != nil {
|
||||
return nil, fmt.Errorf("s3 bucket check: %w", err)
|
||||
}
|
||||
return &S3Store{client: cli, bucket: bucket, maxBytes: maxBytes, maxPixels: maxPixels}, nil
|
||||
}
|
||||
|
||||
type UploadResult struct {
|
||||
ObjectKey string
|
||||
ContentType string
|
||||
SizeBytes int64
|
||||
SHA256 string
|
||||
Width int
|
||||
Height int
|
||||
}
|
||||
|
||||
// PutValidatedImage читает тело потоково с жёстким лимитом, проверяет
|
||||
// сигнатуру/MIME по содержимому, размеры и пиксели ДО отправки в S3.
|
||||
// Возвращает ошибки ErrTooLarge / ErrUnsupported / ErrNotImage.
|
||||
func (s *S3Store) PutValidatedImage(ctx context.Context, objectKey string, body io.Reader) (*UploadResult, error) {
|
||||
// 1) Читаем не больше maxBytes+1, чтобы отличить «ровно лимит» от «больше».
|
||||
limited := io.LimitReader(body, s.maxBytes+1)
|
||||
data, err := io.ReadAll(limited)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(data)) > s.maxBytes {
|
||||
return nil, ErrTooLarge
|
||||
}
|
||||
if int64(len(data)) == 0 {
|
||||
return nil, ErrUnsupported
|
||||
}
|
||||
|
||||
// 2) MIME по содержимому (http.DetectContentType).
|
||||
ct := http.DetectContentType(data)
|
||||
if ct != "image/png" && ct != "image/jpeg" {
|
||||
return nil, ErrUnsupported
|
||||
}
|
||||
|
||||
// 3) Декодирование с лимитом пикселей через image.DecodeConfig (не полный decode).
|
||||
cfg, _, err := image.DecodeConfig(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, ErrNotImage
|
||||
}
|
||||
if int64(cfg.Width)*int64(cfg.Height) > s.maxPixels {
|
||||
return nil, ErrNotImage
|
||||
}
|
||||
|
||||
// 4) Полный decode для проверки целостности (память ограничена лимитом байт).
|
||||
if _, _, err := image.Decode(bytes.NewReader(data)); err != nil {
|
||||
return nil, ErrNotImage
|
||||
}
|
||||
|
||||
// 5) SHA-256 и отправка в приватный bucket.
|
||||
sum := sha256.Sum256(data)
|
||||
sha := hex.EncodeToString(sum[:])
|
||||
_, err = s.client.PutObject(ctx, s.bucket, objectKey,
|
||||
bytes.NewReader(data), int64(len(data)),
|
||||
minio.PutObjectOptions{ContentType: ct},
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &UploadResult{
|
||||
ObjectKey: objectKey,
|
||||
ContentType: ct,
|
||||
SizeBytes: int64(len(data)),
|
||||
SHA256: sha,
|
||||
Width: cfg.Width,
|
||||
Height: cfg.Height,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *S3Store) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, string, int64, error) {
|
||||
obj, err := s.client.GetObject(ctx, s.bucket, objectKey, minio.GetObjectOptions{})
|
||||
if err != nil {
|
||||
return nil, "", 0, err
|
||||
}
|
||||
info, err := obj.Stat()
|
||||
if err != nil {
|
||||
obj.Close()
|
||||
return nil, "", 0, err
|
||||
}
|
||||
return obj, info.ContentType, info.Size, nil
|
||||
}
|
||||
|
||||
func (s *S3Store) DeleteObject(ctx context.Context, objectKey string) error {
|
||||
return s.client.RemoveObject(ctx, s.bucket, objectKey, minio.RemoveObjectOptions{})
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Свои negative-сценарии lab4: 413, 415, подмена MIME, чужой attachment, вложение чужого чата, restart/failover."""
|
||||
import sys, uuid
|
||||
from pathlib import Path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, Failure, run, LAB, check
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
# 413: > 10 MiB
|
||||
big = b'\x89PNG\r\n\x1a\n' + b'\0' * (10*1024*1024 + 10)
|
||||
c.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=big.png",
|
||||
413, actor=alice, raw=big, headers={'Content-Type':'image/png'})
|
||||
# 415: text/plain, MIME определяется по содержимому
|
||||
c.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=x.txt",
|
||||
415, actor=alice, raw=b'hello', headers={'Content-Type':'text/plain'})
|
||||
# 400: битый PNG
|
||||
c.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=bad.png",
|
||||
400, actor=alice, raw=b'\x89PNG\r\n\x1a\n' + b'\0'*100, headers={'Content-Type':'image/png'})
|
||||
# чужой attachment_id: eve не участник
|
||||
att, content = c.attachment(alice, chat)
|
||||
c.call('GET', f"/api/v1/attachments/{att['id']}", 404, actor=eve)
|
||||
c.call('GET', f"/api/v1/attachments/{att['id']}/content", 404, actor=eve)
|
||||
# привязка вложения к чужому чату: bob создаёт чат с alice, но ссылается на att из другого чата
|
||||
chat2 = c.call('POST','/api/v1/chats',201,actor=bob,
|
||||
body={'title':'второй','member_ids':[bob['id'],alice['id']]},schema='Chat')['json']
|
||||
c.call('POST', f"/api/v1/chats/{chat2['id']}/messages", 404, actor=bob,
|
||||
body={'text':'x','attachment_ids':[att['id']]},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
print('PASS: lab4 negative suite')
|
||||
|
||||
if __name__=='__main__':
|
||||
run(main)
|
||||
Reference in New Issue
Block a user