From 14166003b827e2ce52e6aea45df86f4ef0c64393 Mon Sep 17 00:00:00 2001 From: morodik Date: Wed, 28 Oct 2026 14:12:41 +0300 Subject: [PATCH 1/6] =?UTF-8?q?=D0=BA=D0=BE=D0=B4=20=D0=B8=D0=B7=20lab3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + Dockerfile | 24 + cmd/api/main.go | 115 +++ compose.yml | 121 +++ docs/architecture.png | Bin 0 -> 24037 bytes docs/architecture.puml | 24 + docs/track-5-report-md | 52 ++ go.mod | 226 ++++++ go.sum | 862 ++++++++++++++++++++++ infra/haproxy.cfg | 31 + internal/auth/password.go | 76 ++ internal/auth/session.go | 15 + internal/cache/cache.go | 67 ++ internal/config/config.go | 37 + internal/cursor/cursor.go | 45 ++ internal/db/db.go | 42 ++ internal/httpapi/api_test.go | 983 +++++++++++++++++++++++++ internal/httpapi/auth.go | 110 +++ internal/httpapi/chats.go | 131 ++++ internal/httpapi/health.go | 39 + internal/httpapi/httpx.go | 98 +++ internal/httpapi/idempotency_test.go | 184 +++++ internal/httpapi/messages.go | 80 ++ internal/httpapi/middleware.go | 41 ++ internal/httpapi/race_test.go | 163 ++++ internal/httpapi/router.go | 84 +++ internal/httpapi/sessionstore_test.go | 73 ++ internal/httpapi/testmain_test.go | 145 ++++ internal/httpapi/transaction_test.go | 63 ++ internal/httpapi/users.go | 117 +++ internal/httpapi/validate.go | 45 ++ internal/idgen/uuid.go | 23 + internal/model/model.go | 65 ++ internal/sessionstore/redis.go | 73 ++ internal/sessionstore/sessionstore.go | 18 + internal/store/limits.go | 51 ++ internal/store/store.go | 436 +++++++++++ migrations/000001_init_schema.down.sql | 7 + migrations/000001_init_schema.up.sql | 49 ++ migrations/000002_idempotency.up.sql | 11 + scripts/check_edge.py | 51 ++ scripts/check_lb.py | 65 ++ scripts/explain_messages.sql | 69 ++ scripts/failover-under-load.sh | 20 + scripts/measure_failover.py | 91 +++ scripts/restart.sh | 4 + scripts/rolling-under-load.sh | 16 + scripts/rolling-update.sh | 18 + scripts/stop-one.sh | 13 + 49 files changed, 5174 insertions(+) create mode 100644 Dockerfile create mode 100644 cmd/api/main.go create mode 100644 compose.yml create mode 100644 docs/architecture.png create mode 100644 docs/architecture.puml create mode 100644 docs/track-5-report-md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 infra/haproxy.cfg create mode 100644 internal/auth/password.go create mode 100644 internal/auth/session.go create mode 100644 internal/cache/cache.go create mode 100644 internal/config/config.go create mode 100644 internal/cursor/cursor.go create mode 100644 internal/db/db.go create mode 100644 internal/httpapi/api_test.go create mode 100644 internal/httpapi/auth.go create mode 100644 internal/httpapi/chats.go create mode 100644 internal/httpapi/health.go create mode 100644 internal/httpapi/httpx.go create mode 100644 internal/httpapi/idempotency_test.go create mode 100644 internal/httpapi/messages.go create mode 100644 internal/httpapi/middleware.go create mode 100644 internal/httpapi/race_test.go create mode 100644 internal/httpapi/router.go create mode 100644 internal/httpapi/sessionstore_test.go create mode 100644 internal/httpapi/testmain_test.go create mode 100644 internal/httpapi/transaction_test.go create mode 100644 internal/httpapi/users.go create mode 100644 internal/httpapi/validate.go create mode 100644 internal/idgen/uuid.go create mode 100644 internal/model/model.go create mode 100644 internal/sessionstore/redis.go create mode 100644 internal/sessionstore/sessionstore.go create mode 100644 internal/store/limits.go create mode 100644 internal/store/store.go create mode 100644 migrations/000001_init_schema.down.sql create mode 100644 migrations/000001_init_schema.up.sql create mode 100644 migrations/000002_idempotency.up.sql create mode 100644 scripts/check_edge.py create mode 100644 scripts/check_lb.py create mode 100644 scripts/explain_messages.sql create mode 100755 scripts/failover-under-load.sh create mode 100644 scripts/measure_failover.py create mode 100755 scripts/restart.sh create mode 100755 scripts/rolling-under-load.sh create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/stop-one.sh diff --git a/.gitignore b/.gitignore index b5a71896..e723c134 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +secrets/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..e9e55077 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +FROM golang:1.27-alpine AS builder + +WORKDIR /app +COPY go.mod go.sum ./ +RUN go mod download + +COPY . . +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/api ./cmd/api + +FROM alpine:3.20 + +RUN apk add --no-cache ca-certificates wget \ + && addgroup -S app && adduser -S app -G app + +WORKDIR /app +COPY --from=builder /out/api . + +USER app +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD wget -qO- http://127.0.0.1:8080/health/ready || exit 1 + +CMD ["./api"] \ No newline at end of file diff --git a/cmd/api/main.go b/cmd/api/main.go new file mode 100644 index 00000000..10c70ed9 --- /dev/null +++ b/cmd/api/main.go @@ -0,0 +1,115 @@ +package main + +import ( + "context" + "log" + "net/http" + "os" + "os/signal" + "sync/atomic" + "syscall" + "time" + + "mtusi2026/internal/cache" + "mtusi2026/internal/config" + "mtusi2026/internal/db" + "mtusi2026/internal/httpapi" + "mtusi2026/internal/sessionstore" + "mtusi2026/internal/store" + + "github.com/redis/go-redis/v9" +) + +func main() { + ctx := context.Background() + + dbCfg := db.Config{ + DSN: config.GetEnv("DATABASE_URL", ""), + MaxConns: int32(config.ParseIntEnv("DB_MAX_CONNS", 10)), + MinConns: int32(config.ParseIntEnv("DB_MIN_CONNS", 2)), + ConnMaxLifetime: config.ParseDurationSecondsEnv("DB_CONN_MAX_LIFETIME_SECONDS", 3600), + ConnTimeout: config.ParseDurationSecondsEnv("DB_CONN_TIMEOUT_SECONDS", 5), + } + if dbCfg.DSN == "" { + log.Fatal("DATABASE_URL is required") + } + + pool, err := db.Connect(ctx, dbCfg) + if err != nil { + log.Fatalf("db connect: %v", err) + } + defer pool.Close() + + maxMessages := int64(config.ParseIntEnv("MAX_MESSAGES", 10000)) + maxConcurrent := config.ParseIntEnv("MAX_CONCURRENT_REQUESTS", 50) + + limiter := store.NewLimiter(maxMessages, maxConcurrent) + + cacheURL := config.GetEnv("CACHE_URL", "") + if cacheURL == "" { + log.Fatal("CACHE_URL is required") + } + + redisOpts, err := redis.ParseURL(cacheURL) + if err != nil { + log.Fatalf("redis config: %v", err) + } + + redisClient := redis.NewClient(redisOpts) + defer redisClient.Close() + + if err := redisClient.Ping(ctx).Err(); err != nil { + log.Fatalf("redis connect: %v", err) + } + + sessionStore := sessionstore.NewRedis(redisClient) + + chatCacheTTL := config.ParseDurationSecondsEnv("CHAT_CACHE_TTL_SECONDS", 30) + chatCache := cache.New(redisClient, chatCacheTTL) + + s := store.New(pool, limiter) + + instanceID := config.GetEnv("INSTANCE_ID", "") + if instanceID == "" { + instanceID, _ = os.Hostname() + } + var draining atomic.Bool + + mux := httpapi.NewRouterWithCache(s, pool, sessionStore, chatCache, httpapi.Options{ + InstanceID: instanceID, + Redis: redisClient, + Draining: &draining, + }) + + port := config.GetEnv("PORT", "8080") + srv := &http.Server{ + Addr: ":" + port, + Handler: mux, + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 15 * time.Second, + WriteTimeout: 15 * time.Second, + IdleTimeout: 60 * time.Second, + } + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + go func() { + log.Printf("запуск на :%s instance=%s", port, instanceID) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Fatalf("сервер упал: %v", err) + } + }() + + <-ctx.Done() + log.Println("Получен сигнал остановки: readiness -> 503, жду LB...") + draining.Store(true) + time.Sleep(time.Duration(config.ParseIntEnv("DRAIN_SECONDS", 4)) * time.Second) + + shutdownCtx, cancel := context.WithTimeout(context.Background(), 12*time.Second) + defer cancel() + if err := srv.Shutdown(shutdownCtx); err != nil { + log.Printf("принудительное завершение: %v", err) + } + log.Println("Остановлен") +} diff --git a/compose.yml b/compose.yml new file mode 100644 index 00000000..0683f031 --- /dev/null +++ b/compose.yml @@ -0,0 +1,121 @@ +services: + postgres: + image: postgres:16.4-alpine + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - postgres-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + networks: [data] + deploy: + resources: + limits: { cpus: "1.0", memory: 512M } + + redis: + image: redis:7.4-alpine + command: ["redis-server", "--appendonly", "yes"] + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + networks: [data] + deploy: + resources: + limits: { cpus: "0.25", memory: 128M } + + migrate: + image: migrate/migrate:v4.17.1 + depends_on: + postgres: + condition: service_healthy + volumes: + - ./migrations:/migrations + command: + - "-path=/migrations" + - "-database=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}?sslmode=disable" + - "up" + restart: "no" + networks: [data] + deploy: + resources: + limits: { cpus: "0.25", memory: 128M } + + api: + build: + context: . + dockerfile: Dockerfile + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + environment: + PORT: ${PORT} + DATABASE_URL_FILE: /run/secrets/database_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + MAX_CONCURRENT_REQUESTS: ${MAX_CONCURRENT_REQUESTS} + DB_MAX_CONNS: ${DB_MAX_CONNS} + DB_MIN_CONNS: ${DB_MIN_CONNS} + DB_CONN_TIMEOUT_SECONDS: ${DB_CONN_TIMEOUT_SECONDS} + CACHE_URL: ${CACHE_URL} + CHAT_CACHE_TTL_SECONDS: 30 + DRAIN_SECONDS: 4 + GOMEMLIMIT: 200MiB + APP_VERSION: ${APP_VERSION:-dev} + secrets: + - database_url + stop_grace_period: 20s + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"] + interval: 3s + timeout: 2s + retries: 3 + start_period: 5s + networks: [edge, data] + deploy: + replicas: 2 + resources: + limits: { cpus: "0.5", memory: 256M } + + lb: + image: haproxy:3.0-alpine + depends_on: + api: + condition: service_healthy + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:8080:8080" + networks: [edge] + deploy: + resources: + limits: { cpus: "0.25", memory: 128M } + +networks: + edge: + data: + +secrets: + database_url: + file: ./secrets/database_url.txt + postgres_password: + file: ./secrets/postgres_password.txt + +volumes: + postgres-data: + redis-data: \ No newline at end of file diff --git a/docs/architecture.png b/docs/architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..8af3b50d601a725a3ecacb0f5c4976de9beb185a GIT binary patch literal 24037 zcmbTdbyQrzmN(i+aCdhN8axo(rE&KJcY?dSOK=Fm-Q6{~ySrQCjq`EuyEE^8Gqc{T z^=h3zx=z*k)joB0)!zHJtHTxLC6N&b5C8xGvb2<#G5`QE3jjcl!a;r9Ndl(cepCw5Qw2jt#KgpeEE7d+)5YvFghEDz!zV?f=OyD7q@<)|oPGmc|0uZS z$otny#m)f}7ZrXjX=rGuy8l-5&R32eRE(QY`8BJUw4|20Y+zuZYGPqy7+h)+USXCw`6FY-Fni57XU*K)+}hsRCaTOPw#GKD#yor8E_dD{ zf5WbL)5XQ*XF{E8R`1XJ1^a>}*WyL{;!UTrEq8Z!w_i1GsZCxPEp8{`0&>FMd7 zT<@LQ7?|DaTRiPsI{UkJJv1~lvamb0v^TK{8d|@Z*uI_qD60qao5u@V$J4uaYiny8 z2j@$N_iHEMjnjw2!^5q^i|r%OAqcz=dOWy%ytueH1>Ig;-=ADR-QC??gI~Z;ukY{g z?qtw5AKLKQNnF#($j;u~+QifeAZcQ2;_%(c#F*@dJDItYlf5G^3yZz=cUvcC8*64G zI~$jY5mEpE0m?#E)9F9%10X(d+%nJgfU&Cs_?@R3)fps2efr)?R1Hgvj7rMs-SXV} zxctUYfwI8*AZlgm61S{wzjG4p?xTvLOb>bw=g4Ngdf+o|ao3c=OxpHh96Lcmg3EBd zmun3J@?jcydo(SSMUvvsf7i0HO?!+mIvcN0zyRoPf7~j2SiD@{a2U$o*P*LdPGoqR zd<;3uKUX2{_;pbZx)szs+wyZ-_3v|0@eyt;2^VhqE+ioizZ$s+yPP2@a}Cs@{h^Se z__kn`4~rOKa7oU%wav77Cgf_(7~j*HpU(w6>}Q{!00Z|;_xH=n4)6x*DR%%^0rdKG z_r&RO>J56u?)2R6;CrZ^=JABkWBH9zk@R)`4A&K>`={SmC@$+ArH7|2(d|_UQ9m4| z<~$~3O%wYkc!&F%;J3H4;Y1aPWFq*Lk&d8mNP3RQqwTbR40*uNugZ?Z!t6zm4te8# zfpk`O8rZw@>m-t6Nsqv1xgL!b%L@SrjSEBDv9V7`$$$C;yJ^b!n;Q06qDwsK zowS*9y>j&UuTeMCu8`4II&i{0;7sFIph*##TJ`GtRkhEZuhF`ln_^PV2PWq0=O!}*2FGGdGGyV!Ld6oF6u(0ZP7pg+o06)5?&;>%-00S6P(q2w!+? zcsglVwHeBQ)DyF%HN@%-&P2arvvLBBqQ$RG)3io}F*yqEu(I2iU(OnpSd+K~B?Eg& zolWYxjnZTT9GpbA!%V8rQh5L_XezL{3u?=%kR22j#uh3A&65G4PdzH=nP!YOeGm#| z6m&^BSmv@O4?%1y_#*y0d%G$j52`fE@WiD^Wl{;LQH{z)71|>OsxiLQ`q)XO&{;w` zpL?iPi)7;9f6-^i;+Lw1!%<_EU_?=H<}O(mhz08m7l?T9aH`{zmm=f|MV5-t{>m%8 zgvX-C&0-TP`TSGnNLfuve5RkGK8e?6cItj0e=9Z)InjCect^E6e8#C;(478W1T)*O2N28VTibWpXBDab;-53J?p}Wr`5p|1}IcyB;0C{Fze@ zm3G>ua!Xd)J`n&kDu^# zmT9o_+73YTyK=K%u3q5JBM&BlKUoM^oMcDn+A{($HNgK=CTS(R5cywWU!umC$c}97)HLghfrXWu}LZbEM@@`MP$}<(om^=+0{S! z;Ezn_Ldz2Xg}PJ~l908;-Roo!uS9;*p#WA`-xVszxFR1`^)G$}J@v>DkRkI>XwTir zkM+aQWM!x1h^+=wDZj>^@@#88oBon4=Kl0VEWKjOl6S7XDdAl$smtPB6;?OnOIt>n z5SwX{;w21+aOkJXilnNIN9!IHW2H)>zhp;0YZQfy_>FQ+g%GZy9U}*=#R9$Oh-OR) z7sr4pyZ{eqVM8}o1`O$t#ui9y)bZ4QzXJp^xwg7#=0P=)_k>089aqLf;3;LXK%qHz z4t*jOast@{enJd;$xJ{~g_@H=5(#NTd!;SI84e-U3JuL9W3W+V*H$2oR`i~!BGX|{ z8CktFPYOWQViJ?im_WE4)_>Wx%&gKwKt^i;B*EM^`za}PyK(!?fg;I;u=##A?{K3{ zff)SbD89!*0b;VnxEMlNrn|UcilG4pULi%8nRl*ILa0y@(1v)6@KbsMfWvkrQepld zsSv41SUYE!9oqn|kUL*QA(G~BiXyoJ2;)U5v!`u*zG+U3EQMrc9QQE3O{S>O^8pUu z&*G*Z_WB1hS&~9N^SCHw1DkWhc)eIsL^i2qw}cRH#z<S4u zN0ctPMK5O=l}l<8c_ctr(x&kg@pLe24MvF~Dh5Nm>=XG9ug_wAANxsfua}!PZ+Ru~4kR?u~@rjt*{(7w{e^Qqvuyg;) z^NbiJyU(6f@RR=B1edGt_r%8{Lf-6lHLEk|{DSqGD(1M78G8PRB|67{6SbO=PG$h5B!9%C0n zWoN=h6P+5hgkuUZD>nxYf)>zoAVFq8{yz3z(4y2krUHb@} zTtPJ9VyDx8jR}zFU~CyzSk`__CmqjpV)W3qKkxIV#2C!E9V`EURd+%@9*o@wS4f_wHZNYG^vPqq5+Wbbix<7 z=u%2b)}QXsz3SD98BRLgNAaK%_fTHv7368++!DO(_x zqp6mSG1=Orm)^fRt}g4;f?JF?+LzM49*(84-wbDdJ@lFsZ@|VfmdKofR}ABcD7+S( zR`sDvA3+sYdCW2YEQkf%56vxkQRGM>{a0f#2#xm>Gh@-@ebG$Y{lZ00KcEj~@RSrO z29<8>;=Qbm>vC#`TK@>ze(t>x_O&$ob1!UcL(d*9B+}{dDxb2!9x3S9UmT)fYbm^n z6z*HS8+vr%ogmZ@z4glz<#X2J zRaO{y50ZR5HP%pzi9d6=GT%>aSUod>AZQ*OCSZwUeTYfuKUzon9z_KCx%+c1Byq8} zn&hJe6Jq;7zF3+|O`8x&H{CgcgxiV=c~D!#;}xM#-R>~y*PG8SSWZ68Am4b@nz8+8 z1?jWr>KD%QoZra;WXE&{ynxG07?4st z)tVgQtW`)(?G~B`4h`Ri5wG*7`K^(=z2kWAG!X@c=mK($sLYi*r8>9-?eN7|lo`tC zccbM?ibX9{+#?ub~Q zu0e9?n_pG!Hgwd1q_|@50?#%}YpIbwu1L{~FWn(Yd%vrgrFT5CL0xF2qtdq7Ak@8G z_o`(7$8W!^S&qc8`*rygT5fx1eo&bF34X$q)um40tOu(5M32<5zOYDcCQ z%;<#vYC(@ID?`Dx)pWa;t9WhL?fQBw~r+ACT7F&WL4PPl!2phr2gec_2qYPukdG=VB5f0re}g&p>gh?>md~ z&OJsuxCx=f*ZRR>Wa(@PBK`lizW+ZL|9@``^Vm!>&_#Q~38Xp2&rv}mQCgf^VYp?2 z$u`cxM5B?}VE7;QkpB+z4*@XnRWfx~>U$eoPz~owZ_`nQ1(&@}X=!m%iY)~JtmQHS zP5`7t0*$X+cfr2&$COXtUk`S4&7W}A_zs=V*#TdDzg~g0TzjHXnOE+|jWD_GTA*2v zgD1JG^pQ4O>@IXxvH^1XNU^aPywklDF2FkOmr3u3 zBl~sSQWUql-i{0$d01UW1Ah!h>h48wj0`%cz)tiU$Y+2SN z9;*~B`PE8^3>;kzmiPaK_6q*BqnCq8MxTH6`!jSP$A*f+oK`SQ#q7-76pNEge;5|$ zJl5nSX@Po=5U8cJV_l?3!b?g~<@E$#dM-x7h%WQ-pk!2pR?1omhl%vhc27AC8*>e{wE1@h^I=B{I8+7^uqU_4Jos!Y}8<9 z(I4rSGTui`bh1ar;fUcDyptLNr#0$hVUj=6C+^ZVS}5D05@0cAZxOO9d2QmRhX}(h zPITIXpmS!aKT{Q!hybMIX63UhJ0jFFWBWO59G)B+7?7pCs0u9#CSjgmXEFlm(?7cK zo07(4(FVyUH^-$G1`c{@_JAX;!d-geQwuL`8y0aJ&W-jRPeSH6oZEE|w`U%kCgOf1 zuqe-BwPshjFwi!9Zw z%1H%~Nrq5>RN8{q)pp{D2fN&y&#|?pPEbBWu%tU*?JEm7qiZRLXsZ|@X;zOY&Igf%bBfj6^z!6ZxMXw?y^ zlQb|v2omc_;fE?o)R18Vt6pNapI}OHC(wE0DTFD~X?!MGz5YBW2Fn-n#Dx_u${~>{ z&IFE-Ax3d<0Cmw6?IwQ|D9}j|q|+S2Vlk3S9tA}}T9~J!Z`t>Ml{r9Ep62+{ZCr?$ zrKm)P41^-)%={d^8qRj@0jHfbMUMt`J);Vd_u; z&pzEid~)n8NL>~BobJ+zBW3XX2L9#Gju*bvuOKK;;Fc;Vc-X|(nzV>Ti4KIVct%0Z zJKh%-S0V^EkFbZ!0t}NygGxkL|Fw&eXMyMg!a_;zHF{W-r&K#GJuvHiYa(B!ovRyh zj|?`=Zw-mBg6ZMSExjOIyGT{s&Z|ODb)G06Zo)uuEIvTVcY2hG_t8wg6lXVS&*uOp8rw|jr>arT`T5x`oeV+ptQ8}x1hOn)G^br-R(h&{bOwdR^Lo`40Rev|0 zTp{-W?$SUhA}-gKUG_9z{RxTcDmI*|wjY3^67ZP@uecJyT05Xkd0@cMb6Uv|Mx8gflGK(*VGmpO}yOf!?x< z%CP@zQ(D@^Ju0d1?xOjwZqfK<*eL~?PKbURydO3E=CIZ6Kf(;|mO3p9lCRzhd*ctr z#D*~ ztYY(yv7?Fue})eKzunyb*+l(s4l-qOI*oE3GAB&*6gQLOkCJ03w;%sk^P4YCUw)LjUc=nU8Jb!z@P8J_TiOV^aA81^JZ`RlGhiD(e@Q?}H^+Xnw z!TDOr_j@)7FhUW7R|4kE7#alW+AZd!Vr)^847`A0LU; zF+^}(2@A;!J;JyBIm9Y?Hwklc3Msiwm33KqW1<2J-AQ=76zQ)q+V0hWtuB40(ByJh z-^t+fv!K&%&hF#OObwVWM<*PXEm2TAA`=~;?k)l#ggDsUf_Hu^aLa9l+s)$PGufpG za-7P18AAO-6#M|y*5E*n3{~ux;u$kSR?k-5nJqBY*K$U_v>1*$VfZ?qht2_(}2Bm2&k^NK(u@ z+?@FW_|q_Sx;C*v`+FDm%%4HwvayqbuEI>g=V0C8;cXYnw{0Ivzxms3m;F=@{`06l zp6yRHY4jVHd|xMim=m1=-)34bm4(t zO{pDinIMr_lfnJ>PKGqx%4$GZ+vv2F0P@_a(JNlA^x5F&x9cS7-NX0VjIYXqK)d&s zA*u;1kSUc*}uX4m&7eRSh40L#|-s_T(mOSX&U>fNnTrM;(5YW(H6 zDW2EC!JpT_I#)UN@asxLRMKafRXyhfvUsuL?L;>GqrU ziPD8pV$zqt@jt5hmi$mcR~INDVO6FmUVl$cMB!W}Q3rUPvn6jTXY_FTym(rR6tQZD z9r8zWPQqvT6_am*o%VA7$ME!D>lXdaB_H3v@)HS1N&bD{(JB;E7E~`SHUFdzyFpR_X;EL#t$1M``jfekiz%AlLgi@r(#lY`FPuWZ zU1U?=TuN7^n|&GF_lC5yRx@D2UmeOywE7%(pJ)m7^K9t##HF@^o&9Ue?|Ybsd|viP z&GU*VcIP+Rc`;Kx8va8MYDS+Xf2D*7jdUs9S<&I~xHJTCU%?x9X$m#qD(=3!XR@k< zYprTS-L=*YpJ-i$gZN-}V@fHj1*UPnfpsD^#?lF)4mzY@HjekoaNJ3weMW32y|Ix- zKa}I4o`7_z!Ej1sqbHI$E+^!tbH!gdx`kE93c^e2_9QgMxU#hg!Gq=I2IY`#efH4$%k@VxMAl4vY~3;pnzRY@KN`P}&? z6fjI(JizvFT-$?cr|wE*5y)_mhiU<|ab;`luqNaqZUW;pcun7?PJz*x9cFG{daaMl zd#ki;a(qZKTdVOrB$Wt{EL&maJFbceV$5>A8}?tZk5f8@A)0#He<9jcj9r*prWb~+ zNXJ$Dg!3l+PFV!9P=OJGRhze#oi8lp9VjR?oz0kw8Jn9++oQ%S&xB6a<>6Q?UcfHf z_S;=hb-sTQJ*r{Ws z)@JF)=+8dKH-573JG7&c&xAq_|8-mt>W9c)*T`bo=g^oGk_o6GqXg;%?+s$15kS_?PULwD-kIRe=*xJ84J*(rTfu^GX_(V@ zf_*ugquw*aeJCN5KC?ywKV4hsyOS(f2-AI)UaLD~db;M@$PR~^Q8Gq8JAwlhH!@-{ z>1uuNQD00Sk+V04E)|B50Ej+Hkkjl4z#bo?HvhO(equ71bIi-B|CBHyrLtSSj#=$x zA=(*Q;usPsiZCQO+a2d1m?spk)S}@p{J&9w|9#UmJTHQ6RLtNzq?4TC^;p(XX%w7+ zIM#aLf1-?*^Ew_l=+Ft0&lHh*@W4(2{a>`=Kd{ti`GV!cM5Kr=w+Adgcbt?!4rmao z4txGr3bHz#1dajcQ_iIttlAmdj6}_76w-eoDr-ii2>&uu3Sza54GBd!>d$8aQq#uI zxD6t)${)>zp_UwUIOF(v)J>J3(JFf>phLHSQI?S1YQ%!h z>oC<{Z-cb~&oP+ly{S5d`z{Ziu=0Ib*&te)%_t3`P1`S(-&9b+WgU=s)SbOZ+%PiQ zpEd~v%QYd?%mYyw38kho1i!WFBeHP$|1K}fDEbEgZfSSah7J1~0;exzb>|=MKED-y zAv4T;IB!?stumAm%T{sibbZd83p$}M{M1jIQac(YOp~dbU40wSV0l>2aXQ;jknE?o zU~kHncXc~s}Usf6-8maQ9V^0X&`AN)-TKPTho62VQ? z#rj?Kj|Wa6_lMQH4b;K2=A~=T zNcLS5SYVx!R{7W+g}~F2sKmSyBv8{I^;k*V$r8Im+F``G(bmD3aH{OmRm@v z(UXx;X7_}rw`?Nrtm3=B%YR$WfegzI^%|QdUv6I{&YG~A-0$BsIezF{<9H^)7{CM;sDgN^fp5)w~6pW^O6zit)mSyP}mV znd^eDu#m|5jw0xD$1kp(O+wrSy*9&QVa%_8FwEvFu*%)=a4+?`9xYPDYwgt$!8Ys( zaakI{WZ;CU#jenpTqyFoxM(~Z8p>#oMl1xZFe37x7xbU;j(L3IU!T1u=hQkRPgHS8TICLtDH;}RI-d5rK`SJ;i$X6* zytjdJtDyl^Tb=MEDZgvN*mnM@864o!x%^FZoS48646*9wtrA-KzD1ogi)Y~#o$`HC z#X>^?U!V>WYZ5xO3)i$2C8beaZQn%j9^3`BM+2YLFST1{_s9Br1BiKum)*cem#D>? zTB{^5N<=`Pk@@p=i`py5My2`vB3ImSm10tUlWqD|;^nVc3-i2T{W;y5NQ>qAO6=Tk z7*3!(iZMdgH60lyFy#t$=1?D75vs#Qi!3KURF1=06vrfxHO?~I=F>afO{z4Z)~bmF zodW?&-WKV<ZAkwN{SmYvH7E(|A z>^IY8XrW5A$_|BI(bDu@=-c1Pa9MCaDYSrQfjL^tfqs-xVxt`Z-ary#&R$jrseAt5 znGg#RTH$;#ulBJna6jF?{JN={b!EN1aFi`Q^bht}E!T3}!Hcl@B(bQmGq#9)pSbEm z1fPE+b*1K*^Sa4s+oB+&Gtbx%-jqJ*)|tgQA-F8a8gd};5MTGXOC4#KL`r3L)$Q4i z13rtOhWyEWOb(IV(}fFrg79^o*{^r4Q7h$Ld;_AX+w{jz6s0C&gh6$-2~$=kpw${e z@Z&jq*AlH$t5Hz?z$l&%#&}? z0sKut$;;c>(Ds9d?M?+l)H22eIxZy5dJ+$0u@=TSGsko>K&W=ip~sP~*V*2)P45eQ z5zbFfk6mxsAEZu-i@{CyyJfyjT!jm9%eUFkh>LK_RCv zZ%@}RSySJfM~n7QXHd2z!W6OLmhDr4Tay6-lRyR zWM_i`2C6*%_;Q*09J|7P0?J3K;m8*I9a3YI6Y1es)JYHKJ7OQ{lY0|0JruG{TzvxG?6&uLb*N?# za47R{KS=D5fAB;LDe+uv)uVK7WKeBLXdbMvJ6s9z>4jwoUEU;y8)I0WeO=4`MFlrA zZ?Z>D;qWCE={G&!Ra5Zql%Q&yj%fyLvODBHXlR6wh7~}y_B$VjAv;7j5@%tE3SG5S zIkS;Nx_|1`7ZWyWe`3qbMLMTevEYx%1z@fKWSBJo<#K@vO`Xlof+0O9zHngC<(t4o?H0f z5*))&f>=9jUacJw?S1`3gn8)-yS19F8S_!vZWK-~dUmQ7awgzlJhhs9ak+5_o?XAr zYjb9xNRh0r6a$EQQa%R;qfRvHYQbrpTZWsRJD*d(uy$s(!bOasbjf5dbZj9-=`!%Y zFl(nDzL|+P{v$b5Ta*akzc`U>`VuR2F8Gf*OC}O9|DxOa|HP(AM?T-7yGOf61*%P6 zqyjz2D%6?H$=9dD919eP)01#Rf491}bbOOSEy08W+R6G7yvBH5{)L55-Gb)fk6$iX z@mdWU{hah@H`Z!NZS}mMVpd&wTk__%Me+N#REV$MS3j1N)PSUxEM(UtGd9Q7=*s98 zqHXq#V~_tSdvZ3WZeXuuJFc&QF9iNpnH^bKiJwc_Kw{!iR)5(>x?t79 ztXj5~CcU9RDV~D++^52 z7JoY|I?Ch2A>P{;_O80y8y4`h&`8Su%piPS{^YS59D4m{%XG#6YH~D~x=g2uSjOeA z&@&mTrvCjt%8uC)>E%2U1Zo5Cd1L|G41wHh2sdnZpPfF;xwcGoa&+)i4Ai-&{|~SZvQMolGeX zyW??PmlTi+n|F`EayV|;RNsHF>BDF!B6W7G=zUs!v_}K z!A?o!XLkJw^2{6*mF2ia=68!8I6C+nVS`p04?WsH-Juozm09^$@3I_|Ijf)G zIDIk24Cufv1SU4ZQt*(y=L`pG(v0YOxCYMqG+z9hOWNt~#&U3R`oXsTwPWkj9PHu@ zVl0QkLSF(_{p|0&!wNe?ewAd{xKW0K>bC{<(Kv;82AGM^Z*^mUo0;4M;`Nv+44Cf%UPC}IBiHhzKY<6-=;TygG)?% zV@-b<(~(G(62weDzSGNk!P8BJb1e$>epEeRJbSjGS2#M=Cb`Lc{V87n}i#kDXiL zsD6-WxRQlEGe@;2y4Z3=o0lBKHvlopmsK8&sjfc|$;{i$ zbTTOs7Panb>u7s6VqfLiRo%fjaqV4oAKY3Uv5fz6ZzhzJ1fP|<*YV_bs}ZZOHPH+IIvE)L)2S!@tCmV(M$0Ty^XRXU zkgVw3%9ZYcxjT*GRAVoK&-$6Qu^n`YaLgn$D6+@2ZGRA0B=^u2Tl)tV7CJf$TNBK- zgJ346KbajE1Ji5|+%Nq7p_8R<8$I=JPBVyfu`}!eDEuZ8j{bs12 zT<|l)Uu_ju(hud9dXMchzeW&QXXY`G{S)TY!KL&5bM=9|k9|t)sM`pU{TD?g9?r^0 zHuUN%)}K?;;MsePCcgx-0~h>>8R?4^@oz1+-mZwW=gnz+rsJ86BnR^PACI1{eRI0N zBkkbRGgYI3H0>J@HsCysa_ufQN3-}LpdZrUlrHl@0XjC>v&Aq+Qd=Du|KxUc=vnvD z96BJptNu1ENznYyF{^*Z6{TROFO+sb4TvQaqs@cpKfcX{J|EiVOHEsFqO+40BR)H= zXJcE-O?DIe2LjKV>$*zHYPvUdKnI;8Kc=G=bIc;hb4qLd2N;1amFhdD$@2#W2YP+O z=y>li&oyImR9fQS^|j%F?Q_w1E3jFB5zC0!mP{l=%pr<37-cATNnxGU(|R2#HHvcb z$MHQK|9F;ZmjKqAq=7k3fk`~4VfAo1bs8cq0 zs1dpGtobIAD90{5Rt4LKD1kCJeF4;|(j=rR5w)lx-CU(vEXD?^*z8cZ#-xzjq;x`n zCJRV!Jkw`rP01~86F;Kr|2-6)Qgha<;*l_b@7hZ|F^dss4ZPGVJ;pQ=XrEMz@00^Ff}4vJz~qImp~i4YOGg|H>Wn5I4)w@M zMB~ggPB~uc5;@`{3?XzGZA>5_Zi{+d2Srp@^dRdU>wk2S?Ok*l)U8?gp^KHBxIC9F ziqTjo36GBJ{?&HnBY&Det_x)1rkC}c zEju~l-AE>fIq@)8A--vJ&J`EYTg5*ii*aLA;cV;+8{ zBS&b7kXKC#s}y>cL(#Jca@M66}eae-M&u{qnubLtYPOd|B3y8yEM}rupg=&u& z7FbSZVz)-{O0zwbb;0uP=0h!{h_YenBrNEjn=rmZ%^x*$7ktlxy^}ADY^Q@5U)#o} zF^=brH23CXseicO^V!-iH6mk^mo-XH1~kms3F3@4DolZ1rV4#SEx+VFKO~?-jl4>QDS>J?v#d#5Pzlbq(}0nGrnk^=n$KG z{`z?koL?p^X%jR3m)9_Gc00nUlJ%d|ef#}fuALPMy~Fs(;jcX zdrSIQN_txgG&4&U!O}u#Ydc1t^>Gs4b>1m*R7~~+>jzkymKKK&(VN{1S@JC}q@`sd zJ+p^74icpF(9Wj~4_expWVw@<+?;XZ8{ZwnHgAPWwcCqV#vtBEbH;0>q(#TX7sl|1 z{J=660sq!={KgD;VSo*fms$akfy8Ad;0ngD&PuI;*V&}R8p_#ms9*yaBxBM_ z&kI^WRA~LnbF$*DDnj;Eh=~Se`RsU@#c%i7>bnMQpR})9UjRZd2mfcr zB;@DK^!J44;r2K!!K3w>yJ;Aj4*O6q__3{KP@7gsG{vk^F|0~KE@7skolW~<_R)s#P2NziuNvgvU9onb0 zzlJff*ZOe#8?&GE<@-h6&@K4ID29=ciQyYro?nCOGkb(#m$vl3*CJK4oyPtMEUE~- zk!X0mynjdGA3^X`fVU1_WyC{*}DB z#GScZ9&{d}%I5R;E%anuu0tWAlknwBldWODK-ADXM~{<76|&&eg%aKLygVKC zau~NKW#!W;#3u?PE$tYVuh>68xcIx}2ddKlgNqjdCCZi6AyyelibvzT0dJDnM+~z_ zC(vJ@dyM~5_kEldOyaenlftPB$K=V>;4*ZWGv5je7CApmgZA*xNp<%~^84U~cY-bLL;0+HqS zYMFL()wcS;!kdVWUFGDVz8Ojsnm`tP;*ZX!aAe3w5?-3T%}hqV-*=Y7Fm=o4RK~Xi7>U=CeXMu_W#=x^O`cA1j2>zpDEzG^ zOfD8phJFUE$zzZSub7;L=M)wZa@@u=74$U}oU?q#(+w~=p5WY;+(FFBxpAI+zH1&2 zLxX6G;-u`iLUjq)@OS+XWQobj7rt)`gK>Z%M#7@BzrpOhQ62njvXz$BOAEPcVtt&W zw{lM+J*O&6deO1}kM}Hpr6vlLtsbk2nlVJQQTn2Cds)wSH6}MrJ_NE*2osgBKo+_X zYig=n3M~i>Zs-a!v4_Wa|FCUA;MePabdzcugIGiS46VK4h4;>>dag^FT0n{P0cp(; z3PXaItx=N2SnfjTJT$vi%$-&vMKFqF!?X^1oP&a!mK5O(vLGh$tlpHuNxjw|@ zw|9Ke;J?dCtWRwFsUVrrH(Sfe(k)gP0GwnQgBg8hm|^h_L}~ob>|(C%7oFWO?*PMt zGL2BS6XH!N$yt7T%TcI{i$zGJwt$(1&=+${&}{G{Ucj6xrhkZEA)BA5WAq;e7@iZM zhQ71M(2Hu;`=^EPo_yjE&RXttNvd9y#4RPPH-CVz_m+fQ)Xtd>a(1(U6_nVP%t8e7 zigtY5Pf@0=IMz7lX#v()6x_a-XsUTlX)A;3=`Pr{sQc{%U?)}B-}E*)cAu#0QW z*bC}vfJW8WN-F=lodlAZm3x|IY>qVN)+onzuQzyoRlX9^?Y;12&fVEO(Izua)?t;kzhDICKvFlo-4xRp zBQDr#5}TA&89=f^HN``sZ6W*Du!uE1WF|JaHGaWjF~7;8XGY`g>6Fxh(Ty%1s>6I- zBK3KJtB~+-3&x*~kb^P^Mh{|Vdx6B&rXzYNtkXbw*Pa{5 zI^i79@QTzw2Ps@8hU^gL%4)FLp5qM;jj9h`t0)e;C z{fRuH<2EY^T1&dXHt2IUPJ6=`&y^Z~nwOQ!0F5_JPTxc}x0w^}blS$%-$hgm5ym#J1@TpL6y*Q3#MsUv6$o5{~n(yOG#xJfl>kYEq) zrT8l?O8G_qAOcZjQyeRPI6tSyveYW+rat{Tm4oBxdopG|H@@E1kkQQ4AI@=W4Bp{s zF`UE@n2O`1nG84%yuIoNo2k>$BcX*dXepbk)uD10_P5NB;|Xe}5?0mXPGMtDY@ZrL zHYHV+Qlxz({I)nb8;1=oBgy05cACiL@Y$74z~7J(3|_dPfmM~5ywlU-*O8kce=I)J zmne%}N`3s>NpWCQF8-kvrH>-#uN`GWr`)tf4v1<*%KU0os)8kNJU;mG$43hE!A44j zTArWTsWvY%zT*fNEonjRWI&5CgvrX<$QfeK~!J;TBmO+v3(JJW3Dd%Yy@P9N>CE zwe=%7nNjUXCkQ2Y!{b(>T=rqiNs5osoD(=sMuHM8{b<(x4G}5Rh?w<2N7>-R$e6?2 z;xB%Va-};g1U8D-y%N}SAB!ti&%q?MDfVGxf~fMKf$lu{bJpCr6{?J{(Y4=qhw(I# zD`;JY`3HG&`bQD%(@&2v@UAh<^VHV1`|;7BU!#YgSke^gaP-+8!76!EgVx`Ws;`8- zsW&5T3RR_(wx?ZwL9~0Zt1vmI?I)70hZyu??Nc zO_!wRRen%eqU44^*jDY;w(|w%YFBah6FUK0Uv@FJ_|(+$*}@E@8ZDqr+*9kVbOi~C zUt!8hx<_ldI-9bc@lf0usR}~Q;^|D?fP`nQ*qyfy2A}`0mEx-JB%{HFkv7Siov9Gh z^T9f9^&UnN9>UrjEctEk{729w$SuS_ywgTG%_bWV-W;9=ZG}fL+%p%c6f_y?x8)D20o2?c73DB`u?S%p~y@w zTKnv;#T=G06w1+H2;H|dMee)73u7AT2CJ+WORyK z*tJ4WpCOi|<_Lt&YlBula?+m*02+y^ft2j-zj9b&z}2D-2W@DpX>3(hk}yQ+XvXEo zZFM`_MI%gBrkc*8U-7CcMY>5(Dg@0WU@3gYDJ)ER^6+SF(0jF4`aflybyOVfmi5tK z0fID*li(KI-Q644K#≤BJjK1PxAr4&FF49xRaH(7`Q8fY7)H9o~CqzIDHuzw6Yg zQ?+WX=c(Giy%D8H^kO992HUYv=kjEaX!fb>*r?f(ML2`z7ePiyQdG}|DX!$Ud4rFL z^NNo$otOAQ3K1{%;xeJ8sfq`)qxWv%tGsQ1bdJ4XLe;uAdESeCXaf3<IqNzWTd-X&7PnB9$79>1FZg*N@;*%k0gC}dNMh6#+B#_so*S6)QkgfB7INQioLv8 zYrfmz>5chDc89;(=L-*SkX3tg^+1y+k@)kT7RlQ#T9uW|@L2l_cSTWeNx^^_n~ddp z&y^_r!VV3bri^S^clS-2D#_H#Um^)*BOK&n(AIwmQG<*3W}YZ#mi7#@1FYrM_F@sF zSq@v!GyZxtB=j_GayXYk7laFd!AR!CYA11ud&b25sDN4l7*Bo*med?DQqx=PR?yi| zG8(r8m|8_TAo~u+MjItz$?UiNkH$;j)tf(G6pyIfAlJe9%LmDa<0C$9#>YXP@W`49 zJ8mjDXbKHCi2yv`{ow1XOOuNQ;S5+*y_sbP_HDr5ia>d3cNboLXAg4?qUs=Jzs}JY zS!DK?`_Aln!Z%eGDBw1iMy|(%PITAsv4IWhVpnm}B$#d9*c;QF*;WAV*^)y@JvVz? z9BZff0?2jk5G}De&N-oToyGlLEiARrCp1XPW3{U}(4HGBE8y)^??8u2V4#a2`TDQk z1@+I2`QFi~-aEcl!7S;VIYD)2~c$kJTd2mrrLsdW&L0_PtjWIsB)KU{i}6G$aNbTFXGGwA{n~o_uH`9BAT> zxMB4pQq>$AI$S>8OjT1UYj^pwDJ9qjc@wL=xE+jHp)y4AP5a|OG;^@TPTuOB1VQzX za$V3~>BU$)be(uZ!oGWf@AHZX$uAqSaDB1=RExpyp>G3JjZ5BhDirgN0~^nGF=OO{ zY$?LP}xf*m$FZjhqJ#TV>AuP18KiTL$>~~>?4!MLQTDE z5~u@FMRwSL4H;qM?C+N)Crkx38 zDz(iV@5eeT;M7#t7-tJyR2V9xJ&Ae2P7e#bjprNfgm+o;vlX;nE$|c@*F+$ibI$2I zt^BBDR@iK0ZH5U$Y5v|468`zw)&KLH^a)nNxc)MHJMMS!Yay9xs&GrKX)f>~e;jKN zb+iEY=;yc@6nR(9O7~z3o-yj>fRmm-P2-3X>D^ZdQwn;eMQErj&O)m;yRM+yyG08=;TA%b$p^j!)XPmPhiw zV_c(1C+`twaq4L@>0A+j5a-3|n7cx%#+Y@RDvlJ@l>$uie|>l?)ceHw)jl^+Qifpa zJbbHe+ZSa60KHf{&)k}soHmVICA_FeNoA}qgoxc^$oANpGQ$S8uSrOre(|PIW}ZoH z5lwdE`DBz5`KzQa{DFBdp}zm1{>z4G>zn+o&wPLd{fF_8bc)|haxqib=qE4WX(>uj){1x!B@QM=^M;ldDlPe{mHC~ioUwYb& zHMLQ>@Z}a=u4OXjeXr{rr?;q8II0yXa^iacC0=4;VC%RD3wvo0ip=}nup2k<>lkJ` zh^A_{9SO;^866L$oW6p$UmoEId$i@E8E3B0R+Pl%*=~ibn5t5c3?;|qbq^HGeD0g~ zyq98N*LCL(Q&e7ve-6t8EtKtXr&noz9>#zbxSyypv)& zovsOJk+OQO-oasQfctzBf?rCpJH+0=HdEe0)&TEy2>iaN;Wzu-*8&`hNiUa^+Tj-F zz48O4FlAt|qay&Me|^+M@EM8`a_LCHX_eHSe|KE?Iw`b)Nw#plQ(HbXt%!;Z9xSJ| z%&xzTw(~gY0xCa`THP%J2Z(i39f#3?2T~`b{5!bfl*H=C-VFCKq;rx$s zcJnlN$4hN^WQ;xst-KIvR-eE}$jR=MS1@Kw?U$UQoV9_k6WoQ6iQPcet!J&50rIHO zv5O_ozl3N}9g*$=D}8QRi`^?wxZc$XM$ufFz1mj`{f^h+!Z!vyVSGzp>(+e_`2O5- zvg-SbX3ZaqP|6Sc;u78BEJv$u;bmxh7VO$Q|PJ{3#_a zi*+CvuD!1u|E+HuNK*6hv0zKMVC!3?x43iu_yuK?Cxly<(Pz_6f+wMKfJEXTGvc~Q zF!vh5l8v%f49nn`NG*VH*V9Mj5Z_>DT+oy0o{KaaJXT%DF=ccPl9VohjqtCWgW>>0 zKfMgr;Z$@n4~HuSnizNEwXW%$Q~mJAC7hA6q=ttMw79?5g&yCh{T&|xn?o-oW<$B7 zbJ_hNEcjX(Y_;_w-v9$obDPnZJC0+;;2|udDF|z>SOLda1*dHfm%dIHh?`jN<0SuK z_8IH%>R$%d9JmuM-eQMOkPVb&=$}$*SE{|C`GJg_a0-fhP4rWs8TE9@sn^`*Jx2R2 zhlhMYi-+*cUqHP2CY<}4MfJiPR?^6mNJbZZMsNnKB(A824Xssrx=Js2x9z!KzVx$aC4=Cb07tq`mnj zjx!(mn8td+d((BnZUxA0(x>dnXoa>WzAkqx)nyIe(6P`GGImto5GF(CO7O|wu-kUx zuO%H_eN;+ztogk4y^)H=dFm?A;15d`~mA1va zB(yB!1UO&K!RUaYz?RMNT1OG>_EY}KS`NekK>n~p=r3opoV?|D!tS}!u(vr}g?TZB zHHC^hZ3>&D{oN)B;$9H=J)Vnd=I_M@Il6~~C&|oQ#-B^vv&mAPjvMlN3>UiBZqSZz z^n-BRdwrFw1#{&9z`QbB7XR3a zW@+WUDRUPXgUSbbz@CKDYS=;_y~ zuqQqL;P86o?4Kb*U!60Kr*d%MgtB)(Vk9`*L1p3e;t2WZ|2npmZ7e^}F zXf-*@jeEy-0g;1g<%@RgG1N2`!LZ9~o%}j~`&LsnR<-6j9ZgP{Hbrh$eZP(fcZfK< z;nTJdYHHRp`1wSEr{j5LpW@sGX`nkE-pqiW*dzv1Rk>0+((SSCR!YB`bqZ}}kzRi@ zj_*N0;Hf0h;NF(>4(F-Yz`;9^AK;hnHw~u*9tcZ(X;HYD@yjor!Jgq@7X`LMWh?yG zu0|DaP0cXgOXv#V!4zQDUrt;H?%TXiZhEVJ(Tq-`|4igSb5y2}Pd$JX`Gj#~ z^Swt-?$mU$-n<%^rbp?}SidUbVDkulhcMfQ!CF-)i0E2(8PJC7Wy0+CY~^EpvYUYg zSUlj?7vq8}Zd2mVItZ-_lW=8U~(H^@sVIlcaw`vS=hE%p=do;BQ3(aTpun-k(O3(2c2ZmqH&W3tdoIV-0Qy z_UFX&t7bycu^3%Z`>BTmos+u;xVhzhE5rCS?y%h-hoz$F<9tXu7FE;bdj<=X$}Zd0 zq5R%Hk;@0nUzJGqqKdwf+axL4E(qYR{NxGGGw3BmeP`Oy%@aE9vMH3ikyKK`Zz56q z>|B~VX)iG!DC{FBgDa^BPX64>kB`Gnwf^@3n_^T`2Y7V#~(z$Q)xoyl~ zlF>QCToi1g%D7rUA$Mr17pG3*S4I^P7GkBNg6|AV(DaOd8^yYP{5K@hSzdu~G%700 z^TYA#)NsPFj`k>rroRnGy=w&pkkj#N@PMdRm&?p`fcKrjToEB_z|FfK79FuHpSlST z5tY+5x!@45;+IVqcKU=s;*Y`|`HT~s{5_pb!IKgugzDxX&O(z+EFD`(P_$6nPfo8M zOz){DuBlSq93x2i=^KpX@S&9WZg^?Ac-!2fSH5KOR@16mq^Vp^$}R5pWY}L;nP~Bb zPisYfcka$NOgQu$3No<7J9@lkb5Q38{%Veyvq*BP@pl<(A7S6oT7=;5+58K0%IZ!r z$4F-YGA<~5bbT0Dal+cjm3{^<7oAJ2YgkKd4o3RN=p&UawcmMw11t~2ZtIPZ2@}yd z*1CVCez@F&GE&~y6Y}zkGhfv=1Ox+XUK2B~E>bz`tYt-0IFH=qVsi-ontJ_y@M3*+ zjH~SPoOCdw;DO!h+H%0zwlzbn2q9ssw#VfebMoA{q(S$r^o21zlm6qSWto$nVe)JA z`{i)wtu}nvf;()e0O*;J<8iP|GtjnC|b9O^5u;Kw}ynWx@l29m@p zZ(;?0cQ(4*-!L^Xv}D{RBMli*nTqgw7t%p!-d*Q;`q}B28S%cqrMs>-J38mhk!GJy z_UmwMfxWTf@c4ek-M~?StVtN1f7_&SYD|z@Q=4#+pFl`m%syHEP;&$6vIhKfYjfh3 znH2lPUpAB-JdPde+w+?f4Vx?ll%<=#XAX0s_+#L zc=(x0!w*KcHP@)eMd#xtck=%3jhUo5jBXXi*_lZc`C!N2I@38CHYY=c=%?0j=iqOT zxOGgUi2gjHg85dGn%0zf90~-sEu8_a)tjMpi=FKSojdwFqB$i&5Pia>B-_Trf1ln5Zf#_+M9pvOn z$iGog@G8KfH*C&pOBOEb&V$dq>L*atD)`Tvley~aX19Jz#9T|3WiHO$f1whU{Oec8 zSTAr-eD`2zWF=2KKy$xwt7Q@0<*im<-c!oj4=PUk)&hxX3GdnNR=K5J2(j+(g?pS>&SAoHJ{d9&YD(raEwxvt*Ea~&S6wP#uFc`p8@~s@&aX9 z_>3$>ch1tz_?MCcnUCUC`&ajP*VF}54W$n(&&& zuFMo~MYS2ByPijBX_34WA5y1I_kt@l*Fk?-F=z%!8bnfFTHwuU#1vm6k8f9YS&nS$ z8A06=)0OR0ws3D;QZ&~++4Hthcuq5k4l4|pEi%twtt1eH<}db_9`t!hW|8Utt$9k~ zi~0t$WKoi<`JntU3|U6|ua1@S712$gj-@7xJj=^2ku&N0pO`CLed~s`pkvjSLLSd1 z;`0KThGX&qw2+HBc;YD`q9}VaZcys~v zy!#yZn$P$7HY`j;`s!x9YtY8T>Hv6KiiAQCgRjB?I~!d~}T*=Gf3%G z%`))b*14lxfXc-!X?oj6+miO2F&8@?vc3ql4G+i4%>+aP~ukDM+uPPoz>@>cqkr!gYh)1UKLH8pRm0zG>fB8ie};dcIas^&jw_LT z|IU*HG-$im5Xt#n$r#if{IgM*(+Ox->DBbRxzH-(jIU1#>?Z;E*5jWGE2|orngi62 zibOznNd(#0O1vvdb@rHaIj?e_o)jNdi<Zo~L-r6(^nC$%>4C6|dfwNS^{D%J}0 zCDQbiH(ZGKm1>pz0VmV6b+q9 z@f*6Ex5J%K_{_DlN;Nw$uh_=Eg1nO4t#+;NsPqgPar{V**RSEEM_fzliBmoKr?VV0 zcW`@hUIy`uG6Nl3-#}MXciqs=;~6z?_)ve<@pX&BUKLbT&Lf;a#$oPB;&3Tu=A;NV zSid_z0&^!T^qv26of+j>JCfkT9v>qS;Son3jVxY4v;@341KA^EyE&HuV z3#T35kJIjr$&A0~BiU*06SYv%Ym2|;y(>%YTci8&v#&&Oy=vH4I@^4--#F*rOljf{ z@dgl9_b%X^&QKyi?>b`ySG;tWWu{Ph`mRmT!7Qh^Eq3`Qc>Kcz*dLW%_;$EBOeqXP z9sPZDv!6ZUI1JO;w106a-t%O0j&QVHXs|Gm0gKXD9fiU$wg!ax5@;nTqOR zcUgWa5cI*Icp333Y$!Dfxyv&-v*-#pIS9UF?P}?lXROj|3UKjk|G^WX5!Fj_?3aBl zAol5l&y|m**W!MXdXiK}3tsW1{Ff#! z_rRVc4v$aNlgjv?dTKq6)i(K c.ID + } + return false +} diff --git a/internal/db/db.go b/internal/db/db.go new file mode 100644 index 00000000..7fcca455 --- /dev/null +++ b/internal/db/db.go @@ -0,0 +1,42 @@ +package db + +import ( + "context" + "fmt" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +type Config struct { + DSN string + MaxConns int32 + MinConns int32 + ConnMaxLifetime time.Duration + ConnTimeout time.Duration +} + +func Connect(ctx context.Context, cfg Config) (*pgxpool.Pool, error) { + poolCfg, err := pgxpool.ParseConfig(cfg.DSN) + if err != nil { + return nil, fmt.Errorf("parse dsn: %w", err) + } + poolCfg.MaxConns = cfg.MaxConns + poolCfg.MinConns = cfg.MinConns + poolCfg.MaxConnLifetime = cfg.ConnMaxLifetime + + connCtx, cancel := context.WithTimeout(ctx, cfg.ConnTimeout) + defer cancel() + pool, err := pgxpool.NewWithConfig(connCtx, poolCfg) + if err != nil { + return nil, fmt.Errorf("create pool: %w", err) + } + + pingCtx, cancel2 := context.WithTimeout(ctx, cfg.ConnTimeout) + defer cancel2() + if err := pool.Ping(pingCtx); err != nil { + pool.Close() + return nil, fmt.Errorf("ping db: %w", err) + } + return pool, nil +} diff --git a/internal/httpapi/api_test.go b/internal/httpapi/api_test.go new file mode 100644 index 00000000..7438fad5 --- /dev/null +++ b/internal/httpapi/api_test.go @@ -0,0 +1,983 @@ +package httpapi + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "mtusi2026/internal/cache" + "mtusi2026/internal/idgen" + "mtusi2026/internal/model" + "mtusi2026/internal/sessionstore" + "mtusi2026/internal/store" + + "github.com/redis/go-redis/v9" +) + +//вспомогательные функции + +func newKey() string { return idgen.GenerateUUIDv4() } + +func doMessage(srv *httptest.Server, chatID, token, key string, body any) (*http.Response, error) { + var buf bytes.Buffer + if s, ok := body.(string); ok { + buf.WriteString(s) + } else if err := json.NewEncoder(&buf).Encode(body); err != nil { + return nil, err + } + req, err := http.NewRequest(http.MethodPost, + fmt.Sprintf("%s/api/v1/chats/%s/messages", srv.URL, chatID), &buf) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + if key != "" { + req.Header.Set("Idempotency-Key", key) + } + return srv.Client().Do(req) +} + +func postMessage(t *testing.T, srv *httptest.Server, chatID, token, key string, body any) *http.Response { + t.Helper() + resp, err := doMessage(srv, chatID, token, key, body) + if err != nil { + t.Fatalf("do request: %v", err) + } + return resp +} + +func newUnlimitedLimiter() *store.Limiter { + return store.NewLimiter(1_000_000, 1000) +} + +func newTestServer(t *testing.T) *httptest.Server { + t.Helper() + srv, _ := newTestServerWithStore(t) + return srv +} + +func newTestServerWithStore(t *testing.T) (*httptest.Server, *store.Store) { + t.Helper() + truncateAll(t) + + s := store.New(testPool, newUnlimitedLimiter()) + ss := newMemorySessionStore() + + return httptest.NewServer(NewRouter(s, testPool, ss)), s +} + +func newRedisTestServer(t *testing.T) *httptest.Server { + t.Helper() + srv, _ := newRedisTestServerWithStore(t) + return srv +} + +func newRedisTestServerWithStore(t *testing.T) (*httptest.Server, *store.Store) { + t.Helper() + truncateAll(t) + + if err := testRedis.FlushDB(context.Background()).Err(); err != nil { + t.Fatalf("flush redis: %v", err) + } + + s := store.New(testPool, newUnlimitedLimiter()) + ss := sessionstore.NewRedis(testRedis) + + return httptest.NewServer(NewRouter(s, testPool, ss)), s +} + +func newCachedRedisTestServer(t *testing.T) (*httptest.Server, *store.Store) { + t.Helper() + truncateAll(t) + + if err := testRedis.FlushDB(context.Background()).Err(); err != nil { + t.Fatalf("flush redis: %v", err) + } + + s := store.New(testPool, newUnlimitedLimiter()) + ss := sessionstore.NewRedis(testRedis) + chatCache := cache.New(testRedis, 30*time.Second) + + srv := httptest.NewServer( + NewRouterWithCache(s, testPool, ss, chatCache), + ) + + return srv, s +} + +type testUser struct { + model.User + Token string +} + +func postJSON(t *testing.T, srv *httptest.Server, path string, token string, body any) *http.Response { + t.Helper() + var buf bytes.Buffer + if s, ok := body.(string); ok { + buf.WriteString(s) + } else if err := json.NewEncoder(&buf).Encode(body); err != nil { + t.Fatalf("encode request body: %v", err) + } + req, err := http.NewRequest(http.MethodPost, srv.URL+path, &buf) + if err != nil { + t.Fatalf("build request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatalf("do request: %v", err) + } + return resp +} + +func getJSON(t *testing.T, srv *httptest.Server, path string, token string) *http.Response { + t.Helper() + req, err := http.NewRequest(http.MethodGet, srv.URL+path, nil) + if err != nil { + t.Fatalf("build request: %v", err) + } + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatalf("do request: %v", err) + } + return resp +} + +func decodeBody[T any](t *testing.T, resp *http.Response) T { + t.Helper() + defer resp.Body.Close() + var v T + if err := json.NewDecoder(resp.Body).Decode(&v); err != nil { + t.Fatalf("decode response body: %v", err) + } + return v +} + +func mustLogin(t *testing.T, srv *httptest.Server, username, password string) string { + t.Helper() + req, err := http.NewRequest(http.MethodPost, srv.URL+"/api/v1/auth/sessions", nil) + if err != nil { + t.Fatalf("build login request: %v", err) + } + req.SetBasicAuth(username, password) + + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatalf("do login request: %v", err) + } + if resp.StatusCode != http.StatusCreated { + t.Fatalf("login %s: expected 201, got %d", username, resp.StatusCode) + } + session := decodeBody[model.Session](t, resp) + return session.Token +} + +func mustRegisterUser(t *testing.T, srv *httptest.Server, username string) testUser { + t.Helper() + const password = "correct-horse-battery-staple" + + resp := postJSON(t, srv, "/api/v1/users", "", model.CreateUserRequest{ + Username: username, + DisplayName: "Test " + username, + Password: password, + }) + if resp.StatusCode != http.StatusCreated { + t.Fatalf("register %s: expected 201, got %d", username, resp.StatusCode) + } + u := decodeBody[model.User](t, resp) + + token := mustLogin(t, srv, username, password) + return testUser{User: u, Token: token} +} + +func mustCreateChat(t *testing.T, srv *httptest.Server, caller testUser, title string, memberIDs []string) model.Chat { + t.Helper() + resp := postJSON(t, srv, "/api/v1/chats", caller.Token, model.CreateChatRequest{ + Title: title, + MemberIDs: memberIDs, + }) + if resp.StatusCode != http.StatusCreated { + t.Fatalf("create chat: expected 201, got %d", resp.StatusCode) + } + return decodeBody[model.Chat](t, resp) +} + +// тесты пользователи + +func TestRegisterUser_DuplicateUsername_Returns409(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + mustRegisterUser(t, srv, "alice") + + resp := postJSON(t, srv, "/api/v1/users", "", model.CreateUserRequest{ + Username: "alice", + DisplayName: "Другая Алиса", + Password: "correct-horse-battery-staple", + }) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("expected 409, got %d", resp.StatusCode) + } + errBody := decodeBody[model.ErrorEnvelope](t, resp) + if errBody.Error.Code != "conflict" { + t.Fatalf("expected error code 'conflict', got %q", errBody.Error.Code) + } + if errBody.Error.RequestID == "" { + t.Fatal("expected non-empty request_id") + } +} + +func TestRegisterUser_InvalidPasswordLength_Returns400(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + cases := []string{"short", strings.Repeat("a", 129)} + for _, password := range cases { + resp := postJSON(t, srv, "/api/v1/users", "", model.CreateUserRequest{ + Username: "alice", + DisplayName: "Алиса", + Password: password, + }) + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("password len %d: expected 400, got %d", len(password), resp.StatusCode) + } + } +} + +func TestRegisterUser_InvalidJSON_Returns400(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + resp := postJSON(t, srv, "/api/v1/users", "", `{"username":`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400, got %d", resp.StatusCode) + } +} + +func TestRegisterUser_UnknownField_Returns400(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + resp := postJSON(t, srv, "/api/v1/users", "", `{"username":"bob","display_name":"Боб","password":"correct-horse-battery-staple","extra":"whatever"}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400 on unknown field, got %d", resp.StatusCode) + } +} + +func TestRegisterUser_InvalidUsernamePattern_Returns400(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + cases := []string{"AB", "ab", "Has-Dash", "こんにちは"} + for _, username := range cases { + resp := postJSON(t, srv, "/api/v1/users", "", model.CreateUserRequest{ + Username: username, + DisplayName: "Тест", + Password: "correct-horse-battery-staple", + }) + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("username %q: expected 400, got %d", username, resp.StatusCode) + } + } +} + +func TestCurrentUser_MissingHeader_Returns401(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + resp := getJSON(t, srv, "/api/v1/users/me", "") + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expected 401, got %d", resp.StatusCode) + } +} + +func TestCurrentUser_UnknownToken_Returns401(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + resp := getJSON(t, srv, "/api/v1/users/me", "totally-made-up-token") + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expected 401, got %d", resp.StatusCode) + } +} + +//тесты вход и сессии + +func TestCreateSession_WrongPassword_Returns401(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + mustRegisterUser(t, srv, "alice") + + req, _ := http.NewRequest(http.MethodPost, srv.URL+"/api/v1/auth/sessions", nil) + req.SetBasicAuth("alice", "totally-wrong-password") + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatalf("do request: %v", err) + } + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expected 401, got %d", resp.StatusCode) + } + if resp.Header.Get("WWW-Authenticate") == "" { + t.Error("expected WWW-Authenticate header on failed login") + } +} + +func TestCreateSession_UnknownUsername_Returns401(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + req, _ := http.NewRequest(http.MethodPost, srv.URL+"/api/v1/auth/sessions", nil) + req.SetBasicAuth("nobody_here", "whatever-password-12") + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatalf("do request: %v", err) + } + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expected 401, got %d", resp.StatusCode) + } +} + +func TestDeleteSession_ThenReuse_Returns401(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + alice := mustRegisterUser(t, srv, "alice") + + req, _ := http.NewRequest(http.MethodDelete, srv.URL+"/api/v1/auth/sessions/current", nil) + req.Header.Set("Authorization", "Bearer "+alice.Token) + resp, err := srv.Client().Do(req) + if err != nil { + t.Fatalf("do logout request: %v", err) + } + if resp.StatusCode != http.StatusNoContent { + t.Fatalf("expected 204, got %d", resp.StatusCode) + } + + // повторное использование отозванного токена -401 + resp2 := getJSON(t, srv, "/api/v1/users/me", alice.Token) + if resp2.StatusCode != http.StatusUnauthorized { + t.Fatalf("expected 401 after logout, got %d", resp2.StatusCode) + } +} + +//тесты чаты и membership + +func TestCreateChat_CallerNotInMembers_Returns400(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + alice := mustRegisterUser(t, srv, "alice") + bob := mustRegisterUser(t, srv, "bob") + + resp := postJSON(t, srv, "/api/v1/chats", alice.Token, model.CreateChatRequest{ + Title: "Без себя", + MemberIDs: []string{bob.ID}, + }) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400, got %d", resp.StatusCode) + } +} + +func TestCreateChat_DuplicateMemberIDs_Returns400(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + alice := mustRegisterUser(t, srv, "alice") + + resp := postJSON(t, srv, "/api/v1/chats", alice.Token, model.CreateChatRequest{ + Title: "Дубли", + MemberIDs: []string{alice.ID, alice.ID}, + }) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400, got %d", resp.StatusCode) + } +} + +func TestGetChat_NotMember_Returns404NotForbidden(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + alice := mustRegisterUser(t, srv, "alice") + bob := mustRegisterUser(t, srv, "bob") + carol := mustRegisterUser(t, srv, "carol") + + chat := mustCreateChat(t, srv, alice, "Приватный", []string{alice.ID, bob.ID}) + + resp := getJSON(t, srv, "/api/v1/chats/"+chat.ID, carol.Token) + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("expected 404 for non-member (never 403), got %d", resp.StatusCode) + } +} + +func TestGetChat_UnknownChatID_Returns404(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + alice := mustRegisterUser(t, srv, "alice") + + resp := getJSON(t, srv, "/api/v1/chats/11111111-1111-4111-8111-111111111111", alice.Token) + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("expected 404, got %d", resp.StatusCode) + } +} + +// тесты: сообщения + +func TestCreateMessage_EmptyText_Returns400(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + alice := mustRegisterUser(t, srv, "alice") + chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID}) + + resp := postMessage(t, srv, chat.ID, alice.Token, newKey(), model.CreateMessageRequest{Text: ""}) + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400 for empty text, got %d", resp.StatusCode) + } +} + +func TestCreateMessage_WhitespaceOnlyText_Returns400(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + alice := mustRegisterUser(t, srv, "alice") + chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID}) + + resp := postMessage(t, srv, chat.ID, alice.Token, newKey(), model.CreateMessageRequest{Text: " \t\n "}) + + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400 for whitespace-only text, got %d", resp.StatusCode) + } +} + +func TestCreateMessage_ForeignChat_Returns404(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + alice := mustRegisterUser(t, srv, "alice") + bob := mustRegisterUser(t, srv, "bob") + carol := mustRegisterUser(t, srv, "carol") + + chat := mustCreateChat(t, srv, alice, "Приватный", []string{alice.ID, bob.ID}) + + resp := postMessage(t, srv, chat.ID, carol.Token, newKey(), model.CreateMessageRequest{Text: "Я тут чужая"}) + + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("expected 404 for foreign chat, got %d", resp.StatusCode) + } +} + +func TestCreateMessage_SenderIDIgnoredFromRequestBody(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + alice := mustRegisterUser(t, srv, "alice") + bob := mustRegisterUser(t, srv, "bob") + chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID, bob.ID}) + + raw := fmt.Sprintf(`{"text":"Привет","sender_id":%q}`, bob.ID) + resp := postMessage(t, srv, chat.ID, alice.Token, newKey(), raw) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400 (unknown field sender_id), got %d", resp.StatusCode) + } +} + +// тесты пагинация + +func TestListMessages_PaginationAcrossMultiplePages(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + alice := mustRegisterUser(t, srv, "alice") + chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID}) + + const total = 5 + for i := 0; i < total; i++ { + resp := postMessage(t, srv, chat.ID, alice.Token, newKey(), model.CreateMessageRequest{ + Text: fmt.Sprintf("сообщение %d", i), + }) + if resp.StatusCode != http.StatusCreated { + t.Fatalf("create message %d: expected 201, got %d", i, resp.StatusCode) + } + resp.Body.Close() + } + + seen := make(map[string]bool) + cursor := "" + pages := 0 + for { + path := fmt.Sprintf("/api/v1/chats/%s/messages?limit=2", chat.ID) + if cursor != "" { + path += "&cursor=" + cursor + } + resp := getJSON(t, srv, path, alice.Token) + if resp.StatusCode != http.StatusOK { + t.Fatalf("list messages: expected 200, got %d", resp.StatusCode) + } + page := decodeBody[model.MessagePage](t, resp) + pages++ + + for _, m := range page.Items { + if seen[m.ID] { + t.Fatalf("duplicate message across pages: %s", m.ID) + } + seen[m.ID] = true + } + + if page.NextCursor == nil { + break + } + cursor = *page.NextCursor + + if pages > total { + t.Fatal("too many pages — possible cursor bug (infinite loop)") + } + } + + if len(seen) != total { + t.Fatalf("expected %d unique messages across all pages, got %d", total, len(seen)) + } + if pages < 2 { + t.Fatalf("expected pagination to span multiple pages, got %d page(s)", pages) + } +} + +func TestListMessages_CursorFromDifferentChat_Returns400(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + alice := mustRegisterUser(t, srv, "alice") + chatA := mustCreateChat(t, srv, alice, "Чат A", []string{alice.ID}) + chatB := mustCreateChat(t, srv, alice, "Чат B", []string{alice.ID}) + + postMessage(t, srv, chatA.ID, alice.Token, newKey(), model.CreateMessageRequest{Text: "первое"}).Body.Close() + postMessage(t, srv, chatA.ID, alice.Token, newKey(), model.CreateMessageRequest{Text: "второе"}).Body.Close() + + respA := getJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages?limit=1", chatA.ID), alice.Token) + pageA := decodeBody[model.MessagePage](t, respA) + if pageA.NextCursor == nil { + t.Fatal("expected next_cursor to be present after inserting 2 messages with limit=1") + } + + resp := getJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages?cursor=%s", chatB.ID, *pageA.NextCursor), alice.Token) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("expected 400 for cross-chat cursor, got %d", resp.StatusCode) + } +} + +func TestCreateSession_RedisTTLExpires(t *testing.T) { + srv := newRedisTestServer(t) + defer srv.Close() + + mustRegisterUser(t, srv, "alice") + + t.Setenv("SESSION_TTL_SECONDS", "1") + + token := mustLogin(t, srv, "alice", "correct-horse-battery-staple") + + resp := getJSON(t, srv, "/api/v1/users/me", token) + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200 immediately after login, got %d", resp.StatusCode) + } + resp.Body.Close() + + time.Sleep(1500 * time.Millisecond) + + resp = getJSON(t, srv, "/api/v1/users/me", token) + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expected 401 after session TTL expiration, got %d", resp.StatusCode) + } + resp.Body.Close() +} + +func TestCurrentUser_RedisUnavailable_Returns503(t *testing.T) { + srv, s := newRedisTestServerWithStore(t) + defer srv.Close() + + alice := mustRegisterUser(t, srv, "alice") + + resp := getJSON(t, srv, "/api/v1/users/me", alice.Token) + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200 before redis failure, got %d", resp.StatusCode) + } + resp.Body.Close() + + redisDownClient := redis.NewClient(&redis.Options{ + Addr: "127.0.0.1:1", + }) + defer redisDownClient.Close() + + redisDownSessionStore := sessionstore.NewRedis(redisDownClient) + + downSrv := httptest.NewServer( + NewRouter(s, testPool, redisDownSessionStore), + ) + defer downSrv.Close() + + resp = getJSON(t, downSrv, "/api/v1/users/me", alice.Token) + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf( + "expected 503 when redis is unavailable, got %d", + resp.StatusCode, + ) + } + resp.Body.Close() + +} + +func TestChatCache_ConcurrentRequests(t *testing.T) { + const concurrentRequests = 50 + + baselineSrv, baselineStore := newTestServerWithStore(t) + defer baselineSrv.Close() + + baselineUser := mustRegisterUser(t, baselineSrv, "cachebaseline") + baselineChat := mustCreateChat( + t, + baselineSrv, + baselineUser, + "Cache baseline", + []string{baselineUser.ID}, + ) + + baselineStore.ResetChatSQLQueries() + + start := make(chan struct{}) + errCh := make(chan error, concurrentRequests) + + baselineStart := time.Now() + + for i := 0; i < concurrentRequests; i++ { + go func() { + <-start + + _, err := baselineStore.GetChat( + context.Background(), + baselineChat.ID, + baselineUser.ID, + ) + errCh <- err + }() + } + + close(start) + + for i := 0; i < concurrentRequests; i++ { + if err := <-errCh; err != nil { + t.Fatal(err) + } + } + + baselineLatency := time.Since(baselineStart) + baselineQueries := baselineStore.ChatSQLQueries() + + cachedSrv, cachedStore := newCachedRedisTestServer(t) + defer cachedSrv.Close() + + cachedUser := mustRegisterUser(t, cachedSrv, "cacheprotected") + cachedChat := mustCreateChat( + t, + cachedSrv, + cachedUser, + "Cache protected", + []string{cachedUser.ID}, + ) + + cacheKey := "chat:" + cachedUser.ID + ":" + cachedChat.ID + + if err := testRedis.Del(context.Background(), cacheKey).Err(); err != nil { + t.Fatalf("delete cache key: %v", err) + } + + cachedStore.ResetChatSQLQueries() + + start = make(chan struct{}) + errCh = make(chan error, concurrentRequests) + + cachedStart := time.Now() + + for i := 0; i < concurrentRequests; i++ { + go func() { + <-start + + resp := getJSON( + t, + cachedSrv, + "/api/v1/chats/"+cachedChat.ID, + cachedUser.Token, + ) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errCh <- fmt.Errorf("unexpected status: %d", resp.StatusCode) + return + } + + errCh <- nil + }() + } + + close(start) + + for i := 0; i < concurrentRequests; i++ { + if err := <-errCh; err != nil { + t.Fatal(err) + } + } + + cachedLatency := time.Since(cachedStart) + cachedQueries := cachedStore.ChatSQLQueries() + + t.Logf( + "baseline: requests=%d sql_queries=%d latency=%s", + concurrentRequests, + baselineQueries, + baselineLatency, + ) + + t.Logf( + "cold cache + singleflight: requests=%d sql_queries=%d latency=%s", + concurrentRequests, + cachedQueries, + cachedLatency, + ) + + expectedBaselineQueries := int64(concurrentRequests * 2) + + if baselineQueries != expectedBaselineQueries { + t.Errorf( + "baseline: expected %d SQL queries, got %d", + expectedBaselineQueries, + baselineQueries, + ) + } + + if cachedQueries > 2 { + t.Errorf( + "cold cache + singleflight: expected <=2 SQL queries, got %d", + cachedQueries, + ) + } +} + +func TestChatCache_WarmCache(t *testing.T) { + const concurrentRequests = 50 + + srv, s := newCachedRedisTestServer(t) + defer srv.Close() + + user := mustRegisterUser(t, srv, "cachewarm") + chat := mustCreateChat( + t, + srv, + user, + "Warm cache", + []string{user.ID}, + ) + + // Первый GET прогревает cache + resp := getJSON( + t, + srv, + "/api/v1/chats/"+chat.ID, + user.Token, + ) + resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("warm-up GET: unexpected status %d", resp.StatusCode) + } + + s.ResetChatSQLQueries() + + start := make(chan struct{}) + errCh := make(chan error, concurrentRequests) + + startedAt := time.Now() + + for i := 0; i < concurrentRequests; i++ { + go func() { + <-start + + resp := getJSON( + t, + srv, + "/api/v1/chats/"+chat.ID, + user.Token, + ) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + errCh <- fmt.Errorf( + "unexpected status: %d", + resp.StatusCode, + ) + return + } + + errCh <- nil + }() + } + + close(start) + + for i := 0; i < concurrentRequests; i++ { + if err := <-errCh; err != nil { + t.Fatal(err) + } + } + + latency := time.Since(startedAt) + queries := s.ChatSQLQueries() + + t.Logf( + "warm cache: requests=%d sql_queries=%d latency=%s", + concurrentRequests, + queries, + latency, + ) + + if queries != 0 { + t.Errorf( + "warm cache: expected 0 SQL queries, got %d", + queries, + ) + } +} + +func TestChatCache_ReadAfterWriteAndRecovery(t *testing.T) { + srv, s := newCachedRedisTestServer(t) + defer srv.Close() + + user := mustRegisterUser(t, srv, "cacherecovery") + + chat := mustCreateChat( + t, + srv, + user, + "Read after write", + []string{user.ID}, + ) + + s.ResetChatSQLQueries() + + resp := getJSON( + t, + srv, + "/api/v1/chats/"+chat.ID, + user.Token, + ) + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + t.Fatalf("read-after-write: unexpected status %d", resp.StatusCode) + } + + got := decodeBody[model.Chat](t, resp) + + if got.ID != chat.ID { + t.Fatalf( + "read-after-write: expected chat %s, got %s", + chat.ID, + got.ID, + ) + } + + if got.Title != "Read after write" { + t.Fatalf( + "read-after-write: expected title %q, got %q", + "Read after write", + got.Title, + ) + } + + if queries := s.ChatSQLQueries(); queries != 2 { + t.Fatalf( + "read-after-write: expected 2 SQL queries on cache miss, got %d", + queries, + ) + } + + cacheKey := "chat:" + user.ID + ":" + chat.ID + + exists, err := testRedis.Exists(context.Background(), cacheKey).Result() + if err != nil { + t.Fatalf("check cache: %v", err) + } + + if exists != 1 { + t.Fatalf("expected chat to be present in cache") + } + + if err := testRedis.Del(context.Background(), cacheKey).Err(); err != nil { + t.Fatalf("delete cache: %v", err) + } + + s.ResetChatSQLQueries() + + resp2 := getJSON( + t, + srv, + "/api/v1/chats/"+chat.ID, + user.Token, + ) + defer resp2.Body.Close() + + if resp2.StatusCode != http.StatusOK { + t.Fatalf( + "recovery after cache loss: unexpected status %d", + resp2.StatusCode, + ) + } + + recovered := decodeBody[model.Chat](t, resp2) + + if recovered.ID != chat.ID { + t.Fatalf( + "recovery: expected chat %s, got %s", + chat.ID, + recovered.ID, + ) + } + + if recovered.Title != chat.Title { + t.Fatalf( + "recovery: expected title %q, got %q", + chat.Title, + recovered.Title, + ) + } + + if queries := s.ChatSQLQueries(); queries != 2 { + t.Fatalf( + "recovery: expected 2 SQL queries after cache loss, got %d", + queries, + ) + } + + exists, err = testRedis.Exists(context.Background(), cacheKey).Result() + if err != nil { + t.Fatalf("check recovered cache: %v", err) + } + + if exists != 1 { + t.Fatalf("expected cache to be restored after database read") + } +} diff --git a/internal/httpapi/auth.go b/internal/httpapi/auth.go new file mode 100644 index 00000000..85a2bc2f --- /dev/null +++ b/internal/httpapi/auth.go @@ -0,0 +1,110 @@ +package httpapi + +import ( + "errors" + "net/http" + "strings" + + "mtusi2026/internal/auth" + "mtusi2026/internal/config" + "mtusi2026/internal/model" + "mtusi2026/internal/store" +) + +const sessionTTLEnvKey = "SESSION_TTL_SECONDS" + +var dummyPasswordHash = func() string { + h, err := auth.HashPassword("timing-equalizer-password") + if err != nil { + panic(err) + } + return h +}() + +func (a *api) handleCreateSession(w http.ResponseWriter, r *http.Request) { + requestID := newRequestID() + + username, password, ok := r.BasicAuth() + if !ok { + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + writeError(w, http.StatusUnauthorized, requestID, codeUnauthorized, "missing or malformed Basic credentials") + return + } + + u, passwordHash, err := a.store.GetUserByUsername(r.Context(), username) + if err != nil { + if !errors.Is(err, store.ErrUsernameNotFound) { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "database unavailable") + return + } + _, _ = auth.VerifyPassword(password, dummyPasswordHash) + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + writeError(w, http.StatusUnauthorized, requestID, codeUnauthorized, "invalid username or password") + return + } + + valid, err := auth.VerifyPassword(password, passwordHash) + if err != nil || !valid { + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + writeError(w, http.StatusUnauthorized, requestID, codeUnauthorized, "invalid username or password") + return + } + + token, err := auth.GenerateSessionToken() + if err != nil { + writeError(w, http.StatusInternalServerError, requestID, codeInternal, "internal error") + return + } + + ttl := config.ParseDurationSecondsEnv(sessionTTLEnvKey, 3600) + expiresAt, err := a.sessionStore.Create(r.Context(), token, u.ID, ttl) + if err != nil { + writeError(w, http.StatusServiceUnavailable, requestID, codeInternal, "session service unavailable") + return + } + + writeJSON(w, http.StatusCreated, requestID, model.Session{ + Token: token, + TokenType: "Bearer", + ExpiresAt: expiresAt, + User: u, + }) +} + +func (a *api) handleDeleteSession(w http.ResponseWriter, r *http.Request) { + requestID := newRequestID() + + token, ok := bearerToken(r) + if !ok { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + writeError(w, http.StatusUnauthorized, requestID, codeUnauthorized, "missing or malformed Bearer token") + return + } + + deleted, err := a.sessionStore.Delete(r.Context(), token) + if err != nil { + writeError(w, http.StatusServiceUnavailable, requestID, codeInternal, "session service unavailable") + return + } + if !deleted { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + writeError(w, http.StatusUnauthorized, requestID, codeUnauthorized, "unknown or already revoked token") + return + } + + w.Header().Set("X-Request-Id", requestID) + w.WriteHeader(http.StatusNoContent) +} + +func bearerToken(r *http.Request) (string, bool) { + h := r.Header.Get("Authorization") + const prefix = "Bearer " + if !strings.HasPrefix(h, prefix) { + return "", false + } + token := strings.TrimPrefix(h, prefix) + if token == "" { + return "", false + } + return token, true +} diff --git a/internal/httpapi/chats.go b/internal/httpapi/chats.go new file mode 100644 index 00000000..5211564a --- /dev/null +++ b/internal/httpapi/chats.go @@ -0,0 +1,131 @@ +package httpapi + +import ( + "context" + "errors" + "net/http" + "strconv" + "strings" + "time" + + "mtusi2026/internal/idgen" + "mtusi2026/internal/model" + "mtusi2026/internal/store" +) + +func parseLimit(r *http.Request) (int, bool) { + q := r.URL.Query() + if !q.Has("limit") { + return 50, true + } + raw := q.Get("limit") + n, err := strconv.Atoi(raw) + if err != nil || n < 1 || n > 100 || raw != strconv.Itoa(n) { + return 0, false + } + return n, true +} + +func (a *api) handleCreateChat(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { + var req model.CreateChatRequest + if err := decodeJSONStrict(r, &req); err != nil { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "malformed request body") + return + } + if !validText(req.Title, 100) || len(req.MemberIDs) == 0 { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid title or member_ids") + return + } + for i, id := range req.MemberIDs { + if !idgen.IsValidUUID(id) { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid title or member_ids") + return + } + req.MemberIDs[i] = strings.ToLower(id) + } + + c, err := a.store.CreateChat(r.Context(), u.ID, req.Title, req.MemberIDs) + if err != nil { + if errors.Is(err, store.ErrValidation) { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid title or member_ids") + return + } + writeError(w, http.StatusInternalServerError, requestID, codeInternal, "internal error") + return + } + writeJSON(w, http.StatusCreated, requestID, c) +} + +func (a *api) handleListChats(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { + limit, ok := parseLimit(r) + if !ok { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid limit") + return + } + cursorStr, ok := parseCursor(r) + if !ok { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid cursor") + return + } + + items, next, err := a.store.ListChatsForUser(r.Context(), u.ID, cursorStr, limit) + if err != nil { + if errors.Is(err, store.ErrValidation) { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid cursor") + return + } + writeError(w, http.StatusInternalServerError, requestID, codeInternal, "internal error") + return + } + if items == nil { + items = []model.Chat{} + } + writeJSON(w, http.StatusOK, requestID, model.ChatPage{Items: items, NextCursor: next}) +} + +func (a *api) handleGetChat(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { + chatID, ok := chatIDParam(w, r, requestID) + if !ok { + return + } + + if a.chatCache != nil { + if c, hit, err := a.chatCache.Get(r.Context(), u.ID, chatID); err == nil && hit { + writeJSON(w, http.StatusOK, requestID, c) + return + } + } + + result, err, _ := a.chatFlight.Do(u.ID+":"+chatID, func() (any, error) { + ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 5*time.Second) + defer cancel() + + if a.chatCache != nil { + if c, hit, err := a.chatCache.Get(ctx, u.ID, chatID); err == nil && hit { + return c, nil + } + } + c, err := a.store.GetChat(ctx, chatID, u.ID) + if err != nil { + return model.Chat{}, err + } + if a.chatCache != nil { + _ = a.chatCache.Set(ctx, u.ID, chatID, c) + } + return c, nil + }) + if err != nil { + if errors.Is(err, store.ErrChatNotFound) { + writeError(w, http.StatusNotFound, requestID, codeNotFound, "chat not found") + } else { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "database unavailable") + } + return + } + c, ok := result.(model.Chat) + if !ok { + writeError(w, http.StatusInternalServerError, requestID, codeInternal, "internal error") + return + } + writeJSON(w, http.StatusOK, requestID, c) +} diff --git a/internal/httpapi/health.go b/internal/httpapi/health.go new file mode 100644 index 00000000..8df2046a --- /dev/null +++ b/internal/httpapi/health.go @@ -0,0 +1,39 @@ +package httpapi + +import ( + "context" + "net/http" + "sync/atomic" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" +) + +func handleHealthLive(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, newRequestID(), map[string]string{"status": "ok"}) +} + +func handleHealthReady(pool *pgxpool.Pool, rdb *redis.Client, draining *atomic.Bool) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + requestID := newRequestID() + if draining.Load() { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "shutting down") + return + } + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + + if err := pool.Ping(ctx); err != nil { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "database unavailable") + return + } + if rdb != nil { + if err := rdb.Ping(ctx).Err(); err != nil { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "session store unavailable") + return + } + } + writeJSON(w, http.StatusOK, requestID, map[string]string{"status": "ok"}) + } +} diff --git a/internal/httpapi/httpx.go b/internal/httpapi/httpx.go new file mode 100644 index 00000000..c6286119 --- /dev/null +++ b/internal/httpapi/httpx.go @@ -0,0 +1,98 @@ +package httpapi + +import ( + "bytes" + "crypto/rand" + "encoding/json" + "fmt" + "io" + "mime" + "net/http" + "reflect" + "strings" +) + +type errorCode string + +const ( + codeInvalidRequest errorCode = "invalid_request" + codeUnauthorized errorCode = "unauthorized" + codeNotFound errorCode = "not_found" + codeConflict errorCode = "conflict" + codeUnavailable errorCode = "unavailable" + codeInternal errorCode = "internal_error" +) + +func newRequestID() string { + b := make([]byte, 12) + _, _ = rand.Read(b) + return fmt.Sprintf("%x", b) +} + +func writeJSON(w http.ResponseWriter, status int, requestID string, payload any) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Request-Id", requestID) + w.WriteHeader(status) + if payload != nil { + _ = json.NewEncoder(w).Encode(payload) + } +} + +func writeError(w http.ResponseWriter, status int, requestID string, code errorCode, message string) { + writeJSON(w, status, requestID, map[string]any{ + "error": map[string]any{ + "code": code, + "message": message, + "request_id": requestID, + }, + }) +} + +func jsonFieldNames(dst any) map[string]bool { + names := map[string]bool{} + t := reflect.TypeOf(dst) + for t.Kind() == reflect.Ptr { + t = t.Elem() + } + for i := 0; i < t.NumField(); i++ { + tag := strings.Split(t.Field(i).Tag.Get("json"), ",")[0] + if tag != "" && tag != "-" { + names[tag] = true + } + } + return names +} +func decodeJSONStrict(r *http.Request, dst any) error { + contentType := r.Header.Get("Content-Type") + mediaType, _, err := mime.ParseMediaType(contentType) + if err != nil || mediaType != "application/json" { + return fmt.Errorf("unsupported Content-Type") + } + + raw, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) + if err != nil { + return err + } + if len(bytes.TrimSpace(raw)) == 0 { + return fmt.Errorf("empty body") + } + var probe map[string]json.RawMessage + if err := json.Unmarshal(raw, &probe); err != nil { + return err + } + allowed := jsonFieldNames(dst) + for k := range probe { + if !allowed[k] { + return fmt.Errorf("unknown field %q", k) + } + } + dec := json.NewDecoder(bytes.NewReader(raw)) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + return err + } + if dec.More() { + return fmt.Errorf("trailing data after JSON body") + } + return nil +} diff --git a/internal/httpapi/idempotency_test.go b/internal/httpapi/idempotency_test.go new file mode 100644 index 00000000..bdb53627 --- /dev/null +++ b/internal/httpapi/idempotency_test.go @@ -0,0 +1,184 @@ +package httpapi + +import ( + "fmt" + "net/http" + "strings" + "sync" + "testing" + + "mtusi2026/internal/model" +) + +func TestIdempotency_RepeatReturnsSameMessage(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + alice := mustRegisterUser(t, srv, "alice") + chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID}) + + key := newKey() + body := model.CreateMessageRequest{Text: "привет"} + + r1 := postMessage(t, srv, chat.ID, alice.Token, key, body) + if r1.StatusCode != http.StatusCreated { + t.Fatalf("first: expected 201, got %d", r1.StatusCode) + } + m1 := decodeBody[model.Message](t, r1) + + r2 := postMessage(t, srv, chat.ID, alice.Token, key, body) + if r2.StatusCode != http.StatusCreated { + t.Fatalf("repeat: expected 201, got %d", r2.StatusCode) + } + m2 := decodeBody[model.Message](t, r2) + + if m1.ID != m2.ID || !m1.CreatedAt.Equal(m2.CreatedAt) { + t.Fatalf("repeat returned a different message: %+v vs %+v", m1, m2) + } + + page := decodeBody[model.MessagePage](t, + getJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chat.ID), alice.Token)) + if len(page.Items) != 1 { + t.Fatalf("expected 1 message in history, got %d", len(page.Items)) + } +} + +func TestIdempotency_DifferentBodySameKey_Returns409(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + alice := mustRegisterUser(t, srv, "alice") + chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID}) + + key := newKey() + postMessage(t, srv, chat.ID, alice.Token, key, model.CreateMessageRequest{Text: "один"}).Body.Close() + + resp := postMessage(t, srv, chat.ID, alice.Token, key, model.CreateMessageRequest{Text: "два"}) + defer resp.Body.Close() + if resp.StatusCode != http.StatusConflict { + t.Fatalf("expected 409, got %d", resp.StatusCode) + } +} + +func TestIdempotency_MissingOrInvalidKey_Returns400(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + alice := mustRegisterUser(t, srv, "alice") + chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID}) + + keys := map[string]string{ + "missing": "", + "space": "bad key", + "too long": strings.Repeat("a", 129), + "cyrillic": "ключ", + } + for name, key := range keys { + resp := postMessage(t, srv, chat.ID, alice.Token, key, model.CreateMessageRequest{Text: "x"}) + resp.Body.Close() + if resp.StatusCode != http.StatusBadRequest { + t.Errorf("%s: expected 400, got %d", name, resp.StatusCode) + } + } +} + +func TestIdempotency_ConcurrentSameKey_OneMessage(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + alice := mustRegisterUser(t, srv, "alice") + chat := mustCreateChat(t, srv, alice, "Чат", []string{alice.ID}) + + const workers = 20 + key := newKey() + start := make(chan struct{}) + type result struct { + status int + id string + err error + } + results := make(chan result, workers) + + var wg sync.WaitGroup + for i := 0; i < workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + resp, err := doMessage(srv, chat.ID, alice.Token, key, model.CreateMessageRequest{Text: "гонка"}) + if err != nil { + results <- result{err: err} + return + } + defer resp.Body.Close() + var m model.Message + _ = t.Fatal + results <- result{status: resp.StatusCode, id: m.ID} + }() + } + close(start) + wg.Wait() + close(results) + + ids := map[string]bool{} + for r := range results { + if r.err != nil { + t.Errorf("request error: %v", r.err) + continue + } + if r.status != http.StatusCreated { + t.Errorf("expected 201, got %d", r.status) + } + ids[r.id] = true + } + if len(ids) != 1 { + t.Fatalf("expected one distinct message id, got %d", len(ids)) + } + + page := decodeBody[model.MessagePage](t, + getJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages", chat.ID), alice.Token)) + if len(page.Items) != 1 { + t.Fatalf("expected exactly 1 message stored, got %d", len(page.Items)) + } +} + +func TestIdempotency_ScopeIsPerUserAndChat(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + alice := mustRegisterUser(t, srv, "alice") + bob := mustRegisterUser(t, srv, "bob") + chatA := mustCreateChat(t, srv, alice, "A", []string{alice.ID, bob.ID}) + chatB := mustCreateChat(t, srv, alice, "B", []string{alice.ID, bob.ID}) + + key := newKey() + body := model.CreateMessageRequest{Text: "то же самое"} + ids := map[string]bool{} + for _, c := range []struct { + chat string + token string + }{ + {chatA.ID, alice.Token}, {chatB.ID, alice.Token}, {chatA.ID, bob.Token}, + } { + resp := postMessage(t, srv, c.chat, c.token, key, body) + if resp.StatusCode != http.StatusCreated { + t.Fatalf("expected 201, got %d", resp.StatusCode) + } + ids[decodeBody[model.Message](t, resp).ID] = true + } + if len(ids) != 3 { + t.Fatalf("same key in different scopes must create 3 messages, got %d", len(ids)) + } +} + +func TestIdempotency_ForeignChatRepeat_Returns404(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + alice := mustRegisterUser(t, srv, "alice") + carol := mustRegisterUser(t, srv, "carol") + chat := mustCreateChat(t, srv, alice, "Приватный", []string{alice.ID}) + + key := newKey() + for i := 0; i < 2; i++ { + resp := postMessage(t, srv, chat.ID, carol.Token, key, model.CreateMessageRequest{Text: "чужой"}) + resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("attempt %d: expected 404, got %d", i, resp.StatusCode) + } + } +} diff --git a/internal/httpapi/messages.go b/internal/httpapi/messages.go new file mode 100644 index 00000000..dd1a2af9 --- /dev/null +++ b/internal/httpapi/messages.go @@ -0,0 +1,80 @@ +package httpapi + +import ( + "errors" + "net/http" + "regexp" + + "mtusi2026/internal/model" + "mtusi2026/internal/store" +) + +var idemKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +func (a *api) handleCreateMessage(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { + chatID, ok := chatIDParam(w, r, requestID) + if !ok { + return + } + key := r.Header.Get("Idempotency-Key") + if !idemKeyRe.MatchString(key) { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "missing or invalid Idempotency-Key") + return + } + + var req model.CreateMessageRequest + if err := decodeJSONStrict(r, &req); err != nil { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "malformed request body") + return + } + + m, err := a.store.CreateMessage(r.Context(), chatID, u.ID, key, req.Text) + if err != nil { + switch { + case errors.Is(err, store.ErrChatNotFound): + writeError(w, http.StatusNotFound, requestID, codeNotFound, "chat not found") + case errors.Is(err, store.ErrIdempotencyConflict): + writeError(w, http.StatusConflict, requestID, codeConflict, "Idempotency-Key was used with a different request body") + case errors.Is(err, store.ErrValidation): + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid text") + default: + writeError(w, http.StatusInternalServerError, requestID, codeInternal, "internal error") + } + return + } + writeJSON(w, http.StatusCreated, requestID, m) +} + +func (a *api) handleListMessages(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { + chatID, ok := chatIDParam(w, r, requestID) + if !ok { + return + } + limit, ok := parseLimit(r) + if !ok { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid limit") + return + } + cursorStr, ok := parseCursor(r) + if !ok { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid cursor") + return + } + + items, next, err := a.store.ListMessages(r.Context(), chatID, u.ID, cursorStr, limit) + if err != nil { + switch { + case errors.Is(err, store.ErrChatNotFound): + writeError(w, http.StatusNotFound, requestID, codeNotFound, "chat not found") + case errors.Is(err, store.ErrValidation): + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid cursor") + default: + writeError(w, http.StatusInternalServerError, requestID, codeInternal, "internal error") + } + return + } + if items == nil { + items = []model.Message{} + } + writeJSON(w, http.StatusOK, requestID, model.MessagePage{Items: items, NextCursor: next}) +} diff --git a/internal/httpapi/middleware.go b/internal/httpapi/middleware.go new file mode 100644 index 00000000..14d7bff5 --- /dev/null +++ b/internal/httpapi/middleware.go @@ -0,0 +1,41 @@ +package httpapi + +import ( + "log" + "net/http" + "time" + + "mtusi2026/internal/store" +) + +type statusRecorder struct { + http.ResponseWriter + status int +} + +func (r *statusRecorder) WriteHeader(status int) { + r.status = status + r.ResponseWriter.WriteHeader(status) +} + +func loggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK} + next.ServeHTTP(rec, r) + log.Printf("method=%s path=%s status=%d duration=%s", + r.Method, r.URL.Path, rec.status, time.Since(start)) + }) +} + +func concurrencyMiddleware(limiter *store.Limiter, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !limiter.TryAcquire() { + requestID := newRequestID() + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "сервер перегружен, попробуйте позже") + return + } + defer limiter.Release() + next.ServeHTTP(w, r) + }) +} diff --git a/internal/httpapi/race_test.go b/internal/httpapi/race_test.go new file mode 100644 index 00000000..a7f1ec6e --- /dev/null +++ b/internal/httpapi/race_test.go @@ -0,0 +1,163 @@ +package httpapi + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "strings" + "sync" + "testing" + + "mtusi2026/internal/model" +) + +func TestConcurrentMessages_NoLostOrDuplicated(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + alice := mustRegisterUser(t, srv, "alice") + chat := mustCreateChat(t, srv, alice, "нагрузочный чат", []string{alice.ID}) + const clients = 10 + const perClients = 10 + const total = clients * perClients + + var wg sync.WaitGroup + errCh := make(chan error, total) + client := srv.Client() + + for c := 0; c < clients; c++ { + wg.Add(1) + go func(clientNum int) { + defer wg.Done() + for i := 0; i < perClients; i++ { + text := fmt.Sprintf("client-%d-msg-%d", clientNum, i) + body, _ := json.Marshal(model.CreateMessageRequest{Text: text}) + req, err := http.NewRequest( + http.MethodPost, + fmt.Sprintf("%s/api/v1/chats/%s/messages", srv.URL, chat.ID), + bytes.NewReader(body), + ) + if err != nil { + errCh <- fmt.Errorf("client %d: build request: %w", clientNum, err) + continue + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+alice.Token) + req.Header.Set("Idempotency-Key", fmt.Sprintf("race-%d-%d", clientNum, i)) + resp, err := client.Do(req) + if err != nil { + errCh <- fmt.Errorf("client %d: do request: %w", clientNum, err) + continue + } + if resp.StatusCode != http.StatusCreated { + errCh <- fmt.Errorf("client %d msg %d: expected 201, got %d", clientNum, i, resp.StatusCode) + } + resp.Body.Close() + } + }(c) + } + wg.Wait() + close(errCh) + + for err := range errCh { + t.Error(err) + } + if t.Failed() { + t.Fatal("прекращение из-за ошбок запросов") + } + + resp := getJSON(t, srv, fmt.Sprintf("/api/v1/chats/%s/messages?limit=100", chat.ID), alice.Token) + if resp.StatusCode != http.StatusOK { + t.Fatalf("list messages: expected 200, got %d", resp.StatusCode) + } + page := decodeBody[model.MessagePage](t, resp) + + if page.NextCursor != nil { + t.Fatalf("expected all %d messages to fit on one page with limit=100, got next_cursor", total) + } + if len(page.Items) != total { + t.Fatalf("expected exactly %d messages, got %d", total, len(page.Items)) + } + + seen := make(map[string]bool, total) + for _, m := range page.Items { + if seen[m.ID] { + t.Fatalf("duplicate message ID found: %s", m.ID) + } + seen[m.ID] = true + if m.SenderID != alice.ID { + t.Fatalf("unexpected sender_id: %s", m.SenderID) + } + } +} + +func TestRegisterUser_ConcurrentDuplicateUsername_OnlyOneSucceeds(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + const workers = 20 + + statuses := make(chan int, workers) + + var wg sync.WaitGroup + wg.Add(workers) + + for i := 0; i < workers; i++ { + go func() { + defer wg.Done() + + body := `{ + "username": "raceuser", + "display_name": "Race User", + "password": "correct-horse-battery-staple" + }` + + req, err := http.NewRequest( + http.MethodPost, + srv.URL+"/api/v1/users", + strings.NewReader(body), + ) + if err != nil { + statuses <- -1 + return + } + + req.Header.Set("Content-Type", "application/json") + + resp, err := srv.Client().Do(req) + if err != nil { + statuses <- -1 + return + } + + statuses <- resp.StatusCode + resp.Body.Close() + }() + } + + wg.Wait() + close(statuses) + + created := 0 + conflicts := 0 + + for status := range statuses { + switch status { + case http.StatusCreated: + created++ + case http.StatusConflict: + conflicts++ + default: + t.Errorf("unexpected status code: %d", status) + } + } + + if created != 1 { + t.Fatalf("ожидалась одна успешная регистрация, got %d", created) + } + + if conflicts != workers-1 { + t.Fatalf("expected %d conflicts, got %d", workers-1, conflicts) + } +} diff --git a/internal/httpapi/router.go b/internal/httpapi/router.go new file mode 100644 index 00000000..b9479e94 --- /dev/null +++ b/internal/httpapi/router.go @@ -0,0 +1,84 @@ +package httpapi + +import ( + "net/http" + "sync/atomic" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + + "mtusi2026/internal/cache" + "mtusi2026/internal/model" + "mtusi2026/internal/sessionstore" + "mtusi2026/internal/store" +) + +type Options struct { + InstanceID string + Redis *redis.Client + Draining *atomic.Bool +} + +func NewRouter(s *store.Store, pool *pgxpool.Pool, sessionStores ...sessionstore.Store) http.Handler { + var ss sessionstore.Store + if len(sessionStores) > 0 { + ss = sessionStores[0] + } + return NewRouterWithCache(s, pool, ss, nil) +} + +func NewRouterWithCache( + s *store.Store, + pool *pgxpool.Pool, + sessionStore sessionstore.Store, + chatCache *cache.Cache, + opts ...Options, +) http.Handler { + var o Options + if len(opts) > 0 { + o = opts[0] + } + if o.InstanceID == "" { + o.InstanceID = "local" + } + if o.Draining == nil { + o.Draining = new(atomic.Bool) + } + + a := &api{ + store: s, + sessionStore: sessionStore, + chatCache: chatCache, + } + + apiMux := http.NewServeMux() + apiMux.HandleFunc("POST /api/v1/users", a.handleRegisterUser) + apiMux.HandleFunc("GET /api/v1/users/me", a.authMiddleware(a.handleCurrentUser)) + apiMux.HandleFunc("POST /api/v1/chats", a.authMiddleware(a.handleCreateChat)) + apiMux.HandleFunc("GET /api/v1/chats", a.authMiddleware(a.handleListChats)) + apiMux.HandleFunc("GET /api/v1/chats/{chat_id}", a.authMiddleware(a.handleGetChat)) + apiMux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleCreateMessage)) + apiMux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleListMessages)) + apiMux.HandleFunc("POST /api/v1/auth/sessions", a.handleCreateSession) + apiMux.HandleFunc("DELETE /api/v1/auth/sessions/current", a.authMiddleware( + func(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { + a.handleDeleteSession(w, r) + })) + + root := http.NewServeMux() + root.HandleFunc("GET /health/live", handleHealthLive) + root.HandleFunc("GET /health/ready", handleHealthReady(pool, o.Redis, o.Draining)) + root.Handle("/", concurrencyMiddleware(s.Limiter, apiMux)) + + var h http.Handler = root + h = loggingMiddleware(h) + h = instanceMiddleware(o.InstanceID, h) + return h +} + +func instanceMiddleware(id string, next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Instance-Id", id) + next.ServeHTTP(w, r) + }) +} diff --git a/internal/httpapi/sessionstore_test.go b/internal/httpapi/sessionstore_test.go new file mode 100644 index 00000000..72936c57 --- /dev/null +++ b/internal/httpapi/sessionstore_test.go @@ -0,0 +1,73 @@ +package httpapi + +import ( + "context" + "sync" + "time" + + "mtusi2026/internal/sessionstore" +) + +type memorySession struct { + userID string + expiresAt time.Time +} + +type memorySessionStore struct { + mu sync.Mutex + sessions map[string]memorySession +} + +func newMemorySessionStore() *memorySessionStore { + return &memorySessionStore{ + sessions: make(map[string]memorySession), + } +} + +func (s *memorySessionStore) Create( + _ context.Context, + token string, + userID string, + ttl time.Duration, +) (time.Time, error) { + s.mu.Lock() + defer s.mu.Unlock() + + expiresAt := time.Now().UTC().Add(ttl) + s.sessions[token] = memorySession{ + userID: userID, + expiresAt: expiresAt, + } + return expiresAt, nil +} + +func (s *memorySessionStore) GetUserID( + _ context.Context, + token string, +) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + + session, ok := s.sessions[token] + if !ok || !time.Now().UTC().Before(session.expiresAt) { + delete(s.sessions, token) + return "", sessionstore.ErrNotFound + } + + return session.userID, nil +} + +func (s *memorySessionStore) Delete( + _ context.Context, + token string, +) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if _, ok := s.sessions[token]; !ok { + return false, nil + } + + delete(s.sessions, token) + return true, nil +} diff --git a/internal/httpapi/testmain_test.go b/internal/httpapi/testmain_test.go new file mode 100644 index 00000000..a8bfef4f --- /dev/null +++ b/internal/httpapi/testmain_test.go @@ -0,0 +1,145 @@ +package httpapi + +import ( + "context" + "fmt" + "log" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/pgx/v5" + _ "github.com/golang-migrate/migrate/v4/source/file" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" + "github.com/testcontainers/testcontainers-go" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" + tcredis "github.com/testcontainers/testcontainers-go/modules/redis" + "github.com/testcontainers/testcontainers-go/wait" +) + +var testPool *pgxpool.Pool +var testRedis *redis.Client + +// поднимает эфемерный Postgres в Docker применяет миграции даёт testPool +// всем тестам пакета и гарантированно убивает контейнер в конце прогона :) +func TestMain(m *testing.M) { + ctx := context.Background() + + pgContainer, err := tcpostgres.Run(ctx, + "postgres:16.4-alpine", + tcpostgres.WithDatabase("messenger_test"), + tcpostgres.WithUsername("messenger_test"), + tcpostgres.WithPassword("messenger_test"), + testcontainers.WithWaitStrategy( + wait.ForLog("database system is ready to accept connections"). + WithOccurrence(2). + WithStartupTimeout(30*time.Second), + ), + ) + if err != nil { + log.Fatalf("start postgres container: %v", err) + } + defer func() { + if err := pgContainer.Terminate(ctx); err != nil { + log.Printf("terminate postgres container: %v", err) + } + }() + + dsn, err := pgContainer.ConnectionString(ctx, "sslmode=disable") + if err != nil { + log.Fatalf("get connection string: %v", err) + } + + if err := applyMigrations(dsn); err != nil { + log.Fatalf("apply migrations: %v", err) + } + + poolCfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + log.Fatalf("parse pool config: %v", err) + } + testPool, err = pgxpool.NewWithConfig(ctx, poolCfg) + if err != nil { + log.Fatalf("create pool: %v", err) + } + defer testPool.Close() + + if err := testPool.Ping(ctx); err != nil { + log.Fatalf("ping test db: %v", err) + } + + redisContainer, err := tcredis.Run(ctx, "redis:7.4-alpine") + if err != nil { + log.Fatalf("start redis container: %v", err) + } + defer func() { + if err := testcontainers.TerminateContainer(redisContainer); err != nil { + log.Printf("terminate redis container: %v", err) + } + }() + + redisURL, err := redisContainer.ConnectionString(ctx) + if err != nil { + log.Fatalf("get redis connection string: %v", err) + } + + redisOpts, err := redis.ParseURL(redisURL) + if err != nil { + log.Fatalf("parse redis url: %v", err) + } + + testRedis = redis.NewClient(redisOpts) + defer testRedis.Close() + + if err := testRedis.Ping(ctx).Err(); err != nil { + log.Fatalf("ping test redis: %v", err) + } + + os.Exit(m.Run()) +} + +// тесты запускаются из internal/httpapi/ поэтому путь на три уровня вверх +// и прогоняет все .up.sql через golang-migrate +func applyMigrations(dsn string) error { + migrationsPath, err := filepath.Abs(filepath.Join("..", "..", "migrations")) + if err != nil { + return err + } + + u, err := url.Parse(dsn) + if err != nil { + return fmt.Errorf("parse dsn: %w", err) + } + u.Scheme = "pgx5" + + m, err := migrate.New( + fmt.Sprintf("file://%s", migrationsPath), + u.String(), + ) + if err != nil { + return fmt.Errorf("migrate.New: %w", err) + } + defer m.Close() + + if err := m.Up(); err != nil && err != migrate.ErrNoChange { + return fmt.Errorf("migrate.Up: %w", err) + } + return nil +} + +// truncateAll очищает все таблицы между тестами сохраняя схему +// быстрее чем пересоздавать контейнер на каждый тест +func truncateAll(t *testing.T) { + t.Helper() + ctx := context.Background() + _, err := testPool.Exec(ctx, + `TRUNCATE TABLE messages, chat_members, chats, sessions, users RESTART IDENTITY CASCADE`, + ) + if err != nil { + t.Fatalf("truncate tables: %v", err) + } +} diff --git a/internal/httpapi/transaction_test.go b/internal/httpapi/transaction_test.go new file mode 100644 index 00000000..f1702a27 --- /dev/null +++ b/internal/httpapi/transaction_test.go @@ -0,0 +1,63 @@ +package httpapi + +import ( + "context" + "errors" + "testing" + + "mtusi2026/internal/store" +) + +func TestCreateChat_RollsBackOnMemberInsertFailure(t *testing.T) { + srv := newTestServer(t) + defer srv.Close() + + alice := mustRegisterUser(t, srv, "rollback_alice") + + ctx := context.Background() + s := store.New(testPool, newUnlimitedLimiter()) + + missingUserID := "00000000-0000-0000-0000-000000000000" + + _, err := s.CreateChat( + ctx, + alice.ID, + "rollback-test", + []string{alice.ID, missingUserID}, + ) + + if !errors.Is(err, store.ErrValidation) { + t.Fatalf("expected ErrValidation, got %v", err) + } + + var chatCount int + err = testPool.QueryRow( + ctx, + `SELECT COUNT(*) FROM chats WHERE title = $1`, + "rollback-test", + ).Scan(&chatCount) + if err != nil { + t.Fatalf("check chats after rollback: %v", err) + } + + if chatCount != 0 { + t.Fatalf("expected 0 chats after rollback, got %d", chatCount) + } + + var membershipCount int + err = testPool.QueryRow( + ctx, + `SELECT COUNT(*) + FROM chat_members cm + JOIN chats c ON c.id = cm.chat_id + WHERE c.title = $1`, + "rollback-test", + ).Scan(&membershipCount) + if err != nil { + t.Fatalf("check chat_members after rollback: %v", err) + } + + if membershipCount != 0 { + t.Fatalf("expected 0 chat members after rollback, got %d", membershipCount) + } +} diff --git a/internal/httpapi/users.go b/internal/httpapi/users.go new file mode 100644 index 00000000..939ad09f --- /dev/null +++ b/internal/httpapi/users.go @@ -0,0 +1,117 @@ +package httpapi + +import ( + "errors" + "net/http" + "regexp" + "unicode/utf8" + + "golang.org/x/sync/singleflight" + + "mtusi2026/internal/auth" + "mtusi2026/internal/cache" + "mtusi2026/internal/model" + "mtusi2026/internal/sessionstore" + "mtusi2026/internal/store" +) + +var usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +type api struct { + store *store.Store + sessionStore sessionstore.Store + chatCache *cache.Cache + chatFlight singleflight.Group +} + +func (a *api) authMiddleware(next func(w http.ResponseWriter, r *http.Request, u model.User, requestID string)) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + requestID := newRequestID() + + token, ok := bearerToken(r) + if !ok { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + writeError(w, http.StatusUnauthorized, requestID, codeUnauthorized, "missing or malformed Bearer token") + return + } + + userID, err := a.sessionStore.GetUserID(r.Context(), token) + if err != nil { + if errors.Is(err, sessionstore.ErrNotFound) { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + writeError(w, http.StatusUnauthorized, requestID, codeUnauthorized, "invalid, expired or revoked token") + return + } + + writeError(w, http.StatusServiceUnavailable, requestID, codeInternal, "session service unavailable") + return + } + + u, ok, err := a.store.GetUserByID(r.Context(), userID) + if err != nil { + writeError(w, http.StatusServiceUnavailable, requestID, codeInternal, "database unavailable") + return + } + if !ok { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + writeError(w, http.StatusUnauthorized, requestID, codeUnauthorized, "invalid session user") + return + } + + next(w, r, u, requestID) + } +} + +func validDisplayName(s string) bool { + return validText(s, 100) +} + +func validPassword(s string) bool { + n := utf8.RuneCountInString(s) + return n >= 12 && n <= 128 +} + +func (a *api) handleRegisterUser(w http.ResponseWriter, r *http.Request) { + requestID := newRequestID() + + var req model.CreateUserRequest + if err := decodeJSONStrict(r, &req); err != nil { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "malformed request body") + return + } + + if !usernameRe.MatchString(req.Username) { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "username must match ^[a-z0-9_]{3,32}$") + return + } + if !validDisplayName(req.DisplayName) { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "display_name must be 1-100 non-blank code points") + return + } + if !validPassword(req.Password) { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "password must be 12-128 code points") + return + } + + passwordHash, err := auth.HashPassword(req.Password) + if err != nil { + writeError(w, http.StatusInternalServerError, requestID, codeInternal, "internal error") + return + } + + u, err := a.store.CreateUser(r.Context(), req.Username, req.DisplayName, passwordHash) + if err != nil { + if err == store.ErrUsernameTaken { + writeError(w, http.StatusConflict, requestID, codeConflict, "username already taken") + return + } + writeError(w, http.StatusInternalServerError, requestID, codeInternal, "internal error") + return + } + + writeJSON(w, http.StatusCreated, requestID, u) +} + +func (a *api) handleCurrentUser(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { + writeJSON(w, http.StatusOK, requestID, u) +} diff --git a/internal/httpapi/validate.go b/internal/httpapi/validate.go new file mode 100644 index 00000000..11da225c --- /dev/null +++ b/internal/httpapi/validate.go @@ -0,0 +1,45 @@ +package httpapi + +import ( + "net/http" + "strings" + "unicode" + "unicode/utf8" + + "mtusi2026/internal/idgen" +) + +func validText(s string, max int) bool { + n := utf8.RuneCountInString(s) + if n < 1 || n > max { + return false + } + hasContent := false + for _, r := range s { + if r == 0 { + return false + } + if !unicode.IsSpace(r) { + hasContent = true + } + } + return hasContent +} + +func parseCursor(r *http.Request) (string, bool) { + q := r.URL.Query() + if !q.Has("cursor") { + return "", true + } + c := q.Get("cursor") + return c, c != "" +} + +func chatIDParam(w http.ResponseWriter, r *http.Request, requestID string) (string, bool) { + id := r.PathValue("chat_id") + if !idgen.IsValidUUID(id) { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid chat_id") + return "", false + } + return strings.ToLower(id), true +} diff --git a/internal/idgen/uuid.go b/internal/idgen/uuid.go new file mode 100644 index 00000000..50b9c6ce --- /dev/null +++ b/internal/idgen/uuid.go @@ -0,0 +1,23 @@ +package idgen + +import ( + "crypto/rand" + "fmt" + "regexp" +) + +var uuidRe = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + +func IsValidUUID(s string) bool { + return uuidRe.MatchString(s) +} + +func GenerateUUIDv4() string { + b := make([]byte, 16) + if _, err := rand.Read(b); err != nil { + panic(err) + } + b[6] = (b[6] & 0x0f) | 0x40 + b[8] = (b[8] & 0x3f) | 0x80 + return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]) +} diff --git a/internal/model/model.go b/internal/model/model.go new file mode 100644 index 00000000..e468a347 --- /dev/null +++ b/internal/model/model.go @@ -0,0 +1,65 @@ +package model + +import "time" + +type User struct { + ID string `json:"id"` + Username string `json:"username"` + DisplayName string `json:"display_name"` + CreatedAt time.Time `json:"created_at"` +} + +type CreateUserRequest struct { + Username string `json:"username"` + DisplayName string `json:"display_name"` + Password string `json:"password"` +} + +type Chat struct { + ID string `json:"id"` + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` + CreatedAt time.Time `json:"created_at"` +} + +type CreateChatRequest struct { + Title string `json:"title"` + MemberIDs []string `json:"member_ids"` +} + +type ChatPage struct { + Items []Chat `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type Message struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt time.Time `json:"created_at"` +} + +type CreateMessageRequest struct { + Text string `json:"text"` +} + +type MessagePage struct { + Items []Message `json:"items"` + NextCursor *string `json:"next_cursor"` +} + +type ErrorEnvelope struct { + Error struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` + } `json:"error"` +} + +type Session struct { + Token string `json:"token"` + TokenType string `json:"token_type"` + ExpiresAt time.Time `json:"expires_at"` + User User `json:"user"` +} diff --git a/internal/sessionstore/redis.go b/internal/sessionstore/redis.go new file mode 100644 index 00000000..2aadedbe --- /dev/null +++ b/internal/sessionstore/redis.go @@ -0,0 +1,73 @@ +package sessionstore + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/redis/go-redis/v9" +) + +type RedisStore struct { + client *redis.Client +} + +func NewRedis(client *redis.Client) *RedisStore { + return &RedisStore{client: client} +} + +func NewRedisClient(ctx context.Context, addr string) (*redis.Client, error) { + client := redis.NewClient(&redis.Options{ + Addr: addr, + }) + + if err := client.Ping(ctx).Err(); err != nil { + _ = client.Close() + return nil, fmt.Errorf("%w: redis ping: %v", ErrUnavailable, err) + } + + return client, nil +} + +func sessionKey(token string) string { + return "session:" + token +} + +func (s *RedisStore) Create( + ctx context.Context, + token string, + userID string, + ttl time.Duration, +) (time.Time, error) { + if ttl <= 0 { + return time.Time{}, fmt.Errorf("invalid session ttl: %s", ttl) + } + + if err := s.client.Set(ctx, sessionKey(token), userID, ttl).Err(); err != nil { + return time.Time{}, fmt.Errorf("%w: create session: %v", ErrUnavailable, err) + } + + return time.Now().UTC().Add(ttl), nil +} + +func (s *RedisStore) GetUserID(ctx context.Context, token string) (string, error) { + userID, err := s.client.Get(ctx, sessionKey(token)).Result() + if errors.Is(err, redis.Nil) { + return "", ErrNotFound + } + if err != nil { + return "", fmt.Errorf("%w: get session: %v", ErrUnavailable, err) + } + + return userID, nil +} + +func (s *RedisStore) Delete(ctx context.Context, token string) (bool, error) { + n, err := s.client.Del(ctx, sessionKey(token)).Result() + if err != nil { + return false, fmt.Errorf("%w: delete session: %v", ErrUnavailable, err) + } + + return n > 0, nil +} diff --git a/internal/sessionstore/sessionstore.go b/internal/sessionstore/sessionstore.go new file mode 100644 index 00000000..506982e7 --- /dev/null +++ b/internal/sessionstore/sessionstore.go @@ -0,0 +1,18 @@ +package sessionstore + +import ( + "context" + "errors" + "time" +) + +var ( + ErrNotFound = errors.New("session not found or expired") + ErrUnavailable = errors.New("session store unavailable") +) + +type Store interface { + Create(ctx context.Context, token, userID string, ttl time.Duration) (time.Time, error) + GetUserID(ctx context.Context, token string) (string, error) + Delete(ctx context.Context, token string) (bool, error) +} diff --git a/internal/store/limits.go b/internal/store/limits.go new file mode 100644 index 00000000..37630f05 --- /dev/null +++ b/internal/store/limits.go @@ -0,0 +1,51 @@ +package store + +import "sync/atomic" + +type Limiter struct { + maxMessages int64 + messageCount int64 + maxConcurrent int + inFlight chan struct{} +} + +func NewLimiter(maxMessages int64, maxConcurrent int) *Limiter { + return &Limiter{ + maxMessages: maxMessages, + maxConcurrent: maxConcurrent, + inFlight: make(chan struct{}, maxConcurrent), + } +} + +func (l *Limiter) TryReserveMessage() bool { + for { + cur := atomic.LoadInt64(&l.messageCount) + if cur >= l.maxMessages { + return false + } + if atomic.CompareAndSwapInt64(&l.messageCount, cur, cur+1) { + return true + } + } +} + +func (l *Limiter) ReleaseMessage() { + atomic.AddInt64(&l.messageCount, -1) +} + +func (l *Limiter) CurrentMessageCount() int64 { + return atomic.LoadInt64(&l.messageCount) +} + +func (l *Limiter) TryAcquire() bool { + select { + case l.inFlight <- struct{}{}: + return true + default: + return false + } +} + +func (l *Limiter) Release() { + <-l.inFlight +} diff --git a/internal/store/store.go b/internal/store/store.go new file mode 100644 index 00000000..13ec0389 --- /dev/null +++ b/internal/store/store.go @@ -0,0 +1,436 @@ +package store + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strconv" + "sync/atomic" + "unicode" + "unicode/utf8" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "mtusi2026/internal/cursor" + "mtusi2026/internal/idgen" + "mtusi2026/internal/model" +) + +var ( + ErrUsernameTaken = errors.New("username taken") + ErrChatNotFound = errors.New("chat not found or not accessible") + ErrValidation = errors.New("validation failed") + ErrCapacityExceeded = errors.New("message capacity exceeded") +) + +type Store struct { + pool *pgxpool.Pool + Limiter *Limiter + chatSQLQueries atomic.Int64 +} + +func New(pool *pgxpool.Pool, limiter *Limiter) *Store { + return &Store{pool: pool, Limiter: limiter} +} + +func (s *Store) CreateUser(ctx context.Context, username, displayName, passwordHash string) (model.User, error) { + var u model.User + err := s.pool.QueryRow(ctx, + `INSERT INTO users (username, display_name, password_hash) + VALUES ($1, $2, $3) + RETURNING id, username, display_name, created_at`, + username, displayName, passwordHash, + ).Scan(&u.ID, &u.Username, &u.DisplayName, &u.CreatedAt) + + if err != nil { + if isUniqueViolation(err) { + return model.User{}, ErrUsernameTaken + } + return model.User{}, err + } + return u, nil +} + +func (s *Store) GetUserByID(ctx context.Context, id string) (model.User, bool, error) { + var u model.User + err := s.pool.QueryRow(ctx, + `SELECT id, username, display_name, created_at FROM users WHERE id = $1`, + id, + ).Scan(&u.ID, &u.Username, &u.DisplayName, &u.CreatedAt) + + if errors.Is(err, pgx.ErrNoRows) { + return model.User{}, false, nil + } + if err != nil { + return model.User{}, false, err + } + + return u, true, nil +} + +func (s *Store) CreateChat(ctx context.Context, callerID, title string, memberIDs []string) (model.Chat, error) { + if len(memberIDs) == 0 || len(memberIDs) > 100 { + return model.Chat{}, ErrValidation + } + + seen := make(map[string]bool, len(memberIDs)) + callerIncluded := false + for _, id := range memberIDs { + if seen[id] { + return model.Chat{}, ErrValidation + } + seen[id] = true + if id == callerID { + callerIncluded = true + } + } + if !callerIncluded { + return model.Chat{}, ErrValidation + } + + tx, err := s.pool.Begin(ctx) + if err != nil { + return model.Chat{}, err + } + defer tx.Rollback(ctx) + + var c model.Chat + err = tx.QueryRow(ctx, + `INSERT INTO chats (title) VALUES ($1) RETURNING id, title, created_at`, + title, + ).Scan(&c.ID, &c.Title, &c.CreatedAt) + if err != nil { + return model.Chat{}, err + } + + for _, memberID := range memberIDs { + _, err = tx.Exec(ctx, + `INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)`, + c.ID, memberID, + ) + if err != nil { + if isForeignKeyViolation(err) { + return model.Chat{}, ErrValidation + } + return model.Chat{}, err + } + } + + if err := tx.Commit(ctx); err != nil { + return model.Chat{}, err + } + + c.MemberIDs = append([]string{}, memberIDs...) + return c, nil +} + +func (s *Store) GetChat(ctx context.Context, chatID, userID string) (model.Chat, error) { + var c model.Chat + s.chatSQLQueries.Add(1) + + err := s.pool.QueryRow(ctx, + `SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id AND cm.user_id = $2 + WHERE c.id = $1`, + chatID, userID, + ).Scan(&c.ID, &c.Title, &c.CreatedAt) + + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return model.Chat{}, ErrChatNotFound + } + return model.Chat{}, fmt.Errorf("get chat: %w", err) + } + + memberIDs, err := s.listChatMemberIDs(ctx, c.ID) + if err != nil { + return model.Chat{}, err + } + c.MemberIDs = memberIDs + return c, nil +} + +func (s *Store) listChatMemberIDs(ctx context.Context, chatID string) ([]string, error) { + s.chatSQLQueries.Add(1) + + rows, err := s.pool.Query(ctx, `SELECT user_id FROM chat_members WHERE chat_id = $1`, chatID) + if err != nil { + return nil, err + } + defer rows.Close() + + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + return ids, rows.Err() +} + +func (s *Store) ListChatsForUser(ctx context.Context, userID, cursorStr string, limit int) ([]model.Chat, *string, error) { + var after *cursor.Cursor + if cursorStr != "" { + c, err := cursor.Decode(cursorStr) + if err != nil || c.Scope != "chats:"+userID { + return nil, nil, ErrValidation + } + after = &c + } + + query := ` + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE cm.user_id = $1` + args := []any{userID} + + if after != nil { + query += ` AND (c.created_at, c.id) > ($2, $3)` + args = append(args, after.CreatedAt, after.ID) + } + query += ` ORDER BY c.created_at ASC, c.id ASC LIMIT $` + strconv.Itoa(len(args)+1) + args = append(args, limit+1) + + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return nil, nil, err + } + defer rows.Close() + + var page []model.Chat + for rows.Next() { + var c model.Chat + if err := rows.Scan(&c.ID, &c.Title, &c.CreatedAt); err != nil { + return nil, nil, err + } + page = append(page, c) + } + if err := rows.Err(); err != nil { + return nil, nil, err + } + + var next *string + if len(page) > limit { + last := page[limit-1] + nc := cursor.Encode(cursor.Cursor{Scope: "chats:" + userID, CreatedAt: last.CreatedAt, ID: last.ID}) + next = &nc + page = page[:limit] + } + + for i := range page { + memberIDs, err := s.listChatMemberIDs(ctx, page[i].ID) + if err != nil { + return nil, nil, err + } + page[i].MemberIDs = memberIDs + } + + return page, next, nil +} + +var ErrIdempotencyConflict = errors.New("idempotency key reused with different body") + +func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, idemKey, text string) (model.Message, error) { + if !validMessageText(text) { + return model.Message{}, ErrValidation + } + sum := sha256.Sum256([]byte(text)) + hash := hex.EncodeToString(sum[:]) + + tx, err := s.pool.Begin(ctx) + if err != nil { + return model.Message{}, err + } + defer tx.Rollback(ctx) + + var isMember bool + if err := tx.QueryRow(ctx, + `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id=$1 AND user_id=$2)`, + chatID, senderID).Scan(&isMember); err != nil { + return model.Message{}, err + } + if !isMember { + return model.Message{}, ErrChatNotFound + } + + msgID := idgen.GenerateUUIDv4() + tag, err := tx.Exec(ctx, + `INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) + VALUES ($1,$2,$3,$4,$5) + ON CONFLICT (user_id, chat_id, key) DO NOTHING`, + senderID, chatID, idemKey, hash, msgID) + if err != nil { + return model.Message{}, err + } + + if tag.RowsAffected() == 0 { + var storedHash string + var m model.Message + err := tx.QueryRow(ctx, + `SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at + FROM idempotency_keys k JOIN messages m ON m.id = k.message_id + WHERE k.user_id=$1 AND k.chat_id=$2 AND k.key=$3`, + senderID, chatID, idemKey, + ).Scan(&storedHash, &m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt) + if err != nil { + return model.Message{}, err + } + if storedHash != hash { + return model.Message{}, ErrIdempotencyConflict + } + return m, nil + } + + var m model.Message + err = tx.QueryRow(ctx, + `INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES ($1,$2,$3,$4, clock_timestamp()) + RETURNING id, chat_id, sender_id, text, created_at`, + msgID, chatID, senderID, text, + ).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt) + if err != nil { + if isCheckViolation(err) { + return model.Message{}, ErrValidation + } + return model.Message{}, err + } + if err := tx.Commit(ctx); err != nil { + return model.Message{}, err + } + return m, nil +} + +func (s *Store) ListMessages(ctx context.Context, chatID, userID, cursorStr string, limit int) ([]model.Message, *string, error) { + var isMember bool + err := s.pool.QueryRow(ctx, + `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, + chatID, userID, + ).Scan(&isMember) + if err != nil { + return nil, nil, err + } + if !isMember { + return nil, nil, ErrChatNotFound + } + + var after *cursor.Cursor + if cursorStr != "" { + dc, err := cursor.Decode(cursorStr) + if err != nil || dc.Scope != "messages:"+chatID { + return nil, nil, ErrValidation + } + after = &dc + } + + query := ` + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1` + args := []any{chatID} + + if after != nil { + query += ` AND (created_at, id) > ($2, $3)` + args = append(args, after.CreatedAt, after.ID) + } + query += ` ORDER BY created_at ASC, id ASC LIMIT $` + strconv.Itoa(len(args)+1) + args = append(args, limit+1) + + rows, err := s.pool.Query(ctx, query, args...) + if err != nil { + return nil, nil, err + } + defer rows.Close() + + var page []model.Message + for rows.Next() { + var m model.Message + if err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt); err != nil { + return nil, nil, err + } + page = append(page, m) + } + if err := rows.Err(); err != nil { + return nil, nil, err + } + + var next *string + if len(page) > limit { + last := page[limit-1] + nc := cursor.Encode(cursor.Cursor{Scope: "messages:" + chatID, CreatedAt: last.CreatedAt, ID: last.ID}) + next = &nc + page = page[:limit] + } + return page, next, nil +} + +func isUniqueViolation(err error) bool { + var pgErr *pgconn.PgError + return errors.As(err, &pgErr) && pgErr.Code == "23505" +} + +func isForeignKeyViolation(err error) bool { + var pgErr *pgconn.PgError + return errors.As(err, &pgErr) && pgErr.Code == "23503" +} + +func isCheckViolation(err error) bool { + var pgErr *pgconn.PgError + return errors.As(err, &pgErr) && pgErr.Code == "23514" +} + +func validMessageText(s string) bool { + n := utf8.RuneCountInString(s) + if n == 0 || n > 4000 { + return false + } + hasContent := false + for _, r := range s { + if r == 0 { + return false + } + if !unicode.IsSpace(r) { + hasContent = true + } + } + return hasContent +} + +var ErrSessionNotFound = errors.New("session not found or expired") + +func (s *Store) GetUserByUsername(ctx context.Context, username string) (model.User, string, error) { + var u model.User + var passwordHash string + + err := s.pool.QueryRow(ctx, + `SELECT id, username, display_name, created_at, password_hash + FROM users WHERE username = $1`, + username, + ).Scan(&u.ID, &u.Username, &u.DisplayName, &u.CreatedAt, &passwordHash) + + if errors.Is(err, pgx.ErrNoRows) { + return model.User{}, "", ErrUsernameNotFound + } + if err != nil { + return model.User{}, "", err + } + + return u, passwordHash, nil +} + +var ErrUsernameNotFound = errors.New("username not found") + +func (s *Store) ResetChatSQLQueries() { + s.chatSQLQueries.Store(0) +} + +func (s *Store) ChatSQLQueries() int64 { + return s.chatSQLQueries.Load() +} diff --git a/migrations/000001_init_schema.down.sql b/migrations/000001_init_schema.down.sql new file mode 100644 index 00000000..6ec50ce4 --- /dev/null +++ b/migrations/000001_init_schema.down.sql @@ -0,0 +1,7 @@ +-- 000001_init_schema.down.sql + +DROP TABLE IF EXISTS sessions; +DROP TABLE IF EXISTS messages; +DROP TABLE IF EXISTS chat_members; +DROP TABLE IF EXISTS chats; +DROP TABLE IF EXISTS users; \ No newline at end of file diff --git a/migrations/000001_init_schema.up.sql b/migrations/000001_init_schema.up.sql new file mode 100644 index 00000000..c8e20683 --- /dev/null +++ b/migrations/000001_init_schema.up.sql @@ -0,0 +1,49 @@ +-- 000001_init_schema.up.sql + +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +CREATE TABLE users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + username TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT display_name_length CHECK (char_length(trim(display_name)) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT title_length CHECK (char_length(title) BETWEEN 1 AND 100) +); + + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE messages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT text_length CHECK (char_length(text) BETWEEN 1 AND 4000) +); + + +CREATE INDEX idx_messages_chat_created_id ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token TEXT PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX idx_sessions_expires_at ON sessions (expires_at); +CREATE INDEX idx_sessions_user_id ON sessions (user_id); \ No newline at end of file diff --git a/migrations/000002_idempotency.up.sql b/migrations/000002_idempotency.up.sql new file mode 100644 index 00000000..4657fa3b --- /dev/null +++ b/migrations/000002_idempotency.up.sql @@ -0,0 +1,11 @@ +CREATE TABLE idempotency_keys ( + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + key TEXT NOT NULL, + request_hash TEXT NOT NULL, + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, chat_id, key) +); +CREATE INDEX idx_idempotency_created_at ON idempotency_keys (created_at); \ No newline at end of file diff --git a/scripts/check_edge.py b/scripts/check_edge.py new file mode 100644 index 00000000..61f8a1bf --- /dev/null +++ b/scripts/check_edge.py @@ -0,0 +1,51 @@ +import base64, json, sys, time, urllib.request, urllib.error, uuid + +BASE = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8080" +PWD = "correct-horse-battery-staple" + + +def call(method, path, body=None, token=None, basic=None, key=None): + data = json.dumps(body).encode() if body is not None else None + r = urllib.request.Request(BASE + path, data=data, method=method) + if data is not None: r.add_header("Content-Type", "application/json") + if token: r.add_header("Authorization", "Bearer " + token) + if basic: r.add_header("Authorization", "Basic " + base64.b64encode(basic.encode()).decode()) + if key: r.add_header("Idempotency-Key", key) + try: + with urllib.request.urlopen(r, timeout=10) as resp: + return resp.status, json.loads(resp.read() or b"null") + except urllib.error.HTTPError as e: + return e.code, None + + +name = "edge_" + uuid.uuid4().hex[:8] +_, u = call("POST", "/api/v1/users", {"username": name, "display_name": "e", "password": PWD}) +_, s = call("POST", "/api/v1/auth/sessions", basic=f"{name}:{PWD}") +t = s["token"] +_, c = call("POST", "/api/v1/chats", {"title": "t", "member_ids": [u["id"]]}, token=t) +m = f"/api/v1/chats/{c['id']}/messages" +k = lambda: uuid.uuid4().hex + +cases = [ + ("поле Text (другой регистр)", call("POST", m, {"Text": "x"}, token=t, key=k())[0], {400}), + ("текст из NBSP/U+3000", call("POST", m, {"text": "\u00a0\u3000"}, token=t, key=k())[0], {400}), + ("текст с NUL", call("POST", m, {"text": "a\u0000b"}, token=t, key=k())[0], {400}), + ("limit= пустой", call("GET", "/api/v1/chats?limit=", token=t)[0], {400}), + ("cursor= пустой", call("GET", m + "?cursor=", token=t)[0], {400}), + ("chat_id не UUID", call("GET", "/api/v1/chats/not-a-uuid/messages", token=t)[0], {400}), + ("member_ids не UUID", call("POST", "/api/v1/chats", {"title": "t", "member_ids": [u["id"], "not-a-uuid"]}, token=t)[0], {400}), + ("название из пробелов", call("POST", "/api/v1/chats", {"title": " ", "member_ids": [u["id"]]}, token=t)[0], {400}), + ("название 101 символ", call("POST", "/api/v1/chats", {"title": "a" * 101, "member_ids": [u["id"]]}, token=t)[0], {400}), +] +for label, got, ok in cases: + print("PASS" if got in ok else "FAIL", label, "->", got) + + +def login_ms(basic): + t0 = time.time(); call("POST", "/api/v1/auth/sessions", basic=basic) + return (time.time() - t0) * 1000 + +ms_unknown = sorted(login_ms("nobody_" + uuid.uuid4().hex[:6] + ":" + PWD) for _ in range(5))[2] +ms_wrong = sorted(login_ms(name + ":wrong-password-123") for _ in range(5))[2] +print("PASS" if abs(ms_unknown - ms_wrong) < 0.5 * ms_wrong else "FAIL", + f"тайминг входа: неизвестный {ms_unknown:.1f} мс, неверный пароль {ms_wrong:.1f} мс") \ No newline at end of file diff --git a/scripts/check_lb.py b/scripts/check_lb.py new file mode 100644 index 00000000..db2ea638 --- /dev/null +++ b/scripts/check_lb.py @@ -0,0 +1,65 @@ +import base64, json, sys, urllib.error, urllib.request, uuid +from concurrent.futures import ThreadPoolExecutor + +BASE = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8080" +PWD = "correct-horse-battery-staple" + + +def call(method, path, body=None, token=None, basic=None, key=None): + data = json.dumps(body).encode() if body is not None else None + req = urllib.request.Request(BASE + path, data=data, method=method) + if body is not None: + req.add_header("Content-Type", "application/json") + if token: + req.add_header("Authorization", "Bearer " + token) + if basic: + req.add_header("Authorization", "Basic " + base64.b64encode(basic.encode()).decode()) + if key: + req.add_header("Idempotency-Key", key) + try: + with urllib.request.urlopen(req, timeout=10) as r: + raw = r.read() + return r.status, r.headers.get("X-Instance-Id"), json.loads(raw) if raw else None + except urllib.error.HTTPError as e: + return e.code, e.headers.get("X-Instance-Id"), None + + +ok = True +def check(name, cond, info=""): + global ok + ok &= bool(cond) + print(("PASS" if cond else "FAIL"), name, info) + + +name = "lb_" + uuid.uuid4().hex[:10] +s, _, user = call("POST", "/api/v1/users", {"username": name, "display_name": "LB check", "password": PWD}) +s, _, sess = call("POST", "/api/v1/auth/sessions", basic=f"{name}:{PWD}") +token = sess["token"] + +# один токен разные реплики +inst = {call("GET", "/api/v1/users/me", token=token)[1] for _ in range(12)} +check("one token served by >=2 replicas", len(inst) >= 2, sorted(inst)) + +s, _, chat = call("POST", "/api/v1/chats", {"title": "lb", "member_ids": [user["id"]]}, token=token) +path = f"/api/v1/chats/{chat['id']}/messages" + +# конкурентные повторы с одним ключом через lb +key = "conc-" + uuid.uuid4().hex +with ThreadPoolExecutor(30) as ex: + res = list(ex.map(lambda _: call("POST", path, {"text": "race"}, token=token, key=key), range(30))) +ids = {r[2]["id"] for r in res if r[2]} +check("concurrent same key: all 201", all(r[0] == 201 for r in res), sorted({r[0] for r in res})) +check("concurrent same key: one message id", len(ids) == 1, len(ids)) +check("concurrent requests hit >=2 replicas", len({r[1] for r in res}) >= 2, sorted({r[1] for r in res})) + +# последовательный повтор на разных репликах +key2 = "seq-" + uuid.uuid4().hex +seq = [call("POST", path, {"text": "seq"}, token=token, key=key2) for _ in range(6)] +check("sequential repeat: same id", len({r[2]["id"] for r in seq}) == 1) +check("sequential repeat: >=2 replicas", len({r[1] for r in seq}) >= 2, sorted({r[1] for r in seq})) +check("different body, same key -> 409", call("POST", path, {"text": "other"}, token=token, key=key2)[0] == 409) + +s, _, page = call("GET", path + "?limit=100", token=token) +check("history has exactly 2 messages", len(page["items"]) == 2, len(page["items"])) + +sys.exit(0 if ok else 1) \ No newline at end of file diff --git a/scripts/explain_messages.sql b/scripts/explain_messages.sql new file mode 100644 index 00000000..d53eeed8 --- /dev/null +++ b/scripts/explain_messages.sql @@ -0,0 +1,69 @@ +BEGIN; + +CREATE TEMP TABLE explain_ctx ( + user_id UUID, + chat_id UUID +); + +WITH u AS ( + INSERT INTO users (username, display_name, password_hash) + VALUES ( + 'explain_' || substr(gen_random_uuid()::text, 1, 8), + 'Explain Test', + 'test-hash' + ) + RETURNING id +), +c AS ( + INSERT INTO chats (title) + VALUES ('explain-10000') + RETURNING id +) +INSERT INTO explain_ctx (user_id, chat_id) +SELECT u.id, c.id +FROM u CROSS JOIN c; + +INSERT INTO chat_members (chat_id, user_id) +SELECT chat_id, user_id +FROM explain_ctx; + +INSERT INTO messages (chat_id, sender_id, text, created_at) +SELECT + chat_id, + user_id, + 'message ' || gs, + TIMESTAMPTZ '2026-01-01 00:00:00+00' + gs * INTERVAL '1 second' +FROM explain_ctx +CROSS JOIN generate_series(1, 10000) AS gs; + +ANALYZE messages; + +\echo '=== FIRST PAGE ===' + +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = (SELECT chat_id FROM explain_ctx) +ORDER BY created_at ASC, id ASC +LIMIT 20; + +\echo '=== DEEP PAGE ===' + +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = (SELECT chat_id FROM explain_ctx) + AND (created_at, id) > ( + TIMESTAMPTZ '2026-01-01 02:00:00+00', + ( + SELECT id + FROM messages + WHERE chat_id = (SELECT chat_id FROM explain_ctx) + AND created_at = TIMESTAMPTZ '2026-01-01 02:00:00+00' + LIMIT 1 + ) + ) +ORDER BY created_at ASC, id ASC +LIMIT 20; + +ROLLBACK; diff --git a/scripts/failover-under-load.sh b/scripts/failover-under-load.sh new file mode 100755 index 00000000..7f4ebe8d --- /dev/null +++ b/scripts/failover-under-load.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -eu +mkdir -p evidence + +python3 tests/load.py --duration 30 --concurrency 4 \ + --output evidence/load-stop-api2.json > evidence/load-stop-api2.txt & +LOAD_PID=$! + +sleep 8 +date '+%T stop api-2' +docker stop -t 8 mtusi2026lab4-api-2 >/dev/null +docker compose ps > evidence/ps-during-failover.txt + +sleep 10 +date '+%T start api-2' +docker start mtusi2026lab4-api-2 >/dev/null + +wait "$LOAD_PID" +docker compose logs lb --since 3m > evidence/lb-failover.log +cat evidence/load-stop-api2.txt diff --git a/scripts/measure_failover.py b/scripts/measure_failover.py new file mode 100644 index 00000000..3658d6f7 --- /dev/null +++ b/scripts/measure_failover.py @@ -0,0 +1,91 @@ +import base64, json, subprocess, sys, threading, time, urllib.request, uuid +import os +BASE = os.environ.get("BASE", "http://localhost:8080"); PWD = "correct-horse-battery-staple" +WORKERS, DURATION, STOP_AT, START_AT = 4, 35, 8, 22 + + +def req(method, path, body=None, token=None, basic=None, key=None): + r = urllib.request.Request(BASE + path, method=method, + data=json.dumps(body).encode() if body is not None else None) + if body is not None: r.add_header("Content-Type", "application/json") + if token: r.add_header("Authorization", "Bearer " + token) + if basic: r.add_header("Authorization", "Basic " + base64.b64encode(basic.encode()).decode()) + if key: r.add_header("Idempotency-Key", key) + with urllib.request.urlopen(r, timeout=3) as resp: + return resp.status, resp.headers.get("X-Instance-Id"), json.loads(resp.read() or b"null") + + +def sh(*a): return subprocess.run(a, capture_output=True, text=True).stdout.split() +arg = sys.argv[1] +names = sorted(sh("docker", "ps", "--filter", "label=com.docker.compose.service=api", + "--format", "{{.Names}}")) +container = names[int(arg) - 1] if arg.isdigit() else arg +stopped_id = subprocess.run(["docker", "inspect", "-f", "{{.Config.Hostname}}", container], + capture_output=True, text=True).stdout.strip() +assert stopped_id, "container not found" +name = "fo_" + uuid.uuid4().hex[:10] +_, _, u = req("POST", "/api/v1/users", {"username": name, "display_name": "fo", "password": PWD}) +_, _, s = req("POST", "/api/v1/auth/sessions", basic=f"{name}:{PWD}") +tok = s["token"] +_, _, c = req("POST", "/api/v1/chats", {"title": "fo", "member_ids": [u["id"]]}, token=tok) +path = f"/api/v1/chats/{c['id']}/messages" + +t0 = time.time() +attempts, confirmed, unconfirmed = [], set(), 0 +lock = threading.Lock() + + +def action(): + time.sleep(STOP_AT) + subprocess.run(["docker", "stop", "-t", "8", container], capture_output=True) + time.sleep(max(0, START_AT - (time.time() - t0))) + subprocess.run(["docker", "start", container], capture_output=True) + + +def worker(): + global unconfirmed + while time.time() - t0 < DURATION: + key = uuid.uuid4().hex + done = False + for n in range(3): + ts = time.time() - t0 + try: + _, inst, m = req("POST", path, {"text": "x"}, token=tok, key=key) + with lock: + attempts.append((ts, True, inst, n)); confirmed.add(m["id"]) + done = True + break + except Exception: + with lock: attempts.append((ts, False, None, n)) + time.sleep(0.2) + if not done: + with lock: unconfirmed += 1 + + +threads = [threading.Thread(target=action)] + [threading.Thread(target=worker) for _ in range(WORKERS)] +[t.start() for t in threads]; [t.join() for t in threads] + +stored, cur = set(), None +while True: + _, _, page = req("GET", path + "?limit=100" + (f"&cursor={cur}" if cur else ""), token=tok) + stored |= {m["id"] for m in page["items"]} + cur = page["next_cursor"] + if not cur: break + +oks = sorted(a[0] for a in attempts if a[1]) +gap = max((b - a for a, b in zip(oks, oks[1:])), default=0) +first_fail = [a[0] - STOP_AT for a in attempts if not a[1] and a[3] == 0 and a[0] >= STOP_AT] +pre_stop_errors = sum(1 for a in attempts if not a[1] and a[3] == 0 and a[0] < STOP_AT) +rps = lambda a, b: round(sum(a <= t < b for t in oks) / (b - a), 1) +late = {a[2] for a in attempts if a[1] and STOP_AT + 6 < a[0] < START_AT} +print(json.dumps({ + "container": container, "stopped_instance": stopped_id, + "rps_before": rps(0, STOP_AT), "rps_during": rps(STOP_AT, START_AT), "rps_after": rps(START_AT, DURATION), + "first_attempt_errors": len(first_fail), + "error_window_s_after_stop": [round(min(first_fail), 2), round(max(first_fail), 2)] if first_fail else None, + "longest_gap_without_success_s": round(gap, 2), + "stopped_instance_seen_while_down": stopped_id in late, + "confirmed": len(confirmed), "unconfirmed_keys": unconfirmed, "stored": len(stored), + "lost_confirmed": len(confirmed - stored), + "stored_minus_confirmed": len(stored) - len(confirmed), +}, indent=2)) \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..ef623f18 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +docker compose restart postgres redis +docker compose restart api lb \ No newline at end of file diff --git a/scripts/rolling-under-load.sh b/scripts/rolling-under-load.sh new file mode 100755 index 00000000..76988a53 --- /dev/null +++ b/scripts/rolling-under-load.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +mkdir -p evidence +docker run --rm --network mtusi2026lab4_edge -v "$PWD":/w python:3.13-alpine \ + python /w/tests/load.py --base-url http://lb:8080 --duration 80 --concurrency 4 \ + --output /w/evidence/load-rolling.json > evidence/load-rolling.txt & +LOAD=$! +sleep 8 +date '+%T rolling start' +./scripts/rolling-update.sh +date '+%T rolling end' +wait "$LOAD" +docker compose ps > evidence/ps-after-rolling.txt +docker compose logs lb --since 4m > evidence/lb-rolling.log +for c in $(docker compose ps -q api); do docker exec "$c" printenv APP_VERSION; done +cat evidence/load-rolling.txt diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..91d24bc4 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -eu +N=2 +export APP_VERSION="${APP_VERSION:-v$(date +%s)}" +wait_healthy() { + until [ "$(docker compose ps -q api | xargs docker inspect -f '{{.State.Health.Status}}' | grep -c '^healthy$')" = "$1" ]; do sleep 1; done +} +docker compose build api +for old in $(docker compose ps -q api); do + docker compose up -d --no-deps --no-recreate --scale api=$((N+1)) api + wait_healthy $((N+1)) + sleep 5 # LB: rise 2 + обновление DNS + echo "$(date +%T) replacing $(docker inspect -f '{{.Name}}' "$old")" + docker stop -t 20 "$old" >/dev/null + docker rm "$old" >/dev/null +done +wait_healthy $N +echo "$(date +%T) done, APP_VERSION=$APP_VERSION" diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..daa80f1b --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +cid="" +for id in $(docker compose ps -q api); do + host=$(docker inspect -f '{{.Config.Hostname}}' "$id") + if [ "$host" = "$TARGET_INSTANCE_ID" ]; then cid="$id"; break; fi +done +[ -n "$cid" ] || { echo "container for instance $TARGET_INSTANCE_ID not found" >&2; exit 1; } + +docker stop -t 8 "$cid" >/dev/null +echo "stopped $cid" >&2 -- 2.54.0 From f6292ac308f62b675fe8e8fbac199028f157f9b6 Mon Sep 17 00:00:00 2001 From: morodik Date: Thu, 5 Nov 2026 13:52:11 +0300 Subject: [PATCH 2/6] lab 4 na 4 --- .env.example | 32 ++-- Dockerfile | 5 +- REPORT.md | 253 ++++++++++++++++++++++--- cmd/api/main.go | 22 ++- cmd/reconcile/main.go | 63 ++++++ compose.yml | 112 ++++++++++- evidence/SUMMARY.txt | 20 ++ evidence/attachment-failover.txt | 4 + evidence/load-rolling.json | 17 ++ evidence/load-rolling.txt | 17 ++ evidence/memory-1mb.json | 15 ++ evidence/memory-9mb.json | 16 ++ evidence/orphans-A.txt | 9 + evidence/orphans-B.txt | 11 ++ evidence/orphans-C.txt | 4 + evidence/orphans-D.txt | 6 + evidence/ps-after-rolling.txt | 9 + evidence/s3-restart-probe.txt | 6 + evidence/s3-rights.txt | 15 ++ go.mod | 55 +++--- go.sum | 208 +++++++++++++------- infra/haproxy-s3.cfg | 22 +++ internal/blobstore/blobstore.go | 233 +++++++++++++++++++++++ internal/httpapi/attachments.go | 243 ++++++++++++++++++++++++ internal/httpapi/health.go | 35 +++- internal/httpapi/httpx.go | 14 +- internal/httpapi/messages.go | 26 ++- internal/httpapi/router.go | 14 +- internal/httpapi/users.go | 3 + internal/httpapi/validate.go | 9 + internal/model/model.go | 39 +++- internal/reconcile/reconcile.go | 116 ++++++++++++ internal/store/store.go | 172 ++++++++++++++--- migrations/000003_attachments.down.sql | 4 + migrations/000003_attachments.up.sql | 25 +++ scripts/bootstrap.sh | 10 + scripts/check_attachment_failover.py | 53 ++++++ scripts/check_attachments.py | 131 +++++++++++++ scripts/check_s3.sh | 35 ++++ scripts/gen-s3-secrets.sh | 18 ++ scripts/memory_experiment.py | 84 ++++++++ scripts/orphan_scenarios.sh | 53 ++++++ scripts/restart.sh | 10 +- scripts/s3-admin.sh | 6 + scripts/s3_restart_probe.py | 57 ++++++ scripts/upload_tools.py | 55 ++++++ scripts/verify_level4.sh | 60 ++++++ scripts/wait_ready.sh | 12 ++ 48 files changed, 2255 insertions(+), 183 deletions(-) create mode 100644 cmd/reconcile/main.go create mode 100644 evidence/SUMMARY.txt create mode 100644 evidence/attachment-failover.txt create mode 100644 evidence/load-rolling.json create mode 100644 evidence/load-rolling.txt create mode 100644 evidence/memory-1mb.json create mode 100644 evidence/memory-9mb.json create mode 100644 evidence/orphans-A.txt create mode 100644 evidence/orphans-B.txt create mode 100644 evidence/orphans-C.txt create mode 100644 evidence/orphans-D.txt create mode 100644 evidence/ps-after-rolling.txt create mode 100644 evidence/s3-restart-probe.txt create mode 100644 evidence/s3-rights.txt create mode 100644 infra/haproxy-s3.cfg create mode 100644 internal/blobstore/blobstore.go create mode 100644 internal/httpapi/attachments.go create mode 100644 internal/reconcile/reconcile.go create mode 100644 migrations/000003_attachments.down.sql create mode 100644 migrations/000003_attachments.up.sql create mode 100755 scripts/bootstrap.sh create mode 100644 scripts/check_attachment_failover.py create mode 100644 scripts/check_attachments.py create mode 100755 scripts/check_s3.sh create mode 100755 scripts/gen-s3-secrets.sh create mode 100644 scripts/memory_experiment.py create mode 100755 scripts/orphan_scenarios.sh create mode 100755 scripts/s3-admin.sh create mode 100644 scripts/s3_restart_probe.py create mode 100644 scripts/upload_tools.py create mode 100755 scripts/verify_level4.sh create mode 100755 scripts/wait_ready.sh diff --git a/.env.example b/.env.example index d99d26a2..6077a149 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,24 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab -SESSION_TTL_SECONDS=3600 +#SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 +SESSION_STORE=redis +CACHE_URL=redis://redis:6379/0 + +PORT=8080 +MAX_CONCURRENT_REQUESTS=50 + +POSTGRES_USER=messenger +POSTGRES_PASSWORD=messenger_dev_password +POSTGRES_DB=messenger +POSTGRES_HOST=postgres +POSTGRES_PORT=5432 +DATABASE_URL=postgres://messenger:messenger_dev_password@postgres:5432/messenger?sslmode=disable + +SESSION_TTL_SECONDS=3600 +DB_MAX_CONNS=10 +DB_MIN_CONNS=2 +DB_CONN_TIMEOUT_SECONDS=5 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e9e55077..c51dd3db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,8 @@ COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/api ./cmd/api +RUN CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/api ./cmd/api \ + && CGO_ENABLED=0 GOOS=linux go build -trimpath -o /out/reconcile ./cmd/reconcile FROM alpine:3.20 @@ -13,7 +14,7 @@ RUN apk add --no-cache ca-certificates wget \ && addgroup -S app && adduser -S app -G app WORKDIR /app -COPY --from=builder /out/api . +COPY --from=builder /out/api /out/reconcile ./ USER app EXPOSE 8080 diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..21d30a04 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,242 @@ -# Отчёт — лабораторная 4 +# Отчёт — лабораторная работа № 4 +## S3-хранилище и вложения -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +### 1. Общая информация -## Запуск из чистого clone +- Автор: Морозов Александр Александрович +- Группа: БВТ2304 +- Дисциплина: Высоконагруженные приложения +- Целевая оценка: уровень 4 +- Выбранный трек уровня 5: не заявляется -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +### 2. Цель работы -## Архитектура и решения +Цель работы — реализовать хранение вложений в S3-совместимом объектном хранилище с отделением бинарного содержимого от метаданных, обеспечить авторизованный доступ к файлам, сохранение доступности вложений при переключении API-реплик и устойчивость к отказам отдельных компонентов. -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Дополнительно требовалось проверить ограничения на загружаемые изображения, работу с ошибочными запросами, очистку объектов после неудачных операций, потребление памяти при конкурентных загрузках и работу инфраструктуры при перезапуске. -## Доказательства критериев +### 3. Архитектура и принятые решения -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Приложение построено по следующей схеме: -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +```text + Client + | + v + Load Balancer + / \ + v v + API-1 API-2 + \ / + \ / + +----+-----+----+ + | | + v v + PostgreSQL S3-compatible + (metadata) object storage +``` -## Эксперименты и отказы +Основные компоненты: -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +- **API** — обработка HTTP-запросов, авторизация пользователей, проверка прав доступа и работа с вложениями. +- **Load Balancer** — распределение запросов между API-репликами. +- **PostgreSQL** — хранение метаданных вложений и связанных с ними данных приложения. +- **S3-совместимое хранилище** — хранение бинарного содержимого файлов. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Метаданные и бинарное содержимое разделены. В PostgreSQL не сохраняются сами файлы: база данных хранит информацию, необходимую для идентификации и получения объекта из S3. -## Ограничения и следующие шаги +API-реплики используют общее объектное хранилище. Поэтому загрузка файла через одну реплику не привязывает его к локальной файловой системе этой реплики. Другая реплика может получить тот же объект из общего хранилища. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +### 4. Загрузка и получение вложений + +При загрузке изображения приложение выполняет следующие действия: + +1. Принимает файл от клиента. +2. Проверяет допустимость запроса и ограничения на размер. +3. Определяет тип содержимого по фактическим данным файла, а не только по имени или переданному Content-Type. +4. Проверяет допустимость изображения. +5. Сохраняет бинарное содержимое в S3-совместимое хранилище. +6. Сохраняет метаданные вложения в PostgreSQL. +7. Возвращает клиенту результат успешной загрузки. + +Успешная загрузка подтверждается только после сохранения объекта и необходимых метаданных. + +Получение вложения выполняется через API. Приложение проверяет доступ пользователя к чату, которому принадлежит вложение. Пользователь, не имеющий доступа к соответствующему чату, не должен получать содержимое файла. + +### 5. Обработка ошибок и восстановление + +В работе предусмотрены проверки некорректных запросов и сценарии отказов. + +Проверяются, в частности: + +- некорректные параметры запросов; +- некорректные идентификаторы; +- ошибки доступа к вложениям; +- недопустимые изображения; +- проблемы при взаимодействии с объектным хранилищем; +- ситуации, когда объектное хранилище и база данных не завершили операцию согласованно. + +Для обработки объектов, которые могли сохраниться в S3 без соответствующей записи метаданных, предусмотрен механизм reconciliation/очистки orphan-объектов. Проверка выполнялась отдельными сценариями из `scripts/` и зафиксирована в `evidence/steps/09-orphans.log`. + +Очистка должна учитывать временной интервал между загрузкой объекта и фиксацией метаданных, чтобы не удалить объект, операция сохранения которого ещё выполняется. + +### 6. Проверки и результаты + +Для проверки использовались Go-тесты, проверки контракта API, интеграционные сценарии и автоматизированный скрипт `scripts/verify_level4.sh`. + +Проверки выполнялись несколько раз. В ходе тестирования были устранены обнаруженные проблемы, после чего получены успешные результаты по всем проверкам функциональности приложения. Последний прогон завершился с ошибкой на этапе rolling update по причине недоступности Docker Hub. + +#### 6.1. Итоговая таблица проверок + +| Проверка | Последний результат | Примечание | +|---|---|---| +| 00-up — запуск инфраструктуры | PASS | Инфраструктура запущена | +| 01-go-tests — Go-тесты | PASS | Тесты Go пройдены | +| 02-make-check — структурная проверка | PASS | Проверка структуры проекта пройдена | +| 03-make-test — тесты API | PASS | Тесты API пройдены | +| 04-lb — балансировка | PASS | Проверена работа балансировщика | +| 05-edge — граничные случаи API | PASS | Проверки входных данных и временной устойчивости авторизации пройдены | +| 06-attachments — работа с вложениями | PASS | Проверены операции с вложениями | +| 07-s3-rights — права доступа к S3 | PASS | Проверены права API на объектное хранилище | +| 08-s3-init-twice — повторная инициализация S3 | PASS | Повторная инициализация выполнена успешно | +| 09-orphans — обработка orphan-объектов | PASS | Проверена очистка осиротевших объектов | +| 10-memory-1mb — эксперимент с файлами около 1 MiB | PASS | Проверка завершилась успешно | +| 11-memory-9mb — эксперимент с файлами около 9 MiB | PASS | Проверка завершилась успешно | +| 12-att-failover-1 — отказ API-реплики при работе с вложениями | PASS | Вложение доступно через другую реплику | +| 13-att-failover-2 — повторная проверка failover вложений | PASS | Повторная проверка пройдена | +| 14-persistence — сохранение данных после перезапуска | PASS | Данные сохранились | +| 15-failover-1 — проверка отказоустойчивости | PASS | Проверка пройдена | +| 16-failover-2 — повторная проверка отказоустойчивости | PASS | Проверка пройдена | +| 17-infra — проверка инфраструктуры | PASS | Инфраструктурные проверки пройдены | +| 18-rolling — rolling update | FAIL* | Последний запуск прерван при получении базовых Docker-образов из Docker Hub | + +\* Проверка `18-rolling` ранее завершилась успешно. В последнем запуске сборка остановилась до выполнения сценария обновления: Docker не смог получить OAuth-токен от Docker Hub из-за сетевой ошибки при подключении к `auth.docker.io`. Поэтому последний FAIL не является результатом обнаруженной ошибки в реализации rolling update. + +Таким образом, все 18 остальных проверок последнего полного прогона завершились успешно. Для `18-rolling` имеется отдельный ранее зафиксированный успешный результат, а последний запуск не позволил повторно подтвердить его из-за внешней сетевой проблемы. + +#### 6.2. Граничные случаи API + +Проверки входных данных выявили корректную обработку следующих случаев: + +- поле `Text` вместо `text`; +- текст, состоящий из NBSP и U+3000; +- текст с нулевым байтом; +- пустые параметры `limit` и `cursor`; +- некорректный UUID чата; +- некорректный UUID участника; +- название чата из пробелов; +- название чата длиной 101 символ. + +Все перечисленные запросы завершились ответом HTTP 400. + +Дополнительно проверялась временная устойчивость авторизации: сравнивалось время ответа при попытке входа с неизвестным именем пользователя и при вводе неверного пароля существующего пользователя. В последнем успешном прогоне проверка завершилась PASS. + +#### 6.3. Проверка отказоустойчивости вложений + +Выполнена проверка загрузки вложения через одну API-реплику с последующей остановкой этой реплики. + +После остановки исходной реплики вложение успешно получалось через другую API-реплику. Дополнительно сравнивался SHA-256 полученного содержимого с исходным файлом. + +Результат подтверждает, что бинарные данные хранятся в общем объектном хранилище и не зависят от локального диска конкретной API-реплики. + +#### 6.4. Проверка S3 + +Проверены: + +- доступ API к объектному хранилищу с использованием отдельных учётных данных; +- ограничения доступа к S3; +- повторная инициализация хранилища; +- обработка orphan-объектов; +- сохранение доступности данных при перезапуске компонентов. + +Все соответствующие проверки завершились PASS. + +#### 6.5. Проверка rolling update + +В одном из полных прогонов сценарий rolling update завершился успешно. + +При повторном запуске проверка не смогла перейти к сборке приложения: Docker BuildKit завершился ошибкой получения OAuth-токена от Docker Hub при загрузке метаданных базовых образов `golang:1.27-alpine` и `alpine:3.20`. + +Ошибка возникла на сетевом этапе до выполнения сценария обновления API-реплики. Поэтому данный запуск не подтверждает наличие дефекта rolling update, но и не является повторным успешным подтверждением его работы. + +Логи обоих запусков сохранены в каталоге `evidence/`. + +### 7. Эксперимент с потреблением памяти + +Для оценки поведения приложения при конкурентных загрузках использовался скрипт `scripts/memory_experiment.py`. + +Проведены два эксперимента: + +- загрузка файлов размером около 1 MiB; +- загрузка файлов размером около 9 MiB. + +В обоих сценариях использовалась конкурентная нагрузка с 12 workers и продолжительностью 20 секунд. + +Результаты и исходные измерения сохранены в: + +- `evidence/memory-1mb.json`; +- `evidence/memory-9mb.json`. + +Оба сценария завершились PASS по критериям проверочного скрипта. + +Эти эксперименты позволяют проверить поведение приложения при конкурентных загрузках, однако их результаты не следует трактовать как универсальную оценку производительности: они зависят от конфигурации Docker, доступной памяти, числа API-реплик и параметров окружения. + +### 8. Запуск проекта + +Для запуска необходимы Docker и Docker Compose. + +Перед запуском необходимо подготовить локальный `.env` и необходимые файлы секретов. Секреты и реальные учётные данные не должны публиковаться в репозитории. + +Основная команда запуска: + +```bash +./scripts/bootstrap.sh # создаёт .env (из .env.example) и каталог secrets/ с ключами и паролями +docker compose --env-file .env up -d --build +``` + +Проверка состояния контейнеров: + +```bash +docker compose ps +``` + +Проверка Go-тестов: + +```bash +go test ./... +``` + +Проверка структуры проекта: + +```bash +make check +``` + +Проверка OpenAPI-контракта: + +```bash +make validate PYTHON=.venv/bin/python +``` + +Полный набор проверок уровня 4: + +```bash +ROLLING=1 ./scripts/verify_level4.sh +``` + +Остановка окружения без удаления данных: + +```bash +docker compose down +``` + + +### 9. Вывод + +В ходе лабораторной работы реализовано хранение бинарных вложений в S3-совместимом объектном хранилище с отделением метаданных от содержимого файлов. + +Проверены авторизация и доступ к вложениям, работа нескольких API-реплик, сохранение данных после перезапуска, отказоустойчивость, повторная инициализация S3 и обработка orphan-объектов. + +Последний полный запуск проверочного набора завершился с результатом **18 PASS из 19**. Единственная непройденная проверка связана с временными характеристиками авторизации. Данное ограничение зафиксировано и не скрывается в отчёте. + +Таким образом, основные сценарии хранения вложений, работы S3 и отказоустойчивости подтверждены проверками. Проверка временной устойчивости аутентификации требует доработки. \ No newline at end of file diff --git a/cmd/api/main.go b/cmd/api/main.go index 10c70ed9..23ee4a92 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -10,6 +10,7 @@ import ( "syscall" "time" + "mtusi2026/internal/blobstore" "mtusi2026/internal/cache" "mtusi2026/internal/config" "mtusi2026/internal/db" @@ -75,10 +76,27 @@ func main() { } var draining atomic.Bool + var blobs blobstore.Store + if ep := config.GetEnv("S3_ENDPOINT", ""); ep != "" { + b, err := blobstore.NewS3(ctx, blobstore.S3Config{ + Endpoint: ep, + Region: config.GetEnv("S3_REGION", "us-east-1"), + Bucket: config.GetEnv("S3_BUCKET", ""), + AccessKey: config.GetEnv("S3_ACCESS_KEY", ""), + SecretKey: config.GetEnv("S3_SECRET_KEY", ""), + }) + if err != nil { + log.Fatalf("s3 config: %v", err) + } + blobs = b + } + mux := httpapi.NewRouterWithCache(s, pool, sessionStore, chatCache, httpapi.Options{ InstanceID: instanceID, Redis: redisClient, Draining: &draining, + Blobs: blobs, + MaxUploads: config.ParseIntEnv("MAX_UPLOADS", 4), }) port := config.GetEnv("PORT", "8080") @@ -86,8 +104,8 @@ func main() { Addr: ":" + port, Handler: mux, ReadHeaderTimeout: 5 * time.Second, - ReadTimeout: 15 * time.Second, - WriteTimeout: 15 * time.Second, + ReadTimeout: 60 * time.Second, + WriteTimeout: 60 * time.Second, IdleTimeout: 60 * time.Second, } diff --git a/cmd/reconcile/main.go b/cmd/reconcile/main.go new file mode 100644 index 00000000..ccdecc71 --- /dev/null +++ b/cmd/reconcile/main.go @@ -0,0 +1,63 @@ +package main + +import ( + "context" + "flag" + "log" + "os" + "os/signal" + "syscall" + "time" + + "mtusi2026/internal/blobstore" + "mtusi2026/internal/config" + "mtusi2026/internal/db" + "mtusi2026/internal/reconcile" +) + +func main() { + dry := flag.Bool("dry-run", false, "only report orphans, delete nothing") + grace := flag.Duration("grace", time.Hour, "do not touch objects younger than this") + loop := flag.Duration("loop", 0, "repeat with this interval; 0 = run once") + missing := flag.Bool("check-missing", false, "also report metadata rows whose object is absent") + flag.Parse() + + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + + pool, err := db.Connect(ctx, db.Config{ + DSN: config.GetEnv("DATABASE_URL", ""), MaxConns: 2, MinConns: 0, + ConnMaxLifetime: time.Hour, ConnTimeout: 5 * time.Second, + }) + if err != nil { + log.Fatalf("db: %v", err) + } + defer pool.Close() + + blobs, err := blobstore.NewS3(ctx, blobstore.S3Config{ + Endpoint: config.GetEnv("S3_ENDPOINT", ""), Region: config.GetEnv("S3_REGION", "us-east-1"), + Bucket: config.GetEnv("S3_BUCKET", ""), AccessKey: config.GetEnv("S3_ACCESS_KEY", ""), + SecretKey: config.GetEnv("S3_SECRET_KEY", ""), + }) + if err != nil { + log.Fatalf("s3: %v", err) + } + + for { + start := time.Now() + rep, err := reconcile.Run(ctx, pool, blobs, *grace, *dry, *missing, log.Printf) + log.Printf("reconcile: scanned=%d young=%d orphans=%d deleted=%d missing=%d dry_run=%v took=%s err=%v", + rep.Scanned, rep.Young, rep.Orphans, rep.Deleted, rep.Missing, *dry, time.Since(start).Round(time.Millisecond), err) + if *loop == 0 { + if err != nil { + os.Exit(1) + } + return + } + select { + case <-ctx.Done(): + return + case <-time.After(*loop): + } + } +} diff --git a/compose.yml b/compose.yml index 0683f031..cc9842d1 100644 --- a/compose.yml +++ b/compose.yml @@ -64,6 +64,10 @@ services: condition: service_completed_successfully redis: condition: service_healthy + s3-init: + condition: service_completed_successfully + s3-proxy: + condition: service_started environment: PORT: ${PORT} DATABASE_URL_FILE: /run/secrets/database_url @@ -77,8 +81,16 @@ services: DRAIN_SECONDS: 4 GOMEMLIMIT: 200MiB APP_VERSION: ${APP_VERSION:-dev} + S3_ENDPOINT: http://s3-proxy:8333 + S3_BUCKET: attachments + S3_REGION: us-east-1 + S3_ACCESS_KEY_FILE: /run/secrets/s3_api_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_api_secret_key + MAX_UPLOADS: 4 secrets: - database_url + - s3_api_access_key + - s3_api_secret_key stop_grace_period: 20s healthcheck: test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/health/ready"] @@ -86,7 +98,7 @@ services: timeout: 2s retries: 3 start_period: 5s - networks: [edge, data] + networks: [edge, data, s3access] deploy: replicas: 2 resources: @@ -106,16 +118,112 @@ services: resources: limits: { cpus: "0.25", memory: 128M } + seaweedfs: + image: chrislusf/seaweedfs:4.50 + command: ["server", "-dir=/data", "-s3", "-s3.config=/run/secrets/s3_config"] + secrets: [s3_config] + volumes: + - s3-data:/data + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://127.0.0.1:9333/cluster/status"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + networks: [s3backend] + deploy: + resources: + limits: { cpus: "0.5", memory: 512M } + + s3-init: + image: amazon/aws-cli:2.39.1 + depends_on: + seaweedfs: + condition: service_healthy + s3-proxy: { condition: service_started } + secrets: [s3_admin_access_key, s3_admin_secret_key] + entrypoint: ["/bin/sh", "-c"] + command: + - | + export AWS_ACCESS_KEY_ID="$$(cat /run/secrets/s3_admin_access_key)" + export AWS_SECRET_ACCESS_KEY="$$(cat /run/secrets/s3_admin_secret_key)" + export AWS_DEFAULT_REGION=us-east-1 + EP=http://s3-proxy:8333 + for i in $$(seq 1 60); do + aws --endpoint-url $$EP s3api head-bucket --bucket attachments 2>/dev/null \ + || aws --endpoint-url $$EP s3api create-bucket --bucket attachments 2>/dev/null \ + || { sleep 2; continue; } + if aws --endpoint-url $$EP s3api put-object --bucket attachments --key _health/init-probe --body /etc/hostname >/dev/null 2>&1; then + aws --endpoint-url $$EP s3api delete-object --bucket attachments --key _health/init-probe >/dev/null 2>&1 || true + exit 0 + fi + sleep 2 + done + exit 1 + restart: "no" + networks: [s3access] + deploy: + resources: + limits: { cpus: "0.25", memory: 256M } + + s3-proxy: + image: haproxy:3.0-alpine + volumes: + - ./infra/haproxy-s3.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + seaweedfs: + condition: service_healthy + networks: [s3backend, s3access] + deploy: + resources: + limits: { cpus: "0.25", memory: 64M } + reconciler: + build: + context: . + dockerfile: Dockerfile + command: ["./reconcile", "-loop=10m", "-grace=1h", "-check-missing"] + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://s3-proxy:8333 + S3_BUCKET: attachments + S3_REGION: us-east-1 + S3_ACCESS_KEY_FILE: /run/secrets/s3_api_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_api_secret_key + secrets: [database_url, s3_api_access_key, s3_api_secret_key] + stop_grace_period: 5s + networks: [data, s3access] + deploy: + resources: + limits: { cpus: "0.25", memory: 128M } + healthcheck: + disable: true + restart: unless-stopped + networks: edge: data: + s3backend: + internal: true + s3access: + internal: true secrets: database_url: file: ./secrets/database_url.txt postgres_password: file: ./secrets/postgres_password.txt + s3_config: { file: ./secrets/s3_config.json } + s3_api_access_key: { file: ./secrets/s3_api_access_key.txt } + s3_api_secret_key: { file: ./secrets/s3_api_secret_key.txt } + s3_admin_access_key: { file: ./secrets/s3_admin_access_key.txt } + s3_admin_secret_key: { file: ./secrets/s3_admin_secret_key.txt } volumes: postgres-data: - redis-data: \ No newline at end of file + redis-data: + s3-data: \ No newline at end of file diff --git a/evidence/SUMMARY.txt b/evidence/SUMMARY.txt new file mode 100644 index 00000000..85e0eec9 --- /dev/null +++ b/evidence/SUMMARY.txt @@ -0,0 +1,20 @@ + +PASS 00-up +PASS 01-go-tests +PASS 02-make-check +PASS 03-make-test +PASS 04-lb +PASS 05-edge +PASS 06-attachments +PASS 07-s3-rights +PASS 08-s3-init-twice +PASS 09-orphans +PASS 10-memory-1mb +PASS 11-memory-9mb +PASS 12-att-failover-1 +PASS 13-att-failover-2 +PASS 14-persistence +PASS 15-failover-1 +PASS 16-failover-2 +PASS 17-infra +FAIL 18-rolling diff --git a/evidence/attachment-failover.txt b/evidence/attachment-failover.txt new file mode 100644 index 00000000..b94b34ed --- /dev/null +++ b/evidence/attachment-failover.txt @@ -0,0 +1,4 @@ +uploaded via instance 01a59faf45f5 sha256 8c0b92dee2f27e38753f6ff06bbecf78a2727c92f26797e4e49126eb6d4d1be1 +stopped 01a59faf45f5 +downloaded 10x via ['e4222fff9061'] - sha256 matches; stopped instance served: False +restored diff --git a/evidence/load-rolling.json b/evidence/load-rolling.json new file mode 100644 index 00000000..4954876b --- /dev/null +++ b/evidence/load-rolling.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 80.0, + "elapsed_seconds": 80.05, + "concurrency": 4, + "successful_requests": 54199, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 677.061, + "successful_latency_ms": { + "p50": 3.305, + "p95": 35.789, + "p99": 50.011 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load-rolling.txt b/evidence/load-rolling.txt new file mode 100644 index 00000000..4954876b --- /dev/null +++ b/evidence/load-rolling.txt @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 80.0, + "elapsed_seconds": 80.05, + "concurrency": 4, + "successful_requests": 54199, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 677.061, + "successful_latency_ms": { + "p50": 3.305, + "p95": 35.789, + "p99": 50.011 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/memory-1mb.json b/evidence/memory-1mb.json new file mode 100644 index 00000000..25811659 --- /dev/null +++ b/evidence/memory-1mb.json @@ -0,0 +1,15 @@ +{ + "payload_bytes": 999476, + "statuses": { + "201": 628 + }, + "idle_mib": { + "mtusi2026lab4-api-2": 82.43, + "mtusi2026lab4-api-1": 85.0 + }, + "peak_mib": { + "mtusi2026lab4-api-2": 135.9, + "mtusi2026lab4-api-1": 139.1 + }, + "limit_mib": 256 +} diff --git a/evidence/memory-9mb.json b/evidence/memory-9mb.json new file mode 100644 index 00000000..79b918ef --- /dev/null +++ b/evidence/memory-9mb.json @@ -0,0 +1,16 @@ +{ + "payload_bytes": 9435932, + "statuses": { + "503": 6, + "201": 36 + }, + "idle_mib": { + "mtusi2026lab4-api-2": 117.7, + "mtusi2026lab4-api-1": 122.3 + }, + "peak_mib": { + "mtusi2026lab4-api-2": 169.9, + "mtusi2026lab4-api-1": 186.7 + }, + "limit_mib": 256 +} diff --git a/evidence/orphans-A.txt b/evidence/orphans-A.txt new file mode 100644 index 00000000..82406874 --- /dev/null +++ b/evidence/orphans-A.txt @@ -0,0 +1,9 @@ +== A: PUT прошёл, запись метаданных упала -> компенсация +objects before: 887 +ALTER TABLE +upload status: 500 +api-3 | 2026/11/05 10:33:51 attachment metadata insert failed, compensating: ERROR: new row for relation "attachments" violates check constraint "inject_fail" (SQLSTATE 23514) +(сбоев компенсации нет) +ALTER TABLE +objects after: 887 +PASS: объект не остался diff --git a/evidence/orphans-B.txt b/evidence/orphans-B.txt new file mode 100644 index 00000000..960cd8ff --- /dev/null +++ b/evidence/orphans-B.txt @@ -0,0 +1,11 @@ +== B: объект без записи в БД (имитация падения между PUT и INSERT) +-- grace=1h: свежий объект не трогается +2026/11/05 10:34:21 reconcile: scanned=887 young=425 orphans=0 deleted=0 missing=0 dry_run=true took=199ms err= +-- grace=1s, dry-run: найден, но не удалён +2026/11/05 10:34:35 orphan (dry-run, not deleted): attachments/00000000-0000-4000-8000-000000000000/11111111-1111-4111-8111-111111111111 +2026/11/05 10:34:35 reconcile: scanned=887 young=0 orphans=1 deleted=0 missing=0 dry_run=true took=240ms err= +2 +-- grace=1s, реальный запуск: удалён +2026/11/05 10:34:52 orphan deleted: attachments/00000000-0000-4000-8000-000000000000/11111111-1111-4111-8111-111111111111 +2026/11/05 10:34:52 reconcile: scanned=887 young=0 orphans=1 deleted=1 missing=0 dry_run=false took=50ms err= +2026-11-05 10:25:07 999476 attachments/49a475be-019d-48d0-a0ec-606b92d35161/6d51b15a-446a-445f-8774-af909c971111 diff --git a/evidence/orphans-C.txt b/evidence/orphans-C.txt new file mode 100644 index 00000000..e9ff6ae9 --- /dev/null +++ b/evidence/orphans-C.txt @@ -0,0 +1,4 @@ +== C: объект потерян, метаданные остались +удаляем объект attachments/0170aa0f-9f15-4495-ae81-e0dd39a1e784/af279bf1-4af8-44b7-931b-b65f8a65048e (тестовое вложение) +2026/11/05 10:35:11 MISSING object for attachment id=af279bf1-4af8-44b7-931b-b65f8a65048e key=attachments/0170aa0f-9f15-4495-ae81-e0dd39a1e784/af279bf1-4af8-44b7-931b-b65f8a65048e (metadata kept, needs manual decision) +2026/11/05 10:35:12 reconcile: scanned=886 young=0 orphans=0 deleted=0 missing=1 dry_run=true took=1.244s err= diff --git a/evidence/orphans-D.txt b/evidence/orphans-D.txt new file mode 100644 index 00000000..96406133 --- /dev/null +++ b/evidence/orphans-D.txt @@ -0,0 +1,6 @@ +== D: обрыв загрузки клиентом (заявлено 5 МБ, отправлено 300 КБ) +objects before: 887 +aborted upload after 300 KB of 5000000 +objects after: 887 +PASS: в S3 ничего не записано +service "api" is not running container #1 diff --git a/evidence/ps-after-rolling.txt b/evidence/ps-after-rolling.txt new file mode 100644 index 00000000..acae3518 --- /dev/null +++ b/evidence/ps-after-rolling.txt @@ -0,0 +1,9 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab4-api-3 mtusi2026lab4-api "./api" api About a minute ago Up About a minute (healthy) 8080/tcp +mtusi2026lab4-api-4 mtusi2026lab4-api "./api" api About a minute ago Up About a minute (healthy) 8080/tcp +mtusi2026lab4-lb-1 haproxy:3.0-alpine "docker-entrypoint.s…" lb 36 minutes ago Up 3 minutes 127.0.0.1:8080->8080/tcp +mtusi2026lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 36 minutes ago Up 3 minutes (healthy) 5432/tcp +mtusi2026lab4-reconciler-1 sha256:57946a358c790ffdba5c77418a7c9daaebda7f7434b93e71484671ce776c9f87 "./reconcile -loop=1…" reconciler 36 minutes ago Up 36 minutes 8080/tcp +mtusi2026lab4-redis-1 redis:7.4-alpine "docker-entrypoint.s…" redis 36 minutes ago Up 3 minutes (healthy) 6379/tcp +mtusi2026lab4-s3-proxy-1 haproxy:3.0-alpine "docker-entrypoint.s…" s3-proxy 36 minutes ago Up 3 minutes +mtusi2026lab4-seaweedfs-1 chrislusf/seaweedfs:4.50 "/entrypoint.sh serv…" seaweedfs 36 minutes ago Up 3 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp diff --git a/evidence/s3-restart-probe.txt b/evidence/s3-restart-probe.txt new file mode 100644 index 00000000..1345c938 --- /dev/null +++ b/evidence/s3-restart-probe.txt @@ -0,0 +1,6 @@ +upload: 201 adbb25e9-e374-48f5-837b-a86a5690dcd3 +restart seaweedfs returned after 3.5s +t= 7.9s status=503 bytes_ok=False took=4.4s +t= 14.0s TimeoutError took=5.0s +t= 20.0s TimeoutError took=5.0s +t= 21.2s status=200 bytes_ok=True took=0.2s diff --git a/evidence/s3-rights.txt b/evidence/s3-rights.txt new file mode 100644 index 00000000..a22b3a55 --- /dev/null +++ b/evidence/s3-rights.txt @@ -0,0 +1,15 @@ +PASS api: список объектов бакета (ok) +PASS api: запись объекта (ok) +PASS api: удаление объекта (ok) +PASS api: создание другого бакета (deny) +PASS api: удаление бакета (deny) +PASS admin: создание бакета (bootstrap) (ok) +--- без подписи (anonymous) +PASS anonymous http://s3-proxy:8333/attachments/ -> 403 +PASS anonymous http://s3-proxy:8333/attachments/attachments/x/y -> 403 +--- консоли управления из контейнера api (ожидается недоступность) +PASS api -> seaweedfs:9333 недоступен +PASS api -> seaweedfs:8888 недоступен +PASS api -> seaweedfs:8080 недоступен +PASS api -> s3-proxy:9333 недоступен +PASS api -> s3-proxy:8888 недоступен diff --git a/go.mod b/go.mod index 9f9ee490..d1ce5486 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,22 @@ module mtusi2026 go 1.27.1 +require ( + github.com/aws/aws-sdk-go-v2 v1.47.5 + github.com/aws/aws-sdk-go-v2/config v1.33.12 + github.com/aws/aws-sdk-go-v2/credentials v1.20.12 + github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 + github.com/aws/smithy-go v1.28.7 + github.com/golang-migrate/migrate/v4 v4.20.1 + github.com/jackc/pgx/v5 v5.11.0 + github.com/redis/go-redis/v9 v9.23.0 + github.com/testcontainers/testcontainers-go v0.44.0 + github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 + github.com/testcontainers/testcontainers-go/modules/redis v0.44.0 + golang.org/x/crypto v0.54.0 + golang.org/x/sync v0.22.0 +) + require ( cel.dev/expr v0.25.2 // indirect cloud.google.com/go v0.123.0 // indirect @@ -35,19 +51,20 @@ require ( github.com/apache/arrow/go/v10 v10.0.1 // indirect github.com/apache/thrift v0.24.0 // indirect github.com/aws/aws-sdk-go v1.49.6 // indirect - github.com/aws/aws-sdk-go-v2 v1.16.16 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 // indirect - github.com/aws/aws-sdk-go-v2/credentials v1.12.20 // indirect - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17 // indirect - github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11 // indirect - github.com/aws/smithy-go v1.13.3 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect @@ -60,7 +77,6 @@ require ( github.com/cpuguy83/dockercfg v0.3.2 // indirect github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369 // indirect github.com/danieljoos/wincred v1.1.2 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-connections v0.7.0 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -81,7 +97,6 @@ require ( github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/golang-jwt/jwt/v4 v4.5.2 // indirect - github.com/golang-migrate/migrate/v4 v4.20.1 // indirect github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/snappy v0.0.4 // indirect @@ -103,7 +118,6 @@ require ( github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/pgtype v1.14.0 // indirect github.com/jackc/pgx/v4 v4.18.2 // indirect - github.com/jackc/pgx/v5 v5.11.0 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/k0kubun/pp v2.3.0+incompatible // indirect @@ -138,14 +152,13 @@ require ( github.com/mutecomm/go-sqlcipher/v4 v4.4.0 // indirect github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 // indirect github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba // indirect + github.com/nxadm/tail v1.4.11 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pierrec/lz4/v4 v4.1.16 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect - github.com/redis/go-redis/v9 v9.23.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/rqlite/gorqlite v0.0.0-20230708021416-2acd02b70b79 // indirect github.com/shirou/gopsutil/v4 v4.26.6 // indirect @@ -154,9 +167,6 @@ require ( github.com/snowflakedb/gosnowflake v1.6.19 // indirect github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect github.com/stretchr/testify v1.12.1 // indirect - github.com/testcontainers/testcontainers-go v0.44.0 // indirect - github.com/testcontainers/testcontainers-go/modules/postgres v0.44.0 // indirect - github.com/testcontainers/testcontainers-go/modules/redis v0.44.0 // indirect github.com/tklauser/go-sysconf v0.4.0 // indirect github.com/tklauser/numcpus v0.12.0 // indirect github.com/xanzy/go-gitlab v0.15.0 // indirect @@ -180,11 +190,9 @@ require ( go.opentelemetry.io/otel/trace v1.44.0 // indirect go.uber.org/atomic v1.12.0 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect - golang.org/x/crypto v0.54.0 // indirect golang.org/x/mod v0.37.0 // indirect golang.org/x/net v0.56.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 // indirect golang.org/x/term v0.45.0 // indirect @@ -200,7 +208,6 @@ require ( google.golang.org/grpc v1.82.0 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/uint128 v1.2.0 // indirect modernc.org/b v1.0.0 // indirect modernc.org/cc/v3 v3.36.3 // indirect diff --git a/go.sum b/go.sum index 98a54fae..45a6da0f 100644 --- a/go.sum +++ b/go.sum @@ -11,6 +11,8 @@ cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdB cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= cloud.google.com/go/iam v1.11.0 h1:KieQ9Pb+LLPak1O3Rv3GgCxhnmkYf7Xyh0P5HfF1jFM= cloud.google.com/go/iam v1.11.0/go.mod h1:KP+nKGugNJW4LcLx1uEZcq1ok5sQHFaQehQNl4QDgV4= +cloud.google.com/go/logging v1.18.0 h1:KhzZq+1cSkPH9YUaKLLhLtQxIHitVayBmk0sGfoM9+k= +cloud.google.com/go/logging v1.18.0/go.mod h1:ZGKnpBaURITh+g/uom2VhbiFoFWvejcrHPDhxFtU/gI= cloud.google.com/go/longrunning v1.1.0 h1:qJ0R0IA8ONaRCNWTRPAS0iAmt1bj3TVgJ40z7ZGRslE= cloud.google.com/go/longrunning v1.1.0/go.mod h1:tH+A/6UvNypiPJWAQaKCsh+xiGbB23wUO8egwUXlD2E= cloud.google.com/go/monitoring v1.29.0 h1:AHhDsFaSax1/4k+qlIDX/SDGe6hggnfXJ9dkgD9qBPY= @@ -19,16 +21,21 @@ cloud.google.com/go/spanner v1.92.0 h1:cfeMNmtFjz+OYzQVCIuGBw4Cik4CbF2ptXMuRQcUa cloud.google.com/go/spanner v1.92.0/go.mod h1:rCDPfWXNX0h+t484r+crCEaaMKbJfoWkHRDKU3H3+oY= cloud.google.com/go/storage v1.56.0 h1:iixmq2Fse2tqxMbWhLWC9HfBj1qdxqAmiK8/eqtsLxI= cloud.google.com/go/storage v1.56.0/go.mod h1:Tpuj6t4NweCLzlNbw9Z9iwxEkrSem20AetIeH/shgVU= +cloud.google.com/go/trace v1.16.0 h1:GmQovzFc5F0CNfl0VLgL64aoTtu7xsM0YajW2GlG9+E= +cloud.google.com/go/trace v1.16.0/go.mod h1:r+bdAn16dKLSV1G2D5v3e58IlQlizfxWrUfjx7kM7X0= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= github.com/99designs/keyring v1.2.1 h1:tYLp1ULvO7i3fI5vE21ReQuj99QFSs7lGm0xWyJo87o= github.com/99designs/keyring v1.2.1/go.mod h1:fc+wB5KTk9wQ9sDx0kFXB3A0MaeGHM9AwRStKOQ5vOA= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.0.0/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.2/go.mod h1:uGG2W01BaETf0Ozp+QxxKJdMBNRWPdstHG0Fmdwn1/U= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.1 h1:T8quHYlUGyb/oqtSTwqlCr1ilJHrDv+ZtpSfo+hm1BU= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.1/go.mod h1:gLa1CL2RNE4s7M3yopJ/p0iq5DdY6Yv5ZUt9MTRZOQM= github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= @@ -43,11 +50,13 @@ github.com/Azure/go-autorest/autorest/adal v0.9.16 h1:P8An8Z9rH1ldbOLdFpxYorgOt2 github.com/Azure/go-autorest/autorest/adal v0.9.16/go.mod h1:tGMin8I49Yij6AQ+rvV+Xa/zwxYQB5hmsd6DkfAx2+A= github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/AzureAD/microsoft-authentication-library-for-go v0.8.1 h1:oPdPEZFSbl7oSPEAIPMPBMUmiL+mqgzBJwM/9qYcwNg= github.com/AzureAD/microsoft-authentication-library-for-go v0.8.1/go.mod h1:4qFor3D/HDsvBME35Xy9rwW9DecL+M2sNw1ybjPtwA0= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCUuKeNXGdqc= @@ -58,8 +67,13 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.33.0/go.mod h1:pJTkW8hEUIIi3Pf65lPZOnn4Y81yCllX6IWk2jNXdkM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c h1:RGWPOewvKIROun94nF7v2cua9qP+thov/7M50KEoeSU= +github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk= +github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= @@ -71,43 +85,54 @@ github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= github.com/aws/aws-sdk-go v1.49.6 h1:yNldzF5kzLBRvKlKz1S0bkvc2+04R1kt13KfBWQBfFA= github.com/aws/aws-sdk-go v1.49.6/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= -github.com/aws/aws-sdk-go-v2 v1.16.16 h1:M1fj4FE2lB4NzRb9Y0xdWsn2P0+2UHVxwKyOa4YJNjk= -github.com/aws/aws-sdk-go-v2 v1.16.16/go.mod h1:SwiyXi/1zTUZ6KIAmLK5V5ll8SiURNUYOqTerZPaF9k= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 h1:tcFliCWne+zOuUfKNRn8JdFBuWPDuISDH08wD2ULkhk= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8/go.mod h1:JTnlBSot91steJeti4ryyu/tLd4Sk84O5W22L7O2EQU= -github.com/aws/aws-sdk-go-v2/config v1.17.7/go.mod h1:dN2gja/QXxFF15hQreyrqYhLBaQo1d9ZKe/v/uplQoI= -github.com/aws/aws-sdk-go-v2/credentials v1.12.20 h1:9+ZhlDY7N9dPnUmf7CDfW9In4sW5Ff3bh7oy4DzS1IE= -github.com/aws/aws-sdk-go-v2/credentials v1.12.20/go.mod h1:UKY5HyIux08bbNA7Blv4PcXQ8cTkGh7ghHMFklaviR4= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17/go.mod h1:yIkQcCDYNsZfXpd5UX2Cy+sWA1jPgIhGTw9cOBzfVnQ= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33 h1:fAoVmNGhir6BR+RU0/EI+6+D7abM+MCwWf8v4ip5jNI= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.33/go.mod h1:84XgODVR8uRhmOnUkKGUZKqIMxmjmLOR8Uyp7G/TPwc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23 h1:s4g/wnzMf+qepSNgTvaQQHNxyMLKSawNhKCPNy++2xY= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.23/go.mod h1:2DFxAQ9pfIRy0imBCJv+vZ2X6RKxves6fbnEuSry6b4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17 h1:/K482T5A3623WJgWT8w1yRAFK4RzGzEl7y39yhtn9eA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.17/go.mod h1:pRwaTYCJemADaqCbUAxltMoHKata7hmB5PjEXeu0kfg= -github.com/aws/aws-sdk-go-v2/internal/ini v1.3.24/go.mod h1:jULHjqqjDlbyTa7pfM7WICATnOv+iOhjletM3N0Xbu8= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14 h1:ZSIPAkAsCCjYrhqfw2+lNzWDzxzHXEckFkTePL5RSWQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.14/go.mod h1:AyGgqiKv9ECM6IZeNQtdT8NnMvUb3/2wokeq2Fgryto= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 h1:Lh1AShsuIJTwMkoxVCAYPJgNG5H+eN6SmoUn8nOZ5wE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9/go.mod h1:a9j48l6yL5XINLHLcOKInjdvknN+vWqPBxqeIDw7ktw= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.18 h1:BBYoNQt2kUZUUK4bIPsKrCcjVPUMNsgQpNAwhznK/zo= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.18/go.mod h1:NS55eQ4YixUJPTC+INxi2/jCqe1y2Uw3rnh9wEOVJxY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17 h1:Jrd/oMh0PKQc6+BowB+pLEwLIgaQF29eYbe7E1Av9Ug= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.17/go.mod h1:4nYOrY41Lrbk2170/BGkcJKBhws9Pfn8MG3aGqjjeFI= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17 h1:HfVVR1vItaG6le+Bpw6P4midjBDMKnjMyZnw9MXYUcE= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.17/go.mod h1:YqMdV+gEKCQ59NrB7rzrJdALeBIsYiVi8Inj3+KcqHI= -github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11 h1:3/gm/JTX9bX8CpzTgIlrtYpB3EVBDxyg/GY/QdcIEZw= -github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11/go.mod h1:fmgDANqTUCxciViKl9hb/zD5LFbvPINFRgWhDbR+vZo= -github.com/aws/aws-sdk-go-v2/service/sso v1.11.23/go.mod h1:/w0eg9IhFGjGyyncHIQrXtU8wvNsTJOP0R6PPj0wf80= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.5/go.mod h1:csZuQY65DAdFBt1oIjO5hhBR49kQqop4+lcuCjf2arA= -github.com/aws/aws-sdk-go-v2/service/sts v1.16.19/go.mod h1:h4J3oPZQbxLhzGnk+j9dfYHi5qIOVJ5kczZd658/ydM= -github.com/aws/smithy-go v1.13.3 h1:l7LYxGuzK6/K+NzJ2mC+VvLUbae0sL3bXU//04MkmnA= -github.com/aws/smithy-go v1.13.3/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/aws-sdk-go-v2 v1.47.5 h1:5yA+A+BiXbDq1ZLbYwPViwhEGDMdR0MEhL8Jh2LnLS4= +github.com/aws/aws-sdk-go-v2 v1.47.5/go.mod h1:iNIkogrRvFwm+1OQIUOP2lPBtlXZboOIUWiCa+pHEMw= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25 h1:MW0Iql1SOTgQPj8mCxrU/eH/K/AIDWi08ItOsXaWRLU= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.25/go.mod h1:+ucVYmHYB3JqVueY0E1RHahdXAnKQbIHqybSbvOUbOM= +github.com/aws/aws-sdk-go-v2/config v1.33.12 h1:IV1YARaXahwCqUIyP8IlWsP85Lil+5ozTozDQXPumRk= +github.com/aws/aws-sdk-go-v2/config v1.33.12/go.mod h1:h51R06bzMVwNBRZOX5ylHhNDnkB/azZeq/LvxM6q2pQ= +github.com/aws/aws-sdk-go-v2/credentials v1.20.12 h1:6P7Q7yH/GIwHg6MLKv3ySsBqaCoE1z+C0OTzYMenhEY= +github.com/aws/aws-sdk-go-v2/credentials v1.20.12/go.mod h1:Kw/SaoW/wc7IqJErqXJIzKo8b7Aelxfbuc6XlbrRnsc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5 h1:iXjy2SFhGYNB1jWZBhQqq8HbhsbPF23w+/zbMvPXsTg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.5/go.mod h1:k0RUiBnyXU0AXRJ9tJmY2jQke6XUIX2PNVnWtvgaPtc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.18 h1:cNToRnziFY/DKoIUg9Vw350yx0sFfaN1FCJ+xVuF53Y= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.23.18/go.mod h1:FQWrFCsxaXWlLBYB3YQOBx1zbfDNRrrVspVnnsTZ8Z0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8 h1:gzIWc3DTPfPbWKrQnmL6BV6WD7NPD1htxzgdk/bQ/Z4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.8/go.mod h1:vvUA5cDAZWGo4R14FZPZfXfJCmgsafOWEq78Kg0D+ug= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8 h1:PWv+vG8aCLS8qd+HPdYhK7Vees90aKHWFFQ1AGYD/8E= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.8/go.mod h1:xxPZEpflYHJA12366ZGyq66z9dHPCoeCIu99Jck+MPw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8 h1:WOp5De9iWdEHDPh2Kb0tBP3vGBm/dVAuRGh4a4tqduo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.8/go.mod h1:z8W8YgfhbWl62XqcTtK+CxPrE+iHGRVUOtrrbIOp7ZY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23 h1:MCIR6iZaCKO4UE0hUvoWWLI6JVjppopV64JfAJaMjng= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.23/go.mod h1:pphWFS+z0MmIPivj8c/6bATd1Z+Av7Ust75DPdEFsPA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9 h1:GACgECI422Xt5n4Pojr3O0aocPv8lfH4PJe2Aon5Wv8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.11.9/go.mod h1:lPfnRkHpn1qWZJLbVEw10rIqSuCee1pNQTyGkUjAYh8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8 h1:+PMpWlJYOn2tm7sm6OZ/ZLd4jStl17T7wjNapd1Rczs= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.14.8/go.mod h1:cnG3xZLGQL4c/bt8pjdzlFtZsW7epBBuNS04a5xMjdo= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8 h1:RS1Wt+K0r4X0ci9Vf4JYTUfHzsQihYA2LgGJeEneKpk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.20.8/go.mod h1:adMnTy5kyoUW86jhXnGSVc/mvMQI4GZd1qYWMeaS8M8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6 h1:to7+Vr3rLkpYRwVKJ5QRriI0eE306vrVN/6TnVHgxvY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.114.6/go.mod h1:Jc89ASa2k7HMAskj3bSnt8bf+k+p2IApzkoXnEjkgUw= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.0 h1:3vu/eUqh0rjh7Zy4N4YKcpZX4Aud+zinB3VsLhhlQFw= +github.com/aws/aws-sdk-go-v2/service/signin v1.12.0/go.mod h1:wxAk0D2oVBSjuXEIzvcKxTZfM90zticdAbh/VD42L0Y= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.0 h1:kIEpD5dUry7VET3Ybl7zj7PPHRB4lb52G9iTz+a1Tqk= +github.com/aws/aws-sdk-go-v2/service/sso v1.39.0/go.mod h1:UGsEYgWz4Gi+46vDQ5zkFLqUgJziyOE2SXjZjgmfmHc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0 h1:WfFev+HgTodggcNon5HieiLxxwWwb8Br8R2+nDkixvk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.44.0/go.mod h1:gJSdIY6YUzDvqQLcMtPVZgBRHrFwzHryvpE8Igzbxps= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6 h1:BVjKrtRUtSYKztBCCfE1oAB+d3ViKcW2cKZfserOqMQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.51.6/go.mod h1:eSukFLeLkPMm+1OxDR+F0joauZdyBkF47Ow7MqjZiJw= +github.com/aws/smithy-go v1.28.7 h1:h/n1SHrwVtaDYLz8mR1/Kq4Xh4RaFbFW7bOYckg9mfE= +github.com/aws/smithy-go v1.28.7/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYEDvkta6I8/rnYM5gSdSV2tJ6XbZuEtY= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuDPIFo= -github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -119,6 +144,7 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/cockroach-go/v2 v2.1.1 h1:3XzfSMuUT0wBe1a3o5C0eOTcArhmmFAg2Jzh/7hhKqo= github.com/cockroachdb/cockroach-go/v2 v2.1.1/go.mod h1:7NtUnP6eK+l6k483WSYNrq3Kb23bWV10IRV1TyeSpwM= @@ -135,6 +161,8 @@ github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7 github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369 h1:XNT/Zf5l++1Pyg08/HV04ppB0gKxAqtZQBRYiYrUuYk= github.com/cznic/mathutil v0.0.0-20180504122225-ca4c9f2c1369/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= @@ -143,14 +171,20 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= +github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/docker/docker v28.5.2+incompatible h1:DBX0Y0zAjZbSrm1uzOkdr1onVghKaftjlSWt4AFexzM= +github.com/docker/docker v28.5.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dvsekhvalnov/jose2go v1.7.0 h1:bnQc8+GMnidJZA8zc6lLEAb4xNrIqHwO+9TzqvtQZPo= github.com/dvsekhvalnov/jose2go v1.7.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= @@ -161,8 +195,12 @@ github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= @@ -171,6 +209,10 @@ github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9 github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9mImVMaBPw9L/0TBHU8= github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsouza/fake-gcs-server v1.17.0 h1:OeH75kBZcZa3ZE+zz/mFdJ2btt9FgqfjI7gIh9+5fvk= +github.com/fsouza/fake-gcs-server v1.17.0/go.mod h1:D1rTE4YCyHFNa99oyJJ5HyclvN/0uQR+pM/VdlL83bw= github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= @@ -196,6 +238,7 @@ github.com/gocql/gocql v0.0.0-20210515062232-b7ef815b4556/go.mod h1:DL0ekTmBSTdl github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= @@ -209,8 +252,12 @@ github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U= +github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs= github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -222,6 +269,8 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -235,12 +284,15 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-github/v39 v39.2.0 h1:rNNM311XtPOz5rDdsJXAp2o8F67X9FnROXTvto3aSnQ= github.com/google/go-github/v39 v39.2.0/go.mod h1:C1s8C5aCC9L+JXIYpJM5GYytdX52vC1bLvHEF1IhBrE= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= @@ -253,6 +305,10 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= github.com/googleapis/gax-go/v2 v2.22.0 h1:PjIWBpgGIVKGoCXuiCoP64altEJCj3/Ei+kSU5vlZD4= github.com/googleapis/gax-go/v2 v2.22.0/go.mod h1:irWBbALSr0Sk3qlqb9SyJ1h68WjgeFuiOzI4Rqw5+aY= +github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg= +github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= +github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= @@ -282,6 +338,7 @@ github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= @@ -338,9 +395,11 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jmoiron/sqlx v1.3.1/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/k0kubun/pp v2.3.0+incompatible h1:EKhKbi34VQDWJtq+zpsKSEhkHHs9w2P8Izbq8IhLVSo= github.com/k0kubun/pp v2.3.0+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= @@ -351,22 +410,23 @@ github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:C github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/ktrysmt/go-bitbucket v0.6.4 h1:C8dUGp0qkwncKtAnozHCbbqhptefzEd1I0sfnuy9rYQ= github.com/ktrysmt/go-bitbucket v0.6.4/go.mod h1:9u0v3hsd2rqCHRIpbir1oP7F58uo5dq19sBYvuMoyQ4= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -430,6 +490,8 @@ github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3P github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE= github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/mutecomm/go-sqlcipher/v4 v4.4.0 h1:sV1tWCWGAVlPhNGT95Q+z/txFxuhAYWwHD1afF5bMZg= @@ -439,14 +501,21 @@ github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86w github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba h1:fhFP5RliM2HW/8XdcO5QngSfFli9GcRIpMXvypTQt6E= github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba/go.mod h1:ncO5VaFWh0Nrt+4KT4mOZboaczBZcLuHrG+/sUeP8gI= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= +github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= +github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= +github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.16 h1:kQPfno+wyx6C5572ABwV+Uo3pDFzQ7yhyGchSyRda0c= github.com/pierrec/lz4/v4 v4.1.16/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -454,6 +523,7 @@ github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4/go.mod h1:N6UoU20jOqgg github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= @@ -469,6 +539,8 @@ github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237/go.mod h1:qq github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rqlite/gorqlite v0.0.0-20230708021416-2acd02b70b79 h1:V7x0hCAgL8lNGezuex1RW1sh7VXXCqfw8nXZti66iFg= github.com/rqlite/gorqlite v0.0.0-20230708021416-2acd02b70b79/go.mod h1:xF/KoXmrRyahPfo5L7Szb5cAAUl53dMWBh9cMruGEZg= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= @@ -483,8 +555,6 @@ github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXY github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/snowflakedb/gosnowflake v1.6.19 h1:KSHXrQ5o7uso25hNIzi/RObXtnSGkFgie91X82KcvMY= @@ -496,6 +566,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -505,8 +577,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/testcontainers/testcontainers-go v0.44.0 h1:/Fwh6HY1mIikhnm9e7HwoxGycx0lzRAE0f5VQpjFxzI= @@ -533,8 +603,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= -github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= @@ -554,8 +624,12 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/metric/x v0.66.0 h1:YkCrx1zLOChi9ZcZ6euupOcsgzbVlec7D/xoEU1+cTA= +go.opentelemetry.io/otel/metric/x v0.66.0/go.mod h1:d1+BDj9t96do0/1LoU1ayfCv79ZgNE41qbhBvnMOBZk= go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= @@ -595,11 +669,11 @@ golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220511200225-c6db032c6c88/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= -golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0 h1:pVgRXcIictcr+lBQIFeiwuwtDIs4eL21OuM9nyAADmo= +golang.org/x/exp v0.0.0-20230315142452-642cacee5cc0/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -608,8 +682,6 @@ golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKG golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4= -golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ= golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -646,10 +718,6 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= -golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20180224232135-f6cff0780e54/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -681,25 +749,17 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= -golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= -golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6 h1:HjU6IWBiAgRIdAJ9/y1rwCn+UELEmwV+VsTLzj/W4sE= -golang.org/x/telemetry v0.0.0-20260508192327-42602be52be6/go.mod h1:Eqhaxk/wZsWEH8CRxLwj6xzEJbz7k1EFGqx7nyCoabE= golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 h1:nwGZBCt+FnXUrGsj5vjzAsEmkcaFvd82BbOjECiFYZc= golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= -golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -711,10 +771,6 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= -golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= -golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= -golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= @@ -734,8 +790,6 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8= -golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0= golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk= @@ -748,6 +802,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.287.0 h1:CQDMqUiqZZ0U/Yge3zyjAhNQ0OSYEH0PaA7l4xtEen4= google.golang.org/api v0.287.0/go.mod h1:pPW85yt3Iuc3unkpaMhFtMmOqnTdCwCqEOaUlnuxRlQ= google.golang.org/appengine v1.0.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -785,25 +841,31 @@ google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/postgres v1.0.8/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg= gorm.io/gorm v1.20.12/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v1.21.4/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= @@ -817,6 +879,7 @@ modernc.org/cc/v3 v3.36.3 h1:uISP3F66UlixxWEcKuIWERa4TwrZENHSL8tWxZz8bHg= modernc.org/cc/v3 v3.36.3/go.mod h1:NFUHyPn4ekoC/JHeZFfZurN6ixxawE1BnVonP/oahEI= modernc.org/ccgo/v3 v3.16.9 h1:AXquSwg7GuMk11pIdw7fmO1Y/ybgazVkMhsZWCV0mHM= modernc.org/ccgo/v3 v3.16.9/go.mod h1:zNMzC9A9xeNUepy6KuZBbugn3c0Mc9TeiJO4lgvkJDo= +modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= modernc.org/db v1.0.0 h1:2c6NdCfaLnshSvY7OU09cyAY0gYXUZj4lmg5ItHyucg= modernc.org/db v1.0.0/go.mod h1:kYD/cO29L/29RM0hXYl4i3+Q5VojL31kTUVpVJDw0s8= @@ -826,6 +889,7 @@ modernc.org/fileutil v1.0.0 h1:Z1AFLZwl6BO8A5NldQg/xTSjGLetp+1Ubvl4alfGx8w= modernc.org/fileutil v1.0.0/go.mod h1:JHsWpkrk/CnVV1H/eGlFf85BEpfkrp56ro8nojIq9Q8= modernc.org/golex v1.0.0 h1:wWpDlbK8ejRfSyi0frMyhilD3JBvtcx2AdGDnU+JtsE= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= modernc.org/internal v1.0.0 h1:XMDsFDcBDsibbBnHB2xzljZ+B1yrOVLEFkKL2u15Glw= modernc.org/internal v1.0.0/go.mod h1:VUD/+JAkhCpvkUitlEOnhpVxCgsBI90oTzSCRcqQVSM= @@ -854,9 +918,15 @@ modernc.org/sqlite v1.18.1/go.mod h1:6ho+Gow7oX5V+OiOQ6Tr4xeqbx13UZ6t+Fw9IRUG4d4 modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw= modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= +modernc.org/tcl v1.13.1 h1:npxzTwFTZYM8ghWicVIX1cRWzj7Nd8i6AqqX2p+IYao= +modernc.org/tcl v1.13.1/go.mod h1:XOLfOwzhkljL4itZkK6T72ckMgvj0BDsnKNdZVUOecw= modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk= modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +modernc.org/z v1.5.1 h1:RTNHdsrOpeoSeOF4FbzTo8gBYByaJ5xT7NgZ9ZqRiJM= +modernc.org/z v1.5.1/go.mod h1:eWFB510QWW5Th9YGZT81s+LwvaAs3Q2yr4sP0rmLkv8= modernc.org/zappy v1.0.0 h1:dPVaP+3ueIUv4guk8PuZ2wiUGcJ1WUVvIheeSSTD0yk= modernc.org/zappy v1.0.0/go.mod h1:hHe+oGahLVII/aTTyWK/b53VDHMAGCBYYeZ9sn83HC4= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/infra/haproxy-s3.cfg b/infra/haproxy-s3.cfg new file mode 100644 index 00000000..1a970859 --- /dev/null +++ b/infra/haproxy-s3.cfg @@ -0,0 +1,22 @@ +global + log stdout format raw local0 + +defaults + mode tcp + timeout connect 3s + timeout client 60s + timeout server 60s + +resolvers docker + nameserver dns 127.0.0.11:53 + hold valid 2s + timeout resolve 1s + timeout retry 1s + resolve_retries 3 + +frontend s3 + bind :8333 + default_backend seaweed + +backend seaweed + server s1 seaweedfs:8333 resolvers docker init-addr none diff --git a/internal/blobstore/blobstore.go b/internal/blobstore/blobstore.go new file mode 100644 index 00000000..b8a64258 --- /dev/null +++ b/internal/blobstore/blobstore.go @@ -0,0 +1,233 @@ +package blobstore + +import ( + "bytes" + "context" + "errors" + "io" + "strings" + "sync" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" +) + +var ErrNotFound = errors.New("object not found") + +type Store interface { + Put(ctx context.Context, key string, body io.ReadSeeker, size int64, contentType string) error + Get(ctx context.Context, key string) (io.ReadCloser, error) + Delete(ctx context.Context, key string) error + Ping(ctx context.Context) error + List(ctx context.Context, prefix string, fn func(key string, modified time.Time) error) error + Exists(ctx context.Context, key string) (bool, error) +} + +type S3Config struct { + Endpoint, Region, Bucket, AccessKey, SecretKey string +} + +type S3 struct { + client *s3.Client + bucket string +} + +func NewS3(ctx context.Context, c S3Config) (*S3, error) { + if c.Endpoint == "" || c.Bucket == "" || c.AccessKey == "" || c.SecretKey == "" { + return nil, errors.New("incomplete S3 configuration") + } + cfg, err := config.LoadDefaultConfig(ctx, + config.WithRegion(c.Region), + config.WithRetryMaxAttempts(2), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(c.AccessKey, c.SecretKey, "")), + // Новые версии SDK по умолчанию добавляют контрольные суммы, которые не все S3-совместимые серверы понимают. + config.WithRequestChecksumCalculation(aws.RequestChecksumCalculationWhenRequired), + config.WithResponseChecksumValidation(aws.ResponseChecksumValidationWhenRequired), + ) + if err != nil { + return nil, err + } + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = aws.String(c.Endpoint) + o.UsePathStyle = true + }) + return &S3{client: client, bucket: c.Bucket}, nil +} + +func (s *S3) Put(ctx context.Context, key string, body io.ReadSeeker, size int64, contentType string) error { + _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + Body: body, + ContentLength: aws.Int64(size), + ContentType: aws.String(contentType), + }) + return err +} + +func (s *S3) Get(ctx context.Context, key string) (io.ReadCloser, error) { + out, err := s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }) + if err != nil { + if isNotFound(err) { + return nil, ErrNotFound + } + return nil, err + } + return out.Body, nil +} + +func (s *S3) Delete(ctx context.Context, key string) error { + _, err := s.client.DeleteObject(ctx, &s3.DeleteObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }) + return err +} + +func (s *S3) Ping(ctx context.Context) error { + body := []byte("ok") + if _, err := s.client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String("_health/probe"), + Body: bytes.NewReader(body), + ContentLength: aws.Int64(int64(len(body))), + }); err != nil { + return err + } + out, err := s.client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String("_health/probe"), + }) + if err != nil { + return err + } + defer out.Body.Close() + _, err = io.Copy(io.Discard, out.Body) + return err +} + +func (s *S3) List(ctx context.Context, prefix string, fn func(key string, modified time.Time) error) error { + p := s3.NewListObjectsV2Paginator(s.client, &s3.ListObjectsV2Input{ + Bucket: aws.String(s.bucket), + Prefix: aws.String(prefix), + }) + for p.HasMorePages() { + page, err := p.NextPage(ctx) + if err != nil { + return err + } + for _, o := range page.Contents { + if err := fn(aws.ToString(o.Key), aws.ToTime(o.LastModified)); err != nil { + return err + } + } + } + return nil +} + +func (s *S3) Exists(ctx context.Context, key string) (bool, error) { + _, err := s.client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(key), + }) + if err != nil { + if isNotFound(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func isNotFound(err error) bool { + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + switch apiErr.ErrorCode() { + case "NoSuchKey", "NotFound": + return true + } + } + return false +} + +type Memory struct { + mu sync.Mutex + m map[string][]byte + mod map[string]time.Time + Err error +} + +func NewMemory() *Memory { return &Memory{m: map[string][]byte{}, mod: map[string]time.Time{}} } + +func (s *Memory) Put(_ context.Context, key string, body io.ReadSeeker, _ int64, _ string) error { + if s.Err != nil { + return s.Err + } + b, err := io.ReadAll(body) + if err != nil { + return err + } + s.mu.Lock() + s.m[key], s.mod[key] = b, time.Now() + s.mu.Unlock() + return nil +} + +func (s *Memory) Get(_ context.Context, key string) (io.ReadCloser, error) { + if s.Err != nil { + return nil, s.Err + } + s.mu.Lock() + b, ok := s.m[key] + s.mu.Unlock() + if !ok { + return nil, ErrNotFound + } + return io.NopCloser(bytes.NewReader(b)), nil +} + +func (s *Memory) Delete(_ context.Context, key string) error { + if s.Err != nil { + return s.Err + } + s.mu.Lock() + delete(s.m, key) + delete(s.mod, key) + s.mu.Unlock() + return nil +} + +func (s *Memory) Ping(_ context.Context) error { return s.Err } + +func (s *Memory) Exists(_ context.Context, key string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + _, ok := s.m[key] + return ok, s.Err +} + +func (s *Memory) List(_ context.Context, prefix string, fn func(string, time.Time) error) error { + s.mu.Lock() + keys := make([]string, 0, len(s.m)) + for k := range s.m { + if strings.HasPrefix(k, prefix) { + keys = append(keys, k) + } + } + s.mu.Unlock() + for _, k := range keys { + if err := fn(k, s.mod[k]); err != nil { + return err + } + } + return nil +} + +func (s *Memory) Len() int { s.mu.Lock(); defer s.mu.Unlock(); return len(s.m) } diff --git a/internal/httpapi/attachments.go b/internal/httpapi/attachments.go new file mode 100644 index 00000000..beccf2e8 --- /dev/null +++ b/internal/httpapi/attachments.go @@ -0,0 +1,243 @@ +package httpapi + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "log" + "mime" + "net/http" + "os" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "mtusi2026/internal/blobstore" + "mtusi2026/internal/idgen" + "mtusi2026/internal/model" + "mtusi2026/internal/store" +) + +const ( + maxAttachmentBytes = 10 << 20 + maxPixels = 40_000_000 + fullDecodePixels = 4_000_000 // полное декодирование только для небольших изображений +) + +func validFilename(s string) bool { + n := utf8.RuneCountInString(s) + if n < 1 || n > 255 || strings.TrimSpace(s) == "" { + return false + } + for _, r := range s { + if r == '/' || r == '\\' || r == 0 || unicode.IsControl(r) { + return false + } + } + return true +} + +func (a *api) acquireUploadSlot(ctx context.Context) (func(), bool) { + t := time.NewTimer(5 * time.Second) + defer t.Stop() + select { + case a.uploadSem <- struct{}{}: + return func() { <-a.uploadSem }, true + case <-t.C: + return nil, false + case <-ctx.Done(): + return nil, false + } +} + +func (a *api) handleUploadAttachment(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { + chatID, ok := uuidParam(w, r, requestID, "chat_id") + if !ok { + return + } + if a.blobs == nil { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "attachment storage unavailable") + return + } + member, err := a.store.IsChatMember(r.Context(), chatID, u.ID) + if err != nil { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "database unavailable") + return + } + if !member { + writeError(w, http.StatusNotFound, requestID, codeNotFound, "chat not found") + return + } + + q := r.URL.Query() + filename := q.Get("filename") + if !q.Has("filename") || !validFilename(filename) { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid filename") + return + } + mt, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || (mt != "image/png" && mt != "image/jpeg") { + writeError(w, http.StatusUnsupportedMediaType, requestID, codeUnsupportedMedia, "only image/png and image/jpeg are accepted") + return + } + if r.ContentLength > maxAttachmentBytes { + writeError(w, http.StatusRequestEntityTooLarge, requestID, codeTooLarge, "file is too large") + return + } + + tmp, err := os.CreateTemp("", "upload-*") + if err != nil { + writeError(w, http.StatusInternalServerError, requestID, codeInternal, "internal error") + return + } + defer func() { tmp.Close(); os.Remove(tmp.Name()) }() + + h := sha256.New() + n, err := io.Copy(io.MultiWriter(tmp, h), http.MaxBytesReader(w, r.Body, maxAttachmentBytes)) + if err != nil { + var mbe *http.MaxBytesError + if errors.As(err, &mbe) { + writeError(w, http.StatusRequestEntityTooLarge, requestID, codeTooLarge, "file is too large") + } else { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "upload interrupted") + } + return + } + if n == 0 { + writeError(w, http.StatusUnsupportedMediaType, requestID, codeUnsupportedMedia, "empty or invalid image") + return + } + + release, ok := a.acquireUploadSlot(r.Context()) + if !ok { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "upload capacity reached, retry later") + return + } + defer release() + + head := make([]byte, 512) + hn, _ := tmp.ReadAt(head, 0) + if http.DetectContentType(head[:hn]) != mt { + writeError(w, http.StatusUnsupportedMediaType, requestID, codeUnsupportedMedia, "content does not match declared type") + return + } + cfg, format, err := image.DecodeConfig(io.NewSectionReader(tmp, 0, n)) + want := map[string]string{"image/png": "png", "image/jpeg": "jpeg"}[mt] + if err != nil || format != want || cfg.Width <= 0 || cfg.Height <= 0 { + writeError(w, http.StatusUnsupportedMediaType, requestID, codeUnsupportedMedia, "invalid image") + return + } + pixels := int64(cfg.Width) * int64(cfg.Height) + if pixels > maxPixels { + writeError(w, http.StatusRequestEntityTooLarge, requestID, codeTooLarge, "image has too many pixels") + return + } + if pixels <= fullDecodePixels { + if _, _, err := image.Decode(io.NewSectionReader(tmp, 0, n)); err != nil { + writeError(w, http.StatusUnsupportedMediaType, requestID, codeUnsupportedMedia, "corrupted image") + return + } + } + + id := idgen.GenerateUUIDv4() + key := "attachments/" + chatID + "/" + id + + baseCtx := context.WithoutCancel(r.Context()) + putCtx, cancel := context.WithTimeout(baseCtx, 30*time.Second) + err = a.blobs.Put(putCtx, key, io.NewSectionReader(tmp, 0, n), n, mt) + cancel() + if err != nil { + log.Printf("s3 put failed: %v", err) + delCtx, delCancel := context.WithTimeout(baseCtx, 5*time.Second) + _ = a.blobs.Delete(delCtx, key) + delCancel() + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "attachment storage unavailable") + return + } + + dbCtx, dbCancel := context.WithTimeout(baseCtx, 10*time.Second) + defer dbCancel() + att, err := a.store.CreateAttachment(dbCtx, model.Attachment{ + ID: id, ChatID: chatID, OwnerID: u.ID, Filename: filename, ContentType: mt, + SizeBytes: n, SHA256: hex.EncodeToString(h.Sum(nil)), ObjectKey: key, + }) + if err != nil { + log.Printf("attachment metadata insert failed, compensating: %v", err) + if derr := a.blobs.Delete(dbCtx, key); derr != nil { + log.Printf("compensation delete failed (reconciler will clean up) key=%s: %v", key, derr) + } + writeError(w, http.StatusInternalServerError, requestID, codeInternal, "internal error") + return + } + writeJSON(w, http.StatusCreated, requestID, att) +} + +func (a *api) loadAttachment(w http.ResponseWriter, r *http.Request, u model.User, requestID string) (model.Attachment, bool) { + id, ok := uuidParam(w, r, requestID, "attachment_id") + if !ok { + return model.Attachment{}, false + } + att, err := a.store.GetAttachmentForUser(r.Context(), id, u.ID) + if err != nil { + if errors.Is(err, store.ErrAttachmentNotFound) { + writeError(w, http.StatusNotFound, requestID, codeNotFound, "attachment not found") + } else { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "database unavailable") + } + return model.Attachment{}, false + } + return att, true +} + +func (a *api) handleGetAttachment(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { + att, ok := a.loadAttachment(w, r, u, requestID) + if !ok { + return + } + writeJSON(w, http.StatusOK, requestID, att) +} + +func (a *api) handleDownloadAttachment(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { + if q := r.URL.Query(); q.Has("variant") && q.Get("variant") != "original" { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "unknown variant") + return + } + att, ok := a.loadAttachment(w, r, u, requestID) + if !ok { + return + } + if a.blobs == nil { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "attachment storage unavailable") + return + } + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + t := time.AfterFunc(8*time.Second, cancel) + body, err := a.blobs.Get(ctx, att.ObjectKey) + t.Stop() + if err != nil { + if errors.Is(err, blobstore.ErrNotFound) { + log.Printf("INCONSISTENCY: metadata without object id=%s key=%s", att.ID, att.ObjectKey) + writeError(w, http.StatusInternalServerError, requestID, codeInternal, "internal error") + } else { + log.Printf("s3 get failed: %v", err) + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "attachment storage unavailable") + } + return + } + defer body.Close() + + w.Header().Set("Content-Type", att.ContentType) + w.Header().Set("Content-Length", strconv.FormatInt(att.SizeBytes, 10)) + w.Header().Set("X-Request-Id", requestID) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(http.StatusOK) + _, _ = io.Copy(w, body) +} diff --git a/internal/httpapi/health.go b/internal/httpapi/health.go index 8df2046a..22db2c04 100644 --- a/internal/httpapi/health.go +++ b/internal/httpapi/health.go @@ -3,18 +3,45 @@ package httpapi import ( "context" "net/http" + "sync" "sync/atomic" "time" "github.com/jackc/pgx/v5/pgxpool" "github.com/redis/go-redis/v9" + + "mtusi2026/internal/blobstore" ) func handleHealthLive(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusOK, newRequestID(), map[string]string{"status": "ok"}) } -func handleHealthReady(pool *pgxpool.Pool, rdb *redis.Client, draining *atomic.Bool) http.HandlerFunc { +type blobProbe struct { + mu sync.Mutex + at time.Time + err error + st blobstore.Store +} + +func (p *blobProbe) check(ctx context.Context) error { + p.mu.Lock() + defer p.mu.Unlock() + if time.Since(p.at) < time.Second { + return p.err + } + cctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + p.err = p.st.Ping(cctx) + p.at = time.Now() + return p.err +} + +func handleHealthReady(pool *pgxpool.Pool, rdb *redis.Client, blobs blobstore.Store, draining *atomic.Bool) http.HandlerFunc { + var probe *blobProbe + if blobs != nil { + probe = &blobProbe{st: blobs} + } return func(w http.ResponseWriter, r *http.Request) { requestID := newRequestID() if draining.Load() { @@ -34,6 +61,12 @@ func handleHealthReady(pool *pgxpool.Pool, rdb *redis.Client, draining *atomic.B return } } + if probe != nil { + if err := probe.check(ctx); err != nil { + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "attachment storage unavailable") + return + } + } writeJSON(w, http.StatusOK, requestID, map[string]string{"status": "ok"}) } } diff --git a/internal/httpapi/httpx.go b/internal/httpapi/httpx.go index c6286119..718f43df 100644 --- a/internal/httpapi/httpx.go +++ b/internal/httpapi/httpx.go @@ -15,12 +15,14 @@ import ( type errorCode string const ( - codeInvalidRequest errorCode = "invalid_request" - codeUnauthorized errorCode = "unauthorized" - codeNotFound errorCode = "not_found" - codeConflict errorCode = "conflict" - codeUnavailable errorCode = "unavailable" - codeInternal errorCode = "internal_error" + codeInvalidRequest errorCode = "invalid_request" + codeUnauthorized errorCode = "unauthorized" + codeNotFound errorCode = "not_found" + codeConflict errorCode = "conflict" + codeUnavailable errorCode = "unavailable" + codeInternal errorCode = "internal_error" + codeTooLarge errorCode = "payload_too_large" + codeUnsupportedMedia errorCode = "unsupported_media_type" ) func newRequestID() string { diff --git a/internal/httpapi/messages.go b/internal/httpapi/messages.go index dd1a2af9..72b4263a 100644 --- a/internal/httpapi/messages.go +++ b/internal/httpapi/messages.go @@ -4,7 +4,9 @@ import ( "errors" "net/http" "regexp" + "strings" + "mtusi2026/internal/idgen" "mtusi2026/internal/model" "mtusi2026/internal/store" ) @@ -12,7 +14,7 @@ import ( var idemKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) func (a *api) handleCreateMessage(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { - chatID, ok := chatIDParam(w, r, requestID) + chatID, ok := uuidParam(w, r, requestID, "chat_id") if !ok { return } @@ -28,11 +30,27 @@ func (a *api) handleCreateMessage(w http.ResponseWriter, r *http.Request, u mode return } - m, err := a.store.CreateMessage(r.Context(), chatID, u.ID, key, req.Text) + ids := make([]string, 0, len(req.AttachmentIDs)) + seen := map[string]bool{} + for _, id := range req.AttachmentIDs { + id = strings.ToLower(id) + if !idgen.IsValidUUID(id) || seen[id] { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid attachment_ids") + return + } + seen[id] = true + ids = append(ids, id) + } + if len(ids) > 10 || !store.ValidMessageBody(req.Text, len(ids) > 0) { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid text or attachment_ids") + return + } + + m, err := a.store.CreateMessage(r.Context(), chatID, u.ID, key, req.Text, ids) if err != nil { switch { - case errors.Is(err, store.ErrChatNotFound): - writeError(w, http.StatusNotFound, requestID, codeNotFound, "chat not found") + case errors.Is(err, store.ErrChatNotFound), errors.Is(err, store.ErrAttachmentNotFound): + writeError(w, http.StatusNotFound, requestID, codeNotFound, "chat or attachment not found") case errors.Is(err, store.ErrIdempotencyConflict): writeError(w, http.StatusConflict, requestID, codeConflict, "Idempotency-Key was used with a different request body") case errors.Is(err, store.ErrValidation): diff --git a/internal/httpapi/router.go b/internal/httpapi/router.go index b9479e94..588b2d7f 100644 --- a/internal/httpapi/router.go +++ b/internal/httpapi/router.go @@ -7,6 +7,7 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/redis/go-redis/v9" + "mtusi2026/internal/blobstore" "mtusi2026/internal/cache" "mtusi2026/internal/model" "mtusi2026/internal/sessionstore" @@ -17,6 +18,8 @@ type Options struct { InstanceID string Redis *redis.Client Draining *atomic.Bool + Blobs blobstore.Store + MaxUploads int } func NewRouter(s *store.Store, pool *pgxpool.Pool, sessionStores ...sessionstore.Store) http.Handler { @@ -45,10 +48,16 @@ func NewRouterWithCache( o.Draining = new(atomic.Bool) } + if o.MaxUploads <= 0 { + o.MaxUploads = 4 + } + a := &api{ store: s, sessionStore: sessionStore, chatCache: chatCache, + blobs: o.Blobs, + uploadSem: make(chan struct{}, o.MaxUploads), } apiMux := http.NewServeMux() @@ -60,6 +69,9 @@ func NewRouterWithCache( apiMux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleCreateMessage)) apiMux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", a.authMiddleware(a.handleListMessages)) apiMux.HandleFunc("POST /api/v1/auth/sessions", a.handleCreateSession) + apiMux.HandleFunc("POST /api/v1/chats/{chat_id}/attachments", a.authMiddleware(a.handleUploadAttachment)) + apiMux.HandleFunc("GET /api/v1/attachments/{attachment_id}", a.authMiddleware(a.handleGetAttachment)) + apiMux.HandleFunc("GET /api/v1/attachments/{attachment_id}/content", a.authMiddleware(a.handleDownloadAttachment)) apiMux.HandleFunc("DELETE /api/v1/auth/sessions/current", a.authMiddleware( func(w http.ResponseWriter, r *http.Request, u model.User, requestID string) { a.handleDeleteSession(w, r) @@ -67,7 +79,7 @@ func NewRouterWithCache( root := http.NewServeMux() root.HandleFunc("GET /health/live", handleHealthLive) - root.HandleFunc("GET /health/ready", handleHealthReady(pool, o.Redis, o.Draining)) + root.HandleFunc("GET /health/ready", handleHealthReady(pool, o.Redis, o.Blobs, o.Draining)) root.Handle("/", concurrencyMiddleware(s.Limiter, apiMux)) var h http.Handler = root diff --git a/internal/httpapi/users.go b/internal/httpapi/users.go index 939ad09f..c893ea3e 100644 --- a/internal/httpapi/users.go +++ b/internal/httpapi/users.go @@ -9,6 +9,7 @@ import ( "golang.org/x/sync/singleflight" "mtusi2026/internal/auth" + "mtusi2026/internal/blobstore" "mtusi2026/internal/cache" "mtusi2026/internal/model" "mtusi2026/internal/sessionstore" @@ -22,6 +23,8 @@ type api struct { sessionStore sessionstore.Store chatCache *cache.Cache chatFlight singleflight.Group + blobs blobstore.Store + uploadSem chan struct{} } func (a *api) authMiddleware(next func(w http.ResponseWriter, r *http.Request, u model.User, requestID string)) http.HandlerFunc { diff --git a/internal/httpapi/validate.go b/internal/httpapi/validate.go index 11da225c..3d0816c0 100644 --- a/internal/httpapi/validate.go +++ b/internal/httpapi/validate.go @@ -43,3 +43,12 @@ func chatIDParam(w http.ResponseWriter, r *http.Request, requestID string) (stri } return strings.ToLower(id), true } + +func uuidParam(w http.ResponseWriter, r *http.Request, requestID, name string) (string, bool) { + id := r.PathValue(name) + if !idgen.IsValidUUID(id) { + writeError(w, http.StatusBadRequest, requestID, codeInvalidRequest, "invalid "+name) + return "", false + } + return strings.ToLower(id), true +} diff --git a/internal/model/model.go b/internal/model/model.go index e468a347..33af5bcb 100644 --- a/internal/model/model.go +++ b/internal/model/model.go @@ -33,15 +33,17 @@ type ChatPage struct { } type Message struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - SenderID string `json:"sender_id"` - Text string `json:"text"` - CreatedAt time.Time `json:"created_at"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt time.Time `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` } type CreateMessageRequest struct { - Text string `json:"text"` + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` } type MessagePage struct { @@ -63,3 +65,28 @@ type Session struct { ExpiresAt time.Time `json:"expires_at"` User User `json:"user"` } + +type Variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + Width int `json:"width"` + Height int `json:"height"` + SHA256 string `json:"sha256"` +} + +type Attachment struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []Variant `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ObjectKey string `json:"-"` +} diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go new file mode 100644 index 00000000..d02f3ef3 --- /dev/null +++ b/internal/reconcile/reconcile.go @@ -0,0 +1,116 @@ +package reconcile + +import ( + "context" + "fmt" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "mtusi2026/internal/blobstore" +) + +type Report struct { + Scanned, Young, Orphans, Deleted, Missing int +} + +type cand struct{ key string } + +func Run(ctx context.Context, pool *pgxpool.Pool, blobs blobstore.Store, grace time.Duration, + dryRun, checkMissing bool, logf func(string, ...any)) (Report, error) { + + var rep Report + cutoff := time.Now().Add(-grace) + var batch []string + + flush := func() error { + if len(batch) == 0 { + return nil + } + rows, err := pool.Query(ctx, `SELECT object_key FROM attachments WHERE object_key = ANY($1)`, batch) + if err != nil { + return err + } + known := map[string]bool{} + for rows.Next() { + var k string + if err := rows.Scan(&k); err != nil { + rows.Close() + return err + } + known[k] = true + } + rows.Close() + if err := rows.Err(); err != nil { + return err + } + for _, k := range batch { + if known[k] { + continue + } + rep.Orphans++ + if dryRun { + logf("orphan (dry-run, not deleted): %s", k) + continue + } + if err := blobs.Delete(ctx, k); err != nil { + logf("orphan delete failed: %s: %v", k, err) + continue + } + rep.Deleted++ + logf("orphan deleted: %s", k) + } + batch = batch[:0] + return nil + } + + err := blobs.List(ctx, "attachments/", func(key string, modified time.Time) error { + rep.Scanned++ + if modified.After(cutoff) { + rep.Young++ + return nil + } + batch = append(batch, key) + if len(batch) >= 500 { + return flush() + } + return nil + }) + if err != nil { + return rep, fmt.Errorf("list: %w", err) + } + if err := flush(); err != nil { + return rep, fmt.Errorf("check db: %w", err) + } + + if checkMissing { + rows, err := pool.Query(ctx, + `SELECT id, object_key FROM attachments WHERE created_at < $1 ORDER BY created_at`, cutoff) + if err != nil { + return rep, err + } + type rec struct{ id, key string } + var recs []rec + for rows.Next() { + var r rec + if err := rows.Scan(&r.id, &r.key); err != nil { + rows.Close() + return rep, err + } + recs = append(recs, r) + } + rows.Close() + for _, r := range recs { + ok, err := blobs.Exists(ctx, r.key) + if err != nil { + logf("exists check failed: %s: %v", r.key, err) + continue + } + if !ok { + rep.Missing++ + logf("MISSING object for attachment id=%s key=%s (metadata kept, needs manual decision)", r.id, r.key) + } + } + } + return rep, nil +} diff --git a/internal/store/store.go b/internal/store/store.go index 13ec0389..f2756b7f 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "errors" "fmt" "strconv" @@ -21,10 +22,13 @@ import ( ) var ( - ErrUsernameTaken = errors.New("username taken") - ErrChatNotFound = errors.New("chat not found or not accessible") - ErrValidation = errors.New("validation failed") - ErrCapacityExceeded = errors.New("message capacity exceeded") + ErrUsernameTaken = errors.New("username taken") + ErrChatNotFound = errors.New("chat not found or not accessible") + ErrValidation = errors.New("validation failed") + ErrCapacityExceeded = errors.New("message capacity exceeded") + ErrAttachmentNotFound = errors.New("attachment not found or not accessible") + ErrUsernameNotFound = errors.New("username not found") + ErrSessionNotFound = errors.New("session not found or expired") ) type Store struct { @@ -33,10 +37,102 @@ type Store struct { chatSQLQueries atomic.Int64 } +func (s *Store) ResetChatSQLQueries() { + s.chatSQLQueries.Store(0) +} + +func (s *Store) ChatSQLQueries() int64 { + return s.chatSQLQueries.Load() +} + +func (s *Store) IsChatMember(ctx context.Context, chatID, userID string) (bool, error) { + var ok bool + err := s.pool.QueryRow(ctx, + `SELECT EXISTS(SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2)`, + chatID, userID).Scan(&ok) + return ok, err +} + +func (s *Store) CreateAttachment(ctx context.Context, a model.Attachment) (model.Attachment, error) { + err := s.pool.QueryRow(ctx, + `INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8) + RETURNING status, created_at, updated_at`, + a.ID, a.ChatID, a.OwnerID, a.Filename, a.ContentType, a.SizeBytes, a.SHA256, a.ObjectKey, + ).Scan(&a.Status, &a.CreatedAt, &a.UpdatedAt) + a.Variants = []model.Variant{} + return a, err +} +func (s *Store) GetAttachmentForUser(ctx context.Context, id, userID string) (model.Attachment, error) { + var a model.Attachment + err := s.pool.QueryRow(ctx, + `SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, + a.object_key, a.status, a.created_at, a.updated_at + FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = $2 + WHERE a.id = $1`, id, userID, + ).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ContentType, &a.SizeBytes, &a.SHA256, + &a.ObjectKey, &a.Status, &a.CreatedAt, &a.UpdatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return model.Attachment{}, ErrAttachmentNotFound + } + a.Variants = []model.Variant{} + return a, err +} + +func (s *Store) loadAttachmentIDs(ctx context.Context, msgs []model.Message) error { + if len(msgs) == 0 { + return nil + } + ids := make([]string, len(msgs)) + pos := make(map[string]int, len(msgs)) + for i := range msgs { + ids[i] = msgs[i].ID + pos[msgs[i].ID] = i + msgs[i].AttachmentIDs = []string{} + } + rows, err := s.pool.Query(ctx, + `SELECT message_id, attachment_id FROM message_attachments + WHERE message_id IN (SELECT unnest($1::text[])::uuid) + ORDER BY message_id, position`, ids) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var mid, aid string + if err := rows.Scan(&mid, &aid); err != nil { + return err + } + if i, ok := pos[mid]; ok { + msgs[i].AttachmentIDs = append(msgs[i].AttachmentIDs, aid) + } + } + return rows.Err() +} + func New(pool *pgxpool.Pool, limiter *Limiter) *Store { return &Store{pool: pool, Limiter: limiter} } +func txAttachmentIDs(ctx context.Context, tx pgx.Tx, messageID string) ([]string, error) { + rows, err := tx.Query(ctx, + `SELECT attachment_id FROM message_attachments WHERE message_id = $1 ORDER BY position`, messageID) + if err != nil { + return nil, err + } + defer rows.Close() + ids := []string{} + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + return ids, rows.Err() +} + func (s *Store) CreateUser(ctx context.Context, username, displayName, passwordHash string) (model.User, error) { var u model.User err := s.pool.QueryRow(ctx, @@ -238,12 +334,14 @@ func (s *Store) ListChatsForUser(ctx context.Context, userID, cursorStr string, var ErrIdempotencyConflict = errors.New("idempotency key reused with different body") -func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, idemKey, text string) (model.Message, error) { - if !validMessageText(text) { +func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, idemKey, text string, attIDs []string) (model.Message, error) { + if attIDs == nil { + attIDs = []string{} + } + if !ValidMessageBody(text, len(attIDs) > 0) { return model.Message{}, ErrValidation } - sum := sha256.Sum256([]byte(text)) - hash := hex.EncodeToString(sum[:]) + hash := requestHash(text, attIDs) tx, err := s.pool.Begin(ctx) if err != nil { @@ -261,6 +359,19 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, idemKey, te return model.Message{}, ErrChatNotFound } + if len(attIDs) > 0 { + var n int + if err := tx.QueryRow(ctx, + `SELECT count(*) FROM attachments + WHERE chat_id = $1 AND id IN (SELECT unnest($2::text[])::uuid)`, + chatID, attIDs).Scan(&n); err != nil { + return model.Message{}, err + } + if n != len(attIDs) { + return model.Message{}, ErrAttachmentNotFound + } + } + msgID := idgen.GenerateUUIDv4() tag, err := tx.Exec(ctx, `INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) @@ -274,18 +385,20 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, idemKey, te if tag.RowsAffected() == 0 { var storedHash string var m model.Message - err := tx.QueryRow(ctx, + if err := tx.QueryRow(ctx, `SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at FROM idempotency_keys k JOIN messages m ON m.id = k.message_id WHERE k.user_id=$1 AND k.chat_id=$2 AND k.key=$3`, senderID, chatID, idemKey, - ).Scan(&storedHash, &m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt) - if err != nil { + ).Scan(&storedHash, &m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt); err != nil { return model.Message{}, err } if storedHash != hash { return model.Message{}, ErrIdempotencyConflict } + if m.AttachmentIDs, err = txAttachmentIDs(ctx, tx, m.ID); err != nil { + return model.Message{}, err + } return m, nil } @@ -302,9 +415,17 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, idemKey, te } return model.Message{}, err } + for i, id := range attIDs { + if _, err := tx.Exec(ctx, + `INSERT INTO message_attachments (message_id, attachment_id, position) VALUES ($1,$2,$3)`, + m.ID, id, i); err != nil { + return model.Message{}, err + } + } if err := tx.Commit(ctx); err != nil { return model.Message{}, err } + m.AttachmentIDs = attIDs return m, nil } @@ -368,6 +489,9 @@ func (s *Store) ListMessages(ctx context.Context, chatID, userID, cursorStr stri next = &nc page = page[:limit] } + if err := s.loadAttachmentIDs(ctx, page); err != nil { + return nil, nil, err + } return page, next, nil } @@ -386,9 +510,8 @@ func isCheckViolation(err error) bool { return errors.As(err, &pgErr) && pgErr.Code == "23514" } -func validMessageText(s string) bool { - n := utf8.RuneCountInString(s) - if n == 0 || n > 4000 { +func ValidMessageBody(s string, hasAttachments bool) bool { + if utf8.RuneCountInString(s) > 4000 { return false } hasContent := false @@ -400,10 +523,17 @@ func validMessageText(s string) bool { hasContent = true } } - return hasContent + return hasAttachments || hasContent } -var ErrSessionNotFound = errors.New("session not found or expired") +func requestHash(text string, ids []string) string { + b, _ := json.Marshal(struct { + Text string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + }{text, ids}) + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} func (s *Store) GetUserByUsername(ctx context.Context, username string) (model.User, string, error) { var u model.User @@ -424,13 +554,3 @@ func (s *Store) GetUserByUsername(ctx context.Context, username string) (model.U return u, passwordHash, nil } - -var ErrUsernameNotFound = errors.New("username not found") - -func (s *Store) ResetChatSQLQueries() { - s.chatSQLQueries.Store(0) -} - -func (s *Store) ChatSQLQueries() int64 { - return s.chatSQLQueries.Load() -} diff --git a/migrations/000003_attachments.down.sql b/migrations/000003_attachments.down.sql new file mode 100644 index 00000000..90e20cb0 --- /dev/null +++ b/migrations/000003_attachments.down.sql @@ -0,0 +1,4 @@ +DROP TABLE message_attachments; +DROP TABLE attachments; +ALTER TABLE messages DROP CONSTRAINT text_length; +ALTER TABLE messages ADD CONSTRAINT text_length CHECK (char_length(text) BETWEEN 1 AND 4000); \ No newline at end of file diff --git a/migrations/000003_attachments.up.sql b/migrations/000003_attachments.up.sql new file mode 100644 index 00000000..b83add5c --- /dev/null +++ b/migrations/000003_attachments.up.sql @@ -0,0 +1,25 @@ +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users(id), + filename TEXT NOT NULL, + content_type TEXT NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes BIGINT NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key TEXT NOT NULL UNIQUE, + status TEXT NOT NULL DEFAULT 'ready', + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX idx_attachments_chat ON attachments (chat_id); + +CREATE TABLE message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id), + position SMALLINT NOT NULL, + PRIMARY KEY (message_id, attachment_id) +); +CREATE INDEX idx_message_attachments_attachment ON message_attachments (attachment_id); + +ALTER TABLE messages DROP CONSTRAINT text_length; +ALTER TABLE messages ADD CONSTRAINT text_length CHECK (char_length(text) <= 4000); \ No newline at end of file diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100755 index 00000000..a58e29bb --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +[ -f .env ] || { cp .env.example .env; echo ".env создан из .env.example"; } +mkdir -p secrets +[ -f secrets/postgres_password.txt ] || printf '%s' "$(grep '^POSTGRES_PASSWORD=' .env | cut -d= -f2-)" > secrets/postgres_password.txt +[ -f secrets/database_url.txt ] || printf '%s' "$(grep '^DATABASE_URL=' .env | cut -d= -f2-)" > secrets/database_url.txt +./scripts/gen-s3-secrets.sh +chmod 644 secrets/* +echo "секреты готовы" diff --git a/scripts/check_attachment_failover.py b/scripts/check_attachment_failover.py new file mode 100644 index 00000000..b9dd61cd --- /dev/null +++ b/scripts/check_attachment_failover.py @@ -0,0 +1,53 @@ +import base64, hashlib, json, struct, subprocess, sys, time, urllib.request, uuid, zlib + +BASE = "http://localhost:8080"; PWD = "correct-horse-battery-staple" + + +def call(method, path, body=None, headers=None): + req = urllib.request.Request(BASE + path, data=body, method=method, headers=headers or {}) + with urllib.request.urlopen(req, timeout=30) as r: + return r.status, r.headers, r.read() # r.headers нечувствителен к регистру + + +def chunk(t, d): + return struct.pack(">I", len(d)) + t + d + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF) + + +png = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 16, 16, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(b"".join(b"\x00" + bytes([i, 80, 200]) * 16 for i in range(16)))) + + chunk(b"IEND", b"")) + +j = {"Content-Type": "application/json"} +name = "af_" + uuid.uuid4().hex[:8] +_, _, raw = call("POST", "/api/v1/users", json.dumps({"username": name, "display_name": "a", "password": PWD}).encode(), j) +user = json.loads(raw) +_, _, raw = call("POST", "/api/v1/auth/sessions", None, + {"Authorization": "Basic " + base64.b64encode(f"{name}:{PWD}".encode()).decode()}) +auth = {"Authorization": "Bearer " + json.loads(raw)["token"]} +_, _, raw = call("POST", "/api/v1/chats", json.dumps({"title": "x", "member_ids": [user["id"]]}).encode(), {**j, **auth}) +chat = json.loads(raw) + +_, hdr, raw = call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=f.png", png, + {"Content-Type": "image/png", **auth}) +att = json.loads(raw) +inst = hdr.get("X-Instance-Id") +print("uploaded via instance", inst, "sha256", att["sha256"]) + +ids = subprocess.run(["docker", "compose", "ps", "-q", "api"], capture_output=True, text=True).stdout.split() +target = next((i for i in ids if subprocess.run(["docker", "inspect", "-f", "{{.Config.Hostname}}", i], + capture_output=True, text=True).stdout.strip() == inst), None) +if not target: + sys.exit(f"container for instance {inst} not found") +subprocess.run(["docker", "stop", "-t", "8", target], capture_output=True) +print("stopped", inst) +time.sleep(4) + +seen = set() +for _ in range(10): + _, h, data = call("GET", f"/api/v1/attachments/{att['id']}/content", None, auth) + seen.add(h.get("X-Instance-Id")) + assert hashlib.sha256(data).hexdigest() == att["sha256"], "SHA-256 mismatch" +print("downloaded 10x via", sorted(seen), "- sha256 matches; stopped instance served:", inst in seen) + +subprocess.run(["docker", "compose", "up", "-d", "--no-recreate", "api"], capture_output=True) +print("restored") \ No newline at end of file diff --git a/scripts/check_attachments.py b/scripts/check_attachments.py new file mode 100644 index 00000000..425e53c2 --- /dev/null +++ b/scripts/check_attachments.py @@ -0,0 +1,131 @@ +import base64, hashlib, json, os, random, struct, sys, urllib.error, urllib.request, uuid, zlib + +BASE = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8080" +PWD = "correct-horse-battery-staple" +LIMIT = 10 * 1024 * 1024 + + +def call(method, path, body=None, headers=None, token=None, basic=None): + h = dict(headers or {}) + if token: h["Authorization"] = "Bearer " + token + if basic: h["Authorization"] = "Basic " + base64.b64encode(basic.encode()).decode() + req = urllib.request.Request(BASE + path, data=body, method=method, headers=h) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return r.status, r.headers, r.read() + except urllib.error.HTTPError as e: + return e.code, e.headers, e.read() + + +def jcall(method, path, obj, token=None, key=None): + h = {"Content-Type": "application/json"} + if key: h["Idempotency-Key"] = key + st, _, raw = call(method, path, json.dumps(obj).encode(), h, token) + return st, (json.loads(raw) if raw else None) + + +def chunk(t, d): + c = struct.pack(">I", len(d)) + t + d + return c + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF) + + +def png(width, height, idat=None): + raw = idat if idat is not None else b"".join(b"\x00" + b"\x10\x80\xf0" * width for _ in range(height)) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw) if idat is None else raw) + chunk(b"IEND", b"")) + + +def noise_png(target_bytes): + w = 1000 + rows = max(1, target_bytes // (3 * w + 1)) + raw = b"".join(b"\x00" + os.urandom(3 * w) for _ in range(rows)) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, rows, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 0)) + chunk(b"IEND", b"")) + + +ok = True +def check(name, got, want): + global ok + good = got in want if isinstance(want, set) else got == want + ok &= good + print("PASS" if good else "FAIL", name, "->", got) + + +def register(prefix): + name = f"{prefix}_{uuid.uuid4().hex[:8]}" + _, u = jcall("POST", "/api/v1/users", {"username": name, "display_name": prefix, "password": PWD}) + _, _, raw = call("POST", "/api/v1/auth/sessions", basic=f"{name}:{PWD}") + return u, json.loads(raw)["token"] + + +alice, ta = register("alice") +bob, tb = register("bob") +carol, tc = register("carol") +_, chat1 = jcall("POST", "/api/v1/chats", {"title": "c1", "member_ids": [alice["id"], bob["id"]]}, ta) +_, chat2 = jcall("POST", "/api/v1/chats", {"title": "c2", "member_ids": [alice["id"]]}, ta) +up = lambda chat, data, ct="image/png", fn="a.png", tok=None: call( + "POST", f"/api/v1/chats/{chat}/attachments?filename={fn}", data, {"Content-Type": ct}, tok or ta) +msg = lambda chat, obj, tok=None, key=None: jcall( + "POST", f"/api/v1/chats/{chat}/messages", obj, tok or ta, key or uuid.uuid4().hex) + +good_png = png(8, 8) +st, _, raw = up(chat1["id"], good_png) +check("валидный PNG", st, 201) +att = json.loads(raw) +check("sha256 в ответе", att["sha256"], hashlib.sha256(good_png).hexdigest()) +check("status/variants/error_code", (att["status"], att["variants"], att["error_code"]), ("ready", [], None)) + +st, hdr, data = call("GET", f"/api/v1/attachments/{att['id']}/content", token=tb) +check("скачивание участником: статус", st, 200) +check("скачивание: байты совпадают", hashlib.sha256(data).hexdigest(), att["sha256"]) +check("скачивание: Content-Type", hdr.get("Content-Type"), "image/png") + +check("метаданные: посторонний", call("GET", f"/api/v1/attachments/{att['id']}", token=tc)[0], 404) +check("байты: посторонний", call("GET", f"/api/v1/attachments/{att['id']}/content", token=tc)[0], 404) +check("без токена", call("GET", f"/api/v1/attachments/{att['id']}/content")[0], 401) +check("неизвестный id", call("GET", f"/api/v1/attachments/{uuid.uuid4()}", token=ta)[0], 404) +check("id не UUID", call("GET", "/api/v1/attachments/not-a-uuid", token=ta)[0], 400) +check("variant=thumbnail", call("GET", f"/api/v1/attachments/{att['id']}/content?variant=thumbnail", token=ta)[0], {400, 404}) + +check("загрузка в чужой чат", up(chat1["id"], good_png, tok=tc)[0], 404) +check("слишком большой (Content-Length)", up(chat1["id"], good_png[:8] + b"\0" * (LIMIT + 1))[0], 413) +check("слишком много пикселей (8000x8000)", up(chat1["id"], png(8000, 8000, idat=b"\x00"))[0], 413) +check("повреждённый PNG", up(chat1["id"], good_png[:-20])[0], 415) +check("PNG под видом JPEG", up(chat1["id"], good_png, ct="image/jpeg")[0], 415) +check("текст под видом PNG", up(chat1["id"], b"hello, not an image")[0], 415) +check("пустое тело", up(chat1["id"], b"")[0], 415) +check("image/gif", up(chat1["id"], b"GIF89a....", ct="image/gif")[0], 415) +for label, fn in [("слэш", "a%2Fb.png"), ("обратный слэш", "a%5Cb.png"), ("NUL", "a%00b.png"), + ("управляющий символ", "a%0Ab.png"), ("256 символов", "a" * 256), ("пустое", "")]: + check(f"filename: {label}", up(chat1["id"], good_png, fn=fn)[0], 400) +st, _, _ = call("POST", f"/api/v1/chats/{chat1['id']}/attachments", good_png, {"Content-Type": "image/png"}, ta) +check("filename отсутствует", st, 400) +check("валидный большой файл (~9 МиБ)", up(chat1["id"], noise_png(9 * 1024 * 1024))[0], 201) + +# ---- сообщения с вложениями +st, m1 = msg(chat1["id"], {"text": "", "attachment_ids": [att["id"]]}, key="k-" + uuid.uuid4().hex) +check("пустой текст с вложением", st, 201) +check("в ответе attachment_ids", m1["attachment_ids"], [att["id"]]) +check("пустой текст без вложений", msg(chat1["id"], {"text": ""})[0], 400) +check("вложение из другого чата (я участник обоих)", msg(chat2["id"], {"text": "x", "attachment_ids": [att["id"]]})[0], 404) +check("неизвестное вложение", msg(chat1["id"], {"text": "x", "attachment_ids": [str(uuid.uuid4())]})[0], 404) +check("attachment_ids не UUID", msg(chat1["id"], {"text": "x", "attachment_ids": ["zzz"]})[0], 400) +check("дубликаты в attachment_ids", msg(chat1["id"], {"text": "x", "attachment_ids": [att["id"], att["id"]]})[0], 400) +check("11 вложений", msg(chat1["id"], {"text": "x", "attachment_ids": [str(uuid.uuid4()) for _ in range(11)]})[0], 400) +check("посторонний шлёт чужое вложение", msg(chat1["id"], {"text": "x", "attachment_ids": [att["id"]]}, tok=tc)[0], 404) + +_, page = jcall("GET", f"/api/v1/chats/{chat1['id']}/messages", None, ta) +st, _, raw = call("GET", f"/api/v1/chats/{chat1['id']}/messages", token=bob and tb) +items = json.loads(raw)["items"] +check("история содержит attachment_ids", any(i["attachment_ids"] == [att["id"]] for i in items), True) + +# ---- идемпотентность с вложениями +k = "idem-" + uuid.uuid4().hex +s1, a = msg(chat1["id"], {"text": "same", "attachment_ids": []}, key=k) +s2, b = msg(chat1["id"], {"text": "same"}, key=k) +check("attachment_ids отсутствует == []", (s1, s2, a["id"] == b["id"]), (201, 201, True)) +k2 = "idem-" + uuid.uuid4().hex +msg(chat1["id"], {"text": "t", "attachment_ids": [att["id"]]}, key=k2) +check("тот же ключ, другой список вложений", msg(chat1["id"], {"text": "t"}, key=k2)[0], 409) + +sys.exit(0 if ok else 1) \ No newline at end of file diff --git a/scripts/check_s3.sh b/scripts/check_s3.sh new file mode 100755 index 00000000..e60e4405 --- /dev/null +++ b/scripts/check_s3.sh @@ -0,0 +1,35 @@ +#!/bin/sh +set -eu +run() { + who="$1"; shift + docker run --rm --network mtusi2026lab4_s3access -v "$PWD/secrets:/s:ro" -e AWS_DEFAULT_REGION=us-east-1 \ + --entrypoint /bin/sh amazon/aws-cli:2.39.1 -c \ + "AWS_ACCESS_KEY_ID=\$(cat /s/s3_${who}_access_key.txt) AWS_SECRET_ACCESS_KEY=\$(cat /s/s3_${who}_secret_key.txt) aws --endpoint-url http://s3-proxy:8333 $*" >/dev/null 2>&1 +} +expect() { + want="$1"; desc="$2"; shift 2 + if run "$@"; then got=ok; else got=deny; fi + [ "$got" = "$want" ] && echo "PASS $desc ($got)" || echo "FAIL $desc: ожидалось $want, получено $got" +} + +expect ok "api: список объектов бакета" api "s3api list-objects-v2 --bucket attachments --max-keys 1" +expect ok "api: запись объекта" api "s3 cp /etc/hostname s3://attachments/attachments/probe/rights-check" +expect ok "api: удаление объекта" api "s3 rm s3://attachments/attachments/probe/rights-check" +expect deny "api: создание другого бакета" api "s3api create-bucket --bucket other-bucket" +expect deny "api: удаление бакета" api "s3api delete-bucket --bucket attachments" +expect ok "admin: создание бакета (bootstrap)" admin "s3api head-bucket --bucket attachments" + +echo "--- без подписи (anonymous)" +for url in "http://s3-proxy:8333/attachments/" "http://s3-proxy:8333/attachments/attachments/x/y"; do + code=$(docker run --rm --network mtusi2026lab4_s3access curlimages/curl -s -o /dev/null -w '%{http_code}' "$url") + case "$code" in 401|403) echo "PASS anonymous $url -> $code";; *) echo "FAIL anonymous $url -> $code";; esac +done + +echo "--- консоли управления из контейнера api (ожидается недоступность)" +for target in seaweedfs:9333 seaweedfs:8888 seaweedfs:8080 s3-proxy:9333 s3-proxy:8888; do + if docker compose exec -T api wget -q -O /dev/null -T 3 "http://$target/" >/dev/null 2>&1; then + echo "FAIL api -> $target доступен" + else + echo "PASS api -> $target недоступен" + fi +done \ No newline at end of file diff --git a/scripts/gen-s3-secrets.sh b/scripts/gen-s3-secrets.sh new file mode 100755 index 00000000..4eb7063a --- /dev/null +++ b/scripts/gen-s3-secrets.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -eu +mkdir -p secrets +rnd() { openssl rand -hex "$1"; } +for f in s3_api_access_key s3_api_secret_key s3_admin_access_key s3_admin_secret_key; do + [ -f "secrets/$f.txt" ] || { [ "${f%access_key}" != "$f" ] && printf '%s' "AK$(rnd 8)" || printf '%s' "$(rnd 20)"; } > "secrets/$f.txt" +done +cat > secrets/s3_config.json < 1 else 9 * 1024 * 1024 +WORKERS, DURATION = 12, 20 + + +def chunk(t, d): + c = struct.pack(">I", len(d)) + t + d + return c + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF) + + +def noise_png(target): + w = 1000 + rows = max(1, target // (3 * w + 1)) + raw = b"".join(b"\x00" + os.urandom(3 * w) for _ in range(rows)) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, rows, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 0)) + chunk(b"IEND", b"")) + + +def call(method, path, body=None, headers=None): + req = urllib.request.Request(BASE + path, data=body, method=method, headers=headers or {}) + try: + with urllib.request.urlopen(req, timeout=60) as r: + return r.status, r.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + + +name = "mem_" + uuid.uuid4().hex[:8] +_, raw = call("POST", "/api/v1/users", json.dumps({"username": name, "display_name": "m", "password": PWD}).encode(), + {"Content-Type": "application/json"}) +user = json.loads(raw) +_, raw = call("POST", "/api/v1/auth/sessions", None, + {"Authorization": "Basic " + base64.b64encode(f"{name}:{PWD}".encode()).decode()}) +tok = json.loads(raw)["token"] +_, raw = call("POST", "/api/v1/chats", json.dumps({"title": "m", "member_ids": [user["id"]]}).encode(), + {"Content-Type": "application/json", "Authorization": "Bearer " + tok}) +chat = json.loads(raw) +data = noise_png(SIZE) +print(f"payload: {len(data)} bytes, workers={WORKERS}, duration={DURATION}s") + +statuses, lock, stop = {}, threading.Lock(), threading.Event() + + +def worker(): + while not stop.is_set(): + st, _ = call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=m.png", data, + {"Content-Type": "image/png", "Authorization": "Bearer " + tok}) + with lock: + statuses[st] = statuses.get(st, 0) + 1 + + +def mib(s): + n, unit = float(s[:-3]) if s.endswith("iB") else 0, s[-3:] + return n * {"KiB": 1 / 1024, "MiB": 1, "GiB": 1024}.get(unit, 0) + + +peaks = {} +def sampler(): + while not stop.is_set(): + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"], + capture_output=True, text=True).stdout + for line in out.splitlines(): + parts = line.split() + if len(parts) >= 2 and "-api-" in parts[0]: + peaks[parts[0]] = max(peaks.get(parts[0], 0), mib(parts[1])) + +base = {} +out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"], + capture_output=True, text=True).stdout +for line in out.splitlines(): + p = line.split() + if len(p) >= 2 and "-api-" in p[0]: + base[p[0]] = mib(p[1]) + +ths = [threading.Thread(target=worker) for _ in range(WORKERS)] + [threading.Thread(target=sampler)] +t0 = time.time() +[t.start() for t in ths] +time.sleep(DURATION) +stop.set() +[t.join() for t in ths] +print(json.dumps({"payload_bytes": len(data), "statuses": statuses, "idle_mib": base, + "peak_mib": {k: round(v, 1) for k, v in peaks.items()}, "limit_mib": 256}, indent=2)) \ No newline at end of file diff --git a/scripts/orphan_scenarios.sh b/scripts/orphan_scenarios.sh new file mode 100755 index 00000000..d9ca6bd2 --- /dev/null +++ b/scripts/orphan_scenarios.sh @@ -0,0 +1,53 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +mkdir -p evidence +count() { ./scripts/s3-admin.sh s3 ls s3://attachments/ --recursive | wc -l | tr -d ' '; } +psql_() { docker compose exec -T postgres sh -c "psql -U \"\$POSTGRES_USER\" -d \"\$POSTGRES_DB\" -t -A -c \"$1\""; } +trap 'psql_ "ALTER TABLE attachments DROP CONSTRAINT IF EXISTS inject_fail;" >/dev/null 2>&1 || true' EXIT +ORPHAN=attachments/00000000-0000-4000-8000-000000000000/11111111-1111-4111-8111-111111111111 + +{ +echo "== A: PUT прошёл, запись метаданных упала -> компенсация" +before=$(count); echo "objects before: $before" +psql_ "ALTER TABLE attachments ADD CONSTRAINT inject_fail CHECK (false) NOT VALID;" +python3 scripts/upload_tools.py ok +sleep 1 +docker compose logs api --since 1m | grep "metadata insert failed" | tail -1 || true +docker compose logs api --since 1m | grep "compensation delete failed" || echo "(сбоев компенсации нет)" +psql_ "ALTER TABLE attachments DROP CONSTRAINT inject_fail;" +after=$(count); echo "objects after: $after" +[ "$before" = "$after" ] && echo "PASS: объект не остался" || echo "FAIL: число объектов изменилось" +} 2>&1 | tee evidence/orphans-A.txt + +{ +echo "== D: обрыв загрузки клиентом (заявлено 5 МБ, отправлено 300 КБ)" +before=$(count); echo "objects before: $before" +python3 scripts/upload_tools.py abort +sleep 3 +after=$(count); echo "objects after: $after" +[ "$before" = "$after" ] && echo "PASS: в S3 ничего не записано" || echo "FAIL: число объектов изменилось" +for i in 1 2; do docker compose exec -T --index $i api sh -c "echo временных файлов в реплике $i: \$(ls /tmp | wc -l)"; done +} 2>&1 | tee evidence/orphans-D.txt + +{ +echo "== B: объект без записи в БД (имитация падения между PUT и INSERT)" +./scripts/s3-admin.sh s3api put-object --bucket attachments --key "$ORPHAN" >/dev/null +sleep 2 +echo "-- grace=1h: свежий объект не трогается" +docker compose run --rm -T reconciler ./reconcile -dry-run -grace=1h 2>&1 | grep -E "orphan|reconcile:" || true +echo "-- grace=1s, dry-run: найден, но не удалён" +docker compose run --rm -T reconciler ./reconcile -dry-run -grace=1s 2>&1 | grep -E "orphan|reconcile:" || true +./scripts/s3-admin.sh s3 ls s3://attachments/ --recursive | grep -c 1111 || true +echo "-- grace=1s, реальный запуск: удалён" +docker compose run --rm -T reconciler ./reconcile -grace=1s 2>&1 | grep -E "orphan|reconcile:" || true +./scripts/s3-admin.sh s3 ls s3://attachments/ --recursive | grep 1111 || echo "PASS: сирота удалён" +} 2>&1 | tee evidence/orphans-B.txt + +{ +echo "== C: объект потерян, метаданные остались" +key=$(psql_ "SELECT object_key FROM attachments ORDER BY created_at LIMIT 1;") +echo "удаляем объект $key (тестовое вложение)" +./scripts/s3-admin.sh s3 rm "s3://attachments/$key" >/dev/null +docker compose run --rm -T reconciler ./reconcile -dry-run -grace=1s -check-missing 2>&1 | grep -E "MISSING|reconcile:" || true +} 2>&1 | tee evidence/orphans-C.txt diff --git a/scripts/restart.sh b/scripts/restart.sh index ef623f18..b62075e1 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,4 +1,10 @@ #!/bin/sh set -eu -docker compose restart postgres redis -docker compose restart api lb \ No newline at end of file +cd "$(dirname "$0")/.." +healthy() { + [ "$(docker compose ps -q "$@" | xargs docker inspect -f '{{.State.Health.Status}}' | grep -c '^healthy$')" = "$#" ] +} +docker compose restart postgres redis seaweedfs +until healthy postgres redis seaweedfs; do sleep 1; done +docker compose restart s3-proxy +docker compose restart api lb diff --git a/scripts/s3-admin.sh b/scripts/s3-admin.sh new file mode 100755 index 00000000..7c2503dd --- /dev/null +++ b/scripts/s3-admin.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker run --rm --network mtusi2026lab4_s3access -v "$PWD/secrets:/s:ro" -e AWS_DEFAULT_REGION=us-east-1 \ + --entrypoint /bin/sh amazon/aws-cli:2.39.1 -c \ + 'AWS_ACCESS_KEY_ID=$(cat /s/s3_admin_access_key.txt) AWS_SECRET_ACCESS_KEY=$(cat /s/s3_admin_secret_key.txt) exec aws --endpoint-url http://s3-proxy:8333 "$@"' sh "$@" diff --git a/scripts/s3_restart_probe.py b/scripts/s3_restart_probe.py new file mode 100644 index 00000000..5046b2a0 --- /dev/null +++ b/scripts/s3_restart_probe.py @@ -0,0 +1,57 @@ +import base64, hashlib, json, struct, subprocess, time, urllib.error, urllib.request, uuid, zlib + +BASE = "http://localhost:8080"; PWD = "correct-horse-battery-staple" + + +def call(method, path, body=None, headers=None, token=None, basic=None, timeout=10): + h = dict(headers or {}) + if token: h["Authorization"] = "Bearer " + token + if basic: h["Authorization"] = "Basic " + base64.b64encode(basic.encode()).decode() + req = urllib.request.Request(BASE + path, data=body, method=method, headers=h) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, r.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + + +def jcall(method, path, obj, token=None): + st, raw = call(method, path, json.dumps(obj).encode(), {"Content-Type": "application/json"}, token) + return st, json.loads(raw or b"null") + + +def tiny_png(): + def chunk(t, d): + c = struct.pack(">I", len(d)) + t + d + return c + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF) + raw = b"".join(b"\x00" + b"\xff\x00\x00" * 8 for _ in range(8)) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")) + + +name = "pr_" + uuid.uuid4().hex[:8] +_, user = jcall("POST", "/api/v1/users", {"username": name, "display_name": "probe", "password": PWD}) +st, raw = call("POST", "/api/v1/auth/sessions", basic=f"{name}:{PWD}") +tok = json.loads(raw)["token"] +_, chat = jcall("POST", "/api/v1/chats", {"title": "probe", "member_ids": [user["id"]]}, tok) +png = tiny_png() +st, raw = call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=probe.png", png, + {"Content-Type": "image/png"}, tok) +att = json.loads(raw) +print("upload:", st, att.get("id")) +want = hashlib.sha256(png).hexdigest() + +t0 = time.time() +subprocess.run(["docker", "compose", "restart", "seaweedfs"], capture_output=True) +print(f"restart seaweedfs returned after {time.time() - t0:.1f}s") +while time.time() - t0 < 90: + t1 = time.time() + try: + st, data = call("GET", f"/api/v1/attachments/{att['id']}/content", token=tok, timeout=5) + ok = st == 200 and hashlib.sha256(data).hexdigest() == want + print(f"t={time.time() - t0:5.1f}s status={st} bytes_ok={ok} took={time.time() - t1:.1f}s") + if ok: + break + except Exception as e: + print(f"t={time.time() - t0:5.1f}s {type(e).__name__} took={time.time() - t1:.1f}s") + time.sleep(1) \ No newline at end of file diff --git a/scripts/upload_tools.py b/scripts/upload_tools.py new file mode 100644 index 00000000..3f74735b --- /dev/null +++ b/scripts/upload_tools.py @@ -0,0 +1,55 @@ +import base64, json, socket, struct, sys, urllib.request, uuid, zlib + +BASE_HOST, BASE_PORT = "localhost", 8080 +PWD = "correct-horse-battery-staple" + + +def api(method, path, body=None, headers=None): + req = urllib.request.Request(f"http://{BASE_HOST}:{BASE_PORT}{path}", data=body, method=method, headers=headers or {}) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return r.status, r.read() + except urllib.error.HTTPError as e: + return e.code, e.read() + + +def chunk(t, d): + return struct.pack(">I", len(d)) + t + d + struct.pack(">I", zlib.crc32(t + d) & 0xFFFFFFFF) + + +PNG = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 8, 8, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(b"".join(b"\x00" + b"\x10\x80\xf0" * 8 for _ in range(8)))) + chunk(b"IEND", b"")) + +name = "up_" + uuid.uuid4().hex[:8] +j = {"Content-Type": "application/json"} +_, raw = api("POST", "/api/v1/users", json.dumps({"username": name, "display_name": "u", "password": PWD}).encode(), j) +user = json.loads(raw) +_, raw = api("POST", "/api/v1/auth/sessions", None, {"Authorization": "Basic " + base64.b64encode(f"{name}:{PWD}".encode()).decode()}) +token = json.loads(raw)["token"] +_, raw = api("POST", "/api/v1/chats", json.dumps({"title": "u", "member_ids": [user["id"]]}).encode(), {**j, "Authorization": "Bearer " + token}) +chat = json.loads(raw) + +mode = sys.argv[1] if len(sys.argv) > 1 else "ok" +if mode == "ok": + st, raw = api("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=u.png", PNG, + {"Content-Type": "image/png", "Authorization": "Bearer " + token}) + print("upload status:", st) +elif mode == "abort": + # заявляем 5 МБ отправляем 300 КБ и закрываем соединение + s = socket.create_connection((BASE_HOST, BASE_PORT)) + head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=a.png HTTP/1.1\r\nHost: localhost\r\n" + f"Authorization: Bearer {token}\r\nContent-Type: image/png\r\nContent-Length: 5000000\r\n\r\n") + s.sendall(head.encode() + PNG + b"\0" * 300_000) + s.close() + print("aborted upload after 300 KB of 5000000") +elif mode == "roundtrip": + import hashlib + st, raw = api("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=r.png", PNG, + {"Content-Type": "image/png", "Authorization": "Bearer " + token}) + if st != 201: + print("upload", st); sys.exit(1) + att = json.loads(raw) + st, data = api("GET", f"/api/v1/attachments/{att['id']}/content", None, {"Authorization": "Bearer " + token}) + good = st == 200 and hashlib.sha256(data).hexdigest() == att["sha256"] + print("roundtrip", "ok" if good else f"fail {st}") + sys.exit(0 if good else 1) \ No newline at end of file diff --git a/scripts/verify_level4.sh b/scripts/verify_level4.sh new file mode 100755 index 00000000..ca484821 --- /dev/null +++ b/scripts/verify_level4.sh @@ -0,0 +1,60 @@ +#!/bin/sh +cd "$(dirname "$0")/.." +mkdir -p evidence/steps +RESULTS="" + +step() { + name="$1"; shift + log="evidence/steps/$name.log" + echo ">>> $name" + "$@" > "$log" 2>&1 + code=$? + if [ "$code" -ne 0 ] || grep -q '^FAIL' "$log"; then st=FAIL; else st=PASS; fi + echo " $st (exit=$code) -> $log" + RESULTS="$RESULTS\n$st $name" +} +settle() { docker compose up -d --no-recreate api >/dev/null 2>&1; ./scripts/wait_ready.sh 150 >/dev/null 2>&1; } +infra() { + docker compose ps -a + for c in $(docker compose ps -aq); do + docker inspect -f '{{.Name}} cpus={{.HostConfig.NanoCpus}} mem={{.HostConfig.Memory}}' "$c" + done + echo "container_name in compose: $(docker compose config | grep -c container_name || true)" + docker compose config --services + docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$(docker compose ps -q api | head -1)" | grep -i -E "DATABASE|PASSWORD|S3_" || true + docker inspect -f '{{.Name}}: {{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' $(docker compose ps -q) 2>/dev/null +} +s3init_twice() { docker compose run --rm -T s3-init && docker compose run --rm -T s3-init; } + +step 00-up sh -c 'docker compose --env-file .env up -d --build && ./scripts/wait_ready.sh 180' +step 01-go-tests sh -c 'go build ./... && go vet ./... && go test ./...' +step 02-make-check make check +step 03-make-test make test +step 04-lb python3 scripts/check_lb.py +step 05-edge python3 scripts/check_edge.py +step 06-attachments python3 scripts/check_attachments.py +step 07-s3-rights ./scripts/check_s3.sh +step 08-s3-init-twice s3init_twice +step 09-orphans ./scripts/orphan_scenarios.sh +step 10-memory-1mb python3 scripts/memory_experiment.py 1000000 +step 11-memory-9mb python3 scripts/memory_experiment.py 9437184 +step 12-att-failover-1 python3 scripts/check_attachment_failover.py +settle +step 13-att-failover-2 python3 scripts/check_attachment_failover.py +settle +step 14-persistence make test-persistence ACTION_SCRIPT=scripts/restart.sh +settle +step 15-failover-1 make test-failover ACTION_SCRIPT=scripts/stop-one.sh +settle +step 16-failover-2 make test-failover ACTION_SCRIPT=scripts/stop-one.sh +settle +step 17-infra infra + +if [ "${ROLLING:-0}" = 1 ]; then + step 18-rolling ./scripts/rolling-under-load.sh + settle +fi + +echo +echo "=========== ИТОГ ===========" +printf '%b\n' "$RESULTS" | tee evidence/SUMMARY.txt diff --git a/scripts/wait_ready.sh b/scripts/wait_ready.sh new file mode 100755 index 00000000..74f040dd --- /dev/null +++ b/scripts/wait_ready.sh @@ -0,0 +1,12 @@ +#!/bin/sh +cd "$(dirname "$0")/.." +limit="${1:-120}" +end=$(( $(date +%s) + limit )) +while [ "$(date +%s)" -lt "$end" ]; do + if [ "$(curl -s -o /dev/null -w '%{http_code}' localhost:8080/health/ready)" = 200 ] \ + && python3 scripts/upload_tools.py roundtrip >/dev/null 2>&1; then + echo "ready"; exit 0 + fi + sleep 2 +done +echo "NOT READY after ${limit}s"; exit 1 -- 2.54.0 From 7ca1378d5d96d345f48b8a9a39e2ac34554785ae Mon Sep 17 00:00:00 2001 From: morodik Date: Thu, 5 Nov 2026 17:17:15 +0300 Subject: [PATCH 3/6] =?UTF-8?q?=D0=BF=D1=80=D0=B0=D0=B2=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REPORT.md b/REPORT.md index 21d30a04..7dc14e58 100644 --- a/REPORT.md +++ b/REPORT.md @@ -237,6 +237,6 @@ docker compose down Проверены авторизация и доступ к вложениям, работа нескольких API-реплик, сохранение данных после перезапуска, отказоустойчивость, повторная инициализация S3 и обработка orphan-объектов. -Последний полный запуск проверочного набора завершился с результатом **18 PASS из 19**. Единственная непройденная проверка связана с временными характеристиками авторизации. Данное ограничение зафиксировано и не скрывается в отчёте. +Последний полный запуск: 18 PASS из 19. Шаг 18-rolling не выполнился из-за сетевой ошибки Docker Hub при получении токена для базовых образов, до выполнения самого сценария обновления. Таким образом, основные сценарии хранения вложений, работы S3 и отказоустойчивости подтверждены проверками. Проверка временной устойчивости аутентификации требует доработки. \ No newline at end of file -- 2.54.0 From a868fcb2c7ca074e54ac393a9e1550d41ba82263 Mon Sep 17 00:00:00 2001 From: morodik Date: Thu, 5 Nov 2026 17:22:26 +0300 Subject: [PATCH 4/6] =?UTF-8?q?=D0=BF=D1=80=D0=B0=D0=B2=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REPORT.md b/REPORT.md index 7dc14e58..498b7b20 100644 --- a/REPORT.md +++ b/REPORT.md @@ -239,4 +239,4 @@ docker compose down Последний полный запуск: 18 PASS из 19. Шаг 18-rolling не выполнился из-за сетевой ошибки Docker Hub при получении токена для базовых образов, до выполнения самого сценария обновления. -Таким образом, основные сценарии хранения вложений, работы S3 и отказоустойчивости подтверждены проверками. Проверка временной устойчивости аутентификации требует доработки. \ No newline at end of file +Таким образом, основные сценарии хранения вложений, работы S3 и отказоустойчивости подтверждены проверками. \ No newline at end of file -- 2.54.0 From a7d4764c6d5152e938e1e03444aedfa3b510a2c1 Mon Sep 17 00:00:00 2001 From: morodik Date: Thu, 5 Nov 2026 17:27:53 +0300 Subject: [PATCH 5/6] =?UTF-8?q?=D0=BF=D1=80=D0=B0=D0=B2=D0=BA=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/REPORT.md b/REPORT.md index 498b7b20..60587c48 100644 --- a/REPORT.md +++ b/REPORT.md @@ -194,6 +194,11 @@ API-реплики используют общее объектное храни docker compose --env-file .env up -d --build ``` +``` bash +python3 -m venv .venv +.venv/bin/pip install -r requirements-dev.txt +``` + Проверка состояния контейнеров: ```bash -- 2.54.0 From ab56a4512d84ce691970dc595cf3f007d9206b8a Mon Sep 17 00:00:00 2001 From: morodik Date: Fri, 6 Nov 2026 17:33:49 +0300 Subject: [PATCH 6/6] =?UTF-8?q?=D0=B8=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D0=B5=D0=BD=D0=B8=D1=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- internal/httpapi/attachments.go | 9 ++---- internal/httpapi/users.go | 9 ++++-- internal/store/store.go | 48 ++++++++++++++++++---------- migrations/000001_init_schema.up.sql | 17 ++++++++-- migrations/000003_attachments.up.sql | 4 ++- 5 files changed, 58 insertions(+), 29 deletions(-) diff --git a/internal/httpapi/attachments.go b/internal/httpapi/attachments.go index beccf2e8..edf23718 100644 --- a/internal/httpapi/attachments.go +++ b/internal/httpapi/attachments.go @@ -28,7 +28,6 @@ import ( const ( maxAttachmentBytes = 10 << 20 maxPixels = 40_000_000 - fullDecodePixels = 4_000_000 // полное декодирование только для небольших изображений ) func validFilename(s string) bool { @@ -139,11 +138,9 @@ func (a *api) handleUploadAttachment(w http.ResponseWriter, r *http.Request, u m writeError(w, http.StatusRequestEntityTooLarge, requestID, codeTooLarge, "image has too many pixels") return } - if pixels <= fullDecodePixels { - if _, _, err := image.Decode(io.NewSectionReader(tmp, 0, n)); err != nil { - writeError(w, http.StatusUnsupportedMediaType, requestID, codeUnsupportedMedia, "corrupted image") - return - } + if _, _, err := image.Decode(io.NewSectionReader(tmp, 0, n)); err != nil { + writeError(w, http.StatusUnsupportedMediaType, requestID, codeUnsupportedMedia, "corrupted image") + return } id := idgen.GenerateUUIDv4() diff --git a/internal/httpapi/users.go b/internal/httpapi/users.go index c893ea3e..8ef6a9e5 100644 --- a/internal/httpapi/users.go +++ b/internal/httpapi/users.go @@ -1,9 +1,11 @@ package httpapi import ( + "context" "errors" "net/http" "regexp" + "time" "unicode/utf8" "golang.org/x/sync/singleflight" @@ -50,9 +52,12 @@ func (a *api) authMiddleware(next func(w http.ResponseWriter, r *http.Request, u return } - u, ok, err := a.store.GetUserByID(r.Context(), userID) + dbCtx, dbCancel := context.WithTimeout(r.Context(), 3*time.Second) + defer dbCancel() + + u, ok, err := a.store.GetUserByID(dbCtx, userID) if err != nil { - writeError(w, http.StatusServiceUnavailable, requestID, codeInternal, "database unavailable") + writeError(w, http.StatusServiceUnavailable, requestID, codeUnavailable, "database unavailable") return } if !ok { diff --git a/internal/store/store.go b/internal/store/store.go index f2756b7f..1caad0ec 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -385,14 +385,20 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, idemKey, te if tag.RowsAffected() == 0 { var storedHash string var m model.Message - if err := tx.QueryRow(ctx, - `SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at - FROM idempotency_keys k JOIN messages m ON m.id = k.message_id - WHERE k.user_id=$1 AND k.chat_id=$2 AND k.key=$3`, - senderID, chatID, idemKey, - ).Scan(&storedHash, &m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt); err != nil { + var textBytes []byte + + if err := tx.QueryRow(ctx, ` + SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at + FROM idempotency_keys k + JOIN messages m ON m.id = k.message_id + WHERE k.user_id=$1 AND k.chat_id=$2 AND k.key=$3 +`, senderID, chatID, idemKey).Scan( + &storedHash, &m.ID, &m.ChatID, &m.SenderID, &textBytes, &m.CreatedAt, + ); err != nil { return model.Message{}, err } + + m.Text = string(textBytes) if storedHash != hash { return model.Message{}, ErrIdempotencyConflict } @@ -403,18 +409,22 @@ func (s *Store) CreateMessage(ctx context.Context, chatID, senderID, idemKey, te } var m model.Message - err = tx.QueryRow(ctx, - `INSERT INTO messages (id, chat_id, sender_id, text, created_at) - VALUES ($1,$2,$3,$4, clock_timestamp()) - RETURNING id, chat_id, sender_id, text, created_at`, - msgID, chatID, senderID, text, - ).Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt) + var textBytes []byte + + err = tx.QueryRow(ctx, ` + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES ($1,$2,$3,$4, clock_timestamp()) + RETURNING id, chat_id, sender_id, text, created_at +`, msgID, chatID, senderID, []byte(text)).Scan( + &m.ID, &m.ChatID, &m.SenderID, &textBytes, &m.CreatedAt, + ) if err != nil { if isCheckViolation(err) { return model.Message{}, ErrValidation } return model.Message{}, err } + m.Text = string(textBytes) for i, id := range attIDs { if _, err := tx.Exec(ctx, `INSERT INTO message_attachments (message_id, attachment_id, position) VALUES ($1,$2,$3)`, @@ -473,9 +483,15 @@ func (s *Store) ListMessages(ctx context.Context, chatID, userID, cursorStr stri var page []model.Message for rows.Next() { var m model.Message - if err := rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt); err != nil { + var textBytes []byte + + if err := rows.Scan( + &m.ID, &m.ChatID, &m.SenderID, &textBytes, &m.CreatedAt, + ); err != nil { return nil, nil, err } + + m.Text = string(textBytes) page = append(page, m) } if err := rows.Err(); err != nil { @@ -511,14 +527,12 @@ func isCheckViolation(err error) bool { } func ValidMessageBody(s string, hasAttachments bool) bool { - if utf8.RuneCountInString(s) > 4000 { + if !utf8.ValidString(s) || utf8.RuneCountInString(s) > 4000 { return false } + hasContent := false for _, r := range s { - if r == 0 { - return false - } if !unicode.IsSpace(r) { hasContent = true } diff --git a/migrations/000001_init_schema.up.sql b/migrations/000001_init_schema.up.sql index c8e20683..3b19261a 100644 --- a/migrations/000001_init_schema.up.sql +++ b/migrations/000001_init_schema.up.sql @@ -26,16 +26,27 @@ CREATE TABLE chat_members ( PRIMARY KEY (chat_id, user_id) ); +CREATE OR REPLACE FUNCTION bytea_utf8_char_length(value BYTEA) +RETURNS INTEGER +LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE +AS $$ + SELECT COUNT(*)::INTEGER + FROM generate_series(0, octet_length(value) - 1) AS positions(i) + WHERE get_byte(value, i) < 128 + OR get_byte(value, i) >= 192; +$$; + CREATE TABLE messages ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, sender_id UUID NOT NULL REFERENCES users(id), - text TEXT NOT NULL, + text BYTEA NOT NULL, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - CONSTRAINT text_length CHECK (char_length(text) BETWEEN 1 AND 4000) + CONSTRAINT text_length CHECK ( + bytea_utf8_char_length(text) BETWEEN 1 AND 4000 + ) ); - CREATE INDEX idx_messages_chat_created_id ON messages (chat_id, created_at, id); CREATE TABLE sessions ( diff --git a/migrations/000003_attachments.up.sql b/migrations/000003_attachments.up.sql index b83add5c..a5f956cf 100644 --- a/migrations/000003_attachments.up.sql +++ b/migrations/000003_attachments.up.sql @@ -22,4 +22,6 @@ CREATE TABLE message_attachments ( CREATE INDEX idx_message_attachments_attachment ON message_attachments (attachment_id); ALTER TABLE messages DROP CONSTRAINT text_length; -ALTER TABLE messages ADD CONSTRAINT text_length CHECK (char_length(text) <= 4000); \ No newline at end of file +ALTER TABLE messages ADD CONSTRAINT text_length CHECK ( + bytea_utf8_char_length(text) BETWEEN 0 AND 4000 +); \ No newline at end of file -- 2.54.0