From dfa8784afaebee0fe12e8a9169d33d4b79b8f656 Mon Sep 17 00:00:00 2001 From: gera Date: Fri, 6 Nov 2026 01:59:07 +0300 Subject: [PATCH 1/5] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D0=BD=D1=91=D1=81?= =?UTF-8?q?=20=D0=BF=D1=80=D0=B8=D0=BB=D0=BE=D0=B6=D0=B5=D0=BD=D0=B8=D0=B5?= =?UTF-8?q?=20=D0=B8=D0=B7=20lab4,=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D0=BB=20=D0=BE=D1=87=D0=B5=D1=80=D0=B5=D0=B4=D1=8C=20=D0=B7?= =?UTF-8?q?=D0=B0=D0=B4=D0=B0=D0=BD=D0=B8=D0=B9,=20outbox=20=D0=B8=20worke?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 19 + attachments.go | 530 +++++++++++++++ auth.go | 131 ++++ broker.go | 135 ++++ cache.go | 104 +++ db.go | 57 ++ go.mod | 39 ++ go.sum | 85 +++ handlers.go | 1092 +++++++++++++++++++++++++++++++ imaging.go | 176 +++++ main.go | 199 ++++++ migrate.go | 84 +++ migrations/0001_init.sql | 38 ++ migrations/0002_idempotency.sql | 10 + migrations/0003_attachments.sql | 22 + migrations/0004_jobs.sql | 60 ++ models.go | 33 + s3.go | 109 +++ worker.go | 448 +++++++++++++ 19 files changed, 3371 insertions(+) create mode 100644 Dockerfile create mode 100644 attachments.go create mode 100644 auth.go create mode 100644 broker.go create mode 100644 cache.go create mode 100644 db.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 handlers.go create mode 100644 imaging.go create mode 100644 main.go create mode 100644 migrate.go create mode 100644 migrations/0001_init.sql create mode 100644 migrations/0002_idempotency.sql create mode 100644 migrations/0003_attachments.sql create mode 100644 migrations/0004_jobs.sql create mode 100644 models.go create mode 100644 s3.go create mode 100644 worker.go diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..94f0316e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM golang:1.26-alpine@sha256:c95332c2af86b6d89b91bd0500f4b9529ccbd090a0d1855c6d1ceaa142ae8615 AS build +WORKDIR /src +COPY go.mod go.sum ./ +RUN go mod download +COPY *.go ./ +COPY migrations ./migrations +RUN CGO_ENABLED=0 GOOS=linux go build -tags nodynamic -trimpath -ldflags="-s -w" -o /out/messenger . + +FROM alpine:3.21@sha256:ce64758a109eb420d874a118f87920e625e12d3634e03b4a5573fd9f6e5d3507 +RUN adduser -D -u 10001 app +WORKDIR /app +COPY --from=build /out/messenger /app/messenger +USER 10001 +EXPOSE 8080 +ENV APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +STOPSIGNAL SIGTERM +HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \ + CMD wget -qO- http://127.0.0.1:8080/health/ready >/dev/null || exit 1 +ENTRYPOINT ["/app/messenger"] diff --git a/attachments.go b/attachments.go new file mode 100644 index 00000000..b8894990 --- /dev/null +++ b/attachments.go @@ -0,0 +1,530 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "hash" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "log/slog" + "net/http" + "os" + "strconv" + "strings" + "time" + "unicode" + "unicode/utf8" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + objectPrefix = "attachments/" + stagingPrefix = "staging/" + probeHead = 256 << 10 + tailLen = 12 +) + +var ( + pngMagic = []byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'} + pngTail = []byte{0, 0, 0, 0, 'I', 'E', 'N', 'D', 0xae, 0x42, 0x60, 0x82} + jpegMagic = []byte{0xff, 0xd8, 0xff} + jpegTail = []byte{0xff, 0xd9} +) + +type limits struct { + maxBytes int64 + maxPixels int64 + uploadSlots int + maxAttempts int +} + +type attachment struct { + ID, ChatID, OwnerID, Filename, ContentType, SHA256, Key, Status string + ErrorCode *string + Size int64 + Created, Updated time.Time + Variants []variant +} + +type variant struct { + Name string `json:"name"` + ContentType string `json:"content_type"` + Size int64 `json:"size_bytes"` + Width int `json:"width"` + Height int `json:"height"` + SHA256 string `json:"sha256"` + key string +} + +func (x *attachment) json() map[string]any { + variants := x.Variants + if variants == nil { + variants = []variant{} + } + return map[string]any{ + "id": x.ID, + "chat_id": x.ChatID, + "owner_id": x.OwnerID, + "filename": x.Filename, + "content_type": x.ContentType, + "size_bytes": x.Size, + "sha256": x.SHA256, + "status": x.Status, + "variants": variants, + "error_code": x.ErrorCode, + "created_at": formatTime(x.Created), + "updated_at": formatTime(x.Updated), + } +} + +type probe struct { + sum hash.Hash + size int64 + head []byte + tail []byte +} + +func newProbe() *probe { + return &probe{sum: sha256.New()} +} + +func (p *probe) Write(b []byte) (int, error) { + p.sum.Write(b) + p.size += int64(len(b)) + if room := probeHead - len(p.head); room > 0 { + if room > len(b) { + room = len(b) + } + p.head = append(p.head, b[:room]...) + } + p.tail = append(p.tail, b...) + if len(p.tail) > tailLen { + p.tail = append(p.tail[:0], p.tail[len(p.tail)-tailLen:]...) + } + return len(b), nil +} + +func (p *probe) digest() string { + return hex.EncodeToString(p.sum.Sum(nil)) +} + +var ( + errNotImage = errors.New("not an image") + errTooBig = errors.New("too large") +) + +func (p *probe) verify(contentType string, maxPixels int64) error { + cfg, format, err := image.DecodeConfig(bytes.NewReader(p.head)) + if err != nil { + return errNotImage + } + var tail []byte + switch contentType { + case "image/png": + if format != "png" { + return errNotImage + } + tail = pngTail + case "image/jpeg": + if format != "jpeg" { + return errNotImage + } + tail = jpegTail + } + if !bytes.HasSuffix(p.tail, tail) { + return errNotImage + } + if int64(cfg.Width)*int64(cfg.Height) > maxPixels { + return errTooBig + } + return nil +} + +func magicMatches(contentType string, head []byte) bool { + switch contentType { + case "image/png": + return bytes.HasPrefix(head, pngMagic) + case "image/jpeg": + return bytes.HasPrefix(head, jpegMagic) + } + return false +} + +func validFilename(name string) bool { + n := utf8.RuneCountInString(name) + if n < 1 || n > 255 || !utf8.ValidString(name) { + return false + } + for _, r := range name { + if r == '/' || r == '\\' || r == 0 || unicode.IsControl(r) { + return false + } + } + return true +} + +type trackedReader struct { + r io.Reader + err error +} + +func (t *trackedReader) Read(b []byte) (int, error) { + n, err := t.r.Read(b) + if err != nil && err != io.EOF && t.err == nil { + t.err = err + } + return n, err +} + +func isTooLarge(err error) bool { + var mbe *http.MaxBytesError + return errors.As(err, &mbe) +} + +func (a *App) dropObject(key string) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := a.store.Remove(ctx, key); err != nil { + a.log.Warn("object cleanup failed", "key", key, "err", err.Error()) + } +} + +func (a *App) handleUploadAttachment(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + chatID := r.PathValue("chat_id") + if !isUUID(chatID) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id") + return + } + ok, err := a.isMember(r.Context(), chatID, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if !ok { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + select { + case a.uploads <- struct{}{}: + defer func() { <-a.uploads }() + default: + w.Header().Set("Retry-After", "1") + a.writeErr(w, reqID, 429, "rate_limited", "too many concurrent uploads") + return + } + filename := r.URL.Query().Get("filename") + if !validFilename(filename) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid filename") + return + } + contentType := strings.ToLower(strings.TrimSpace(strings.Split(r.Header.Get("Content-Type"), ";")[0])) + if contentType != "image/png" && contentType != "image/jpeg" { + a.writeErr(w, reqID, 415, "unsupported_media_type", "Content-Type must be image/png or image/jpeg") + return + } + if r.ContentLength > a.limits.maxBytes { + a.writeErr(w, reqID, 413, "payload_too_large", "file too large") + return + } + body := &trackedReader{r: http.MaxBytesReader(w, r.Body, a.limits.maxBytes)} + head := make([]byte, 16) + n, _ := io.ReadFull(body, head) + if body.err != nil { + if isTooLarge(body.err) { + a.writeErr(w, reqID, 413, "payload_too_large", "file too large") + } else { + a.writeErr(w, reqID, 400, "invalid_request", "upload interrupted") + } + return + } + head = head[:n] + if !magicMatches(contentType, head) { + a.writeErr(w, reqID, 415, "unsupported_media_type", "content does not match image type") + return + } + + id := newUUID() + key := stagingPrefix + id + pr := newProbe() + size := r.ContentLength + if size < 0 { + size = -1 + } + src := io.TeeReader(io.MultiReader(bytes.NewReader(head), body), pr) + putCtx, cancel := context.WithTimeout(r.Context(), 60*time.Second) + defer cancel() + if err := a.store.Put(putCtx, key, src, size, contentType); err != nil { + a.dropObject(key) + switch { + case isTooLarge(body.err): + a.writeErr(w, reqID, 413, "payload_too_large", "file too large") + case body.err != nil: + a.writeErr(w, reqID, 400, "invalid_request", "upload interrupted") + default: + a.log.Error("s3 put failed", "request_id", reqID, "err", err.Error()) + a.writeErr(w, reqID, 503, "unavailable", "storage unavailable") + } + return + } + if err := pr.verify(contentType, a.limits.maxPixels); err != nil { + a.dropObject(key) + if err == errTooBig { + a.writeErr(w, reqID, 413, "payload_too_large", "image too large") + } else { + a.writeErr(w, reqID, 415, "unsupported_media_type", "invalid image") + } + return + } + + now := nowUTC() + att := &attachment{ + ID: id, ChatID: chatID, OwnerID: u.ID, Filename: filename, ContentType: contentType, + SHA256: pr.digest(), Size: pr.size, Status: "queued", Created: now, Updated: now, + } + if err := a.enqueue(r.Context(), att, key); err != nil { + a.dropObject(key) + a.log.Error("enqueue failed", "request_id", reqID, "err", err.Error()) + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if a.fault == "after_commit" { + a.log.Error("fault injected", "point", "after_commit", "attachment_id", att.ID) + os.Exit(3) + } + a.writeJSON(w, 202, att.json()) +} + +func (a *App) enqueue(ctx context.Context, att *attachment, key string) error { + ctx, cancel := dbCtx(ctx) + defer cancel() + tx, err := a.db.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + if _, err := tx.Exec(ctx, ` + insert into attachments(id, chat_id, owner_id, filename, content_type, size_bytes, sha256, staging_key, status, created_at, updated_at) + values($1::uuid, $2::uuid, $3::uuid, $4, $5, $6, $7, $8, 'queued', $9, $9)`, + att.ID, att.ChatID, att.OwnerID, att.Filename, att.ContentType, att.Size, att.SHA256, key, att.Created); err != nil { + return err + } + jobID := newUUID() + if _, err := tx.Exec(ctx, ` + insert into jobs(id, attachment_id, pipeline_version, status, attempt, max_attempts, next_attempt_at, created_at, updated_at) + values($1::uuid, $2::uuid, $3, 'queued', 1, $4, $5, $5, $5)`, + jobID, att.ID, pipelineVersion, a.limits.maxAttempts, att.Created); err != nil { + return err + } + if _, err := tx.Exec(ctx, ` + insert into outbox(job_id, attempt, available_at, created_at) values($1::uuid, 1, $2, $2)`, jobID, att.Created); err != nil { + return err + } + return tx.Commit(ctx) +} + +func (a *App) loadAttachment(ctx context.Context, id, userID string) (*attachment, error) { + var x attachment + err := a.db.QueryRow(ctx, ` + select a.id::text, a.chat_id::text, a.owner_id::text, a.filename, a.content_type, + a.size_bytes, a.sha256, coalesce(a.object_key, ''), a.status, a.error_code, a.created_at, a.updated_at + from attachments a + join chat_members m on m.chat_id = a.chat_id and m.user_id = $2::uuid + where a.id = $1::uuid`, id, userID). + Scan(&x.ID, &x.ChatID, &x.OwnerID, &x.Filename, &x.ContentType, &x.Size, &x.SHA256, &x.Key, &x.Status, &x.ErrorCode, &x.Created, &x.Updated) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if x.Status != "ready" { + return &x, nil + } + rows, err := a.db.Query(ctx, ` + select name, content_type, size_bytes, width, height, sha256, object_key + from attachment_variants where attachment_id = $1::uuid and pipeline_version = $2 + order by array_position(array['thumbnail', 'crop', 'optimized'], name)`, id, pipelineVersion) + if err != nil { + return nil, err + } + x.Variants, err = pgx.CollectRows(rows, func(row pgx.CollectableRow) (variant, error) { + var v variant + return v, row.Scan(&v.Name, &v.ContentType, &v.Size, &v.Width, &v.Height, &v.SHA256, &v.key) + }) + if err != nil { + return nil, err + } + return &x, nil +} + +func (a *App) attachmentFor(w http.ResponseWriter, r *http.Request) *attachment { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return nil + } + id := r.PathValue("attachment_id") + if !isUUID(id) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid attachment_id") + return nil + } + ctx, cancel := dbCtx(r.Context()) + defer cancel() + att, err := a.loadAttachment(ctx, id, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return nil + } + if att == nil { + a.writeErr(w, reqID, 404, "not_found", "attachment not found") + return nil + } + return att +} + +func (a *App) handleGetAttachment(w http.ResponseWriter, r *http.Request) { + if att := a.attachmentFor(w, r); att != nil { + a.writeJSON(w, 200, att.json()) + } +} + +func (a *App) handleDownloadAttachment(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + name := r.URL.Query().Get("variant") + if name == "" { + name = "original" + } + if name != "original" && name != "thumbnail" && name != "crop" && name != "optimized" { + a.writeErr(w, reqID, 400, "invalid_request", "invalid variant") + return + } + att := a.attachmentFor(w, r) + if att == nil { + return + } + if att.Status != "ready" { + a.writeErr(w, reqID, 409, "conflict", "attachment is not ready") + return + } + key, contentType := att.Key, att.ContentType + if name != "original" { + key = "" + for _, v := range att.Variants { + if v.Name == name { + key, contentType = v.key, v.ContentType + } + } + if key == "" { + a.writeErr(w, reqID, 404, "not_found", "variant not found") + return + } + } + ctx, cancel := context.WithTimeout(r.Context(), 30*time.Second) + defer cancel() + obj, info, err := a.store.Get(ctx, key) + if err != nil { + if isNoSuchKey(err) { + a.writeErr(w, reqID, 404, "not_found", "attachment not found") + return + } + a.log.Error("s3 get failed", "request_id", reqID, "err", err.Error()) + a.writeErr(w, reqID, 503, "unavailable", "storage unavailable") + return + } + defer obj.Close() + first := make([]byte, 32<<10) + n, err := io.ReadFull(obj, first) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + a.log.Error("s3 read failed", "request_id", reqID, "err", err.Error()) + a.writeErr(w, reqID, 503, "unavailable", "storage unavailable") + return + } + w.Header().Set("Content-Type", contentType) + w.Header().Set("Content-Length", strconv.FormatInt(info.Size, 10)) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(200) + _, _ = w.Write(first[:n]) + _, _ = io.Copy(w, obj) +} + +var keepQueries = map[string]string{ + objectPrefix: ` + select k from unnest($1::text[]) k + where exists(select 1 from attachments where object_key = k) + or exists(select 1 from attachment_variants where object_key = k)`, + stagingPrefix: ` + select k from unnest($1::text[]) k + where exists(select 1 from attachments where staging_key = k and (status in ('queued', 'processing') + or (status = 'failed' and updated_at > now() - make_interval(secs => $2))))`, +} + +func cleanupOrphans(ctx context.Context, db *pgxpool.Pool, store *Store, grace, retention time.Duration, log *slog.Logger) (int, error) { + cutoff := time.Now().Add(-grace) + removed := 0 + flush := func(prefix string, keys []string) error { + if len(keys) == 0 { + return nil + } + args := []any{keys} + if prefix == stagingPrefix { + args = append(args, retention.Seconds()) + } + rows, err := db.Query(ctx, keepQueries[prefix], args...) + if err != nil { + return err + } + kept, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return err + } + known := map[string]bool{} + for _, k := range kept { + known[k] = true + } + for _, k := range keys { + if known[k] { + continue + } + if err := store.Remove(ctx, k); err != nil { + return err + } + log.Info("orphan object removed", "key", k) + removed++ + } + return nil + } + for _, prefix := range []string{objectPrefix, stagingPrefix} { + var batch []string + for obj := range store.List(ctx, prefix) { + if obj.Err != nil { + return removed, obj.Err + } + if obj.LastModified.After(cutoff) { + continue + } + batch = append(batch, obj.Key) + if len(batch) == 500 { + if err := flush(prefix, batch); err != nil { + return removed, err + } + batch = batch[:0] + } + } + if err := flush(prefix, batch); err != nil { + return removed, err + } + } + return removed, nil +} diff --git a/auth.go b/auth.go new file mode 100644 index 00000000..cacc42d0 --- /dev/null +++ b/auth.go @@ -0,0 +1,131 @@ +package main + +import ( + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" + "unicode/utf8" + + "golang.org/x/crypto/argon2" +) + +const ( + argonTime = 2 + argonMemory = 19456 + argonThreads = 1 + argonKeyLen = 32 + argonSaltLen = 16 +) + +func hashPassword(password string) (string, error) { + salt := make([]byte, argonSaltLen) + if _, err := rand.Read(salt); err != nil { + return "", err + } + hash := argon2.IDKey([]byte(password), salt, argonTime, argonMemory, argonThreads, argonKeyLen) + b64Salt := base64.RawStdEncoding.EncodeToString(salt) + b64Hash := base64.RawStdEncoding.EncodeToString(hash) + return fmt.Sprintf("$argon2id$v=%d$m=%d,t=%d,p=%d$%s$%s", + argon2.Version, argonMemory, argonTime, argonThreads, b64Salt, b64Hash), nil +} + +func checkPassword(encoded, password string) (bool, error) { + parts := strings.Split(encoded, "$") + if len(parts) != 6 || parts[1] != "argon2id" { + return false, errors.New("bad hash") + } + var version int + if _, err := fmt.Sscanf(parts[2], "v=%d", &version); err != nil { + return false, err + } + var memory, timeCost, threads int + for _, p := range strings.Split(parts[3], ",") { + kv := strings.SplitN(p, "=", 2) + if len(kv) != 2 { + return false, errors.New("bad params") + } + n, err := strconv.Atoi(kv[1]) + if err != nil { + return false, err + } + switch kv[0] { + case "m": + memory = n + case "t": + timeCost = n + case "p": + threads = n + } + } + salt, err := base64.RawStdEncoding.DecodeString(parts[4]) + if err != nil { + return false, err + } + want, err := base64.RawStdEncoding.DecodeString(parts[5]) + if err != nil { + return false, err + } + got := argon2.IDKey([]byte(password), salt, uint32(timeCost), uint32(memory), uint8(threads), uint32(len(want))) + return subtle.ConstantTimeCompare(got, want) == 1, nil +} + +func newToken() (string, []byte, string, error) { + raw := make([]byte, 32) + if _, err := rand.Read(raw); err != nil { + return "", nil, "", err + } + token := base64.RawURLEncoding.EncodeToString(raw) + sum := sha256.Sum256([]byte(token)) + hash := make([]byte, len(sum)) + copy(hash, sum[:]) + return token, hash, hex.EncodeToString(hash), nil +} + +func tokenHash(token string) ([]byte, string) { + sum := sha256.Sum256([]byte(token)) + hash := make([]byte, len(sum)) + copy(hash, sum[:]) + return hash, hex.EncodeToString(hash) +} + +func parseBasic(header string) (username, password string, ok bool) { + const prefix = "Basic " + if !strings.HasPrefix(header, prefix) { + return "", "", false + } + decoded, err := base64.StdEncoding.DecodeString(strings.TrimSpace(header[len(prefix):])) + if err != nil { + return "", "", false + } + s := string(decoded) + i := strings.IndexByte(s, ':') + if i < 0 { + return "", "", false + } + username = s[:i] + password = s[i+1:] + return username, password, true +} + +func parseBearer(header string) (string, bool) { + const prefix = "Bearer " + if !strings.HasPrefix(header, prefix) { + return "", false + } + tok := strings.TrimSpace(header[len(prefix):]) + if tok == "" { + return "", false + } + return tok, true +} + +func validPassword(p string) bool { + n := utf8.RuneCountInString(p) + return n >= 12 && n <= 128 +} diff --git a/broker.go b/broker.go new file mode 100644 index 00000000..1b6cad9b --- /dev/null +++ b/broker.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "encoding/json" + "log/slog" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" +) + +const ( + jobStream = "attachments.jobs" + jobGroup = "workers" + jobType = "attachment.process.v1" + pipelineVersion = 1 +) + +type jobMessage struct { + SchemaVersion int `json:"schema_version"` + JobID string `json:"job_id"` + Type string `json:"type"` + AttachmentID string `json:"attachment_id"` + PipelineVersion int `json:"pipeline_version"` + Attempt int `json:"attempt"` + CreatedAt string `json:"created_at"` +} + +type Broker struct { + rdb *redis.Client +} + +func openBroker() (*Broker, error) { + raw, err := readSecret("BROKER_URL", "BROKER_URL_FILE") + if err != nil { + return nil, err + } + opt, err := redis.ParseURL(raw) + if err != nil { + return nil, err + } + opt.DialTimeout = 2 * time.Second + opt.ReadTimeout = 3 * time.Second + opt.WriteTimeout = 3 * time.Second + opt.MaxRetries = 0 + return &Broker{rdb: redis.NewClient(opt)}, nil +} + +func (b *Broker) ensureGroup(ctx context.Context) error { + err := b.rdb.XGroupCreateMkStream(ctx, jobStream, jobGroup, "0").Err() + if err != nil && !strings.Contains(err.Error(), "BUSYGROUP") { + return err + } + return nil +} + +func (b *Broker) publish(ctx context.Context, m jobMessage) error { + body, err := json.Marshal(m) + if err != nil { + return err + } + return b.rdb.XAdd(ctx, &redis.XAddArgs{Stream: jobStream, Values: map[string]any{"job": string(body)}}).Err() +} + +func relayOnce(ctx context.Context, db *pgxpool.Pool, b *Broker) (int, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + tx, err := db.Begin(ctx) + if err != nil { + return 0, err + } + defer tx.Rollback(ctx) + rows, err := tx.Query(ctx, ` + select o.id, o.job_id::text, o.attempt, j.attachment_id::text, j.pipeline_version, j.created_at + from outbox o + join jobs j on j.id = o.job_id + where o.published_at is null and o.available_at <= now() + order by o.id + limit 50 + for update of o skip locked`) + if err != nil { + return 0, err + } + var ids []int64 + var msgs []jobMessage + for rows.Next() { + var id int64 + var created time.Time + m := jobMessage{SchemaVersion: 1, Type: jobType} + if err := rows.Scan(&id, &m.JobID, &m.Attempt, &m.AttachmentID, &m.PipelineVersion, &created); err != nil { + rows.Close() + return 0, err + } + m.CreatedAt = formatTime(created) + ids = append(ids, id) + msgs = append(msgs, m) + } + rows.Close() + if err := rows.Err(); err != nil { + return 0, err + } + if len(ids) == 0 { + return 0, nil + } + if err := b.ensureGroup(ctx); err != nil { + return 0, err + } + for _, m := range msgs { + if err := b.publish(ctx, m); err != nil { + return 0, err + } + } + if _, err := tx.Exec(ctx, `update outbox set published_at = now() where id = any($1)`, ids); err != nil { + return 0, err + } + return len(ids), tx.Commit(ctx) +} + +func runRelay(ctx context.Context, db *pgxpool.Pool, b *Broker, log *slog.Logger) { + for ctx.Err() == nil { + n, err := relayOnce(ctx, db, b) + if err != nil { + log.Warn("outbox relay failed", "err", err.Error()) + time.Sleep(2 * time.Second) + continue + } + if n > 0 { + log.Info("outbox published", "count", n) + continue + } + time.Sleep(300 * time.Millisecond) + } +} diff --git a/cache.go b/cache.go new file mode 100644 index 00000000..9afc5e3f --- /dev/null +++ b/cache.go @@ -0,0 +1,104 @@ +package main + +import ( + "context" + "encoding/json" + "log/slog" + "time" + + "github.com/redis/go-redis/v9" +) + +type cachedSession struct { + UserID string `json:"user_id"` + ExpiresAt time.Time `json:"expires_at"` +} + +type Cache struct { + rdb *redis.Client + log *slog.Logger +} + +func openCache(log *slog.Logger) *Cache { + url, err := readSecret("CACHE_URL", "CACHE_URL_FILE") + if err != nil { + log.Warn("cache disabled", "err", err.Error()) + return &Cache{log: log} + } + opt, err := redis.ParseURL(url) + if err != nil { + log.Warn("cache parse failed", "err", err.Error()) + return &Cache{log: log} + } + opt.DialTimeout = 200 * time.Millisecond + opt.ReadTimeout = 200 * time.Millisecond + opt.WriteTimeout = 200 * time.Millisecond + opt.PoolTimeout = 200 * time.Millisecond + rdb := redis.NewClient(opt) + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + if err := rdb.Ping(ctx).Err(); err != nil { + log.Warn("cache ping failed, will bypass", "err", err.Error()) + } + return &Cache{rdb: rdb, log: log} +} + +func (c *Cache) Close() { + if c.rdb != nil { + _ = c.rdb.Close() + } +} + +func sessKey(hashHex string) string { + return "sess:" + hashHex +} + +func (c *Cache) GetSession(ctx context.Context, hashHex string) (*cachedSession, bool) { + if c.rdb == nil { + return nil, false + } + ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) + defer cancel() + val, err := c.rdb.Get(ctx, sessKey(hashHex)).Result() + if err != nil { + return nil, false + } + var s cachedSession + if err := json.Unmarshal([]byte(val), &s); err != nil { + return nil, false + } + if time.Now().UTC().After(s.ExpiresAt) { + return nil, false + } + return &s, true +} + +func (c *Cache) SetSession(ctx context.Context, hashHex string, userID string, expiresAt time.Time) { + if c.rdb == nil { + return + } + ttl := time.Until(expiresAt) + if ttl <= 0 { + return + } + body, err := json.Marshal(cachedSession{UserID: userID, ExpiresAt: expiresAt.UTC()}) + if err != nil { + return + } + ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) + defer cancel() + if err := c.rdb.Set(ctx, sessKey(hashHex), body, ttl).Err(); err != nil { + c.log.Warn("cache set failed", "err", err.Error()) + } +} + +func (c *Cache) DelSession(ctx context.Context, hashHex string) { + if c.rdb == nil { + return + } + ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) + defer cancel() + if err := c.rdb.Del(ctx, sessKey(hashHex)).Err(); err != nil { + c.log.Warn("cache del failed", "err", err.Error()) + } +} diff --git a/db.go b/db.go new file mode 100644 index 00000000..5b051b33 --- /dev/null +++ b/db.go @@ -0,0 +1,57 @@ +package main + +import ( + "context" + "fmt" + "os" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func readSecret(envKey, fileKey string) (string, error) { + if v := os.Getenv(envKey); v != "" { + return strings.TrimSpace(v), nil + } + path := os.Getenv(fileKey) + if path == "" { + return "", fmt.Errorf("%s or %s required", envKey, fileKey) + } + b, err := os.ReadFile(path) + if err != nil { + return "", err + } + return strings.TrimSpace(string(b)), nil +} + +func openDB(ctx context.Context) (*pgxpool.Pool, error) { + dsn, err := readSecret("DATABASE_URL", "DATABASE_URL_FILE") + if err != nil { + return nil, err + } + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + return nil, err + } + cfg.MaxConns = 10 + cfg.MinConns = 2 + cfg.MaxConnLifetime = 5 * time.Minute + cfg.ConnConfig.ConnectTimeout = 3 * time.Second + + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + return nil, err + } + pingCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + if err := pool.Ping(pingCtx); err != nil { + pool.Close() + return nil, err + } + return pool, nil +} + +func dbCtx(parent context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(parent, 3*time.Second) +} diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..8f85fe5a --- /dev/null +++ b/go.mod @@ -0,0 +1,39 @@ +module mtusi-messenger + +go 1.26.0 + +require ( + github.com/gen2brain/webp v0.6.4 + github.com/jackc/pgx/v5 v5.11.0 + github.com/minio/minio-go/v7 v7.4.1 + github.com/redis/go-redis/v9 v9.23.0 + github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd + golang.org/x/crypto v0.58.0 + golang.org/x/image v0.47.0 +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/ebitengine/purego v0.10.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect + github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/klauspost/compress v1.19.2 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect + github.com/klauspost/crc32 v1.3.0 // indirect + github.com/minio/crc64nvme v1.1.1 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/philhofer/fwd v1.2.0 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/tinylib/msgp v1.6.4 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.uber.org/atomic v1.12.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/net v0.60.0 // indirect + golang.org/x/sync v0.24.0 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/text v0.43.0 // indirect + gopkg.in/ini.v1 v1.67.3 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..c2500014 --- /dev/null +++ b/go.sum @@ -0,0 +1,85 @@ +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/gen2brain/webp v0.6.4 h1:SUDdmxADOAiPQ+5ylNmuHhuYf2dOi0KgKZHL5vpVCNU= +github.com/gen2brain/webp v0.6.4/go.mod h1:iGWMaCSw7t3I/Cv9llzEKmpnR36S8lS8VL/ZVjxU0JE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= +github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.11.0 h1:IzBBtyK9AHqf98cctWFifYSci2hgQR/cd56wB4p+ogg= +github.com/jackc/pgx/v5 v5.11.0/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= +github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= +github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/klauspost/compress v1.19.2 h1:hMRETovs/pu/dVWN7zIT1PGG8t509MwT6bO7XSi26R8= +github.com/klauspost/compress v1.19.2/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= +github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM= +github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw= +github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI= +github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.4.1 h1:PtU/zIOfQRWZ+QydXAZysc+xZYHdG+BBR4Z6TUvy73w= +github.com/minio/minio-go/v7 v7.4.1/go.mod h1:nQnQi7zTfXR4lREtYXZQWqo5aw0Rqv+ZqCGrTORygRk= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.23.0 h1:/TTVdJa8BRNsybwwGGnmiYktHNlUXtbOQxlWpHBPL3s= +github.com/redis/go-redis/v9 v9.23.0/go.mod h1:EWP2UUk+XDaI1s5nQB5mablZN6vccXlmBQTyfnF87D0= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= +github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +github.com/tinylib/msgp v1.6.4 h1:mOwYbyYDLPj35mkA2BjjYejgJk9BuHxDdvRnb6v2ZcQ= +github.com/tinylib/msgp v1.6.4/go.mod h1:RSp0LW9oSxFut3KzESt5Voq4GVWyS+PSulT77roAqEA= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.uber.org/atomic v1.12.0 h1:BvcXdFKuviU4fTL/f+SxdQ5qJX/Jix8pAkgdUcb3XOE= +go.uber.org/atomic v1.12.0/go.mod h1:I6c4cg+6HCxRjfjSsYtApoFILnpc0CGUdGkXVqbYVNk= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/image v0.47.0 h1:za/8gVz++PPZqmZJBNSISICN5Rrj5zixP/8K++fgmqo= +golang.org/x/image v0.47.0/go.mod h1:IkN5o4A7vq55vXN8FWjjnhg9Rs2GOtffqsSi9mWAKYE= +golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= +golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/ini.v1 v1.67.3 h1:iM9Lhz5MRSGhHVGGwCuzG9KO8PoirCXj/m/qTmOJJQw= +gopkg.in/ini.v1 v1.67.3/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/handlers.go b/handlers.go new file mode 100644 index 00000000..7fc5c807 --- /dev/null +++ b/handlers.go @@ -0,0 +1,1092 @@ +package main + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "os" + "regexp" + "strconv" + "strings" + "sync/atomic" + "time" + "unicode/utf8" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +var idempotencyKeyRe = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) + +var instanceID = func() string { + h, err := os.Hostname() + if err != nil || h == "" { + return newUUID() + } + return h +}() + +var usernameRe = regexp.MustCompile(`^[a-z0-9_]{3,32}$`) + +type apiError struct { + Code string `json:"code"` + Message string `json:"message"` + ReqID string `json:"request_id"` +} + +type errorBody struct { + Error apiError `json:"error"` +} + +type App struct { + db *pgxpool.Pool + cache *Cache + log *slog.Logger + ready atomic.Bool + sessionTTL time.Duration + idempotencyTTL time.Duration + store *Store + limits limits + uploads chan struct{} + fault string +} + +func NewApp(db *pgxpool.Pool, cache *Cache, log *slog.Logger, sessionTTL, idempotencyTTL time.Duration, store *Store, lim limits) *App { + a := &App{db: db, cache: cache, log: log, sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, store: store, limits: lim, uploads: make(chan struct{}, lim.uploadSlots), fault: os.Getenv("FAULT_POINT")} + a.ready.Store(true) + return a +} + +func (a *App) SetReady(v bool) { + a.ready.Store(v) +} + +func newUUID() string { + var b [16]byte + _, _ = rand.Read(b[:]) + b[6] = (b[6] & 0x0f) | 0x40 + b[8] = (b[8] & 0x3f) | 0x80 + return fmt.Sprintf("%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]) +} + +func nowUTC() time.Time { + return time.Now().UTC().Truncate(time.Microsecond) +} + +func formatTime(t time.Time) string { + return t.UTC().Format("2006-01-02T15:04:05.000000Z") +} + +func parseTime(s string) (time.Time, error) { + return time.Parse("2006-01-02T15:04:05.000000Z", s) +} + +type statusWriter struct { + http.ResponseWriter + status int +} + +func (w *statusWriter) WriteHeader(code int) { + w.status = code + w.ResponseWriter.WriteHeader(code) +} + +type ctxKey int + +const reqIDKey ctxKey = 1 + +func (a *App) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + reqID := newUUID() + w.Header().Set("X-Request-Id", reqID) + w.Header().Set("X-Instance-Id", instanceID) + sw := &statusWriter{ResponseWriter: w, status: 200} + defer func() { + if rec := recover(); rec != nil { + a.writeErr(sw, reqID, 500, "internal_error", "internal error") + a.log.Error("panic", "request_id", reqID, "err", fmt.Sprint(rec)) + } + a.log.Info("request", + "method", r.Method, + "path", r.URL.Path, + "status", sw.status, + "duration_ms", time.Since(start).Milliseconds(), + "request_id", reqID, + ) + }() + next.ServeHTTP(sw, r.WithContext(context.WithValue(r.Context(), reqIDKey, reqID))) + }) +} + +func reqIDFrom(r *http.Request) string { + if v, ok := r.Context().Value(reqIDKey).(string); ok { + return v + } + return "" +} + +func (a *App) writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + enc := json.NewEncoder(w) + enc.SetEscapeHTML(false) + _ = enc.Encode(v) +} + +func (a *App) writeErr(w http.ResponseWriter, reqID string, status int, code, msg string) { + a.writeJSON(w, status, errorBody{Error: apiError{Code: code, Message: msg, ReqID: reqID}}) +} + +func (a *App) writeUnauthorized(w http.ResponseWriter, r *http.Request, realm string) { + reqID := reqIDFrom(r) + if realm == "Basic" { + w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`) + } else { + w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`) + } + a.writeErr(w, reqID, 401, "unauthorized", "unauthorized") +} + +func (a *App) Routes() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("GET /health/live", a.handleLive) + mux.HandleFunc("GET /health/ready", a.handleReady) + mux.HandleFunc("POST /api/v1/users", a.handleCreateUser) + mux.HandleFunc("GET /api/v1/users/me", a.handleMe) + mux.HandleFunc("POST /api/v1/chats", a.handleCreateChat) + mux.HandleFunc("GET /api/v1/chats", a.handleListChats) + mux.HandleFunc("GET /api/v1/chats/{chat_id}", a.handleGetChat) + mux.HandleFunc("POST /api/v1/chats/{chat_id}/messages", a.handleCreateMessage) + mux.HandleFunc("GET /api/v1/chats/{chat_id}/messages", a.handleListMessages) + mux.HandleFunc("POST /api/v1/chats/{chat_id}/attachments", a.handleUploadAttachment) + mux.HandleFunc("GET /api/v1/attachments/{attachment_id}", a.handleGetAttachment) + mux.HandleFunc("GET /api/v1/attachments/{attachment_id}/content", a.handleDownloadAttachment) + mux.HandleFunc("POST /api/v1/auth/sessions", a.handleCreateSession) + mux.HandleFunc("DELETE /api/v1/auth/sessions/current", a.handleRevokeSession) + return a.Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, pattern := mux.Handler(r) + if pattern == "" { + a.writeErr(w, reqIDFrom(r), 404, "not_found", "not found") + return + } + mux.ServeHTTP(w, r) + })) +} + +func (a *App) handleLive(w http.ResponseWriter, r *http.Request) { + a.writeJSON(w, 200, map[string]string{"status": "ok"}) +} + +func (a *App) handleReady(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + if !a.ready.Load() { + a.writeErr(w, reqID, 503, "unavailable", "shutting down") + return + } + ctx, cancel := dbCtx(r.Context()) + defer cancel() + if err := a.db.Ping(ctx); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 200, map[string]string{"status": "ok"}) +} + +func (a *App) readJSON(w http.ResponseWriter, r *http.Request, dst any) bool { + reqID := reqIDFrom(r) + ct := r.Header.Get("Content-Type") + media := strings.ToLower(strings.TrimSpace(strings.Split(ct, ";")[0])) + if media != "application/json" { + a.writeErr(w, reqID, 415, "unsupported_media_type", "Content-Type must be application/json") + return false + } + r.Body = http.MaxBytesReader(w, r.Body, 1<<20) + dec := json.NewDecoder(r.Body) + dec.DisallowUnknownFields() + if err := dec.Decode(dst); err != nil { + if _, ok := err.(*http.MaxBytesError); ok { + a.writeErr(w, reqID, 413, "payload_too_large", "body too large") + return false + } + a.writeErr(w, reqID, 400, "invalid_request", "invalid json") + return false + } + var extra json.RawMessage + if err := dec.Decode(&extra); err != io.EOF { + a.writeErr(w, reqID, 400, "invalid_request", "invalid json") + return false + } + return true +} + +func isBlank(s string) bool { + return strings.TrimSpace(s) == "" +} + +func isUUID(s string) bool { + if len(s) != 36 { + return false + } + for i, c := range s { + switch i { + case 8, 13, 18, 23: + if c != '-' { + return false + } + default: + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F')) { + return false + } + } + } + raw := strings.ReplaceAll(s, "-", "") + _, err := hex.DecodeString(raw) + return err == nil +} + +func isPgCode(err error, code string) bool { + var pgErr *pgconn.PgError + return errors.As(err, &pgErr) && pgErr.Code == code +} + +func userJSON(u *User) map[string]any { + return map[string]any{ + "id": u.ID, + "username": u.Username, + "display_name": u.DisplayName, + "created_at": formatTime(u.CreatedAt), + } +} + +func (a *App) getUserByID(ctx context.Context, id string) (*User, error) { + ctx, cancel := dbCtx(ctx) + defer cancel() + var u User + err := a.db.QueryRow(ctx, ` + select id::text, username, display_name, password_hash, created_at + from users where id = $1::uuid`, id).Scan( + &u.ID, &u.Username, &u.DisplayName, &u.PasswordHash, &u.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + return &u, nil +} + +func (a *App) authUser(w http.ResponseWriter, r *http.Request) (*User, string) { + token, ok := parseBearer(r.Header.Get("Authorization")) + if !ok { + a.writeUnauthorized(w, r, "Bearer") + return nil, "" + } + hash, hashHex := tokenHash(token) + + if cs, hit := a.cache.GetSession(r.Context(), hashHex); hit { + u, err := a.getUserByID(r.Context(), cs.UserID) + if err != nil { + a.writeErr(w, reqIDFrom(r), 503, "unavailable", "database unavailable") + return nil, "" + } + if u == nil { + a.writeUnauthorized(w, r, "Bearer") + return nil, "" + } + return u, hashHex + } + + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var userID string + var expiresAt time.Time + err := a.db.QueryRow(ctx, ` + select user_id::text, expires_at from sessions + where token_hash = $1 and expires_at > now()`, hash).Scan(&userID, &expiresAt) + if errors.Is(err, pgx.ErrNoRows) { + a.writeUnauthorized(w, r, "Bearer") + return nil, "" + } + if err != nil { + a.writeErr(w, reqIDFrom(r), 503, "unavailable", "database unavailable") + return nil, "" + } + u, err := a.getUserByID(r.Context(), userID) + if err != nil { + a.writeErr(w, reqIDFrom(r), 503, "unavailable", "database unavailable") + return nil, "" + } + if u == nil { + a.writeUnauthorized(w, r, "Bearer") + return nil, "" + } + a.cache.SetSession(r.Context(), hashHex, userID, expiresAt) + return u, hashHex +} + +func (a *App) handleCreateUser(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + var body struct { + Username *string `json:"username"` + DisplayName *string `json:"display_name"` + Password *string `json:"password"` + } + if !a.readJSON(w, r, &body) { + return + } + if body.Username == nil || body.DisplayName == nil || body.Password == nil { + a.writeErr(w, reqID, 400, "invalid_request", "username, display_name and password required") + return + } + username := *body.Username + display := *body.DisplayName + password := *body.Password + if !usernameRe.MatchString(username) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid username") + return + } + n := utf8.RuneCountInString(display) + if n < 1 || n > 100 || isBlank(display) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid display_name") + return + } + if !validPassword(password) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid password") + return + } + ph, err := hashPassword(password) + if err != nil { + a.writeErr(w, reqID, 500, "internal_error", "internal error") + return + } + id := newUUID() + createdAt := nowUTC() + ctx, cancel := dbCtx(r.Context()) + defer cancel() + _, err = a.db.Exec(ctx, ` + insert into users(id, username, display_name, password_hash, created_at) + values($1::uuid, $2, $3, $4, $5)`, id, username, display, ph, createdAt) + if isPgCode(err, "23505") { + a.writeErr(w, reqID, 409, "conflict", "username taken") + return + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 201, map[string]any{ + "id": id, + "username": username, + "display_name": display, + "created_at": formatTime(createdAt), + }) +} + +func (a *App) handleCreateSession(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + username, password, ok := parseBasic(r.Header.Get("Authorization")) + if !ok { + a.writeUnauthorized(w, r, "Basic") + return + } + if !usernameRe.MatchString(username) || !validPassword(password) { + a.writeUnauthorized(w, r, "Basic") + return + } + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var u User + err := a.db.QueryRow(ctx, ` + select id::text, username, display_name, password_hash, created_at + from users where username = $1`, username).Scan( + &u.ID, &u.Username, &u.DisplayName, &u.PasswordHash, &u.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + a.writeUnauthorized(w, r, "Basic") + return + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + match, err := checkPassword(u.PasswordHash, password) + if err != nil || !match { + a.writeUnauthorized(w, r, "Basic") + return + } + token, hash, hashHex, err := newToken() + if err != nil { + a.writeErr(w, reqID, 500, "internal_error", "internal error") + return + } + createdAt := nowUTC() + expiresAt := createdAt.Add(a.sessionTTL) + _, err = a.db.Exec(ctx, ` + insert into sessions(token_hash, user_id, created_at, expires_at) + values($1, $2::uuid, $3, $4)`, hash, u.ID, createdAt, expiresAt) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.cache.SetSession(r.Context(), hashHex, u.ID, expiresAt) + a.writeJSON(w, 201, map[string]any{ + "token": token, + "token_type": "Bearer", + "expires_at": formatTime(expiresAt), + "user": userJSON(&u), + }) +} + +func (a *App) handleRevokeSession(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + token, ok := parseBearer(r.Header.Get("Authorization")) + if !ok { + a.writeUnauthorized(w, r, "Bearer") + return + } + hash, hashHex := tokenHash(token) + ctx, cancel := dbCtx(r.Context()) + defer cancel() + tag, err := a.db.Exec(ctx, ` + delete from sessions where token_hash = $1 and expires_at > now()`, hash) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.cache.DelSession(r.Context(), hashHex) + if tag.RowsAffected() == 0 { + a.writeUnauthorized(w, r, "Bearer") + return + } + w.Header().Set("X-Request-Id", reqID) + w.WriteHeader(204) +} + +func (a *App) handleMe(w http.ResponseWriter, r *http.Request) { + u, _ := a.authUser(w, r) + if u == nil { + return + } + a.writeJSON(w, 200, userJSON(u)) +} + +func (a *App) handleCreateChat(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + var body struct { + Title *string `json:"title"` + MemberIDs *[]string `json:"member_ids"` + } + if !a.readJSON(w, r, &body) { + return + } + if body.Title == nil || body.MemberIDs == nil { + a.writeErr(w, reqID, 400, "invalid_request", "title and member_ids required") + return + } + title := *body.Title + n := utf8.RuneCountInString(title) + if n < 1 || n > 100 || isBlank(title) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid title") + return + } + members := *body.MemberIDs + if len(members) < 1 || len(members) > 100 { + a.writeErr(w, reqID, 400, "invalid_request", "invalid member_ids") + return + } + seen := make(map[string]struct{}, len(members)) + hasCaller := false + for _, id := range members { + if !isUUID(id) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid member id") + return + } + if _, dup := seen[id]; dup { + a.writeErr(w, reqID, 400, "invalid_request", "duplicate member") + return + } + seen[id] = struct{}{} + if id == u.ID { + hasCaller = true + } + } + if !hasCaller { + a.writeErr(w, reqID, 400, "invalid_request", "caller must be member") + return + } + + chatID := newUUID() + createdAt := nowUTC() + ctx, cancel := dbCtx(r.Context()) + defer cancel() + tx, err := a.db.Begin(ctx) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + defer tx.Rollback(ctx) + + _, err = tx.Exec(ctx, `insert into chats(id, title, created_at) values($1::uuid, $2, $3)`, + chatID, title, createdAt) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + for _, mid := range members { + _, err = tx.Exec(ctx, `insert into chat_members(chat_id, user_id) values($1::uuid, $2::uuid)`, + chatID, mid) + if isPgCode(err, "23503") { + a.writeErr(w, reqID, 400, "invalid_request", "unknown member") + return + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + } + if err := tx.Commit(ctx); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 201, map[string]any{ + "id": chatID, + "title": title, + "member_ids": members, + "created_at": formatTime(createdAt), + }) +} + +func encodeCursor(parts ...string) string { + return base64.RawURLEncoding.EncodeToString([]byte(strings.Join(parts, "|"))) +} + +func decodeCursor(s string) ([]string, error) { + b, err := base64.RawURLEncoding.DecodeString(s) + if err != nil { + return nil, err + } + return strings.Split(string(b), "|"), nil +} + +func parseLimit(r *http.Request) (int, error) { + q := r.URL.Query().Get("limit") + if q == "" { + return 50, nil + } + n, err := strconv.Atoi(q) + if err != nil || n < 1 || n > 100 { + return 0, fmt.Errorf("bad limit") + } + return n, nil +} + +func (a *App) loadChatMembers(ctx context.Context, chatID string) ([]string, error) { + ctx, cancel := dbCtx(ctx) + defer cancel() + rows, err := a.db.Query(ctx, ` + select user_id::text from chat_members where chat_id = $1::uuid order by user_id`, chatID) + if err != nil { + return nil, err + } + defer rows.Close() + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + return ids, rows.Err() +} + +func (a *App) isMember(ctx context.Context, chatID, userID string) (bool, error) { + ctx, cancel := dbCtx(ctx) + defer cancel() + var ok bool + err := a.db.QueryRow(ctx, ` + select exists( + select 1 from chat_members where chat_id = $1::uuid and user_id = $2::uuid + )`, chatID, userID).Scan(&ok) + return ok, err +} + +func (a *App) handleListChats(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + limit, err := parseLimit(r) + if err != nil { + a.writeErr(w, reqID, 400, "invalid_request", "invalid limit") + return + } + var afterTS *time.Time + var afterID string + if cur := r.URL.Query().Get("cursor"); cur != "" { + parts, err := decodeCursor(cur) + if err != nil || len(parts) != 3 { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + if parts[0] != u.ID { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + ts, err := parseTime(parts[1]) + if err != nil || !isUUID(parts[2]) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + afterTS = &ts + afterID = parts[2] + } + + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var rows pgx.Rows + if afterTS == nil { + rows, err = a.db.Query(ctx, ` + select c.id::text, c.title, c.created_at + from chats c + join chat_members m on m.chat_id = c.id + where m.user_id = $1::uuid + order by c.created_at, c.id + limit $2`, u.ID, limit+1) + } else { + rows, err = a.db.Query(ctx, ` + select c.id::text, c.title, c.created_at + from chats c + join chat_members m on m.chat_id = c.id + where m.user_id = $1::uuid + and (c.created_at, c.id) > ($2, $3::uuid) + order by c.created_at, c.id + limit $4`, u.ID, *afterTS, afterID, limit+1) + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + defer rows.Close() + + type row struct { + id, title string + created time.Time + } + var list []row + for rows.Next() { + var x row + if err := rows.Scan(&x.id, &x.title, &x.created); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + list = append(list, x) + } + if err := rows.Err(); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + hasMore := len(list) > limit + if hasMore { + list = list[:limit] + } + out := make([]map[string]any, 0, len(list)) + for _, c := range list { + members, err := a.loadChatMembers(r.Context(), c.id) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + out = append(out, map[string]any{ + "id": c.id, + "title": c.title, + "member_ids": members, + "created_at": formatTime(c.created), + }) + } + var next any = nil + if hasMore && len(list) > 0 { + last := list[len(list)-1] + next = encodeCursor(u.ID, formatTime(last.created), last.id) + } + a.writeJSON(w, 200, map[string]any{"items": out, "next_cursor": next}) +} + +func (a *App) handleGetChat(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + chatID := r.PathValue("chat_id") + if !isUUID(chatID) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id") + return + } + ok, err := a.isMember(r.Context(), chatID, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if !ok { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var title string + var createdAt time.Time + err = a.db.QueryRow(ctx, `select title, created_at from chats where id = $1::uuid`, chatID). + Scan(&title, &createdAt) + if errors.Is(err, pgx.ErrNoRows) { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + members, err := a.loadChatMembers(r.Context(), chatID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 200, map[string]any{ + "id": chatID, + "title": title, + "member_ids": members, + "created_at": formatTime(createdAt), + }) +} + +func (a *App) handleCreateMessage(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + chatID := r.PathValue("chat_id") + if !isUUID(chatID) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id") + return + } + key := r.Header.Get("Idempotency-Key") + if !idempotencyKeyRe.MatchString(key) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid Idempotency-Key") + return + } + ok, err := a.isMember(r.Context(), chatID, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if !ok { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + var body struct { + Text *string `json:"text"` + AttachmentIDs []string `json:"attachment_ids"` + } + if !a.readJSON(w, r, &body) { + return + } + if body.Text == nil { + a.writeErr(w, reqID, 400, "invalid_request", "text required") + return + } + text := *body.Text + attachmentIDs := body.AttachmentIDs + if attachmentIDs == nil { + attachmentIDs = []string{} + } + if len(attachmentIDs) > 10 { + a.writeErr(w, reqID, 400, "invalid_request", "too many attachments") + return + } + seen := make(map[string]bool, len(attachmentIDs)) + for _, id := range attachmentIDs { + if !isUUID(id) || seen[id] { + a.writeErr(w, reqID, 400, "invalid_request", "invalid attachment_ids") + return + } + seen[id] = true + } + n := utf8.RuneCountInString(text) + if n > 4000 || (len(attachmentIDs) == 0 && (n < 1 || isBlank(text))) || (n > 0 && isBlank(text)) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid text") + return + } + if len(attachmentIDs) > 0 { + var found, failed int + cctx, ccancel := dbCtx(r.Context()) + err := a.db.QueryRow(cctx, ` + select count(*), count(*) filter (where status = 'failed') + from attachments where id = any($1::uuid[]) and chat_id = $2::uuid`, attachmentIDs, chatID).Scan(&found, &failed) + ccancel() + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if found != len(attachmentIDs) { + a.writeErr(w, reqID, 404, "not_found", "attachment not found") + return + } + if failed > 0 { + a.writeErr(w, reqID, 409, "conflict", "attachment processing failed") + return + } + } + hashInput := text + if len(attachmentIDs) > 0 { + hashInput += "\x00" + strings.Join(attachmentIDs, ",") + } + sum := sha256.Sum256([]byte(hashInput)) + bodyHash := hex.EncodeToString(sum[:]) + path := r.URL.Path + ctx, cancel := dbCtx(r.Context()) + defer cancel() + tx, err := a.db.Begin(ctx) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + defer tx.Rollback(ctx) + + if _, err := tx.Exec(ctx, ` + delete from idempotency_keys + where user_id=$1::uuid and method='POST' and path=$2 and key=$3 + and created_at < $4`, + u.ID, path, key, time.Now().Add(-a.idempotencyTTL)); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + msgID := newUUID() + createdAt := nowUTC() + resp := map[string]any{ + "id": msgID, + "chat_id": chatID, + "sender_id": u.ID, + "text": text, + "created_at": formatTime(createdAt), + "attachment_ids": attachmentIDs, + } + respJSON, _ := json.Marshal(resp) + tag, err := tx.Exec(ctx, ` + insert into idempotency_keys(user_id, method, path, key, body_hash, response, created_at) + values($1::uuid, 'POST', $2, $3, $4, $5, $6) + on conflict do nothing`, + u.ID, path, key, bodyHash, respJSON, createdAt) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if tag.RowsAffected() == 0 { + var storedHash string + var stored []byte + err := tx.QueryRow(ctx, ` + select body_hash, response from idempotency_keys + where user_id=$1::uuid and method='POST' and path=$2 and key=$3`, + u.ID, path, key).Scan(&storedHash, &stored) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if storedHash != bodyHash { + a.writeErr(w, reqID, 409, "conflict", "idempotency key reused with different body") + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(201) + _, _ = w.Write(append(stored, '\n')) + return + } + if _, err := tx.Exec(ctx, ` + insert into messages(id, chat_id, sender_id, text, created_at) + values($1::uuid, $2::uuid, $3::uuid, $4, $5)`, + msgID, chatID, u.ID, text, createdAt); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if len(attachmentIDs) > 0 { + if _, err := tx.Exec(ctx, ` + insert into message_attachments(message_id, attachment_id, position) + select $1::uuid, t.id, t.ord - 1 + from unnest($2::uuid[]) with ordinality as t(id, ord)`, + msgID, attachmentIDs); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + } + if err := tx.Commit(ctx); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + a.writeJSON(w, 201, resp) +} + +func (a *App) handleListMessages(w http.ResponseWriter, r *http.Request) { + reqID := reqIDFrom(r) + u, _ := a.authUser(w, r) + if u == nil { + return + } + chatID := r.PathValue("chat_id") + if !isUUID(chatID) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid chat_id") + return + } + ok, err := a.isMember(r.Context(), chatID, u.ID) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + if !ok { + a.writeErr(w, reqID, 404, "not_found", "chat not found") + return + } + limit, err := parseLimit(r) + if err != nil { + a.writeErr(w, reqID, 400, "invalid_request", "invalid limit") + return + } + var afterTS *time.Time + var afterID string + if cur := r.URL.Query().Get("cursor"); cur != "" { + parts, err := decodeCursor(cur) + if err != nil || len(parts) != 3 { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + if parts[0] != chatID { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + ts, err := parseTime(parts[1]) + if err != nil || !isUUID(parts[2]) { + a.writeErr(w, reqID, 400, "invalid_request", "invalid cursor") + return + } + afterTS = &ts + afterID = parts[2] + } + + ctx, cancel := dbCtx(r.Context()) + defer cancel() + var rows pgx.Rows + if afterTS == nil { + rows, err = a.db.Query(ctx, ` + select id::text, chat_id::text, sender_id::text, text, created_at + from messages + where chat_id = $1::uuid + order by created_at, id + limit $2`, chatID, limit+1) + } else { + rows, err = a.db.Query(ctx, ` + select id::text, chat_id::text, sender_id::text, text, created_at + from messages + where chat_id = $1::uuid + and (created_at, id) > ($2, $3::uuid) + order by created_at, id + limit $4`, chatID, *afterTS, afterID, limit+1) + } + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + defer rows.Close() + + type row struct { + id, chatID, senderID, text string + created time.Time + } + var list []row + for rows.Next() { + var x row + if err := rows.Scan(&x.id, &x.chatID, &x.senderID, &x.text, &x.created); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + list = append(list, x) + } + if err := rows.Err(); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + hasMore := len(list) > limit + if hasMore { + list = list[:limit] + } + attachments := map[string][]string{} + if len(list) > 0 { + ids := make([]string, len(list)) + for i, m := range list { + ids[i] = m.id + } + arows, err := a.db.Query(ctx, ` + select message_id::text, attachment_id::text + from message_attachments + where message_id = any($1::uuid[]) + order by message_id, position`, ids) + if err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + for arows.Next() { + var mid, aid string + if err := arows.Scan(&mid, &aid); err != nil { + arows.Close() + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + attachments[mid] = append(attachments[mid], aid) + } + arows.Close() + if err := arows.Err(); err != nil { + a.writeErr(w, reqID, 503, "unavailable", "database unavailable") + return + } + } + out := make([]map[string]any, 0, len(list)) + for _, m := range list { + ids := attachments[m.id] + if ids == nil { + ids = []string{} + } + out = append(out, map[string]any{ + "id": m.id, + "chat_id": m.chatID, + "sender_id": m.senderID, + "text": m.text, + "created_at": formatTime(m.created), + "attachment_ids": ids, + }) + } + var next any = nil + if hasMore && len(list) > 0 { + last := list[len(list)-1] + next = encodeCursor(chatID, formatTime(last.created), last.id) + } + a.writeJSON(w, 200, map[string]any{"items": out, "next_cursor": next}) +} + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} diff --git a/imaging.go b/imaging.go new file mode 100644 index 00000000..6ccc5f28 --- /dev/null +++ b/imaging.go @@ -0,0 +1,176 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "image" + "image/draw" + "image/jpeg" + _ "image/png" + + "github.com/gen2brain/webp" + "github.com/rwcarlsen/goexif/exif" + xdraw "golang.org/x/image/draw" +) + +const ( + jpegQuality = 85 + webpQuality = 80 + webpMethod = 4 +) + +var errInvalidImage = errors.New("invalid image") + +type variantSpec struct { + name, contentType, ext string + box int + crop bool +} + +var variantSpecs = []variantSpec{ + {"thumbnail", "image/jpeg", "jpg", 256, false}, + {"crop", "image/jpeg", "jpg", 128, true}, + {"optimized", "image/webp", "webp", 1280, false}, +} + +type rendered struct { + spec variantSpec + data []byte + width, height int + sha256 string +} + +func renderVariants(ctx context.Context, data []byte, maxPixels int64) ([]rendered, error) { + cfg, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || cfg.Width < 1 || cfg.Height < 1 { + return nil, errInvalidImage + } + if int64(cfg.Width)*int64(cfg.Height) > maxPixels { + return nil, errInvalidImage + } + src, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return nil, errInvalidImage + } + orient := 1 + if format == "jpeg" { + orient = orientation(data) + } + src = shrink(src, 1280) + var out []rendered + for _, spec := range variantSpecs { + if err := ctx.Err(); err != nil { + return nil, err + } + img := render(src, cfg.Width, cfg.Height, orient, spec) + var buf bytes.Buffer + if spec.contentType == "image/webp" { + err = webp.Encode(&buf, img, webp.Options{Quality: webpQuality, Method: webpMethod}) + } else { + err = jpeg.Encode(&buf, img, &jpeg.Options{Quality: jpegQuality}) + } + if err != nil { + return nil, err + } + sum := sha256.Sum256(buf.Bytes()) + b := img.Bounds() + out = append(out, rendered{spec: spec, data: buf.Bytes(), width: b.Dx(), height: b.Dy(), sha256: hex.EncodeToString(sum[:])}) + } + return out, nil +} + +func orientation(data []byte) int { + x, err := exif.Decode(bytes.NewReader(data)) + if err != nil { + return 1 + } + tag, err := x.Get(exif.Orientation) + if err != nil { + return 1 + } + v, err := tag.Int(0) + if err != nil || v < 1 || v > 8 { + return 1 + } + return v +} + +func fit(w, h, box int) (int, int) { + if w <= box && h <= box { + return w, h + } + if w >= h { + return box, max(1, h*box/w) + } + return max(1, w*box/h), box +} + +func shrink(src image.Image, limit int) image.Image { + for { + b := src.Bounds() + if max(b.Dx(), b.Dy())/2 < limit { + return src + } + dst := image.NewRGBA(image.Rect(0, 0, max(1, b.Dx()/2), max(1, b.Dy()/2))) + xdraw.ApproxBiLinear.Scale(dst, dst.Bounds(), src, b, xdraw.Src, nil) + src = dst + } +} + +func render(src image.Image, sw, sh, orient int, spec variantSpec) *image.RGBA { + b := src.Bounds() + rect := b + var tw, th int + if spec.crop { + side := min(b.Dx(), b.Dy()) + x0, y0 := b.Min.X+(b.Dx()-side)/2, b.Min.Y+(b.Dy()-side)/2 + rect = image.Rect(x0, y0, x0+side, y0+side) + tw, th = spec.box, spec.box + } else if orient >= 5 { + th, tw = fit(sh, sw, spec.box) + } else { + tw, th = fit(sw, sh, spec.box) + } + dst := image.NewRGBA(image.Rect(0, 0, tw, th)) + draw.Draw(dst, dst.Bounds(), image.White, image.Point{}, draw.Src) + xdraw.CatmullRom.Scale(dst, dst.Bounds(), src, rect, xdraw.Over, nil) + return orientImage(dst, orient) +} + +func orientImage(img *image.RGBA, o int) *image.RGBA { + if o <= 1 { + return img + } + w, h := img.Bounds().Dx(), img.Bounds().Dy() + dw, dh := w, h + if o >= 5 { + dw, dh = h, w + } + dst := image.NewRGBA(image.Rect(0, 0, dw, dh)) + for y := 0; y < h; y++ { + for x := 0; x < w; x++ { + var nx, ny int + switch o { + case 2: + nx, ny = w-1-x, y + case 3: + nx, ny = w-1-x, h-1-y + case 4: + nx, ny = x, h-1-y + case 5: + nx, ny = y, x + case 6: + nx, ny = h-1-y, x + case 7: + nx, ny = h-1-y, w-1-x + case 8: + nx, ny = y, w-1-x + } + copy(dst.Pix[dst.PixOffset(nx, ny):dst.PixOffset(nx, ny)+4], img.Pix[img.PixOffset(x, y):img.PixOffset(x, y)+4]) + } + } + return dst +} diff --git a/main.go b/main.go new file mode 100644 index 00000000..e72de1f6 --- /dev/null +++ b/main.go @@ -0,0 +1,199 @@ +package main + +import ( + "context" + "encoding/json" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func main() { + log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelInfo})) + + host := envOr("APP_HOST", "0.0.0.0") + port := envOr("APP_PORT", "8080") + shutdownSec, _ := strconv.Atoi(envOr("SHUTDOWN_TIMEOUT_SECONDS", "5")) + if shutdownSec < 1 { + shutdownSec = 5 + } + ttlSec, _ := strconv.Atoi(envOr("SESSION_TTL_SECONDS", "3600")) + if ttlSec < 1 { + ttlSec = 3600 + } + + idemSec, _ := strconv.Atoi(envOr("IDEMPOTENCY_TTL_SECONDS", "86400")) + if idemSec < 1 { + idemSec = 86400 + } + drainSec, _ := strconv.Atoi(envOr("DRAIN_SECONDS", "3")) + + ctx := context.Background() + if len(os.Args) > 1 && os.Args[1] == "s3-init" { + if err := s3Init(ctx); err != nil { + log.Error("s3 init failed", "err", err.Error()) + os.Exit(1) + } + log.Info("bucket ready") + return + } + db, err := openDB(ctx) + if err != nil { + log.Error("database open failed", "err", err.Error()) + os.Exit(1) + } + defer db.Close() + + if len(os.Args) > 1 && os.Args[1] == "migrate" { + if err := migrate(ctx, db); err != nil { + log.Error("migrate failed", "err", err.Error()) + os.Exit(1) + } + log.Info("migrations applied") + return + } + + store, err := openStore() + if err != nil { + log.Error("s3 open failed", "err", err.Error()) + os.Exit(1) + } + + if len(os.Args) > 1 && (os.Args[1] == "cleanup" || os.Args[1] == "janitor") { + runCleanup(ctx, db, store, log, os.Args[1] == "janitor") + return + } + + maxBytes, _ := strconv.ParseInt(envOr("MAX_UPLOAD_BYTES", "10485760"), 10, 64) + maxPixels, _ := strconv.ParseInt(envOr("MAX_IMAGE_PIXELS", "40000000"), 10, 64) + maxAttempts := envInt("JOB_MAX_ATTEMPTS", 5) + + if len(os.Args) > 1 && (os.Args[1] == "replay" || os.Args[1] == "failed") { + out := json.NewEncoder(os.Stdout) + if os.Args[1] == "failed" { + items, err := listFailed(ctx, db) + if err != nil { + log.Error("failed jobs query failed", "err", err.Error()) + os.Exit(1) + } + for _, item := range items { + _ = out.Encode(item) + } + return + } + if len(os.Args) != 3 || !isUUID(os.Args[2]) { + log.Error("usage: replay ") + os.Exit(2) + } + result, err := replayJob(ctx, db, os.Args[2], maxAttempts) + if err != nil { + log.Error("replay failed", "err", err.Error()) + os.Exit(1) + } + _ = out.Encode(map[string]string{"attachment_id": os.Args[2], "result": result}) + return + } + + broker, err := openBroker() + if err != nil { + log.Error("broker open failed", "err", err.Error()) + os.Exit(1) + } + + if len(os.Args) > 1 && os.Args[1] == "worker" { + timeout := time.Duration(envInt("JOB_TIMEOUT_SECONDS", 60)) * time.Second + owner, _ := os.Hostname() + w := &Worker{ + db: db, store: store, broker: broker, log: log, + owner: owner + "-" + newUUID()[:8], + timeout: timeout, + lease: timeout + 15*time.Second, + backoffBase: time.Duration(envInt("JOB_BACKOFF_BASE_MS", 1000)) * time.Millisecond, + backoffMax: time.Duration(envInt("JOB_BACKOFF_MAX_MS", 30000)) * time.Millisecond, + maxBytes: maxBytes, + maxPixels: maxPixels, + fault: os.Getenv("FAULT_POINT"), + } + runWorker(db, store, broker, log, w, envInt("WORKER_CONCURRENCY", 2)) + return + } + + cache := openCache(log) + defer cache.Close() + + slots := envInt("UPLOAD_CONCURRENCY", 10) + app := NewApp(db, cache, log, time.Duration(ttlSec)*time.Second, time.Duration(idemSec)*time.Second, store, + limits{maxBytes: maxBytes, maxPixels: maxPixels, uploadSlots: slots, maxAttempts: maxAttempts}) + relayCtx, stopRelay := context.WithCancel(ctx) + defer stopRelay() + go runRelay(relayCtx, db, broker, log) + + addr := net.JoinHostPort(host, port) + srv := &http.Server{ + Addr: addr, + Handler: app.Routes(), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 10 * time.Second, + WriteTimeout: 15 * time.Second, + IdleTimeout: 30 * time.Second, + } + + go func() { + log.Info("listening", "addr", addr, "env", envOr("APP_ENV", "lab"), "session_ttl_s", ttlSec) + if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { + log.Error("server error", "err", err.Error()) + os.Exit(1) + } + }() + + stop := make(chan os.Signal, 1) + signal.Notify(stop, syscall.SIGTERM, syscall.SIGINT) + sig := <-stop + log.Info("shutdown started", "signal", sig.String()) + app.SetReady(false) + time.Sleep(time.Duration(drainSec) * time.Second) + + shCtx, cancel := context.WithTimeout(context.Background(), time.Duration(shutdownSec)*time.Second) + defer cancel() + if err := srv.Shutdown(shCtx); err != nil { + log.Error("shutdown error", "err", err.Error()) + os.Exit(1) + } + log.Info("shutdown finished") +} + +func runCleanup(ctx context.Context, db *pgxpool.Pool, store *Store, log *slog.Logger, loop bool) { + graceSec, _ := strconv.Atoi(envOr("CLEANUP_GRACE_SECONDS", "600")) + intervalSec, _ := strconv.Atoi(envOr("CLEANUP_INTERVAL_SECONDS", "300")) + retention := time.Duration(envInt("STAGING_RETENTION_SECONDS", 86400)) * time.Second + for { + removed, err := cleanupOrphans(ctx, db, store, time.Duration(graceSec)*time.Second, retention, log) + if err != nil { + log.Error("cleanup failed", "err", err.Error()) + if !loop { + os.Exit(1) + } + } else { + log.Info("cleanup finished", "removed", removed) + } + if !loop { + return + } + time.Sleep(time.Duration(intervalSec) * time.Second) + } +} + +func envInt(key string, def int) int { + v, err := strconv.Atoi(os.Getenv(key)) + if err != nil || v < 1 { + return def + } + return v +} diff --git a/migrate.go b/migrate.go new file mode 100644 index 00000000..d2b953fe --- /dev/null +++ b/migrate.go @@ -0,0 +1,84 @@ +package main + +import ( + "context" + "embed" + "fmt" + "sort" + "strings" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +//go:embed migrations/*.sql +var migrationFS embed.FS + +func migrate(ctx context.Context, pool *pgxpool.Pool) error { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + conn, err := pool.Acquire(ctx) + if err != nil { + return err + } + defer conn.Release() + if _, err := conn.Exec(ctx, `select pg_advisory_lock(727001)`); err != nil { + return err + } + defer conn.Exec(context.Background(), `select pg_advisory_unlock(727001)`) + + _, err = conn.Exec(ctx, ` + create table if not exists schema_migrations ( + version text primary key, + applied_at timestamptz not null + )`) + if err != nil { + return fmt.Errorf("schema_migrations: %w", err) + } + + entries, err := migrationFS.ReadDir("migrations") + if err != nil { + return err + } + var names []string + for _, e := range entries { + if !e.IsDir() && strings.HasSuffix(e.Name(), ".sql") { + names = append(names, e.Name()) + } + } + sort.Strings(names) + + for _, name := range names { + var exists bool + err := conn.QueryRow(ctx, `select exists(select 1 from schema_migrations where version=$1)`, name).Scan(&exists) + if err != nil { + return err + } + if exists { + continue + } + + body, err := migrationFS.ReadFile("migrations/" + name) + if err != nil { + return err + } + + tx, err := conn.Begin(ctx) + if err != nil { + return err + } + if _, err := tx.Exec(ctx, string(body)); err != nil { + _ = tx.Rollback(ctx) + return fmt.Errorf("migration %s: %w", name, err) + } + if _, err := tx.Exec(ctx, `insert into schema_migrations(version, applied_at) values($1, now())`, name); err != nil { + _ = tx.Rollback(ctx) + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + } + return nil +} diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 00000000..a50290e5 --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,38 @@ +create table users ( + id uuid primary key, + username text not null unique, + display_name text not null, + password_hash text not null, + created_at timestamptz not null +); + +create table chats ( + id uuid primary key, + title text not null, + created_at timestamptz not null +); + +create table chat_members ( + chat_id uuid not null references chats(id), + user_id uuid not null references users(id), + primary key (chat_id, user_id) +); + +create index chat_members_user_idx on chat_members (user_id); + +create table messages ( + id uuid primary key, + chat_id uuid not null references chats(id), + sender_id uuid not null references users(id), + text text not null, + created_at timestamptz not null +); + +create index messages_history_idx on messages (chat_id, created_at, id); + +create table sessions ( + token_hash bytea primary key, + user_id uuid not null references users(id) on delete cascade, + created_at timestamptz not null, + expires_at timestamptz not null +); diff --git a/migrations/0002_idempotency.sql b/migrations/0002_idempotency.sql new file mode 100644 index 00000000..0cfbc2d6 --- /dev/null +++ b/migrations/0002_idempotency.sql @@ -0,0 +1,10 @@ +create table idempotency_keys ( + user_id uuid not null references users(id) on delete cascade, + method text not null, + path text not null, + key text not null, + body_hash text not null, + response jsonb not null, + created_at timestamptz not null, + primary key (user_id, method, path, key) +); diff --git a/migrations/0003_attachments.sql b/migrations/0003_attachments.sql new file mode 100644 index 00000000..00d00fbe --- /dev/null +++ b/migrations/0003_attachments.sql @@ -0,0 +1,22 @@ +create table attachments ( + id uuid primary key, + chat_id uuid not null references chats(id), + owner_id uuid not null references users(id), + filename text not null, + content_type text not null, + size_bytes bigint not null, + sha256 text not null, + object_key text not null unique, + created_at timestamptz not null +); + +create index attachments_chat_idx on attachments (chat_id); + +create table message_attachments ( + message_id uuid not null references messages(id), + attachment_id uuid not null references attachments(id), + position int not null, + primary key (message_id, attachment_id) +); + +create index message_attachments_attachment_idx on message_attachments (attachment_id); diff --git a/migrations/0004_jobs.sql b/migrations/0004_jobs.sql new file mode 100644 index 00000000..feddbf0a --- /dev/null +++ b/migrations/0004_jobs.sql @@ -0,0 +1,60 @@ +alter table attachments + add column status text not null default 'ready', + add column error_code text, + add column staging_key text, + add column updated_at timestamptz; + +update attachments set updated_at = created_at; + +alter table attachments + alter column status drop default, + alter column updated_at set not null, + alter column object_key drop not null; + +create table attachment_variants ( + attachment_id uuid not null references attachments(id), + pipeline_version int not null, + name text not null, + object_key text not null unique, + content_type text not null, + size_bytes bigint not null, + width int not null, + height int not null, + sha256 text not null, + primary key (attachment_id, pipeline_version, name) +); + +create table jobs ( + id uuid primary key, + attachment_id uuid not null unique references attachments(id), + pipeline_version int not null, + status text not null, + attempt int not null, + max_attempts int not null, + next_attempt_at timestamptz not null, + lease_owner text, + lease_until timestamptz, + last_error text, + created_at timestamptz not null, + updated_at timestamptz not null +); + +create index jobs_lease_idx on jobs (lease_until) where status = 'processing'; + +create table outbox ( + id bigserial primary key, + job_id uuid not null references jobs(id), + attempt int not null, + available_at timestamptz not null, + published_at timestamptz, + created_at timestamptz not null +); + +create index outbox_pending_idx on outbox (available_at) where published_at is null; + +create table job_replays ( + id bigserial primary key, + job_id uuid not null references jobs(id), + previous_error text, + replayed_at timestamptz not null +); diff --git a/models.go b/models.go new file mode 100644 index 00000000..c6fd8042 --- /dev/null +++ b/models.go @@ -0,0 +1,33 @@ +package main + +import "time" + +type User struct { + ID string + Username string + DisplayName string + PasswordHash string + CreatedAt time.Time +} + +type Chat struct { + ID string + Title string + MemberIDs []string + CreatedAt time.Time +} + +type Message struct { + ID string + ChatID string + SenderID string + Text string + CreatedAt time.Time +} + +type Session struct { + TokenHash []byte + UserID string + CreatedAt time.Time + ExpiresAt time.Time +} diff --git a/s3.go b/s3.go new file mode 100644 index 00000000..5f4e9bdd --- /dev/null +++ b/s3.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "io" + "net" + "net/http" + "net/url" + "time" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type Store struct { + client *minio.Client + bucket string +} + +func openStore() (*Store, error) { + endpoint, err := url.Parse(envOr("S3_ENDPOINT", "http://s3:9000")) + if err != nil { + return nil, err + } + keyID, err := readSecret("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE") + if err != nil { + return nil, err + } + secret, err := readSecret("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE") + if err != nil { + return nil, err + } + transport := &http.Transport{ + DialContext: (&net.Dialer{Timeout: 2 * time.Second}).DialContext, + ResponseHeaderTimeout: 10 * time.Second, + MaxIdleConnsPerHost: 16, + IdleConnTimeout: 30 * time.Second, + } + client, err := minio.New(endpoint.Host, &minio.Options{ + Creds: credentials.NewStaticV4(keyID, secret, ""), + Secure: endpoint.Scheme == "https", + Region: envOr("S3_REGION", "us-east-1"), + Transport: transport, + MaxRetries: 1, + }) + if err != nil { + return nil, err + } + return &Store{client: client, bucket: envOr("S3_BUCKET", "messenger-attachments")}, nil +} + +func (s *Store) Put(ctx context.Context, key string, r io.Reader, size int64, contentType string) error { + _, err := s.client.PutObject(ctx, s.bucket, key, r, size, minio.PutObjectOptions{ + ContentType: contentType, + PartSize: 5 << 20, + }) + return err +} + +func (s *Store) Get(ctx context.Context, key string) (*minio.Object, minio.ObjectInfo, error) { + obj, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, minio.ObjectInfo{}, err + } + info, err := obj.Stat() + if err != nil { + obj.Close() + return nil, minio.ObjectInfo{}, err + } + return obj, info, nil +} + +func (s *Store) Remove(ctx context.Context, key string) error { + return s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}) +} + +func (s *Store) List(ctx context.Context, prefix string) <-chan minio.ObjectInfo { + return s.client.ListObjects(ctx, s.bucket, minio.ListObjectsOptions{Prefix: prefix, Recursive: true}) +} + +func isNoSuchKey(err error) bool { + return minio.ToErrorResponse(err).Code == "NoSuchKey" +} + +func s3Init(ctx context.Context) error { + s, err := openStore() + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + for { + exists, err := s.client.BucketExists(ctx, s.bucket) + if err == nil { + if exists { + return nil + } + err = s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: envOr("S3_REGION", "us-east-1")}) + if err == nil || minio.ToErrorResponse(err).Code == "BucketAlreadyOwnedByYou" { + return nil + } + } + select { + case <-ctx.Done(): + return err + case <-time.After(2 * time.Second): + } + } +} diff --git a/worker.go b/worker.go new file mode 100644 index 00000000..d8cfe749 --- /dev/null +++ b/worker.go @@ -0,0 +1,448 @@ +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "math/rand/v2" + "os" + "os/signal" + "strings" + "sync" + "syscall" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/redis/go-redis/v9" +) + +var errSource = errors.New("staging object missing or corrupted") + +type Worker struct { + db *pgxpool.Pool + store *Store + broker *Broker + log *slog.Logger + owner string + timeout time.Duration + lease time.Duration + backoffBase time.Duration + backoffMax time.Duration + maxBytes int64 + maxPixels int64 + fault string +} + +type claimedJob struct { + id, attachmentID, owner string + attempt int +} + +type source struct { + stagingKey, contentType, sha256 string + size int64 +} + +func finalKey(attachmentID string) string { + return objectPrefix + attachmentID + "/original" +} + +func variantKey(attachmentID string, spec variantSpec) string { + return fmt.Sprintf("%s%s/v%d/%s.%s", objectPrefix, attachmentID, pipelineVersion, spec.name, spec.ext) +} + +func (w *Worker) inject(point string) { + if w.fault == point { + w.log.Error("fault injected", "point", point) + os.Exit(3) + } +} + +func (w *Worker) backoff(attempt int) time.Duration { + d := w.backoffBase << (attempt - 1) + if d <= 0 || d > w.backoffMax { + d = w.backoffMax + } + return d/2 + rand.N(d/2+1) +} + +func (w *Worker) claim(ctx context.Context, jobID string) (*claimedJob, error) { + tx, err := w.db.Begin(ctx) + if err != nil { + return nil, err + } + defer tx.Rollback(ctx) + job := &claimedJob{id: jobID, owner: w.owner} + err = tx.QueryRow(ctx, ` + update jobs set status = 'processing', lease_owner = $2, + lease_until = now() + make_interval(secs => $3), updated_at = now() + where id = $1::uuid and status = 'queued' and next_attempt_at <= now() + returning attachment_id::text, attempt`, jobID, w.owner, w.lease.Seconds()).Scan(&job.attachmentID, &job.attempt) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if _, err := tx.Exec(ctx, `update attachments set status = 'processing', updated_at = now() where id = $1::uuid`, job.attachmentID); err != nil { + return nil, err + } + return job, tx.Commit(ctx) +} + +func (w *Worker) fetch(ctx context.Context, src source) ([]byte, error) { + obj, info, err := w.store.Get(ctx, src.stagingKey) + if err != nil { + if isNoSuchKey(err) { + return nil, errSource + } + return nil, err + } + defer obj.Close() + if info.Size != src.size || info.Size > w.maxBytes { + return nil, errSource + } + data, err := io.ReadAll(io.LimitReader(obj, w.maxBytes+1)) + if err != nil { + return nil, err + } + sum := sha256.Sum256(data) + if int64(len(data)) != src.size || hex.EncodeToString(sum[:]) != src.sha256 { + return nil, errSource + } + return data, nil +} + +func (w *Worker) execute(ctx context.Context, job *claimedJob) error { + ctx, cancel := context.WithTimeout(ctx, w.timeout) + defer cancel() + var src source + err := w.db.QueryRow(ctx, ` + select coalesce(staging_key, ''), content_type, sha256, size_bytes from attachments where id = $1::uuid`, + job.attachmentID).Scan(&src.stagingKey, &src.contentType, &src.sha256, &src.size) + if err != nil { + return err + } + data, err := w.fetch(ctx, src) + if err != nil { + return err + } + variants, err := renderVariants(ctx, data, w.maxPixels) + if err != nil { + return err + } + original := finalKey(job.attachmentID) + if err := w.store.Put(ctx, original, bytes.NewReader(data), int64(len(data)), src.contentType); err != nil { + return err + } + for _, v := range variants { + if err := w.store.Put(ctx, variantKey(job.attachmentID, v.spec), bytes.NewReader(v.data), int64(len(v.data)), v.spec.contentType); err != nil { + return err + } + } + w.inject("after_store") + + tx, err := w.db.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + tag, err := tx.Exec(ctx, ` + update jobs set status = 'ready', lease_owner = null, lease_until = null, last_error = null, updated_at = now() + where id = $1::uuid and status = 'processing' and lease_owner = $2 and attempt = $3`, job.id, job.owner, job.attempt) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + w.log.Warn("lease lost, result discarded", "job_id", job.id, "attempt", job.attempt) + return nil + } + for _, v := range variants { + _, err := tx.Exec(ctx, ` + insert into attachment_variants(attachment_id, pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256) + values($1::uuid, $2, $3, $4, $5, $6, $7, $8, $9) + on conflict (attachment_id, pipeline_version, name) do update + set object_key = excluded.object_key, content_type = excluded.content_type, size_bytes = excluded.size_bytes, + width = excluded.width, height = excluded.height, sha256 = excluded.sha256`, + job.attachmentID, pipelineVersion, v.spec.name, variantKey(job.attachmentID, v.spec), v.spec.contentType, + len(v.data), v.width, v.height, v.sha256) + if err != nil { + return err + } + } + if _, err := tx.Exec(ctx, ` + update attachments set status = 'ready', error_code = null, object_key = $2, updated_at = now() + where id = $1::uuid`, job.attachmentID, original); err != nil { + return err + } + if err := tx.Commit(ctx); err != nil { + return err + } + w.log.Info("job ready", "job_id", job.id, "attachment_id", job.attachmentID, "attempt", job.attempt) + w.inject("before_ack") + if err := w.store.Remove(ctx, src.stagingKey); err != nil { + w.log.Warn("staging cleanup deferred to janitor", "attachment_id", job.attachmentID, "err", err.Error()) + } + return nil +} + +func (w *Worker) settle(ctx context.Context, job *claimedJob, permanent bool, code, reason string) error { + tx, err := w.db.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + var maxAttempts int + if err := tx.QueryRow(ctx, `select max_attempts from jobs where id = $1::uuid`, job.id).Scan(&maxAttempts); err != nil { + return err + } + fence := ` where id = $1::uuid and status = 'processing' and lease_owner = $2 and attempt = $3` + if !permanent && job.attempt < maxAttempts { + delay := w.backoff(job.attempt) + tag, err := tx.Exec(ctx, ` + update jobs set status = 'queued', attempt = attempt + 1, next_attempt_at = now() + make_interval(secs => $4), + lease_owner = null, lease_until = null, last_error = $5, updated_at = now()`+fence, + job.id, job.owner, job.attempt, delay.Seconds(), reason) + if err != nil || tag.RowsAffected() == 0 { + return err + } + if _, err := tx.Exec(ctx, ` + insert into outbox(job_id, attempt, available_at, created_at) + values($1::uuid, $2, now() + make_interval(secs => $3), now())`, job.id, job.attempt+1, delay.Seconds()); err != nil { + return err + } + if _, err := tx.Exec(ctx, `update attachments set status = 'queued', updated_at = now() where id = $1::uuid`, job.attachmentID); err != nil { + return err + } + w.log.Warn("job retry scheduled", "job_id", job.id, "attempt", job.attempt, "next_attempt", job.attempt+1, + "delay_ms", delay.Milliseconds(), "reason", reason) + return tx.Commit(ctx) + } + if !permanent { + code = "retry_exhausted" + } + tag, err := tx.Exec(ctx, ` + update jobs set status = 'failed', lease_owner = null, lease_until = null, last_error = $4, updated_at = now()`+fence, + job.id, job.owner, job.attempt, reason) + if err != nil || tag.RowsAffected() == 0 { + return err + } + if _, err := tx.Exec(ctx, ` + update attachments set status = 'failed', error_code = $2, updated_at = now() where id = $1::uuid`, job.attachmentID, code); err != nil { + return err + } + w.log.Error("job failed", "job_id", job.id, "attempt", job.attempt, "error_code", code, "reason", reason) + return tx.Commit(ctx) +} + +func (w *Worker) handle(ctx context.Context, msg redis.XMessage) bool { + var m jobMessage + raw, _ := msg.Values["job"].(string) + if err := json.Unmarshal([]byte(raw), &m); err != nil || m.SchemaVersion != 1 || m.Type != jobType || !isUUID(m.JobID) { + w.log.Warn("invalid job message dropped", "stream_id", msg.ID) + return true + } + job, err := w.claim(ctx, m.JobID) + if err != nil { + w.log.Warn("claim failed", "job_id", m.JobID, "err", err.Error()) + return false + } + if job == nil { + var status string + var attempt int + _ = w.db.QueryRow(ctx, `select status, attempt from jobs where id = $1::uuid`, m.JobID).Scan(&status, &attempt) + w.log.Info("job delivery skipped", "job_id", m.JobID, "message_attempt", m.Attempt, "status", status, "attempt", attempt) + return true + } + err = w.execute(ctx, job) + if err == nil { + return true + } + permanent, code := false, "processing_failed" + switch { + case errors.Is(err, errInvalidImage): + permanent, code = true, "invalid_image" + case errors.Is(err, errSource): + permanent = true + } + if err := w.settle(ctx, job, permanent, code, err.Error()); err != nil { + w.log.Warn("settle failed", "job_id", job.id, "err", err.Error()) + return false + } + return true +} + +func (w *Worker) read(ctx context.Context, consumer, id string, block time.Duration) ([]redis.XMessage, error) { + res, err := w.broker.rdb.XReadGroup(ctx, &redis.XReadGroupArgs{ + Group: jobGroup, Consumer: consumer, Streams: []string{jobStream, id}, Count: 1, Block: block, + }).Result() + if len(res) == 0 { + return nil, err + } + return res[0].Messages, err +} + +func (w *Worker) consume(ctx context.Context, consumer string) { + pending := true + nextClaim := time.Now().Add(w.lease) + for ctx.Err() == nil { + var msgs []redis.XMessage + var err error + switch { + case pending: + msgs, err = w.read(ctx, consumer, "0", -1) + pending = len(msgs) > 0 + case time.Now().After(nextClaim): + msgs, _, err = w.broker.rdb.XAutoClaim(ctx, &redis.XAutoClaimArgs{ + Stream: jobStream, Group: jobGroup, Consumer: consumer, MinIdle: w.lease, Start: "0-0", Count: 1, + }).Result() + nextClaim = time.Now().Add(15 * time.Second) + default: + msgs, err = w.read(ctx, consumer, ">", 5*time.Second) + } + if err != nil && !errors.Is(err, redis.Nil) { + if ctx.Err() != nil { + return + } + if strings.Contains(err.Error(), "NOGROUP") { + _ = w.broker.ensureGroup(ctx) + } + w.log.Warn("broker read failed", "consumer", consumer, "err", err.Error()) + time.Sleep(2 * time.Second) + continue + } + for _, msg := range msgs { + if !w.handle(context.WithoutCancel(ctx), msg) { + pending = false + continue + } + if err := w.broker.rdb.XAck(context.WithoutCancel(ctx), jobStream, jobGroup, msg.ID).Err(); err != nil { + w.log.Warn("ack failed", "stream_id", msg.ID, "err", err.Error()) + } + } + } +} + +func (w *Worker) reap(ctx context.Context) error { + rows, err := w.db.Query(ctx, ` + select id::text, attachment_id::text, attempt, coalesce(lease_owner, '') + from jobs where status = 'processing' and lease_until < now() limit 50`) + if err != nil { + return err + } + jobs, err := pgx.CollectRows(rows, func(row pgx.CollectableRow) (*claimedJob, error) { + j := &claimedJob{} + return j, row.Scan(&j.id, &j.attachmentID, &j.attempt, &j.owner) + }) + if err != nil { + return err + } + for _, j := range jobs { + w.log.Warn("lease expired", "job_id", j.id, "attempt", j.attempt) + if err := w.settle(ctx, j, false, "", "lease expired"); err != nil { + return err + } + } + return nil +} + +func runWorker(db *pgxpool.Pool, store *Store, broker *Broker, log *slog.Logger, w *Worker, concurrency int) { + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT) + defer stop() + for { + err := broker.ensureGroup(ctx) + if err == nil { + break + } + log.Warn("broker not ready", "err", err.Error()) + select { + case <-ctx.Done(): + return + case <-time.After(2 * time.Second): + } + } + host, _ := os.Hostname() + var wg sync.WaitGroup + wg.Go(func() { + for ctx.Err() == nil { + if err := w.reap(ctx); err != nil && ctx.Err() == nil { + log.Warn("lease reaper failed", "err", err.Error()) + } + select { + case <-ctx.Done(): + case <-time.After(5 * time.Second): + } + } + }) + for i := range concurrency { + consumer := fmt.Sprintf("%s-%d", host, i) + wg.Go(func() { w.consume(ctx, consumer) }) + } + log.Info("worker started", "owner", w.owner, "concurrency", concurrency, "lease_s", w.lease.Seconds()) + <-ctx.Done() + log.Info("worker stopping") + wg.Wait() +} + +func replayJob(ctx context.Context, db *pgxpool.Pool, attachmentID string, maxAttempts int) (string, error) { + tx, err := db.Begin(ctx) + if err != nil { + return "", err + } + defer tx.Rollback(ctx) + var jobID, status string + var attempt int + var lastErr *string + err = tx.QueryRow(ctx, `select id::text, status, attempt, last_error from jobs where attachment_id = $1::uuid for update`, + attachmentID).Scan(&jobID, &status, &attempt, &lastErr) + if errors.Is(err, pgx.ErrNoRows) { + return "not_found", nil + } + if err != nil { + return "", err + } + if status != "failed" { + return "skipped_" + status, nil + } + if _, err := tx.Exec(ctx, `insert into job_replays(job_id, previous_error, replayed_at) values($1::uuid, $2, now())`, jobID, lastErr); err != nil { + return "", err + } + if _, err := tx.Exec(ctx, ` + update jobs set status = 'queued', attempt = attempt + 1, max_attempts = attempt + $2, next_attempt_at = now(), + last_error = null, updated_at = now() + where id = $1::uuid`, jobID, maxAttempts); err != nil { + return "", err + } + if _, err := tx.Exec(ctx, `insert into outbox(job_id, attempt, available_at, created_at) values($1::uuid, $2, now(), now())`, jobID, attempt+1); err != nil { + return "", err + } + if _, err := tx.Exec(ctx, `update attachments set status = 'queued', error_code = null, updated_at = now() where id = $1::uuid`, attachmentID); err != nil { + return "", err + } + return "queued", tx.Commit(ctx) +} + +func listFailed(ctx context.Context, db *pgxpool.Pool) ([]map[string]any, error) { + rows, err := db.Query(ctx, ` + select j.attachment_id::text, j.id::text, j.attempt, coalesce(a.error_code, ''), coalesce(j.last_error, ''), j.updated_at + from jobs j join attachments a on a.id = j.attachment_id + where j.status = 'failed' order by j.updated_at`) + if err != nil { + return nil, err + } + return pgx.CollectRows(rows, func(row pgx.CollectableRow) (map[string]any, error) { + var att, job, code, reason string + var attempt int + var updated time.Time + err := row.Scan(&att, &job, &attempt, &code, &reason, &updated) + return map[string]any{"attachment_id": att, "job_id": job, "attempt": attempt, "error_code": code, + "last_error": reason, "failed_at": formatTime(updated)}, err + }) +} -- 2.54.0 From 60f0f2bf13d0a4efe5e37d5c3b8bc000fb261620 Mon Sep 17 00:00:00 2001 From: gera Date: Fri, 6 Nov 2026 01:59:07 +0300 Subject: [PATCH 2/5] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D0=BB?= =?UTF-8?q?=20=D0=B1=D1=80=D0=BE=D0=BA=D0=B5=D1=80=20=D0=B8=20worker=20?= =?UTF-8?q?=D0=B2=20compose-=D1=81=D1=82=D0=B5=D0=BD=D0=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 19 +-- compose.yaml | 321 ++++++++++++++++++++++++++++++++++++++ lb/haproxy.cfg | 38 +++++ scripts/mksecrets.sh | 28 ++++ scripts/restart.sh | 7 + scripts/rolling-update.sh | 16 ++ scripts/stop-one.sh | 10 ++ 7 files changed, 425 insertions(+), 14 deletions(-) create mode 100644 compose.yaml create mode 100644 lb/haproxy.cfg create mode 100755 scripts/mksecrets.sh create mode 100755 scripts/restart.sh create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/stop-one.sh diff --git a/.env.example b/.env.example index 119384b5..f6ece76a 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,16 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +API_REPLICAS=2 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url +CLEANUP_GRACE_SECONDS=600 +CLEANUP_INTERVAL_SECONDS=300 WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_BACKOFF_BASE_MS=1000 +JOB_BACKOFF_MAX_MS=30000 STAGING_RETENTION_SECONDS=86400 diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..faeb251c --- /dev/null +++ b/compose.yaml @@ -0,0 +1,321 @@ +x-api-env: &api-env + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + SESSION_STORE: postgres + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-10} + GOMEMLIMIT: 150MiB + SHUTDOWN_TIMEOUT_SECONDS: 10 + DRAIN_SECONDS: 3 + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + BROKER_URL_FILE: /run/secrets/broker_url + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + FAULT_POINT: ${API_FAULT_POINT:-} + +services: + postgres: + image: postgres:17-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24 + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_DB: messenger + secrets: + - postgres_password + volumes: + - pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + networks: + - backend + restart: unless-stopped + + valkey: + image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: ["valkey-server", "--appendonly", "yes", "--save", "60", "1"] + volumes: + - valkey_data:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: unless-stopped + + broker: + image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: ["valkey-server", "/run/secrets/broker_conf", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory-policy", "noeviction"] + secrets: + - broker_conf + volumes: + - broker_data:/data + healthcheck: + test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$(sed -n 's/^requirepass //p' /run/secrets/broker_conf)\" ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: unless-stopped + + s3: + image: chrislusf/seaweedfs@sha256:c2658820e763294a9e9de7b36c81e59b879e3459148351aee4e31a126ca64860 + command: + - server + - -dir=/data + - -ip=s3 + - -master.volumeSizeLimitMB=128 + - -volume.max=30 + - -s3 + - -s3.port=9000 + - -s3.config=/run/secrets/s3_config + secrets: + - s3_config + volumes: + - s3_data:/data + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9333/dir/assign | grep -q fid"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + networks: + - storage + restart: unless-stopped + + s3-init: + image: mtusi-messenger:lab5 + command: ["s3-init"] + environment: + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + depends_on: + s3: + condition: service_healthy + migrate: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - storage + restart: "no" + + janitor: + image: mtusi-messenger:lab5 + command: ["janitor"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + CLEANUP_GRACE_SECONDS: ${CLEANUP_GRACE_SECONDS:-600} + CLEANUP_INTERVAL_SECONDS: ${CLEANUP_INTERVAL_SECONDS:-300} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3-init: + condition: service_completed_successfully + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.25" + memory: 64M + networks: + - backend + - storage + restart: unless-stopped + + worker: + image: mtusi-messenger:lab5 + command: ["worker"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_BACKOFF_BASE_MS: ${JOB_BACKOFF_BASE_MS:-1000} + JOB_BACKOFF_MAX_MS: ${JOB_BACKOFF_MAX_MS:-30000} + GOMEMLIMIT: 600MiB + FAULT_POINT: ${WORKER_FAULT_POINT:-} + secrets: + - database_url + - broker_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3-init: + condition: service_completed_successfully + broker: + condition: service_healthy + healthcheck: + disable: true + stop_grace_period: 30s + deploy: + resources: + limits: + cpus: "1.0" + memory: 768M + networks: + - backend + - storage + restart: unless-stopped + + migrate: + build: . + image: mtusi-messenger:lab5 + command: ["migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: "no" + + api: + image: mtusi-messenger:lab5 + environment: *api-env + secrets: + - database_url + - cache_url + - broker_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3-init: + condition: service_completed_successfully + valkey: + condition: service_healthy + broker: + condition: service_healthy + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "0.5" + memory: 192M + networks: + - backend + - storage + restart: unless-stopped + + lb: + image: haproxy:3.0-alpine@sha256:cf3af0facf7eb22b9fee463daf0fbc07ead740b8150696dce724264faf7b4318 + ports: + - "127.0.0.1:8080:8080" + volumes: + - ./lb/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + - api + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + networks: + - backend + restart: unless-stopped + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + broker_conf: + file: ./secrets/broker.conf + broker_url: + file: ./secrets/broker_url + s3_config: + file: ./secrets/s3_config.json + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + +volumes: + pg_data: + valkey_data: + broker_data: + s3_data: + +networks: + backend: + driver: bridge + storage: + driver: bridge diff --git a/lb/haproxy.cfg b/lb/haproxy.cfg new file mode 100644 index 00000000..d63fed5b --- /dev/null +++ b/lb/haproxy.cfg @@ -0,0 +1,38 @@ +global + log stdout format raw local0 + maxconn 2000 + +resolvers docker + nameserver dns 127.0.0.11:53 + accepted_payload_size 8192 + hold valid 1s + hold nx 1s + hold obsolete 1s + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + +defaults + mode http + log global + option httplog + option redispatch 1 + retries 3 + retry-on conn-failure empty-response + timeout connect 1s + timeout client 20s + timeout server 15s + timeout http-request 10s + timeout queue 5s + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr none on-marked-down shutdown-sessions + server-template api 1-8 api:8080 diff --git a/scripts/mksecrets.sh b/scripts/mksecrets.sh new file mode 100755 index 00000000..a036162f --- /dev/null +++ b/scripts/mksecrets.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +mkdir -p "$ROOT/secrets" +cd "$ROOT/secrets" +gen() { + [ -f "$1" ] || openssl rand -hex "$2" | tr -d '\n' > "$1" +} +if [ ! -f postgres_password ]; then + openssl rand -base64 24 | tr -d '\n' > postgres_password +fi +PASS=$(cat postgres_password) +ENC=$(printf '%s' "$PASS" | python3 -c 'import sys,urllib.parse; print(urllib.parse.quote(sys.stdin.read(), safe=""))') +printf 'postgres://messenger:%s@postgres:5432/messenger?sslmode=disable\n' "$ENC" > database_url +printf 'redis://valkey:6379/0\n' > cache_url +gen broker_password 24 +printf 'requirepass %s\n' "$(cat broker_password)" > broker.conf +printf 'redis://:%s@broker:6379/0\n' "$(cat broker_password)" > broker_url +gen s3_admin_access_key_id 8 +gen s3_admin_secret_access_key 20 +gen s3_access_key_id 8 +gen s3_secret_access_key 20 +BUCKET=$(grep -E '^S3_BUCKET=' "$ROOT/.env" 2>/dev/null | cut -d= -f2 || true) +BUCKET=${BUCKET:-messenger-attachments} +printf '{"identities":[{"name":"admin","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","Write","List","Tagging"]},{"name":"api","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Read:%s","Write:%s","List:%s"]}]}\n' \ + "$(cat s3_admin_access_key_id)" "$(cat s3_admin_secret_access_key)" "$(cat s3_access_key_id)" "$(cat s3_secret_access_key)" "$BUCKET" "$BUCKET" "$BUCKET" > s3_config.json +chmod 644 ./* +echo "secrets ready in $ROOT/secrets" diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..dc4cf7bc --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +docker compose --env-file .env restart postgres valkey broker s3 +docker compose --env-file .env up -d --wait postgres valkey broker s3 +docker compose --env-file .env restart api lb janitor worker diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..9cee552c --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +dc="docker compose --env-file .env" +old=$($dc ps -q api) +n=$(echo "$old" | wc -w) +$dc build migrate +$dc up -d --no-deps migrate janitor worker +$dc up -d --no-deps --no-recreate --scale api=$((n * 2)) api +$dc up -d --wait --no-deps --no-recreate --scale api=$((n * 2)) api +for c in $old; do + docker stop -t 10 "$c" >/dev/null + docker rm "$c" >/dev/null +done +$dc up -d --wait --no-deps --no-recreate --scale api=$n api diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..1283ec1b --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?}" +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +cd "$ROOT" +id=$(docker compose --env-file .env ps -q api | while read -r c; do + case "$c" in "$TARGET_INSTANCE_ID"*) echo "$c";; esac +done) +[ -n "$id" ] || exit 1 +docker stop -t 10 "$id" >/dev/null -- 2.54.0 From d9e47a47d4e2f5f8fea33034e90ce57cce1f0206 Mon Sep 17 00:00:00 2001 From: gera Date: Fri, 6 Nov 2026 01:59:07 +0300 Subject: [PATCH 3/5] =?UTF-8?q?=D0=9D=D0=B0=D0=BF=D0=B8=D1=81=D0=B0=D0=BB?= =?UTF-8?q?=20=D1=82=D0=B5=D1=81=D1=82=D1=8B=20=D0=B2=D0=B0=D1=80=D0=B8?= =?UTF-8?q?=D0=B0=D0=BD=D1=82=D0=BE=D0=B2,=20=D0=B2=D0=BE=D1=81=D1=81?= =?UTF-8?q?=D1=82=D0=B0=D0=BD=D0=BE=D0=B2=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F=20?= =?UTF-8?q?=D0=BE=D1=87=D0=B5=D1=80=D0=B5=D0=B4=D0=B8=20=D0=B8=20=D1=81?= =?UTF-8?q?=D0=B1=D0=BE=D0=B5=D0=B2=20worker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/attachments.py | 89 +++++++++++++++++++++++ tests/faults.py | 157 ++++++++++++++++++++++++++++++++++++++++ tests/queue_recovery.py | 68 +++++++++++++++++ tests/stack.py | 92 +++++++++++++++++++++++ tests/variants.py | 76 +++++++++++++++++++ tests/worker_load.py | 84 +++++++++++++++++++++ 6 files changed, 566 insertions(+) create mode 100644 tests/attachments.py create mode 100755 tests/faults.py create mode 100755 tests/queue_recovery.py create mode 100644 tests/stack.py create mode 100755 tests/variants.py create mode 100755 tests/worker_load.py diff --git a/tests/attachments.py b/tests/attachments.py new file mode 100644 index 00000000..871480c1 --- /dev/null +++ b/tests/attachments.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +import hashlib +import uuid +from pathlib import Path +from client import Client, arguments, check, run +from image_gen import png, png_header_only +from stack import wait_status + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() +MAX_BYTES = 10 * 1024 * 1024 + + +def upload(c, actor, chat, data, expected, content_type='image/png', filename='file.png'): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, expected, + actor=actor, raw=data, headers={'Content-Type': content_type}, + schema='Attachment' if expected == 202 else None) + + +def post_message(c, actor, chat, ids, expected, text=''): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', expected, actor=actor, + body={'text': text, 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message' if expected == 201 else None) + + +def main(): + args = arguments('Attachment validation and access checks') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'second', 'member_ids': [alice['id'], bob['id']]}, schema='Chat')['json'] + eve_chat = c.call('POST', '/api/v1/chats', 201, actor=eve, body={'title': 'eve', 'member_ids': [eve['id'], bob['id']]}, schema='Chat')['json'] + + upload(c, alice, chat, b'x' * (MAX_BYTES + 1), 413) + upload(c, alice, chat, SAMPLE + b'\0' * MAX_BYTES, 413) + upload(c, alice, chat, png_header_only(8000, 8000), 413) + upload(c, alice, chat, SAMPLE[:len(SAMPLE) // 2], 415) + upload(c, alice, chat, SAMPLE[:-1], 415) + upload(c, alice, chat, b'\x89PNG\r\n\x1a\n' + b'garbage' * 100, 415) + upload(c, alice, chat, b'', 415) + upload(c, alice, chat, b'GIF89a' + b'\0' * 64, 415) + upload(c, alice, chat, SAMPLE, 415, content_type='image/jpeg') + upload(c, alice, chat, SAMPLE, 415, content_type='text/plain') + upload(c, alice, chat, SAMPLE, 415, content_type='application/octet-stream') + + for name in ('..%2Fx.png', 'a%5Cb.png', 'a%00b.png', 'a%0Ab.png', 'a' * 256, ''): + upload(c, alice, chat, SAMPLE, 400, filename=name) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments', 400, actor=alice, raw=SAMPLE, headers={'Content-Type': 'image/png'}) + upload(c, eve, chat, SAMPLE, 404) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=a.png', 401, raw=SAMPLE, headers={'Content-Type': 'image/png'}) + + ok = upload(c, alice, chat, SAMPLE, 202)['json'] + check(ok['sha256'] == hashlib.sha256(SAMPLE).hexdigest(), 'checksum mismatch') + check(ok['filename'] == 'file.png' and ok['status'] == 'queued' and ok['variants'] == [], 'attachment fields') + unicode_name = upload(c, alice, chat, SAMPLE, 202, filename='%D1%84%D0%BE%D1%82%D0%BE.png')['json'] + check(unicode_name['filename'] == 'фото.png', 'unicode filename changed') + + post_message(c, alice, other, [ok['id']], 404) + post_message(c, alice, chat, [str(uuid.uuid4())], 404) + post_message(c, alice, chat, [ok['id'], ok['id']], 400) + post_message(c, alice, chat, ['not-a-uuid'], 400) + post_message(c, alice, chat, [str(uuid.uuid4()) for _ in range(11)], 400) + post_message(c, alice, chat, [], 400) + post_message(c, eve, chat, [ok['id']], 404) + post_message(c, eve, eve_chat, [ok['id']], 404) + + shared = post_message(c, bob, chat, [ok['id'], unicode_name['id']], 201, text='from bob')['json'] + check(shared['attachment_ids'] == [ok['id'], unicode_name['id']], 'attachment order changed') + history = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=alice, schema='MessagePage')['json']['items'] + check(any(m['id'] == shared['id'] and m['attachment_ids'] == shared['attachment_ids'] for m in history), 'history lost attachment_ids') + + for suffix in ('', '/content'): + c.call('GET', '/api/v1/attachments/' + ok['id'] + suffix, 404, actor=eve) + c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()) + suffix, 404, actor=alice) + c.call('GET', '/api/v1/attachments/bad' + suffix, 400, actor=alice) + c.call('GET', '/api/v1/attachments/' + ok['id'] + suffix, 401) + c.call('GET', '/api/v1/attachments/' + ok['id'] + '/content?variant=bogus', 400, actor=bob) + wait_status(c, bob, ok['id'], ('ready',), 60) + download = c.call('GET', '/api/v1/attachments/' + ok['id'] + '/content', actor=bob) + check(download['bytes'] == SAMPLE, 'download differs') + + big = png(1800, 1800, noise=True) + check(len(big) < MAX_BYTES, 'generated image exceeds limit') + large = upload(c, alice, chat, big, 202)['json'] + wait_status(c, bob, large['id'], ('ready',), 60) + check(c.call('GET', '/api/v1/attachments/' + large['id'] + '/content', actor=bob)['bytes'] == big, 'large download differs') + print('PASS: attachment validation, limits and access control') + + +if __name__ == '__main__': + run(main) diff --git a/tests/faults.py b/tests/faults.py new file mode 100755 index 00000000..d4edb607 --- /dev/null +++ b/tests/faults.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +import json +import re +import time +import uuid +from client import Client, Failure, arguments, check, run +from stack import (SAMPLE, broker, compose, job_row, logs, now_iso, psql, restore_api, restore_worker, upload, + variant_count, wait_api, wait_log, wait_status) + + +def job_id_of(att_id): + return psql("select id from jobs where attachment_id='%s'" % att_id) + + +def worker_with(fault, **extra): + compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env=dict({'WORKER_FAULT_POINT': fault}, **extra)) + + +def wait_s3(timeout=90): + deadline = time.monotonic() + timeout + while '(healthy)' not in compose('ps', 's3', '--format', '{{.Status}}').stdout: + check(time.monotonic() < deadline, 's3 did not recover') + time.sleep(1) + + +def ops(*args): + out = compose('run', '--rm', '--no-deps', '-T', 'worker', *args).stdout + return [json.loads(line) for line in out.splitlines() if line.startswith('{')] + + +def crash_after_commit(c, alice, bob, chat): + since = now_iso() + compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': 'after_commit'}) + compose('restart', 'lb') + wait_api(c) + name = 'crash-' + uuid.uuid4().hex[:12] + '.png' + answered = None + try: + answered = upload(c, alice, chat, filename=name) + except Failure: + pass + check(answered is None, 'api answered 202 despite injected crash') + wait_log('api', since, '"point":"after_commit"', 20) + att_id = psql("select id from attachments where filename='%s'" % name) + check(att_id, 'committed attachment not found') + restore_api(c) + ready = wait_status(c, bob, att_id, ('ready', 'failed'), 60) + check(ready['status'] == 'ready' and job_row(att_id)[:2] == ('ready', 1), 'outbox did not recover the job') + published = psql("select count(*) from outbox where job_id='%s' and published_at is not null" % job_id_of(att_id)) + print('1) api crashed after COMMIT before 202/publish: attachment=%s recovered by outbox relay (published rows=%s) -> ready attempt=1' + % (att_id, published)) + + +def crash_after_store(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + since = now_iso() + worker_with('after_store', JOB_TIMEOUT_SECONDS='10') + wait_log('worker', since, '"point":"after_store"', 60) + status, attempt, _ = job_row(att['id']) + check(status == 'processing' and attempt == 1, 'job must stay processing under expired-to-be lease') + since = now_iso() + worker_with('', JOB_TIMEOUT_SECONDS='10') + wait_log('worker', since, '"lease expired","job_id":"%s"' % job_id_of(att['id']), 90) + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + status, attempt, _ = job_row(att['id']) + check(ready['status'] == 'ready' and status == 'ready' and attempt >= 2, 'job did not finish after lease expiry') + check(variant_count(att['id']) == 3 and len(ready['variants']) == 3, 'retry duplicated variants') + thumb = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0] + body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)['bytes'] + check(len(body) == thumb['size_bytes'], 'thumbnail differs from metadata') + restore_worker() + print('2) worker crashed after S3 write before COMMIT: lease expired, reaper requeued, attempt=%d ready, variants=3' % attempt) + + +def crash_before_ack(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + job_id = job_id_of(att['id']) + since = now_iso() + worker_with('before_ack') + wait_log('worker', since, '"point":"before_ack"', 60) + wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 60) + status, attempt, _ = job_row(att['id']) + check(status == 'ready' and attempt == 1 and variant_count(att['id']) == 3, 'redelivery after commit changed result') + check(broker('XPENDING', 'attachments.jobs', 'workers').splitlines()[0] == '0', 'redelivered message not acked') + check(wait_status(c, bob, att['id'], ('ready',), 10)['status'] == 'ready', 'attachment not ready') + restore_worker() + print('3) worker crashed after COMMIT before ack: message redelivered after restart, skipped as no-op, attempt=1, variants=3') + + +def retries_and_replay(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + job_id = job_id_of(att['id']) + compose('stop', 's3') + since = now_iso() + try: + compose('up', '-d', '--no-deps', 'worker') + check('(healthy)' not in compose('ps', '-a', 's3', '--format', '{{.Status}}').stdout, 's3 must stay stopped') + failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 120) + check(failed['status'] == 'failed' and failed['error_code'] == 'retry_exhausted', 'expected retry_exhausted') + text = logs('worker', since) + delays = [int(d) for d in re.findall(r'"job retry scheduled","job_id":"%s".*?"delay_ms":(\d+)' % job_id, text)] + check(len(delays) == 4, 'expected 4 scheduled retries, got %d' % len(delays)) + for n, d in enumerate(delays): + base = 1000 * 2 ** n + check(base // 2 <= d <= base, 'delay %d outside jittered window for retry %d' % (d, n + 1)) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, body={'text': '', 'attachment_ids': [att['id']]}, + headers={'Idempotency-Key': uuid.uuid4().hex}) + check(any(x['attachment_id'] == att['id'] for x in ops('failed')), 'failed registry misses job') + print('4) s3 down: transient errors retried with backoff+jitter delays_ms=%s, then failed retry_exhausted, listed in registry' % delays) + finally: + compose('start', 's3') + wait_s3() + check(ops('replay', att['id'])[0]['result'] == 'queued', 'replay did not queue job') + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 120) + check(ready['status'] == 'ready' and variant_count(att['id']) == 3, 'replayed job did not finish') + check(ops('replay', att['id'])[0]['result'] == 'skipped_ready', 'replay of ready job must be a no-op') + check(psql("select count(*) from job_replays where job_id='%s'" % job_id) == '1', 'replay audit missing') + check(variant_count(att['id']) == 3, 'second replay created new effects') + print(' replay: failed -> queued -> ready (audit rows=1); replay of ready job skipped, variants=3') + + +def permanent_error(c, alice, bob, chat): + data = bytearray(SAMPLE) + mid = len(data) // 2 + data[mid:mid + 64] = b'\x00' * 64 + att = upload(c, alice, chat, bytes(data))['json'] + failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 60) + status, attempt, _ = job_row(att['id']) + check(failed['status'] == 'failed' and failed['error_code'] == 'invalid_image' and attempt == 1, 'corrupt image must fail without retries') + check(failed['variants'] == [], 'failed attachment has variants') + print('5) corrupted IDAT (passes header/tail check in API): failed invalid_image on attempt 1, no retries') + + +def main(): + args = arguments('Fault injection: COMMIT/publish gap, crash before COMMIT and before ack, retries, DLQ replay') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + try: + crash_after_commit(c, alice, bob, chat) + crash_after_store(c, alice, bob, chat) + crash_before_ack(c, alice, bob, chat) + retries_and_replay(c, alice, bob, chat) + permanent_error(c, alice, bob, chat) + finally: + compose('up', '-d', '--no-deps', 's3') + restore_worker() + if 'after_commit' in compose('exec', '-T', 'api', 'printenv', 'FAULT_POINT', check_rc=False).stdout: + restore_api(c) + print('PASS: outbox recovery, lease recovery, redelivery after commit, backoff with jitter, retry_exhausted, replay, permanent error') + + +if __name__ == '__main__': + run(main) diff --git a/tests/queue_recovery.py b/tests/queue_recovery.py new file mode 100755 index 00000000..0c420d8a --- /dev/null +++ b/tests/queue_recovery.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +import json +import time +from client import Client, arguments, check, run +from image_probe import jpeg_size +from stack import broker, compose, job_row, logs, now_iso, psql, upload, variant_count, wait_api, wait_log, wait_status + + +def stream_has(job_id): + return job_id in broker('XRANGE', 'attachments.jobs', '-', '+') + + +def main(): + args = arguments('Worker stop, API/broker restart and redelivery scenario') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + + compose('stop', 'worker') + try: + att = upload(c, alice, chat)['json'] + check(att['status'] == 'queued' and att['variants'] == [], 'upload must return queued snapshot') + msg = c.message(alice, chat, '', attachments=[att['id']])['json'] + check(msg['attachment_ids'] == [att['id']], 'message lost queued attachment') + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 409, actor=bob) + c.message(bob, chat, 'text works while worker is down') + job_id = psql("select id from jobs where attachment_id='%s'" % att['id']) + deadline = time.monotonic() + 10 + while not stream_has(job_id): + check(time.monotonic() < deadline, 'job was not published to broker') + time.sleep(0.5) + check(psql("select count(*) from outbox where job_id='%s' and published_at is null" % job_id) == '0', 'outbox not marked') + print('accepted: attachment=%s job=%s status=queued, job in stream, worker stopped' % (att['id'], job_id)) + + compose('restart', 'broker', 'api') + compose('up', '-d', '--wait', '--no-deps', 'broker', 'api') + compose('restart', 'lb') + wait_api(c) + check(stream_has(job_id), 'job lost after broker restart') + check(c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status'] == 'queued', 'status changed without worker') + print('restarted broker and api: job still in stream, attachment still queued') + finally: + compose('start', 'worker') + + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + check(ready['status'] == 'ready', 'job did not finish after worker start') + thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob) + check(jpeg_size(thumb['bytes']) == (256, 160), 'thumbnail size') + check(job_row(att['id'])[:2] == ('ready', 1) and variant_count(att['id']) == 3, 'unexpected job state or variants') + print('worker started: attachment ready, attempt=1, variants=3') + + since = now_iso() + before = psql("select updated_at from attachments where id='%s'" % att['id']) + payload = json.dumps({'schema_version': 1, 'job_id': job_id, 'type': 'attachment.process.v1', 'attachment_id': att['id'], + 'pipeline_version': 1, 'attempt': 1, 'created_at': ready['created_at']}) + broker('XADD', 'attachments.jobs', '*', 'job', payload) + wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 30) + time.sleep(1) + check(psql("select updated_at from attachments where id='%s'" % att['id']) == before, 'duplicate delivery changed attachment') + check(variant_count(att['id']) == 3 and job_row(att['id'])[:2] == ('ready', 1), 'duplicate delivery created new effects') + pending = broker('XPENDING', 'attachments.jobs', 'workers') + check(pending.splitlines()[0] == '0', 'duplicate message was not acked') + print('duplicate delivery: skipped as no-op, acked, variants=3, updated_at unchanged') + print('PASS: worker stop, broker/api restart, redelivery idempotency') + + +if __name__ == '__main__': + run(main) diff --git a/tests/stack.py b/tests/stack.py new file mode 100644 index 00000000..f0b645f6 --- /dev/null +++ b/tests/stack.py @@ -0,0 +1,92 @@ +import os +import subprocess +import time +import uuid +from pathlib import Path +from client import Failure, check + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() + + +def compose(*args, env=None, check_rc=True): + full = dict(os.environ, **(env or {})) + return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=check_rc, + capture_output=True, text=True, env=full) + + +def psql(sql): + return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', sql).stdout.strip() + + +def broker(*args): + cmd = 'valkey-cli --no-auth-warning -a "$(sed -n "s/^requirepass //p" /run/secrets/broker_conf)" ' + ' '.join("'%s'" % a for a in args) + return compose('exec', '-T', 'broker', 'sh', '-c', cmd).stdout.strip() + + +def logs(service, since): + return compose('logs', '--no-color', '--since', since, service).stdout + + +def now_iso(): + return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) + + +def upload(c, actor, chat, data=SAMPLE, content_type='image/png', filename=None, expected=202): + filename = filename or 'f-' + uuid.uuid4().hex[:12] + '.png' + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, expected, actor=actor, + raw=data, headers={'Content-Type': content_type}, schema='Attachment' if expected == 202 else None) + + +def wait_status(c, actor, att_id, statuses, timeout=120): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + att = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json'] + if att['status'] in statuses: + return att + except Failure: + pass + time.sleep(0.5) + raise Failure('attachment %s did not reach %s' % (att_id, statuses)) + + +def wait_api(c, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + c.call('GET', '/health/ready') + return + except Failure: + time.sleep(1) + raise Failure('api did not become ready') + + +def wait_log(service, since, needle, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if needle in logs(service, since): + return + time.sleep(1) + raise Failure('%s did not log %r' % (service, needle)) + + +def job_row(att_id): + out = psql("select status, attempt, coalesce(last_error, '') from jobs where attachment_id='%s'" % att_id) + check(out, 'job not found') + status, attempt, error = out.split('|', 2) + return status, int(attempt), error + + +def variant_count(att_id): + return int(psql("select count(*) from attachment_variants where attachment_id='%s'" % att_id)) + + +def restore_worker(): + compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'WORKER_FAULT_POINT': ''}) + + +def restore_api(c): + compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': ''}) + compose('restart', 'lb') + wait_api(c) diff --git a/tests/variants.py b/tests/variants.py new file mode 100755 index 00000000..4c009050 --- /dev/null +++ b/tests/variants.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +import hashlib +import io +from PIL import Image +from client import Client, arguments, check, run +from stack import SAMPLE, upload, wait_status + +EXPECTED = {'thumbnail': ('image/jpeg', 'JPEG'), 'crop': ('image/jpeg', 'JPEG'), 'optimized': ('image/webp', 'WEBP')} + + +def encode(img, fmt, **kw): + buf = io.BytesIO() + img.save(buf, fmt, **kw) + return buf.getvalue() + + +def halves(w, h): + img = Image.new('RGB', (w, h), (0, 0, 255)) + img.paste((255, 0, 0), (0, 0, w // 2, h)) + return img + + +def process(c, alice, bob, chat, data, content_type='image/png'): + att = upload(c, alice, chat, data, content_type)['json'] + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + check(ready['status'] == 'ready' and ready['error_code'] is None, 'attachment failed: %s' % ready['error_code']) + check(sorted(v['name'] for v in ready['variants']) == sorted(EXPECTED), 'unexpected variant set') + original = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] + check(original == data and ready['sha256'] == hashlib.sha256(data).hexdigest(), 'original changed') + out = {} + for v in ready['variants']: + r = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + v['name'], actor=bob) + body = r['bytes'] + check(r['headers'].get_content_type() == v['content_type'] == EXPECTED[v['name']][0], v['name'] + ' content type') + check(len(body) == v['size_bytes'] and hashlib.sha256(body).hexdigest() == v['sha256'], v['name'] + ' checksum') + img = Image.open(io.BytesIO(body)) + check(img.format == EXPECTED[v['name']][1] and img.size == (v['width'], v['height']), v['name'] + ' real size/format') + check('exif' not in img.info, v['name'] + ' keeps EXIF') + img.load() + out[v['name']] = img.convert('RGB') + return att, out + + +def main(): + args = arguments('Thumbnail, crop and optimized variant rules') + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + + att, v = process(c, alice, bob, chat, SAMPLE) + check(v['thumbnail'].size == (256, 160) and v['crop'].size == (128, 128) and v['optimized'].size == (320, 200), 'fixture sizes') + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=bogus', 400, actor=bob) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 404, actor=eve) + + _, v = process(c, alice, bob, chat, encode(halves(2000, 1000), 'PNG')) + check(v['thumbnail'].size == (256, 128) and v['optimized'].size == (1280, 640) and v['crop'].size == (128, 128), 'large image sizes') + + _, v = process(c, alice, bob, chat, encode(halves(100, 50), 'PNG')) + check(v['thumbnail'].size == (100, 50) and v['optimized'].size == (100, 50), 'small image must not be upscaled') + check(v['crop'].size == (128, 128), 'crop must be 128x128') + + _, v = process(c, alice, bob, chat, encode(Image.new('RGBA', (300, 300), (0, 0, 0, 0)), 'PNG')) + for name, img in v.items(): + check(min(img.getpixel((img.width // 2, img.height // 2))) >= 250, name + ': alpha not composited on white') + + exif = Image.Exif() + exif[0x0112] = 6 + _, v = process(c, alice, bob, chat, encode(halves(300, 200), 'JPEG', quality=95, exif=exif.tobytes()), 'image/jpeg') + t = v['thumbnail'] + check(t.size == (170, 256) and v['optimized'].size == (200, 300), 'EXIF orientation not applied to dimensions') + top, bottom = t.getpixel((85, 20)), t.getpixel((85, 236)) + check(top[0] > 200 and top[2] < 60 and bottom[2] > 200 and bottom[0] < 60, 'EXIF rotation direction wrong') + print('PASS: thumbnail/crop/optimized sizes, formats, checksums, no upscale, alpha on white, EXIF orientation, no EXIF in variants') + + +if __name__ == '__main__': + run(main) diff --git a/tests/worker_load.py b/tests/worker_load.py new file mode 100755 index 00000000..97bdfd7e --- /dev/null +++ b/tests/worker_load.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +import json +import subprocess +import threading +import time +from pathlib import Path +from client import Client, arguments, check, run +from image_gen import png +from stack import SAMPLE, compose, job_row, psql, restore_worker, upload, wait_status + +LIMIT_MIB = 768 + + +def sample(stop, state): + while not stop.is_set(): + out = compose('exec', '-T', 'worker', 'grep', 'VmRSS', '/proc/1/status', check_rc=False).stdout.split() + if len(out) > 1: + state['rss'] = max(state['rss'], int(out[1]) / 1024) + n = psql("select count(*) from jobs where status = 'processing'") + state['processing'] = max(state['processing'], int(n or 0)) + time.sleep(0.2) + + +def main(): + def configure(p): + p.add_argument('--huge', type=int, default=6) + p.add_argument('--small', type=int, default=20) + p.add_argument('--output') + args = arguments('Worker concurrency, memory and time limits', configure) + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + concurrency = int(compose('exec', '-T', 'worker', 'printenv', 'WORKER_CONCURRENCY').stdout.strip()) + huge = png(6320, 6320) + + state, stop = {'rss': 0, 'processing': 0}, threading.Event() + t = threading.Thread(target=sample, args=(stop, state)) + t.start() + started = time.monotonic() + ids = [upload(c, alice, chat, huge)['json']['id'] for _ in range(args.huge)] + ids += [upload(c, alice, chat, SAMPLE)['json']['id'] for _ in range(args.small)] + accepted = time.monotonic() - started + results = [wait_status(c, bob, i, ('ready', 'failed'), 300)['status'] for i in ids] + elapsed = time.monotonic() - started + stop.set() + t.join() + container = compose('ps', '-q', 'worker').stdout.strip() + oom = subprocess.run(['docker', 'inspect', '-f', '{{.RestartCount}} {{.State.OOMKilled}}', container], + capture_output=True, text=True).stdout.split() + + compose('stop', 'worker') + slow = upload(c, alice, chat, huge)['json'] + compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'JOB_TIMEOUT_SECONDS': '1'}) + timed = wait_status(c, bob, slow['id'], ('ready', 'failed'), 120) + _, attempts, last_error = job_row(slow['id']) + restore_worker() + + report = { + 'huge_images': {'count': args.huge, 'pixels': 6320 * 6320, 'bytes': len(huge)}, + 'small_images': args.small, + 'worker_concurrency': concurrency, + 'max_processing_jobs_observed': state['processing'], + 'all_ready': results.count('ready') == len(results), + 'accept_seconds': round(accepted, 2), + 'all_ready_seconds': round(elapsed, 1), + 'worker_peak_rss_mib': round(state['rss'], 1), + 'worker_memory_limit_mib': LIMIT_MIB, + 'worker_restarts_oom': oom, + 'timeout_case': {'job_timeout_seconds': 1, 'status': timed['status'], 'error_code': timed['error_code'], + 'attempts': attempts, 'last_error': last_error}, + } + text = json.dumps(report, indent=2) + if args.output: + Path(args.output).write_text(text + '\n') + print(text) + check(report['all_ready'], 'some jobs failed under load') + check(state['processing'] <= concurrency, 'more jobs processing than WORKER_CONCURRENCY') + check(state['rss'] < LIMIT_MIB and oom == ['0', 'false'], 'worker reached memory limit') + check(timed['status'] == 'failed' and timed['error_code'] == 'retry_exhausted' and 'deadline' in last_error, + 'timeout must be a bounded transient failure') + print('PASS: bounded worker concurrency, memory below limit, job timeout enforced') + + +if __name__ == '__main__': + run(main) -- 2.54.0 From 27fafc6725e9010b36101b422e3f62533452e22a Mon Sep 17 00:00:00 2001 From: gera Date: Fri, 6 Nov 2026 01:59:07 +0300 Subject: [PATCH 4/5] =?UTF-8?q?=D0=9F=D1=80=D0=B8=D0=BB=D0=BE=D0=B6=D0=B8?= =?UTF-8?q?=D0=BB=20=D1=80=D0=B5=D0=B7=D1=83=D0=BB=D1=8C=D1=82=D0=B0=D1=82?= =?UTF-8?q?=D1=8B=20=D0=BF=D1=80=D0=BE=D0=B2=D0=B5=D1=80=D0=BE=D0=BA=20?= =?UTF-8?q?=D0=B8=20=D0=BD=D0=B0=D0=BF=D0=B8=D1=81=D0=B0=D0=BB=20=D0=BE?= =?UTF-8?q?=D1=82=D1=87=D1=91=D1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 168 ++++++++++++++++++++++++++++++++---- evidence/attachments.txt | 3 + evidence/failover-1.txt | 8 ++ evidence/failover-2.txt | 7 ++ evidence/failover-each.json | 71 +++++++++++++++ evidence/faults.txt | 9 ++ evidence/idempotency.txt | 3 + evidence/persistence.txt | 25 ++++++ evidence/queue-recovery.txt | 7 ++ evidence/rolling.json | 24 ++++++ evidence/s3-checks.txt | 3 + evidence/smoke.txt | 6 ++ evidence/upload-memory.json | 18 ++++ evidence/variants.txt | 3 + evidence/worker-load.json | 26 ++++++ 15 files changed, 366 insertions(+), 15 deletions(-) create mode 100644 evidence/attachments.txt create mode 100644 evidence/failover-1.txt create mode 100644 evidence/failover-2.txt create mode 100644 evidence/failover-each.json create mode 100644 evidence/faults.txt create mode 100644 evidence/idempotency.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/queue-recovery.txt create mode 100644 evidence/rolling.json create mode 100644 evidence/s3-checks.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/upload-memory.json create mode 100644 evidence/variants.txt create mode 100644 evidence/worker-load.json diff --git a/REPORT.md b/REPORT.md index e98f75e9..e8ef995e 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,175 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Березин Герман / БВТ2304 +- Целевая оценка: 4 +- Commit/PR: после публикации +- Использованные источники, библиотеки и помощь ИИ: Go 1.26, pgx, go-redis (клиент Valkey Streams), minio-go v7, golang.org/x/image/draw (ресемплинг), github.com/gen2brain/webp (libwebp, транспилированный в Go), github.com/rwcarlsen/goexif (EXIF orientation); PostgreSQL 17, Valkey 8.1, SeaweedFS, HAProxy 3.0; документация RabbitMQ reliability и Redis Streams из README; код lab4 перенесён за основу; ИИ-ассистент использован при разработке ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker 29 с Compose v2 и Python 3.9+. Для `tests/variants.py` нужен Pillow (проверяет реальные пиксели JPEG/WebP). + +```sh +cp .env.example .env +sh scripts/mksecrets.sh +docker compose --env-file .env up -d --build --wait +make check && make test +``` + +Порядок старта: postgres, valkey, broker, s3 → `migrate` → `s3-init` → `api` × 2, `worker`, `janitor` → `lb`. Публичный адрес: `http://localhost:8080`. После первого старта S3 запись становится доступна через 10–30 s. + +Остановка без потери данных: `docker compose --env-file .env stop`. Намеренное удаление учебных данных: `docker compose --env-file .env down -v`. + +Операторские команды: + +```sh +docker compose --env-file .env stop worker +docker compose --env-file .env up -d --no-deps worker +docker compose --env-file .env run --rm --no-deps worker failed +docker compose --env-file .env run --rm --no-deps worker replay +``` + +Проверки (стенд запущен): + +```sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/variants.py +python3 tests/queue_recovery.py +python3 tests/faults.py +python3 tests/worker_load.py --output evidence/worker-load.json +python3 tests/attachments.py +python3 tests/s3_checks.py +python3 tests/upload_memory.py --output evidence/upload-memory.json +python3 tests/idempotency.py +python3 tests/scenario.py --mode failover-each --output evidence/failover-each.json +python3 tests/scenario.py --mode rolling --output evidence/rolling.json +``` + +После `make test-failover` реплику возвращают командой `docker compose --env-file .env up -d --wait api`. `faults.py`, `queue_recovery.py`, `worker_load.py` сами останавливают и пересоздают worker/api/broker/s3 и возвращают их в исходное состояние. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +клиент → HAProxy → api ×2 ──(1) S3 staging/ + └──(2) PostgreSQL: attachments(queued) + jobs + outbox — одна транзакция → 202 +api: outbox relay ──XADD──→ broker (Valkey Stream attachments.jobs, group workers) +worker ──XREADGROUP──→ claim (lease) → S3 attachments//... → COMMIT ready → XACK +``` + +| Роль | Сервис | Образ | Готовность | Volume | Сеть | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| api ×2 | `api` | собирается из исходников (`mtusi-messenger:lab5`) | `/health/ready` (PostgreSQL) | нет | backend, storage | secrets: database_url, cache_url, broker_url, S3-ключи API | +| worker | `worker` | тот же образ, команда `worker` | нет HTTP, healthcheck отключён | нет | backend, storage | database_url, broker_url, S3-ключи API | +| postgres | `postgres` | postgres:17-alpine@sha256:b0f9560a… | `pg_isready` | pg_data | backend | postgres_password | +| broker | `broker` | valkey/valkey:8.1-alpine@sha256:081c2f5c… | `valkey-cli ping` с паролем | broker_data | backend | broker.conf (`requirepass`) | +| s3 | `s3` | chrislusf/seaweedfs@sha256:c2658820… | `/dir/assign` выдаёт fid | s3_data | storage | s3_config.json | +| load_balancer | `lb` | haproxy:3.0-alpine@sha256:cf3af0fa… | проверки `/health/ready` бэкендов | нет | backend, 127.0.0.1:8080 | нет | +| кэш сессий | `valkey` | valkey/valkey:8.1-alpine | `valkey-cli ping` | valkey_data | backend | нет | + +Bootstrap: `migrate` применяет `migrations/*.sql` под advisory lock, повторный запуск пропускает применённые. `s3-init` создаёт bucket, если его нет. Consumer group `workers` создают API (перед первой публикацией) и worker при старте (`XGROUP CREATE … 0 MKSTREAM`, ошибка BUSYGROUP игнорируется). + +- **Брокер.** Valkey Streams в отдельном сервисе `broker` с `appendonly yes`, `appendfsync always`, `noeviction`. Доставка at least once: сообщение остаётся в pending (PEL) до `XACK`. Publisher confirm: ответ на `XADD` приходит после записи в AOF с fsync. Consumer ack: `XACK` после COMMIT результата в PostgreSQL. В сообщении только поля `jobs.schema.json` (поле `job` — JSON с `schema_version`, `job_id`, `type`, `attachment_id`, `pipeline_version`, `attempt`, `created_at`), без bytes, URL и credentials. +- **Что сохранено к 202.** Полный проверенный оригинал лежит в приватном S3-prefix `staging/` (тот же bucket, volume s3_data). Одной транзакцией сохранены строки `attachments` (status `queued`, staging_key, sha256, размер), `jobs` (attempt 1) и `outbox`. Проверки на приёме те же, что в lab4: 413/415/400/404, хвост и заголовок изображения, не более 40 млн пикселей. +- **Transactional outbox.** В каждой API-реплике работает relay: `select … from outbox where published_at is null and available_at <= now() for update skip locked`, затем `XADD` и `published_at = now()` в той же транзакции. Падение между COMMIT и publish оставляет неопубликованную строку, её публикует любая живая реплика. Падение после `XADD` до COMMIT пометки даёт повторную публикацию: это дубль, worker обработает его как no-op. +- **Worker.** Отдельный процесс (`messenger worker`), `WORKER_CONCURRENCY=2` потребителя, у каждого `COUNT 1`: prefetch 1, не больше двух задач одновременно. При старте потребитель дочитывает свои pending-сообщения (`XREADGROUP … 0`), каждые 15 s забирает зависшие чужие (`XAUTOCLAIM` с min-idle = lease). +- **Ключ идемпотентности.** Логическая работа — `jobs.id` (job_id), он один на вложение и не меняется при redelivery. Ключ эффекта — `(attachment_id, pipeline_version, name)`: первичный ключ `attachment_variants`, вставка через `on conflict do update`. S3-ключи детерминированы: `attachments//original`, `attachments//v1/{thumbnail.jpg,crop.jpg,optimized.webp}`. Повтор перезаписывает те же объекты и не создаёт второго набора. +- **Захват и lease.** `update jobs set status='processing', lease_owner, lease_until = now() + lease where status='queued' and next_attempt_at <= now()`. Сообщение для задачи в любом другом состоянии подтверждается без работы (`job delivery skipped`), поэтому повторная доставка ready-задачи — no-op. Lease = `JOB_TIMEOUT_SECONDS` + 15 s. Фиксация результата и перевод в retry/failed выполняются только при совпадении `lease_owner` и `attempt`. Worker, потерявший lease, не перезапишет состояние (`lease lost, result discarded`). +- **Зависший или долгий.** Задача ограничена контекстом `JOB_TIMEOUT_SECONDS` (60 s), поэтому живой worker не может законно работать дольше lease. Каждые 5 s reaper в worker переводит задачи с истёкшим lease в `queued` со следующей попыткой или в `failed`. Новый захват получает новый `attempt`. Старый процесс не сможет зафиксировать результат из-за fencing. Запись тех же детерминированных объектов безвредна. +- **Retry.** Временные ошибки (S3, PostgreSQL, timeout, истёкший lease) дают `queued` с `attempt+1`, новой строкой outbox с `available_at = now() + delay` и записью причины в `jobs.last_error`. Задержка `d = min(JOB_BACKOFF_BASE_MS·2^(n−1), JOB_BACKOFF_MAX_MS)` выбирается случайно в `[d/2, d]` (equal jitter). Всего `JOB_MAX_ATTEMPTS = 5` попыток, затем `failed` с `retry_exhausted`. Постоянные ошибки сразу дают `failed` без повторов: `invalid_image` (файл не декодируется, превышен лимит пикселей), `processing_failed` (staging отсутствует или не совпадает sha256/размер). +- **Реестр failed и replay.** Реестр — `jobs` со статусом `failed` (`worker failed` выводит attachment_id, attempt, error_code, last_error). `worker replay ` переводит в `queued` только задачу в `failed`: даёт новый бюджет из `JOB_MAX_ATTEMPTS` попыток, пишет аудит в `job_replays` и строку outbox. Для `ready`/`queued`/`processing` результат `skipped_`, дубля нет. +- **Варианты.** Все варианты строятся из пикселей и кодируются заново, поэтому EXIF и прочие метаданные не копируются. EXIF orientation (JPEG) применяется, alpha компонуется на белый, ресемплинг Catmull-Rom. Изображения больше 2560 px сначала уменьшаются вдвое блоком 2×2, чтобы ограничить промежуточный буфер. Итоговые размеры считаются от исходных. + - `thumbnail`: JPEG q=85, вписан в 256×256, без увеличения, размеры округляются вниз, минимум 1 px; fixture 320×200 → 256×160. + - `crop`: JPEG q=85, центральный квадрат, 128×128, маленькие исходники увеличиваются. + - `optimized`: WebP lossy, quality 80, method 4, вписан в 1280×1280 без увеличения. Уменьшение каждого маленького файла не обещается. +- **Состояния API.** POST upload → 202 `queued`. GET метаданных возвращает `queued` / `processing` / `ready` / `failed`. Для `ready` в `variants` три записи с размерами и sha256 реального файла. `/content` для не-ready → 409. `variant=` с неизвестным именем → 400. Отсутствующий вариант ready-объекта → 404. Прикрепить `failed`-вложение к сообщению → 409, `queued`/`processing` можно. Ready ставится одной транзакцией после записи оригинала и всех трёх вариантов. +- **Staging.** Удаляется worker'ом после COMMIT ready. Если удаление не прошло, `janitor` удаляет staging ready-вложений и staging без строки (старше `CLEANUP_GRACE_SECONDS`). Staging `failed` хранится `STAGING_RETENTION_SECONDS` (24 ч) ради replay. Staging `queued`/`processing` не удаляется. +- **Критичные зависимости.** `/health/ready` проверяет только PostgreSQL. Брокер не нужен синхронному пути: при его недоступности загрузки принимаются, задания копятся в outbox. Upload и download требуют S3 (иначе 503). +- **Внедрение отказов.** `FAULT_POINT` по умолчанию пуст. В тестах через `API_FAULT_POINT=after_commit` (выход процесса после COMMIT, до ответа и publish) и `WORKER_FAULT_POINT=after_store|before_ack` (выход после записи в S3 до COMMIT / после COMMIT до ack). +- Перенесено из lab4 без изменения смысла: сессии, идемпотентность сообщений, HAProxy, приватный bucket и отдельные ключи, janitor (дополнен очисткой staging), лимиты загрузки. Изменено: upload 201 → 202, `attachments.object_key` заполняется worker'ом, добавлена миграция `0004_jobs.sql`. + +### Окна отказа + +| Окно | Что происходит | Восстановление | +| --- | --- | --- | +| после staging до записи задания | объект в `staging/` без строки, клиент получает 503 или обрыв | janitor удаляет его после grace period | +| после COMMIT до publish | строка outbox не опубликована | relay любой API-реплики публикует; проверено (`faults.py`, 1) | +| после publish до confirm/пометки | сообщение в stream, строка outbox не помечена | повторная публикация → дубль → no-op при захвате | +| после записи S3 до COMMIT результата | задача `processing`, объекты записаны | lease истекает → reaper → попытка 2 перезаписывает те же ключи; проверено (`faults.py`, 2) | +| после COMMIT результата до ack | ready, сообщение в PEL | после рестарта worker дочитывает pending → no-op → ack; проверено (`faults.py`, 3) | + +### Соответствие .env.example + +| Переменная | Где используется | +| --- | --- | +| `API_REPLICAS` | `deploy.replicas` сервиса api | +| `S3_REGION`, `S3_BUCKET` | api, worker, janitor, s3-init, генерация прав в `mksecrets.sh` | +| `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` | api (приём) и worker (повторная проверка) | +| `WORKER_CONCURRENCY` | число потребителей worker | +| `JOB_MAX_ATTEMPTS` | api записывает в `jobs.max_attempts`, replay выдаёт новый бюджет | +| `JOB_TIMEOUT_SECONDS` | таймаут задачи, lease = значение + 15 s | +| `JOB_BACKOFF_BASE_MS`, `JOB_BACKOFF_MAX_MS` | backoff повторов | +| `STAGING_RETENTION_SECONDS` | хранение staging failed-вложений (janitor) | +| `CLEANUP_GRACE_SECONDS`, `CLEANUP_INTERVAL_SECONDS` | janitor | +| `BROKER_URL_FILE`, `DATABASE_URL_FILE`, `S3_*_FILE` | задаются в compose как пути к secrets | + +### Лимиты worker + +| Ограничение | Значение | +| --- | --- | +| Bytes | размер staging должен совпасть с метаданными; чтение не более `MAX_UPLOAD_BYTES`, sha256 сверяется | +| Пиксели | `image.DecodeConfig` до декодирования, не более `MAX_IMAGE_PIXELS` (40 млн), иначе `invalid_image` | +| Время | `JOB_TIMEOUT_SECONDS` на задачу (S3, БД, проверки между этапами) | +| Память | 2 задачи одновременно, `GOMEMLIMIT=600MiB`, лимит контейнера 768 MiB, CPU 1.0 | +| Конкуренция | `WORKER_CONCURRENCY=2`, prefetch (`COUNT`) 1 на потребителя | ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke (202 → ready, оригинал, настоящий JPEG 256×160) | `make check && make test` | PASS | evidence/smoke.txt | +| Минимум lab4 сохранён (валидация, доступ, приватный bucket, отказ S3, идемпотентность) | `tests/attachments.py`, `tests/s3_checks.py`, `tests/idempotency.py` | PASS: 413/415/400/404, анонимный доступ 403, 503 при остановленном S3 | evidence/attachments.txt, evidence/s3-checks.txt, evidence/idempotency.txt | +| Restart всех stores с volumes | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; перезапущены postgres, valkey, broker, s3, api, lb, janitor, worker (состояния до/после в файле) | evidence/persistence.txt | +| Failover каждой API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS для api-1 и api-2, 0–1 неуспешная попытка, восстановление 3.5 s | evidence/failover-1.txt, evidence/failover-2.txt | +| Worker остановлен → upload 202 → сообщение → content 409 → рестарт broker и API → worker запущен → ready | `tests/queue_recovery.py` | задача есть в stream до и после рестарта broker, статус `queued` без worker, после старта ready, attempt 1, 3 варианта | evidence/queue-recovery.txt | +| Повторная доставка не размножает варианты | `tests/queue_recovery.py` (ручной `XADD` дубля) | `job delivery skipped`, ack, `updated_at` не изменился, 3 варианта | evidence/queue-recovery.txt | +| Thumbnail, crop 128×128, optimized WebP, качество/метаданные | `tests/variants.py` | размеры и форматы по реальным байтам, sha256 совпадает, без увеличения, alpha → белый, EXIF orientation 6 применена, EXIF в вариантах нет | evidence/variants.txt | +| Crash между COMMIT и publish | `tests/faults.py`, сценарий 1 | API упал после COMMIT (202 не получен), outbox опубликован другой репликой, ready attempt 1 | evidence/faults.txt | +| Остановка worker после записи объекта до COMMIT, timeout/lease | `tests/faults.py`, сценарий 2 | lease истёк, reaper вернул в очередь, ready attempt 2, вариантов 3 | evidence/faults.txt | +| Остановка worker после COMMIT до ack | `tests/faults.py`, сценарий 3 | после рестарта pending доставлен повторно, no-op, attempt 1, PEL пуст | evidence/faults.txt | +| Backoff с jitter, ограниченный retry, failed, replay | `tests/faults.py`, сценарий 4 | S3 остановлен: 4 повтора с задержками 931/1284/2760/7484 мс (окна [d/2, d]), `retry_exhausted`, задача в реестре, content и прикрепление → 409; replay → ready, повторный replay → `skipped_ready`, аудит 1 строка | evidence/faults.txt | +| Постоянная ошибка без повторов | `tests/faults.py`, сценарий 5 | PNG с повреждённым IDAT (проходит проверку API) → `invalid_image` на попытке 1 | evidence/faults.txt | +| Ограничение конкуренции, памяти и времени worker | `tests/worker_load.py` | 6 изображений по 39.9 млн пикселей + 20 малых: одновременно не более 2 processing, пик RSS 617 MiB при лимите 768 MiB, без OOM; при `JOB_TIMEOUT_SECONDS=1` — `context deadline exceeded`, 5 попыток, `retry_exhausted` | evidence/worker-load.json | +| Очистка staging после безопасной фиксации | `tests/s3_checks.py` | staging ready-вложения удалён, staging queued-задачи сохранён janitor'ом, staging без строки удалён | evidence/s3-checks.txt | +| Память API при загрузках | `tests/upload_memory.py` | 12 потоков × 3 файла по 9.3 MiB, 36/36 → 202, пик api 119 / 100 MiB при лимите 192 | evidence/upload-memory.json | +| Отказ реплики и rolling update под нагрузкой | `tests/scenario.py` | см. эксперименты | evidence/failover-each.json, evidence/rolling.json | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Стенд: один хост, Docker 29.9.2, Compose 5.6.1, лимиты контейнеров как выше. Нагрузка сообщений — 8 потоков, closed-loop, повтор с тем же ключом. Нагрузка worker — 6 PNG 6320×6320 (116 545 байт, однотонные: проверка лимита пикселей, а не сети) и 20 копий fixture. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Worker остановлен, upload | 202, работа ждёт | `queued`, content 409, текст работает | после старта worker ready, потерь нет | +| Restart broker и API с volumes при остановленном worker | задача не теряется | job_id в stream до и после | потерь нет | +| Падение API после COMMIT | задача не теряется без ручного вмешательства | outbox опубликован живой репликой | ready в пределах секунд, потерь нет | +| Падение worker после записи S3 | повтор того же результата | ready с attempt 2 после истечения lease 25 s (тест с `JOB_TIMEOUT_SECONDS=10`) | ≈ lease + 5 s, дублей нет | +| Падение worker после COMMIT | повторная доставка = no-op | skipped, ack | сразу после рестарта | +| S3 недоступен для worker | ограниченные повторы, failed | 5 попыток за ≈ 13 s, `retry_exhausted`, replay → ready | после replay, потерь нет | +| 6×40 Мп + 20 малых | не более 2 задач одновременно, память в лимите | все ready за 13.1 s, пик RSS 617 MiB / 768 | без OOM и рестартов | +| Остановка каждой API-реплики под нагрузкой | запись продолжается | 2 и 0 неуспешных попыток, RPS 439 / 447 против 524 до отказа | возврат реплики около 6.5 s | +| Rolling update | без простоя | 4 неуспешные попытки, RPS 615 | старые реплики не отвечают | + +Первая версия без предварительного уменьшения упиралась в лимит 512 MiB (RSS 780 MiB при двух 40-Мп задачах). Причина — промежуточный буфер Catmull-Rom размером ширина_цели × высота_источника × 32 байта. После уменьшения блоком 2×2 пик стал около 600 MiB, а время двух таких задач сократилось с 11 до 4.6 s. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Декодирование и ресемплинг одного этапа нельзя прервать: timeout проверяется между этапами и на S3/БД-операциях. Длительность этапа ограничена лимитом пикселей (около 2 s на 40 Мп). +- Broker, PostgreSQL, S3 и HAProxy — единичные экземпляры. `appendfsync always` надёжнее, но медленнее everysec. Stream не обрезается (`XTRIM` не реализован), он растёт на одну запись на попытку. +- Relay работает только в API: если остановлены обе реплики API, принятые задания ждут в outbox. Задержка relay до 300 ms. +- Имена потребителей привязаны к hostname контейнера: после пересоздания worker старые pending переходят к новым через `XAUTOCLAIM` только после lease (75 s), а старые consumer'ы остаются в группе. +- EXIF orientation читается только у JPEG; eXIf-чанк PNG не учитывается. +- `pipeline_version` фиксирован (1). Версионирование нескольких pipeline (трек 5) не делалось. +- Replay — операторская CLI-команда без HTTP API. `last_error` хранит внутренний текст ошибки только в БД и наружу не выдаётся. +- WebP кодируется транспилированным libwebp (без CGo): медленнее нативного, но без системных библиотек. +- Секреты лежат в `./secrets` вне Git. Для чтения процессом с UID 10001 у них права 644: это допустимо только для локального стенда. diff --git a/evidence/attachments.txt b/evidence/attachments.txt new file mode 100644 index 00000000..8ec36d20 --- /dev/null +++ b/evidence/attachments.txt @@ -0,0 +1,3 @@ +# 2026-11-05T21:56:06Z $ python3 tests/attachments.py +PASS: attachment validation, limits and access control +# exit=0 diff --git a/evidence/failover-1.txt b/evidence/failover-1.txt new file mode 100644 index 00000000..7f7a0445 --- /dev/null +++ b/evidence/failover-1.txt @@ -0,0 +1,8 @@ +# 2026-11-05T21:57:18Z $ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 139, "failed_attempts": 1, "first_recovery_seconds": 3.475} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +# exit=0 +# api containers after hook +mtusi2026lab5-fork-api-1 Exited (0) 12 seconds ago +mtusi2026lab5-fork-api-2 Up 35 seconds (healthy) diff --git a/evidence/failover-2.txt b/evidence/failover-2.txt new file mode 100644 index 00000000..537bfe2c --- /dev/null +++ b/evidence/failover-2.txt @@ -0,0 +1,7 @@ +# 2026-11-05T22:01:19Z $ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 140, "failed_attempts": 0, "first_recovery_seconds": 3.465} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +# api containers after hook +mtusi2026lab5-fork-api-1 Up 35 seconds (healthy) +mtusi2026lab5-fork-api-2 Exited (0) 12 seconds ago diff --git a/evidence/failover-each.json b/evidence/failover-each.json new file mode 100644 index 00000000..1ecd3d31 --- /dev/null +++ b/evidence/failover-each.json @@ -0,0 +1,71 @@ +{ + "baseline": { + "window_seconds": 6, + "successful_requests": 3146, + "failed_attempts": 0, + "successful_rps": 524.3, + "last_error_offset_seconds": null, + "instances": [ + "4337e4656268", + "eb2165af370d" + ] + }, + "replicas": [ + { + "stopped_replica": "eb2165af370d", + "during_failure": { + "window_seconds": 15.35, + "successful_requests": 6738, + "failed_attempts": 2, + "successful_rps": 438.9, + "last_error_offset_seconds": 0.66, + "instances": [ + "4337e4656268", + "eb2165af370d" + ] + }, + "seen_stopped_replica_after_stop": false, + "stop_command_seconds": 3.35, + "restore_seconds": 6.48, + "after_restore": { + "window_seconds": 7.0, + "successful_requests": 3915, + "failed_attempts": 0, + "successful_rps": 559.3, + "last_error_offset_seconds": null, + "instances": [ + "4337e4656268", + "eb2165af370d" + ] + } + }, + { + "stopped_replica": "4337e4656268", + "during_failure": { + "window_seconds": 15.5, + "successful_requests": 6926, + "failed_attempts": 0, + "successful_rps": 446.9, + "last_error_offset_seconds": null, + "instances": [ + "4337e4656268", + "eb2165af370d" + ] + }, + "seen_stopped_replica_after_stop": false, + "stop_command_seconds": 3.5, + "restore_seconds": 6.52, + "after_restore": { + "window_seconds": 7.0, + "successful_requests": 3608, + "failed_attempts": 0, + "successful_rps": 515.4, + "last_error_offset_seconds": null, + "instances": [ + "4337e4656268", + "eb2165af370d" + ] + } + } + ] +} diff --git a/evidence/faults.txt b/evidence/faults.txt new file mode 100644 index 00000000..5a5824d3 --- /dev/null +++ b/evidence/faults.txt @@ -0,0 +1,9 @@ +# 2026-11-05T22:02:15Z $ python3 tests/faults.py +1) api crashed after COMMIT before 202/publish: attachment=50f6ace7-1393-4c19-b47a-1def389a62c3 recovered by outbox relay (published rows=1) -> ready attempt=1 +2) worker crashed after S3 write before COMMIT: lease expired, reaper requeued, attempt=2 ready, variants=3 +3) worker crashed after COMMIT before ack: message redelivered after restart, skipped as no-op, attempt=1, variants=3 +4) s3 down: transient errors retried with backoff+jitter delays_ms=[931, 1284, 2760, 7484], then failed retry_exhausted, listed in registry + replay: failed -> queued -> ready (audit rows=1); replay of ready job skipped, variants=3 +5) corrupted IDAT (passes header/tail check in API): failed invalid_image on attempt 1, no retries +PASS: outbox recovery, lease recovery, redelivery after commit, backoff with jitter, retry_exhausted, replay, permanent error +# exit=0 diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt new file mode 100644 index 00000000..0830e230 --- /dev/null +++ b/evidence/idempotency.txt @@ -0,0 +1,3 @@ +# 2026-11-05T21:56:12Z $ python3 tests/idempotency.py +PASS: 32 parallel requests, 1 message, 2 instances +# exit=0 diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..c70f73f7 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,25 @@ +# before +api Up 2 minutes (healthy) +api Up 2 minutes (healthy) +broker Up 16 minutes (healthy) +janitor Up 2 minutes +lb Up 12 minutes +postgres Up 19 minutes (healthy) +s3 Up 7 minutes (healthy) +valkey Up 19 minutes (healthy) +worker Up 37 seconds +# 2026-11-05T21:56:14Z $ make test-persistence ACTION_SCRIPT=scripts/restart.sh +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +# exit=0 +# after +api Up 2 seconds (health: starting) +api Up 2 seconds (health: starting) +broker Up 46 seconds (healthy) +janitor Up 5 seconds +lb Up 5 seconds +postgres Up 46 seconds (healthy) +s3 Up 36 seconds (healthy) +valkey Up 46 seconds (healthy) +worker Up Less than a second diff --git a/evidence/queue-recovery.txt b/evidence/queue-recovery.txt new file mode 100644 index 00000000..9b024075 --- /dev/null +++ b/evidence/queue-recovery.txt @@ -0,0 +1,7 @@ +# 2026-11-05T22:01:54Z $ python3 tests/queue_recovery.py +accepted: attachment=d4c108ad-3da8-467c-aee0-eb2ee25759fc job=0208b98f-c201-4686-aa66-62b0bbf703ae status=queued, job in stream, worker stopped +restarted broker and api: job still in stream, attachment still queued +worker started: attachment ready, attempt=1, variants=3 +duplicate delivery: skipped as no-op, acked, variants=3, updated_at unchanged +PASS: worker stop, broker/api restart, redelivery idempotency +# exit=0 diff --git a/evidence/rolling.json b/evidence/rolling.json new file mode 100644 index 00000000..ee0b09e9 --- /dev/null +++ b/evidence/rolling.json @@ -0,0 +1,24 @@ +{ + "old_replicas": [ + "eb2165af370d", + "4337e4656268" + ], + "new_replicas": [ + "2ce132adb5c9", + "2fe5ec734588" + ], + "update_window": { + "window_seconds": 16.4, + "successful_requests": 10091, + "failed_attempts": 4, + "successful_rps": 615.4, + "last_error_offset_seconds": 13.57, + "instances": [ + "2ce132adb5c9", + "2fe5ec734588", + "4337e4656268", + "eb2165af370d" + ] + }, + "old_replicas_served_after_update": [] +} diff --git a/evidence/s3-checks.txt b/evidence/s3-checks.txt new file mode 100644 index 00000000..4dcdd84c --- /dev/null +++ b/evidence/s3-checks.txt @@ -0,0 +1,3 @@ +# 2026-11-05T22:04:42Z $ python3 tests/s3_checks.py +PASS: private bucket, limited credentials, s3 outage, orphan and staging cleanup +# exit=0 diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..687a0371 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,6 @@ +# 2026-11-05T21:56:01Z $ sh -c make check && make test +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +# exit=0 diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 00000000..12f58996 --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,18 @@ +{ + "file_bytes": 9722608, + "concurrent_uploads": 12, + "rounds": 3, + "uploads_ok": 36, + "uploads_failed": 0, + "elapsed_seconds": 12.65, + "api_memory_limit_mib": 192, + "idle_memory_mib": { + "mtusi2026lab5-fork-api-1": 62.4, + "mtusi2026lab5-fork-api-2": 62.2 + }, + "peak_memory_mib": { + "mtusi2026lab5-fork-api-1": 118.8, + "mtusi2026lab5-fork-api-2": 99.5 + }, + "buffering_all_files_would_need_mib": 111.3 +} diff --git a/evidence/variants.txt b/evidence/variants.txt new file mode 100644 index 00000000..bf08edf0 --- /dev/null +++ b/evidence/variants.txt @@ -0,0 +1,3 @@ +# 2026-11-05T21:56:02Z $ python3 tests/variants.py +PASS: thumbnail/crop/optimized sizes, formats, checksums, no upscale, alpha on white, EXIF orientation, no EXIF in variants +# exit=0 diff --git a/evidence/worker-load.json b/evidence/worker-load.json new file mode 100644 index 00000000..00cc42e1 --- /dev/null +++ b/evidence/worker-load.json @@ -0,0 +1,26 @@ +{ + "huge_images": { + "count": 6, + "pixels": 39942400, + "bytes": 116545 + }, + "small_images": 20, + "worker_concurrency": 2, + "max_processing_jobs_observed": 2, + "all_ready": true, + "accept_seconds": 0.32, + "all_ready_seconds": 13.1, + "worker_peak_rss_mib": 616.7, + "worker_memory_limit_mib": 768, + "worker_restarts_oom": [ + "0", + "false" + ], + "timeout_case": { + "job_timeout_seconds": 1, + "status": "failed", + "error_code": "retry_exhausted", + "attempts": 5, + "last_error": "context deadline exceeded" + } +} -- 2.54.0 From 583013fa06eca3580bea148bc1ba3d785edbcdcc Mon Sep 17 00:00:00 2001 From: gera Date: Thu, 19 Nov 2026 21:38:31 +0300 Subject: [PATCH 5/5] =?UTF-8?q?=D0=98=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D0=BB=20=D0=BE=D1=82=D0=B7=D1=8B=D0=B2=20=D1=81=D0=B5=D1=81?= =?UTF-8?q?=D1=81=D0=B8=D0=B8=20=D0=B1=D0=B5=D0=B7=20=D0=B4=D0=BE=D0=B2?= =?UTF-8?q?=D0=B5=D1=80=D0=B8=D1=8F=20=D0=BA=20=D0=BA=D1=8D=D1=88=D1=83=20?= =?UTF-8?q?=D0=B8=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D0=BB=20=D0=BD?= =?UTF-8?q?=D0=B5=D0=B4=D0=BE=D1=81=D1=82=D0=B0=D1=8E=D1=89=D0=B8=D0=B5=20?= =?UTF-8?q?=D1=82=D0=B5=D1=81=D1=82=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGES.md | 91 +++++++++++++++++++ REPORT.md | 4 +- cache.go | 104 --------------------- evidence/session-revocation.txt | 1 + handlers.go | 61 +++---------- main.go | 5 +- tests/idempotency.py | 42 +++++++++ tests/image_gen.py | 23 +++++ tests/s3_checks.py | 134 +++++++++++++++++++++++++++ tests/scenario.py | 155 ++++++++++++++++++++++++++++++++ tests/session_revocation.py | 31 +++++++ tests/upload_memory.py | 106 ++++++++++++++++++++++ 12 files changed, 597 insertions(+), 160 deletions(-) create mode 100644 CHANGES.md delete mode 100644 cache.go create mode 100644 evidence/session-revocation.txt create mode 100644 tests/idempotency.py create mode 100644 tests/image_gen.py create mode 100644 tests/s3_checks.py create mode 100644 tests/scenario.py create mode 100644 tests/session_revocation.py create mode 100644 tests/upload_memory.py diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 00000000..bf7cd075 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,91 @@ +# Изменения lab5 относительно lab4 + +## API + +- `POST /chats/{chat_id}/attachments` возвращает `202` со статусом `queued` вместо `201 ready`. Проверки при приёме те же, что в lab4. +- Оригинал сначала сохраняется в приватный S3-prefix `staging/`, а не сразу в конечный `attachments/`. +- `GET /attachments/{id}` возвращает статусы `queued`, `processing`, `ready`, `failed`, а также `error_code` и `variants`. +- `GET /attachments/{id}/content`: + - принимает `variant=original|thumbnail|crop|optimized`, неизвестное имя → 400; + - для не-ready вложения → 409; + - отсутствующий вариант ready-вложения → 404. +- `POST /chats/{chat_id}/messages`: прикрепить `failed`-вложение нельзя (409), `queued`/`processing` можно. + +## База данных + +Новая миграция `migrations/0004_jobs.sql`: + +- В `attachments` добавлены `status`, `error_code`, `staging_key`, `updated_at`. `object_key` теперь заполняет worker. +- `attachment_variants`: варианты с первичным ключом `(attachment_id, pipeline_version, name)`. +- `jobs`: задача обработки (статус, попытка, лимит попыток, время следующей попытки, lease, последняя ошибка). +- `outbox`: задания на публикацию в брокер. +- `job_replays`: аудит операторских повторов. + +## Новые компоненты + +- `broker`: Valkey Streams, отдельный сервис с volume, `appendfsync always`, `noeviction`, паролем из secrets. Stream `attachments.jobs`, группа `workers`. +- `worker`: отдельный процесс (`messenger worker`), 2 потребителя, prefetch 1, лимит 768 MiB. +- `broker.go`: формат сообщения по `jobs.schema.json`, публикация, outbox relay (работает в каждой API-реплике). +- `worker.go`: + - захват задачи с lease и fencing по `lease_owner` и `attempt`; + - обработка и фиксация результата; + - retry с backoff и jitter; + - reaper для задач с истёкшим lease; + - повторное чтение своих pending и `XAUTOCLAIM`; + - команды `failed` и `replay`. +- `imaging.go`: + - thumbnail JPEG 256, crop JPEG 128×128, optimized WebP quality 80 (вписан в 1280); + - применение EXIF orientation, alpha на белом; + - предварительное уменьшение больших изображений для экономии памяти. + +## Изменения существующего кода + +- `attachments.go`: приём в staging, транзакция `attachments` + `jobs` + `outbox`, ответ 202, загрузка вариантов, скачивание по `variant`. Janitor дополнительно чистит `staging/`. +- `handlers.go`: проверка failed-вложений при создании сообщения, точка внедрения отказа. +- `main.go`: подкоманды `worker`, `failed`, `replay`; запуск relay в API; новые переменные окружения. +- `s3.go`: listing по произвольному prefix. +- `Dockerfile`: сборка с тегом `nodynamic` (WebP без системных библиотек). +- `go.mod`: добавлены `golang.org/x/image`, `github.com/gen2brain/webp`, `github.com/rwcarlsen/goexif`. + +## Инфраструктура и скрипты + +- `compose.yaml`: сервисы `broker` и `worker`, volume `broker_data`, secrets `broker_conf` и `broker_url`, образ `mtusi-messenger:lab5`, переменные для точек внедрения отказа (по умолчанию пусты). +- `scripts/mksecrets.sh`: генерирует пароль брокера, `broker.conf` и `broker_url`. +- `scripts/restart.sh`: перезапускает также `broker` и `worker`. +- `scripts/rolling-update.sh`: обновляет также `worker`. +- `.env.example`: `WORKER_CONCURRENCY`, `JOB_MAX_ATTEMPTS`, `JOB_TIMEOUT_SECONDS`, `JOB_BACKOFF_BASE_MS`, `JOB_BACKOFF_MAX_MS`, `STAGING_RETENTION_SECONDS`. + +## Надёжность + +- Работа не теряется после ответа 202: оригинал в S3, задача и outbox в одной транзакции PostgreSQL. +- Разрыв COMMIT/publish закрыт transactional outbox. +- Повторная доставка и повторы не создают второго набора вариантов: детерминированные S3-ключи и upsert по ключу эффекта. +- Зависшие задачи возвращаются в очередь по истечении lease. +- Постоянные ошибки (`invalid_image`, `processing_failed`) уходят в failed сразу, временные повторяются до `retry_exhausted`. +- Failed-задачи можно повторить через `replay`. Повтор уже готовой задачи ничего не делает. +- Повреждение внутри PNG, которое lab4 пропускал (проверялись только заголовок и конец), теперь ловит worker при полном декодировании. + +## Тесты + +Новые: +- `tests/stack.py`: общие хелперы. +- `tests/variants.py`: размеры, форматы, sha256, без увеличения, alpha, EXIF. +- `tests/queue_recovery.py`: сценарий на 3 и повторная доставка. +- `tests/faults.py`: 5 сценариев сбоев на 4. +- `tests/worker_load.py`: конкуренция, память и timeout worker. + +Адаптированы под 202: `tests/attachments.py`, `tests/s3_checks.py` (плюс проверка очистки staging), `tests/upload_memory.py`. + +Без изменений: `tests/idempotency.py`, `tests/scenario.py`, `tests/image_gen.py`. + +## Отчёт и доказательства + +- `REPORT.md` переписан под lab5. +- Новые файлы в `evidence/`: `variants.txt`, `queue-recovery.txt`, `faults.txt`, `worker-load.json`. +- Остальные файлы evidence получены заново на стенде lab5. + +## Исправления после ревью + +- Авторизация проверяет сессию только в PostgreSQL одним запросом (`sessions` ⋈ `users`, `expires_at > now()`); запись кэша не принимается за доказательство действительности токена. Код кэша сессий (`cache.go`) удалён. +- Добавлен `tests/session_revocation.py`: Valkey остановлен → logout 204 → Valkey запущен → прежний токен 401. +- Тесты `image_gen.py`, `idempotency.py`, `s3_checks.py`, `scenario.py`, `upload_memory.py` были не добавлены в git, из-за чего `attachments.py` и `worker_load.py` падали с `ModuleNotFoundError: image_gen`. diff --git a/REPORT.md b/REPORT.md index e8ef995e..bc4e3983 100644 --- a/REPORT.md +++ b/REPORT.md @@ -42,6 +42,7 @@ python3 tests/attachments.py python3 tests/s3_checks.py python3 tests/upload_memory.py --output evidence/upload-memory.json python3 tests/idempotency.py +python3 tests/session_revocation.py python3 tests/scenario.py --mode failover-each --output evidence/failover-each.json python3 tests/scenario.py --mode rolling --output evidence/rolling.json ``` @@ -65,7 +66,7 @@ worker ──XREADGROUP──→ claim (lease) → S3 attachments//... → C | broker | `broker` | valkey/valkey:8.1-alpine@sha256:081c2f5c… | `valkey-cli ping` с паролем | broker_data | backend | broker.conf (`requirepass`) | | s3 | `s3` | chrislusf/seaweedfs@sha256:c2658820… | `/dir/assign` выдаёт fid | s3_data | storage | s3_config.json | | load_balancer | `lb` | haproxy:3.0-alpine@sha256:cf3af0fa… | проверки `/health/ready` бэкендов | нет | backend, 127.0.0.1:8080 | нет | -| кэш сессий | `valkey` | valkey/valkey:8.1-alpine | `valkey-cli ping` | valkey_data | backend | нет | +| Valkey (сессии проверяются только в PostgreSQL) | `valkey` | valkey/valkey:8.1-alpine | `valkey-cli ping` | valkey_data | backend | нет | Bootstrap: `migrate` применяет `migrations/*.sql` под advisory lock, повторный запуск пропускает применённые. `s3-init` создаёт bucket, если его нет. Consumer group `workers` создают API (перед первой публикацией) и worker при старте (`XGROUP CREATE … 0 MKSTREAM`, ошибка BUSYGROUP игнорируется). @@ -128,6 +129,7 @@ Bootstrap: `migrate` применяет `migrations/*.sql` под advisory lock, | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | | Публичный smoke (202 → ready, оригинал, настоящий JPEG 256×160) | `make check && make test` | PASS | evidence/smoke.txt | +| Отозванная сессия не воскресает после перезапуска Valkey | `tests/session_revocation.py` | Valkey остановлен → logout 204 → Valkey запущен → прежний токен 401 | evidence/session-revocation.txt | | Минимум lab4 сохранён (валидация, доступ, приватный bucket, отказ S3, идемпотентность) | `tests/attachments.py`, `tests/s3_checks.py`, `tests/idempotency.py` | PASS: 413/415/400/404, анонимный доступ 403, 503 при остановленном S3 | evidence/attachments.txt, evidence/s3-checks.txt, evidence/idempotency.txt | | Restart всех stores с volumes | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; перезапущены postgres, valkey, broker, s3, api, lb, janitor, worker (состояния до/после в файле) | evidence/persistence.txt | | Failover каждой API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS для api-1 и api-2, 0–1 неуспешная попытка, восстановление 3.5 s | evidence/failover-1.txt, evidence/failover-2.txt | diff --git a/cache.go b/cache.go deleted file mode 100644 index 9afc5e3f..00000000 --- a/cache.go +++ /dev/null @@ -1,104 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "log/slog" - "time" - - "github.com/redis/go-redis/v9" -) - -type cachedSession struct { - UserID string `json:"user_id"` - ExpiresAt time.Time `json:"expires_at"` -} - -type Cache struct { - rdb *redis.Client - log *slog.Logger -} - -func openCache(log *slog.Logger) *Cache { - url, err := readSecret("CACHE_URL", "CACHE_URL_FILE") - if err != nil { - log.Warn("cache disabled", "err", err.Error()) - return &Cache{log: log} - } - opt, err := redis.ParseURL(url) - if err != nil { - log.Warn("cache parse failed", "err", err.Error()) - return &Cache{log: log} - } - opt.DialTimeout = 200 * time.Millisecond - opt.ReadTimeout = 200 * time.Millisecond - opt.WriteTimeout = 200 * time.Millisecond - opt.PoolTimeout = 200 * time.Millisecond - rdb := redis.NewClient(opt) - ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) - defer cancel() - if err := rdb.Ping(ctx).Err(); err != nil { - log.Warn("cache ping failed, will bypass", "err", err.Error()) - } - return &Cache{rdb: rdb, log: log} -} - -func (c *Cache) Close() { - if c.rdb != nil { - _ = c.rdb.Close() - } -} - -func sessKey(hashHex string) string { - return "sess:" + hashHex -} - -func (c *Cache) GetSession(ctx context.Context, hashHex string) (*cachedSession, bool) { - if c.rdb == nil { - return nil, false - } - ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) - defer cancel() - val, err := c.rdb.Get(ctx, sessKey(hashHex)).Result() - if err != nil { - return nil, false - } - var s cachedSession - if err := json.Unmarshal([]byte(val), &s); err != nil { - return nil, false - } - if time.Now().UTC().After(s.ExpiresAt) { - return nil, false - } - return &s, true -} - -func (c *Cache) SetSession(ctx context.Context, hashHex string, userID string, expiresAt time.Time) { - if c.rdb == nil { - return - } - ttl := time.Until(expiresAt) - if ttl <= 0 { - return - } - body, err := json.Marshal(cachedSession{UserID: userID, ExpiresAt: expiresAt.UTC()}) - if err != nil { - return - } - ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) - defer cancel() - if err := c.rdb.Set(ctx, sessKey(hashHex), body, ttl).Err(); err != nil { - c.log.Warn("cache set failed", "err", err.Error()) - } -} - -func (c *Cache) DelSession(ctx context.Context, hashHex string) { - if c.rdb == nil { - return - } - ctx, cancel := context.WithTimeout(ctx, 200*time.Millisecond) - defer cancel() - if err := c.rdb.Del(ctx, sessKey(hashHex)).Err(); err != nil { - c.log.Warn("cache del failed", "err", err.Error()) - } -} diff --git a/evidence/session-revocation.txt b/evidence/session-revocation.txt new file mode 100644 index 00000000..38dab274 --- /dev/null +++ b/evidence/session-revocation.txt @@ -0,0 +1 @@ +PASS: token revoked while valkey was stopped stays revoked after valkey restart (401) diff --git a/handlers.go b/handlers.go index 7fc5c807..0dadcec8 100644 --- a/handlers.go +++ b/handlers.go @@ -49,7 +49,6 @@ type errorBody struct { type App struct { db *pgxpool.Pool - cache *Cache log *slog.Logger ready atomic.Bool sessionTTL time.Duration @@ -60,8 +59,8 @@ type App struct { fault string } -func NewApp(db *pgxpool.Pool, cache *Cache, log *slog.Logger, sessionTTL, idempotencyTTL time.Duration, store *Store, lim limits) *App { - a := &App{db: db, cache: cache, log: log, sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, store: store, limits: lim, uploads: make(chan struct{}, lim.uploadSlots), fault: os.Getenv("FAULT_POINT")} +func NewApp(db *pgxpool.Pool, log *slog.Logger, sessionTTL, idempotencyTTL time.Duration, store *Store, lim limits) *App { + a := &App{db: db, log: log, sessionTTL: sessionTTL, idempotencyTTL: idempotencyTTL, store: store, limits: lim, uploads: make(chan struct{}, lim.uploadSlots), fault: os.Getenv("FAULT_POINT")} a.ready.Store(true) return a } @@ -268,23 +267,6 @@ func userJSON(u *User) map[string]any { } } -func (a *App) getUserByID(ctx context.Context, id string) (*User, error) { - ctx, cancel := dbCtx(ctx) - defer cancel() - var u User - err := a.db.QueryRow(ctx, ` - select id::text, username, display_name, password_hash, created_at - from users where id = $1::uuid`, id).Scan( - &u.ID, &u.Username, &u.DisplayName, &u.PasswordHash, &u.CreatedAt) - if errors.Is(err, pgx.ErrNoRows) { - return nil, nil - } - if err != nil { - return nil, err - } - return &u, nil -} - func (a *App) authUser(w http.ResponseWriter, r *http.Request) (*User, string) { token, ok := parseBearer(r.Header.Get("Authorization")) if !ok { @@ -293,26 +275,15 @@ func (a *App) authUser(w http.ResponseWriter, r *http.Request) (*User, string) { } hash, hashHex := tokenHash(token) - if cs, hit := a.cache.GetSession(r.Context(), hashHex); hit { - u, err := a.getUserByID(r.Context(), cs.UserID) - if err != nil { - a.writeErr(w, reqIDFrom(r), 503, "unavailable", "database unavailable") - return nil, "" - } - if u == nil { - a.writeUnauthorized(w, r, "Bearer") - return nil, "" - } - return u, hashHex - } - ctx, cancel := dbCtx(r.Context()) defer cancel() - var userID string + var u User var expiresAt time.Time err := a.db.QueryRow(ctx, ` - select user_id::text, expires_at from sessions - where token_hash = $1 and expires_at > now()`, hash).Scan(&userID, &expiresAt) + select u.id::text, u.username, u.display_name, u.password_hash, u.created_at, s.expires_at + from sessions s join users u on u.id = s.user_id + where s.token_hash = $1 and s.expires_at > now()`, hash).Scan( + &u.ID, &u.Username, &u.DisplayName, &u.PasswordHash, &u.CreatedAt, &expiresAt) if errors.Is(err, pgx.ErrNoRows) { a.writeUnauthorized(w, r, "Bearer") return nil, "" @@ -321,17 +292,7 @@ func (a *App) authUser(w http.ResponseWriter, r *http.Request) (*User, string) { a.writeErr(w, reqIDFrom(r), 503, "unavailable", "database unavailable") return nil, "" } - u, err := a.getUserByID(r.Context(), userID) - if err != nil { - a.writeErr(w, reqIDFrom(r), 503, "unavailable", "database unavailable") - return nil, "" - } - if u == nil { - a.writeUnauthorized(w, r, "Bearer") - return nil, "" - } - a.cache.SetSession(r.Context(), hashHex, userID, expiresAt) - return u, hashHex + return &u, hashHex } func (a *App) handleCreateUser(w http.ResponseWriter, r *http.Request) { @@ -423,7 +384,7 @@ func (a *App) handleCreateSession(w http.ResponseWriter, r *http.Request) { a.writeUnauthorized(w, r, "Basic") return } - token, hash, hashHex, err := newToken() + token, hash, _, err := newToken() if err != nil { a.writeErr(w, reqID, 500, "internal_error", "internal error") return @@ -437,7 +398,6 @@ func (a *App) handleCreateSession(w http.ResponseWriter, r *http.Request) { a.writeErr(w, reqID, 503, "unavailable", "database unavailable") return } - a.cache.SetSession(r.Context(), hashHex, u.ID, expiresAt) a.writeJSON(w, 201, map[string]any{ "token": token, "token_type": "Bearer", @@ -453,7 +413,7 @@ func (a *App) handleRevokeSession(w http.ResponseWriter, r *http.Request) { a.writeUnauthorized(w, r, "Bearer") return } - hash, hashHex := tokenHash(token) + hash, _ := tokenHash(token) ctx, cancel := dbCtx(r.Context()) defer cancel() tag, err := a.db.Exec(ctx, ` @@ -462,7 +422,6 @@ func (a *App) handleRevokeSession(w http.ResponseWriter, r *http.Request) { a.writeErr(w, reqID, 503, "unavailable", "database unavailable") return } - a.cache.DelSession(r.Context(), hashHex) if tag.RowsAffected() == 0 { a.writeUnauthorized(w, r, "Bearer") return diff --git a/main.go b/main.go index e72de1f6..a3383716 100644 --- a/main.go +++ b/main.go @@ -125,11 +125,8 @@ func main() { return } - cache := openCache(log) - defer cache.Close() - slots := envInt("UPLOAD_CONCURRENCY", 10) - app := NewApp(db, cache, log, time.Duration(ttlSec)*time.Second, time.Duration(idemSec)*time.Second, store, + app := NewApp(db, log, time.Duration(ttlSec)*time.Second, time.Duration(idemSec)*time.Second, store, limits{maxBytes: maxBytes, maxPixels: maxPixels, uploadSlots: slots, maxAttempts: maxAttempts}) relayCtx, stopRelay := context.WithCancel(ctx) defer stopRelay() diff --git a/tests/idempotency.py b/tests/idempotency.py new file mode 100644 index 00000000..951d4385 --- /dev/null +++ b/tests/idempotency.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import threading +import uuid +from client import Client, Failure, arguments, check, run + + +def main(): + args = arguments('Concurrent Idempotency-Key check through the load balancer') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/messages' + parallel = 32 + key = 'idem-' + uuid.uuid4().hex + barrier = threading.Barrier(parallel) + + def send(_): + local = Client(args.base_url, args.ca_file) + barrier.wait() + return local.call('POST', path, 201, actor=alice, body={'text': 'same'}, headers={'Idempotency-Key': key}, schema='Message') + + with ThreadPoolExecutor(max_workers=parallel) as pool: + results = list(pool.map(send, range(parallel))) + ids = {r['json']['id'] for r in results} + instances = {r['headers']['X-Instance-Id'] for r in results} + check(len(ids) == 1, 'parallel requests produced %d messages' % len(ids)) + check(len(instances) >= 2, 'requests were served by one instance only') + check(len({r['json']['created_at'] for r in results}) == 1, 'created_at differs between replays') + + listing = c.call('GET', path, actor=alice, schema='MessagePage')['json']['items'] + check(sum(1 for m in listing if m['text'] == 'same') == 1, 'duplicate message in history') + + c.call('POST', path, 409, actor=alice, body={'text': 'other'}, headers={'Idempotency-Key': key}) + c.call('POST', path, 400, actor=alice, body={'text': 'x'}) + c.call('POST', path, 400, actor=alice, body={'text': 'x'}, headers={'Idempotency-Key': 'bad key!'}) + other = c.call('POST', path, 201, actor=bob, body={'text': 'same'}, headers={'Idempotency-Key': key}, schema='Message')['json'] + check(other['id'] not in ids, 'key must be scoped per user') + print('PASS: %d parallel requests, 1 message, %d instances' % (parallel, len(instances))) + + +if __name__ == '__main__': + run(main) diff --git a/tests/image_gen.py b/tests/image_gen.py new file mode 100644 index 00000000..aeb7de0f --- /dev/null +++ b/tests/image_gen.py @@ -0,0 +1,23 @@ +import os +import struct +import zlib + + +def chunk(kind, data): + body = kind + data + return struct.pack('>I', len(data)) + body + struct.pack('>I', zlib.crc32(body) & 0xffffffff) + + +def png(width, height, noise=False): + header = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + raw = bytearray() + row = width * 3 + 1 + for _ in range(height): + raw += b'\x00' + (os.urandom(row - 1) if noise else bytes(row - 1)) + data = zlib.compress(bytes(raw), 0 if noise else 6) + return header + chunk(b'IDAT', data) + chunk(b'IEND', b'') + + +def png_header_only(width, height): + header = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)) + return header + chunk(b'IDAT', zlib.compress(b'\x00')) + chunk(b'IEND', b'') diff --git a/tests/s3_checks.py b/tests/s3_checks.py new file mode 100644 index 00000000..25413909 --- /dev/null +++ b/tests/s3_checks.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +import hashlib +import subprocess +import time +import uuid +from pathlib import Path +from client import Client, Failure, arguments, check, run +from stack import wait_status + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() +BUCKET = 'messenger-attachments' + + +def compose(*args, check_rc=True, stdin=None): + return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=check_rc, + capture_output=True, text=True, input=stdin) + + +def secret(name): + return (ROOT / 'secrets' / name).read_text().strip() + + +def s3_status(method, path, user=None, data=None): + cmd = ['exec', '-T', 's3', 'curl', '-s', '-o', '/dev/null', '-w', '%{http_code}', '-X', method] + if user: + cmd += ['--aws-sigv4', 'aws:amz:us-east-1:s3', '--user', user[0] + ':' + user[1]] + if data is not None: + cmd += ['--data-binary', data] + cmd.append('http://127.0.0.1:9000' + path) + return compose(*cmd).stdout.strip() + + +def upload(c, actor, chat, expected): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=a.png', expected, actor=actor, + raw=SAMPLE, headers={'Content-Type': 'image/png'}, schema='Attachment' if expected == 202 else None) + + +def wait_api(c): + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + try: + c.call('GET', '/health/ready') + return + except Failure: + time.sleep(1) + raise Failure('api did not recover') + + +def wait_s3(timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if compose('ps', 's3', '--format', '{{.Status}}').stdout.count('(healthy)'): + return + time.sleep(1) + raise Failure('s3 did not recover') + + +def object_key(attachment_id): + out = compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', + "select object_key from attachments where id='%s'" % attachment_id).stdout.strip() + check(out.startswith('attachments/'), 'object key not found') + return out + + +def main(): + args = arguments('S3 privacy, outage and orphan cleanup checks') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + att = upload(c, alice, chat, 202)['json'] + wait_status(c, bob, att['id'], ('ready',), 60) + key = object_key(att['id']) + check(key != 'attachments/' + att['filename'] and att['filename'] not in key, 'object key derived from filename') + + api = (secret('s3_access_key_id'), secret('s3_secret_access_key')) + admin = (secret('s3_admin_access_key_id'), secret('s3_admin_secret_access_key')) + path = '/' + BUCKET + '/' + key + check(s3_status('GET', path) == '403', 'anonymous GET must be denied') + check(s3_status('GET', '/' + BUCKET + '/') == '403', 'anonymous listing must be denied') + check(s3_status('PUT', path + '-anon', data='x') == '403', 'anonymous PUT must be denied') + check(s3_status('GET', path, ('wrong', 'wrong')) == '403', 'wrong credentials must be denied') + check(s3_status('GET', path, api) == '200', 'api credentials cannot read') + check(s3_status('PUT', '/other-bucket-' + uuid.uuid4().hex[:8], api) == '403', 'api credentials can create buckets') + ports = compose('ps', '--format', '{{.Service}} {{.Ports}}').stdout + published = [line.split()[0] for line in ports.splitlines() if '->' in line] + check(published == ['lb'], 'only lb may publish ports, found %s' % published) + + compose('stop', 's3') + try: + upload(c, alice, chat, 503) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 503, actor=bob) + c.call('GET', '/api/v1/attachments/' + att['id'], 200, actor=bob, schema='Attachment') + c.message(alice, chat, 'text works while s3 is down') + finally: + compose('start', 's3') + wait_s3() + wait_api(c) + deadline = time.monotonic() + 60 + while True: + try: + body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] + check(body == SAMPLE, 'content differs after s3 restart') + break + except Failure: + check(time.monotonic() < deadline, 'download did not recover') + time.sleep(1) + upload(c, alice, chat, 202) + + orphan = '/' + BUCKET + '/attachments/' + uuid.uuid4().hex + check(s3_status('PUT', orphan, admin, 'orphan') == '200', 'cannot create orphan') + compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=3600', 'janitor', 'cleanup') + check(s3_status('GET', orphan, admin) == '200', 'fresh object was removed inside grace period') + time.sleep(1) + compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=0', 'janitor', 'cleanup') + check(s3_status('GET', orphan, admin) == '404', 'orphan was not removed') + check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == SAMPLE, 'referenced object was removed') + + check(s3_status('GET', '/' + BUCKET + '/staging/' + att['id'], admin) == '404', 'staging kept after ready') + stale = '/' + BUCKET + '/staging/' + str(uuid.uuid4()) + check(s3_status('PUT', stale, admin, 'stale') == '200', 'cannot create stale staging object') + compose('stop', 'worker') + try: + queued = upload(c, alice, chat, 202)['json'] + compose('run', '--rm', '--no-deps', '-e', 'CLEANUP_GRACE_SECONDS=0', 'janitor', 'cleanup') + check(s3_status('GET', '/' + BUCKET + '/staging/' + queued['id'], admin) == '200', 'staging of queued job was removed') + check(s3_status('GET', stale, admin) == '404', 'unreferenced staging object was not removed') + finally: + compose('up', '-d', '--no-deps', 'worker') + check(wait_status(c, bob, queued['id'], ('ready', 'failed'), 60)['status'] == 'ready', 'queued job lost staging') + print('PASS: private bucket, limited credentials, s3 outage, orphan and staging cleanup') + + +if __name__ == '__main__': + run(main) diff --git a/tests/scenario.py b/tests/scenario.py new file mode 100644 index 00000000..f7ddd700 --- /dev/null +++ b/tests/scenario.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import json +import subprocess +import threading +import time +import uuid +from pathlib import Path +from client import Client, Failure, arguments, check, run + +ROOT = Path(__file__).resolve().parent.parent + + +def sh(*cmd): + return subprocess.run(cmd, cwd=ROOT, check=True, capture_output=True, text=True).stdout.strip() + + +def api_containers(): + out = sh('docker', 'compose', '--env-file', '.env', 'ps', '-q', 'api') + return [x[:12] for x in out.split()] + + +def wait_healthy(count, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + out = sh('docker', 'compose', '--env-file', '.env', 'ps', 'api', '--format', '{{.Status}}') + if out.count('(healthy)') >= count: + return + time.sleep(1) + raise Failure('replicas did not become healthy') + + +class Load: + def __init__(self, base, ca, concurrency): + c = Client(base, ca) + self.alice, _, _, self.chat = c.bootstrap() + self.base, self.ca, self.n = base, ca, concurrency + self.events = [] + self.lock = threading.Lock() + self.stop = threading.Event() + self.t0 = time.monotonic() + + def worker(self): + c = Client(self.base, self.ca, timeout=3) + while not self.stop.is_set(): + key = uuid.uuid4().hex + first_error = None + while not self.stop.is_set(): + try: + r = c.message(self.alice, self.chat, 'k-' + key, key=key) + t = time.monotonic() - self.t0 + with self.lock: + self.events.append((t, True, r['headers']['X-Instance-Id'], first_error)) + break + except Failure: + t = time.monotonic() - self.t0 + first_error = first_error or t + with self.lock: + self.events.append((t, False, None, None)) + time.sleep(0.05) + + def start(self): + self.pool = ThreadPoolExecutor(max_workers=self.n) + self.futures = [self.pool.submit(self.worker) for _ in range(self.n)] + + def finish(self): + self.stop.set() + self.pool.shutdown(wait=True) + + def summary(self, begin, end): + ev = [e for e in self.events if begin <= e[0] < end] + ok = [e for e in ev if e[1]] + bad = [e for e in ev if not e[1]] + span = max(end - begin, 1e-9) + return { + 'window_seconds': round(span, 2), + 'successful_requests': len(ok), + 'failed_attempts': len(bad), + 'successful_rps': round(len(ok) / span, 1), + 'last_error_offset_seconds': round(max(e[0] for e in bad) - begin, 2) if bad else None, + 'instances': sorted({e[2] for e in ok}), + } + + def instances_between(self, begin, end=float('inf')): + return sorted({e[2] for e in self.events if e[1] and begin <= e[0] < end}) + + +def failover_each(args): + ids = api_containers() + check(len(ids) >= 2, 'need at least two replicas') + load = Load(args.base_url, args.ca_file, args.concurrency) + load.start() + time.sleep(args.warmup) + results = [] + base = load.summary(2, args.warmup) + for target in ids: + before = time.monotonic() - load.t0 + sh('docker', 'stop', '-t', '10', target) + stopped = time.monotonic() - load.t0 + time.sleep(args.hold) + after_stop = time.monotonic() - load.t0 + window = load.summary(before, after_stop) + sh('docker', 'start', target) + wait_healthy(len(ids)) + restored = time.monotonic() - load.t0 + time.sleep(args.warmup) + results.append({ + 'stopped_replica': target, + 'during_failure': window, + 'seen_stopped_replica_after_stop': target in load.instances_between(stopped + 0.5, after_stop), + 'stop_command_seconds': round(stopped - before, 2), + 'restore_seconds': round(restored - after_stop, 2), + 'after_restore': load.summary(restored + 1, restored + args.warmup), + }) + load.finish() + return {'baseline': base, 'replicas': results} + + +def rolling(args): + old = api_containers() + check(len(old) >= 2, 'need at least two replicas') + n = len(old) + load = Load(args.base_url, args.ca_file, args.concurrency) + load.start() + time.sleep(args.warmup) + begin = time.monotonic() - load.t0 + sh('scripts/rolling-update.sh') + end = time.monotonic() - load.t0 + time.sleep(args.warmup) + load.finish() + return { + 'old_replicas': old, + 'new_replicas': api_containers(), + 'update_window': load.summary(begin, end), + 'old_replicas_served_after_update': [i for i in load.instances_between(end) if i in old], + } + + +def main(): + def configure(p): + p.add_argument('--mode', choices=['failover-each', 'rolling'], required=True) + p.add_argument('--concurrency', type=int, default=8) + p.add_argument('--warmup', type=float, default=8) + p.add_argument('--hold', type=float, default=12) + p.add_argument('--output') + args = arguments('Failure and rolling update under load', configure) + report = failover_each(args) if args.mode == 'failover-each' else rolling(args) + text = json.dumps(report, indent=2) + if args.output: + Path(args.output).write_text(text + '\n') + print(text) + + +if __name__ == '__main__': + run(main) diff --git a/tests/session_revocation.py b/tests/session_revocation.py new file mode 100644 index 00000000..2e2a80de --- /dev/null +++ b/tests/session_revocation.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +import time +from client import Client, arguments, check, run +from stack import compose + + +def wait_valkey(timeout=60): + deadline = time.monotonic() + timeout + while '(healthy)' not in compose('ps', 'valkey', '--format', '{{.Status}}').stdout: + check(time.monotonic() < deadline, 'valkey did not recover') + time.sleep(1) + + +def main(): + args = arguments('Session revocation must not depend on the cache') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + c.call('GET', '/api/v1/chats', 200, actor=alice) + compose('stop', 'valkey') + try: + c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice) + finally: + compose('start', 'valkey') + wait_valkey() + c.call('GET', '/api/v1/chats', 401, actor=alice) + c.call('GET', '/api/v1/chats', 200, actor=bob) + print('PASS: token revoked while valkey was stopped stays revoked after valkey restart (401)') + + +if __name__ == '__main__': + run(main) diff --git a/tests/upload_memory.py b/tests/upload_memory.py new file mode 100644 index 00000000..aa88929f --- /dev/null +++ b/tests/upload_memory.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +import hashlib +import json +import re +import subprocess +import threading +import time +from pathlib import Path +from client import Client, Failure, arguments, check, run +from image_gen import png + +ROOT = Path(__file__).resolve().parent.parent +UNITS = {'B': 1, 'KiB': 1024, 'MiB': 1024 ** 2, 'GiB': 1024 ** 3} + + +def to_mib(text): + value, unit = re.match(r'([\d.]+)\s*([A-Za-z]+)', text).groups() + return float(value) * UNITS[unit] / 1024 ** 2 + + +def sample(stop, peaks): + proc = subprocess.Popen(['docker', 'stats', '--format', '{{.Name}} {{.MemUsage}}'], cwd=ROOT, + stdout=subprocess.PIPE, text=True) + try: + for line in proc.stdout: + line = re.sub(r'\x1b\[[0-9;]*[A-Za-z]', '', line).strip() + m = re.match(r'(\S+-api-\d+)\s+(\S+ ?\S+) /', line) + if m: + peaks[m.group(1)] = max(peaks.get(m.group(1), 0), to_mib(m.group(2))) + if stop.is_set(): + break + finally: + proc.terminate() + + +def main(): + def configure(p): + p.add_argument('--concurrency', type=int, default=12) + p.add_argument('--rounds', type=int, default=3) + p.add_argument('--output') + args = arguments('Concurrent upload memory experiment', configure) + check(args.concurrency >= 10, 'use at least 10 concurrent uploads') + c = Client(args.base_url, args.ca_file, timeout=60) + alice, bob, eve, chat = c.bootstrap() + data = png(1800, 1800, noise=True) + digest = hashlib.sha256(data).hexdigest() + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=load.png' + + idle, stop = {}, threading.Event() + t = threading.Thread(target=sample, args=(stop, idle)) + t.start() + time.sleep(4) + stop.set() + t.join() + + peaks, stop = {}, threading.Event() + t = threading.Thread(target=sample, args=(stop, peaks)) + t.start() + time.sleep(2) + barrier = threading.Barrier(args.concurrency) + results = [] + + def worker(_): + local = Client(args.base_url, args.ca_file, timeout=60) + barrier.wait() + out = [] + for _ in range(args.rounds): + try: + r = local.call('POST', path, 202, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment') + out.append(r['json']['sha256'] == digest) + except Failure: + out.append(False) + return out + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + for out in pool.map(worker, range(args.concurrency)): + results.extend(out) + elapsed = time.monotonic() - started + time.sleep(2) + stop.set() + t.join() + limit = 192 + report = { + 'file_bytes': len(data), + 'concurrent_uploads': args.concurrency, + 'rounds': args.rounds, + 'uploads_ok': sum(results), + 'uploads_failed': len(results) - sum(results), + 'elapsed_seconds': round(elapsed, 2), + 'api_memory_limit_mib': limit, + 'idle_memory_mib': {k: round(v, 1) for k, v in sorted(idle.items())}, + 'peak_memory_mib': {k: round(v, 1) for k, v in sorted(peaks.items())}, + 'buffering_all_files_would_need_mib': round(args.concurrency * len(data) / 1024 ** 2, 1), + } + text = json.dumps(report, indent=2) + if args.output: + Path(args.output).write_text(text + '\n') + print(text) + check(report['uploads_failed'] == 0, 'some uploads failed') + check(max(peaks.values()) < limit, 'peak memory reached the limit') + + +if __name__ == '__main__': + run(main) -- 2.54.0