From e3cd2516bad8a73c155402abf94ac23b7ae07ee6 Mon Sep 17 00:00:00 2001 From: TAskMAster339 Date: Sun, 18 Oct 2026 18:06:27 +0300 Subject: [PATCH] feat: add minio --- Makefile | 9 +- compose.yml | 47 +++- infra/haproxy.cfg | 4 +- infra/minio.Dockerfile | 19 ++ scripts/check_grade3.py | 165 +++++++++++++ scripts/init_env.py | 6 +- scripts/restart.sh | 4 +- src/backend/handlers/attachment_http.go | 145 ++++++++++++ src/backend/handlers/chat_http.go | 6 +- src/backend/handlers/http.go | 17 +- src/backend/handlers/message_http.go | 14 +- src/backend/main.go | 24 +- src/backend/repositories/attachment.go | 42 ++++ src/backend/repositories/idempotency.go | 32 ++- src/backend/repositories/message.go | 26 ++- .../migrations/000004_attachments.down.sql | 5 + .../migrations/000004_attachments.up.sql | 22 ++ src/backend/repositories/s3.go | 99 ++++++++ src/backend/services/attachment.go | 148 ++++++++++++ src/backend/services/health.go | 17 +- src/backend/services/idempotency.go | 19 +- .../tests/attachment_validation_test.go | 94 ++++++++ src/backend/tests/idempotency_test.go | 3 +- src/backend/tests/test_fixture_test.go | 7 + src/frontend/src/App.tsx | 71 +++++- src/frontend/src/AttachmentImage.tsx | 35 +++ src/frontend/src/api.ts | 25 +- src/frontend/src/styles.css | 218 +++++++++--------- src/go.mod | 17 +- src/go.sum | 50 ++-- 30 files changed, 1213 insertions(+), 177 deletions(-) create mode 100644 infra/minio.Dockerfile create mode 100644 scripts/check_grade3.py create mode 100644 src/backend/handlers/attachment_http.go create mode 100644 src/backend/repositories/attachment.go create mode 100644 src/backend/repositories/migrations/000004_attachments.down.sql create mode 100644 src/backend/repositories/migrations/000004_attachments.up.sql create mode 100644 src/backend/repositories/s3.go create mode 100644 src/backend/services/attachment.go create mode 100644 src/backend/tests/attachment_validation_test.go create mode 100644 src/frontend/src/AttachmentImage.tsx diff --git a/Makefile b/Makefile index cfc4b94..a73e72c 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ ACTION_SCRIPT ?= COMPOSE = docker compose --project-name highload-lab4 --env-file .env -f compose.yml TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover test-grade3 test-grade3-failures .PHONY: app-init app-build app-up app-start app-stop app-down app-restart app-logs app-status test-backend test-baseline test-baseline-persistence help: @@ -13,6 +13,7 @@ help: @echo "app-start: start without rebuilding; app-stop: stop and keep containers" @echo "app-down: remove containers and networks, keeping data volumes" @echo "app-restart: restart the stack; app-logs / app-status: inspect services" + @echo "test-grade3 / test-grade3-failures: attachment checks; second target temporarily stops this stack services" @echo "test-backend: Go race tests; test-baseline: transferred API checks" @echo "test-baseline-persistence: transferred API checks across a restart" @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -54,6 +55,12 @@ test-baseline: test-baseline-persistence: $(PYTHON) scripts/check_baseline.py --base-url "$(BASE_URL)" --restart +test-grade3: + $(PYTHON) scripts/check_grade3.py --base-url "$(BASE_URL)" + +test-grade3-failures: + $(PYTHON) scripts/check_grade3.py --base-url "$(BASE_URL)" --failures + check: $(PYTHON) scripts/check_contract.py diff --git a/compose.yml b/compose.yml index d1f2c44..4ff6c75 100644 --- a/compose.yml +++ b/compose.yml @@ -53,17 +53,57 @@ services: depends_on: db: condition: service_healthy - api: + s3: + cpus: 1.0 + mem_limit: 512m + networks: [data] + build: + context: ./infra + dockerfile: minio.Dockerfile + environment: + MINIO_ROOT_USER_FILE: /run/secrets/s3_access_key_id + MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_secret_access_key + MINIO_BROWSER: "off" + volumes: + - s3_data:/data + - ./secrets/s3_access_key_id:/run/secrets/s3_access_key_id:ro + - ./secrets/s3_secret_access_key:/run/secrets/s3_secret_access_key:ro + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/ready"] + interval: 3s + timeout: 2s + retries: 30 + s3-bootstrap: cpus: 0.5 mem_limit: 128m + networks: [data] + build: ./src + environment: &s3-environment + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id} + S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key} + volumes: + - ./secrets/s3_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro + - ./secrets/s3_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro + command: ["./messenger", "s3-bootstrap"] + depends_on: + s3: + condition: service_healthy + api: + cpus: 0.5 + mem_limit: 512m networks: [proxy, data] build: ./src deploy: replicas: ${API_REPLICAS:-2} stop_grace_period: 15s environment: + <<: *s3-environment # Go 1.22 sees host CPUs; keep parallelism suitable for this 0.5 CPU limit. GOMAXPROCS: ${API_GOMAXPROCS:-1} + GOMEMLIMIT: 384MiB APP_HOST: ${APP_HOST:-0.0.0.0} APP_PORT: ${APP_PORT:-8080} APP_ENV: ${APP_ENV:-lab} @@ -74,9 +114,13 @@ services: REDIS_ADDR: redis:6379 DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@db:5432/${POSTGRES_DB}?sslmode=disable volumes: + - ./secrets/s3_access_key_id:${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}:ro + - ./secrets/s3_secret_access_key:${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}:ro - ./secrets/database_url:${DATABASE_URL_FILE:-/run/secrets/database_url}:ro - ./secrets/cache_url:${CACHE_URL_FILE:-/run/secrets/cache_url}:ro depends_on: + s3-bootstrap: + condition: service_completed_successfully migrate: condition: service_completed_successfully db: @@ -165,6 +209,7 @@ services: volumes: postgres_data: redis_data: + s3_data: networks: edge: diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg index 85e2a99..9e63581 100644 --- a/infra/haproxy.cfg +++ b/infra/haproxy.cfg @@ -9,8 +9,8 @@ defaults mode http option httplog timeout connect 1s - timeout client 15s - timeout server 12s + timeout client 35s + timeout server 35s timeout check 1s # Let the client decide which failed POST is safe to repeat. retries 0 diff --git a/infra/minio.Dockerfile b/infra/minio.Dockerfile new file mode 100644 index 0000000..2ab1be7 --- /dev/null +++ b/infra/minio.Dockerfile @@ -0,0 +1,19 @@ +FROM golang:1.24.2-bookworm AS builder +WORKDIR /source +RUN git init . \ + && git remote add origin https://github.com/minio/minio.git \ + && git fetch --depth 1 origin 0d7408fc9969caf07de6a8c3a84f9fbb10a6739e \ + && git checkout --detach FETCH_HEAD +RUN CGO_ENABLED=0 go build -trimpath \ + -ldflags="-s -w -X github.com/minio/minio/cmd.Version=2025-04-22T22:12:26Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-04-22T22-12-26Z -X github.com/minio/minio/cmd.CommitID=0d7408fc9969caf07de6a8c3a84f9fbb10a6739e" \ + -o /out/minio . + +FROM debian:12.15-slim +RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates curl \ + && rm -rf /var/lib/apt/lists/* \ + && groupadd --system storage && useradd --system --gid storage storage \ + && mkdir /data && chown storage:storage /data +COPY --from=builder /out/minio /usr/local/bin/minio +USER storage:storage +ENTRYPOINT ["minio"] +CMD ["server", "/data", "--console-address", ":9001"] diff --git a/scripts/check_grade3.py b/scripts/check_grade3.py new file mode 100644 index 0000000..857d155 --- /dev/null +++ b/scripts/check_grade3.py @@ -0,0 +1,165 @@ +import argparse +import base64 +import concurrent.futures +import hashlib +import http.client +import json +import os +import struct +import subprocess +import sys +import time +import urllib.parse +import uuid +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, check, run + +COMPOSE = ["docker", "compose", "--project-name", "highload-lab4", "--env-file", ".env", "-f", "compose.yml"] + +def command(args, **kwargs): + return subprocess.check_output(args, cwd=ROOT, text=True, **kwargs).strip() + +def ready(client): + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Exception: + time.sleep(1) + raise AssertionError("readiness did not recover") + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--failures", action="store_true") + args = parser.parse_args() + c = Client(args.base_url) + alice, bob, eve, chat = c.bootstrap() + png = (ROOT / "tests/fixtures/sample.png").read_bytes() + path = "/api/v1/chats/" + chat["id"] + "/attachments" + def upload(data=png, mime="image/png", filename="photo.png", status=201, actor=alice, target=path): + return c.call("POST", target + "?" + urllib.parse.urlencode({"filename": filename}), status, + actor=actor, raw=data, headers={"Content-Type": mime}, schema="Attachment" if status == 201 else None) + jpeg = base64.b64decode('/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAACAAMDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDj6KKKyPJP/9k=') + jpeg_result = upload(data=jpeg,mime="image/jpeg",filename="photo.jpg")["json"] + check(c.call("GET", "/api/v1/attachments/"+jpeg_result["id"]+"/content",actor=bob)["bytes"] == jpeg,"JPEG changed") + first = upload()["json"] + second = upload(filename="Второе фото.png")["json"] + for a in (first, second): + content = c.call("GET", "/api/v1/attachments/" + a["id"] + "/content", actor=bob) + check(content["bytes"] == png, "download bytes changed") + check(a["sha256"] == hashlib.sha256(png).hexdigest(), "checksum differs") + for suffix in ("", "/content"): + c.call("GET", "/api/v1/attachments/" + first["id"] + suffix, 404, actor=eve) + c.call("GET", "/api/v1/attachments/" + first["id"] + suffix, 401) + c.call("GET", "/api/v1/attachments/" + first["id"] + "/content?variant=thumbnail", 404, actor=alice) + upload(actor=eve, status=404) + upload(data=b"x" * (10485760 + 1), status=413) + upload(data=png[:40], status=415) + upload(mime="image/jpeg", status=415) + upload(mime="text/plain", status=415) + for filename in ("", " ", "../file.png", "a\\b.png", "a\x00.png", "a\n.png", "x" * 256): + upload(filename=filename, status=400) + huge = bytearray(png) + huge[16:24] = struct.pack(">II", 10001, 4000) + huge[29:33] = struct.pack(">I", zlib.crc32(huge[12:29]) & 0xffffffff) + upload(data=bytes(huge), status=413) + + # Unknown-length uploads must obey the same byte limit. + endpoint = urllib.parse.urlsplit(args.base_url) + conn = http.client.HTTPConnection(endpoint.hostname, endpoint.port or 80, timeout=35) + conn.request("POST", path + "?filename=chunked.png", body=iter([png, b"x" * 10485760]), + headers={"Authorization": "Bearer " + alice["token"], "Content-Type": "image/png"}, encode_chunked=True) + response = conn.getresponse() + check(response.status == 413, "chunked body limit not enforced") + response.read(); conn.close() + + ids = [first["id"], second["id"]] + key = uuid.uuid4().hex + def send(_): + return c.message(alice, chat, "", key=key, attachments=ids) + with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool: + responses = list(pool.map(send, range(24))) + check(len({r["headers"]["X-Instance-Id"] for r in responses}) >= 2, "did not use two API replicas") + check(all(r["json"] == responses[0]["json"] for r in responses), "replay changed message") + messages_path = "/api/v1/chats/" + chat["id"] + "/messages" + history = c.call("GET", messages_path, actor=bob, schema="MessagePage")["json"]["items"] + check(len(history) == 1 and history[0]["attachment_ids"] == ids, "duplicate or lost attachments") + for body, expected in (({"text": "", "attachment_ids": ids[::-1]}, 409), + ({"text": "changed", "attachment_ids": ids}, 409)): + c.call("POST", messages_path, expected, actor=alice, body=body, headers={"Idempotency-Key": key}) + for bad_ids, expected in (([first["id"], first["id"]], 400), ([str(uuid.uuid4())], 404), (["invalid"], 400), (None, 400), (ids * 6, 400)): + c.call("POST", messages_path, expected, actor=alice, body={"text": "x", "attachment_ids": bad_ids}, headers={"Idempotency-Key": uuid.uuid4().hex}) + other_chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "Other", "member_ids": [alice["id"]]}, schema="Chat")["json"] + c.call("POST", "/api/v1/chats/" + other_chat["id"] + "/messages", 404, actor=alice, + body={"text": "", "attachment_ids": [first["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex}) + text_key = uuid.uuid4().hex + text = c.message(alice, chat, "text", key=text_key)["json"] + check(c.message(alice, chat, "text", key=text_key, attachments=[])["json"] == text, "omitted/empty array differ") + print("PASS: image limits, chunked upload, ACL, message attachments, concurrent replay across replicas") + + if args.failures: + records = [] + for _ in range(2): + result = upload() + target = result["headers"]["X-Instance-Id"] + a = result["json"] + check(not records or target != records[0]["upload_instance"], "repeat did not target the other replica") + try: + command([str(ROOT / "scripts/stop-one.sh")], env={**os.environ, "TARGET_INSTANCE_ID": target}) + time.sleep(3) + downloaded = c.call("GET", "/api/v1/attachments/" + a["id"] + "/content", actor=bob) + check(downloaded["bytes"] == png, "original lost across failover") + served = downloaded["headers"]["X-Instance-Id"] + check(served != target, "download came from stopped instance") + records.append({"upload_instance": target, "download_instance": served, "sha256": a["sha256"]}) + if len(records) == 1: + # Upload the next fixture while only the second replica runs. + # Restore below, then select that same replica via LB uploads. + next_target = served + finally: + command(COMPOSE + ["up", "-d", "--no-build", "--wait"], stderr=subprocess.STDOUT) + ready(c) + if len(records) == 1: + # Round-robin state may change after recovery; keep requesting until it + # chooses the other replica, then use that upload in the next pass. + original_upload = upload + def upload(*a, **kw): + for attempt in range(30): + candidate = original_upload(*a, **kw) + if candidate["headers"]["X-Instance-Id"] == next_target: + return candidate + raise AssertionError("could not reach second replica") + print("PASS: stop the uploading replica, download from the other (both replicas)") + try: + command(COMPOSE + ["stop", "s3"], stderr=subprocess.STDOUT) + # Probe through an existing image on the data network; no API port exposure. + image = command(["docker", "inspect", "--format", "{{.Config.Image}}", command(COMPOSE + ["ps", "--all", "-q", "s3"])]) + probe = ["docker", "run", "--rm", "--network", "highload-lab4_data", "--entrypoint", "curl", image, "-s", "-o", "/dev/null", "-w", "%{http_code}"] + for container in command(COMPOSE + ["ps", "-q", "api"]).splitlines(): + # Container hostname aliases are not DNS names; use the data-network IP. + info = json.loads(command(["docker", "inspect", container]))[0] + ip = info["NetworkSettings"]["Networks"]["highload-lab4_data"]["IPAddress"] + check(command(probe + [f"http://{ip}:8080/health/live"]) == "200", "liveness depends on S3") + check(command(probe + [f"http://{ip}:8080/health/ready"]) == "503", "readiness ignores S3 failure") + records.append({"s3_failure": "live=200 ready=503 on both API replicas"}) + finally: + command(COMPOSE + ["up", "-d", "--no-build", "--wait"], stderr=subprocess.STDOUT) + ready(c) + check(c.call("GET", "/api/v1/attachments/" + first["id"] + "/content", actor=bob)["bytes"] == png, "S3 recovery lost bytes") + image = command(["docker", "inspect", "--format", "{{.Config.Image}}", command(COMPOSE + ["ps", "--all", "-q", "s3"])]) + code = command(["docker", "run", "--rm", "--network", "highload-lab4_data", "--entrypoint", "curl", image, "-s", "-o", "/dev/null", "-w", "%{http_code}", "http://s3:9000/messenger-attachments/originals/" + first["id"]]) + check(code == "403", "anonymous object access was permitted") + records.append({"anonymous_object_get": code}) + directory = ROOT / ".memory/grade3" + directory.mkdir(parents=True, exist_ok=True) + (directory / "failures.json").write_text(json.dumps(records, indent=2) + "\n") + print("PASS: S3 unavailable/recovery, private bucket") + +if __name__ == "__main__": + run(main) diff --git a/scripts/init_env.py b/scripts/init_env.py index ac6ead1..7566029 100644 --- a/scripts/init_env.py +++ b/scripts/init_env.py @@ -47,11 +47,13 @@ def main(): quote(values["POSTGRES_USER"], safe=""), quote(values["POSTGRES_PASSWORD"], safe=""), quote(values["POSTGRES_DB"], safe="")) for name, content in (("DATABASE_URL_FILE", database_url), - ("CACHE_URL_FILE", "redis://redis:6379/0")): + ("CACHE_URL_FILE", "redis://redis:6379/0"), + ("S3_ACCESS_KEY_ID_FILE", secrets.token_hex(12)), + ("S3_SECRET_ACCESS_KEY_FILE", secrets.token_hex(32))): container_path = Path(values[name]) if container_path.parent != Path("/run/secrets") or not container_path.name: raise SystemExit(f"{name} must name a file directly inside /run/secrets") - target = secret_dir / ("database_url" if name == "DATABASE_URL_FILE" else "cache_url") + target = secret_dir / {"DATABASE_URL_FILE":"database_url", "CACHE_URL_FILE":"cache_url", "S3_ACCESS_KEY_ID_FILE":"s3_access_key_id", "S3_SECRET_ACCESS_KEY_FILE":"s3_secret_access_key"}[name] if not target.exists(): # Containers run as an unprivileged user; the private host directory # protects the files, while read permission lets the API consume them. diff --git a/scripts/restart.sh b/scripts/restart.sh index 5145fa5..49581a1 100644 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,7 +1,7 @@ #!/bin/sh set -eu cd "$(dirname "$0")/.." -docker compose --project-name highload-lab4 --env-file .env -f compose.yml restart db redis >/dev/null -docker compose --project-name highload-lab4 --env-file .env -f compose.yml up -d --wait db redis >/dev/null +docker compose --project-name highload-lab4 --env-file .env -f compose.yml restart db redis s3 >/dev/null +docker compose --project-name highload-lab4 --env-file .env -f compose.yml up -d --wait db redis s3 >/dev/null docker compose --project-name highload-lab4 --env-file .env -f compose.yml restart api >/dev/null docker compose --project-name highload-lab4 --env-file .env -f compose.yml up -d --wait >/dev/null diff --git a/src/backend/handlers/attachment_http.go b/src/backend/handlers/attachment_http.go new file mode 100644 index 0000000..d78b6b4 --- /dev/null +++ b/src/backend/handlers/attachment_http.go @@ -0,0 +1,145 @@ +package handlers + +import ( + "context" + "errors" + "github.com/google/uuid" + "io" + "log" + "messenger/src/backend/services" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +func uploadAttachmentHandler(chats *services.ChatService, attachments *services.AttachmentService) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + requestID := newRequestID() + w.Header().Set("X-Request-Id", requestID) + if r.Method != http.MethodPost { + writeError(w, 405, "method_not_allowed", "method not allowed", requestID) + return + } + userID, ok := requestUserID(w, r, requestID) + if !ok { + return + } + raw := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/api/v1/chats/"), "/attachments") + chatID, err := uuid.Parse(raw) + if err != nil { + writeError(w, 400, "invalid_request", "invalid chat id", requestID) + return + } + if _, err = chats.Get(r.Context(), userID, chatID.String()); err != nil { + writeServiceError(w, err, requestID) + return + } + query, err := urlQuery(r) + if err != nil { + writeServiceError(w, err, requestID) + return + } + filenames := query["filename"] + if len(filenames) != 1 { + writeError(w, 400, "invalid_request", "exactly one filename is required", requestID) + return + } + if err = services.ValidateFilename(filenames[0]); err != nil { + writeServiceError(w, err, requestID) + return + } + if r.ContentLength > services.MaxUploadBytes { + writeServiceError(w, services.ErrPayloadTooLarge, requestID) + return + } + r.Body = http.MaxBytesReader(w, r.Body, services.MaxUploadBytes) + ctx, cancel := context.WithTimeout(r.Context(), 25*time.Second) + defer cancel() + attachment, err := attachments.Upload(ctx, userID, chatID.String(), filenames[0], r.Header.Get("Content-Type"), r.Body) + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + err = services.ErrPayloadTooLarge + } + if err != nil { + writeServiceError(w, err, requestID) + return + } + writeJSON(w, 201, attachment) + } +} + +func attachmentHandler(attachments *services.AttachmentService) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + requestID := newRequestID() + w.Header().Set("X-Request-Id", requestID) + if r.Method != http.MethodGet { + writeError(w, 405, "method_not_allowed", "method not allowed", requestID) + return + } + userID, ok := requestUserID(w, r, requestID) + if !ok { + return + } + path := strings.TrimPrefix(r.URL.Path, "/api/v1/attachments/") + parts := strings.Split(path, "/") + if len(parts) > 2 || (len(parts) == 2 && parts[1] != "content") { + writeServiceError(w, services.ErrAttachmentNotFound, requestID) + return + } + id, err := uuid.Parse(parts[0]) + if err != nil { + writeError(w, 400, "invalid_request", "invalid attachment id", requestID) + return + } + a, err := attachments.Get(r.Context(), id.String(), userID) + if err != nil { + writeServiceError(w, err, requestID) + return + } + if len(parts) == 1 { + writeJSON(w, 200, a) + return + } + query, err := urlQuery(r) + if err != nil { + writeServiceError(w, err, requestID) + return + } + if variants, exists := query["variant"]; exists { + if len(variants) != 1 { + writeError(w, 400, "invalid_request", "invalid variant", requestID) + return + } + if variants[0] != "original" { + writeServiceError(w, services.ErrAttachmentNotFound, requestID) + return + } + } + ctx, cancel := context.WithTimeout(r.Context(), 25*time.Second) + defer cancel() + content, err := attachments.Content(ctx, a) + if err != nil { + writeServiceError(w, err, requestID) + return + } + defer content.Close() + w.Header().Set("Content-Type", a.ContentType) + w.Header().Set("Content-Length", strconv.FormatInt(a.SizeBytes, 10)) + w.Header().Set("X-Content-Type-Options", "nosniff") + w.Header().Set("Cache-Control", "private, no-store") + w.WriteHeader(200) + if _, err = io.Copy(w, content); err != nil { + log.Printf("attachment download interrupted, request %s", requestID) + } + } +} + +func urlQuery(r *http.Request) (url.Values, error) { + q, err := url.ParseQuery(r.URL.RawQuery) + if err != nil { + return nil, services.InvalidInput{Message: "invalid query"} + } + return q, nil +} diff --git a/src/backend/handlers/chat_http.go b/src/backend/handlers/chat_http.go index ef5edf8..b55bb99 100644 --- a/src/backend/handlers/chat_http.go +++ b/src/backend/handlers/chat_http.go @@ -110,11 +110,15 @@ func chatCollectionHandler(chats *services.ChatService) http.HandlerFunc { } } -func chatSubresourceHandler(chats *services.ChatService, messages *services.MessageService) http.HandlerFunc { +func chatSubresourceHandler(chats *services.ChatService, messages *services.MessageService, attachments *services.AttachmentService) http.HandlerFunc { get := getChatHandler(chats) createMessage := createMessageHandler(chats, messages) listMessages := listMessagesHandler(chats, messages) return func(w http.ResponseWriter, r *http.Request) { + if attachments != nil && strings.HasSuffix(r.URL.Path, "/attachments") { + uploadAttachmentHandler(chats, attachments)(w, r) + return + } if strings.HasSuffix(r.URL.Path, "/messages") { if r.Method == http.MethodGet { listMessages(w, r) diff --git a/src/backend/handlers/http.go b/src/backend/handlers/http.go index e486e41..1f51b45 100644 --- a/src/backend/handlers/http.go +++ b/src/backend/handlers/http.go @@ -48,6 +48,12 @@ func writeServiceError(w http.ResponseWriter, err error, requestID string) { switch { case errors.As(err, &invalid): writeError(w, 400, "invalid_request", invalid.Message, requestID) + case errors.Is(err, services.ErrPayloadTooLarge): + writeError(w, 413, "payload_too_large", "image exceeds upload limits", requestID) + case errors.Is(err, services.ErrUnsupportedMedia): + writeError(w, 415, "unsupported_media_type", "a valid PNG or JPEG matching Content-Type is required", requestID) + case errors.Is(err, services.ErrAttachmentNotFound): + writeError(w, 404, "not_found", "attachment not found", requestID) case errors.Is(err, services.ErrConflict): writeError(w, 409, "conflict", "request conflicts with an existing resource", requestID) case errors.Is(err, services.ErrUnknownUser): @@ -130,7 +136,11 @@ func readyHandler(health *services.HealthService) http.HandlerFunc { } } -func NewRouter(health *services.HealthService, users *services.UserService, sessions *services.SessionService, chats *services.ChatService, messages *services.MessageService) http.Handler { +func NewRouter(health *services.HealthService, users *services.UserService, sessions *services.SessionService, chats *services.ChatService, messages *services.MessageService, attachmentServices ...*services.AttachmentService) http.Handler { + var attachments *services.AttachmentService + if len(attachmentServices) > 0 { + attachments = attachmentServices[0] + } mux := http.NewServeMux() mux.HandleFunc("/health/live", healthHandler) mux.HandleFunc("/health/ready", readyHandler(health)) @@ -139,6 +149,9 @@ func NewRouter(health *services.HealthService, users *services.UserService, sess mux.Handle("/api/v1/auth/sessions/current", authRequired(sessions, logoutHandler(sessions))) mux.Handle("/api/v1/users/me", authRequired(sessions, currentUserHandler(users))) mux.Handle("/api/v1/chats", authRequired(sessions, chatCollectionHandler(chats))) - mux.Handle("/api/v1/chats/", authRequired(sessions, chatSubresourceHandler(chats, messages))) + mux.Handle("/api/v1/chats/", authRequired(sessions, chatSubresourceHandler(chats, messages, attachments))) + if attachments != nil { + mux.Handle("/api/v1/attachments/", authRequired(sessions, attachmentHandler(attachments))) + } return withInstance(mux) } diff --git a/src/backend/handlers/message_http.go b/src/backend/handlers/message_http.go index a2f6aed..0658b21 100644 --- a/src/backend/handlers/message_http.go +++ b/src/backend/handlers/message_http.go @@ -10,7 +10,7 @@ import ( ) type createMessageRequest struct { - Text string `json:"text"` + Text *string `json:"text"` AttachmentIDs json.RawMessage `json:"attachment_ids"` } @@ -62,10 +62,14 @@ func createMessageHandler(chats *services.ChatService, messages *services.Messag writeError(w, 400, "invalid_request", "invalid JSON body", requestID) return } + ids := []string{} + if request.Text == nil { + writeError(w, 400, "invalid_request", "text must be a string", requestID) + return + } if len(request.AttachmentIDs) > 0 { - var ids []string - if err := json.Unmarshal(request.AttachmentIDs, &ids); err != nil || ids == nil || len(ids) != 0 { - writeError(w, 400, "invalid_request", "attachments are not supported in lab3; use an empty array or omit attachment_ids", requestID) + if err := json.Unmarshal(request.AttachmentIDs, &ids); err != nil || ids == nil { + writeError(w, 400, "invalid_request", "attachment_ids must be an array of UUIDs", requestID) return } } @@ -74,7 +78,7 @@ func createMessageHandler(chats *services.ChatService, messages *services.Messag writeError(w, 400, "invalid_request", "exactly one Idempotency-Key is required", requestID) return } - message, err := messages.CreateIdempotent(r.Context(), userID, chat, request.Text, keys[0]) + message, err := messages.CreateIdempotent(r.Context(), userID, chat, *request.Text, keys[0], ids) if err != nil { writeServiceError(w, err, requestID) return diff --git a/src/backend/main.go b/src/backend/main.go index 6bcaf8b..c089535 100644 --- a/src/backend/main.go +++ b/src/backend/main.go @@ -35,6 +35,19 @@ func main() { if environment := os.Getenv("APP_ENV"); environment != "" { log.SetPrefix("[" + environment + "] ") } + if len(os.Args) == 2 && os.Args[1] == "s3-bootstrap" { + objects, err := repositories.NewObjectStoreFromEnvironment() + if err != nil { + log.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err = objects.Bootstrap(ctx); err != nil { + log.Fatal("S3 bootstrap failed") + } + log.Print("private S3 bucket ready") + return + } if len(os.Args) > 1 { if len(os.Args) == 2 && os.Args[1] == "healthcheck" { client := http.Client{Timeout: 2 * time.Second} @@ -109,14 +122,19 @@ func main() { sessionService := services.NewSessionService(userService, sessionStore, ttl) chatService := services.NewChatService(chatStore) messageService := services.NewMessageService(messageStore) - mux := handlers.NewRouter(services.NewHealthService(repositories.NewHealthStore(db), sessionStore), userService, sessionService, chatService, messageService) + objects, err := repositories.NewObjectStoreFromEnvironment() + if err != nil { + log.Fatal(err) + } + attachmentService := services.NewAttachmentService(repositories.NewAttachmentStore(db), objects) + mux := handlers.NewRouter(services.NewHealthService(repositories.NewHealthStore(db), sessionStore, objects), userService, sessionService, chatService, messageService, attachmentService) server := &http.Server{ Addr: serverAddress(), Handler: mux, ReadHeaderTimeout: 5 * time.Second, - ReadTimeout: 10 * time.Second, - WriteTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, IdleTimeout: 60 * time.Second, } diff --git a/src/backend/repositories/attachment.go b/src/backend/repositories/attachment.go new file mode 100644 index 0000000..f1d829c --- /dev/null +++ b/src/backend/repositories/attachment.go @@ -0,0 +1,42 @@ +package repositories + +import ( + "context" + "github.com/jackc/pgx/v5/pgxpool" + "time" +) + +type Attachment struct { + ID string `json:"id"` + ChatID string `json:"chat_id"` + OwnerID string `json:"owner_id"` + Filename string `json:"filename"` + ContentType string `json:"content_type"` + SizeBytes int64 `json:"size_bytes"` + SHA256 string `json:"sha256"` + Status string `json:"status"` + Variants []any `json:"variants"` + ErrorCode *string `json:"error_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ObjectKey string `json:"-"` +} + +type AttachmentStore struct{ db *pgxpool.Pool } + +func NewAttachmentStore(db *pgxpool.Pool) *AttachmentStore { return &AttachmentStore{db: db} } +func (s *AttachmentStore) Create(ctx context.Context, a Attachment) error { + ctx, cancel := boundedQuery(ctx) + defer cancel() + _, err := s.db.Exec(ctx, `INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256,created_at,updated_at) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`, a.ID, a.ChatID, a.OwnerID, a.Filename, a.ObjectKey, a.ContentType, a.SizeBytes, a.SHA256, a.CreatedAt, a.UpdatedAt) + return err +} +func (s *AttachmentStore) GetForMember(ctx context.Context, id, userID string) (Attachment, error) { + ctx, cancel := boundedQuery(ctx) + defer cancel() + a := Attachment{Status: "ready", Variants: []any{}} + err := s.db.QueryRow(ctx, `SELECT a.id,a.chat_id,a.owner_id,a.filename,a.object_key,a.content_type,a.size_bytes,a.sha256,a.created_at,a.updated_at FROM attachments a JOIN chat_members cm ON cm.chat_id=a.chat_id WHERE a.id=$1 AND cm.user_id=$2`, id, userID).Scan(&a.ID, &a.ChatID, &a.OwnerID, &a.Filename, &a.ObjectKey, &a.ContentType, &a.SizeBytes, &a.SHA256, &a.CreatedAt, &a.UpdatedAt) + a.CreatedAt = a.CreatedAt.UTC() + a.UpdatedAt = a.UpdatedAt.UTC() + return a, err +} diff --git a/src/backend/repositories/idempotency.go b/src/backend/repositories/idempotency.go index b99064b..86cb2b4 100644 --- a/src/backend/repositories/idempotency.go +++ b/src/backend/repositories/idempotency.go @@ -3,16 +3,23 @@ package repositories import ( "context" "errors" + "reflect" "time" "github.com/google/uuid" "github.com/jackc/pgx/v5" ) +var ErrAttachmentNotFound = errors.New("attachment not found") + var ErrIdempotencyConflict = errors.New("idempotency key belongs to a different message") // CreateIdempotent commits the message and its replay record together. -func (s *MessageStore) CreateIdempotent(ctx context.Context, chatID, senderID, text, key string) (Message, error) { +func (s *MessageStore) CreateIdempotent(ctx context.Context, chatID, senderID, text, key string, attachments ...[]string) (Message, error) { + ids := []string{} + if len(attachments) > 0 && attachments[0] != nil { + ids = attachments[0] + } ctx, cancel := boundedQuery(ctx) defer cancel() tx, err := s.db.Begin(ctx) @@ -27,16 +34,26 @@ func (s *MessageStore) CreateIdempotent(ctx context.Context, chatID, senderID, t if _, err = tx.Exec(ctx, "SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", scope); err != nil { return Message{}, err } + // Validate attachments in this same transaction, including replay. + for _, id := range ids { + var exists bool + if err = tx.QueryRow(ctx, "SELECT EXISTS(SELECT 1 FROM attachments WHERE id=$1 AND chat_id=$2)", id, chatID).Scan(&exists); err != nil { + return Message{}, err + } + if !exists { + return Message{}, ErrAttachmentNotFound + } + } var message Message err = tx.QueryRow(ctx, ` - SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, ARRAY(SELECT attachment_id::text FROM message_attachments ma WHERE ma.message_id=m.id ORDER BY position) FROM message_idempotency i JOIN messages m ON m.id = i.message_id WHERE i.sender_id=$1 AND i.chat_id=$2 AND i.key=$3 AND i.expires_at > clock_timestamp()`, senderID, chatID, key). - Scan(&message.ID, &message.ChatID, &message.SenderID, &message.Text, &message.CreatedAt) + Scan(&message.ID, &message.ChatID, &message.SenderID, &message.Text, &message.CreatedAt, &message.AttachmentIDs) if err == nil { message.CreatedAt = message.CreatedAt.UTC() - if message.Text != text { + if message.Text != text || !reflect.DeepEqual(message.AttachmentIDs, ids) { return Message{}, ErrIdempotencyConflict } return message, tx.Commit(ctx) @@ -50,12 +67,17 @@ func (s *MessageStore) CreateIdempotent(ctx context.Context, chatID, senderID, t return Message{}, err } message = Message{ - ID: uuid.NewString(), ChatID: chatID, SenderID: senderID, Text: text, + AttachmentIDs: ids, ID: uuid.NewString(), ChatID: chatID, SenderID: senderID, Text: text, CreatedAt: time.Now().UTC().Truncate(time.Microsecond), } if _, err = tx.Exec(ctx, "INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)", message.ID, chatID, senderID, text, message.CreatedAt); err != nil { return Message{}, err } + for position, id := range ids { + if _, err = tx.Exec(ctx, "INSERT INTO message_attachments(message_id,attachment_id,position) VALUES($1,$2,$3)", message.ID, id, position); err != nil { + return Message{}, err + } + } if _, err = tx.Exec(ctx, `INSERT INTO message_idempotency(sender_id,chat_id,key,message_id,expires_at) VALUES($1,$2,$3,$4,clock_timestamp() + $5 * interval '1 second')`, senderID, chatID, key, message.ID, s.idempotencyTTL.Seconds()); err != nil { return Message{}, err diff --git a/src/backend/repositories/message.go b/src/backend/repositories/message.go index 8552dc4..3edafc2 100644 --- a/src/backend/repositories/message.go +++ b/src/backend/repositories/message.go @@ -9,11 +9,12 @@ import ( ) type Message struct { - ID string `json:"id"` - ChatID string `json:"chat_id"` - SenderID string `json:"sender_id"` - Text string `json:"text"` - CreatedAt time.Time `json:"created_at"` + ID string `json:"id"` + ChatID string `json:"chat_id"` + SenderID string `json:"sender_id"` + Text string `json:"text"` + CreatedAt time.Time `json:"created_at"` + AttachmentIDs []string `json:"attachment_ids"` } type MessageStore struct { @@ -38,11 +39,12 @@ func (s *MessageStore) Create( ctx, cancel := boundedQuery(ctx) defer cancel() message := Message{ - ID: uuid.New().String(), - ChatID: chatID, - SenderID: senderID, - Text: text, - CreatedAt: time.Now().UTC().Truncate(time.Microsecond), + AttachmentIDs: []string{}, + ID: uuid.New().String(), + ChatID: chatID, + SenderID: senderID, + Text: text, + CreatedAt: time.Now().UTC().Truncate(time.Microsecond), } _, err := s.db.Exec(ctx, "INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES($1,$2,$3,$4,$5)", message.ID, message.ChatID, message.SenderID, message.Text, message.CreatedAt) @@ -63,7 +65,7 @@ func (s *MessageStore) ListPageByChat( afterTime = cursor.CreatedAt afterID = cursor.MessageID } - rows, err := s.db.Query(ctx, `SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=$1 AND (created_at,id)>($2,$3::uuid) ORDER BY created_at,id LIMIT $4`, chatID, afterTime, afterID, limit+1) + rows, err := s.db.Query(ctx, `SELECT id,chat_id,sender_id,text,created_at,ARRAY(SELECT attachment_id::text FROM message_attachments ma WHERE ma.message_id=messages.id ORDER BY position) FROM messages WHERE chat_id=$1 AND (created_at,id)>($2,$3::uuid) ORDER BY created_at,id LIMIT $4`, chatID, afterTime, afterID, limit+1) if err != nil { return nil, false, err } @@ -71,7 +73,7 @@ func (s *MessageStore) ListPageByChat( items := []Message{} for rows.Next() { var m Message - if err = rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt); err != nil { + if err = rows.Scan(&m.ID, &m.ChatID, &m.SenderID, &m.Text, &m.CreatedAt, &m.AttachmentIDs); err != nil { return nil, false, err } items = append(items, m) diff --git a/src/backend/repositories/migrations/000004_attachments.down.sql b/src/backend/repositories/migrations/000004_attachments.down.sql new file mode 100644 index 0000000..e6d4d2a --- /dev/null +++ b/src/backend/repositories/migrations/000004_attachments.down.sql @@ -0,0 +1,5 @@ +-- An operator must first remove attachment-only messages before this rollback. +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) BETWEEN 1 AND 4000); +DROP TABLE message_attachments; +DROP TABLE attachments; diff --git a/src/backend/repositories/migrations/000004_attachments.up.sql b/src/backend/repositories/migrations/000004_attachments.up.sql new file mode 100644 index 0000000..fcaac35 --- /dev/null +++ b/src/backend/repositories/migrations/000004_attachments.up.sql @@ -0,0 +1,22 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename text NOT NULL, + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL +); +CREATE INDEX attachments_chat_idx ON attachments(chat_id); +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments(id), + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE(message_id, position) +); +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000); diff --git a/src/backend/repositories/s3.go b/src/backend/repositories/s3.go new file mode 100644 index 0000000..5b66e7e --- /dev/null +++ b/src/backend/repositories/s3.go @@ -0,0 +1,99 @@ +package repositories + +import ( + "context" + "errors" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +type ObjectStore struct { + client *minio.Client + bucket string +} + +func NewObjectStoreFromEnvironment() (*ObjectStore, error) { + endpoint, err := url.Parse(os.Getenv("S3_ENDPOINT")) + if err != nil || endpoint.Host == "" || (endpoint.Scheme != "http" && endpoint.Scheme != "https") || endpoint.Path != "" { + return nil, errors.New("invalid S3_ENDPOINT") + } + read := func(name string) (string, error) { + b, e := os.ReadFile(os.Getenv(name)) + if e != nil { + return "", errors.New("cannot read " + name) + } + v := strings.TrimSpace(string(b)) + if v == "" { + return "", errors.New("empty " + name) + } + return v, nil + } + access, err := read("S3_ACCESS_KEY_ID_FILE") + if err != nil { + return nil, err + } + secret, err := read("S3_SECRET_ACCESS_KEY_FILE") + if err != nil { + return nil, err + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.ResponseHeaderTimeout = 5 * time.Second + transport.MaxIdleConnsPerHost = 8 + client, err := minio.New(endpoint.Host, &minio.Options{Creds: credentials.NewStaticV4(access, secret, ""), Secure: endpoint.Scheme == "https", Region: os.Getenv("S3_REGION"), Transport: transport}) + if err != nil { + return nil, err + } + bucket := os.Getenv("S3_BUCKET") + if bucket == "" { + return nil, errors.New("S3_BUCKET is required") + } + return &ObjectStore{client: client, bucket: bucket}, nil +} +func (s *ObjectStore) Ping(ctx context.Context) error { + exists, err := s.client.BucketExists(ctx, s.bucket) + if err != nil { + return err + } + if !exists { + return errors.New("bucket missing") + } + return nil +} +func (s *ObjectStore) Bootstrap(ctx context.Context) error { + exists, err := s.client.BucketExists(ctx, s.bucket) + if err != nil { + return err + } + if !exists { + if err = s.client.MakeBucket(ctx, s.bucket, minio.MakeBucketOptions{Region: os.Getenv("S3_REGION")}); err != nil { + return err + } + } + // No anonymous policy: every object operation requires signed credentials. + return s.client.SetBucketPolicy(ctx, s.bucket, "") +} +func (s *ObjectStore) Put(ctx context.Context, key string, body io.Reader, size int64, contentType string) error { + _, err := s.client.PutObject(ctx, s.bucket, key, body, size, minio.PutObjectOptions{ContentType: contentType, DisableMultipart: true}) + return err +} +func (s *ObjectStore) Delete(ctx context.Context, key string) error { + return s.client.RemoveObject(ctx, s.bucket, key, minio.RemoveObjectOptions{}) +} +func (s *ObjectStore) Get(ctx context.Context, key string) (io.ReadCloser, error) { + object, err := s.client.GetObject(ctx, s.bucket, key, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + // GetObject is lazy: force the request before writing HTTP 200 headers. + if _, err = object.Stat(); err != nil { + object.Close() + return nil, err + } + return object, nil +} diff --git a/src/backend/services/attachment.go b/src/backend/services/attachment.go new file mode 100644 index 0000000..1623f85 --- /dev/null +++ b/src/backend/services/attachment.go @@ -0,0 +1,148 @@ +package services + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "image" + _ "image/jpeg" + _ "image/png" + "io" + "log" + "messenger/src/backend/repositories" + "mime" + "strings" + "time" + "unicode" + "unicode/utf8" +) + +const MaxUploadBytes int64 = 10485760 +const MaxImagePixels int64 = 40000000 + +var ErrPayloadTooLarge = errors.New("payload too large") +var ErrUnsupportedMedia = errors.New("unsupported image") +var ErrAttachmentNotFound = errors.New("attachment not found") + +type AttachmentService struct { + metadata *repositories.AttachmentStore + objects *repositories.ObjectStore + uploadSlot chan struct{} +} + +func NewAttachmentService(metadata *repositories.AttachmentStore, objects *repositories.ObjectStore) *AttachmentService { + return &AttachmentService{metadata: metadata, objects: objects, uploadSlot: make(chan struct{}, 1)} +} +func ValidateFilename(name string) error { + if !utf8.ValidString(name) || strings.TrimSpace(name) == "" || utf8.RuneCountInString(name) > 255 { + return InvalidInput{"filename must contain 1-255 characters"} + } + for _, r := range name { + if r == '/' || r == '\\' || unicode.IsControl(r) { + return InvalidInput{"invalid filename"} + } + } + return nil +} +func ValidateImage(data []byte, declared string) (string, error) { + contentType, _, err := mime.ParseMediaType(declared) + if err != nil || (contentType != "image/png" && contentType != "image/jpeg") { + return "", ErrUnsupportedMedia + } + config, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil { + return "", ErrUnsupportedMedia + } + actual := "image/" + format + if actual != contentType { + return "", ErrUnsupportedMedia + } + if config.Width <= 0 || config.Height <= 0 || int64(config.Width) > MaxImagePixels/int64(config.Height) { + return "", ErrPayloadTooLarge + } + // DecodeConfig alone accepts truncated headers; decode only after bounding pixels. + decoded, _, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return "", ErrUnsupportedMedia + } + _ = decoded + return actual, nil +} +func (s *AttachmentService) Upload(ctx context.Context, userID, chatID, filename, contentType string, body io.Reader) (repositories.Attachment, error) { + select { + case s.uploadSlot <- struct{}{}: + defer func() { <-s.uploadSlot }() + case <-ctx.Done(): + return repositories.Attachment{}, ctx.Err() + } + data, err := io.ReadAll(io.LimitReader(body, MaxUploadBytes+1)) + if err != nil { + return repositories.Attachment{}, err + } + if int64(len(data)) > MaxUploadBytes { + return repositories.Attachment{}, ErrPayloadTooLarge + } + actual, err := ValidateImage(data, contentType) + if err != nil { + return repositories.Attachment{}, err + } + sum := sha256.Sum256(data) + now := time.Now().UTC().Truncate(time.Microsecond) + a := repositories.Attachment{ID: uuid.NewString(), ChatID: chatID, OwnerID: userID, Filename: filename, ContentType: actual, SizeBytes: int64(len(data)), SHA256: hex.EncodeToString(sum[:]), Status: "ready", Variants: []any{}, CreatedAt: now, UpdatedAt: now} + a.ObjectKey = "originals/" + a.ID + if err = s.objects.Put(ctx, a.ObjectKey, bytes.NewReader(data), a.SizeBytes, a.ContentType); err != nil { + return repositories.Attachment{}, err + } + if err = s.metadata.Create(ctx, a); err != nil { + // A connection error can hide a successful autocommit. Delete only after + // a definite SQL rejection; otherwise preserve bytes for later reconciliation. + var rejected *pgconn.PgError + if errors.As(err, &rejected) { + cleanup, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if cleanupErr := s.objects.Delete(cleanup, a.ObjectKey); cleanupErr != nil { + log.Printf("attachment compensation failed for %s", a.ID) + } + } else { + log.Printf("attachment metadata outcome uncertain for %s; preserving S3 object", a.ID) + } + return repositories.Attachment{}, err + } + return a, nil +} +func (s *AttachmentService) Get(ctx context.Context, id, userID string) (repositories.Attachment, error) { + a, err := s.metadata.GetForMember(ctx, id, userID) + if errors.Is(err, pgx.ErrNoRows) { + return a, ErrAttachmentNotFound + } + return a, err +} +func (s *AttachmentService) Content(ctx context.Context, a repositories.Attachment) (io.ReadCloser, error) { + return s.objects.Get(ctx, a.ObjectKey) +} + +func NormalizeAttachmentIDs(ids []string) ([]string, error) { + if len(ids) > 10 { + return nil, InvalidInput{"at most 10 attachments are allowed"} + } + result := make([]string, 0, len(ids)) + seen := map[string]bool{} + for _, id := range ids { + parsed, err := uuid.Parse(id) + if err != nil { + return nil, InvalidInput{"invalid attachment id"} + } + id = strings.ToLower(parsed.String()) + if seen[id] { + return nil, InvalidInput{"duplicate attachment id"} + } + seen[id] = true + result = append(result, id) + } + return result, nil +} diff --git a/src/backend/services/health.go b/src/backend/services/health.go index b7c311b..462e750 100644 --- a/src/backend/services/health.go +++ b/src/backend/services/health.go @@ -9,15 +9,26 @@ import ( type HealthService struct { health *repositories.HealthStore sessions *repositories.SessionStore + objects interface{ Ping(context.Context) error } } -func NewHealthService(health *repositories.HealthStore, sessions *repositories.SessionStore) *HealthService { - return &HealthService{health: health, sessions: sessions} +func NewHealthService(health *repositories.HealthStore, sessions *repositories.SessionStore, objects ...interface{ Ping(context.Context) error }) *HealthService { + service := &HealthService{health: health, sessions: sessions} + if len(objects) > 0 { + service.objects = objects[0] + } + return service } func (s *HealthService) Ready(ctx context.Context) error { if err := s.health.Ping(ctx); err != nil { return err } - return s.sessions.Ping(ctx) + if err := s.sessions.Ping(ctx); err != nil { + return err + } + if s.objects != nil { + return s.objects.Ping(ctx) + } + return nil } diff --git a/src/backend/services/idempotency.go b/src/backend/services/idempotency.go index 93d86e8..0bc2ba6 100644 --- a/src/backend/services/idempotency.go +++ b/src/backend/services/idempotency.go @@ -12,17 +12,28 @@ import ( var idempotencyKeyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`) -func (s *MessageService) CreateIdempotent(ctx context.Context, userID string, chat repositories.Chat, text, key string) (repositories.Message, error) { +func (s *MessageService) CreateIdempotent(ctx context.Context, userID string, chat repositories.Chat, text, key string, attachments ...[]string) (repositories.Message, error) { if !ChatHasMember(chat, userID) { return repositories.Message{}, ErrChatNotFound } if !idempotencyKeyPattern.MatchString(key) { return repositories.Message{}, InvalidInput{"Idempotency-Key must contain 1-128 ASCII letters, digits or ._:-"} } - if strings.TrimSpace(text) == "" || utf8.RuneCountInString(text) > 4000 { - return repositories.Message{}, InvalidInput{"text must contain 1-4000 characters and must not be blank"} + var ids []string + if len(attachments) > 0 { + ids = attachments[0] + } + ids, err := NormalizeAttachmentIDs(ids) + if err != nil { + return repositories.Message{}, err + } + if (strings.TrimSpace(text) == "" && len(ids) == 0) || utf8.RuneCountInString(text) > 4000 { + return repositories.Message{}, InvalidInput{"text must be at most 4000 characters; nonblank text or attachments required"} + } + message, err := s.messages.CreateIdempotent(ctx, chat.ID, userID, text, key, ids) + if errors.Is(err, repositories.ErrAttachmentNotFound) { + return repositories.Message{}, ErrAttachmentNotFound } - message, err := s.messages.CreateIdempotent(ctx, chat.ID, userID, text, key) if errors.Is(err, repositories.ErrIdempotencyConflict) { return repositories.Message{}, ErrConflict } diff --git a/src/backend/tests/attachment_validation_test.go b/src/backend/tests/attachment_validation_test.go new file mode 100644 index 0000000..34698c0 --- /dev/null +++ b/src/backend/tests/attachment_validation_test.go @@ -0,0 +1,94 @@ +package tests + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "github.com/google/uuid" + "hash/crc32" + "image" + "image/jpeg" + "image/png" + "messenger/src/backend/repositories" + "messenger/src/backend/services" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestAttachmentImageValidation(t *testing.T) { + img := image.NewRGBA(image.Rect(0, 0, 3, 2)) + var pngData, jpegData bytes.Buffer + if err := png.Encode(&pngData, img); err != nil { + t.Fatal(err) + } + if err := jpeg.Encode(&jpegData, img, nil); err != nil { + t.Fatal(err) + } + for _, sample := range []struct { + data []byte + mime string + }{{pngData.Bytes(), "image/png"}, {jpegData.Bytes(), "image/jpeg"}} { + if actual, err := services.ValidateImage(sample.data, sample.mime); err != nil || actual != sample.mime { + t.Fatalf("valid image: %s %v", actual, err) + } + } + huge := append([]byte(nil), pngData.Bytes()...) + binary.BigEndian.PutUint32(huge[16:20], 10001) + binary.BigEndian.PutUint32(huge[20:24], 4000) + binary.BigEndian.PutUint32(huge[29:33], crc32.ChecksumIEEE(huge[12:29])) + if _, err := services.ValidateImage(huge, "image/png"); !errors.Is(err, services.ErrPayloadTooLarge) { + t.Fatalf("pixel limit: %v", err) + } + for _, sample := range []struct { + data []byte + mime string + }{{pngData.Bytes(), "image/jpeg"}, {jpegData.Bytes(), "image/png"}, {pngData.Bytes()[:40], "image/png"}, {[]byte("not an image"), "image/png"}, {pngData.Bytes(), "text/plain"}} { + if _, err := services.ValidateImage(sample.data, sample.mime); !errors.Is(err, services.ErrUnsupportedMedia) { + t.Fatalf("invalid image accepted: %v", err) + } + } +} + +func TestAttachmentFilenameValidation(t *testing.T) { + for _, name := range []string{"", " ", "../photo.png", `a\b.png`, "photo\x00.png", "photo\n.png"} { + if services.ValidateFilename(name) == nil { + t.Fatalf("accepted %q", name) + } + } + if err := services.ValidateFilename("Фотография 1.png"); err != nil { + t.Fatal(err) + } +} + +func TestAttachmentMessageRequiresStringText(t *testing.T) { + f := newTestFixture(t) + user, _ := createTestUser(t, f.users, "alice", "Alice") + chat := createTestChat(t, f.chats, "Images", []string{user.ID}) + id := uuid.NewString() + // Repository-level fixture: message validation does not fetch S3 bytes. + err := repositories.NewAttachmentStore(f.db).Create(context.Background(), repositories.Attachment{ + ID: id, ChatID: chat.ID, OwnerID: user.ID, Filename: "photo.png", ObjectKey: "test/" + id, + ContentType: "image/png", SizeBytes: 1, SHA256: strings.Repeat("a", 64), CreatedAt: time.Now().UTC(), UpdatedAt: time.Now().UTC(), + }) + if err != nil { + t.Fatal(err) + } + for _, body := range []string{`{"attachment_ids":["` + id + `"]}`, `{"text":null,"attachment_ids":["` + id + `"]}`} { + r := httptest.NewRequest(http.MethodPost, "/api/v1/chats/"+chat.ID+"/messages", strings.NewReader(body)) + r.Header.Set("Content-Type", "application/json") + r.Header.Set("Idempotency-Key", uuid.NewString()) + authorizeTestRequest(t, f, r, user.ID) + w := httptest.NewRecorder() + f.router.ServeHTTP(w, r) + if w.Code != 400 { + t.Fatalf("expected 400, got %d: %s", w.Code, w.Body.String()) + } + } + if len(listTestMessages(t, f.messages, chat.ID)) != 0 { + t.Fatal("invalid request saved a message") + } +} diff --git a/src/backend/tests/idempotency_test.go b/src/backend/tests/idempotency_test.go index 5811ee0..27c32f0 100644 --- a/src/backend/tests/idempotency_test.go +++ b/src/backend/tests/idempotency_test.go @@ -3,6 +3,7 @@ package tests import ( "context" "errors" + "reflect" "sync" "testing" @@ -44,7 +45,7 @@ func TestIdempotencyConcurrentStoresAndScopes(t *testing.T) { for message := range results { if original.ID == "" { original = message - } else if message != original { + } else if !reflect.DeepEqual(message, original) { t.Fatalf("replay changed the message: %v != %v", message, original) } } diff --git a/src/backend/tests/test_fixture_test.go b/src/backend/tests/test_fixture_test.go index ade688a..1dca000 100644 --- a/src/backend/tests/test_fixture_test.go +++ b/src/backend/tests/test_fixture_test.go @@ -85,6 +85,13 @@ func newTestFixture(t *testing.T) *testFixture { if _, err = db.Exec(ctx, string(ddl)); err != nil { t.Fatal(err) } + ddl, err = os.ReadFile("../repositories/migrations/000004_attachments.up.sql") + if err != nil { + t.Fatal(err) + } + if _, err = db.Exec(ctx, string(ddl)); err != nil { + t.Fatal(err) + } t.Cleanup(func() { _ = redisClient.Close() }) if err := redisClient.Ping(ctx).Err(); err != nil { t.Fatal(err) diff --git a/src/frontend/src/App.tsx b/src/frontend/src/App.tsx index cb8d933..691beff 100644 --- a/src/frontend/src/App.tsx +++ b/src/frontend/src/App.tsx @@ -1,6 +1,7 @@ import { useCallback, useEffect, useRef, useState, type FormEvent } from "react"; -import { api, ApiError, type Chat, type Session, type User } from "./api"; +import { api, ApiError, type Attachment, type Chat, type Session, type User } from "./api"; +import AttachmentImage from "./AttachmentImage"; import Pagination from "./Pagination"; import { useCursorPage } from "./useCursorPage"; @@ -111,6 +112,8 @@ function AuthForm({ onSuccess }: { onSuccess: (session: Session) => void }) { ); } +type UploadDraft = { localId: string; file: File; status: "uploading" | "ready" | "error"; attachment?: Attachment; error?: string }; + function Workspace({ token, user, onLogout }: { token: string; user: User; onLogout: (notice?: string) => void }) { const [selectedChat, setSelectedChat] = useState(null); const [showCreate, setShowCreate] = useState(false); @@ -119,7 +122,9 @@ function Workspace({ token, user, onLogout }: { token: string; user: User; onLog const [drafts, setDrafts] = useState>({}); const [creating, setCreating] = useState(false); const [sending, setSending] = useState(false); - const pendingMessages = useRef>({}); + const pendingMessages = useRef>({}); + const [uploads, setUploads] = useState>({}); + const [uploading, setUploading] = useState(false); const [error, setError] = useState(""); const [notice, setNotice] = useState(""); const reportError = useCallback((cause: unknown) => { @@ -144,19 +149,55 @@ function Workspace({ token, user, onLogout }: { token: string; user: User; onLog finally { setCreating(false); } } + async function uploadFiles(targetChat: string, items: UploadDraft[]) { + setUploading(true); setError(""); + const update = (id: string, patch: Partial) => setUploads(current => ({ ...current, + [targetChat]: (current[targetChat] || []).map(item => item.localId === id ? { ...item, ...patch } : item) })); + try { + for (const item of items) { + update(item.localId, { status: "uploading", error: undefined }); + try { + const attachment = await api.uploadAttachment(token, targetChat, item.file); + update(item.localId, { status: "ready", attachment }); + } catch (cause) { + update(item.localId, { status: "error", error: errorText(cause) }); + reportError(cause); + if (cause instanceof ApiError && cause.status === 401) break; + } + } + } finally { setUploading(false); } + } + + function selectFiles(files: FileList | null) { + if (!files || uploading || sending) return; + const current = uploads[chatId] || []; + const chosen = Array.from(files); + if (current.length + chosen.length > 10) { setError("Можно прикрепить не более 10 изображений."); return; } + if (chosen.some(file => file.size > 10485760 || !["image/png", "image/jpeg"].includes(file.type))) { + setError("Выберите PNG или JPEG размером до 10 МиБ каждый."); return; + } + const items: UploadDraft[] = chosen.map(file => ({ localId: crypto.randomUUID(), file, status: "uploading" })); + setUploads(previous => ({ ...previous, [chatId]: [...current, ...items] })); + void uploadFiles(chatId, items); + } + async function sendMessage(event: FormEvent) { event.preventDefault(); const text = (drafts[chatId] || "").trim(); - if (!text) return; + const selected = uploads[chatId] || []; + if (uploading || selected.some(item => item.status !== "ready") || (!text && selected.length === 0)) return; + const attachmentIds = selected.map(item => item.attachment!.id); + const body = JSON.stringify({ text, attachment_ids: attachmentIds }); const previous = pendingMessages.current[chatId]; - const pending = previous?.text === text ? previous : { text, key: crypto.randomUUID() }; + const pending = previous?.body === body ? previous : { body, key: crypto.randomUUID() }; // A manual retry after an uncertain response keeps the original key/body. pendingMessages.current[chatId] = pending; setSending(true); setError(""); try { - await api.sendMessage(token, chatId, text, pending.key); + await api.sendMessage(token, chatId, text, pending.key, attachmentIds); delete pendingMessages.current[chatId]; setDrafts((previous) => ({ ...previous, [chatId]: "" })); + setUploads(previous => ({ ...previous, [chatId]: [] })); messages.reload(); setNotice("Сообщение отправлено"); } catch (cause) { reportError(cause); } finally { setSending(false); } @@ -189,7 +230,7 @@ function Workspace({ token, user, onLogout }: { token: string; user: User; onLog
{chats.items.length} на странице
{chats.busy ?

Загрузка…

: chats.failed ?

Не удалось загрузить чаты

: !chats.items.length &&

Нет чатов

} - {chats.items.map((chat) => )} + {chats.items.map((chat) => )}
Мой ID{user.id}
@@ -199,10 +240,24 @@ function Workspace({ token, user, onLogout }: { token: string; user: User; onLog

{selectedChat.title}

{selectedChat.member_ids.length} участн.{selectedChat.member_ids.map((id) => {id}{id === user.id ? " · вы" : ""})}
{messages.busy ?

Загрузка…

: messages.failed ?

Не удалось загрузить сообщения

: !messages.items.length &&

Нет сообщений

} - {messages.items.map((message) =>
{message.sender_id === user.id ? "Вы" : message.sender_id.slice(0, 8)}

{message.text}

)} + {messages.items.map((message) =>
{message.sender_id === user.id ? "Вы" : message.sender_id.slice(0, 8)}
{message.text &&

{message.text}

}{message.attachment_ids.map(id => )}
)}
-