From 335df4b2b29fdb82e0b6714eb6fcb04eba145123 Mon Sep 17 00:00:00 2001 From: Alexey Bodanyuk <144894236+wfa22@users.noreply.github.com> Date: Sun, 25 Oct 2026 13:26:01 +0300 Subject: [PATCH] =?UTF-8?q?lab4:=20=D0=B2=D0=BB=D0=BE=D0=B6=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D1=8F=20=D0=B2=20=D0=BF=D1=80=D0=B8=D0=B2=D0=B0=D1=82?= =?UTF-8?q?=D0=BD=D0=BE=D0=BC=20S3=20(SeaweedFS),=20=D0=BC=D0=B5=D1=82?= =?UTF-8?q?=D0=B0=D0=B4=D0=B0=D0=BD=D0=BD=D1=8B=D0=B5=20=D0=B2=20PostgreSQ?= =?UTF-8?q?L?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - compose: сервис s3 (SeaweedFS 4.46 по digest, volume s3data, внутренняя сеть storage, web-интерфейсы только на 127.0.0.1 внутри контейнера), одноразовый s3-init; - три пары ключей S3: admin (только bootstrap), api (Read+Write своего bucket), reconciler (+List); bootstrap идемпотентен и проверяет приватность и права; - миграция 0004: attachments (pending -> ready, случайный object_key), message_attachments, post_message_v2 с проверкой вложений в той же транзакции; - POST /chats/{id}/attachments: лимиты до чтения/декодирования, потоковая проверка PNG (CRC + zlib до точного размера), JPEG через djpeg -strict, семафор загрузок; - GET метаданных и /content (стриминг из S3, без redirect), readiness с S3, HAProxy: деградированный режим без S3 через /health/ready/core; - janitor незавершённых загрузок с grace period. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 11 +- compose.yaml | 176 ++++++- infra/haproxy/haproxy.cfg | 37 +- infra/seaweedfs/start.sh | 40 ++ migrations/0004_attachments.sql | 139 +++++ requirements-test.txt | 2 + requirements.in | 4 + requirements.txt | 906 +++++++++++++++++++++++++++++++- scripts/init_secrets.py | 7 +- scripts/restart.sh | 9 +- src/messenger/app.py | 172 +++++- src/messenger/attachments.py | 285 ++++++++++ src/messenger/config.py | 64 +++ src/messenger/errors.py | 6 + src/messenger/images.py | 317 +++++++++++ src/messenger/repo.py | 96 +++- src/messenger/s3_bootstrap.py | 151 ++++++ src/messenger/storage.py | 173 ++++++ src/messenger/validation.py | 60 ++- 19 files changed, 2586 insertions(+), 69 deletions(-) create mode 100755 infra/seaweedfs/start.sh create mode 100644 migrations/0004_attachments.sql create mode 100644 src/messenger/attachments.py create mode 100644 src/messenger/images.py create mode 100644 src/messenger/s3_bootstrap.py create mode 100644 src/messenger/storage.py diff --git a/Dockerfile b/Dockerfile index f46ac0b..9da4841 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ # syntax=docker/dockerfile:1 -# Messenger API, lab 3 (replicas behind HAProxy, PostgreSQL + Valkey). Built by compose.yaml (services api, migrate). +# Messenger API, lab 4 (replicas behind HAProxy, PostgreSQL + Valkey + S3). Built by compose.yaml +# (services api, migrate, s3-init, reconciler: one image, different commands). # # Python is interpreted, so there is no separate compile stage; the image # contains only the interpreter, locked dependencies and the source code. @@ -19,7 +20,13 @@ WORKDIR /app COPY requirements.txt ./ RUN pip install --root-user-action=ignore --require-hashes --no-deps -r requirements.txt -# 2. Unprivileged runtime user. Code stays owned by root, so the process +# 2. JPEG integrity check: libjpeg-turbo's djpeg (strict mode, see src/messenger/images.py). +# Version pinned to the Debian 12 package; apt lists are removed in the same layer. +RUN apt-get update \ + && apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 \ + && rm -rf /var/lib/apt/lists/* + +# 3. Unprivileged runtime user. Code stays owned by root, so the process # cannot modify its own code. RUN groupadd --system --gid 10001 app \ && useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app diff --git a/compose.yaml b/compose.yaml index b8cc3a8..580fd27 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,14 +1,15 @@ -# Lab 3 stand: client -> HAProxy (lb) -> API x N (one scalable service `api`) -> PostgreSQL + Valkey. +# Lab 4 stand: client -> HAProxy (lb) -> API x N (one scalable service `api`) +# -> PostgreSQL (metadata, source of truth) + Valkey (sessions) + SeaweedFS S3 (bytes). # From a clean clone (details and Windows notes: REPORT.md, docs/WINDOWS.md): # python scripts/init_secrets.py # once: random local credentials into ./secrets (git-ignored) # cp .env.example .env # optional overrides (API_REPLICAS, limits, ...) # docker compose --env-file .env up -d --build --wait -# Order: postgres/valkey healthy -> migrate (ONE run, exits 0) -> api replicas healthy -> lb. -# Images are pinned by version AND digest. Passwords are passed only as secret files, and +# Order: postgres/valkey/s3 healthy -> migrate + s3-init (ONE run each, exit 0) -> api healthy -> lb. +# Images are pinned by version AND digest. Passwords/keys are passed only as secret files, and # every service gets only the secrets it needs. Only the LB publishes ports (loopback only). -# Two networks: `edge` (lb <-> api) and `data` (api/migrate <-> postgres/valkey, internal): -# the LB cannot reach the stores, and the stores are not reachable from the edge. -name: mtusi-lab3 +# Three networks: `edge` (lb <-> api), `data` (api/migrate/reconciler <-> postgres/valkey) and +# `storage` (api/s3-init/reconciler <-> s3); data and storage are internal (no route outside). +name: mtusi-lab4 x-api-env: &api-env APP_ENV: ${APP_ENV:-lab} @@ -27,8 +28,25 @@ x-api-env: &api-env DRAIN_SECONDS: ${DRAIN_SECONDS:-3} SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-8} DOCS_ENABLED: ${DOCS_ENABLED:-true} - # Track 1 improvement: `function` = one SQL call per send (default), `multi` = old path (A/B). MESSAGE_WRITE_PATH: ${MESSAGE_WRITE_PATH:-function} + # Lab 4: S3 (the API's own Read+Write key pair, see infra/seaweedfs/start.sh) and upload limits. + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-4} + UPLOAD_QUEUE_TIMEOUT_SECONDS: ${UPLOAD_QUEUE_TIMEOUT_SECONDS:-5} + UPLOAD_READ_TIMEOUT_SECONDS: ${UPLOAD_READ_TIMEOUT_SECONDS:-30} + IMAGE_VERIFY_TIMEOUT_SECONDS: ${IMAGE_VERIFY_TIMEOUT_SECONDS:-10} + JPEG_MAX_MEMORY_MB: ${JPEG_MAX_MEMORY_MB:-256} + PENDING_GRACE_SECONDS: ${PENDING_GRACE_SECONDS:-900} + JANITOR_INTERVAL_SECONDS: ${JANITOR_INTERVAL_SECONDS:-60} + # Level 4 failure experiments only (compose.faults.yaml); empty = no injected failures. + FAULT_POINTS: ${FAULT_POINTS:-} services: postgres: @@ -37,7 +55,7 @@ services: POSTGRES_USER: messenger POSTGRES_DB: messenger POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password - # pg_stat_statements: per-statement counters for the track 5 experiments (bench/track1). + # pg_stat_statements: per-statement counters for the experiments. command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"] secrets: [postgres_password] volumes: @@ -83,6 +101,39 @@ services: resources: limits: {cpus: "${VALKEY_CPUS:-0.5}", memory: 256m} + # Lab 4: S3-compatible object storage (SeaweedFS: master + volume + filer + S3 gateway in one + # process). Bytes of the attachments live ONLY here, on the named volume s3data. Identities + # and exposure: infra/seaweedfs/start.sh. Not published on the host; its web UIs listen on + # 127.0.0.1 inside the container only. + s3: + image: chrislusf/seaweedfs:4.46@sha256:08d516132314207d10c8e37cbffc1f32b147d870169688734cc61c6231625b62 + entrypoint: ["/bin/sh", "/etc/seaweedfs/start.sh"] + environment: + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + - s3_api_access_key_id + - s3_api_secret_access_key + - s3_reconciler_access_key_id + - s3_reconciler_secret_access_key + volumes: + - ./infra/seaweedfs/start.sh:/etc/seaweedfs/start.sh:ro + - s3data:/data + healthcheck: + # Unauthenticated health endpoint of the S3 gateway (200 once it serves requests). + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 10s + stop_grace_period: 30s + restart: unless-stopped + networks: [storage] + deploy: + resources: + limits: {cpus: "${S3_CPUS:-1.0}", memory: "${S3_MEMORY:-512m}"} + # One-shot schema migration: runs ONCE per `up`, before any API replica starts # (api depends_on service_completed_successfully). Replicas never migrate themselves. migrate: @@ -90,7 +141,7 @@ services: context: . args: BUILD_REVISION: ${BUILD_REVISION:-dev} - image: ${API_IMAGE:-mtusi-messenger:lab3} + image: ${API_IMAGE:-mtusi-messenger:lab4} command: ["python", "-m", "messenger.migrate"] environment: DATABASE_URL_FILE: /run/secrets/database_url @@ -106,14 +157,55 @@ services: resources: limits: {cpus: "0.5", memory: 256m} + # One-shot, idempotent S3 bootstrap: bucket, sentinel object, verification of the access model + # (private bucket, least-privilege keys). The ONLY holder of the admin key pair besides s3. + s3-init: + image: ${API_IMAGE:-mtusi-messenger:lab4} + command: ["python", "-m", "messenger.s3_bootstrap"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_access_key + S3_RECONCILER_ACCESS_KEY_ID_FILE: /run/secrets/s3_reconciler_access_key_id + S3_RECONCILER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_reconciler_secret_access_key + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + - s3_api_access_key_id + - s3_api_secret_access_key + - s3_reconciler_access_key_id + - s3_reconciler_secret_access_key + depends_on: + s3: + condition: service_healthy + migrate: + condition: service_completed_successfully # same image: built once by `migrate` + healthcheck: + disable: true + restart: "no" + networks: [storage] + deploy: + resources: + limits: {cpus: "0.5", memory: 256m} + api: build: context: . args: BUILD_REVISION: ${BUILD_REVISION:-dev} - image: ${API_IMAGE:-mtusi-messenger:lab3} + image: ${API_IMAGE:-mtusi-messenger:lab4} environment: *api-env - secrets: [database_url, cache_url] + secrets: + - database_url + - cache_url + - source: s3_api_access_key_id + target: s3_access_key_id + - source: s3_api_secret_access_key + target: s3_secret_access_key # No `ports` and no container_name: replicas are reachable only through `lb`, # and `--scale api=N` / deploy.replicas works. X-Instance-Id = container hostname. expose: ["8080"] @@ -128,15 +220,21 @@ services: condition: service_healthy valkey: condition: service_healthy + s3: + condition: service_healthy migrate: condition: service_completed_successfully + s3-init: + condition: service_completed_successfully stop_grace_period: 15s # > DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (11 s) restart: unless-stopped - networks: [edge, data] + networks: [edge, data, storage] deploy: replicas: ${API_REPLICAS:-2} resources: - limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-256m}"} + # 512m: UPLOAD_CONCURRENCY (4) x 10 MiB bodies + one djpeg (<= JPEG_MAX_MEMORY_MB) + runtime; + # measured in evidence/memory/ (bench/upload_memory.py). + limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-512m}"} lb: image: haproxy:3.2.26-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 @@ -145,7 +243,7 @@ services: ports: # The ONLY public entry point, on the loopback interfaces only. Both IPv4 and IPv6: # on Windows `localhost` resolves to ::1 first, and a refused ::1 connection costs ~2 s - # per request before the fallback to 127.0.0.1 (measured: 2.0 s vs 0.007 s). + # per request before the fallback to 127.0.0.1 (measured in lab 3: 2.0 s vs 0.007 s). - "127.0.0.1:${APP_PUBLISH_PORT:-8080}:8080" - "[::1]:${APP_PUBLISH_PORT:-8080}:8080" - "127.0.0.1:${LB_STATS_PORT:-8404}:8404" # local status page (UP/DOWN of replicas) @@ -164,14 +262,52 @@ services: resources: limits: {cpus: "${LB_CPUS:-1.0}", memory: 128m} + # Track 2 (lab 4): reconciler of S3 objects vs PostgreSQL metadata. Not a daemon: started on + # demand, dry-run by default (read-only on both stores): + # docker compose run --rm reconciler # report only + # docker compose run --rm reconciler --apply # fix: orphans, stale intents, quarantine + reconciler: + profiles: ["tools"] + image: ${API_IMAGE:-mtusi-messenger:lab4} + entrypoint: ["python", "-m", "messenger.reconcile"] + command: [] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + DB_TIMEOUT_SECONDS: ${RECONCILE_DB_TIMEOUT_SECONDS:-10} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3} + secrets: + - database_url + - source: s3_reconciler_access_key_id + target: s3_access_key_id + - source: s3_reconciler_secret_access_key + target: s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + s3: + condition: service_healthy + restart: "no" + networks: [data, storage] + deploy: + resources: + limits: {cpus: "${RECONCILER_CPUS:-1.0}", memory: 256m} + networks: edge: data: internal: true # no route outside: stores talk only to api/migrate + storage: + internal: true # S3 API reachable only by api, s3-init, reconciler volumes: pgdata: valkeydata: + s3data: secrets: postgres_password: @@ -182,3 +318,15 @@ secrets: file: ./secrets/database_url cache_url: file: ./secrets/cache_url + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + s3_api_access_key_id: + file: ./secrets/s3_api_access_key_id + s3_api_secret_access_key: + file: ./secrets/s3_api_secret_access_key + s3_reconciler_access_key_id: + file: ./secrets/s3_reconciler_access_key_id + s3_reconciler_secret_access_key: + file: ./secrets/s3_reconciler_secret_access_key diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg index ac95280..13172af 100644 --- a/infra/haproxy/haproxy.cfg +++ b/infra/haproxy/haproxy.cfg @@ -1,4 +1,4 @@ -# HAProxy 3.2 LTS — the single public entry point of the lab 3 stand (compose service `lb`). +# HAProxy 3.2 LTS — the single public entry point of the lab 4 stand (compose service `lb`). # # Backend discovery: Docker DNS name `api` returns one A record per RUNNING replica of the # compose service `api` (deploy.replicas / --scale). server-template keeps 8 slots and @@ -6,10 +6,18 @@ # full stack restart are picked up without restarting the LB. A stopped container drops # out of DNS and its slot goes to MAINT. # -# Readiness: an ACTIVE check `GET /health/ready` (PostgreSQL + Valkey reachable, not draining) -# every 1 s; 2 failures -> DOWN, 2 successes -> UP. Passive errors do not decide anything on -# their own, and there are NO backup/fallback servers without checks: a replica that answers -# /health/live but not /health/ready never gets business traffic. +# Readiness: an ACTIVE check `GET /health/ready` (PostgreSQL + Valkey + S3 reachable, not +# draining) every 1 s; 2 failures -> DOWN, 2 successes -> UP. Passive errors do not decide +# anything on their own, and there are NO fallback servers without checks: a replica that +# answers /health/live but not /health/ready never gets traffic while a fully ready one exists. +# +# Degraded mode (lab 4): S3 is a SHARED dependency. If it is down, /health/ready is 503 on +# every replica and backend `api` has no server. Instead of answering 503 to everything, the +# frontend then switches to backend `api_core`: the same replicas, checked by +# /health/ready/core (PostgreSQL + Valkey, not draining). Chats and text messages keep working, +# every attachment request is answered 503 by the API itself (fast, S3_TIMEOUT_SECONDS). +# A replica that alone lost S3 (e.g. its storage network) is DOWN in `api` while the others are +# UP, so it gets no traffic at all. # Worst case detection of a failed replica (a check answered at once counts at once, a check # that hangs counts only after `timeout check`): # fast explicit 503 (drain after SIGTERM): inter 1 s + fastinter 0.5 s ~= 1.5 s; @@ -72,16 +80,33 @@ frontend public http-request set-header X-Request-Id %[uuid()] unless { req.hdr(x-request-id) -m reg ^[A-Za-z0-9._:-]{1,128}$ } http-request set-var(txn.request_id) req.hdr(x-request-id) log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.request_id)] \"%HM %HPO\"" - default_backend api + use_backend api if { nbsrv(api) gt 0 } + default_backend api_core backend api balance roundrobin + # Uploads (<= 10 MiB body + verification + S3 PUT): the API's own budget is + # UPLOAD_READ_TIMEOUT 30 s + IMAGE_VERIFY 10 s + S3; the LB must not cut it at 10 s (a 504 + # after a successful upload would make the client upload a duplicate). + acl is_upload path_reg ^/api/v1/chats/[^/]+/attachments$ + http-request set-timeout server 60s if is_upload METH_POST option httpchk http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api http-check expect status 200 default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 maxconn 200 server-template api 8 api:8080 +# Degraded mode only (see the header): used when NO replica is fully ready. +backend api_core + balance roundrobin + acl is_upload path_reg ^/api/v1/chats/[^/]+/attachments$ + http-request set-timeout server 60s if is_upload METH_POST + option httpchk + http-check send meth GET uri /health/ready/core ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 maxconn 200 + server-template core 8 api:8080 + # Local-only status page (127.0.0.1:8404 on the host): which replica is UP/DOWN right now. frontend stats bind :8404 diff --git a/infra/seaweedfs/start.sh b/infra/seaweedfs/start.sh new file mode 100755 index 0000000..80446b5 --- /dev/null +++ b/infra/seaweedfs/start.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# Entry point of the compose service `s3` (SeaweedFS 4.46, one container: master + volume + +# filer + S3 gateway). Renders the S3 identities from Docker secrets, then execs `weed`. +# +# Identities (SeaweedFS s3.json, actions scoped to the bucket): +# bootstrap-admin Admin (create bucket) -> only s3-init has these keys +# messenger-api Read + Write on S3_BUCKET -> API replicas (no List, no Admin) +# messenger-reconciler Read + Write + List on S3_BUCKET -> reconciler CLI (needs the listing) +# There is NO anonymous identity, so every unsigned request is refused: the bucket is private. +# +# Exposure: master (9333), volume (8080), filer (8888) with their web UIs and gRPC listen on +# 127.0.0.1 INSIDE the container only (-ip.bind); only the S3 API port 8333 listens on the +# container network `storage` (internal, not published on the host). Telemetry is off. +set -eu +umask 077 +secret() { tr -d '\r\n' < "/run/secrets/$1"; } +bucket="${S3_BUCKET:?S3_BUCKET is required}" +cat > /tmp/s3.json < 404, cannot be attached), never deleted, and the mark + -- is removed again if the object comes back (restore from backup). + lost_at timestamptz, + lost_reason text, + -- The uploader must be a member of the chat (enforced by the database too). + CONSTRAINT attachments_owner_is_member FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members (chat_id, user_id), + CONSTRAINT attachments_object_key_key UNIQUE (object_key), + CONSTRAINT attachments_object_key_format CHECK (object_key ~ '^o/[0-9a-f]{32}$'), + CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready')), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256_len CHECK (octet_length(sha256) = 32), + CONSTRAINT attachments_pixels CHECK (width >= 1 AND height >= 1 AND width::bigint * height <= 40000000), + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_lost CHECK ((lost_at IS NULL) = (lost_reason IS NULL) AND (lost_at IS NULL OR status = 'ready')) +); +-- Janitor: stale intents, oldest first; tiny partial index (only rows of unfinished uploads). +CREATE INDEX attachments_pending_created ON attachments (created_at) WHERE status = 'pending'; + +-- Which attachments a message references, in the order of attachment_ids. +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id), + position smallint NOT NULL, + attachment_id uuid NOT NULL REFERENCES attachments (id), + PRIMARY KEY (message_id, position), + CONSTRAINT message_attachments_unique UNIQUE (message_id, attachment_id), + CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9) +); +-- "Is this attachment used by any message?" (janitor/reconciler never touch used ones). +CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id); + +-- Lab 4: text may be "" when the message has attachments (the API enforces "text or files"). +-- DROP CONSTRAINT takes a short exclusive lock on messages and the new CHECK scans the table +-- once, inside this migration's transaction (seconds for a lab-sized table; for a big table +-- this would be ADD ... NOT VALID + VALIDATE in a separate migration). +ALTER TABLE messages DROP CONSTRAINT messages_text_len; +ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (octet_length(text) <= 16000); + +-- post_message_v2: post_message (0003) + attachment_ids, still ONE statement and ONE transaction. +-- The attachments are checked BEFORE the key is claimed and the key is claimed only if they are +-- fine, so a failed check never leaves a key behind. A replay (same key, same body) returns the +-- stored message even if an attachment was quarantined since: the result of a key is final. +-- +-- o_status: created | replayed | conflict | not_found | retry | attachment_not_found | attachment_conflict +CREATE FUNCTION post_message_v2(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid, + p_text bytea, p_ttl_seconds integer, p_attachments uuid[]) +RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz, o_attachments uuid[]) +LANGUAGE plpgsql VOLATILE AS $$ +DECLARE + v_created timestamptz; + v_found integer; + v_usable integer; + v_check text := 'ok'; +BEGIN + PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user; + IF NOT FOUND THEN + RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[]; + RETURN; + END IF; + + IF cardinality(p_attachments) > 0 THEN + -- Visible = ready attachment of THIS chat (pending, foreign or unknown -> 404). + -- Usable = visible and not quarantined (lost -> 409, like a failed attachment). + -- FOR SHARE: a concurrent quarantine waits until this message is committed. + SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL) INTO v_found, v_usable + FROM (SELECT a.lost_at FROM attachments a + WHERE a.id = ANY (p_attachments) AND a.chat_id = p_chat AND a.status = 'ready' + FOR SHARE) a; + IF v_found <> cardinality(p_attachments) THEN + v_check := 'attachment_not_found'; + ELSIF v_usable <> v_found THEN + v_check := 'attachment_conflict'; + END IF; + END IF; + + IF v_check = 'ok' THEN + INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id) + VALUES (p_user, p_chat, p_key, p_hash, p_message) + ON CONFLICT (user_id, chat_id, key) DO UPDATE + SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp() + WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds); + IF FOUND THEN + UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond') + WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created; + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES (p_message, p_chat, p_user, p_text, v_created); + INSERT INTO message_attachments (message_id, position, attachment_id) + SELECT p_message, (a.ord - 1)::smallint, a.id + FROM unnest(p_attachments) WITH ORDINALITY AS a(id, ord); + RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created, p_attachments; + RETURN; + END IF; + END IF; + + RETURN QUERY + SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at, + ARRAY(SELECT ma.attachment_id FROM message_attachments ma + WHERE ma.message_id = m.id ORDER BY ma.position) + FROM idempotency_keys k JOIN messages m ON m.id = k.message_id + WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key; + IF NOT FOUND THEN + RETURN QUERY SELECT CASE WHEN v_check = 'ok' THEN 'retry' ELSE v_check END, + NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[]; + END IF; +END +$$; diff --git a/requirements-test.txt b/requirements-test.txt index c5ab68f..fea533d 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -2,3 +2,5 @@ pytest==9.1.1 httpx2==2.13.1 jsonschema==4.26.0 +# Lab 4 tests: generate PNG/JPEG fixtures (valid, corrupt, oversized) on the fly. +pillow==12.3.0 diff --git a/requirements.in b/requirements.in index bdaae42..22ea682 100644 --- a/requirements.in +++ b/requirements.in @@ -14,3 +14,7 @@ psycopg-pool==3.3.3 redis==8.1.0 # Argon2id password KDF. argon2-cffi==25.1.0 +# Lab 4: S3 client (async botocore over aiohttp; requests are cancellable, unlike libpq cancel). +aiobotocore==3.9.2 +# Images are checked without Pillow: PNG by src/messenger/images.py, JPEG by libjpeg-turbo +# `djpeg -strict` (Debian package libjpeg-turbo-progs, see Dockerfile). diff --git a/requirements.txt b/requirements.txt index 78a3b4a..c7c583c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,185 @@ # # pip-compile --generate-hashes --no-emit-index-url --no-index --output-file=requirements.txt --strip-extras requirements.in # +aiobotocore==3.9.2 \ + --hash=sha256:363b4892423b272eb84afbb58000da0be6ccd0cf8a68b98438c957d05b841efc \ + --hash=sha256:8e32238c5bd77717ab1ac90077ff51ab141d025528115ccc7cc9b50b2d176e5f + # via -r requirements.in +aiohappyeyeballs==2.7.1 \ + --hash=sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d \ + --hash=sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472 + # via aiohttp +aiohttp==3.14.5 \ + --hash=sha256:0133c3c3b54a0bf1e71fa5c1ad95c93f07fd54e24ef1fe182f5122e1573d2bf1 \ + --hash=sha256:038c2c7e8caa26b6c8423779b5eaf1893904048a512c19b32fe841ffa5592b50 \ + --hash=sha256:0684952aeae1f5dbfe02d46039338513b94009baecd15d8e4098a357c4c4a2a6 \ + --hash=sha256:0790ec66fa4013e83c53b9025a45d454723da1a2fce28b3208c9b32d08af162f \ + --hash=sha256:083673c7a94c3ea035caaa5ca04288bdb44887abfe1f5ba23294e6a4b03efd2d \ + --hash=sha256:09e0eb18c7e0c8777e2f9149de63799195b9b3ca1b5c81ba6f32f2c6b8628210 \ + --hash=sha256:09ec102b4b8c9a920275733bbc11fdbb615efe6f9231a06007c0218d336fb77a \ + --hash=sha256:0a8ea271867e360ac985ae607f4a23ad9a38414b9aca1d49ec98839ae660e49f \ + --hash=sha256:0e6f16f5e49c4b8267988c05ab07760d7064cea57d077c3d068d04b0fbb992cb \ + --hash=sha256:1061b364556e8172e8d46b0b183adeeb73e8c42d30ebc745591e1bd89acad52e \ + --hash=sha256:1220353657ad49493551f089ce02f1a348fd57ffd585bfec77f2f3c4fe3a7346 \ + --hash=sha256:137351bf20bbed9a65e839f4a4452ac377389bdb2f2857d2acffef38f5e9f2d1 \ + --hash=sha256:149fb56caf7acb67073126f675d0958d9c4b3125fcd3f6d4877df98aa8a97ce9 \ + --hash=sha256:14f04769cfefe4734016a856a83af36133cd17779cef9ae817f812b8ba9d6d51 \ + --hash=sha256:15a310d3c71398e3d7bfc93a1a73fbe664315cd9e9016b8efc1cff85eeab7155 \ + --hash=sha256:1612fa5857b37bf32e5c1eaeefb96e3b01e9c70679eec81f0934e8a600080863 \ + --hash=sha256:16c8abd5bca220a47efe667d26f8460124c81810787e79ee87b242677563d9dd \ + --hash=sha256:19e2ba471507c34f8252402ab50f5ab512398b9ea8c8f1cb26beb3f75793ba30 \ + --hash=sha256:1aead151c3abbac6b32942e452020cb66d7efc099d253cc6c20f748e926c858b \ + --hash=sha256:1b438b73c38111818d0c9d6a5c2bfed8584c8e503a49ef085d70e874ec846738 \ + --hash=sha256:1b5416552740edf07234cc9437d0706f2acb67b93c198670b1a68e1b2b587dec \ + --hash=sha256:1d2d981b53dd09a319e3570ef8cc3bbc3ef86f5a7abef0f6b2bff3867db3a9e7 \ + --hash=sha256:1eb8167961ec4dfcc8cb9dd50bd0ee72519f7ef496be95203e49e27b01618382 \ + --hash=sha256:1ffa3a523a36d8628f98c06492ae16a31a23d14c0b4ec721757b477319f656d6 \ + --hash=sha256:20064a177a070d789ee64a50b01a9161d3468e989baacfc6c714aa685c4b332f \ + --hash=sha256:20726f9782d5c2744c1c66255842d1d163bb3edcf768b8de25216bf47f7b6ccf \ + --hash=sha256:20f085697d7e911f1f73c43ed03fafbed1e7121797e2eb5428efa80398060584 \ + --hash=sha256:225c579c23b68b343cccea27a7e06e3bd8ec23a09c30b427eb3f1e4ca6239b20 \ + --hash=sha256:24409db442e2fb6e766bc7f3943851a8381dec3098140e43bb2e843b79e31b12 \ + --hash=sha256:248d779ad720b49d4fb355720e60c9e5f444f95887bc16974fea48fc56c41789 \ + --hash=sha256:2528cb4c6b92008c76ac9ac6298624069bb2db91ff4929905512d1d84485f658 \ + --hash=sha256:276a4fc00b1d9ae492b802763a789c5b86328b989c5ea169f2faa447d6a11c7c \ + --hash=sha256:27c2322e03f66101acb09869ce1cf1efc04994ee95e1735b69827bf8c8b9d781 \ + --hash=sha256:293d3ae7c6a0ed176a42e59a1b5fde825ead65c835360f734148e96729f928d2 \ + --hash=sha256:2cc38a4f2b516bef1714e690df87a0e043faf1a7693c82d860091684453d5111 \ + --hash=sha256:3093b72c215bda16ce961a6d073f6e71d46e022962a9d5d457c5d4d421c78b57 \ + --hash=sha256:32e8fa6644e541fcd7e02430588c7fc93b602c1778ea0bc345505db76b61cfb4 \ + --hash=sha256:330900acd0dc4cb8b27f9c127fbaad770964845338493e7906ae3822e82dbf8d \ + --hash=sha256:33f706574e32c6e694f352a856e05caf18f7f2c871b3e87b41c55ea452b409ab \ + --hash=sha256:3ae800a20947e2c2e53088047d021e6bf7d51560cc49f6a0737a1f79d2e3a13c \ + --hash=sha256:3be7dd397d64ca3e1869626fa9318aaebb54b7bf93bc72d7a205448d83e4f748 \ + --hash=sha256:3e0eb43bed3c6801a6cee315195377789e90b2a72c2277a475b578535312488d \ + --hash=sha256:3e51a27980c3788e6e6b3325d694fdd4898087fa8a86b2763af77b39353da41e \ + --hash=sha256:3edbece0379b8b4aaa67619b8aa2399bb66fce372cd5911098a434ea77220aa0 \ + --hash=sha256:3f2dcc00191fd563e9075181a14ec31d7dd63223ced7582cc70a15a499de0c79 \ + --hash=sha256:42b5e616946dbaf505e2bff18c9af2cd4ef9e7ef300ee58a6e951a5b7cf147ae \ + --hash=sha256:42f320d4a5b00b9af0bddcfec5407dc6f2d9816f006b2f79ebbaa31f16895df3 \ + --hash=sha256:43351bdb5e4c3cb7d1772368e988534e869a74db7778079a83782c11c69535c7 \ + --hash=sha256:43e1b7994a8b038125f722bff07492ef501110722c2727c408995d9fb864c421 \ + --hash=sha256:4887d130a7bbfed3a85493bb5a25e5b5b558d40c1d986dd16970d2bb26d63793 \ + --hash=sha256:4f07fe3ac408d8b3f768be471dc3f56d43843c47d97c66120534467a15ead197 \ + --hash=sha256:50343c1757b4b6f6708eeaf24534b32f19dfb99fb1b762c00420867a62fc81e0 \ + --hash=sha256:50983e3be33d8c0942ab88cec3905b10602f64c469b20153c48c5d4e558dd016 \ + --hash=sha256:50a195903119008fe9cc68710535eb37f556ffffd6a7759afe70a2c145587045 \ + --hash=sha256:5558a7f5a05af9ecf744af91e5baefc436f93c9333e656c27ec253f9a6bbe178 \ + --hash=sha256:56572c42e3ecd636de8d2c3dd54cf5fc939cb5c32eb56297f176a0d366fac622 \ + --hash=sha256:56d9828f204331a5ca8850fcfe2bcce95a149f1f223f60cc7216e5524978e480 \ + --hash=sha256:579f97d5120f2971876d2ddca2968135f6944d00c44c3a6590ad7d86ca9b403f \ + --hash=sha256:5af42135fdfebdadbc2bcd9c0842a48ccf0d62794c36a260b21dc4b94d1e0119 \ + --hash=sha256:5ba14a839fbe87cf7c12a6b5661c05f324a296eb8363141edb3944ba63d4c9d3 \ + --hash=sha256:5c76f1802bab718a68ac3cce447160605c734551f95c67ae90fa1132b215cb29 \ + --hash=sha256:5e8f97c0488ffda3082766ac0f2c8150a9a58c4d05788330e479cfd449b37939 \ + --hash=sha256:5f3e96071686755d9cd3600c3880183eb94b012178e92746d68101800f0ed8a3 \ + --hash=sha256:5fb6a6e919bfb703227bc1ce6579281b84b1a2ba57deb9794dfdbec7dcd1e40c \ + --hash=sha256:602c1e9b718a3275c580149f947e7fac65044c0a20e599553fb12e9700da9eca \ + --hash=sha256:657291433bf4dd3142f3abac495764cd47d0c7c92087751e6666c6447e65fcef \ + --hash=sha256:6774814fd5c338e72ee0da5cbb9432816df450e69c019f72b5d29bdec2a1792d \ + --hash=sha256:6da32b5ff3fd78d244e37300463434c7145162bfd2b6e9e915ab164da37f7343 \ + --hash=sha256:6e1d8637cf73eebc92eba2e11d4cfff98a3b562f2505bd75bba766d908926e8d \ + --hash=sha256:6e4251c0ba4624a68a2c11471a1ac54c3306876c21f0ae86de085cc9241c8905 \ + --hash=sha256:6f275c11d1aa6d4c458e05a68be084efe3c55a113d99e3f46a318098e52948fc \ + --hash=sha256:6f967dde489ca6a8c02d093ab245d2cbf50ccb5c36adf0188b17b0ca39d24b67 \ + --hash=sha256:6fdcd6af7e2e51d1ba1b4bea16e97b074bcb7b5dd0246a9d8201341bb28085a0 \ + --hash=sha256:70cb4008ac2ed1e0ca9e824deb4b53d3aa0d939109698ebf1e723a84337bd794 \ + --hash=sha256:7457580535e019e1247ea35d6a02bf081ad30c26d0cbc210c93f6c3ab67a0835 \ + --hash=sha256:74b0a9c8270f9b0a11410e124ff8d4f18bfc1f1837440ec84da5ae7b50927b5d \ + --hash=sha256:74efb69332b85675b1eabd760a8cfc2e2cf42c60607c66f88014c1bdfb40942d \ + --hash=sha256:755933b107ea7a6a9ac916f635a70595a5b1a32fac10a8ff0b9f2ab88555550c \ + --hash=sha256:7779cd97e61ebe583ec2f1c5616cdd038aa08a4453b1848c67842176d054948e \ + --hash=sha256:788ecaa9c10533b786ce5ba70c4f2df78ad41819fd00a6c99d92b66f9a32e1da \ + --hash=sha256:7bd8ac754ebd6733a3e2a0dd1674c4d8ab086196803fd8dcd776f07b4e2607d9 \ + --hash=sha256:81c2b3dfd56c62bee6108e4852d5970b4cf9086390b6983f52b666e878c1f115 \ + --hash=sha256:823c910f046f23f4c713b8d99a2242dc65f591cb45ee86418fa11762a3c2963c \ + --hash=sha256:82c7583cd3dfdc7dcc927835b4f6c7faae7ecc1ba3ca5879321621ae2e6f8e84 \ + --hash=sha256:8966ecac808dd5f473c9c4cefd10cd3ffda71c18a4d3493b7c7d2ae1803bf2cc \ + --hash=sha256:8df7d481654ac96fe1ba9a02a9f67770fdd367823e0d5ef01b922725c4bd2cfa \ + --hash=sha256:8e317e0fb6b16212c881d2205a7d87414c29acd69320b3aa6dce9d9c7b86fe4f \ + --hash=sha256:932ce7e694bbc29b2bf6f64f2343c27d148d4997c771d01bdade4639b6749ff4 \ + --hash=sha256:939042d5cda21d41a6f512e7cc8b8e33a2aebff863352251da495fbd91b673b5 \ + --hash=sha256:94684b879ac1d71e4238850c99b62dc1b28d9086b156a2555f082010b85a865c \ + --hash=sha256:96a2e584f0b9ed8f1fa33211397dcf67bb7069866402cb405d191c2f0defb9a3 \ + --hash=sha256:9ad7e6aa38c20da1be697874349c4c273c8a03b7887169665081706398d0439a \ + --hash=sha256:9b42db919715e91eb76acf3bc492a9a7ccd8bd9adc6745c1412b689735269f14 \ + --hash=sha256:9bab2045550c4fe0f7baf89574db1b455c195750702ba96fef1f16972b146617 \ + --hash=sha256:9bf1d5dcc15204d9ec8b8ea4c18fd66e6b80e5de1f4ecbafb3a2f2740f8039d4 \ + --hash=sha256:9c061aa954daaf57d2a4b8374f9fca621ef0e1b603584431c220c22458c59b6d \ + --hash=sha256:9c428eb2bd8817588d16a0ab898aa4eb5d141f896aa2b394cc79a4cf61d9a8e2 \ + --hash=sha256:9cc882cf8619109583c906b4d4a85d6a111a98afa34b7a450d1e08118d016820 \ + --hash=sha256:9ce66feae6ac65327379460380549bf1b8df8e17c4e25df2a2bcf168272e3bed \ + --hash=sha256:a23fe35d776bc03cb495938b9594450d047e3bc08c5255315a82323e9cb7d2dd \ + --hash=sha256:a2c473a355f9239efcb72c92d5abfd8fcdb0cc78c8e9af607e72ca12dbb36593 \ + --hash=sha256:a63afd1f757de949028387e65a7127b61ad0f775432dbb0e62816ae619fe69ac \ + --hash=sha256:a6d02b4c38de03d9c7617813433e6a0fb6b522797974177d69d9dad431900833 \ + --hash=sha256:a7d470cf7b206e6359fc77b1b860632fde400d5a2ed59cd0181b93a686bc81ee \ + --hash=sha256:a95529a92a446db351675f4aab518feaf5e99842f63f5dd17160c2b74f382db3 \ + --hash=sha256:a9918e58faf62ba2c7147927d06057aec78f42475aff5048047ec47e7265a600 \ + --hash=sha256:a9d3983bd6ab7aa1cfd573544ae98df9b6cb6912a5185a198263e024a636861d \ + --hash=sha256:ab52d8f1fc1b64821c1fbad64a647ed6203627004059a6d1ed4f0858a1499703 \ + --hash=sha256:abfda5cb094a829f7bc25216a32f7db2e85cc65bd59910f8e7b40b3d9b224764 \ + --hash=sha256:adbeee7d6fd4cf5fe0aece2fb3edc4243615d3180430ba8149d01a90670cac99 \ + --hash=sha256:ae53924aa853a7a2ca20ed4142c7c6b56338e4d4cd999e2980075b9efc2e257a \ + --hash=sha256:b032a0023eb41d768ce77d83210ab2a3c389bc0b09313273c7e1eca48c10a755 \ + --hash=sha256:b1b8ece1e71132d2afba4dbc0c3d62c766e25165990b25db1196c04969eb3d84 \ + --hash=sha256:b2966998927d7bed9db12c0a4647b0c7b179755878fc9c357fe1ffd3e3b0c1a5 \ + --hash=sha256:b3cc509327c7b27f6f4727a8830f4004f6df7766e179f2f4b8e54e65c0bec5d3 \ + --hash=sha256:b7806e804889231b0e06469fd4a5c06313d1c0a3377322b6d9237fa5e0fe4167 \ + --hash=sha256:bea559ad70218d230663e4210875735076a9bfea5994cef34a55a25faeaf2544 \ + --hash=sha256:bf163cc701f3d4ac43ba7d97771bf5fd955220ef5500ef3ee847bc0ecfbf4ec1 \ + --hash=sha256:c147451b4a58e7050f7f7394e6c467867c84161560001f9ad4fb2d1446743946 \ + --hash=sha256:c172db893e516e1358e65a95ee20b7ce7173963eefe318b6ab2a2220688b999e \ + --hash=sha256:c1d60eafd9c7e8e74abd03a5b00df44e7febfe6d9b89b559c0a6551eef0699d4 \ + --hash=sha256:c2c30484dd1417ef98b51021ffa2cc0d7f3c78918adaaaab7e70817335ab3e02 \ + --hash=sha256:c32e26310cc10e547f53cd13d39a369034f69dcb7d749d5cb0e5f67bc196b6ba \ + --hash=sha256:c5ed596aedb9c42afd3fe0aae3117725378ac73d2cc5ddc735056fbdb96c5d02 \ + --hash=sha256:c8859a013ae0de1074660992139a1a440df3e6b219b86cf0d3f11c2692bb4fe3 \ + --hash=sha256:c8c4478bef6d57fcfda15dae461ea3c9f06aa7b257c58df3f2300174ccbb185a \ + --hash=sha256:cb11a971a3aea10f9b8373be628f1df932964fc6c6b174516d318a48c3ac4412 \ + --hash=sha256:cb131d775a1573c1aee66656bd78b023577bbdb6cb8349a07773bd4f73e68a6e \ + --hash=sha256:cd88b01f3d37b7a2a34f91d98f14720206f1ea3d540843fab2d649dd5fb91fec \ + --hash=sha256:d05e94cdfe0d15d0206f970722d2554780ce562787b21b218b275447f8751319 \ + --hash=sha256:d079c0a0135c36e7beb6f1c88087c8f108dc5891cdd0b5eafa778421bda70ed2 \ + --hash=sha256:d3112585250b199296c26ca6e0131640b6a8d01bab8b232d2eb3763ed469de11 \ + --hash=sha256:d418ce2af40c6bb685b3f663e9e8de27cb0a22431d8e88a167348d7f01878073 \ + --hash=sha256:d51db97c96384fbfcaf8f4c65922183a68b94f891c3c10c862ef5f6df2adbb1f \ + --hash=sha256:d94e44be379e569758fee8a9a58431cfc3c2598c708b92b1cfe96c66b4c94aef \ + --hash=sha256:dab9ac5a67c8d1f070c00fa8fccb7cbd1b8dcc1a8d6b42f37540df9b3d4cc603 \ + --hash=sha256:dbf53ae2601b7fd5a93c3944deea3a78d40f495226d582c35ef7a433425ce2b2 \ + --hash=sha256:dca3fa8d8a0a26679862eccb0b1a9151b2b9f1cd2c212e7a6335778faaff5833 \ + --hash=sha256:df37b620684e19b5e25724412518ccafc3b1a49cdac706fdbd2f983fad943450 \ + --hash=sha256:e1cc2bfaee8c214f06080a7c7d5772419b8a1108e8e5349236189811823fb02a \ + --hash=sha256:e29347c142cf6e99e0dff5e2995ead1d50fa3b51bf37a7c726a7ccfe5419745a \ + --hash=sha256:e4f5cf4dc72a71c4cfa9751b4950be22f733626670230d46e7d606592aa22d59 \ + --hash=sha256:e724a7b6091f0b1ac064f9d1b15ff9ec52e6033a86cdae649e5f086e32a3c0db \ + --hash=sha256:e87046c8ff77a8decdb6a41d8ab25824b47531b2da933aeab0c1e21c7acff329 \ + --hash=sha256:e95c8def4b81c5d68d5cf1f54c07acd7c0d2577af244e5b6da802120825737c6 \ + --hash=sha256:eb324e2009fb54db30a071dad7caf6998ee2879c4704007efb244514dad1fec1 \ + --hash=sha256:ef60869969180ec2464f1349aff07138ae35ca2200f0946cb3552e49e8f301a8 \ + --hash=sha256:ef692a24087a699c0a4a26af45e746e0c1eae2116f6d8a5ff91d8aae2b867b45 \ + --hash=sha256:efc21a454892828368b11c2c780de0ff8bc991f73f6b99c6b66e56205470929b \ + --hash=sha256:f001b571ead90ca1770f1e616db255351a1703317f20374c361ef22f12c06d09 \ + --hash=sha256:f2a7966bda23dd85051f1661ce0ace38d6890e05ec6c357ecae9d2479cba377e \ + --hash=sha256:f2ebb54b3f932210503072f09974b4fb574d823e497a944adfdcd140a6a00255 \ + --hash=sha256:f2ed8b64dc0c651c0f5a9c926777719770021251b8f336d97c4b80b660836ce1 \ + --hash=sha256:f375db73a39f5cf83696d500e21a67f418dc9a988955756f254be8f03b7b3651 \ + --hash=sha256:f59c7673465908cbe506117176156c127f29f917677afceada34957179221d91 \ + --hash=sha256:f8d40ce41991e9d56fab4f5dc4a51fe59bc3b5c77c27f4b148963064d00232e8 \ + --hash=sha256:f9033b43f511f27547c557dcaba0177649e10a3725336ccd2cce0fdc1dc4850d \ + --hash=sha256:fba47bc2c3d7303c3d027c6cf4d07626c37b1314ac81f5820c31032e0ca1f677 \ + --hash=sha256:fbdc5ec49f9ca3cd24955cf3520b10a4d4c901ba2572094c84274e9e7eb30534 \ + --hash=sha256:fce9523df31cea6284f3e2c479876750d7687cf671d7b25d32b19effc0e86441 \ + --hash=sha256:ff75a7537413a86e7cafe98e0e1d6e3dc4b15c6349896e7d5c6b881bfdb6d550 + # via aiobotocore +aioitertools==0.13.0 \ + --hash=sha256:0be0292b856f08dfac90e31f4739432f4cb6d7520ab9eb73e143f4f2fa5259be \ + --hash=sha256:620bd241acc0bbb9ec819f1ab215866871b4bbd1f73836a55f799200ee86950c + # via aiobotocore +aiosignal==1.4.0 \ + --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ + --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7 + # via aiohttp anyio==4.15.1 \ --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \ --hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94 @@ -55,6 +234,14 @@ argon2-cffi-bindings==26.1.0 \ --hash=sha256:f9c4420a7a864fe1b86ce35befc95b8e39fb852493b81cf798671ddc265de638 \ --hash=sha256:ffff613aaa9ce6236766e2fc6dc560bb5abde7a2e2416e3db1f9ae395a2b4dd4 # via argon2-cffi +attrs==26.1.0 \ + --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ + --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 + # via aiohttp +botocore==1.43.106 \ + --hash=sha256:006870b3b4e40547232ad12c3bb4faec91bbbe0659aafaa3b7fa48a112c4ee97 \ + --hash=sha256:c1fb8818f9957cb5037f73db34ac1a12ba43562e31d6e7f9cb7e5fc64e1dba78 + # via aiobotocore cffi==2.1.1 \ --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ @@ -161,6 +348,140 @@ click==8.5.0 \ --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 # via uvicorn +frozenlist==1.8.0 \ + --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 \ + --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 \ + --hash=sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121 \ + --hash=sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd \ + --hash=sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7 \ + --hash=sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c \ + --hash=sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84 \ + --hash=sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d \ + --hash=sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b \ + --hash=sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79 \ + --hash=sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967 \ + --hash=sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f \ + --hash=sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4 \ + --hash=sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7 \ + --hash=sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef \ + --hash=sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9 \ + --hash=sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3 \ + --hash=sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd \ + --hash=sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087 \ + --hash=sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068 \ + --hash=sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7 \ + --hash=sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed \ + --hash=sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b \ + --hash=sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f \ + --hash=sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25 \ + --hash=sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe \ + --hash=sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143 \ + --hash=sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e \ + --hash=sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930 \ + --hash=sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37 \ + --hash=sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128 \ + --hash=sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2 \ + --hash=sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675 \ + --hash=sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f \ + --hash=sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746 \ + --hash=sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df \ + --hash=sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8 \ + --hash=sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c \ + --hash=sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0 \ + --hash=sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad \ + --hash=sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82 \ + --hash=sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29 \ + --hash=sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c \ + --hash=sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30 \ + --hash=sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf \ + --hash=sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62 \ + --hash=sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5 \ + --hash=sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383 \ + --hash=sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c \ + --hash=sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52 \ + --hash=sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d \ + --hash=sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1 \ + --hash=sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a \ + --hash=sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714 \ + --hash=sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65 \ + --hash=sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95 \ + --hash=sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1 \ + --hash=sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506 \ + --hash=sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888 \ + --hash=sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6 \ + --hash=sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41 \ + --hash=sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459 \ + --hash=sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a \ + --hash=sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608 \ + --hash=sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa \ + --hash=sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8 \ + --hash=sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1 \ + --hash=sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186 \ + --hash=sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6 \ + --hash=sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed \ + --hash=sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e \ + --hash=sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52 \ + --hash=sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231 \ + --hash=sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450 \ + --hash=sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496 \ + --hash=sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a \ + --hash=sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3 \ + --hash=sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24 \ + --hash=sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178 \ + --hash=sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695 \ + --hash=sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7 \ + --hash=sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4 \ + --hash=sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e \ + --hash=sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e \ + --hash=sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61 \ + --hash=sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca \ + --hash=sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad \ + --hash=sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b \ + --hash=sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a \ + --hash=sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8 \ + --hash=sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51 \ + --hash=sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011 \ + --hash=sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8 \ + --hash=sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103 \ + --hash=sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b \ + --hash=sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda \ + --hash=sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806 \ + --hash=sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042 \ + --hash=sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e \ + --hash=sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b \ + --hash=sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef \ + --hash=sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d \ + --hash=sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567 \ + --hash=sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a \ + --hash=sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2 \ + --hash=sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0 \ + --hash=sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e \ + --hash=sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b \ + --hash=sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d \ + --hash=sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a \ + --hash=sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52 \ + --hash=sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47 \ + --hash=sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1 \ + --hash=sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94 \ + --hash=sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f \ + --hash=sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff \ + --hash=sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822 \ + --hash=sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a \ + --hash=sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11 \ + --hash=sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581 \ + --hash=sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51 \ + --hash=sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565 \ + --hash=sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40 \ + --hash=sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92 \ + --hash=sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2 \ + --hash=sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5 \ + --hash=sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4 \ + --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \ + --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \ + --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd + # via + # aiohttp + # aiosignal h11==0.16.0 \ --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 @@ -288,7 +609,333 @@ httptools==0.9.0 \ idna==3.20 \ --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c - # via anyio + # via + # anyio + # yarl +jmespath==1.1.0 \ + --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \ + --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 + # via + # aiobotocore + # botocore +multidict==6.9.1 \ + --hash=sha256:006c4478de0a1876f4834e14255776286f09b9846b505fe63f67f9d173a9487c \ + --hash=sha256:024123f0ab402ab33828e24eb80fa8f25167d0d3783ba5f287e39ed741e6abf9 \ + --hash=sha256:02f6d0c4b70f783305e73f9944d8efe6be1022550f0974ba0ae9d8893c0350fa \ + --hash=sha256:02fe09dc197b8ae7e355371e51e5dce2f39060cd5a8badf94904527e23a3f188 \ + --hash=sha256:03731f6fc036180700c9dc2308205a48e5ca6f3ff03087739ab746f294022201 \ + --hash=sha256:03d47df72f084f757c1cb771188d5f4e3a805e4abc4d67e32509272343ae9382 \ + --hash=sha256:03fac50ddfd8302175b77863a015eccfae76767cda5506eef86df559ba861e1f \ + --hash=sha256:042fb0196047e786936a730bd302de83143950da45f2c16078da8f35e1cf7919 \ + --hash=sha256:083735b7f395894e43adb278d5dae901448883a835ff8f1977e285fefdb10418 \ + --hash=sha256:095d900c242e00fbe5f321ee072e7278b4153e78c5ce9c1efde167d62c1e4771 \ + --hash=sha256:0a5769559e3312dd96731fbe15b4abb6033368ac1cad5a98dadd21946a4c7d6c \ + --hash=sha256:0b2fb8c349d1103863750b5d8cb5ace766917f4b35f3d883c8f778853eaa9f76 \ + --hash=sha256:0dd655518f136febd96c05131a76a863e32fc2a1d7acd4e3c959e3ceb77d8345 \ + --hash=sha256:0f06e60fa190aa7abd0914c2a766736fdc8e9f34878c4346338534b73d1b20e2 \ + --hash=sha256:0f2ce963299d42fa3f22a90adc0fdf174792ffef5ff4c7ffb68260548fb05580 \ + --hash=sha256:0f3bd290711c6e9486173a6ee7cd4e7f00c3971c7908c1ec1b6e5437c5e4c6f9 \ + --hash=sha256:10083a8a0f4e1b26b599889e90b9802504ce5d3f7722f925bbb7ca47dd22a7c1 \ + --hash=sha256:13849a1d4f54c3809ae721e9e83ab28f5ea602f33660cb84eb6ef261eac706c1 \ + --hash=sha256:14c56f73e78faa1f68bbb826197cd5871994e70e841b8590829c35912ece5c64 \ + --hash=sha256:15db6a102cbaf1949cf028ecf080aac76d20bcd29ad4e092574db6c6b7af78a5 \ + --hash=sha256:19e31815d41cefc365489e591d105d2baceb2f65aa75d29471fbdbda8651e006 \ + --hash=sha256:1a53de2772cfb74559df2eb4456ec4eeb908435ec55a84b69370d9d745d62aa8 \ + --hash=sha256:1a8adfcaf96f587ab138476eaddef95f29b8a2a8a9afbfea8d2fd62180995d02 \ + --hash=sha256:1a938761c77e0e6edb0c93d02f4e988d44a69e5195e5a3b893e5553311347132 \ + --hash=sha256:1c9f3c25df6c9d3bbae4f6fd3f514b1c5c740a2110f56ccf36069c85417e289c \ + --hash=sha256:1ca5ebe6d454f1e5496cf052386a559f1080bf2de75bf327ebca0a6003b79f19 \ + --hash=sha256:1d804e4caf5d5da37d6dac1325da5629ebef1e27a294c2b568b295814aa36c7b \ + --hash=sha256:1fee9a16d88a1c4865610de31ef5c666671020d7a81d1f510eaf3c97d00ebeba \ + --hash=sha256:23136f5a564654eb61061ec6d5620a4c1ea32c8f552b65e9982a12b72bff601b \ + --hash=sha256:2784090c30a586d5b45197bd9c32f87fb927216cde302f6cfd76d76577e90f08 \ + --hash=sha256:29138fef49828542e859828107e42e50d0e587c513b7eb4b2d92bade2b0860fe \ + --hash=sha256:2c1aeb92eea59d824f004341b26d5e4b47a8a441cf9726769b9a90abf9d0e08f \ + --hash=sha256:2c5d675da8f1cb5650271c8ad5e95c0a3e5a183c105e72d953b12877b1c8d0fd \ + --hash=sha256:3100d169ceb7bc8f05f89a6db11d1b21f119975fc26f29dc45a472e3569f0879 \ + --hash=sha256:31199204b3ced121ff5407a2c342326a5d27e3870abbf94bd80dbd2451b7bc8f \ + --hash=sha256:32217133dddc58c927805cf6c0731d8144584176b768042ee886d51e71860bc9 \ + --hash=sha256:33fa55b990f81c2927e01399ace0d18926c69d69baa8cdaa819424132fb97987 \ + --hash=sha256:35ba0263bae5dd3ad5aad767cc9afc01a8598c7dae30f1b3b2de98b1b32c28bd \ + --hash=sha256:35fc236507fb1b3138f0af5ecd5f94ed752d4d6d826248eae425f86204013eea \ + --hash=sha256:361f7206cf341ba94fb015688f5c8b480f8e63bd58a4c14a48aeca7851a241cc \ + --hash=sha256:369b5aa01b241cd3fea6890bdbb11a1425d87bf1515831500d518f4223e9d72c \ + --hash=sha256:37245ca4105386194dd1d292a6f2aae09bfe1bd7ac6f9ec25093e3cf8e9b143b \ + --hash=sha256:372c063f37480f62c1ae32dc3a1a0a5942b180c883f99789075a8a8ec3c4e709 \ + --hash=sha256:37a9ebe00c698279213d56e6c64e1962ab1e092918270649b397cac3dc196ca4 \ + --hash=sha256:38c9986f9ce50c459b10de216a05f4bd7ed5ac63887d56e500a52bb464b861ce \ + --hash=sha256:3adf06c66041aa21eeb8a71e82379b74773298c8e6d3d839b151aae441a99b94 \ + --hash=sha256:3c95601ed98fad3f6e2f8fe809c3b526b0fab31ef525e00a155e227f3d17f58a \ + --hash=sha256:40aec5299e1ed71fbb988389059da381c3c1a60e0c649acceb2a35d9b128848e \ + --hash=sha256:43124fe172ada86d03ac3c8dc8179091341f6724d5e5d5b160e1587e4cd3761b \ + --hash=sha256:4736d350371337825cac1793c9f7c40701c32a03912548c2a7608e51679cbb96 \ + --hash=sha256:475d04d5192eba487a3e2f935976340baa24529046e9c1c9c7a3b7bf80445ae1 \ + --hash=sha256:4a409ccc42aefec904038695d5d7fd6d8f3af2721b7b6401d55135ec7d6d298e \ + --hash=sha256:4d718fa1b5f0d0dd75e86fbbc5b0c93ea3a5d65216c85c61cd5d7cdddfe08455 \ + --hash=sha256:501ed8b02a5990c67a91c732843609d43a6be1f7576fcdfc867331239f37fbd3 \ + --hash=sha256:539c2cd5fed0947c135cd7eabaaac55f48300dfa1de0f3ca4edb5efa6606f471 \ + --hash=sha256:557a4e1708df428ebe6c3081c83a273d275dad2446ce0d81fc648ac71afdb18d \ + --hash=sha256:56d834b74c993a7d7cb2b8ab33a0d55c3e0d4a3d2f2da2808a4ad3d79189711b \ + --hash=sha256:5800368526647146978389dfaa46da3356291e9f0fff9a4ef12e8c2bef964a0d \ + --hash=sha256:59c123d0e948d760a5f930f316cfefa07e8d632ab84327c0693ee6a88171154f \ + --hash=sha256:5b30ddf7234e611ca877575b62840e6af5977f92f1f9d532eedbb05a44ff8004 \ + --hash=sha256:5ecace251ccfa705bf3d7d35c5032cf750f5c629809740405697bce5c118c4a4 \ + --hash=sha256:5f89dad732280e7a10b74d40b91364f88e13c3f2c08c2ef83a8cd42f7a61af2e \ + --hash=sha256:637f4ae36264bd7b8d9a60193acddc1d735ad52e8ed53a19931ea6d921fea8e5 \ + --hash=sha256:63ada7ee2e9345695f9e9bc4c65d72222253f07b1ac94fd0e37555cc6f3c7f60 \ + --hash=sha256:6441cc837aea58be7d9baef1b2383eb8311ab9303f500f99ac90b584cd78bb14 \ + --hash=sha256:658f90f49cf5af2441cad0a2b801c3ef520471989a1ec55bcb25b255b2ca8d2f \ + --hash=sha256:66987aa68b0f7c2a1cc5f388ca962b8ed92b10f79de38d6d0d8c716154f519d9 \ + --hash=sha256:696477ad71385c4795e3b8e4cf10b0d2c28c2a1ca6a955e031cb1e62993e9ee3 \ + --hash=sha256:6b7e54fd883671d1a8810704851c044b7173287c552b9f5c3d9e0eb9f00ae194 \ + --hash=sha256:6bc94fe17c3c56e5418f79515b786b101845f70609b0d19d0c1ba13448e5633a \ + --hash=sha256:6e6b7e3a1520c39a2772f414cd9ddf5995f77e4e823dfa1522af383addd465a3 \ + --hash=sha256:6ed30be8918e18c8bed0a2e8b70639ecf02feb61ed00ca2e41cfcb2a50fa3f42 \ + --hash=sha256:73533644e1f69ea1164d56cc6505f564c6c44072b646b66d67df2d31fed0348c \ + --hash=sha256:73bcafa21a78d0776b3ee7cd2a63c66f968eb7db8e8d594e32d7329950f6e828 \ + --hash=sha256:7814bbee202acd3bd240204c17d8b87a4c48c81064fd8674dbe527d94d5a4290 \ + --hash=sha256:783ba7d845d79ce976afd9c1e91a4e5714671defa198ee789e8b23316083a485 \ + --hash=sha256:79da2491348b30810728050b4a8ec0416f85884125c2fd44655b3d01150d9c3e \ + --hash=sha256:7a0c98f6a636adf0d7edd60c61589eaf52d149239af754d0bfeb0effedba53a8 \ + --hash=sha256:7ab379f95caee071a37cbd8be86d4c65accc651d391f9f97748fef006f38769c \ + --hash=sha256:7bf6478188f4e47bf5686e8a33da4ae28bf43b1b2528d9ee144d28492bfac60b \ + --hash=sha256:7c6eecfab7ce4cd9487ff8ba936fe38cdfd68c04faf3d5c710363c6a8e695659 \ + --hash=sha256:7c708566da8014b120a64b1eb6d200c6c0c8cb36296383723cdb6fc82038270b \ + --hash=sha256:7c8d5882ba25ac0282258be435d8a05aa0cbcacfce15799154a338f847f159b9 \ + --hash=sha256:7fc59e9ba821b220944ccfe0f89c9dc4745f6d097569992356eb03869f21e953 \ + --hash=sha256:7fd79c521f6290c69125fa2b85fa65d9e657e6a8ffaf722dc881b926bef4aa5c \ + --hash=sha256:803f8b575a71b1b299d677c28db0653459c79b5308874efec813f17b7457c7f1 \ + --hash=sha256:8050e75af7e4c6e2d5260b84eeedb618f3e452e66473432e085b5d1b81429299 \ + --hash=sha256:81cdc537e0a42e3c0170752fcadbe450246d5c3b4b7231d6eb9672456605ac94 \ + --hash=sha256:827c92145b3b976b39430129c89d213b250dacbe5fce678e9a03940e6e848983 \ + --hash=sha256:854fd2f1bc6e8a56b89910b5cd7261a8b40f13ebb31572da985ce59c7da0886d \ + --hash=sha256:877ca17fdcfdf5c397493a71e5ff97a87bb181417fe717fdadc77c08c09301ac \ + --hash=sha256:87cc632c88ee5dc80e12681047839304d98ee5c9a708d686505767001c9b8b9a \ + --hash=sha256:8879510a76940670517ea1cb589978da44b86e286ec3e50d664ef330817afce7 \ + --hash=sha256:8885e3808aedbd6725b921fb67dacaae0678933561ddd47c2b01315198c70e2e \ + --hash=sha256:8b193bf7a443c97d81c47052f60c486071a4bdef4a573fa2514f920089414d45 \ + --hash=sha256:8ed78ccad4c7b421804f5d524b7740946a7ef75d89dc0dac52e9d7c24c472410 \ + --hash=sha256:8f2973bbd2bebd9d2e0cd6394c1292a1a19ccd56bdcbe1e174059f1a39be5b40 \ + --hash=sha256:910d4260512660484c0dc1588a316fbb35a40c081c36fc51d1225351af17cfe4 \ + --hash=sha256:95052e8777a86bae87c0bd0b5ab22d809e3d1d02bf69e3e66ddda5ba75a05805 \ + --hash=sha256:95f273bea318a194f656527ee2ed19494327bc500b8e87d9358e3222579aa28d \ + --hash=sha256:963a8d8f97057082679523d0fd4c53a38f86bc58cabe4556faef682ae53fa2fa \ + --hash=sha256:976fd7689d69ec78d67d31d38d396d8adb562f7e8368279f76aed4aa451fa06d \ + --hash=sha256:989261c5f1735a165f2e4e87cf6d5f17ab734fa18f9ad0383d5adfcdaefce701 \ + --hash=sha256:9ae9614c317c50836689ce2dfde07c05fe0b16378562e2221746c3913ede3c80 \ + --hash=sha256:9b0124b9c17e9890f0819b2e7a5f65ec9a2f5aabd6c8e7ad090c1675cf70dee6 \ + --hash=sha256:9c4880d017555d70dea367dd49271830842d48e3891c2da97da7ce8c4abcee40 \ + --hash=sha256:9c62e71e6289d78c0108d8aeb495f8bd3cad4bc1632fedddc9297ddf287ecc20 \ + --hash=sha256:9d0a21cf76153de8f2d96a877991d6bc59b9ab5180b949e50db73cc193b4a694 \ + --hash=sha256:9e14d17773b1b3c758ff153659a1824608a0cb562c45f484b5ed8a433428444a \ + --hash=sha256:a16a1dc8529f9e734a41c3b856f3eae7ebacdc061dde3f8a844e0c7889c97203 \ + --hash=sha256:a2212a0c842c723d919ea4a22a9296cb6b244b386e4e6ea92adfc7fbf3095519 \ + --hash=sha256:a313bad717dde740959d50850c315b75fd4eb0c5e6b4dc8535db0f1c369be125 \ + --hash=sha256:a32b78c1e52ebd8e247bb68300b90b233300d8816faa008ed0713bc539fb6af0 \ + --hash=sha256:a3ffe881246d28a862f1985824f484cb7361f44d6b99c4c620436ef56462f38d \ + --hash=sha256:a49ff5cdb33654cb7d6a3c377aa2a83ddefaa1db31eb10bcf3c180aa84f9af8a \ + --hash=sha256:a8b75dd3d3638d9a19f23e84af4ffab3b8940422c0df2da2a77005ef5aa3d7ea \ + --hash=sha256:ab64ace1a68682d191d9bedd9d4c939406ad86b1d9f628180410644249ad46c2 \ + --hash=sha256:abc7c2e4b47bfe6a9aea434d3fdebb9597ee636e914e92ba352f2068b9142f4c \ + --hash=sha256:ac348379cf4de5538a0a213be1532d289aa801ec5d267c5909446b9ea2f8e2c3 \ + --hash=sha256:ac51cd64bae51c462ea58ad2492c9b8209667a4ef60c45c4a304518b67598d5d \ + --hash=sha256:acddcac38adc8342ba48aba98896faa7928854bebb62542362138655b5367ee3 \ + --hash=sha256:ad4528cdce058b684f75fad1faf4a6a6c992fe2f08376370ca66e4ce5916a84a \ + --hash=sha256:aec65b53a07f580606593f877eefbb29a45939bfc0d3fe6e6d9f42b41b749f68 \ + --hash=sha256:aef74e9beabbd6c4aafc091dabff86d046ccf013ce1e4396c0fbb01b4cad9de8 \ + --hash=sha256:af1b5a92315048c3e36bbebfa7d4760a9c3e910bc4166f11b20d77d20d6bcfca \ + --hash=sha256:b22ff30006a2f28f8bff878fb93413cbe3a4d1fd517c28081d848c90e9cfd2c8 \ + --hash=sha256:b2483da477932ad1983d1d33c18bc3771c6fb00cfbaaed70a875fd547ef8e840 \ + --hash=sha256:b2f0adc22a4eb31e545221d93fc73a0f6a8cc2379d0f4309f71d1d17ba938b82 \ + --hash=sha256:b4c9e5d05b126b267ac048a89a0e2d9b48b1b648add62d4872906304a8590610 \ + --hash=sha256:b4da208a63434d21a3df64d29758e650fc4aa8cb05848554b76949c296539cca \ + --hash=sha256:b5f8771aaaed7f80e84a4e471d2f29ab6721e4595075e54d03ae1ed951b2000a \ + --hash=sha256:b65121091567847a8cb520d364ab22ba90e00d3cc55fa9eb34bb439f0684bcd1 \ + --hash=sha256:b66ccc5c2cdd26e74fa5d4c29ffae424cc6148bf93ce574821783fb3b6d452c5 \ + --hash=sha256:b69651732c64afb691e50cdc3387cae305e0eeff8804fe3e3ce203876494932a \ + --hash=sha256:b828cf64d62dc09ac183f03c1aeedd164ade96a2ce4934109452edf29de1dd13 \ + --hash=sha256:b8a65621b98984a62e59403009591b8a5a7736273aefe1cab64cfb85b365cc07 \ + --hash=sha256:bb0d664505f4b112f384cffeee82e91e3f6448d8e574989479db4439b68cba05 \ + --hash=sha256:bb58ba73a3f96f9a3e46b1fab69929d7edbbddb3133ee74b5c3c54074a53c4f1 \ + --hash=sha256:bc94a68ea5e18f8e85dc6b522bcb53093f692c8eb62b4837ac047da73956cbe4 \ + --hash=sha256:bd82c4977196681a499bb6ca9e462afbc5c91c1c15b6a991dfbd72733fea4dd2 \ + --hash=sha256:c0c88085affd35c33e124e36930c5ad96aff9294ce195eaa0fd9cec962b64a82 \ + --hash=sha256:c148e8b596000dd3e4bfe206e70f3e666be18d72032e0012555f2373c52e35d6 \ + --hash=sha256:c57541034d12b215ab0a2bfa371d1a8a198da18176d0426c27105b9161a6862d \ + --hash=sha256:c9648ed33dc8179e4ec04bbc73bd7f0038e1e81217a69467f61f02a78bf07e88 \ + --hash=sha256:ca65cced0d67a9039e93bcd98a369920e499bf98296844ff56ead01c9085321f \ + --hash=sha256:cf22b43f35b7dbb9f71e8ee2041b5c00029cdfc28ede3a2f31cf8906f9a6c126 \ + --hash=sha256:d35a4f1c63f07fbb8c8f9946dea98b21eddf6c57421585f71d91864be3ba2a24 \ + --hash=sha256:d3b6e6840421c83ccb60398e333b44f910b0907bb409597685ab2eedd1e22eab \ + --hash=sha256:d73fed4e37158ff00cd271871170b79e40138db51e625fd352fa17c6acb34f67 \ + --hash=sha256:d7bf9e43282d69561618e8a0ea33368d532ebef42f15c096f427090521dd74f3 \ + --hash=sha256:d9d6544790ba50438a9c1a903c3c4afb1ec8a7832db5518549275b40ef0dd4b1 \ + --hash=sha256:db4d697b18b6ef5528b1f36bfa25072cd2a421869f5963bc0e92c8a34b9e2800 \ + --hash=sha256:dd9a137a4a9becda3094f3831cd026380f75f6855e051eefe4c73ade524f1cc3 \ + --hash=sha256:de7738b8c0bb74c4cc16bbd7fb49fc2bcf6430dba11b3432cd52768ae40933e8 \ + --hash=sha256:e3b1aa25f01238886a6baed9e13b9a9240ed344346c79ae280ae65e8603b21d5 \ + --hash=sha256:e58952f04772f59f11c6e007471449809a30165188669bca8fdb19dde40a8f24 \ + --hash=sha256:e5ccad4b7bac125722f48d6f862bed3b514d8526deea06316bb72f152cd30a7c \ + --hash=sha256:e7386aa18d98d6b8af44b92173654ec469237fda35f8e8523e43b581a86f476a \ + --hash=sha256:e81ae656b9935ac4528a71f96bb7a14d949778ed1897c573d3e7ebb9187f8841 \ + --hash=sha256:e8f1e362c9352b50ed120f001046fdbb80810c9d56580f4c3fc13bbe30823387 \ + --hash=sha256:e96ca64383efa107262ee3949f047af5ee4f1845ba09463466c04d35a83bd3ec \ + --hash=sha256:ea999ae6e80e66ad5eea287860951b033d0104ca34d6d87c7b5125ebe0e12721 \ + --hash=sha256:ecc89dbd4155b2f8a47f4bbd89242a35ed15e8e0ec581cab5ac65fa38407329d \ + --hash=sha256:ef01d29fca550ab871fd99154f82c6472aacf8e7def272dfb07b460123850390 \ + --hash=sha256:f04551dce5a7db8c9659f2e4245494c182d0663b83661803e08d46bfcae5eda1 \ + --hash=sha256:f0700527dd5bfa8b7204b08330542f4f388899d3c14d885d8a368992e0eb562d \ + --hash=sha256:f2524ec55b3e65cbe235a8b3c36e2af3b635be02ed05e20c94e70c5c943c009e \ + --hash=sha256:f5844e7befc707367807586f550fa97e23dcfef0728f02b98c7bc498a961a5df \ + --hash=sha256:f9dad513626a33670f17cddc6078e30e311f444c957e8dbfc5b2b4603c8b4edb \ + --hash=sha256:fc0dcb22fa9aeabfe3fa4e0430099acff985ec5d77a851382f76cc6146e780e5 \ + --hash=sha256:fd882aa29bf402b62bf1fd7c19fd5df4b6528cf468a908864b39368572b662a9 + # via + # aiobotocore + # aiohttp + # yarl +propcache==0.5.4 \ + --hash=sha256:004e685b315646c410771836e72a44f143bbe624f29653a42687815069a303d5 \ + --hash=sha256:02c0a34f16889cf800f10f0247a564d8ce6eeab6ffcd7c87198f769067eb8432 \ + --hash=sha256:03969626faf0783a592dfa17e28eac06018bd0b44dafae6943d53b92421a7f72 \ + --hash=sha256:03b229037d25b801e7af53fd52b9fc49d9439b036fca1e087e02780631adfa97 \ + --hash=sha256:0951315a6b3142ee2167404d707743f0157c110091342b1aa0accac5cf0e4acf \ + --hash=sha256:0a095db8e15a6020db149ecbed6461939fe74f6acaa3ae8b702a1fe8c38cd983 \ + --hash=sha256:0c889f6fa84957bc7e8b4eab71fd16a0455068d5045e3aa40c733071d2b2fd77 \ + --hash=sha256:0d21d0d2c82bbfeb1677a9711f38df968f9837576102bb4add1bd449d28d88f1 \ + --hash=sha256:10ef33a68a61ce317e095fd2e202a592ea92392b90944a78c993f0d9a73ab06c \ + --hash=sha256:12682126712ddc19b70ff819debbd279e58adf1f0c8f8f8138c18ade2044b284 \ + --hash=sha256:135036c5cfc93864affb0f9af9a27e5d7a71cb7bd745e7b6dbfc2d56cc30e827 \ + --hash=sha256:13e52b6e0bde97dee98ab66552dbff2931649c96f1ac432eac299fe689ec373b \ + --hash=sha256:141fdbd73748db0cf7636035030aaac383d2efde8f34e7bc24594cc776d225b8 \ + --hash=sha256:146f48a9e4812611a7581003b1a39de56c34967046310c4171a68ef908c9a745 \ + --hash=sha256:174507f82d3594622acb1dd2dafecf2d899d6d506335494e7107767bf05f3aae \ + --hash=sha256:1783582065a1f07f9d9ee1e992e13f15d7dc8fb1eb3a7476d43eb3f2e69d26bb \ + --hash=sha256:17a7400cec0256f0a71ae71f9da398f9894c956ff6668a1c9d317b3367316320 \ + --hash=sha256:1b2f3bec4261a94019575481c726c29850f72e27907773c75b1de421e20e9f9d \ + --hash=sha256:1d759d05634f1b038fb625a66662a8c85e5a8fec912da381b5149ddac107482b \ + --hash=sha256:1df8d8561b21465c5dd56110a01caf897e026d065b4b84e98a488209094272ec \ + --hash=sha256:213bb68d9ced5cf2bf717b1071bf2b09b4b04c426256f9fe6d054c60318424c4 \ + --hash=sha256:23278f808cd81d5ada7184a76606b925fb3389c60e1077b2cd7da7b1fcf0553c \ + --hash=sha256:251c63dd46a0659bb875cb254dc4c1e79ee91a847c737cd62373295afc2235dc \ + --hash=sha256:279655a16973f1ee2bd2fe79973137681642fd9ae0d89215bba263726eb0dc3a \ + --hash=sha256:2814ecd8e818f487bee4b0f921bc4d1c176cc5fc71ac0f072d0fa67eda4ac14b \ + --hash=sha256:286867fb156488c251a3721766e380ac4495e4fd6b51aaa1403d89ce7f4359d9 \ + --hash=sha256:2dba2f02d2d5c09ef8a0e6c1a42aeaa451f4be9898cb00b04fe98717da2eb23b \ + --hash=sha256:30cc1cebaf9aef49db06357a50398323ae04d70460c0491837d026ab7d6452ea \ + --hash=sha256:31eb43ba2edc704ab2ec27815315dd8a19def0fb16215be4cfe8d32fe78ffd51 \ + --hash=sha256:350b272b2279f4135a64fc0c304a5d08e28a137c9573442c606152446638a831 \ + --hash=sha256:36c0d9db44b523ef93d03341b1c42d69ff01d673c053d1b1c6c3a363bcaa39ba \ + --hash=sha256:3af0c8642b2da4815d86e631232ac8286e17644fad907c19508aa8e7cb4ba8ad \ + --hash=sha256:3cd3a7edb6b95b9b33998135ebfa18d709da82290fb8f27c858970b5a12c8b56 \ + --hash=sha256:3d605bb239b796e82a81c6709548b2bd460ab73b4590cb0c83de8a2dd9694d0f \ + --hash=sha256:3e413d7a4a9b4866b7a761d6060d434b64d23cd35122eda3b026a0bbe8196b25 \ + --hash=sha256:3eb2e820e8e2101407da93f17c57cbb7d225461955fc60105daaba14cd421ee2 \ + --hash=sha256:3fa15757fea1dfcd5b7745cad9f4638929605531bd4018ab2adff7955f1a403d \ + --hash=sha256:3fc24f209c1b7f7f688b66b98293954f5504279760999b58920ee12dd8471c1d \ + --hash=sha256:4054acf80d40456a0537f2913b349718649d8d6458a14ab7f48d0ce28c30869d \ + --hash=sha256:40e94adb1e7d39ff28a8bd8d8b8fbd1df6b9f40976dbe379134f1ce058e532dd \ + --hash=sha256:420162a77f94eb1cf5ef7893f500016dabd548e73de956785a1dd899cc73006a \ + --hash=sha256:425f8cc86ab5018b4b8d4a23bc8e74d964bd3d757c3702e301aa79be76c53f6c \ + --hash=sha256:44149f46500a0a41b95b4d99c2e586a77319539730607b9892974a092788b111 \ + --hash=sha256:445ee3bfb46e85838387fb3c536a73cc0b994dc192b004e40e170adc54aa2a7e \ + --hash=sha256:45488d1a5f9ab5bd90aaa1ca20f50fe1922b8ffad71a2009d2adf41355897aac \ + --hash=sha256:45bebbe252550fec975ba3b62bc6f931643cfd3b5464ef47619cf3fef154e01c \ + --hash=sha256:45bf2e730ab8905d0527fe05a86500f406e64305c34cc81ebe64b4617cab9760 \ + --hash=sha256:48cb48c5346a97de792254af77715aa2529c2a1ebc5f586aa0aae44a02f1fe57 \ + --hash=sha256:4a1f4f5ffa55dce6307631f3cb2948e117e665966ea512e0d502b16c24f567e7 \ + --hash=sha256:4cfe0a92ae30151869e67a4b5f5e105e4e03ad30b3f38e5211b5bf77d0881993 \ + --hash=sha256:4d86476a935c88963d9b8e1a9a0d38188790e9622169bfbafa173046846709d3 \ + --hash=sha256:4e985382be6d15da8d0c2710a6fa7b9070fc9ecdeefb7f580e88373984ec8be3 \ + --hash=sha256:4f2d880ff60f45898f4acfa152aac8d04e3ee627d90ff4003491bf92239d5757 \ + --hash=sha256:4fbc1a15dc8cd1689508758d626b372b1f09d28d9577667feaf9e6bfcd8efcbc \ + --hash=sha256:50e337653721d20ead710da33bf44487fbe8a0db8782714b60306481e9f95b51 \ + --hash=sha256:53eaa697c4d0422ff4cb714d00231b43352064d97b944033b30c1d57cc506ec0 \ + --hash=sha256:56fc3f7599528db40b1efa0889a620116e2704144495273d66066e8164e45838 \ + --hash=sha256:58134228927cee6c047d626c08e60a81be604a20578a12ce752cc5c9a84d4826 \ + --hash=sha256:594eb4c6ec35e7179b058481f4e9f02521b56de16fa577c4b85c76fb1bf8a9f8 \ + --hash=sha256:5cacf3c9efd09df409dc33654dd077e1c245ba8fb747b0f0236ef41b7c49b589 \ + --hash=sha256:60a64cbccaa11b7760ce705a14ada17ba459e7ca9f23ba587eb013821032d7ef \ + --hash=sha256:62530ca89187827e4a4fe733f971abe81a7542eeea48ff61995f19b64d7199c8 \ + --hash=sha256:62c60aec739ed00124573cce1178138fd690c7676352d67a37328c1cf51d7468 \ + --hash=sha256:69fc35c0779522da366c563e5faf203ffc1f8ff0021d5b1337fa4efa5be73177 \ + --hash=sha256:6af4693716bfb03f1752ef1b30faa593db2c01d5272e9b8564a1549452a979ab \ + --hash=sha256:6c7599df2b57ebeea8de011b5f2f7b85de95e76037d43d34b95e328430275487 \ + --hash=sha256:6e9368e87a3efc285e559131092c5db643eb8e56de4ee42064d5baec22ef2bb5 \ + --hash=sha256:6f0093ac3e9daada202c2082439d414a625c57184727a46e112a3fb2a81cb788 \ + --hash=sha256:7177c43eddf10a0893c4fec52ebb408fdcd7f7d63962caace9180d8f81b14ece \ + --hash=sha256:720cf832eb2d0b0dfee129cb3335a26f6ce3cc45ee1187e8f0731758caa16792 \ + --hash=sha256:770e8209d018175fc0063936fa9583b6d27e88c5ad31543f3383d66080efdd62 \ + --hash=sha256:7a8d5ff04eb1f85698a78d20c62a14676e7b960dcafde09a388d60ad377d355d \ + --hash=sha256:7b9100a93b372418d8688f3f2a3e5b45c64d70ca4d6176e121aca1e3bfc1e32f \ + --hash=sha256:7cc528e760a8af06f2b13e9b9f362cd90c7c718ea61228a96dbd31ba16ed7f47 \ + --hash=sha256:7ffafcbfc7b549ab940047e505c831eabac5e67de53e1bc174adbc5285c55944 \ + --hash=sha256:87a3caecf8095e48dc72f84bfa42e23a848cf410cc9cc13031fba4869b706a21 \ + --hash=sha256:886b59c4d28ca97dd23b025fdfc50a0356be934efbbbca89ad26230067f86fe5 \ + --hash=sha256:8876b39961e33d912afe3c1bee18ee564fdad0206f873cc15d522756b7f50737 \ + --hash=sha256:897d1ddf6716e8f47200f7aad9a0efa6cc7586df66c6defa572f9eab379c078e \ + --hash=sha256:8a1fc236528c457cd739c88abe823da851b7ab645d72792f88658114cc340c12 \ + --hash=sha256:8a235f73d6e020855dc29dff012d920c02ee0feab8d73a24185a7569f4be1161 \ + --hash=sha256:8f911c395cef73c510bac566da9507bb6a43e7763d0c79138dc60ee53f11207e \ + --hash=sha256:96f7c5c15656040ddcbc51e56dc59b58aa25999d743c126abd425b9766ab43e9 \ + --hash=sha256:978f28401afbc76cdc3df9e1717b4229a06b626a1dcc75db4e1f2beb3884c3e9 \ + --hash=sha256:98914de2c4d7f0f9f4a8c6ea4bf05841f4175796941e3ef7d47eb718f22311fb \ + --hash=sha256:9a2a8a50a93dee0268a860a07fa3b4bd968f8ce4dbd794957da772f395368526 \ + --hash=sha256:9cbfff4423eef4cc6cafc021469641a2b835f610b2647a6c5281903e21b8670d \ + --hash=sha256:9e9ab13760aa8b6d0881ae7cb04fd891d8d490cd2554ea8e79bb278399169bcc \ + --hash=sha256:9f3551b8a35c1df3e7ea4d2d86edee15f0dde1bddd434a71744048683544d0ef \ + --hash=sha256:9f86f7259efe2c951f43e57d471c9b41daa5bfc7db9f67189059cf1ae6d77fd9 \ + --hash=sha256:9fb0a5be8d9aa213150e8d8148a42aca4984b285bcad1e69587dc4298edd929b \ + --hash=sha256:a219f0ac59817a9114dd2aa57c13180f993e819ba658c7ddab4b66ed1ee0d370 \ + --hash=sha256:a419ee85e654927baabda3929c03c0cc1112bf472ff0dfd6142f4e3a81ca4162 \ + --hash=sha256:a4d7a54719b67338a305dca2ce6aafe366817df94ddfd4b5514374356f5ca546 \ + --hash=sha256:a5793c7698a53f56f4a1889a4737c7eeb1b7ad0842fa6b1abca22913ff79c8c1 \ + --hash=sha256:a5e8ef588c109725dc713ba69aadcac00a1ef90c2ce9c0a8c7075128f569f47f \ + --hash=sha256:a74bfa37147cc08fb29df10bd9c16f40fa7f860cd3a6d2fff853323a94f6e17f \ + --hash=sha256:ada748108a43d29b7c328ba7db3755327cd94f028bcc1a7ee3f0addcfacd9c38 \ + --hash=sha256:ae58f361bd5dae942717c65d3413b478c70aea9c462599e7b9adad3731db3894 \ + --hash=sha256:b28f41fa3b8c6900457f858ec5b03998f3a6d535fbc1bb2edec5961ea05ec429 \ + --hash=sha256:b3083bfe87f95c756e610bd8025f26cbd1cd4aaa03a422f2d65efb7a97cd53d8 \ + --hash=sha256:b61805357d966680acf68b3b6d49772631ed9df44ebece10ff1460e117a7da8a \ + --hash=sha256:b77c313314524ca9c38fbd70f73515d04597ac58c40c939bc0e71eeb4abff680 \ + --hash=sha256:bee7d3aed13d56f54e681df38c3a23031bc9e3863f687d9d598825c9146acd7d \ + --hash=sha256:c02c0e570c5c7e077b0181a9f3cdb7d4c3617d1cda6b5c95bd5d34022923d82c \ + --hash=sha256:c174bfd1c48a1b51a3078e95586dde718374bac79719ab3541ec9e74aec40574 \ + --hash=sha256:c2ba30a89035b57b73e00475de948521602f543d79ce01db10b04b36c4c76fc8 \ + --hash=sha256:c3e98c55bde2bcf7db3c70d1aed7ae9aa8aebbf19a250c66645cde44cdb8b867 \ + --hash=sha256:c3ef2818d63bc86071e9d2989ae75a1bc32b8f7059cfd9f5abbbee70c32e2ed6 \ + --hash=sha256:c83acbce9f2b5e3f5f5eda9e53d2001fed22fcdfef81274a9e02d8fd53b70a30 \ + --hash=sha256:c9281e922c072158c91974d4589f1dbe0fee6d467f284c28e463f9f5a4d933f4 \ + --hash=sha256:cc07876cfb079b6f6f36d21ce75784ad6c2c6b563eeac0ed26c2fa2669b85df9 \ + --hash=sha256:ccf4f7a79e26bb7efb06ecd50c177833b71df05cbc748701372325e6bcc17f6f \ + --hash=sha256:cdee8205a44d0be91bbac4c41b95d86641b72dfc7aef1279400e4fda3f26a937 \ + --hash=sha256:ceb3e879afac028f93d272c957814695dc5569e4904262dbee92f6c41bd5e4a3 \ + --hash=sha256:d1f5a500bfcbb2c0ab85e98a0dcd70f5899d34efe365a0187700369a79603031 \ + --hash=sha256:d42a9a856a4a6e2f6c10f1318c07e7daa498d6593abe745c71dae4521a26ca39 \ + --hash=sha256:d83b12902eb8bce151259c86c03ba746600b2d994543de46e370cecf96c452f2 \ + --hash=sha256:d8e017eeb7482bed34cdb0d61cf2bcfc88d104bbab296a17cd16a6af8aabc70e \ + --hash=sha256:db3ae52ccc150dbc84704e9d642743897f3e1c54742ff34cacb661e52e3818a9 \ + --hash=sha256:dbab5f5ff6897c81f355d079010cdae85b02e5a0b518b5251523b8ad8ae9ac3c \ + --hash=sha256:dc4242ca653c9b30ab51c5f8193323e7bc0928f897ee9103201e59a43abcb72e \ + --hash=sha256:dcbf346a318a5e30063f547630b02bb787ce2f45b6368d5da143660b6a3835d8 \ + --hash=sha256:dd2ac8f5b643454c2cc6b6118b13da16e88f4a6434fc3ba61aca384029f04f36 \ + --hash=sha256:e1d52a05dc417279f7e5c7618c5dfbbc29923aaf9bc0a5c1802ddcebf54c61a0 \ + --hash=sha256:e6720ba44ad7e72174314d0e1fb0172494cff5c73a3a8a2159c3d2402ff15565 \ + --hash=sha256:e738ab81179510ce79b2eac9a6ecf47feffd9e76d1c72e403005dddb6e36c06c \ + --hash=sha256:e904d4d01f36bd6e197590be1533c44e06058771e0746dd073a8ebb3ef880858 \ + --hash=sha256:e9f165403b81fea7e89c932d89046a1e3d9a3a60e8d7ef2f249dccdcb0982bf5 \ + --hash=sha256:ec6a85f424afa8d23e0d9a094e5dbb6eda01da91c92b9183cd433768247ffc97 \ + --hash=sha256:ee19113bce2f3acd46432050688b70f61acd6857d75abb9ec96341b7e9ced123 \ + --hash=sha256:ef3b928d9c984322b5c44e6964d8dbc653da87d2d8ee1647fa6da43072e650a9 \ + --hash=sha256:f273dcf7149a50527c4fd1f55cfe9eac0f60753f5af544b4c9352578e20c0874 \ + --hash=sha256:f5470694918830da62fac9e69133b53d23b736d7070e587b27a4a2be37e08e68 \ + --hash=sha256:f574e460d1c8a08384a016fdb09ccf3543433263ed6b2f97104f979e64ea57c2 \ + --hash=sha256:f85915e00dcb1cd9f2f890ead064ed40a27df06f0db65be427b29482ae357572 \ + --hash=sha256:fc2461ecc45f17893f8207e73b46ea8ba93e33630e51cf4af3fbc21d47462b1a \ + --hash=sha256:ff6b113f50bc066a698db5d944d2c6dc7507168dd3341e255a8892fd0715a558 + # via + # aiohttp + # yarl psycopg==3.3.6 \ --hash=sha256:a1db9f7148b06a28606767efaca51fa6f9398c5c0a3810519be69d7000bdb631 \ --hash=sha256:c081f2250df751a943036e42db6df4571c66cd0aabe8291a7a506512b12007d2 @@ -369,10 +1016,20 @@ pycparser==3.11 \ --hash=sha256:51d5a8ba2be0bbe440b99d2112604c95bbbc3c2748a64260186c541e1729cd80 \ --hash=sha256:d875f09c3507d00e1aba0eecc6dcadc1352f30fff09dc6bff2f1c2935e97c2bc # via cffi +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via + # aiobotocore + # botocore redis==8.1.0 \ --hash=sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25 \ --hash=sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb # via -r requirements.in +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil starlette==1.7.0 \ --hash=sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e \ --hash=sha256:c79f74ea63cff761804fbbfb182f1e0b440c2d07b164d24700c5a1bab5d6ff5d @@ -383,6 +1040,10 @@ typing-extensions==4.16.0 \ # via # anyio # psycopg-pool +urllib3==2.8.0 \ + --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ + --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 + # via botocore uvicorn==0.54.0 \ --hash=sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf \ --hash=sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620 @@ -450,3 +1111,246 @@ uvloop==0.23.0 ; sys_platform != "win32" \ --hash=sha256:fefea5cf8cdda9053b962ca8a90216fb0b1d40907dcb6819382b42e483e6e9f6 \ --hash=sha256:ff7144d8167e513fe39fbb46bffb4f6f192dfb1f4b0b4e9102e1fd4f212e4747 # via -r requirements.in +wrapt==2.5.1 \ + --hash=sha256:016602dd8827d190280a707c5e67f9a80038f54bac1782cc8ff68a2a16c618bc \ + --hash=sha256:03aa7d2256309b57ddbf317bff2cae5f47e50ea9ae8d582780ebe0b554347b42 \ + --hash=sha256:051220e5071fdfb1a6678707c8abb7bbf4824d40f99758394b2b4d64855fb284 \ + --hash=sha256:0591e6eace0d186c9ef1ecd1244be5a04e98041424cfca425b684ffe4f0d8030 \ + --hash=sha256:05f6138d5833edf68d88f950ea71bd96daf0a9505b53abd48aa002a0b6d05765 \ + --hash=sha256:06740dbf984af8a26d4b63b75a6ee4e88846c068dc865486ad906448079f50d4 \ + --hash=sha256:094b847491b813b6e6c1775e03770930d75078c0821adf929ac712830951ef25 \ + --hash=sha256:09b1893ee4063706574c1813abf479b8b51926633fbdb6f96aab8dc7b0976668 \ + --hash=sha256:0a526227efe17dd94bd16b123d170f879bce42c15f10eb92495a745f54caa943 \ + --hash=sha256:0c9480bdee340a1602cae5a777146ab4be3e384fdcb569fffdf8721032314645 \ + --hash=sha256:129cab3c7b21e68e693c2819a95c47f3b1c41a834b931154688c83b6aef6bdab \ + --hash=sha256:12bee472452019706fa1d4ead093f52a9683b4fe6617953e15bab9acdfdc013f \ + --hash=sha256:12d3d2b9d6553df6e2421ab99e1cc5413509076788f57fcb3169f5ce100a19d1 \ + --hash=sha256:1425fcf0e70b27053bd610d57bae975856e7897e3f6ba1456d2b80b9d7fd15d1 \ + --hash=sha256:183bf0bb893f783c9d22f953cb01fababb9f618e098763f8e66337b575b0647a \ + --hash=sha256:1910be5adc0232cc6e8c0673bf3f41c2ee724547543526bed8d00734458e7bc5 \ + --hash=sha256:1a96e2671c60f9f09ae547b5a815cecb29af16caa68d73693387d0028788cb32 \ + --hash=sha256:22300c5f254627f24ad2197998fde26db6eacbb0f879162944bf7bd79dd5ee5b \ + --hash=sha256:22a9fda6ac53536ec74e3e334f3568af2535a3df1ae70e8f2816f77160c386d9 \ + --hash=sha256:25eb4d928a9abeaf70ca786a35861b46d1ab37cc4ce49ea70a070dacdead4dfe \ + --hash=sha256:25ed8b1b39234140d5b5c6a273130c7595e0abece417c3ca3cb378fcea5cd0fe \ + --hash=sha256:26313f38d18d40a9975123a4ebff9da125ec63ab9ece4f05320a3d8d37d2c1fe \ + --hash=sha256:26d8ea2ec6818aeb656bd8a9e745a6f1fb0edfcd8f54291ccd94f62eb5f5e3bd \ + --hash=sha256:29b62e87fcd6a1893f669abfd02a596a7fc5cfa79fa57e42c4e650a6c170c67b \ + --hash=sha256:2c642a83b6703804b571caa3b8b205aacd341b1b37e2b2d89cd70e03e0e9caa6 \ + --hash=sha256:36d7d0ad593c4f1a651e4032de834db59aee1a929ee396cd483895b673328e51 \ + --hash=sha256:380f72610181883f66b41442cfc7c0f7552b42169efb2113def26e6380013d37 \ + --hash=sha256:3cf273b7e8d2038abb7f0a8c6550aff4f617b9d486a9965c8e8acc96a3a04de9 \ + --hash=sha256:3f93ceb0ac4896de45d5a45a8f4e69474da583440589de10b362ddc1db4691ed \ + --hash=sha256:4b3f410c416752e1dba53d361e2e6562f22c2c3ec855740dfa5836e061b22571 \ + --hash=sha256:521bd5ef2a33171fac08a0a302d51a983c19c3519406c1ee8da7ce29285488da \ + --hash=sha256:5ad562c23e61e626f9d27aa37aa5679f1c29085de1f998466d107854048bba9e \ + --hash=sha256:5b53000b424dc2133eaaf22838a2352d3497f5d7c2e7d9a2acfe675ab7225bb1 \ + --hash=sha256:5be9816d9de88f02fce23cf55f392403411d9bd9c7ae57fdc965a43b22e2de5e \ + --hash=sha256:6201c7e122f40060a9b50696d80deec8f93b1a235ec0443f51d7a8a42f7044a6 \ + --hash=sha256:6405ff2160af9d59132ebb076eda0304db44d9d09809582932412ef7c0788a36 \ + --hash=sha256:69fd0fbb3daf7c8c6f5e062847a0061f880f347374d74cf1daba57220fb64cd0 \ + --hash=sha256:6e3eff05ae616671b40d7ad0a504210329e4adc9fb91415663570aca93c5f5cc \ + --hash=sha256:711e73da3d7983547fc9dd208973b6b0c52640822f5d477910ba24622df6ba64 \ + --hash=sha256:729d644b6acaf4846a4ef81b037857b66a01dea6d227f827c6d71c0b6d656d6c \ + --hash=sha256:736c1de0230c6d24327b14684794214167b2c5ebb6332e28a10f504641b600df \ + --hash=sha256:76f230a9b07e3cb66646d265398f579abb6128b1bb4cb97c74b1ae5d09e96f31 \ + --hash=sha256:7fa321270b40f3e8cdfd954b3a8dcafc6db1d8bbd4d681b92dfa6b9ef91a9a99 \ + --hash=sha256:8078186f719a92693199f1e06c4ec72e1e6d374c2e459da18ed5c39d6966d727 \ + --hash=sha256:859f67bfc31eb7ab55f237b629cd4ab0441b075912446481f910f7d02066811e \ + --hash=sha256:8922821f66ec08a39f72247776c6158db5bfaa09d0c8f607cd854bdf6b2a2c10 \ + --hash=sha256:89d9a8607b7028054bb6fd01d437f205534a5d59d53c3665d15949a99a2fce0d \ + --hash=sha256:8a7c078323e6e1534968cb85488c5eb7ee2b9bbd0f8a291095213a763da40dab \ + --hash=sha256:8bdf4696fb5bb141a7f96710ac6d9a6aa9a57a14c54075f9c7d3946869d457df \ + --hash=sha256:920f700ef41ee774a1e4778c1f4295e117f1ff3435a7e0cd3e997d10da819d32 \ + --hash=sha256:9a34640eb6295f33ca23462977de275fe8f3a50ab339b8918b96d69a7451e2e1 \ + --hash=sha256:9aa7660684d73925c0d1e4f8536ccbaf233cef3897e33a8c2ec462f83b338323 \ + --hash=sha256:9bad4dbb4e61624fcce5f301e37f9e743ecae4f1259a3777b3207eb7eba3dccd \ + --hash=sha256:9bc472825027b276d4bf678d2ac64149db0b122f80ae6f59c423e6d31f0c4bb7 \ + --hash=sha256:9f0750cbc2e29e4f3c9529d3587d4e7ed8f60638ceafb80b87a95833b0c5acd9 \ + --hash=sha256:9f437dd704abc4ee1bd03bb2d796d362d0e75915e8f3113a7900b3b7ec5f8b47 \ + --hash=sha256:a18e63910252eb75d8806b4baefbc3a03612502f63eab042e3741b00b719f043 \ + --hash=sha256:a1e823aecb3746b8f9e0aee2e1413887871ee2f5c502a3e0ef8d466dbd4adde1 \ + --hash=sha256:a424e8a9776c06aef6313af1d0e3fe6e0838af4241d0c09eb0a3b46f2c9a5ff3 \ + --hash=sha256:a88370a7d89fcb1c4953a87673fdd7b4a0eb14a1a4dfce49771f0c827ef44893 \ + --hash=sha256:ab6db7d2a18d366cc57c2228253cf26443190aba0a6dd0939b3c1e8ac6e29e2c \ + --hash=sha256:ad81bf81b0a0b6c6ec74169638202851962843e86749570c463eecc55072f93b \ + --hash=sha256:aed178902c2386d7c5d3d23eb96d32c100e34cb8c2390e7ece0e4901ae43f0e7 \ + --hash=sha256:b0c82c19baca8ddeb4f513f584f53f6d3aa96b1a273f1a507d6d70620b01ba92 \ + --hash=sha256:b238e955ba34ef2b8897f358b7b868b41b9a02ffd338014b62985fa91898cc4a \ + --hash=sha256:b40f814df9e106371fea48911814383284e99df34ec1aa1fdd9b07d2055345d0 \ + --hash=sha256:b40fb47d637df8da7b02d76f242688416c23e53195ea5748895db671c01759d2 \ + --hash=sha256:bc5c0203d383403043fb86c964bd0bab4fcbfb26004ff4bb9c6d02ebc1d608ae \ + --hash=sha256:bde5d1b37101b1e9dd3da1f35072e2e7028e9c5e3511f7d76d3fdd4d071b7663 \ + --hash=sha256:bfaa998ceeea4d0aa72b40cdd0023d19409504e244b439ff2aa9f01729341c5f \ + --hash=sha256:c25c594f58ecb676358d6d6b0ff068b8bbbc506dc831c6d17876460c66ce39c2 \ + --hash=sha256:c39c7130ea0702c4ab0faf12da1df1e02d5174305c17edf02309e2f058c4114f \ + --hash=sha256:c40f3b1cd3ff9dd9f4ae829e4301f0d3a553e3467058b8c3f5528fee2c768a20 \ + --hash=sha256:c44dd9881626da7d621c23805f26726f6b023cf3e9755f48d092bc9cbef4a8e7 \ + --hash=sha256:c4d9c76e9a16a8bae0bdcc57efabad499192565bd9a95258b01fb0b49a62bd63 \ + --hash=sha256:c6e6c226b1ca5402d7ae5fb34a0d21f1b49124fe4200e5884d1e19e53c47ac1d \ + --hash=sha256:ca7b967e96384abdf7e7182c79f71529997981ece8169f8a8ddb31bc5b57cbec \ + --hash=sha256:cab37b82ec328173222e4f9da5eec4f2ec9e8e506f83557c8be8e1bffad351cc \ + --hash=sha256:ce3889e3815f97d46414eb574bffdd9bdb41ff70f503097e2707615a87d4e92c \ + --hash=sha256:cef2a8f006410b6134a0d273ec037fea8cc7a6a914f1bd7555ad9788ad788c6e \ + --hash=sha256:cf63fffcdcd8c60f223d3967bb92cc4fc2e8b46f09e75b67a6a75e6f47c0fc43 \ + --hash=sha256:d5b665a43fe0d3b390cbdd3c003d61c92fa07bd5e3fb1ed3f47920c2d03cd9fd \ + --hash=sha256:d6d274ec50a5b208be75596dc44ea253e65deaa6ee3a600babc86dafbb957dfc \ + --hash=sha256:d800c7689154622b0ba2922ceca44a3cf2ef61c3b9a4c4eeb1d8b3050d7ededa \ + --hash=sha256:d90c91cb4ef83b2ff00db4e0a7bdd9602902504ef9b26d0f9d7ecf6cd05c7554 \ + --hash=sha256:da42395e7add724c1f7caf18a2977b1fbdfd5aab314e5622731f0ed66731eaaf \ + --hash=sha256:da847332447db5505162759a4cd5ac374eb8b74841fe97a98ef3de14edd2586d \ + --hash=sha256:dc401274fcc7b15b3b2c12df2ff34024a11925243a7d3daee91c6d7d14f9addf \ + --hash=sha256:df6e3a36170cda0d313be50fe5065948e7f12f3a181b38cbc262e9f2ee4824e1 \ + --hash=sha256:e089a22ff5af1290b8c759a610830bdb2a829ef9c3d7797e4ee32c2f795ed482 \ + --hash=sha256:e85a9db9e5a5ccc326edb19e35a5106ba16e451d570a2ec8ea9deb1ea52a3c42 \ + --hash=sha256:ea27bcf5c56b13463ba5b9bbfa4d6544997e47ba6db77c59a259b09daa802d4d \ + --hash=sha256:f063c696328408fc4f259b9d7d439398d36b709e12445a904e7b047f0a84c3c5 \ + --hash=sha256:f1630201b0e2a96bb26304b7adfbd91a4ef486abb5a4c48377444a0bed749f37 \ + --hash=sha256:f1c911818fb076910ef509f2298dfcb966a54a6ff068eebd459632102cf589fb \ + --hash=sha256:f280c115ea64eff3dcbd68a668ce3f63476a4ba386bbabb318017e286196ea2c \ + --hash=sha256:f595bb0185aab3e9dc31950c95d914f56ea8278810c3b928f3426e12ed6d27bc \ + --hash=sha256:f98eaf784cd12bc69c77af398084174531007cd81849c962163ccfc6e791f3ea \ + --hash=sha256:fc0eb73b450b53950b7879ac7642889c82918d17bd2d877fd7270348dfd5550c \ + --hash=sha256:fcccaa1484f7dd1091602970988ab741491f9f974013c844f70e45ac1196b80d \ + --hash=sha256:fd3f878a4aac3c262447ddf43c5f4c18fc67dfc3ba69c4fb1c7a4c4af96abe7e + # via aiobotocore +yarl==1.25.1 \ + --hash=sha256:0136d640dfa9b0523853e411430a99f8a91eca85774c6420285a33b755bc6de3 \ + --hash=sha256:03dd38de09bc213e9a8b29761eec33ee1d5318dac0e49d8af36e4d27830e23a7 \ + --hash=sha256:0a191bfdb30a79b98e5d175d75285f9fcb78bf0e46ba5efda042e1c72071a0de \ + --hash=sha256:0a66db89ea473abeac4b70523cafd94db3772380e565f9d28af7a179b7af71fa \ + --hash=sha256:0ae12ff2b805fa02c4dab838005caef735e39986322698c48588d3beacb65c62 \ + --hash=sha256:0f12afda4eea8c8994a76d4df1875c765194f5fbe8a9d197929ea303caee29ec \ + --hash=sha256:10b2fd95332f0d716d5eee3c9fb2ce8eada19082de7fee83d32e37992fd75c26 \ + --hash=sha256:126a2533570c554719ca40a1288fdee1700b6bc82e7131aa69fa85252d92e651 \ + --hash=sha256:12b6bc4906e11f5e1a1cdcb12296e7afbd366c783cc8073403cd2fb74334e453 \ + --hash=sha256:142c06c4d6a35ee3ec5da08499805e879cb3ca7c1fbfbecb0140fe72403818d6 \ + --hash=sha256:14b79a30a93a3ce2e8832603fd0ab780ada281b0ba5110b519a634f2d7d7d1fc \ + --hash=sha256:17c9877a89fb6e2bca6f9087eb24cd7fb434653946ef5075e470d23d49b52287 \ + --hash=sha256:192a866877a49993949ef1975864ad8728bea28ee810f6abe1a0729c2b500426 \ + --hash=sha256:1ab7618921a93767387a4b83776f751588f5b5ae9bb5bc96620e2e2e00bca868 \ + --hash=sha256:1e80dcf1446e1b080b1932b0d103c464a04112f5bc31f0f983ad418172063cde \ + --hash=sha256:1f51020b2eb8a003c84925638ec63c21a750a4bddd3a22ec8eac6a742dadf1b9 \ + --hash=sha256:1fb2a01ba8cd9c5d2c5dc1ec35e0fc951d04b4f037541d4ac090c993ce58b3d7 \ + --hash=sha256:2239a02249d9326655419e0168a28ca9008938eaab31dc29fc875c217927a6c0 \ + --hash=sha256:23bf5b403c879a54964e0feac7285688e04bb220074878d737d331522da0a5bf \ + --hash=sha256:24ce942011a61953e7d313438038f4d32ff21387b775f58a957f7a07dd55ef95 \ + --hash=sha256:25868beca8b6765f8f7d0e11fe6dd7c66dd4b0793b9500286d20cc92352126a5 \ + --hash=sha256:287e99ff5aa4dc1c7630bfc683ded6f106d756c99dec432a2d7f197a784f51c6 \ + --hash=sha256:29273edf1530e397bd07cb784db1fbe0d2590b77569f2e24679a9c0a2d763b94 \ + --hash=sha256:2b49375d22299b0a834c2bca72f39aaecc270d96fb24c30424899676f487b22a \ + --hash=sha256:30eec96e8a91bd588ce897c9543f6d5d8d34b28fbcba28a4dedf20ebeae9fe57 \ + --hash=sha256:319e070a01db9920fb63761843f96a104c8e2b9427266731810dc1e22595b17c \ + --hash=sha256:35dcbea443fafb3eece757ad4e514560ddeb6c34cfae1582c620d7b293d7feee \ + --hash=sha256:3f4d48a6112712973e676bd792121fee470e432d749177162d9949d5c9460a1b \ + --hash=sha256:3feb99222553a8cbedfa52c2f59dd84c3f50d5b582c728d522caf8d72769a54b \ + --hash=sha256:419f392a1da624877975709e3864dfe833af6cc7671b39318086d456e288380c \ + --hash=sha256:42a66563d8cc056ee32e6191e05097a7b2b3bc302e0bc3133daf8710eb18bd26 \ + --hash=sha256:48796ea00a303961507dc6c8437c4b325a6fc3f95f7c36c71b91ea9a8150963c \ + --hash=sha256:4bd6340d20ae2c7ca719b87b426e808e90743b676d05d4c26c4fb5ca71f41184 \ + --hash=sha256:4ca89e4e21854ed27ec753297dde84b16c9f8e53b14a4866fb44457d643c19f8 \ + --hash=sha256:4d781294bb815ecb5ea57ff6bbf8038e0a31a95fdf3e1788f66e0dc100d64b58 \ + --hash=sha256:4f1c91f5a5980a937ff8e238e98e6897e1ad74a4b1e2c0d68c73b5ffbb3f5c0b \ + --hash=sha256:564fdc7085d2245ab84f88882fdb1d6ac0723124bff6ded35bfb1c00f812630d \ + --hash=sha256:59ba3a6e1aa8cfe5adf4bd270fd965db21955401b7ca6f1696010c55ed4daec2 \ + --hash=sha256:5df89f769cc8ff94c3d7e7603386fba309d25ce5240132d26c15baa8d0e96c4c \ + --hash=sha256:632da579b2d879f6bad20f2cfa35ded1efe2f4f77f8abb26a6234a5b236acd2f \ + --hash=sha256:65b5b2066651b7432d389e9799d979c703bcc6ef44266bb8153ef54e91e4aab3 \ + --hash=sha256:664ec6a520b74a1df2810666eb67695fcb77fa663e6ea0a25aaf2e529cb24dfa \ + --hash=sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3 \ + --hash=sha256:683e362b8ba453080f7489c66f4ea794e751c35b72e7eab3575ef784c2fbc7fb \ + --hash=sha256:68782fdb4027b8d1eee25ec35e9a6db05e863b899eb0310b3a33b6c3fef55707 \ + --hash=sha256:6efaf45df6a849cef613a03a94c845647456662f85438c886bb67a9c027c8c2c \ + --hash=sha256:71f42c5b9a948c113bbdebfa544598321431d064ff959d32e99b1feb61d68345 \ + --hash=sha256:72849d892954be4d09e569b8b831ac39ce58417fedc767d4308a0fe542018a40 \ + --hash=sha256:72c34ac7ad4314c19362d5ce27626dcc8429bd30bbf8c179f4234078851f9492 \ + --hash=sha256:734f6e5400352ac4254456003d462866c684703570929cff7a7bde015d0cb371 \ + --hash=sha256:75baa6cf9b6d1c52f3e111a130e202fd8cf0a5b3a066c3f73d615e885092e4ec \ + --hash=sha256:77716e245c90f058466a05e6a465bb8600f767a8f4b18b4d40f3aff958e5f73c \ + --hash=sha256:77e5099b99b37f3cf79c246998ca9f7313a78054cd1809ec46bc1afad47e1c4c \ + --hash=sha256:783dd1467083f4d3f7722ad6a313f24c173e7571372738fcb7a6e6d1ba48df25 \ + --hash=sha256:7a5c3115595995779ee21f2567035793911c3802a43c74f3fbb0314929ec67ac \ + --hash=sha256:7c88edaec8c349ad4c5ad4c486a3defcc4b80ceb2f074436ffa0a87caf5e76a6 \ + --hash=sha256:7cb414a73e21a7ab58254926073f2930cb22f5b4314ea4260a687e2b3fd4dce3 \ + --hash=sha256:7d42e7e3ca399555578b4d617e3a6ecf13371b3743a115995fa010c7bf341459 \ + --hash=sha256:7d575b54cb3863ef9bc290ea4b009999d55dc237326131e4853cf33e888fee03 \ + --hash=sha256:7e4de3ac4adbad3d0bc7c6f4360a7dbff5de2f15e3b723be3198074e17fd9c40 \ + --hash=sha256:7efc9f082dfed77c316edffa9deb52888e1bc6789171887cc1f68e06d65465c8 \ + --hash=sha256:80a063f8297fc796296f00f100be520f209b23dc98f93ce8eba6ee7122598209 \ + --hash=sha256:80e47012e730da131c9f059c80936783f9659aae22dc31c03c0595590d11ed54 \ + --hash=sha256:83d4a37e4b95da4d8bda930d6d35b75b4cdadbacbb4980cae290ea3100b5d51d \ + --hash=sha256:83e9f4a25085bd4b7214701a0794ff1f50fc633ffb8bdfebf07abdd81c2db126 \ + --hash=sha256:85a18376073f8a39aa07be34f9fc77e2869aa72c55c441efdd2cf79a0407504d \ + --hash=sha256:87796fedc3ba97ec14fab55acb48584276e6c1e4c1e89c422bda62c838e754a9 \ + --hash=sha256:882569ff613758cac762a457a5d72d6e211b28d4bcfea89d1d71ea942b02eac0 \ + --hash=sha256:8ce4d6ccafb33d39bd78444612d14938ead674c25702ded2ee9c54a47735d225 \ + --hash=sha256:8e7d98cdbb6d71e726f7d525952867096053d1f290dd4e3c50d7d313a136f414 \ + --hash=sha256:8ee202350cf57abf0e9502a41601841019c25d3db7ff52d980aaf31446254059 \ + --hash=sha256:8fb0eb4955adf0579001581f2f71a126e8781ba61bcd120f127b0401163c6c2d \ + --hash=sha256:90c30ed53546da833c700115c0064c22120d1b1560f474699fd31f22dd668233 \ + --hash=sha256:9489e6abf47ba37f332075a91444c7cfedb03e6ce99fbb2f116bfe1ce810da3b \ + --hash=sha256:94d7aa6debf92a1dd14cb5280b083a764169a13cfb23a452111160274ed989f4 \ + --hash=sha256:98d370568f393215d605304cdb77b3d5539bd192c75b623c7304c42c8d6d8273 \ + --hash=sha256:9b1bdaae98bc016825dd3c9d8ee1832f829b3341f9cc6ebd1a1b0a7fef7367cc \ + --hash=sha256:9d693bf4bf534e9ba3ae2780cfd577f5135629f7b5ac653490859d0b77864865 \ + --hash=sha256:9d6ed3d17bccce4c05343e1ca8da13bc5c02c812a4e7282ddd05e8769322d3fc \ + --hash=sha256:9e23c82b63cd7652fc24d33ed6cc17099d607aa3b4fc4ddc75e95062f3d82df4 \ + --hash=sha256:a1daf47cd95a7c3a63456336bc5aaa8c86dd3a47d07ed3d0e76132ae4666a5a1 \ + --hash=sha256:a1e32763e641a1566507d90a8d3b19bfc3cc04a9d4e5ae3e32189874ed4b58a3 \ + --hash=sha256:a2059a2d891bd156bc5184e7ab7a56e78a84dfcfdeac8c501b552533ad1c36ee \ + --hash=sha256:a2ed0ba415ccdf08f14bf544cb78346d0f76086707ffee24921a2c84dbf1305a \ + --hash=sha256:a3faadac7d812ddac258feb57b9846b60c1b437c4f4b9ad42595c6f6fe4390df \ + --hash=sha256:a5877f2255aab518ebe528289037699201d5dc5f045f2396cb30aa02db22f57f \ + --hash=sha256:a78b50b4f7918a3de71105d5c0b93bbc57bb8339a4d03a9dfd449f9068e76f3d \ + --hash=sha256:a8c2b841478068440d8b733005d13a5ef535b9928cbc05f17182d410f32ba449 \ + --hash=sha256:a9ca696eb02e5c02a8afd872ada510eba9b7fe6e68b9572c2e9a9b1941e31e2e \ + --hash=sha256:aa4ed3dd308548f9e707d9caaf005d2d7f8c1e7868f858dfeb47fe76e16b391d \ + --hash=sha256:ab2054c5531af2a9ba7b69b8ec91e4f884420e83a8c5e579b013084cb57e5e5d \ + --hash=sha256:abb1384477f5901d436b5d2e5465954de46ea6098f59163d243660b5c4461d35 \ + --hash=sha256:acae6b45d1ace09b6ba3876da43b88366ef368f73b988c7f57e14231753d4420 \ + --hash=sha256:acfa7e22aa6c6e7a5996a41d275bfa01efa7ea56ab890590280e9063e2cf5c1b \ + --hash=sha256:af4ea5b37403ef4e30f3927eaed540db942bde01d8d3ff083527c0704d1c9c68 \ + --hash=sha256:b10dd0557ba422715b5206b3743192135a6022acca8baec51aa127d0a75db8fe \ + --hash=sha256:b13b88747769537f3d32e89e3a735da10c0a9e35d7322928c701b5f93d3afffd \ + --hash=sha256:b51c159a9794633f5e0db7ecec7b2b6e3734eca1f5d17dc989ff3552a43ff78b \ + --hash=sha256:b5402a340723fa7da00b5cff987ddab61276be6d11251ea71ae02bcac54890d8 \ + --hash=sha256:b7abffdf37af1cec6a2ad69b827aa84320db5894791bc8ed932dc93fb274b7e9 \ + --hash=sha256:b8075fe90bc08e40b8b8a1874fab42ee4c7b56af05c5886e9cc841397f916908 \ + --hash=sha256:bc3ac7bf569f6b64dad04dd7808c7872dae8a97df657856eac05e9b7e3614a85 \ + --hash=sha256:bd0912757081f89b107d6c00b2ff8a194401b0b87eadcf4481de2b865a8fd44f \ + --hash=sha256:bdc8d8b8c22e9e43ac68316b5e6cf083dec537f4ec213cb4aa967b583bc3fa64 \ + --hash=sha256:be80550d9bfe83d9b62398a37081a90434e6df2d978ec345c3d2820de6beddab \ + --hash=sha256:c6f117789d22dce188e5754e8bc65b7e6ebf8cb73963b9fa761f672a5883769d \ + --hash=sha256:ca32926d7d77bcc8838425c4c95e040a3ace1cb7dfdae599013458dcda2607ca \ + --hash=sha256:cce0727fd5ac04d372fa9bbfde9febc2bcf209aadfcf0468e45dec72719895d1 \ + --hash=sha256:d0f1489233a254bb3643d2f05de7d59019254d81daeca6b9162fe9edef57e0c7 \ + --hash=sha256:d1c557dfd5e3db046053a0bdc72261ade790ebe8e2c7a41b36b0ca1f14cb95f3 \ + --hash=sha256:d21f0fa80a02d05299207eeaafef345d812ace96d5306e4ef265e1d419a615fa \ + --hash=sha256:d45673badd08456d0340e9364eddafe1c53a9d2896424294de4d7dd71ad3ee57 \ + --hash=sha256:d5add7b4ca7afeea91d52e4d4e4db3b1fe9885b71f07054560d8c4296b7441a2 \ + --hash=sha256:d5f90e44653c4e0f78501ed9bb7d3fce835a8d62b7c6ed0cb16557534087e743 \ + --hash=sha256:d7306dee25b8a0e737363f347362b875094b4dc4e367311470656ae420fdbf8e \ + --hash=sha256:dbcef5a9119ef653653132cccaf999b30a0af6f33bb0a4ba80bec30056868487 \ + --hash=sha256:def538065f9e4d4cf1ae164bd59aba00dfa84f03923e0de4c3788f252d6bcd17 \ + --hash=sha256:df23df54b5114a17c2d0ef192433e2e5a9f0c5178c32375e90b7cfc965f349d0 \ + --hash=sha256:dfbf531053a0935f2e871bcd4753f90313688772ff8c017f5ea402e315a78c1f \ + --hash=sha256:e029648f9c951db30e98a7d7ec90835db88ec4b32820efe2a9bdc2287e032eb6 \ + --hash=sha256:e07595c7d6f4db270ceede356a1bd1c07a34f1c26f958d1ed0cd7b48e0d2bba3 \ + --hash=sha256:e12c538e00e7c1b286a07061046b90e8124e6a9793efae2c70db6a4aad07faad \ + --hash=sha256:e546fe1d4a93ebc2910f0d768baff19faa09843ab3f2036a67ed6e69fae4419d \ + --hash=sha256:e5637ca8d0bd7fb72648a6c7934af4baaccb697657f7438c9d264fc2abb8b0b1 \ + --hash=sha256:e636b64d24fd9c38053c5e389a1174c66361fa49dcfd220f4dd35b4abde7cb89 \ + --hash=sha256:e7011b8fb8c4054bf0c12e5edc6cd83778b0028e99ce59b18586ed036f92cfdc \ + --hash=sha256:e80f557716fd765439577131e526b8942ffc2c07bdbc5e39fa62f660ba1e963f \ + --hash=sha256:e92b6bcc741b86d67606c40d3cb9c7cc8e6c737f81e31f4a94efc204456c92e3 \ + --hash=sha256:eb96ed1ae6c7d072d60840c0434aef07a2df611812810807fbc54263a6053e9a \ + --hash=sha256:eda19ea5ee88742f47a2340816e6f2d40b53bed3ab5b69794769f36af9f35bb4 \ + --hash=sha256:ef74070ac553c59eb4f04258722066d6c6135b7baa03b2e9f2da65c096e96d98 \ + --hash=sha256:efb01a106f971cb3752856bca2318bbdf7f01bd8823779c461586cbe5ffd5258 \ + --hash=sha256:f074e8d4aa0a5798920ddb6de3d08b228c614ff3724c3e8bd7577f4bafea867b \ + --hash=sha256:f38a70074041d3b7e138e452799f5174198bae5bd5ab2000917badf403908c5f \ + --hash=sha256:f41753a76f4f63927d03a0d8ba8f5ce0f2083bec29a8cfaccc55371b1564b96b \ + --hash=sha256:f53dcd26694f148f738edc052b5a69234833e739f10f4c3287bdfd8ec0f7b326 \ + --hash=sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc \ + --hash=sha256:fe01645169a2112aa1d4ebc3e4c5f029c5c8f97adfc32e5d37c993b39a994d75 + # via aiohttp diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py index b02f635..04e5d62 100644 --- a/scripts/init_secrets.py +++ b/scripts/init_secrets.py @@ -24,7 +24,7 @@ def write(name: str, value: str) -> None: print(f"keep secrets/{name}") return path.write_text(value + "\n", encoding="utf-8", newline="\n") - # Readable by the container users (postgres, valkey, app uid 10001) through the + # Readable by the container users (postgres, valkey, seaweedfs, app uid 10001) through the # compose bind mount. Local synthetic lab credentials only. os.chmod(path, 0o644) print(f"created secrets/{name}") @@ -40,6 +40,11 @@ def main() -> int: write("valkey_password", secrets.token_urlsafe(24)) write("database_url", f"postgresql://messenger:{read('postgres_password')}@postgres:5432/messenger") write("cache_url", f"redis://:{read('valkey_password')}@valkey:6379/0") + # Lab 4: three separate S3 key pairs (SeaweedFS identities, infra/seaweedfs/start.sh): + # admin = bucket bootstrap only; api = Read+Write on the bucket; reconciler = + List. + for who in ("admin", "api", "reconciler"): + write(f"s3_{who}_access_key_id", "AK" + secrets.token_hex(9).upper()) + write(f"s3_{who}_secret_access_key", secrets.token_hex(20)) return 0 diff --git a/scripts/restart.sh b/scripts/restart.sh index b641607..f00dcb4 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,6 +1,6 @@ #!/bin/sh # Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`. -# Restarts EVERY process of the stand: LB, all API replicas, PostgreSQL and Valkey. +# Restarts EVERY process of the stand: LB, all API replicas, PostgreSQL, Valkey and S3 (SeaweedFS). # Volumes, schema and data are preserved: no `down -v`, no FLUSH, no re-seed. # Output is suppressed by the test; run it by hand to see the steps. set -eu @@ -8,10 +8,11 @@ cd "$(dirname "$0")/.." # 1. Entry point and API first: no request is served while the stores restart underneath. docker compose stop lb api -# 2. Stores: clean shutdown (PostgreSQL checkpoint, Valkey AOF fsync), then start again. -docker compose restart postgres valkey +# 2. Stores: clean shutdown (PostgreSQL checkpoint, Valkey AOF fsync, SeaweedFS volume/filer +# flush), then start again. The S3 bytes live on the named volume s3data. +docker compose restart postgres valkey s3 # 3. API replicas (wait for healthy stores via depends_on), then the LB. HAProxy resolves # the new replica addresses through Docker DNS by itself - no stale backend list. docker compose start api docker compose start lb -echo "restart.sh: lb, api (all replicas), postgres and valkey restarted with volumes preserved" +echo "restart.sh: lb, api (all replicas), postgres, valkey and s3 restarted with volumes preserved" diff --git a/src/messenger/app.py b/src/messenger/app.py index aa4852a..cc32393 100644 --- a/src/messenger/app.py +++ b/src/messenger/app.py @@ -6,6 +6,7 @@ import contextlib import hashlib import json import logging +import random import re import time import uuid @@ -14,15 +15,16 @@ from typing import Any from starlette.applications import Starlette from starlette.exceptions import HTTPException from starlette.requests import Request -from starlette.responses import JSONResponse, Response +from starlette.responses import JSONResponse, Response, StreamingResponse from starlette.routing import Route from starlette.types import ASGIApp, Message as AsgiMessage, Receive, Scope, Send -from . import authhdr, cursor, docs, passwords, repo, validation +from . import attachments, authhdr, cursor, docs, passwords, repo, validation from .config import Settings from .db import Database from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unauthorized, Unavailable from .sessions import SessionStore, build_store +from .storage import ObjectMissing, Storage from .timefmt import rfc3339 log = logging.getLogger("messenger") @@ -126,6 +128,10 @@ def sessions_of(request: Request) -> SessionStore: return request.app.state.sessions +def storage_of(request: Request) -> Storage: + return request.app.state.storage + + async def authenticate(request: Request) -> str: """User id of a valid Bearer session, otherwise 401. X-User-Id is ignored since lab 2. @@ -206,15 +212,32 @@ async def live(request: Request) -> JSONResponse: async def ready(request: Request) -> JSONResponse: - """200 only if the mandatory synchronous path can be served: PostgreSQL and, - with SESSION_STORE=valkey, Valkey answer. 503 while draining after SIGTERM.""" + """200 only if the WHOLE mandatory synchronous path can be served: PostgreSQL, Valkey + (SESSION_STORE=valkey) and, since lab 4, the S3 bucket (upload/download of attachments) + answer. 503 while draining after SIGTERM. HAProxy routes to such replicas first.""" if request.app.state.draining: raise Unavailable("Сервер завершает работу") - db_ok, store_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping()) + db_ok, store_ok, s3_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping(), + storage_of(request).ping()) if not db_ok: raise Unavailable("PostgreSQL недоступен") if not store_ok: raise Unavailable("Хранилище сессий недоступно") + if not s3_ok: + raise Unavailable("Хранилище файлов (S3) недоступно") + return ok({"status": "ok"}) + + +async def ready_core(request: Request) -> JSONResponse: + """Not in the contract: readiness WITHOUT S3, for the load balancer's degraded mode only. + If S3 is down for every replica, /health/ready is 503 everywhere; HAProxy then routes to + replicas that are at least core-ready, so chats and messages keep working while every + attachment request answers 503 itself (see infra/haproxy/haproxy.cfg).""" + if request.app.state.draining: + raise Unavailable("Сервер завершает работу") + db_ok, store_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping()) + if not (db_ok and store_ok): + raise Unavailable("PostgreSQL или хранилище сессий недоступно") return ok({"status": "ok"}) @@ -288,12 +311,12 @@ async def get_chat(request: Request) -> JSONResponse: def parse_message(data: dict[str, Any]) -> tuple[str, list[str]]: + """`text` is required and a string (also with attachments); `attachment_ids`, if present, an + array of 0-10 unique UUIDs. "" or whitespace-only text is allowed only with attachments.""" validation.check_fields(data, {"text"}, frozenset({"attachment_ids"})) - text = validation.text_field(data, "text", 4000) attachment_ids = validation.attachment_ids_field(data) - if attachment_ids: - # Lab 3 has no attachments: every referenced ID is unknown (lab 4 semantics: 404). - raise NotFound("Вложение не найдено") + text = validation.text_field(data, "text", 4000, min_len=0 if attachment_ids else 1, + allow_blank=bool(attachment_ids)) return text, attachment_ids @@ -309,7 +332,7 @@ async def create_message(request: Request) -> JSONResponse: key = idempotency_key(request) text, attachment_ids = parse_message(await read_json(request)) result = await repo.post_message(db, user_id, chat_id, key, request_fingerprint(text, attachment_ids), text, - settings.idempotency_ttl_seconds) + settings.idempotency_ttl_seconds, attachment_ids) else: try: key = idempotency_key(request) @@ -321,7 +344,7 @@ async def create_message(request: Request) -> JSONResponse: # Membership check, key claim, chat clock and message: one SQL call, one transaction. result = await repo.post_message_single_call(db, user_id, chat_id, key, request_fingerprint(text, attachment_ids), text, - settings.idempotency_ttl_seconds) + settings.idempotency_ttl_seconds, attachment_ids) if result.replayed: log.info("idempotent_replay", extra={"fields": {"request_id": request.state.request_id, "message_id": result.message.id}}) @@ -340,10 +363,111 @@ async def list_messages(request: Request) -> JSONResponse: return ok({"items": [m.to_json() for m in items], "next_cursor": next_cursor}) +# ------------------------------------------------------------------ attachments (lab 4) +def fault(request: Request, point: str) -> None: + """Failure injection for the level 4 experiments (FAULT_POINTS, compose.faults.yaml only).""" + if point in request.app.state.settings.fault_points: + log.warning("fault_injected", extra={"fields": {"request_id": request.state.request_id, "point": point}}) + raise Unavailable("Внедрённый отказ: " + point) + + +async def upload_attachment(request: Request) -> JSONResponse: + """POST /chats/{chat_id}/attachments?filename=...: raw PNG/JPEG -> 201 ready (attachments.py).""" + settings: Settings = request.app.state.settings + reader = attachments.BodyReader(request) + try: + user_id = await authenticate(request) + chat_id = path_uuid(request, "chat_id") + db = db_of(request) + await repo.ensure_member(db, user_id, chat_id) # outsiders: 404 before anything about the body + filename = validation.filename_param(validation.query_items(request.scope["query_string"])) + declared = attachments.declared_type(request) + attachments.declared_length(request, settings.max_upload_bytes) + slots: asyncio.Semaphore = request.app.state.upload_slots + try: + await asyncio.wait_for(slots.acquire(), settings.upload_queue_timeout_seconds) + except asyncio.TimeoutError: + raise attachments.Busy("Слишком много одновременных загрузок, повторите позже") from None + try: + upload = await attachments.read_image(reader, settings, declared) + finally: + slots.release() + except ApiError: + # Early rejection: drop (not store) a bounded rest of the body so the client reads our error. + await reader.discard(settings.max_upload_bytes + (1 << 20), timeout=5.0) + raise + attachment_id = uuid.uuid4() + object_key = attachments.new_object_key() + storage = storage_of(request) + await attachments.create_intent(db, attachment_id, chat_id, user_id, filename, upload.info, len(upload.data), + upload.sha256, object_key) + await storage.put(object_key, upload.data, upload.info.content_type, + {"attachment-id": str(attachment_id), "sha256": upload.sha256.hex()}) + fault(request, "after_s3_put") + attachment = await attachments.mark_ready(db, attachment_id) + if attachment is None: + raise Unavailable("Загрузка не завершена, повторите запрос") + fault(request, "after_ready_commit") + log.info("attachment_created", extra={"fields": {"request_id": request.state.request_id, + "attachment_id": attachment.id, "chat_id": chat_id, + "size": attachment.size_bytes, "type": attachment.content_type}}) + return ok(attachment.to_json(), 201) + + +async def get_attachment(request: Request) -> JSONResponse: + user_id = await authenticate(request) + attachment_id = path_uuid(request, "attachment_id") + return ok((await attachments.get_visible(db_of(request), user_id, attachment_id)).to_json()) + + +def content_disposition(filename: str) -> str: + """RFC 6266/5987: ASCII fallback + UTF-8 name; the stored name itself is never interpreted.""" + from urllib.parse import quote + fallback = "".join(c if 32 <= ord(c) < 127 and c not in '"\\%;' else "_" for c in filename) + return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{quote(filename, safe='')}" + + +async def download_attachment(request: Request) -> Response: + """Original bytes through the authorized API: no redirect, no public S3 URL. Streamed from + S3 in 64 KiB chunks (memory per download does not depend on the file size).""" + user_id = await authenticate(request) + attachment_id = path_uuid(request, "attachment_id") + query = validation.query_items(request.scope["query_string"]) + try: + validation.variant_param(query) + except validation.NotFoundVariant: + await attachments.get_visible(db_of(request), user_id, attachment_id) # 404 for outsiders first + raise NotFound("Такого варианта у вложения нет") from None + attachment = await attachments.get_visible(db_of(request), user_id, attachment_id) + if attachment.lost: + raise NotFound("Оригинал вложения утрачен") + try: + size, chunks = await storage_of(request).open_stream(attachment.object_key) + except ObjectMissing: + # A ready row without its object: data loss, reported by the reconciler (quarantine). + log.error("attachment_object_missing", extra={"fields": {"request_id": request.state.request_id, + "attachment_id": attachment.id}}) + raise NotFound("Оригинал вложения утрачен") from None + if size != attachment.size_bytes: + log.error("attachment_size_mismatch", extra={"fields": {"request_id": request.state.request_id, + "attachment_id": attachment.id}}) + await chunks.aclose() + raise Unavailable("Оригинал вложения временно недоступен") + headers = { + "Content-Length": str(size), + "Content-Disposition": content_disposition(attachment.filename), + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + "ETag": '"' + attachment.sha256 + '"', + } + return StreamingResponse(chunks, media_type=attachment.content_type, headers=headers) + + def create_app(settings: Settings | None = None) -> Starlette: settings = settings or Settings.from_env() db = Database(settings) sessions = build_store(settings, db) + storage = Storage(settings) async def purge_loop() -> None: """Housekeeping: expired Idempotency-Key rows (older than IDEMPOTENCY_TTL_SECONDS).""" @@ -356,6 +480,18 @@ def create_app(settings: Settings | None = None) -> Starlette: except ApiError: pass # database unavailable right now; next round + async def janitor_loop(every: float) -> None: + """Level 4 cleanup: intents of failed uploads older than PENDING_GRACE_SECONDS (attachments.py). + Every replica runs it; the advisory lock lets one round run at a time.""" + while True: + await asyncio.sleep(every * (0.5 + random.random())) + try: + removed = await attachments.clean_stale_pending(db, storage, settings.pending_grace_seconds) + if removed: + log.info("stale_uploads_removed", extra={"fields": {"count": removed}}) + except ApiError: + pass # PostgreSQL or S3 unavailable right now; the rows stay pending for the next round + stats_log = logging.getLogger("messenger.stats") async def stats_loop(every: float) -> None: @@ -376,12 +512,17 @@ def create_app(settings: Settings | None = None) -> Starlette: async def lifespan(app: Starlette): passwords.configure(settings.hash_concurrency) await db.open() + await storage.open() log.info("dependencies_configured", extra={"fields": { "session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds, "db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds, "instance_id": settings.instance_id, "drain_s": settings.drain_seconds, - "message_write_path": settings.message_write_path}}) + "message_write_path": settings.message_write_path, "s3_bucket": settings.s3_bucket, + "upload_concurrency": settings.upload_concurrency, "pending_grace_s": settings.pending_grace_seconds, + "fault_points": sorted(settings.fault_points)}}) tasks = [asyncio.create_task(purge_loop())] + if settings.janitor_interval_seconds > 0: + tasks.append(asyncio.create_task(janitor_loop(settings.janitor_interval_seconds))) if settings.stats_log_seconds > 0: tasks.append(asyncio.create_task(stats_loop(settings.stats_log_seconds))) try: @@ -390,11 +531,13 @@ def create_app(settings: Settings | None = None) -> Starlette: for task in tasks: task.cancel() await sessions.close() + await storage.close() await db.close() routes = [ Route("/health/live", live, methods=["GET"]), Route("/health/ready", ready, methods=["GET"]), + Route("/health/ready/core", ready_core, methods=["GET"]), Route("/api/v1/users", register_user, methods=["POST"]), Route("/api/v1/users/me", me, methods=["GET"]), Route("/api/v1/auth/sessions", create_session, methods=["POST"]), @@ -404,6 +547,9 @@ def create_app(settings: Settings | None = None) -> Starlette: Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]), Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]), Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]), + Route("/api/v1/chats/{chat_id}/attachments", upload_attachment, methods=["POST"]), + Route("/api/v1/attachments/{attachment_id}", get_attachment, methods=["GET"]), + Route("/api/v1/attachments/{attachment_id}/content", download_attachment, methods=["GET"]), ] if settings.docs_enabled: # Swagger UI for manual experiments, not part of the contract routes += [ @@ -419,5 +565,7 @@ def create_app(settings: Settings | None = None) -> Starlette: app.state.settings = settings app.state.db = db app.state.sessions = sessions + app.state.storage = storage + app.state.upload_slots = asyncio.Semaphore(settings.upload_concurrency) app.state.draining = False return app diff --git a/src/messenger/attachments.py b/src/messenger/attachments.py new file mode 100644 index 0000000..fd9804f --- /dev/null +++ b/src/messenger/attachments.py @@ -0,0 +1,285 @@ +"""Attachments (lab 4): metadata in PostgreSQL, original bytes in the private S3 bucket. + +Upload (POST /chats/{id}/attachments, raw PNG/JPEG body), in this order: + + auth -> membership (404) -> filename / Content-Type / Content-Length (400/415/413) + -> upload slot (at most UPLOAD_CONCURRENCY bodies per replica, else 503 + Retry-After) + -> body in chunks: size limit, read deadline, format + pixel count from the first bytes + -> full integrity check (images.verify) -> SHA-256 / size known + -> INSERT row status='pending' with a fresh random object_key, COMMIT (intent) + -> PUT object to S3 (Content-MD5) + -> UPDATE status='ready' WHERE status='pending', COMMIT -> 201 ready. + +Why 201 is honest: it is sent only after BOTH the object and the ready row are durable. +What a failure between the steps leaves behind, and who removes it (docs/CONSISTENCY.md): + +* before the intent COMMIT: nothing (no object was written); +* after the intent, before/while PUT, before the ready COMMIT: a 'pending' row (+ maybe the + object). The client got 503 and may retry (a new attachment). The janitor (below) deletes the + object - idempotent, a missing key is fine - and then the row, once it is older than + PENDING_GRACE_SECONDS. Nothing is compensated in the request path: after a lost answer of + COMMIT we do not know whether the row is ready, and deleting the object could destroy a + ready attachment. Only the janitor decides, under a row lock, from the committed state; +* after the ready COMMIT (answer lost): a correct ready attachment the client does not know. +""" +from __future__ import annotations + +import asyncio +import datetime as dt +import hashlib +import logging +import secrets +import uuid +from dataclasses import dataclass + +import psycopg +from psycopg import errors as pg_errors +from starlette.requests import Request + +from . import images +from .config import Settings +from .db import Database +from .errors import ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType +from .storage import OBJECT_PREFIX, Storage +from .timefmt import rfc3339 + +log = logging.getLogger("messenger.attachments") + +JANITOR_LOCK_ID = 2026_04 # advisory lock: one janitor round at a time across all replicas + + +@dataclass(frozen=True, slots=True) +class Attachment: + id: str + chat_id: str + owner_id: str + filename: str + content_type: str + size_bytes: int + sha256: str + object_key: str + status: str + created_at: dt.datetime + updated_at: dt.datetime + lost: bool = False + + def to_json(self) -> dict: + """Public representation: no object key, no bucket, no internal state.""" + return {"id": self.id, "chat_id": self.chat_id, "owner_id": self.owner_id, "filename": self.filename, + "content_type": self.content_type, "size_bytes": self.size_bytes, "sha256": self.sha256, + "status": "ready", "variants": [], "error_code": None, + "created_at": rfc3339(self.created_at), "updated_at": rfc3339(self.updated_at)} + + +COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.object_key, " + "a.status, a.created_at, a.updated_at, a.lost_at IS NOT NULL") + + +def _attachment(row) -> Attachment: + return Attachment(str(row[0]), str(row[1]), str(row[2]), row[3], row[4], row[5], bytes(row[6]).hex(), row[7], + row[8], row[9], row[10], row[11]) + + +def new_object_key() -> str: + """Server-generated, 128 random bits, unrelated to the attachment id and the filename.""" + return OBJECT_PREFIX + secrets.token_hex(16) + + +# ------------------------------------------------------------------ SQL +async def create_intent(db: Database, attachment_id: uuid.UUID, chat_id: str, owner_id: str, filename: str, + info: images.ImageInfo, size: int, sha256: bytes, object_key: str) -> None: + async def work(conn: psycopg.AsyncConnection) -> None: + await conn.execute( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, " + "width, height, object_key, status) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending')", + (attachment_id, chat_id, owner_id, filename, info.content_type, size, sha256, info.width, info.height, + object_key)) + try: + await db.run(work, transaction=False) + except pg_errors.ForeignKeyViolation: + raise NotFound("Чат не найден") from None + + +async def mark_ready(db: Database, attachment_id: uuid.UUID) -> Attachment | None: + """pending -> ready. None if the row is no longer pending (the janitor removed a stale intent).""" + async def work(conn: psycopg.AsyncConnection) -> Attachment | None: + cur = await conn.execute( + "UPDATE attachments a SET status = 'ready', updated_at = clock_timestamp() " + f"WHERE a.id = %s AND a.status = 'pending' RETURNING {COLUMNS}", (attachment_id,)) + row = await cur.fetchone() + return _attachment(row) if row else None + return await db.run(work, transaction=False) + + +async def get_visible(db: Database, user_id: str, attachment_id: str) -> Attachment: + """A READY attachment of a chat the user is a member of; anything else is 404 (unknown, + foreign chat, unfinished upload) - existence of foreign objects is never revealed.""" + async def work(conn: psycopg.AsyncConnection) -> Attachment: + cur = await conn.execute( + f"SELECT {COLUMNS} FROM attachments a " + "JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s " + "WHERE a.id = %s AND a.status = 'ready'", (user_id, attachment_id)) + row = await cur.fetchone() + if row is None: + raise NotFound("Вложение не найдено") + return _attachment(row) + return await db.run(work, transaction=False) + + +async def clean_stale_pending(db: Database, storage: Storage, grace_seconds: int, batch: int = 20) -> int: + """Janitor round: remove intents of uploads that never became ready. + + Under FOR UPDATE: an upload that is (impossibly) still running would block on the row in + mark_ready and then find it gone (-> 503), so a ready attachment can never lose its object. + Order: object first (idempotent), row second; a crash in between leaves the pending row and + the next round repeats. Only status='pending' rows older than the grace period are touched: + fresh uploads and ready/used attachments are never selected. + """ + async def work(conn: psycopg.AsyncConnection) -> int: + cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (JANITOR_LOCK_ID,)) + if not (await cur.fetchone())[0]: + return 0 + cur = await conn.execute( + "SELECT id, object_key FROM attachments WHERE status = 'pending' " + "AND created_at < clock_timestamp() - make_interval(secs => %s) " + "ORDER BY created_at LIMIT %s FOR UPDATE SKIP LOCKED", (grace_seconds, batch)) + rows = await cur.fetchall() + if not rows: + return 0 + await asyncio.gather(*(storage.delete(key) for _, key in rows)) + await conn.execute("DELETE FROM attachments WHERE id = ANY(%s) AND status = 'pending'", + ([r[0] for r in rows],)) + return len(rows) + return await db.run(work, deadline=storage.timeout + 5) + + +# ------------------------------------------------------------------ request body +class Busy(Unavailable): + """All upload slots of this replica are taken: retry shortly (no body was read).""" + + @property + def headers(self) -> dict[str, str]: + return {"Retry-After": "1"} + + +@dataclass +class Upload: + data: bytes + info: images.ImageInfo + sha256: bytes + + +class BodyReader: + """Reads the raw ASGI body ourselves (not request.stream()), so we always know how much of it + is consumed and can discard the rest after an early rejection.""" + + def __init__(self, request: Request): + self.receive = request.receive + self.done = False + + async def chunk(self, timeout: float) -> bytes | None: + if self.done: + return None + message = await asyncio.wait_for(self.receive(), timeout) + if message["type"] == "http.disconnect": + self.done = True + raise InvalidRequest("Клиент прервал загрузку") + if not message.get("more_body", False): + self.done = True + return message.get("body", b"") + + async def discard(self, limit: int, timeout: float) -> None: + """After an early 4xx: read and DROP up to `limit` more bytes, so an ordinary client that + writes the whole body before reading the answer still gets our JSON error instead of a + connection reset. Nothing is kept in memory; beyond the limit the connection is dropped.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + seen = 0 + try: + while not self.done and seen <= limit: + remaining = deadline - loop.time() + if remaining <= 0: + return + seen += len(await self.chunk(remaining) or b"") + except (ApiError, asyncio.TimeoutError): + return + + +def declared_type(request: Request) -> str: + """Content-Type of the upload must be image/png or image/jpeg (parameters ignored).""" + media_type = request.headers.get("content-type", "").partition(";")[0].strip().lower() + if media_type not in (images.PNG, images.JPEG): + raise UnsupportedMediaType("Content-Type должен быть image/png или image/jpeg") + return media_type + + +def declared_length(request: Request, limit: int) -> int | None: + raw = request.headers.get("content-length") + if raw is None: + return None # chunked transfer: the limit is enforced while reading + if not raw.isdigit(): + raise InvalidRequest("Неверный Content-Length") + length = int(raw) + if length > limit: + raise PayloadTooLarge(f"Файл больше {limit} байт") + if length == 0: + raise InvalidRequest("Тело запроса пустое: нужны байты изображения") + return length + + +async def read_image(reader: BodyReader, settings: Settings, declared: str) -> Upload: + """Body -> verified image. Memory: at most MAX_UPLOAD_BYTES per upload (bytearray); the format + and the pixel count are known after the first bytes, so a picture that is too big or not a + PNG/JPEG is rejected without reading/decoding the rest.""" + loop = asyncio.get_running_loop() + deadline = loop.time() + settings.upload_read_timeout_seconds + limit = settings.max_upload_bytes + buffer = bytearray() + digest = hashlib.sha256() + info: images.ImageInfo | None = None + while True: + remaining = deadline - loop.time() + if remaining <= 0: + raise InvalidRequest("Тело запроса не получено вовремя") + try: + chunk = await reader.chunk(remaining) + except asyncio.TimeoutError: + raise InvalidRequest("Тело запроса не получено вовремя") from None + if chunk is None: + break + if len(buffer) + len(chunk) > limit: + raise PayloadTooLarge(f"Файл больше {limit} байт") + buffer += chunk + digest.update(chunk) + if info is None and buffer: + info = _sniff(buffer, settings, declared) + if not buffer: + raise InvalidRequest("Тело запроса пустое: нужны байты изображения") + if info is None: + raise UnsupportedMediaType("Файл не является корректным изображением PNG или JPEG") + data = bytes(buffer) + del buffer + try: + await images.verify(data, info, timeout=settings.image_verify_timeout_seconds, + jpeg_max_memory_mb=settings.jpeg_max_memory_mb) + except images.ImageError as exc: + raise _image_error(exc) from None + except asyncio.TimeoutError: + raise UnsupportedMediaType("Изображение не удалось проверить за отведённое время") from None + return Upload(data, info, digest.digest()) + + +def _sniff(buffer: bytearray, settings: Settings, declared: str) -> images.ImageInfo | None: + try: + info = images.sniff(buffer, settings.max_image_pixels) + except images.ImageError as exc: + raise _image_error(exc) from None + if info is not None and info.content_type != declared: + # "MIME spoofing": the header says PNG, the bytes are JPEG (or vice versa). + raise UnsupportedMediaType("Содержимое не совпадает с Content-Type") + return info + + +def _image_error(exc: images.ImageError) -> ApiError: + return PayloadTooLarge(exc.message) if exc.status == 413 else UnsupportedMediaType(exc.message) + diff --git a/src/messenger/config.py b/src/messenger/config.py index 1c10990..1e3c1dc 100644 --- a/src/messenger/config.py +++ b/src/messenger/config.py @@ -115,6 +115,25 @@ class Settings: idempotency_ttl_seconds: int = 86400 stats_log_seconds: float = 0.0 message_write_path: str = "function" + # Lab 4: private S3 bucket and attachment limits (see storage.py, attachments.py). + s3_endpoint: str = "http://s3:8333" + s3_region: str = "us-east-1" + s3_bucket: str = "messenger-attachments" + s3_access_key_id: str = "" + s3_secret_access_key: str = "" + s3_timeout_seconds: float = 3.0 + s3_connect_timeout_seconds: float = 1.0 + s3_max_connections: int = 20 + max_upload_bytes: int = 10 * 1024 * 1024 + max_image_pixels: int = 40_000_000 + upload_concurrency: int = 4 + upload_queue_timeout_seconds: float = 5.0 + upload_read_timeout_seconds: float = 30.0 + image_verify_timeout_seconds: float = 10.0 + jpeg_max_memory_mb: int = 256 + pending_grace_seconds: int = 900 + janitor_interval_seconds: float = 60.0 + fault_points: frozenset[str] = frozenset() @property def db_statement_timeout_ms(self) -> int: @@ -165,4 +184,49 @@ class Settings: # Track 1: `function` = one SQL call per send (migration 0003), `multi` = the previous # multi-statement transaction; both have identical semantics (A/B on one image). message_write_path=_choice("MESSAGE_WRITE_PATH", "function", {"function", "multi"}), + **s3_settings(), ) + + +# Contract limits (contracts/README.md): configurable DOWN for experiments, never above them. +CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024 +CONTRACT_MAX_IMAGE_PIXELS = 40_000_000 +# Failure points for the level 4 experiments only (compose.faults.yaml); empty in normal runs. +FAULT_POINTS = {"after_s3_put", "after_ready_commit"} + + +def s3_settings(*, credentials_required: bool = True) -> dict: + """S3 part of Settings; also used by the reconciler/bootstrap CLIs with their own keys.""" + faults = frozenset(f.strip() for f in os.environ.get("FAULT_POINTS", "").split(",") if f.strip()) + if faults - FAULT_POINTS: + raise ConfigError(f"FAULT_POINTS may contain only {sorted(FAULT_POINTS)}") + endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").strip() or "http://s3:8333" + if not re.fullmatch(r"https?://[A-Za-z0-9.\-]+(:[0-9]{1,5})?/?", endpoint): + raise ConfigError(f"S3_ENDPOINT must be http(s)://host[:port], got {endpoint!r}") + bucket = os.environ.get("S3_BUCKET", "messenger-attachments").strip() or "messenger-attachments" + if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket): + raise ConfigError(f"S3_BUCKET is not a valid bucket name: {bucket!r}") + return dict( + s3_endpoint=endpoint, + s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1", + s3_bucket=bucket, + s3_access_key_id=_secret("S3_ACCESS_KEY_ID", required=credentials_required), + s3_secret_access_key=_secret("S3_SECRET_ACCESS_KEY", required=credentials_required), + s3_timeout_seconds=_float("S3_TIMEOUT_SECONDS", 3.0, 0.2, 30.0), + s3_connect_timeout_seconds=_float("S3_CONNECT_TIMEOUT_SECONDS", 1.0, 0.1, 10.0), + s3_max_connections=_int("S3_MAX_CONNECTIONS", 20, 1, 200), + max_upload_bytes=_int("MAX_UPLOAD_BYTES", CONTRACT_MAX_UPLOAD_BYTES, 1024, CONTRACT_MAX_UPLOAD_BYTES), + max_image_pixels=_int("MAX_IMAGE_PIXELS", CONTRACT_MAX_IMAGE_PIXELS, 1, CONTRACT_MAX_IMAGE_PIXELS), + # At most this many request bodies of one replica are buffered/checked at once (RAM bound: + # UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES); further uploads wait, then get 503. + upload_concurrency=_int("UPLOAD_CONCURRENCY", 4, 1, 64), + upload_queue_timeout_seconds=_float("UPLOAD_QUEUE_TIMEOUT_SECONDS", 5.0, 0.0, 60.0), + upload_read_timeout_seconds=_float("UPLOAD_READ_TIMEOUT_SECONDS", 30.0, 1.0, 300.0), + image_verify_timeout_seconds=_float("IMAGE_VERIFY_TIMEOUT_SECONDS", 10.0, 1.0, 120.0), + jpeg_max_memory_mb=_int("JPEG_MAX_MEMORY_MB", 256, 16, 2048), + # An intent row still `pending` after this long belongs to a failed upload: the janitor + # deletes its object and the row. Far above the longest possible upload (~45 s). + pending_grace_seconds=_int("PENDING_GRACE_SECONDS", 900, 1, 7 * 24 * 3600), + janitor_interval_seconds=_float("JANITOR_INTERVAL_SECONDS", 60.0, 0.0, 3600.0), + fault_points=faults, + ) diff --git a/src/messenger/errors.py b/src/messenger/errors.py index 22dbe74..fa3ce51 100644 --- a/src/messenger/errors.py +++ b/src/messenger/errors.py @@ -59,6 +59,12 @@ class PayloadTooLarge(ApiError): default_message = "Тело запроса слишком большое" +class UnsupportedMediaType(ApiError): + status = 415 + code = "unsupported_media_type" + default_message = "Неподдерживаемый тип содержимого" + + class Unavailable(ApiError): status = 503 code = "unavailable" diff --git a/src/messenger/images.py b/src/messenger/images.py new file mode 100644 index 0000000..4e1a6fb --- /dev/null +++ b/src/messenger/images.py @@ -0,0 +1,317 @@ +"""Content checks of uploaded images (lab 4): what the bytes ARE, not what the client says. + +Two stages, both with bounded memory and both BEFORE anything is stored: + +1. `sniff(head)` — runs on the first bytes while the body is still arriving. Recognises + PNG/JPEG by the signature and reads the dimensions from the header (PNG IHDR, JPEG SOFn). + A foreign format is rejected (415) and a picture with more than MAX_IMAGE_PIXELS pixels + (413) before the rest of the body is read and long before any pixel is decoded. +2. `verify(data, info)` — full integrity check of the complete file, run in a worker thread: + * PNG: our own streaming validator. Every chunk CRC is checked, the IDAT zlib stream is + inflated in 1 MiB steps and DISCARDED (no raster in memory), its size must be exactly + height x (1 + row bytes) (Adam7 per pass), every row filter byte must be 0..4, IEND + must close the file. A truncated file, a broken IDAT or extra/missing pixel data fail. + Pillow is not used for PNG: it would allocate the whole raster (40 Mpx RGBA = 160 MB). + * JPEG: libjpeg-turbo `djpeg -strict` decodes the whole entropy-coded data at 1/8 scale + (DCT scaling: the output raster is 64 times smaller) in a child process; any decoder + warning is fatal, so a truncated file or a file without scan data after SOS fails. + +Every problem of the CONTENT is an ImageError carrying the HTTP status (415 or 413); +nothing here may turn user input into a 500. +""" +from __future__ import annotations + +import asyncio +import os +import shutil +import struct +import zlib +from dataclasses import dataclass + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +JPEG_SOI = b"\xff\xd8\xff" + +PNG = "image/png" +JPEG = "image/jpeg" + +# Bits per pixel channel count by PNG colour type, and the bit depths allowed for it. +PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}} +# Adam7 passes: (x start, y start, x step, y step). +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +# JPEG start-of-frame markers that carry the frame size (not DHT C4, JPG C8, DAC CC). +JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF} +# Standalone markers without a length field. +JPEG_STANDALONE = {0x01, *range(0xD0, 0xD8)} + +DJPEG = shutil.which("djpeg") or "djpeg" # libjpeg-turbo-progs, installed in the Dockerfile +INFLATE_STEP = 1 << 20 # bytes of raw PNG data produced per zlib call +MAX_DIMENSION = 2**31 - 1 + + +class ImageError(Exception): + """The uploaded bytes are not an acceptable image. `status`: 415 (format/corrupt) or 413.""" + + def __init__(self, status: int, message: str): + super().__init__(message) + self.status = status + self.message = message + + +def unsupported(message: str = "Файл не является корректным изображением PNG или JPEG") -> ImageError: + return ImageError(415, message) + + +@dataclass(frozen=True, slots=True) +class ImageInfo: + content_type: str + width: int + height: int + + +def detect(head: bytes) -> str | None: + """Format by signature; None while fewer bytes than a signature have arrived.""" + if head[:8] == PNG_SIGNATURE: + return PNG + if head[:3] == JPEG_SOI: + return JPEG + probe = min(len(head), 8) + if PNG_SIGNATURE.startswith(head[:probe]) or JPEG_SOI.startswith(head[:min(probe, 3)]): + return None + raise unsupported() + + +def sniff(head: bytes, max_pixels: int) -> ImageInfo | None: + """Format and size from the beginning of the file, or None if more bytes are needed. + + Raises ImageError: 415 for a foreign/broken header, 413 for too many pixels.""" + kind = detect(head) + if kind is None: + return None + info = _png_header(head) if kind == PNG else _jpeg_header(head) + if info is None: + return None + if info.width * info.height > max_pixels: + raise ImageError(413, f"Изображение больше {max_pixels} пикселей") + return info + + +def _png_header(head: bytes) -> ImageInfo | None: + if len(head) < 33: + return None + length, ctype = struct.unpack(">I4s", head[8:16]) + if ctype != b"IHDR" or length != 13: + raise unsupported() + if zlib.crc32(head[12:29]) != struct.unpack(">I", head[29:33])[0]: + raise unsupported() + width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", head[16:29]) + if not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION): + raise unsupported() + if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1: + raise unsupported() + return ImageInfo(PNG, width, height) + + +def _jpeg_header(head: bytes) -> ImageInfo | None: + """Walk the marker segments up to the first SOFn. None = the SOF has not arrived yet.""" + i = 2 + n = len(head) + while True: + if i >= n: + return None + if head[i] != 0xFF: + raise unsupported() + while i < n and head[i] == 0xFF: # fill bytes + i += 1 + if i >= n: + return None + marker = head[i] + i += 1 + if marker in JPEG_STANDALONE: + continue + if marker in (0xD8, 0xD9, 0xDA, 0x00): + # Second SOI, EOI or scan data before any frame header: not a usable JPEG. + raise unsupported() + if i + 2 > n: + return None + length = struct.unpack(">H", head[i:i + 2])[0] + if length < 2: + raise unsupported() + if marker in JPEG_SOF: + if i + 7 > n: + return None + if length < 8: + raise unsupported() + height, width = struct.unpack(">HH", head[i + 3:i + 7]) + if width == 0 or height == 0: # height 0 = DNL-defined height: not supported + raise unsupported() + return ImageInfo(JPEG, width, height) + i += length + + +# ------------------------------------------------------------------ full verification +async def verify(data: bytes | bytearray | memoryview, info: ImageInfo, *, timeout: float = 10.0, + jpeg_max_memory_mb: int = 256) -> None: + """Complete integrity check of the whole file. PNG: our validator in a worker thread (zlib + releases the GIL); JPEG: a strict libjpeg decoder in a child process.""" + if info.content_type == PNG: + await asyncio.wait_for(asyncio.to_thread(verify_png, data), timeout) + else: + await verify_jpeg(data, timeout, jpeg_max_memory_mb) + + +def _png_segments(width: int, height: int, depth: int, color: int, interlace: int) -> list[tuple[int, int]]: + """(byte count, row stride) of every pass of the raw (filtered) image data.""" + bpp = PNG_CHANNELS[color] * depth + passes = ADAM7 if interlace else ((0, 0, 1, 1),) + segments = [] + for xs, ys, dx, dy in passes: + pw = (width - xs + dx - 1) // dx if width > xs else 0 + ph = (height - ys + dy - 1) // dy if height > ys else 0 + if pw and ph: + stride = 1 + (pw * bpp + 7) // 8 + segments.append((ph * stride, stride)) + return segments + + +class _RawChecker: + """Counts inflated bytes against the expected size and checks the filter byte of every row + using C-speed slicing (`piece[first::stride]`), so even 40 000 000 one-pixel rows are fast.""" + + VALID_FILTERS = bytes(range(5)) + + def __init__(self, segments: list[tuple[int, int]]): + self.segments = segments + self.index = 0 # current pass + self.offset = 0 # bytes already seen inside the current pass + self.expected = sum(size for size, _ in segments) + self.total = 0 + + def feed(self, piece: bytes) -> None: + self.total += len(piece) + if self.total > self.expected: + raise unsupported() + pos = 0 + while pos < len(piece): + size, stride = self.segments[self.index] + take = min(size - self.offset, len(piece) - pos) + part = piece[pos:pos + take] + first = (-self.offset) % stride + if part[first::stride].translate(None, self.VALID_FILTERS): + raise unsupported() + self.offset += take + pos += take + if self.offset == size: + self.index += 1 + self.offset = 0 + + +def verify_png(data: bytes | bytearray | memoryview) -> None: + view = memoryview(data) + if bytes(view[:8]) != PNG_SIGNATURE: + raise unsupported() + pos = 8 + n = len(view) + checker: _RawChecker | None = None + inflater = zlib.decompressobj() + color = None + seen_plte = seen_idat = idat_done = False + first = True + while True: + if pos + 12 > n: + raise unsupported() # truncated: no room for another chunk header + CRC + length, ctype = struct.unpack(">I4s", view[pos:pos + 8]) + if length > n - pos - 12: + raise unsupported() + body = view[pos + 8:pos + 8 + length] + crc = struct.unpack(">I", view[pos + 8 + length:pos + 12 + length])[0] + if zlib.crc32(body, zlib.crc32(ctype)) != crc: + raise unsupported() + pos += 12 + length + if not all(65 <= c <= 90 or 97 <= c <= 122 for c in ctype): + raise unsupported() + if first: + if ctype != b"IHDR" or length != 13: + raise unsupported() + width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", body) + if (not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION) or color not in PNG_DEPTHS + or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1): + raise unsupported() + checker = _RawChecker(_png_segments(width, height, depth, color, interlace)) + first = False + continue + assert checker is not None + if ctype == b"IHDR": + raise unsupported() + if ctype == b"PLTE": + if seen_plte or seen_idat or color in (0, 4) or length % 3 or not 3 <= length <= 768: + raise unsupported() + seen_plte = True + elif ctype == b"IDAT": + if idat_done: + raise unsupported() # IDAT chunks must be consecutive + if color == 3 and not seen_plte: + raise unsupported() + seen_idat = True + _inflate(inflater, body, checker) + elif ctype == b"IEND": + if length != 0 or not seen_idat: + raise unsupported() + if pos != n: + raise unsupported() # bytes after IEND (polyglot files) are not accepted + break + elif not 97 <= ctype[0] <= 122: + raise unsupported() # unknown CRITICAL chunk: a decoder could not show the image + if ctype != b"IDAT" and seen_idat: + idat_done = True + # The zlib stream must end exactly with the last pixel row, without trailing data. + if not inflater.eof or inflater.unused_data or checker.total != checker.expected: + raise unsupported() + + +def _inflate(inflater, body: memoryview, checker: _RawChecker) -> None: + if inflater.eof: + if len(body): + raise unsupported() # data after the end of the zlib stream + return + pending: bytes | memoryview = body + try: + while pending: + piece = inflater.decompress(pending, INFLATE_STEP) + checker.feed(piece) + pending = inflater.unconsumed_tail + if inflater.eof: + if pending or inflater.unused_data: + raise unsupported() + break + except zlib.error: + raise unsupported() from None + + +async def verify_jpeg(data: bytes | bytearray | memoryview, timeout: float, max_memory_mb: int) -> None: + """Decode the WHOLE file with the reference decoder (libjpeg-turbo `djpeg`) in strict mode. + + `-strict` turns every libjpeg warning into a failure: "premature end of data segment" + (scan data missing or cut before EOI), "premature end of JPEG file", corrupt Huffman data. + Pillow cannot be used for this: it ignores libjpeg warnings and returns a grey picture for + a JPEG whose scan data is missing. `-scale 1/8` decodes every block but writes a 64x + smaller raster to /dev/null; `-maxmemory` caps libjpeg's working memory (coefficient + buffers of progressive files); the separate process is killed at the deadline. + """ + process = await asyncio.create_subprocess_exec( + DJPEG, "-strict", "-scale", "1/8", "-maxmemory", str(max_memory_mb * 1024), "-outfile", os.devnull, + stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE) + stderr = b"" + try: + _, stderr = await asyncio.wait_for(process.communicate(bytes(data)), timeout) + except (asyncio.TimeoutError, asyncio.CancelledError): + process.kill() + await process.wait() + raise + except (BrokenPipeError, ConnectionResetError): + await process.wait() # djpeg stopped reading: the file is already known to be broken + if process.returncode != 0: + if b"Insufficient memory" in (stderr or b""): + # A valid but extremely memory-hungry file (progressive, 4:4:4, ~40 Mpx): a limit, not corruption. + raise ImageError(413, "Изображение слишком велико для обработки") + raise unsupported() diff --git a/src/messenger/repo.py b/src/messenger/repo.py index 02e1a8c..d1b8aba 100644 --- a/src/messenger/repo.py +++ b/src/messenger/repo.py @@ -62,6 +62,7 @@ class Message: sender_id: str text: str created_at: dt.datetime + attachment_ids: tuple[str, ...] = () @property def key(self) -> SortKey: @@ -69,7 +70,7 @@ class Message: def to_json(self) -> dict: return {"id": self.id, "chat_id": self.chat_id, "sender_id": self.sender_id, "text": self.text, - "created_at": rfc3339(self.created_at)} + "created_at": rfc3339(self.created_at), "attachment_ids": list(self.attachment_ids)} def _user(row) -> User: @@ -205,7 +206,7 @@ class PostResult: async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID, user_id: str, chat_id: str, - text: str) -> dt.datetime: + text: str, attachment_ids: list[str]) -> dt.datetime: """Insert with a per-chat strictly increasing timestamp. `UPDATE chats ... RETURNING` locks the chat row until COMMIT, so writers of one @@ -220,9 +221,39 @@ async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID, await conn.execute( "INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)", (message_id, chat_id, user_id, _b(text), created_at)) + if attachment_ids: + await conn.execute( + "INSERT INTO message_attachments (message_id, position, attachment_id) " + "SELECT %s, a.ord - 1, a.id FROM unnest(%s::uuid[]) WITH ORDINALITY AS a(id, ord)", + (message_id, attachment_ids)) return created_at +# Attachments a message may reference: ready, of THIS chat (else 404), not quarantined (else 409). +# FOR SHARE keeps a concurrent quarantine (reconciler) from slipping in before COMMIT. +CHECK_ATTACHMENTS = ( + "SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL) FROM (" + "SELECT lost_at FROM attachments WHERE id = ANY(%s::uuid[]) AND chat_id = %s AND status = 'ready' " + "FOR SHARE) a") + + +async def _check_attachments(conn: psycopg.AsyncConnection, chat_id: str, attachment_ids: list[str]) -> str: + if not attachment_ids: + return "ok" + cur = await conn.execute(CHECK_ATTACHMENTS, (attachment_ids, chat_id)) + found, usable = await cur.fetchone() + if found != len(attachment_ids): + return "attachment_not_found" + return "ok" if usable == found else "attachment_conflict" + + +def _attachment_error(status: str) -> None: + if status == "attachment_not_found": + raise NotFound("Вложение не найдено") + if status == "attachment_conflict": + raise Conflict("Вложение недоступно и не может быть прикреплено") + + # Claim the key. A row of a concurrent transaction that is not committed yet makes this # INSERT wait on the unique index until that transaction ends: after COMMIT it is a conflict # (the key is taken), after ROLLBACK the INSERT goes through. A key older than the TTL is @@ -235,60 +266,72 @@ CLAIM_KEY = ( "WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => %s) " "RETURNING 1") STORED_RESULT = ( - "SELECT k.request_hash, m.id, m.text, m.created_at FROM idempotency_keys k " + "SELECT k.request_hash, m.id, m.text, m.created_at, " + "ARRAY(SELECT ma.attachment_id FROM message_attachments ma WHERE ma.message_id = m.id ORDER BY ma.position) " + "FROM idempotency_keys k " "JOIN messages m ON m.id = k.message_id " "WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s") async def post_message(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes, text: str, - ttl_seconds: int) -> PostResult: + ttl_seconds: int, attachment_ids: list[str] = ()) -> PostResult: """Idempotent send: the key, the message and the stored result commit in ONE transaction. * first request -> key row + message + chat clock, all or nothing (no "pending" key is ever left behind: a crash or 5xx rolls everything back and the client may retry); * same key, same body (any replica, also concurrently) -> the stored Message, 201, replayed; - * same key, other body -> 409; the membership check comes first, so outsiders always get 404. + * same key, other body -> 409; the membership check comes first, so outsiders always get 404; + * attachments (lab 4) are checked before the key is claimed (same order as post_message_v2). """ message_id = uuid.uuid4() + attachment_ids = list(attachment_ids) async def work(conn: psycopg.AsyncConnection) -> PostResult: await _check_member(conn, user_id, chat_id) - cur = await conn.execute(CLAIM_KEY, (user_id, chat_id, key, request_hash, message_id, ttl_seconds)) - if await cur.fetchone() is not None: - created_at = await _insert_message(conn, message_id, user_id, chat_id, text) - return PostResult(Message(str(message_id), chat_id, user_id, text, created_at), False) + check = await _check_attachments(conn, chat_id, attachment_ids) + if check == "ok": + cur = await conn.execute(CLAIM_KEY, (user_id, chat_id, key, request_hash, message_id, ttl_seconds)) + if await cur.fetchone() is not None: + created_at = await _insert_message(conn, message_id, user_id, chat_id, text, attachment_ids) + return PostResult(Message(str(message_id), chat_id, user_id, text, created_at, + tuple(attachment_ids)), False) cur = await conn.execute(STORED_RESULT, (user_id, chat_id, key)) row = await cur.fetchone() if row is None: + _attachment_error(check) # The key row vanished between the two statements (expired-key cleanup): retryable. raise Unavailable("Повторите запрос с тем же Idempotency-Key") if bytes(row[0]) != request_hash: raise Conflict("Idempotency-Key уже использован с другим сообщением") - return PostResult(Message(str(row[1]), chat_id, user_id, _s(row[2]), row[3]), True) + return PostResult(Message(str(row[1]), chat_id, user_id, _s(row[2]), row[3], + tuple(str(a) for a in row[4])), True) return await db.run(work) -POST_MESSAGE_CALL = "SELECT o_status, o_id, o_text, o_created_at FROM post_message(%s, %s, %s, %s, %s, %s, %s)" +POST_MESSAGE_CALL = ("SELECT o_status, o_id, o_text, o_created_at, o_attachments " + "FROM post_message_v2(%s, %s, %s, %s, %s, %s, %s, %s::uuid[])") async def post_message_single_call(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes, - text: str, ttl_seconds: int) -> PostResult: - """Same contract as post_message, ONE round trip: SQL function post_message (migration 0003) - in autocommit mode, i.e. the call is the whole transaction. Track 1 improvement.""" + text: str, ttl_seconds: int, attachment_ids: list[str] = ()) -> PostResult: + """Same contract as post_message, ONE round trip: SQL function post_message_v2 (migration 0004 = + 0003 + attachment_ids) in autocommit mode, i.e. the call is the whole transaction.""" message_id = uuid.uuid4() async def work(conn: psycopg.AsyncConnection) -> tuple: - cur = await conn.execute(POST_MESSAGE_CALL, - (user_id, chat_id, key, request_hash, message_id, _b(text), ttl_seconds)) + cur = await conn.execute(POST_MESSAGE_CALL, (user_id, chat_id, key, request_hash, message_id, _b(text), + ttl_seconds, list(attachment_ids))) return await cur.fetchone() - status, stored_id, stored_text, created_at = await db.run(work, transaction=False) + status, stored_id, stored_text, created_at, stored_attachments = await db.run(work, transaction=False) if status == "not_found": raise NotFound("Чат не найден") + _attachment_error(status) if status == "conflict": raise Conflict("Idempotency-Key уже использован с другим сообщением") if status == "retry": raise Unavailable("Повторите запрос с тем же Idempotency-Key") - return PostResult(Message(str(stored_id), chat_id, user_id, _s(stored_text), created_at), status == "replayed") + return PostResult(Message(str(stored_id), chat_id, user_id, _s(stored_text), created_at, + tuple(str(a) for a in stored_attachments)), status == "replayed") async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 5000) -> int: @@ -308,12 +351,14 @@ async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 50 # Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR # would push the cursor into a Filter and late pages would scan the whole prefix. -HISTORY_FIRST = ( - "SELECT id, sender_id, text, created_at FROM messages " - "WHERE chat_id = %s ORDER BY created_at, id LIMIT %s") -HISTORY_AFTER = ( - "SELECT id, sender_id, text, created_at FROM messages " - "WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s") +# attachment_ids: a correlated sub-select per returned row (at most limit + 1 = 101 lookups on the +# primary key of message_attachments), so the keyset range scan itself stays unchanged. +HISTORY_COLUMNS = ("SELECT m.id, m.sender_id, m.text, m.created_at, " + "ARRAY(SELECT ma.attachment_id FROM message_attachments ma " + "WHERE ma.message_id = m.id ORDER BY ma.position) FROM messages m ") +HISTORY_FIRST = HISTORY_COLUMNS + "WHERE m.chat_id = %s ORDER BY m.created_at, m.id LIMIT %s" +HISTORY_AFTER = (HISTORY_COLUMNS + + "WHERE m.chat_id = %s AND (m.created_at, m.id) > (%s, %s) ORDER BY m.created_at, m.id LIMIT %s") async def list_messages(db: Database, chat_id: str, after: SortKey | None, limit: int) -> tuple[list[Message], bool]: @@ -324,5 +369,6 @@ async def list_messages(db: Database, chat_id: str, after: SortKey | None, limit cur = await conn.execute(HISTORY_AFTER, (chat_id, from_us(after[0]), after[1], limit + 1)) return await cur.fetchall() rows = await db.run(work) - items = [Message(str(r[0]), chat_id, str(r[1]), _s(r[2]), r[3]) for r in rows[:limit]] + items = [Message(str(r[0]), chat_id, str(r[1]), _s(r[2]), r[3], tuple(str(a) for a in r[4])) + for r in rows[:limit]] return items, len(rows) > limit diff --git a/src/messenger/s3_bootstrap.py b/src/messenger/s3_bootstrap.py new file mode 100644 index 0000000..91a1b84 --- /dev/null +++ b/src/messenger/s3_bootstrap.py @@ -0,0 +1,151 @@ +"""One-shot S3 bootstrap: `python -m messenger.s3_bootstrap` (compose service `s3-init`). + +Idempotent, safe to run on every `docker compose up`: + +1. waits until the S3 endpoint answers; +2. creates the bucket with the ADMIN key pair if it does not exist yet (an existing bucket and + its objects are left untouched); +3. writes the sentinel object `health/sentinel` that the API's readiness probe HEADs; +4. proves the access model and FAILS the stand (exit 1) if it is wrong: + * anonymous (unsigned) GET of an object and anonymous LIST of the bucket are refused; + * the API key pair can read/write objects of ITS bucket, but can neither list it, nor create + a bucket, nor see or read another bucket (a canary bucket created here for this check); + * the reconciler key pair can list the bucket but cannot create one. +Only file names and verdicts are printed, never key values. +""" +from __future__ import annotations + +import asyncio +import os +import sys +import time +import urllib.error +import urllib.request + +from botocore.exceptions import ClientError + +from .config import ConfigError, Settings, _secret, s3_settings +from .errors import Unavailable +from .storage import SENTINEL_KEY, Storage + + +def say(text: str) -> None: + print(f"s3-bootstrap: {text}", flush=True) + + +def anonymous_status(url: str) -> int: + """HTTP status of an UNSIGNED request (what anyone on the network could do).""" + try: + with urllib.request.urlopen(url, timeout=3) as response: + return response.status + except urllib.error.HTTPError as exc: + return exc.code + + +async def error_code(coro) -> str: + """S3 error code of a call that is expected to be refused ('' if it succeeded).""" + try: + await coro + return "" + except ClientError as exc: + return str(exc.response.get("Error", {}).get("Code", "error")) + + +def settings_with(keys: tuple[str, str]) -> Settings: + base = s3_settings(credentials_required=False) + base.update(s3_access_key_id=keys[0], s3_secret_access_key=keys[1]) + return Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024, **base) + + +async def run() -> int: + admin = (_secret("S3_ADMIN_ACCESS_KEY_ID", True), _secret("S3_ADMIN_SECRET_ACCESS_KEY", True)) + api = (_secret("S3_ACCESS_KEY_ID", True), _secret("S3_SECRET_ACCESS_KEY", True)) + reconciler = (_secret("S3_RECONCILER_ACCESS_KEY_ID", True), _secret("S3_RECONCILER_SECRET_ACCESS_KEY", True)) + settings = settings_with(admin) + bucket = settings.s3_bucket + admin_s3, api_s3, rec_s3 = Storage(settings), Storage(settings_with(api)), Storage(settings_with(reconciler)) + for s in (admin_s3, api_s3, rec_s3): + await s.open() + try: + deadline = time.monotonic() + 60 + while True: + try: + await admin_s3._call("list_buckets", admin_s3.client.list_buckets()) + break + except Unavailable: + if time.monotonic() > deadline: + say("S3 endpoint does not answer within 60 s") + return 1 + await asyncio.sleep(1) + + code = await error_code(admin_s3.client.head_bucket(Bucket=bucket)) + if code == "": + say(f"bucket {bucket}: exists, kept as is") + elif code in ("404", "NoSuchBucket", "NotFound"): + code = await error_code(admin_s3.client.create_bucket(Bucket=bucket)) + if code not in ("", "BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + say(f"bucket {bucket}: create failed ({code})") + return 1 + say(f"bucket {bucket}: created") + else: + say(f"bucket {bucket}: head failed ({code})") + return 1 + await admin_s3.put(SENTINEL_KEY, b"ok\n", "text/plain", {}) + say(f"sentinel {SENTINEL_KEY}: written") + canary = bucket + "-canary" + if await error_code(admin_s3.client.head_bucket(Bucket=canary)): + await admin_s3.client.create_bucket(Bucket=canary) + await admin_s3.client.put_object(Bucket=canary, Key="canary", Body=b"secret of another bucket") + + failures = [] + base = settings.s3_endpoint.rstrip("/") + for what, url in (("anonymous GET object", f"{base}/{bucket}/{SENTINEL_KEY}"), + ("anonymous LIST bucket", f"{base}/{bucket}?list-type=2")): + status = anonymous_status(url) + verdict = "refused" if status in (401, 403) else "ALLOWED" + say(f"{what}: HTTP {status} -> {verdict}") + if verdict != "refused": + failures.append(what) + + probe = "health/bootstrap-probe" + checks = [ + ("api: read object", await error_code(api_s3.client.head_object(Bucket=bucket, Key=SENTINEL_KEY)), True), + ("api: write object", await error_code(api_s3.client.put_object(Bucket=bucket, Key=probe, Body=b"p")), True), + ("api: delete object", await error_code(api_s3.client.delete_object(Bucket=bucket, Key=probe)), True), + ("api: list bucket", await error_code(api_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), False), + ("api: create bucket", await error_code(api_s3.client.create_bucket(Bucket=bucket + "-x")), False), + ("api: read object of another bucket", + await error_code(api_s3.client.head_object(Bucket=canary, Key="canary")), False), + ("reconciler: list bucket", await error_code(rec_s3.client.list_objects_v2(Bucket=bucket, MaxKeys=1)), True), + ("reconciler: create bucket", await error_code(rec_s3.client.create_bucket(Bucket=bucket + "-x")), False), + ] + for what, code, allowed in checks: + ok = (code == "") == allowed + say(f"{what}: {'allowed' if code == '' else 'refused (' + code + ')'} -> {'OK' if ok else 'WRONG'}") + if not ok: + failures.append(what) + visible = [b["Name"] for b in (await api_s3.client.list_buckets()).get("Buckets", [])] + say(f"api: buckets visible in ListBuckets: {visible} -> {'OK' if canary not in visible else 'WRONG'}") + if canary in visible: + failures.append("api sees another bucket") + if failures: + say("access model is WRONG: " + ", ".join(failures)) + return 1 + say("done: private bucket, least-privilege keys verified") + return 0 + finally: + for s in (admin_s3, api_s3, rec_s3): + await s.close() + + +def main() -> int: + try: + return asyncio.run(run()) + except ConfigError as exc: + print(f"configuration error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + os.environ.setdefault("PYTHONUNBUFFERED", "1") + sys.exit(main()) diff --git a/src/messenger/storage.py b/src/messenger/storage.py new file mode 100644 index 0000000..bf922d1 --- /dev/null +++ b/src/messenger/storage.py @@ -0,0 +1,173 @@ +"""Private S3 bucket (SeaweedFS in compose) through aiobotocore, with a hard deadline per call. + +* Credentials: the API's OWN key pair (S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE -> + Docker secrets). SeaweedFS grants it only Read + Write on this one bucket: no List, no + Admin (cannot create/delete buckets or read other buckets). The admin pair is given only + to the S3 server and the one-shot bootstrap (infra/seaweedfs, messenger.s3_bootstrap). +* Deadlines: botocore connect/read timeouts AND an asyncio deadline around every call, so a + paused or unreachable S3 costs at most S3_TIMEOUT_SECONDS, then 503 `unavailable`. Unlike a + libpq query (db.py), an aiohttp request can be cancelled safely, so the call is cancelled. +* No retries inside botocore (max_attempts=1): the request handler decides; an upload is not + retried behind the client's back (contract: a retry may create another object). +* Checksums: Content-MD5 is sent with every PUT, so S3 rejects bytes damaged on the way. + botocore's newer default (aws-chunked CRC32 trailers) is switched off ("when_required"): + plain, widely supported S3 requests. +""" +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import hashlib +import logging +from collections.abc import AsyncIterator +from dataclasses import dataclass + +from aiobotocore.config import AioConfig +from aiobotocore.session import get_session +from botocore.exceptions import BotoCoreError, ClientError + +from .config import Settings +from .errors import Unavailable + +log = logging.getLogger("messenger.storage") + +MISSING_CODES = {"NoSuchKey", "404", "NotFound"} + + +class ObjectMissing(Exception): + """The key does not exist in the bucket (definitive answer of S3, not an outage).""" + + +@dataclass(frozen=True, slots=True) +class ObjectHead: + size: int + content_type: str + metadata: dict[str, str] + + +@dataclass(frozen=True, slots=True) +class ListedObject: + key: str + size: int + last_modified_ts: float # POSIX seconds + + +class Storage: + def __init__(self, settings: Settings, *, access_key: str | None = None, secret_key: str | None = None): + self.bucket = settings.s3_bucket + self.timeout = settings.s3_timeout_seconds + self._endpoint = settings.s3_endpoint + self._region = settings.s3_region + self._access_key = access_key or settings.s3_access_key_id + self._secret_key = secret_key or settings.s3_secret_access_key + self._config = AioConfig( + connect_timeout=settings.s3_connect_timeout_seconds, + read_timeout=settings.s3_timeout_seconds, + retries={"total_max_attempts": 1, "mode": "standard"}, + s3={"addressing_style": "path"}, + signature_version="s3v4", + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + max_pool_connections=settings.s3_max_connections, + ) + self._stack: contextlib.AsyncExitStack | None = None + self._client = None + + async def open(self) -> None: + self._stack = contextlib.AsyncExitStack() + session = get_session() + self._client = await self._stack.enter_async_context(session.create_client( + "s3", endpoint_url=self._endpoint, region_name=self._region, + aws_access_key_id=self._access_key, aws_secret_access_key=self._secret_key, config=self._config)) + + async def close(self) -> None: + if self._stack is not None: + await self._stack.aclose() + self._stack = None + + @property + def client(self): + assert self._client is not None, "Storage.open() was not called" + return self._client + + async def _call(self, what: str, coro, deadline: float | None = None): + """Await one S3 call under the deadline; outages become Unavailable (503).""" + try: + return await asyncio.wait_for(coro, deadline or self.timeout) + except ClientError as exc: + code = str(exc.response.get("Error", {}).get("Code", "")) + if code in MISSING_CODES: + raise ObjectMissing() from None + log.warning("s3_error", extra={"fields": {"op": what, "code": code}}) + raise Unavailable("Хранилище файлов временно недоступно") from None + except (asyncio.TimeoutError, BotoCoreError, OSError) as exc: + log.warning("s3_unavailable", extra={"fields": {"op": what, "error": type(exc).__name__}}) + raise Unavailable("Хранилище файлов временно недоступно") from None + + async def put(self, key: str, data: bytes | bytearray, content_type: str, metadata: dict[str, str]) -> None: + md5 = base64.b64encode(hashlib.md5(data, usedforsecurity=False).digest()).decode("ascii") + # Time budget grows with the size: 10 MiB over a local network takes well under a second. + deadline = self.timeout + len(data) / (8 * 1024 * 1024) + await self._call("put", self.client.put_object( + Bucket=self.bucket, Key=key, Body=bytes(data), ContentType=content_type, ContentLength=len(data), + ContentMD5=md5, Metadata=metadata), deadline) + + async def head(self, key: str) -> ObjectHead: + response = await self._call("head", self.client.head_object(Bucket=self.bucket, Key=key)) + return ObjectHead(int(response["ContentLength"]), response.get("ContentType", ""), + dict(response.get("Metadata", {}))) + + async def open_stream(self, key: str) -> tuple[int, AsyncIterator[bytes]]: + """Start a GET: (Content-Length, async iterator of chunks). The headers arrive under the + deadline; every chunk read afterwards has its own deadline (a stalled S3 aborts the + download instead of hanging the connection).""" + response = await self._call("get", self.client.get_object(Bucket=self.bucket, Key=key)) + body = response["Body"] + size = int(response["ContentLength"]) + + async def chunks() -> AsyncIterator[bytes]: + try: + while True: + chunk = await self._call("get_body", body.read(64 * 1024)) + if not chunk: + return + yield chunk + finally: + body.close() + return size, chunks() + + async def read_all(self, key: str, limit: int) -> bytes: + """Whole object (reconciler checksum verification); refuses objects above `limit`.""" + size, chunks = await self.open_stream(key) + if size > limit: + raise Unavailable("Объект больше допустимого размера") + parts = [chunk async for chunk in chunks] + return b"".join(parts) + + async def delete(self, key: str) -> None: + """Idempotent: deleting a missing key is a success in S3.""" + await self._call("delete", self.client.delete_object(Bucket=self.bucket, Key=key)) + + async def list_page(self, prefix: str, start_after: str | None, page_size: int) -> tuple[list[ListedObject], bool]: + """One ListObjectsV2 page in key order (needs the List permission: reconciler only).""" + params = {"Bucket": self.bucket, "Prefix": prefix, "MaxKeys": page_size} + if start_after: + params["StartAfter"] = start_after + response = await self._call("list", self.client.list_objects_v2(**params)) + items = [ListedObject(o["Key"], int(o["Size"]), o["LastModified"].timestamp()) + for o in response.get("Contents", [])] + return items, bool(response.get("IsTruncated")) + + async def ping(self, timeout: float = 1.0) -> bool: + """Readiness: HEAD of the sentinel object written by the bootstrap. Proves that S3 + answers, the bucket exists and the API credentials are accepted (Read permission).""" + try: + await self._call("ping", self.client.head_object(Bucket=self.bucket, Key=SENTINEL_KEY), timeout) + return True + except (Unavailable, ObjectMissing): + return False + + +SENTINEL_KEY = "health/sentinel" +OBJECT_PREFIX = "o/" # every attachment original lives under o/ diff --git a/src/messenger/validation.py b/src/messenger/validation.py index b338c9e..3ad26ec 100644 --- a/src/messenger/validation.py +++ b/src/messenger/validation.py @@ -8,7 +8,9 @@ from __future__ import annotations import json import re +import unicodedata from typing import Any +from urllib.parse import parse_qsl from .errors import InvalidRequest @@ -58,7 +60,7 @@ def check_fields(data: dict[str, Any], required: set[str], optional: frozenset[s raise InvalidRequest("Отсутствуют обязательные поля: " + ", ".join(sorted(missing))) -def text_field(data: dict[str, Any], name: str, max_len: int, *, min_len: int = 1) -> str: +def text_field(data: dict[str, Any], name: str, max_len: int, *, min_len: int = 1, allow_blank: bool = False) -> str: value = data[name] if not isinstance(value, str): raise InvalidRequest(f"Поле {name} должно быть строкой") @@ -68,7 +70,7 @@ def text_field(data: dict[str, Any], name: str, max_len: int, *, min_len: int = # len() of a Python str is the number of Unicode code points, as the contract requires. if not min_len <= len(value) <= max_len: raise InvalidRequest(f"Длина поля {name} должна быть от {min_len} до {max_len} символов") - if value.strip() == "": + if value.strip() == "" and not allow_blank: raise InvalidRequest(f"Поле {name} не может состоять только из пробелов") return value @@ -113,8 +115,8 @@ def member_ids_field(data: dict[str, Any]) -> list[str]: def attachment_ids_field(data: dict[str, Any]) -> list[str]: - """Optional `attachment_ids` (lab 4+). Absent and [] are the same message (contracts/README.md); - lab 3 has no attachments yet, so a non-empty list can only reference unknown objects.""" + """Optional `attachment_ids` (lab 4+): absent and [] are the same message (contracts/README.md); + null or any other non-array type is 400.""" value = data.get("attachment_ids", []) if not isinstance(value, list): raise InvalidRequest("attachment_ids должен быть массивом") @@ -145,3 +147,53 @@ def parse_limit(raw: str | None) -> int: if LIMIT_RE.fullmatch(raw) is None or not 1 <= int(raw) <= MAX_LIMIT: raise InvalidRequest(f"limit должен быть целым числом от 1 до {MAX_LIMIT}") return int(raw) + + +# Lab 4: the display name of an attachment. Never used as a path or an S3 key, but it is shown +# to other users, so characters that change how a name LOOKS are refused: control characters +# (C0, DEL, C1 such as U+0085), line/paragraph separators, unpaired surrogates and bidi controls +# ("photognp.exe" would display as "photoexe.png"). +BIDI_CONTROLS = frozenset("\u061c\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069") +FORBIDDEN_CATEGORIES = frozenset({"Cc", "Cs", "Zl", "Zp"}) + + +def query_items(raw: bytes) -> list[tuple[str, str]]: + """Strict query parsing: percent-encoded UTF-8 only (Starlette would replace bad bytes by U+FFFD).""" + try: + return parse_qsl(raw.decode("ascii"), keep_blank_values=True, encoding="utf-8", errors="strict") + except (UnicodeDecodeError, ValueError): + raise InvalidRequest("Параметры запроса должны быть в UTF-8 с percent-encoding") from None + + +def filename_param(query: list[tuple[str, str]]) -> str: + value = single_query_param(query, "filename") + if value is None: + raise InvalidRequest("Параметр filename обязателен") + if not 1 <= len(value) <= 255: + raise InvalidRequest("Длина filename должна быть от 1 до 255 символов") + if value.strip() == "" or value in {".", ".."}: + raise InvalidRequest("filename не может состоять только из пробелов или точек") + if "/" in value or "\\" in value: + raise InvalidRequest("filename не может содержать / или \\") + for char in value: + if unicodedata.category(char) in FORBIDDEN_CATEGORIES or char in BIDI_CONTROLS: + raise InvalidRequest("filename содержит управляющие символы") + return value + + +ATTACHMENT_VARIANTS = frozenset({"thumbnail", "crop", "optimized"}) # exist in the schema, not in lab 4 + + +def variant_param(query: list[tuple[str, str]]) -> str: + """`variant` of GET /attachments/{id}/content: only `original` exists in lab 4. A known variant + name is a missing variant of a ready object (404); anything else is an invalid value (400).""" + value = single_query_param(query, "variant") + if value is None or value == "original": + return "original" + if value in ATTACHMENT_VARIANTS: + raise NotFoundVariant() + raise InvalidRequest("variant должен быть original") + + +class NotFoundVariant(Exception): + """Raised for a variant that lab 4 does not produce; mapped to 404 by the handler."""