From 5be41d86463e5bde53e083ec3aa76cf921774bfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Sun, 15 Nov 2026 22:16:04 +0300 Subject: [PATCH 01/11] feat(pipeline): add durable outbox and versioned image workers --- .dockerignore | 15 + .env.example | 16 +- .gitignore | 5 + Dockerfile | 22 + Makefile | 17 + compose.test.yaml | 13 + compose.yaml | 219 +++++++ infra/fetch-source.go | 107 ++++ infra/haproxy.cfg | 36 ++ infra/s3-bootstrap.sh | 14 + infra/s3-entrypoint.sh | 6 + infra/s3-policy.json | 15 + infra/s3.Dockerfile | 49 ++ migrations/001_core.sql | 36 ++ migrations/002_idempotency.sql | 10 + migrations/003_attachments.sql | 32 + migrations/004_pipeline.sql | 54 ++ pyproject.toml | 44 ++ scripts/bootstrap.py | 63 ++ scripts/cluster.py | 83 +++ scripts/restart.sh | 4 + scripts/rolling-update.sh | 3 + scripts/stand.py | 137 ++++ scripts/stop-one.sh | 3 + src/__init__.py | 0 src/__main__.py | 44 ++ src/api.py | 424 +++++++++++++ src/attachments.py | 144 +++++ src/cache.py | 66 ++ src/cursor.py | 64 ++ src/database.py | 69 ++ src/domain.py | 45 ++ src/errors.py | 16 + src/images.py | 77 +++ src/logging_config.py | 17 + src/migrate.py | 51 ++ src/object_store.py | 84 +++ src/operator.py | 123 ++++ src/password.py | 36 ++ src/pipeline.py | 247 +++++++ src/reconcile.py | 109 ++++ src/repository.py | 296 +++++++++ src/schemas.py | 65 ++ src/service.py | 192 ++++++ src/settings.py | 37 ++ src/worker.py | 428 +++++++++++++ uv.lock | 1095 ++++++++++++++++++++++++++++++++ 47 files changed, 4728 insertions(+), 4 deletions(-) create mode 100644 Dockerfile create mode 100644 compose.test.yaml create mode 100644 compose.yaml create mode 100644 infra/fetch-source.go create mode 100644 infra/haproxy.cfg create mode 100644 infra/s3-bootstrap.sh create mode 100644 infra/s3-entrypoint.sh create mode 100644 infra/s3-policy.json create mode 100644 infra/s3.Dockerfile create mode 100644 migrations/001_core.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_pipeline.sql create mode 100644 pyproject.toml create mode 100644 scripts/bootstrap.py create mode 100644 scripts/cluster.py create mode 100755 scripts/restart.sh create mode 100755 scripts/rolling-update.sh create mode 100644 scripts/stand.py create mode 100755 scripts/stop-one.sh create mode 100644 src/__init__.py create mode 100644 src/__main__.py create mode 100644 src/api.py create mode 100644 src/attachments.py create mode 100644 src/cache.py create mode 100644 src/cursor.py create mode 100644 src/database.py create mode 100644 src/domain.py create mode 100644 src/errors.py create mode 100644 src/images.py create mode 100644 src/logging_config.py create mode 100644 src/migrate.py create mode 100644 src/object_store.py create mode 100644 src/operator.py create mode 100644 src/password.py create mode 100644 src/pipeline.py create mode 100644 src/reconcile.py create mode 100644 src/repository.py create mode 100644 src/schemas.py create mode 100644 src/service.py create mode 100644 src/settings.py create mode 100644 src/worker.py create mode 100644 uv.lock diff --git a/.dockerignore b/.dockerignore index bc16e556..8e5d3661 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,18 @@ __pycache__ node_modules evidence .DS_Store +**/__pycache__ +**/*.py[cod] +.work +.pytest_cache +.ruff_cache +tests +student-tests +research +scripts +contracts +*.md +Makefile +compose.yaml + +.work* diff --git a/.env.example b/.env.example index 119384b5..b385b8fc 100644 --- a/.env.example +++ b/.env.example @@ -13,13 +13,21 @@ API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 S3_ENDPOINT=http://s3:9000 S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_BUCKET=messenger +S3_ACCESS_KEY_FILE=/run/secrets/s3_access_key +S3_SECRET_KEY_FILE=/run/secrets/s3_secret_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 + +PUBLIC_PORT=8085 +API_CPUS=0.5 +API_MEMORY=768m +DB_POOL_SIZE=10 +RECONCILE_GRACE_SECONDS=300 + BROKER_URL_FILE=/run/secrets/broker_url WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 +JOB_TIMEOUT_SECONDS=20 +JOB_LEASE_SECONDS=23 STAGING_RETENTION_SECONDS=86400 diff --git a/.gitignore b/.gitignore index b5a71896..5e1bf8b9 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,8 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. + +.work/ +.work-*.txt +.pytest_cache/ +.ruff_cache/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..15856ef7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,22 @@ +FROM ghcr.io/astral-sh/uv:0.13.7@sha256:96f40d3904aad8d0bd010539001d36c8e2188b981e1334471a90c4a1d93bb490 AS uv +FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS builder +COPY --from=uv /uv /usr/local/bin/uv +WORKDIR /app +ENV UV_PYTHON_DOWNLOADS=never UV_LINK_MODE=copy +COPY pyproject.toml uv.lock ./ +RUN --mount=type=cache,target=/root/.cache/uv uv sync --locked --no-dev --no-install-project + +FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS runtime +ARG VCS_REF=unknown +LABEL org.opencontainers.image.revision=$VCS_REF +WORKDIR /app +ENV PATH="/app/.venv/bin:$PATH" PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 APP_HOST=0.0.0.0 APP_PORT=8080 +RUN groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home app +COPY --from=builder /app/.venv /app/.venv +COPY src /app/src +COPY migrations /app/migrations +USER 10001:10001 +EXPOSE 8080 +HEALTHCHECK --interval=10s --timeout=2s --start-period=10s --start-interval=1s --retries=3 CMD ["python", "-c", "import os,urllib.request; urllib.request.urlopen('http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready',timeout=1).read()"] +STOPSIGNAL SIGTERM +CMD ["python", "-m", "src"] diff --git a/Makefile b/Makefile index 5d86d171..788ce574 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,20 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +UV ?= uv +BASE_URL = http://127.0.0.1:8085 +.PHONY: bootstrap up down lint test-student verify +bootstrap: + $(PYTHON) scripts/bootstrap.py +up: bootstrap + VCS_REF=$$(git rev-parse HEAD) docker compose up -d --build --wait +down: + docker compose down +lint: + $(UV) run --locked ruff format --check src student-tests scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py) + $(UV) run --locked ruff check src student-tests scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py) +test-student: + BASE_URL="$(BASE_URL)" $(UV) run --locked pytest student-tests +verify: + $(UV) run --locked python -m scripts.verify_pipeline --output evidence/pipeline-repeat.json diff --git a/compose.test.yaml b/compose.test.yaml new file mode 100644 index 00000000..094c4c70 --- /dev/null +++ b/compose.test.yaml @@ -0,0 +1,13 @@ +# Explicit test-only fault instrumentation, never enabled by default. +services: + api: + environment: + PIPELINE_FAULTS: "1" + relay: + environment: + PIPELINE_FAULTS: "1" + worker: + environment: + PIPELINE_FAULTS: "1" + JOB_TIMEOUT_SECONDS: "12" + JOB_LEASE_SECONDS: "15" diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..31594949 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,219 @@ +name: mtusi-lab5-maslov +services: + postgres: + image: postgres:18.3@sha256:7e32e9833a6fb1c92c32552794cb6ed569d51b445a54907d35fc112ef39684db + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + PGDATA: /var/lib/postgresql/data/pgdata + secrets: [postgres_password] + volumes: + - postgres_data:/var/lib/postgresql + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 20 + cpus: 1 + mem_limit: 512m + memswap_limit: 512m + cache: + image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571 + command: [valkey-server, /run/secrets/valkey.conf] + secrets: + - cache_password + - source: valkey_config + target: valkey.conf + healthcheck: + test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/cache_password)" valkey-cli ping'] + interval: 2s + timeout: 2s + retries: 20 + cpus: 0.25 + mem_limit: 128m + memswap_limit: 128m + migrate: + image: ${API_IMAGE:-mtusi-messenger:lab5} + pull_policy: never + command: [python, -m, src.migrate] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + postgres: {condition: service_healthy} + cpus: 0.5 + mem_limit: 256m + read_only: true + api: + image: ${API_IMAGE:-mtusi-messenger:lab5} + build: + context: . + args: + VCS_REF: ${VCS_REF:-unknown} + scale: ${API_REPLICAS:-2} + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DB_POOL_SIZE: ${DB_POOL_SIZE:-10} + secrets: [database_url, cache_url, s3_access_key, s3_secret_key] + depends_on: + migrate: {condition: service_completed_successfully} + cache: {condition: service_healthy} + s3-bootstrap: {condition: service_completed_successfully} + cpus: ${API_CPUS:-0.5} + mem_limit: ${API_MEMORY:-384m} + memswap_limit: ${API_MEMORY:-384m} + pids_limit: 128 + read_only: true + tmpfs: ["/tmp:rw,noexec,nosuid,size=128m"] + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + stop_grace_period: 10s + s3: + image: ${S3_IMAGE:-mtusi-storage:minio-20251015} + build: {context: ., dockerfile: infra/s3.Dockerfile, target: server} + secrets: [s3_root_user, s3_root_password] + volumes: [s3_data:/data] + cpus: 1 + mem_limit: 1024m + memswap_limit: 1024m + read_only: true + tmpfs: ["/tmp:rw,nosuid,size=32m"] + environment: + HOME: /tmp + security_opt: [no-new-privileges:true] + s3-bootstrap: + image: ${MC_IMAGE:-mtusi-storage:mc-20250813} + build: {context: ., dockerfile: infra/s3.Dockerfile, target: bootstrap} + secrets: [s3_root_user, s3_root_password, s3_access_key, s3_secret_key] + depends_on: + s3: {condition: service_healthy} + cpus: 0.25 + mem_limit: 128m + read_only: true + tmpfs: ["/tmp:rw,nosuid,size=16m"] + reconciler: + image: ${API_IMAGE:-mtusi-messenger:lab5} + pull_policy: never + command: [python, -m, src.reconcile, --apply, --loop] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + secrets: [database_url, s3_access_key, s3_secret_key] + depends_on: + migrate: {condition: service_completed_successfully} + s3-bootstrap: {condition: service_completed_successfully} + cpus: 0.25 + mem_limit: 256m + memswap_limit: 256m + read_only: true + healthcheck: {disable: true} + broker: + image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571 + command: [valkey-server, /run/secrets/broker.conf] + secrets: + - broker_password + - source: broker_config + target: broker.conf + volumes: [broker_data:/data] + healthcheck: + test: [CMD-SHELL, 'REDISCLI_AUTH="$$(cat /run/secrets/broker_password)" valkey-cli ping'] + interval: 2s + timeout: 2s + retries: 20 + cpus: 0.5 + mem_limit: 256m + memswap_limit: 256m + relay: + image: ${API_IMAGE:-mtusi-messenger:lab5} + pull_policy: never + command: [python, -m, src.pipeline] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + secrets: [database_url, broker_url] + depends_on: + migrate: {condition: service_completed_successfully} + broker: {condition: service_healthy} + cpus: 0.25 + mem_limit: 192m + memswap_limit: 192m + read_only: true + healthcheck: {disable: true} + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + worker: + image: ${API_IMAGE:-mtusi-messenger:lab5} + pull_policy: never + command: [python, -m, src.worker] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-20} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-23} + secrets: [database_url, broker_url, s3_access_key, s3_secret_key] + depends_on: + migrate: {condition: service_completed_successfully} + broker: {condition: service_healthy} + s3-bootstrap: {condition: service_completed_successfully} + cpus: 2 + mem_limit: 1536m + memswap_limit: 1536m + pids_limit: 128 + read_only: true + tmpfs: ["/tmp:rw,noexec,nosuid,size=64m"] + healthcheck: {disable: true} + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + stop_grace_period: 25s + lb: + image: haproxy:3.2.1-alpine@sha256:ac79fe145f2bb6626ff26b584a2d0a34e791906c01015f2ae037aa3137b683d9 + ports: + - "127.0.0.1:${PUBLIC_PORT:-8085}:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: {condition: service_healthy} + cpus: 0.5 + mem_limit: 128m + memswap_limit: 128m + read_only: true + tmpfs: ["/tmp:rw,nosuid,size=4m"] + security_opt: [no-new-privileges:true] + stop_grace_period: 10s +secrets: + broker_password: {file: ./secrets/broker_password} + broker_url: {file: ./secrets/broker_url} + broker_config: {file: ./secrets/broker.conf} + s3_root_user: {file: ./secrets/s3_root_user} + s3_root_password: {file: ./secrets/s3_root_password} + s3_access_key: {file: ./secrets/s3_access_key} + s3_secret_key: {file: ./secrets/s3_secret_key} + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_password: + file: ./secrets/cache_password + cache_url: + file: ./secrets/cache_url + valkey_config: + file: ./secrets/valkey.conf + name: valkey.conf +volumes: + broker_data: + s3_data: + postgres_data: diff --git a/infra/fetch-source.go b/infra/fetch-source.go new file mode 100644 index 00000000..48506cac --- /dev/null +++ b/infra/fetch-source.go @@ -0,0 +1,107 @@ +// Fetch the immutable module ZIP in bounded HTTP ranges. The subsequent +// go mod download validates its h1 checksum through the Go checksum database. +package main + +import ( + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "sync" + "time" +) + +func main() { + const version = "v0.0.0-20251015172955-9e49d5e7a648" + const total int64 = 25342606 + const chunk int64 = 512 * 1024 + const source = "https://proxy.golang.org/github.com/minio/minio/@v/" + version + ".zip" + directory := "/go/pkg/mod/cache/download/github.com/minio/minio/@v" + if err := os.MkdirAll(directory, 0755); err != nil { + panic(err) + } + target := filepath.Join(directory, version+".zip") + if info, err := os.Stat(target); err == nil && info.Size() == total { + fmt.Println("Source ZIP cached; Go will validate checksum") + return + } + client := &http.Client{Timeout: 90 * time.Second} + jobs := make(chan int64) + errors := make(chan error, 64) + var group sync.WaitGroup + for worker := 0; worker < 8; worker++ { + group.Add(1) + go func() { + defer group.Done() + for start := range jobs { + end := min(start+chunk, total) - 1 + part := fmt.Sprintf("%s.part-%d", target, start) + if info, err := os.Stat(part); err == nil && info.Size() == end-start+1 { + continue + } + var last error + for attempt := 0; attempt < 4; attempt++ { + req, _ := http.NewRequest("GET", source, nil) + req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + response, err := client.Do(req) + if err != nil { + last = fmt.Errorf("source range request failed") + continue + } + expected := fmt.Sprintf("bytes %d-%d/%d", start, end, total) + if response.StatusCode != 206 || response.Header.Get("Content-Range") != expected { + response.Body.Close() + last = fmt.Errorf("unexpected source range response: HTTP %d", response.StatusCode) + continue + } + data, err := io.ReadAll(io.LimitReader(response.Body, end-start+2)) + response.Body.Close() + if err != nil || int64(len(data)) != end-start+1 { + last = fmt.Errorf("incomplete source range") + continue + } + if err := os.WriteFile(part, data, 0644); err != nil { + last = err + continue + } + last = nil + fmt.Printf("Source part %d verified\n", start/chunk+1) + break + } + if last != nil { + errors <- last + } + } + }() + } + for start := int64(0); start < total; start += chunk { + jobs <- start + } + close(jobs) + group.Wait() + close(errors) + for err := range errors { + panic(err) + } + out, err := os.Create(target + ".partial") + if err != nil { + panic(err) + } + for start := int64(0); start < total; start += chunk { + part, err := os.Open(fmt.Sprintf("%s.part-%d", target, start)) + if err != nil { + panic(err) + } + if _, err := io.Copy(out, part); err != nil { + panic(err) + } + part.Close() + } + if err := out.Close(); err != nil { + panic(err) + } + if err := os.Rename(target+".partial", target); err != nil { + panic(err) + } +} diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..7528702e --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,36 @@ +global + log stdout format raw local0 + maxconn 512 + stats socket /tmp/admin.sock mode 600 level admin + +defaults + mode http + log global + option httplog + timeout connect 1s + timeout client 15s + timeout server 8s + timeout http-request 10s + timeout http-keep-alive 5s + retries 0 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s + +frontend public + bind :8080 + default_backend messenger + +backend messenger + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server check inter 1s fastinter 250ms fall 2 rise 1 resolvers docker init-addr libc,none resolve-prefer ipv4 + server-template api 1-8 api:8080 diff --git a/infra/s3-bootstrap.sh b/infra/s3-bootstrap.sh new file mode 100644 index 00000000..5732c258 --- /dev/null +++ b/infra/s3-bootstrap.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu +root_user=$(cat /run/secrets/s3_root_user) +root_password=$(cat /run/secrets/s3_root_password) +api_key=$(cat /run/secrets/s3_access_key) +api_secret=$(cat /run/secrets/s3_secret_key) +# Credentials are passed only inside this isolated bootstrap container. +mc alias set local http://s3:9000 "$root_user" "$root_password" >/dev/null 2>&1 +mc mb --ignore-existing local/messenger >/dev/null 2>&1 +mc anonymous set none local/messenger >/dev/null 2>&1 +mc admin user add local "$api_key" "$api_secret" >/dev/null 2>&1 +mc admin policy create local messenger-api /policy.json >/dev/null 2>&1 +mc admin policy attach local messenger-api --user "$api_key" >/dev/null 2>&1 +printf '%s\n' 'Private bucket and scoped application policy ready' diff --git a/infra/s3-entrypoint.sh b/infra/s3-entrypoint.sh new file mode 100644 index 00000000..ea521ab9 --- /dev/null +++ b/infra/s3-entrypoint.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +export MINIO_ROOT_USER="$(cat /run/secrets/s3_root_user)" +export MINIO_ROOT_PASSWORD="$(cat /run/secrets/s3_root_password)" +export MINIO_BROWSER_REDIRECT_URL=http://127.0.0.1:9001 +exec minio server /data --address :9000 --console-address 127.0.0.1:9001 diff --git a/infra/s3-policy.json b/infra/s3-policy.json new file mode 100644 index 00000000..f7ec8fb5 --- /dev/null +++ b/infra/s3-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": ["arn:aws:s3:::messenger"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger/objects/*"] + } + ] +} diff --git a/infra/s3.Dockerfile b/infra/s3.Dockerfile new file mode 100644 index 00000000..d971a03e --- /dev/null +++ b/infra/s3.Dockerfile @@ -0,0 +1,49 @@ +FROM golang:1.24.8-bookworm@sha256:4ed690d6649d63c312b99a6120025ec79ce3b542968a37da53d6236c7c61a848 AS toolchain +ENV CGO_ENABLED=0 GOMAXPROCS=2 GOFLAGS=-p=2 GOTOOLCHAIN=local + +FROM toolchain AS minio-build +WORKDIR /source +COPY infra/fetch-source.go /tmp/fetch-source.go +RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \ + --mount=type=cache,target=/root/.cache/go-build,sharing=locked \ + go run /tmp/fetch-source.go && \ + for attempt in 1 2 3 4; do \ + go mod download -json github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648 && break; \ + test "$attempt" -lt 4 || exit 1; sleep 1; \ + done && \ + cp -a /go/pkg/mod/github.com/minio/minio@v0.0.0-20251015172955-9e49d5e7a648/. /source/ +RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \ + --mount=type=cache,target=/root/.cache/go-build,sharing=locked \ + for attempt in 1 2 3; do \ + timeout 900s go build -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-10-15T17:29:55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.CommitID=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a' -o /out/minio . && break; \ + test "$attempt" -lt 3 || exit 1; sleep 1; \ + done + +FROM toolchain AS mc-build +WORKDIR /source +RUN git init . && git remote add origin https://github.com/minio/mc.git && \ + git fetch --depth 1 origin 7394ce0dd2a80935aded936b09fa12cbb3cb8096 && git checkout --detach FETCH_HEAD +RUN --mount=type=cache,target=/go/pkg/mod,sharing=locked \ + --mount=type=cache,target=/root/.cache/go-build,sharing=locked \ + go build -trimpath -ldflags='-s -w' -o /out/mc . + +FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS server +LABEL org.opencontainers.image.revision=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a org.opencontainers.image.version=RELEASE.2025-10-15T17-29-55Z +COPY --from=minio-build /out/minio /usr/local/bin/minio +COPY --from=minio-build /source/LICENSE /licenses/minio.LICENSE +COPY --chmod=755 infra/s3-entrypoint.sh /usr/local/bin/s3-entrypoint +RUN mkdir /data && chown 10001:10001 /data +USER 10001:10001 +ENTRYPOINT ["/usr/local/bin/s3-entrypoint"] +EXPOSE 9000 +HEALTHCHECK --interval=2s --timeout=2s --start-period=10s --retries=30 CMD ["python","-c","import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready',timeout=1).read()"] + +FROM python:3.14.7-slim-trixie@sha256:51dafde81dbdb6ebde285137a295cf18a47ca95234fe388a343719cb97305b3d AS bootstrap +LABEL org.opencontainers.image.revision=7394ce0dd2a80935aded936b09fa12cbb3cb8096 org.opencontainers.image.version=RELEASE.2025-08-13T08-35-41Z +COPY --from=mc-build /out/mc /usr/local/bin/mc +COPY --from=mc-build /source/LICENSE /licenses/mc.LICENSE +COPY --chmod=755 infra/s3-bootstrap.sh /usr/local/bin/s3-bootstrap +COPY infra/s3-policy.json /policy.json +USER 10001:10001 +ENV MC_CONFIG_DIR=/tmp/mc +ENTRYPOINT ["/usr/local/bin/s3-bootstrap"] diff --git a/migrations/001_core.sql b/migrations/001_core.sql new file mode 100644 index 00000000..f1c3acf1 --- /dev/null +++ b/migrations/001_core.sql @@ -0,0 +1,36 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL +); +CREATE TABLE members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + position integer NOT NULL CHECK (position BETWEEN 0 AND 99), + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); +CREATE INDEX members_user_chat_idx ON members(user_id,chat_id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL, + FOREIGN KEY (chat_id, sender_id) REFERENCES members(chat_id,user_id) +); +CREATE INDEX messages_history_idx ON messages(chat_id,created_at,id); +CREATE TABLE sessions ( + token_digest bytea PRIMARY KEY CHECK (octet_length(token_digest)=32), + user_id uuid NOT NULL REFERENCES users(id), + created_at timestamptz NOT NULL, + expires_at timestamptz NOT NULL CHECK (expires_at > created_at) +); +CREATE INDEX sessions_expiry_idx ON sessions(expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..759451e0 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,10 @@ +CREATE TABLE idempotency ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + body_hash bytea NOT NULL CHECK (octet_length(body_hash)=32), + message_id uuid NOT NULL REFERENCES messages(id), + expires_at timestamptz NOT NULL, + PRIMARY KEY(user_id, chat_id, key) +); +CREATE INDEX idempotency_expiry_idx ON idempotency(expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..f5d88aa8 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,32 @@ +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CHECK (char_length(text) <= 4000); +ALTER TABLE messages ADD UNIQUE(chat_id,id); +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL, + filename text NOT NULL CHECK(char_length(filename) BETWEEN 1 AND 255), + content_type text NOT NULL CHECK(content_type IN ('image/png','image/jpeg')), + size_bytes bigint NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK(sha256 ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL UNIQUE, + state text NOT NULL CHECK(state IN ('pending','ready','failed')), + error_code text, + uploader_instance text NOT NULL, + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE(chat_id,id), + FOREIGN KEY(chat_id,owner_id) REFERENCES members(chat_id,user_id) +); +CREATE INDEX attachments_state_created_idx ON attachments(state,created_at,id); +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + chat_id uuid NOT NULL, + attachment_id uuid NOT NULL, + position integer NOT NULL CHECK(position BETWEEN 0 AND 9), + PRIMARY KEY(message_id,attachment_id), + UNIQUE(message_id,position), + FOREIGN KEY(chat_id,message_id) REFERENCES messages(chat_id,id), + FOREIGN KEY(chat_id,attachment_id) REFERENCES attachments(chat_id,id) +); diff --git a/migrations/004_pipeline.sql b/migrations/004_pipeline.sql new file mode 100644 index 00000000..477316f9 --- /dev/null +++ b/migrations/004_pipeline.sql @@ -0,0 +1,54 @@ +ALTER TABLE attachments DROP CONSTRAINT attachments_state_check; +ALTER TABLE attachments ADD CHECK(state IN ('queued','processing','ready','failed')); +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK(pipeline_version>=1); +ALTER TABLE attachments ADD COLUMN published_version integer; +CREATE TABLE upload_reservations ( + id uuid PRIMARY KEY, object_key text UNIQUE NOT NULL, + expires_at timestamptz NOT NULL DEFAULT clock_timestamp()+interval '120 seconds' +); +CREATE TABLE jobs ( + id uuid PRIMARY KEY, attachment_id uuid NOT NULL REFERENCES attachments(id), + pipeline_version integer NOT NULL CHECK(pipeline_version>=1), source_key text NOT NULL, + state text NOT NULL CHECK(state IN ('queued','processing','ready','failed','obsolete')), + attempt integer NOT NULL DEFAULT 0, cycle integer NOT NULL DEFAULT 1, + fence bigint NOT NULL DEFAULT 0, lease_owner text, lease_until timestamptz, + next_at timestamptz NOT NULL DEFAULT clock_timestamp(), error_code text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + UNIQUE(attachment_id,pipeline_version) +); +CREATE INDEX jobs_due_idx ON jobs(state,next_at); +CREATE TABLE outbox ( + id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES jobs(id), + payload jsonb NOT NULL, due_at timestamptz NOT NULL DEFAULT clock_timestamp(), + published_at timestamptz +); +CREATE INDEX outbox_due_idx ON outbox(due_at,id) WHERE published_at IS NULL; +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments(id), pipeline_version integer NOT NULL, + name text NOT NULL CHECK(name IN ('original','thumbnail','crop','optimized')), + object_key text UNIQUE NOT NULL, content_type text NOT NULL, + size_bytes bigint NOT NULL, sha256 text NOT NULL, + width integer, height integer, + PRIMARY KEY(attachment_id,pipeline_version,name) +); +CREATE TABLE effect_publications ( + attachment_id uuid NOT NULL, pipeline_version integer NOT NULL, + job_id uuid NOT NULL REFERENCES jobs(id), fence bigint NOT NULL, + committed_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY(attachment_id,pipeline_version) +); +CREATE TABLE pipeline_audit ( + id bigserial PRIMARY KEY, job_id uuid NOT NULL, action text NOT NULL, + operator text NOT NULL, reason text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +-- Fault injection is disabled unless PIPELINE_FAULTS=1; no public endpoint. +CREATE TABLE pipeline_faults ( + id bigserial PRIMARY KEY, target uuid NOT NULL, phase text NOT NULL, + action text NOT NULL CHECK(action IN ('wait','crash','transient','permanent','hang')), + remaining integer NOT NULL DEFAULT 1, hit_at timestamptz, released_at timestamptz +); +CREATE TABLE pipeline_events ( + id bigserial PRIMARY KEY, job_id uuid, event text NOT NULL, + detail jsonb NOT NULL DEFAULT '{}', created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..2bd4436c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,44 @@ +[project] +name = "mtusi-messenger-lab5" +version = "1.0.0" +requires-python = ">=3.14,<3.15" +dependencies = [ + "fastapi==0.143.3", + "pydantic==2.14.1", + "uvicorn==0.54.0", + "psycopg[binary,pool]==3.3.6", + "argon2-cffi>=25,<26", + "redis==8.1.0", + "minio==7.2.20", + "pillow==13.0.0", +] + +[dependency-groups] +dev = ["pytest>=8,<10", "httpx>=0.28,<1", "ruff>=0.14,<1", "openapi-spec-validator==0.7.2", "psutil>=7,<8", "matplotlib>=3.10,<4"] + +[tool.ruff] +line-length = 79 +target-version = "py314" +include = ["src/**/*.py", "student-tests/**/*.py", "research/**/*.py", "scripts/verify_*.py"] + +[tool.ruff.lint] +select = ["E", "F", "I", "B", "UP"] + +[tool.ruff.per-file-target-version] +"scripts/*.py" = "py39" + +# SQL и генерируемые Markdown-таблицы в инструментах не разрываем. +[tool.ruff.lint.per-file-ignores] +"research/*.py" = ["E501"] +"scripts/*.py" = ["E501"] + +[tool.pytest.ini_options] +testpaths = ["student-tests"] +addopts = "-ra" + +# Keep SQL statements together; the formatter controls executable statements. +"src/attachments.py" = ["E501"] +"src/pipeline.py" = ["E501"] +"src/worker.py" = ["E501"] +"src/operator.py" = ["E501"] +"src/reconcile.py" = ["E501"] diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..d21913e5 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,63 @@ +"""Создаёт только отсутствующие локальные файлы; секреты не печатает.""" + +import os +import secrets +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def create(path, value): + try: + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) + except FileExistsError: + return + with os.fdopen(fd, "w") as stream: + stream.write(value + "\n") + + +def main(): + directory = ROOT / "secrets" + directory.mkdir(mode=0o700, exist_ok=True) + pg = directory / "postgres_password" + cache = directory / "cache_password" + create(pg, secrets.token_urlsafe(32)) + create(cache, secrets.token_urlsafe(32)) + create( + directory / "database_url", + "postgresql://messenger:" + + pg.read_text().strip() + + "@postgres:5432/messenger", + ) + create( + directory / "cache_url", + "redis://:" + cache.read_text().strip() + "@cache:6379/0", + ) + create( + directory / "valkey.conf", + "bind 0.0.0.0\nprotected-mode yes\nport 6379\nmaxmemory 64mb\n" + 'maxmemory-policy allkeys-lru\nsave ""\nappendonly no\nrequirepass ' + + cache.read_text().strip(), + ) + create(directory / "s3_root_user", secrets.token_hex(10)) + create(directory / "s3_root_password", secrets.token_urlsafe(32)) + create(directory / "s3_access_key", secrets.token_hex(10)) + create(directory / "s3_secret_key", secrets.token_urlsafe(32)) + create(directory / "broker_password", secrets.token_urlsafe(32)) + create( + directory / "broker_url", + "redis://:" + + (directory / "broker_password").read_text().strip() + + "@broker:6379/0", + ) + create( + directory / "broker.conf", + "bind 0.0.0.0\nprotected-mode yes\nport 6379\nappendonly yes\nappendfsync always\nmaxmemory 192mb\nmaxmemory-policy noeviction\ndir /data\nrequirepass " + + (directory / "broker_password").read_text().strip(), + ) + create(ROOT / ".env", (ROOT / ".env.example").read_text().rstrip()) + print("Local configuration ready; existing files preserved") + + +if __name__ == "__main__": + main() diff --git a/scripts/cluster.py b/scripts/cluster.py new file mode 100644 index 00000000..e418d0e0 --- /dev/null +++ b/scripts/cluster.py @@ -0,0 +1,83 @@ +"""Явные операторские действия только над API текущего Compose-проекта.""" + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def command(*args): + return subprocess.check_output(args, cwd=ROOT, text=True).strip() + + +def containers(): + ids = command("docker", "compose", "ps", "-q", "api").split() + return json.loads(command("docker", "inspect", *ids)) if ids else [] + + +def alias(cid): + code = ( + "import urllib.request; r=urllib.request.urlopen(" + "'http://127.0.0.1:8080/health/live',timeout=2); " + "print(r.headers['X-Instance-Id'])" + ) + return command("docker", "exec", cid, "python", "-c", code) + + +def wait_count(count): + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + data = containers() + if len(data) == count and all( + c["State"].get("Health", {}).get("Status") == "healthy" + for c in data + ): + # DNS и проверки HAProxy должны успеть увидеть новый адрес. + time.sleep(3) + return + time.sleep(0.5) + raise TimeoutError("Replica readiness") + + +def main(): + action = sys.argv[1] + data = containers() + if action == "stop": + target = os.environ["TARGET_INSTANCE_ID"] + matches = [c for c in data if alias(c["Id"]) == target] + if len(matches) != 1 or len(data) < 2: + raise RuntimeError("Target must identify one running replica") + command("docker", "stop", "--time", "10", matches[0]["Id"]) + print("Selected API stopped") + elif action == "rolling": + count = len(data) + if count < 2: + raise RuntimeError("Need at least two replicas") + # Сначала добавляется готовая реплика, затем удаляется одна старая. + for old in data: + command( + "docker", + "compose", + "up", + "-d", + "--no-deps", + "--no-recreate", + "--scale", + f"api={count + 1}", + "api", + ) + wait_count(count + 1) + command("docker", "stop", "--time", "10", old["Id"]) + command("docker", "rm", old["Id"]) + wait_count(count) + print("Sequential replacement completed") + else: + raise ValueError("Unknown action") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..f123bd9a --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +docker compose restart api postgres cache s3 broker relay worker lb +docker compose up -d --no-build --no-recreate --wait diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..40ce4909 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +exec python3 scripts/cluster.py rolling diff --git a/scripts/stand.py b/scripts/stand.py new file mode 100644 index 00000000..203486a1 --- /dev/null +++ b/scripts/stand.py @@ -0,0 +1,137 @@ +import json +import os +import subprocess +import time +from pathlib import Path +from uuid import uuid4 + +import httpx + +ROOT = Path(__file__).resolve().parents[1] +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8085") + + +def command(args, *, input=None): + result = subprocess.run( + args, cwd=ROOT, input=input, capture_output=True, text=True + ) + if result.returncode: + raise RuntimeError( + "Command failed: " + + args[0] + + "; " + + result.stderr.replace(str(ROOT), "") + ) + return result.stdout.strip() + + +def compose(*args): + return command(["docker", "compose", *args]) + + +def sql(query, *, database="messenger"): + return compose( + "exec", + "-T", + "postgres", + "psql", + "-U", + "messenger", + "-d", + database, + "-A", + "-t", + "-v", + "ON_ERROR_STOP=1", + "-c", + query, + ) + + +def cache(command_name, *args): + script = ( + 'export REDISCLI_AUTH="$(cat /run/secrets/cache_password)"; ' + 'exec valkey-cli "$@"' + ) + return compose( + "exec", "-T", "cache", "sh", "-c", script, "sh", command_name, *args + ) + + +def ready(base=BASE, *, timeout=30): + end = time.monotonic() + timeout + with httpx.Client(base_url=base, trust_env=False, timeout=6) as client: + while time.monotonic() < end: + try: + if client.get("/health/ready").status_code == 200: + return + except httpx.HTTPError: + pass + time.sleep(0.2) + raise TimeoutError("API readiness") + + +def actor(client): + username = "verify_" + uuid4().hex[:20] + password = "Synthetic-" + uuid4().hex + ":🌍" + result = client.post( + "/api/v1/users", + json={ + "username": username, + "display_name": "Synthetic", + "password": password, + }, + ) + assert result.status_code == 201 + user = result.json() + session = client.post( + "/api/v1/auth/sessions", auth=httpx.BasicAuth(username, password) + ) + assert session.status_code == 201 + return user, password, session.json() + + +def auth(session): + return {"Authorization": "Bearer " + session["token"]} + + +def extra_api(*, ttl=3600, pool_size=1): + name = "mtusi-lab5-test-" + uuid4().hex[:10] + compose( + "run", + "-d", + "--no-deps", + "--name", + name, + "-e", + "SESSION_TTL_SECONDS=" + str(ttl), + "-e", + "DB_POOL_SIZE=" + str(pool_size), + "-p", + "127.0.0.1::8080", + "api", + ) + try: + info = json.loads(command(["docker", "inspect", name]))[0] + base = ( + "http://127.0.0.1:" + + info["NetworkSettings"]["Ports"]["8080/tcp"][0]["HostPort"] + ) + ready(base) + return name, base + except Exception: + command(["docker", "rm", "-f", name]) + raise + + +def instances(): + data = {} + for role in ["api", "postgres", "cache"]: + cid = compose("ps", "-q", role) + info = json.loads(command(["docker", "inspect", cid]))[0] + data[role] = { + "container": cid[:12], + "pid": info["State"]["Pid"], + "started_at": info["State"]["StartedAt"], + } + return data diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..a066590a --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +exec python3 scripts/cluster.py stop diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/__main__.py b/src/__main__.py new file mode 100644 index 00000000..20228f97 --- /dev/null +++ b/src/__main__.py @@ -0,0 +1,44 @@ +import json +import logging +import os +import signal +import sys + + +def early_stop(signum, _frame): + print( + json.dumps({"event": "stopped_before_startup", "signal": signum}), + flush=True, + ) + sys.exit(0) + + +signal.signal(signal.SIGTERM, early_stop) +signal.signal(signal.SIGINT, early_stop) + +import uvicorn # noqa: E402 + +from src.logging_config import JsonFormatter # noqa: E402 + + +def main() -> None: + host = os.environ.get("APP_HOST", "127.0.0.1") + port = int(os.environ.get("APP_PORT", "8080")) + if not 1 <= port <= 65535: + raise ValueError("APP_PORT must be between 1 and 65535") + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(JsonFormatter()) + logging.basicConfig(level=logging.INFO, handlers=[handler], force=True) + uvicorn.run( + "src.api:app", + host=host, + port=port, + workers=1, + access_log=False, + timeout_graceful_shutdown=5, + log_config=None, + ) + + +if __name__ == "__main__": + main() diff --git a/src/api.py b/src/api.py new file mode 100644 index 00000000..e9f287ab --- /dev/null +++ b/src/api.py @@ -0,0 +1,424 @@ +import hashlib +import json +import logging +import tempfile +import time +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from datetime import datetime +from functools import partial +from urllib.parse import quote +from uuid import UUID, uuid4 + +from anyio import Semaphore, fail_after, to_thread +from fastapi import FastAPI, Request +from fastapi.encoders import jsonable_encoder +from pydantic import ValidationError +from starlette.exceptions import HTTPException +from starlette.requests import ClientDisconnect +from starlette.responses import JSONResponse, Response, StreamingResponse + +from src import images +from src.attachments import Attachments +from src.cache import UserCache +from src.cursor import parse_uuid +from src.database import Database +from src.domain import timestamp +from src.errors import DomainError, invalid +from src.object_store import ObjectStore +from src.password import Passwords +from src.repository import PostgresRepository +from src.schemas import CreateChat, CreateMessage, CreateUser, Input +from src.service import MessengerService +from src.settings import Settings + +logger = logging.getLogger("messenger") +INSTANCE_ID = uuid4().hex + + +def response(data: object, *, status: int = 200) -> JSONResponse: + return JSONResponse( + jsonable_encoder( + data, custom_encoder={datetime: timestamp, UUID: str} + ), + status_code=status, + ) + + +async def body(request: Request, model: type[Input]) -> Input: + try: + data = await request.body() + return model.model_validate( + json.loads( + data.decode("utf-8"), + parse_constant=lambda _: (_ for _ in ()).throw(ValueError()), + ) + ) + except ValueError, ValidationError, UnicodeError, RecursionError: + raise invalid() from None + + +def pagination(request: Request) -> tuple[int, str | None]: + limits = request.query_params.getlist("limit") + cursors = request.query_params.getlist("cursor") + if len(limits) > 1 or len(cursors) > 1: + raise invalid() + raw = limits[0] if limits else "50" + if not raw.isascii() or not raw.isdecimal(): + raise invalid() + raw = raw.lstrip("0") or "0" + if len(raw) > 3: + raise invalid() + limit = int(raw) + if not 1 <= limit <= 100: + raise invalid() + return limit, cursors[0] if cursors else None + + +def service(request: Request) -> MessengerService: + return request.app.state.service + + +async def run_io(function, **kwargs): + return await to_thread.run_sync(partial(function, **kwargs)) + + +async def actor(request: Request): + return await run_io( + service(request).identify, value=request.headers.get("authorization") + ) + + +@asynccontextmanager +async def lifespan(app: FastAPI) -> AsyncIterator[None]: + settings = Settings.load() + database = Database(url=settings.database_url, size=settings.pool_size) + cache = UserCache(url=settings.cache_url) + store = ObjectStore() + try: + await run_io(database.open) + passwords = await run_io(Passwords) + app.state.database = database + app.state.store = store + app.state.upload_slots = Semaphore(2) + app.state.attachments = Attachments( + database=database, store=store, instance=INSTANCE_ID + ) + app.state.service = MessengerService( + repository=PostgresRepository(database=database, cache=cache), + passwords=passwords, + session_ttl=settings.session_ttl, + ) + app.state.ready = True + yield + finally: + app.state.ready = False + await run_io(store.close) + await run_io(cache.close) + await run_io(database.close) + + +app = FastAPI( + lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None +) + + +@app.middleware("http") +async def request_context(request: Request, call_next): + request_id = str(uuid4()) + request.state.request_id = request_id + started = time.perf_counter() + try: + result = await call_next(request) + except Exception: + logger.error( + json.dumps({"event": "internal_error", "request_id": request_id}) + ) + result = response( + { + "error": { + "code": "internal_error", + "message": "Internal error", + "request_id": request_id, + } + }, + status=500, + ) + result.headers["X-Request-Id"] = request_id + result.headers["X-Instance-Id"] = INSTANCE_ID + route = request.scope.get("route") + logger.info( + json.dumps( + { + "event": "request", + "request_id": request_id, + "method": request.method, + "route": getattr(route, "path", "unmatched"), + "status": result.status_code, + "elapsed_ms": round((time.perf_counter() - started) * 1000, 3), + } + ) + ) + return result + + +@app.exception_handler(DomainError) +async def domain_error(request: Request, error: DomainError): + result = response( + { + "error": { + "code": error.code, + "message": error.message, + "request_id": request.state.request_id, + } + }, + status=error.status, + ) + result.headers.update(error.headers) + return result + + +@app.exception_handler(HTTPException) +async def http_error(request: Request, error: HTTPException): + code = "not_found" if error.status_code == 404 else "invalid_request" + return response( + { + "error": { + "code": code, + "message": "Resource not found" + if error.status_code == 404 + else "Invalid request", + "request_id": request.state.request_id, + } + }, + status=error.status_code, + ) + + +@app.get("/health/live") +async def live(): + return response({"status": "ok"}) + + +@app.get("/health/ready") +async def ready(): + if not getattr(app.state, "ready", False): + raise DomainError(503, "unavailable", "Not ready") + await run_io(app.state.database.ping) + await run_io(app.state.store.ping) + return response({"status": "ok"}) + + +@app.post("/api/v1/users") +async def register(request: Request): + data = await body(request, CreateUser) + return response( + await run_io(service(request).register, **data.model_dump()), + status=201, + ) + + +@app.get("/api/v1/users/me") +async def current(request: Request): + return response(await actor(request)) + + +@app.post("/api/v1/chats") +async def create_chat(request: Request): + user = await actor(request) + data = await body(request, CreateChat) + return response( + await run_io( + service(request).create_chat, actor_id=user.id, **data.model_dump() + ), + status=201, + ) + + +@app.get("/api/v1/chats") +async def list_chats(request: Request): + user = await actor(request) + limit, cursor = pagination(request) + return response( + await run_io( + service(request).chats, + actor_id=user.id, + limit=limit, + cursor=cursor, + ) + ) + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(request: Request, chat_id: str): + user = await actor(request) + return response( + await run_io( + service(request).get_chat, + actor_id=user.id, + chat_id=parse_uuid(chat_id), + ) + ) + + +@app.post("/api/v1/chats/{chat_id}/messages") +async def send(request: Request, chat_id: str): + user = await actor(request) + cid = parse_uuid(chat_id) + await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid) + data = await body(request, CreateMessage) + return response( + await run_io( + service(request).send, + actor_id=user.id, + chat_id=cid, + idempotency_key=request.headers.get("idempotency-key"), + **data.model_dump(), + ), + status=201, + ) + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def history(request: Request, chat_id: str): + user = await actor(request) + cid = parse_uuid(chat_id) + chat = await run_io( + service(request).get_chat, actor_id=user.id, chat_id=cid + ) + limit, cursor = pagination(request) + return response( + await run_io( + service(request).history, + actor_id=user.id, + chat=chat, + limit=limit, + cursor=cursor, + ) + ) + + +@app.post("/api/v1/auth/sessions") +async def login(request: Request): + return response( + await run_io( + service(request).login, + authorization=request.headers.get("authorization"), + ), + status=201, + ) + + +@app.delete("/api/v1/auth/sessions/current") +async def logout(request: Request): + await run_io( + service(request).logout, + authorization=request.headers.get("authorization"), + ) + return Response(status_code=204) + + +@app.post("/api/v1/chats/{chat_id}/attachments") +async def upload(request: Request, chat_id: str): + user = await actor(request) + cid = parse_uuid(chat_id) + await run_io(service(request).get_chat, actor_id=user.id, chat_id=cid) + names = request.query_params.getlist("filename") + if len(names) != 1: + raise invalid() + name = images.filename(names[0]) + declared = ( + request.headers.get("content-type", "") + .partition(";")[0] + .strip() + .lower() + ) + if declared not in {"image/png", "image/jpeg"}: + raise DomainError( + 415, "unsupported_media_type", "PNG or JPEG required" + ) + length = request.headers.get("content-length") + if length and (not length.isdecimal() or len(length) > 12): + raise invalid() + if length and int(length) > images.MAX_BYTES: + raise DomainError(413, "payload_too_large", "Byte limit exceeded") + try: + with fail_after(20): + async with app.state.upload_slots: + with tempfile.SpooledTemporaryFile( + max_size=1024 * 1024, mode="w+b" + ) as stream: + size = 0 + digest = hashlib.sha256() + async for chunk in request.stream(): + size += len(chunk) + if size > images.MAX_BYTES: + raise DomainError( + 413, "payload_too_large", "Byte limit exceeded" + ) + digest.update(chunk) + stream.write(chunk) + await run_io( + images.verify, stream=stream, declared=declared + ) + result = await run_io( + app.state.attachments.upload, + chat_id=cid, + owner_id=user.id, + filename=name, + content_type=declared, + size=size, + sha256=digest.hexdigest(), + stream=stream, + ) + return response(result, status=202) + except ClientDisconnect: + raise invalid() from None + except TimeoutError: + raise DomainError( + 503, "unavailable", "Upload deadline exceeded" + ) from None + + +@app.get("/api/v1/attachments/{attachment_id}") +async def attachment_metadata(request: Request, attachment_id: str): + user = await actor(request) + return response( + await run_io( + app.state.attachments.metadata, + identity=parse_uuid(attachment_id), + actor_id=user.id, + ) + ) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def attachment_content(request: Request, attachment_id: str): + user = await actor(request) + variants = request.query_params.getlist("variant") + if len(variants) > 1: + raise invalid() + row, upstream = await run_io( + app.state.attachments.content, + identity=parse_uuid(attachment_id), + actor_id=user.id, + variant=variants[0] if variants else "original", + ) + + def chunks(): + try: + while data := upstream.read(65536): + yield data + finally: + upstream.close() + upstream.release_conn() + + return StreamingResponse( + chunks(), + media_type=row["content_type"], + headers={ + "Content-Length": str(row["size_bytes"]), + "Content-Disposition": "inline; filename*=UTF-8''" + + quote(row["filename"], safe=""), + }, + ) diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 00000000..9bd841d8 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,144 @@ +from uuid import uuid4 + +from src.errors import DomainError, not_found +from src.pipeline import enqueue, fault + + +def dto(row, variants=()): + return { + k: row[k] + for k in [ + "id", + "chat_id", + "owner_id", + "filename", + "content_type", + "size_bytes", + "sha256", + "created_at", + "updated_at", + ] + } | { + "status": row["state"], + "variants": [ + { + k: v[k] + for k in [ + "name", + "content_type", + "size_bytes", + "sha256", + "width", + "height", + ] + } + for v in variants + if v["name"] != "original" + ], + "error_code": row["error_code"] if row["state"] == "failed" else None, + } + + +class Attachments: + def __init__(self, *, database, store, instance): + self.database, self.store, self.instance = database, store, instance + + def upload( + self, + *, + chat_id, + owner_id, + filename, + content_type, + size, + sha256, + stream, + ): + identity, job_id = uuid4(), uuid4() + key = f"objects/staging/{identity}/original" + with self.database.connection() as conn: + conn.execute( + "INSERT INTO upload_reservations(id,object_key) VALUES(%s,%s)", + (identity, key), + ) + self.store.put( + key=key, + stream=stream, + size=size, + content_type=content_type, + sha256=sha256, + ) + fault(self.database, chat_id, "api_after_staging") + with self.database.connection() as conn, conn.transaction(): + reservation = conn.execute( + "SELECT * FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp() FOR UPDATE", + (identity,), + ).fetchone() + if reservation is None: + raise DomainError( + 503, "unavailable", "Upload reservation expired" + ) + row = conn.execute( + "INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,uploader_instance,expires_at) " + "VALUES(%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,clock_timestamp()) RETURNING *", + ( + identity, + chat_id, + owner_id, + filename, + content_type, + size, + sha256, + key, + self.instance, + ), + ).fetchone() + conn.execute( + "INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state) VALUES(%s,%s,1,%s,'queued')", + (job_id, identity, key), + ) + enqueue(conn, job_id) + conn.execute( + "DELETE FROM upload_reservations WHERE id=%s", (identity,) + ) + fault(self.database, chat_id, "api_after_commit") + # Captured committed snapshot remains queued even if worker already finished. + return dto(row) + + def row(self, *, identity, actor_id): + with self.database.connection() as conn: + row = conn.execute( + "SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id AND m.user_id=%s WHERE a.id=%s", + (actor_id, identity), + ).fetchone() + if row is None: + raise not_found() + return row + + def metadata(self, *, identity, actor_id): + row = self.row(identity=identity, actor_id=actor_id) + variants = [] + if row["state"] == "ready": + with self.database.connection() as conn: + variants = conn.execute( + "SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name", + (identity, row["published_version"]), + ).fetchall() + return dto(row, variants) + + def content(self, *, identity, actor_id, variant): + row = self.row(identity=identity, actor_id=actor_id) + if row["state"] != "ready": + raise DomainError(409, "conflict", "Attachment is not ready") + with self.database.connection() as conn: + data = conn.execute( + "SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s AND name=%s", + (identity, row["published_version"], variant), + ).fetchone() + if data is None: + raise not_found() + if self.store.head(key=data["object_key"]) is None: + raise DomainError(503, "unavailable", "Variant unavailable") + return {**data, "filename": row["filename"]}, self.store.get( + key=data["object_key"] + ) diff --git a/src/cache.py b/src/cache.py new file mode 100644 index 00000000..adbea82b --- /dev/null +++ b/src/cache.py @@ -0,0 +1,66 @@ +import json +import threading +import time +from datetime import datetime +from uuid import UUID + +import redis +from redis.backoff import NoBackoff +from redis.retry import Retry + +from src.domain import User, timestamp + + +class UserCache: + def __init__(self, *, url: str): + self.client = redis.Redis.from_url( + url, + socket_connect_timeout=0.15, + socket_timeout=0.15, + max_connections=16, + retry=Retry(NoBackoff(), 0), + decode_responses=True, + ) + self.disabled_until = 0.0 + self.lock = threading.Lock() + + def close(self): + self.client.close() + + def _command(self, function, *args, **kwargs): + if time.monotonic() < self.disabled_until: + return None + try: + return function(*args, **kwargs) + except redis.RedisError, OSError: + with self.lock: + self.disabled_until = time.monotonic() + 5 + return None + + def get(self, *, user_id: UUID) -> User | None: + raw = self._command(self.client.get, "user:" + str(user_id)) + if raw is None: + return None + try: + row = json.loads(raw) + if UUID(row["id"]) != user_id: + return None + return User( + user_id, + row["username"], + row["display_name"], + datetime.fromisoformat(row["created_at"]), + ) + except ValueError, TypeError, KeyError: + return None + + def put(self, *, user: User): + raw = json.dumps( + { + "id": str(user.id), + "username": user.username, + "display_name": user.display_name, + "created_at": timestamp(user.created_at), + } + ) + self._command(self.client.set, "user:" + str(user.id), raw, ex=30) diff --git a/src/cursor.py b/src/cursor.py new file mode 100644 index 00000000..9d9c3c9c --- /dev/null +++ b/src/cursor.py @@ -0,0 +1,64 @@ +import base64 +import binascii +import json +import re +from datetime import datetime +from uuid import UUID + +from src.domain import Key, timestamp +from src.errors import invalid + + +def parse_uuid(value: str) -> UUID: + if not isinstance(value, str) or not re.fullmatch( + r"[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}", value + ): + raise invalid() + return UUID(value) + + +def encode_cursor(*, scope: str, key: Key) -> str: + payload = { + "v": 1, + "scope": scope, + "at": timestamp(key[0]), + "id": str(key[1]), + } + return ( + base64.urlsafe_b64encode( + json.dumps(payload, separators=(",", ":")).encode() + ) + .decode() + .rstrip("=") + ) + + +def decode_cursor(*, value: str | None, scope: str) -> Key | None: + if value is None: + return None + try: + if ( + not value + or len(value) > 2048 + or not re.fullmatch(r"[A-Za-z0-9_-]+", value) + ): + raise ValueError + data = json.loads( + base64.b64decode( + value + "=" * (-len(value) % 4), altchars=b"-_", validate=True + ) + ) + if ( + not isinstance(data, dict) + or type(data.get("v")) is not int + or data["v"] != 1 + or data.get("scope") != scope + or not isinstance(data.get("at"), str) + ): + raise ValueError + at = datetime.fromisoformat(data["at"]) + if at.tzinfo is None or timestamp(at) != data["at"]: + raise ValueError + return at, parse_uuid(data["id"]) + except ValueError, TypeError, KeyError, binascii.Error, RecursionError: + raise invalid() from None diff --git a/src/database.py b/src/database.py new file mode 100644 index 00000000..130b5238 --- /dev/null +++ b/src/database.py @@ -0,0 +1,69 @@ +import time +from contextlib import contextmanager + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +from src.errors import DomainError + + +class DeadlineConnection(psycopg.Connection): + """Клиентский срок действует и при замороженном PostgreSQL.""" + + def wait(self, gen, interval=0.1, timeout=None): + deadline = getattr(self, "request_deadline", None) + remaining = ( + max(0.001, deadline - time.monotonic()) if deadline else 3.0 + ) + bound = min(remaining, timeout) if timeout is not None else remaining + try: + return super().wait(gen, interval=interval, timeout=bound) + except psycopg.OperationalError: + self.close() + raise + + +class Database: + def __init__(self, *, url: str, size: int): + self.pool = ConnectionPool( + url, + connection_class=DeadlineConnection, + kwargs={ + "autocommit": True, + "row_factory": dict_row, + "connect_timeout": 2, + "options": ( + "-c statement_timeout=2500 -c lock_timeout=1000 " + "-c idle_in_transaction_session_timeout=6000" + ), + }, + min_size=1, + max_size=size, + timeout=1, + open=False, + ) + + def open(self): + self.pool.open(wait=True, timeout=10) + + def close(self): + self.pool.close(timeout=2) + + @contextmanager + def connection(self): + try: + with self.pool.connection(timeout=1) as conn: + conn.request_deadline = time.monotonic() + 4 + try: + yield conn + finally: + conn.request_deadline = None + except psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout: + raise DomainError( + 503, "unavailable", "Database unavailable" + ) from None + + def ping(self): + with self.connection() as conn: + conn.execute("SELECT 1").fetchone() diff --git a/src/domain.py b/src/domain.py new file mode 100644 index 00000000..e595a2e6 --- /dev/null +++ b/src/domain.py @@ -0,0 +1,45 @@ +from dataclasses import dataclass +from datetime import UTC, datetime +from uuid import UUID + + +def utcnow() -> datetime: + return datetime.now(UTC) + + +def timestamp(value: datetime) -> str: + return value.isoformat(timespec="microseconds").replace("+00:00", "Z") + + +@dataclass(frozen=True, slots=True) +class User: + id: UUID + username: str + display_name: str + created_at: datetime + + +@dataclass(frozen=True, slots=True) +class Chat: + id: UUID + title: str + member_ids: tuple[UUID, ...] + created_at: datetime + + +@dataclass(frozen=True, slots=True) +class Message: + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: tuple[UUID, ...] = () + + +type Record = User | Chat | Message +type Key = tuple[datetime, UUID] + + +def order_key(record: Record) -> Key: + return record.created_at, record.id diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..693b2ccf --- /dev/null +++ b/src/errors.py @@ -0,0 +1,16 @@ +class DomainError(Exception): + def __init__( + self, status: int, code: str, message: str, headers=None + ) -> None: + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def invalid() -> DomainError: + return DomainError(400, "invalid_request", "Invalid request") + + +def not_found() -> DomainError: + return DomainError(404, "not_found", "Resource not found") diff --git a/src/images.py b/src/images.py new file mode 100644 index 00000000..bf7f57bb --- /dev/null +++ b/src/images.py @@ -0,0 +1,77 @@ +import struct +import threading +import unicodedata +import zlib + +from PIL import Image, UnidentifiedImageError + +from src.errors import DomainError, invalid + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +DECODE_SLOT = threading.BoundedSemaphore(1) + + +def filename(value): + if ( + not value + or not 1 <= len(value) <= 255 + or not value.strip() + or any( + c in "/\\" or unicodedata.category(c) in {"Cc", "Cs"} + for c in value + ) + ): + raise invalid() + return value + + +def verify(stream, *, declared): + if declared not in {"image/png", "image/jpeg"}: + raise DomainError( + 415, "unsupported_media_type", "PNG or JPEG required" + ) + with DECODE_SLOT: + try: + stream.seek(0) + with Image.open(stream) as image: + actual = {"PNG": "image/png", "JPEG": "image/jpeg"}.get( + image.format + ) + if actual != declared: + raise DomainError( + 415, "unsupported_media_type", "Image type mismatch" + ) + if image.width * image.height > MAX_PIXELS: + raise DomainError( + 413, "payload_too_large", "Pixel limit exceeded" + ) + image.verify() + # JPEG verify alone does not decode entropy data. + stream.seek(0) + with Image.open(stream) as image: + image.load() + except Image.DecompressionBombError: + raise DomainError( + 413, "payload_too_large", "Pixel limit exceeded" + ) from None + except ( + UnidentifiedImageError, + OSError, + ValueError, + SyntaxError, + struct.error, + zlib.error, + EOFError, + ): + raise DomainError( + 415, "unsupported_media_type", "Invalid image" + ) from None + except MemoryError: + raise DomainError( + 503, "unavailable", "Image memory budget exceeded" + ) from None + finally: + stream.seek(0) + return declared diff --git a/src/logging_config.py b/src/logging_config.py new file mode 100644 index 00000000..e89db63a --- /dev/null +++ b/src/logging_config.py @@ -0,0 +1,17 @@ +import json +import logging +from datetime import UTC, datetime + + +class JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + message = record.getMessage() + try: + event = json.loads(message) + if not isinstance(event, dict): + event = {"event": message} + except ValueError: + event = {"event": message} + event["timestamp"] = datetime.now(UTC).isoformat() + event["level"] = record.levelname.lower() + return json.dumps(event, ensure_ascii=False) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..30363ba6 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,51 @@ +import hashlib +import json +from pathlib import Path + +import psycopg + +from src.settings import secret + + +def main(): + with psycopg.connect( + secret("DATABASE_URL"), autocommit=True, connect_timeout=3 + ) as conn: + conn.execute("SELECT pg_advisory_lock(2301002)") + try: + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations " + "(name text PRIMARY KEY, checksum text NOT NULL, " + "applied_at timestamptz NOT NULL DEFAULT now())" + ) + for path in sorted(Path("migrations").glob("*.sql")): + content = path.read_bytes() + checksum = hashlib.sha256(content).hexdigest() + row = conn.execute( + "SELECT checksum FROM schema_migrations WHERE name=%s", + (path.name,), + ).fetchone() + if row: + if row[0] != checksum: + raise RuntimeError( + "Applied migration checksum mismatch" + ) + continue + with conn.transaction(): + conn.execute(content.decode()) + conn.execute( + "INSERT INTO schema_migrations(name,checksum) " + "VALUES(%s,%s)", + (path.name, checksum), + ) + print( + json.dumps( + {"event": "migration_applied", "name": path.name} + ) + ) + finally: + conn.execute("SELECT pg_advisory_unlock(2301002)") + + +if __name__ == "__main__": + main() diff --git a/src/object_store.py b/src/object_store.py new file mode 100644 index 00000000..998a7467 --- /dev/null +++ b/src/object_store.py @@ -0,0 +1,84 @@ +from contextlib import contextmanager + +import urllib3 +from minio import Minio +from minio.error import S3Error + +from src.errors import DomainError +from src.settings import secret + +BUCKET = "messenger" + + +class ObjectStore: + def __init__(self): + self.http = urllib3.PoolManager( + maxsize=32, + block=True, + timeout=urllib3.Timeout(connect=1, read=3), + retries=urllib3.Retry(total=0), + ) + self.client = Minio( + "s3:9000", + access_key=secret("S3_ACCESS_KEY"), + secret_key=secret("S3_SECRET_KEY"), + secure=False, + region="us-east-1", + http_client=self.http, + ) + + @contextmanager + def operation(self): + try: + yield + except S3Error, urllib3.exceptions.HTTPError, OSError: + raise DomainError( + 503, "unavailable", "Object store unavailable" + ) from None + + def ping(self): + with self.operation(): + if not self.client.bucket_exists(BUCKET): + raise DomainError( + 503, "unavailable", "Object store unavailable" + ) + + def put(self, *, key, stream, size, content_type, sha256): + stream.seek(0) + with self.operation(): + self.client.put_object( + BUCKET, + key, + stream, + length=size, + content_type=content_type, + metadata={"sha256": sha256}, + part_size=10 * 1024 * 1024, + num_parallel_uploads=1, + ) + + def head(self, *, key): + with self.operation(): + try: + return self.client.stat_object(BUCKET, key) + except S3Error as exc: + if exc.code in {"NoSuchKey", "NoSuchObject"}: + return None + raise + + def get(self, *, key): + with self.operation(): + return self.client.get_object(BUCKET, key) + + def delete(self, *, key): + with self.operation(): + self.client.remove_object(BUCKET, key) + + def objects(self): + with self.operation(): + yield from self.client.list_objects( + BUCKET, prefix="objects/", recursive=True + ) + + def close(self): + self.http.clear() diff --git a/src/operator.py b/src/operator.py new file mode 100644 index 00000000..b2c5cdf4 --- /dev/null +++ b/src/operator.py @@ -0,0 +1,123 @@ +"""Explicit local operator actions. There is deliberately no HTTP replay endpoint.""" + +import argparse +import json +from uuid import UUID, uuid4 + +from src.object_store import ObjectStore +from src.pipeline import database, enqueue, event + + +def change(db, store, *, job_id, operator, reason, version=None): + if ( + not operator.strip() + or not reason.strip() + or len(operator) > 100 + or len(reason) > 500 + ): + raise ValueError("Operator and reason are required") + with db.connection() as conn, conn.transaction(): + job = conn.execute( + "SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,) + ).fetchone() + if not job: + raise ValueError("Unknown job") + attachment = conn.execute( + "SELECT * FROM attachments WHERE id=%s FOR UPDATE", + (job["attachment_id"],), + ).fetchone() + if job["pipeline_version"] != attachment["pipeline_version"]: + raise ValueError("Superseded job cannot be replayed") + source = ( + job["source_key"] + if attachment["published_version"] is None + else attachment["object_key"] + ) + if store.head(key=source) is None: + raise ValueError("Source expired or missing; replay refused") + if version is None: + if job["state"] != "failed": + raise ValueError("Only failed jobs can be replayed") + conn.execute( + "UPDATE jobs SET state='queued',attempt=0,cycle=cycle+1,fence=fence+1,error_code=NULL,next_at=clock_timestamp(),lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s", + (job_id,), + ) + action = "replay" + target = job_id + else: + if ( + version not in {1, 2} + or version <= attachment["pipeline_version"] + ): + raise ValueError("Supported version must increase (1 or 2)") + conn.execute( + "UPDATE jobs SET state='obsolete',fence=fence+1,lease_owner=NULL,lease_until=NULL WHERE id=%s", + (job_id,), + ) + target = uuid4() + conn.execute( + "INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state) VALUES(%s,%s,%s,%s,'queued')", + (target, attachment["id"], version, source), + ) + conn.execute( + "UPDATE attachments SET pipeline_version=%s WHERE id=%s", + (version, attachment["id"]), + ) + action = "new_version" + conn.execute( + "UPDATE attachments SET state='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", + (attachment["id"],), + ) + conn.execute( + "INSERT INTO pipeline_audit(job_id,action,operator,reason) VALUES(%s,%s,%s,%s)", + (target, action, operator, reason), + ) + event(conn, target, action, {"previous_job": str(job_id)}) + enqueue(conn, target) + return str(target) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("action", choices=["failed", "replay", "version"]) + p.add_argument("--job-id", type=UUID) + p.add_argument("--operator") + p.add_argument("--reason") + p.add_argument("--version", type=int) + a = p.parse_args() + db, store = database(), ObjectStore() + try: + if a.action == "failed": + with db.connection() as conn: + rows = conn.execute( + "SELECT id,attachment_id,pipeline_version,attempt,cycle,error_code,updated_at FROM jobs WHERE state='failed' ORDER BY updated_at,id" + ).fetchall() + print(json.dumps(rows, default=str)) + else: + if not all([a.job_id, a.operator, a.reason]): + p.error("job-id/operator/reason required") + if a.action == "version" and a.version is None: + p.error("version required") + print( + json.dumps( + { + "job_id": change( + db, + store, + job_id=a.job_id, + operator=a.operator, + reason=a.reason, + version=a.version + if a.action == "version" + else None, + ) + } + ) + ) + finally: + db.close() + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/password.py b/src/password.py new file mode 100644 index 00000000..bce8bd29 --- /dev/null +++ b/src/password.py @@ -0,0 +1,36 @@ +import threading + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from argon2.low_level import Type + +from src.errors import DomainError + + +class Passwords: + def __init__(self): + self.hasher = PasswordHasher( + time_cost=3, memory_cost=65536, parallelism=4, type=Type.ID + ) + self.slots = threading.BoundedSemaphore(2) + self.dummy = self.hasher.hash("Synthetic-dummy-password-not-for-login") + + def _run(self, function, *args): + if not self.slots.acquire(timeout=5): + raise DomainError(503, "unavailable", "Authentication busy") + try: + return function(*args) + finally: + self.slots.release() + + def hash(self, *, password: str) -> str: + return self._run(self.hasher.hash, password) + + def verify(self, *, password: str, encoded: str | None) -> bool: + try: + return ( + self._run(self.hasher.verify, encoded or self.dummy, password) + and encoded is not None + ) + except VerificationError, InvalidHashError: + return False diff --git a/src/pipeline.py b/src/pipeline.py new file mode 100644 index 00000000..2b152a75 --- /dev/null +++ b/src/pipeline.py @@ -0,0 +1,247 @@ +"""Durable job state is authoritative; stream entries are wake-up notifications.""" + +import json +import os +import random +import time +from datetime import UTC, datetime +from uuid import UUID + +import redis +from psycopg.types.json import Jsonb + +from src.database import Database +from src.settings import integer, secret + +STREAM = "attachments" +GROUP = "image-workers" + + +def database(): + db = Database(url=secret("DATABASE_URL"), size=4) + db.open() + return db + + +def broker(): + return redis.Redis.from_url( + secret("BROKER_URL"), + decode_responses=True, + socket_connect_timeout=1, + socket_timeout=3, + max_connections=8, + ) + + +def envelope(row): + return dict( + schema_version=1, + job_id=str(row["id"]), + type="attachment.process.v1", + attachment_id=str(row["attachment_id"]), + pipeline_version=row["pipeline_version"], + attempt=row["attempt"] + 1, + created_at=datetime.now(UTC).isoformat().replace("+00:00", "Z"), + ) + + +def validate_job(data): + required = { + "schema_version", + "job_id", + "type", + "attachment_id", + "pipeline_version", + "attempt", + "created_at", + } + if ( + not isinstance(data, dict) + or not required <= data.keys() + or data.keys() - required - {"traceparent", "tracestate"} + ): + raise ValueError("Invalid job envelope") + if ( + type(data["schema_version"]) is not int + or data["schema_version"] != 1 + or data["type"] != "attachment.process.v1" + ): + raise ValueError("Invalid job type") + for k in ["job_id", "attachment_id"]: + UUID(data[k]) + for k in ["pipeline_version", "attempt"]: + if type(data[k]) is not int or data[k] < 1: + raise ValueError("Invalid job number") + datetime.fromisoformat(data["created_at"].replace("Z", "+00:00")) + if "traceparent" in data: + import re + + if not isinstance(data["traceparent"], str) or not re.fullmatch( + r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", data["traceparent"] + ): + raise ValueError("Invalid trace context") + if "tracestate" in data and ( + not isinstance(data["tracestate"], str) + or len(data["tracestate"]) > 512 + ): + raise ValueError("Invalid trace context") + return data + + +def event(conn, job_id, name, detail=None): + conn.execute( + "INSERT INTO pipeline_events(job_id,event,detail) VALUES(%s,%s,%s)", + (job_id, name, Jsonb(detail or {})), + ) + + +def enqueue(conn, job_id): + row = conn.execute("SELECT * FROM jobs WHERE id=%s", (job_id,)).fetchone() + conn.execute( + "INSERT INTO outbox(job_id,payload,due_at) VALUES(%s,%s,%s)", + (job_id, Jsonb(envelope(row)), row["next_at"]), + ) + + +def fault(db, target, phase): + if os.getenv("PIPELINE_FAULTS") != "1": + return + with db.connection() as conn, conn.transaction(): + row = conn.execute( + "SELECT * FROM pipeline_faults WHERE target=%s AND phase=%s AND remaining>0 ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1", + (target, phase), + ).fetchone() + if row is None: + return + conn.execute( + "UPDATE pipeline_faults SET remaining=remaining-1,hit_at=clock_timestamp() WHERE id=%s", + (row["id"],), + ) + action = row["action"] + if action == "crash": + os._exit(89) + if action == "permanent": + raise PermanentFailure("invalid_image") + if action == "transient": + raise RuntimeError("injected_transient") + if action == "hang": + time.sleep(120) + if action == "wait": + end = time.monotonic() + 60 + while time.monotonic() < end: + with db.connection() as conn: + released = conn.execute( + "SELECT released_at FROM pipeline_faults WHERE id=%s", + (row["id"],), + ).fetchone() + if released and released["released_at"]: + return + time.sleep(0.05) + raise RuntimeError("fault_wait_timeout") + + +class PermanentFailure(Exception): + pass + + +def retry(conn, row, error, *, permanent=False): + maximum = integer("JOB_MAX_ATTEMPTS", 5, 1, 20) + failed = permanent or row["attempt"] >= maximum + delay = random.uniform(0.25, min(30, 2 ** row["attempt"])) + state = "failed" if failed else "queued" + code = error if permanent else ("retry_exhausted" if failed else error) + conn.execute( + "UPDATE jobs SET state=%s,fence=fence+1,lease_owner=NULL,lease_until=NULL,error_code=%s,next_at=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp() WHERE id=%s", + (state, code, delay, row["id"]), + ) + conn.execute( + "UPDATE attachments SET state=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s", + ( + state, + code if failed else None, + row["attachment_id"], + row["pipeline_version"], + ), + ) + event( + conn, + row["id"], + "failed" if failed else "retry", + {"attempt": row["attempt"], "code": code, "backoff_seconds": delay}, + ) + if not failed: + enqueue(conn, row["id"]) + + +def watchdog(db): + with db.connection() as conn, conn.transaction(): + rows = conn.execute( + "SELECT * FROM jobs WHERE state='processing' AND lease_untilclock_timestamp()-interval '15 seconds')) FOR UPDATE OF j SKIP LOCKED LIMIT 16" + ).fetchall() + for row in rows: + enqueue(conn, row["id"]) + + +def relay_once(db, queue): + watchdog(db) + with db.connection() as conn, conn.transaction(): + row = conn.execute( + "SELECT * FROM outbox WHERE published_at IS NULL AND due_at<=clock_timestamp() ORDER BY id FOR UPDATE SKIP LOCKED LIMIT 1" + ).fetchone() + if not row: + return False + validate_job(row["payload"]) + # WAITAOF must use the SAME connection as XADD (not two pooled calls). + with queue.client() as dedicated: + entry = dedicated.xadd( + STREAM, + {"job": json.dumps(row["payload"], separators=(",", ":"))}, + ) + confirmation = dedicated.execute_command("WAITAOF", 1, 0, 2000) + if int(confirmation[0]) < 1: + raise RuntimeError("Broker durability confirmation timed out") + fault(db, row["job_id"], "relay_after_publish") + conn.execute( + "UPDATE outbox SET published_at=clock_timestamp() WHERE id=%s", + (row["id"],), + ) + event( + conn, + row["job_id"], + "published", + {"entry": entry, "outbox": row["id"]}, + ) + return True + + +def relay(): + db, queue = database(), broker() + try: + while True: + try: + if not relay_once(db, queue): + time.sleep(0.2) + except Exception: + print( + json.dumps( + { + "event": "relay_retry", + "code": "dependency_unavailable", + } + ), + flush=True, + ) + time.sleep(1) + finally: + queue.close() + db.close() + + +if __name__ == "__main__": + relay() diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..5a4e9c15 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,109 @@ +"""Retention cleanup; default is dry-run. Active intent always protects bytes.""" + +import argparse +import json +import time + +from src.object_store import ObjectStore +from src.pipeline import database +from src.settings import integer + + +def reconcile(db, store, *, apply=False, grace=None): + grace = ( + grace + if grace is not None + else integer("STAGING_RETENTION_SECONDS", 86400, 300, 604800) + ) + if grace < 300: + raise ValueError("Retention must be at least 300 seconds") + actions = [] + for obj in store.objects(): + # Recheck each candidate in a bounded transaction, including late PUT age. + with db.connection() as conn, conn.transaction(): + now = conn.execute("SELECT clock_timestamp() now").fetchone()[ + "now" + ] + if (now - obj.last_modified).total_seconds() < grace: + continue + reservation = conn.execute( + "SELECT * FROM upload_reservations WHERE object_key=%s FOR UPDATE", + (obj.object_name,), + ).fetchone() + if reservation and reservation["expires_at"] > now: + continue + jobs = conn.execute( + "SELECT * FROM jobs WHERE source_key=%s OR %s LIKE 'objects/' || attachment_id::text || '/v' || pipeline_version::text || '/%%' FOR UPDATE", + (obj.object_name, obj.object_name), + ).fetchall() + if any(j["state"] in {"queued", "processing"} for j in jobs): + continue + current = conn.execute( + "SELECT v.* FROM attachment_variants v JOIN attachments a ON a.id=v.attachment_id AND a.published_version=v.pipeline_version WHERE v.object_key=%s", + (obj.object_name,), + ).fetchone() + if current: + continue + if any( + j["state"] == "failed" + and (now - j["updated_at"]).total_seconds() < grace + for j in jobs + ): + continue + head = store.head(key=obj.object_name) + if ( + head is None + or (now - head.last_modified).total_seconds() < grace + ): + continue + action = { + "action": "delete_expired_object", + "key": obj.object_name, + "applied": apply, + } + if apply: + # Taking the reservation lock fences an expired API before deletion. + if reservation: + conn.execute( + "DELETE FROM upload_reservations WHERE id=%s", + (reservation["id"],), + ) + store.delete(key=obj.object_name) + actions.append(action) + return {"apply": apply, "retention_seconds": grace, "actions": actions} + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--apply", action="store_true") + p.add_argument("--loop", action="store_true") + a = p.parse_args() + db, store = database(), ObjectStore() + try: + while True: + try: + print( + json.dumps(reconcile(db, store, apply=a.apply)), flush=True + ) + except Exception: + if not a.loop: + raise + print( + json.dumps( + { + "event": "cleanup_retry", + "code": "dependency_unavailable", + } + ), + flush=True, + ) + if not a.loop: + break + time.sleep(60) + finally: + db.close() + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/repository.py b/src/repository.py new file mode 100644 index 00000000..b1131404 --- /dev/null +++ b/src/repository.py @@ -0,0 +1,296 @@ +import hashlib +import json +from uuid import UUID + +from psycopg.errors import ForeignKeyViolation, UniqueViolation + +from src.cache import UserCache +from src.database import Database +from src.domain import Chat, Key, Message, User +from src.errors import DomainError, invalid, not_found + +MESSAGE_COLUMNS = ( + "m.*, ARRAY(SELECT attachment_id FROM message_attachments ma " + "WHERE ma.message_id=m.id ORDER BY ma.position) attachment_ids" +) + +USER_COLUMNS = "id, username, display_name, created_at" + + +def user_dto(row) -> User: + return User( + row["id"], row["username"], row["display_name"], row["created_at"] + ) + + +def chat_dto(row) -> Chat: + return Chat( + row["id"], row["title"], tuple(row["member_ids"]), row["created_at"] + ) + + +def message_dto(row) -> Message: + return Message( + row["id"], + row["chat_id"], + row["sender_id"], + row["text"], + row["created_at"], + tuple(row.get("attachment_ids", ())), + ) + + +class PostgresRepository: + def __init__(self, *, database: Database, cache: UserCache): + self.database = database + self.cache = cache + + def add_user(self, *, user: User, password_hash: str) -> User: + try: + with self.database.connection() as conn: + conn.execute( + "INSERT INTO users(id,username,display_name," + "password_hash,created_at) VALUES(%s,%s,%s,%s,%s)", + ( + user.id, + user.username, + user.display_name, + password_hash, + user.created_at, + ), + ) + except UniqueViolation: + raise DomainError( + 409, "conflict", "Username already exists" + ) from None + return user + + def get_user(self, *, user_id: UUID) -> User | None: + cached = self.cache.get(user_id=user_id) + if cached is not None: + return cached + with self.database.connection() as conn: + row = conn.execute( + "SELECT " + USER_COLUMNS + " FROM users WHERE id=%s", + (user_id,), + ).fetchone() + if row is None: + return None + user = user_dto(row) + self.cache.put(user=user) + return user + + def credentials(self, *, username: str): + with self.database.connection() as conn: + row = conn.execute( + "SELECT " + + USER_COLUMNS + + ", password_hash FROM users WHERE username=%s", + (username,), + ).fetchone() + if row is None: + return None, None + return user_dto(row), row["password_hash"] + + def username_exists(self, *, username: str) -> bool: + with self.database.connection() as conn: + return ( + conn.execute( + "SELECT 1 FROM users WHERE username=%s", (username,) + ).fetchone() + is not None + ) + + def add_session( + self, *, digest: bytes, user_id: UUID, created_at, expires_at + ): + with self.database.connection() as conn: + conn.execute( + "INSERT INTO sessions(token_digest,user_id,created_at," + "expires_at) VALUES(%s,%s,%s,%s)", + (digest, user_id, created_at, expires_at), + ) + + def session_user(self, *, digest: bytes) -> UUID | None: + with self.database.connection() as conn: + row = conn.execute( + "SELECT user_id FROM sessions WHERE token_digest=%s " + "AND expires_at>clock_timestamp()", + (digest,), + ).fetchone() + return row["user_id"] if row else None + + def delete_session(self, *, digest: bytes): + with self.database.connection() as conn: + conn.execute( + "DELETE FROM sessions WHERE token_digest=%s", (digest,) + ) + + def add_chat(self, *, chat: Chat, actor_id: UUID) -> Chat: + members = chat.member_ids + if actor_id not in members or len(set(members)) != len(members): + raise invalid() + try: + with self.database.connection() as conn, conn.transaction(): + conn.execute( + "INSERT INTO chats(id,title,created_at) VALUES(%s,%s,%s)", + (chat.id, chat.title, chat.created_at), + ) + with conn.cursor() as cursor: + cursor.executemany( + "INSERT INTO members(chat_id,user_id,position) " + "VALUES(%s,%s,%s)", + [ + (chat.id, user_id, position) + for position, user_id in enumerate(members) + ], + ) + except ForeignKeyViolation: + raise invalid() from None + return chat + + def get_chat(self, *, chat_id: UUID, actor_id: UUID) -> Chat: + with self.database.connection() as conn: + row = conn.execute( + "SELECT c.id,c.title,c.created_at," + "array_agg(m.user_id ORDER BY m.position) member_ids " + "FROM chats c JOIN members m ON m.chat_id=c.id " + "WHERE c.id=%s AND EXISTS(SELECT 1 FROM members own " + "WHERE own.chat_id=c.id AND own.user_id=%s) GROUP BY c.id", + (chat_id, actor_id), + ).fetchone() + if row is None: + raise not_found() + return chat_dto(row) + + def add_message( + self, *, message: Message, idempotency_key: str + ) -> Message: + # Область включает пользователя, метод и нормализованный путь чата. + scope = f"{message.sender_id}:POST:{message.chat_id}:{idempotency_key}" + lock = int.from_bytes( + hashlib.sha256(scope.encode()).digest()[:8], "big", signed=True + ) + fingerprint = hashlib.sha256( + json.dumps( + { + "text": message.text, + "attachment_ids": [str(x) for x in message.attachment_ids], + }, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode() + ).digest() + try: + with self.database.connection() as conn, conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock,)) + prior = conn.execute( + "SELECT i.body_hash,i.message_id," + "i.expires_at>clock_timestamp() active " + "FROM idempotency i WHERE user_id=%s AND chat_id=%s " + "AND key=%s FOR UPDATE", + (message.sender_id, message.chat_id, idempotency_key), + ).fetchone() + if prior and prior["active"]: + if bytes(prior["body_hash"]) != fingerprint: + raise DomainError(409, "conflict", "Key already used") + row = conn.execute( + "SELECT " + + MESSAGE_COLUMNS + + " FROM messages m WHERE m.id=%s", + (prior["message_id"],), + ).fetchone() + return message_dto(row) + if message.attachment_ids: + rows = conn.execute( + "SELECT id,state FROM attachments WHERE chat_id=%s " + "AND id=ANY(%s) FOR SHARE", + (message.chat_id, list(message.attachment_ids)), + ).fetchall() + if len(rows) != len(message.attachment_ids): + raise not_found() + if any(row["state"] == "failed" for row in rows): + raise DomainError( + 409, "conflict", "Attachment is not ready" + ) + conn.execute( + "INSERT INTO messages(id,chat_id,sender_id," + "text,created_at) " + "VALUES(%s,%s,%s,%s,%s)", + ( + message.id, + message.chat_id, + message.sender_id, + message.text, + message.created_at, + ), + ) + if message.attachment_ids: + with conn.cursor() as cursor: + cursor.executemany( + "INSERT INTO message_attachments(" + "message_id,chat_id," + "attachment_id,position) " + "VALUES(%s,%s,%s,%s)", + [ + (message.id, message.chat_id, aid, i) + for i, aid in enumerate(message.attachment_ids) + ], + ) + conn.execute( + "INSERT INTO idempotency(user_id,chat_id,key,body_hash," + "message_id,expires_at) VALUES(%s,%s,%s,%s,%s," + "clock_timestamp()+interval '24 hours') " + "ON CONFLICT(user_id,chat_id,key) DO UPDATE SET " + "body_hash=excluded.body_hash,message_id=excluded.message_id," + "expires_at=excluded.expires_at", + ( + message.sender_id, + message.chat_id, + idempotency_key, + fingerprint, + message.id, + ), + ) + except ForeignKeyViolation: + raise not_found() from None + return message + + def list_chats( + self, *, actor_id: UUID, after: Key | None, limit: int + ) -> list[Chat]: + condition = "AND (c.created_at,c.id)>(%s,%s)" if after else "" + parameters = ( + [actor_id, *after, limit + 1] if after else [actor_id, limit + 1] + ) + with self.database.connection() as conn: + rows = conn.execute( + "SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id " + "FROM members WHERE chat_id=c.id " + "ORDER BY position) member_ids " + "FROM chats c JOIN members own ON own.chat_id=c.id " + "WHERE own.user_id=%s " + + condition + + " ORDER BY c.created_at,c.id LIMIT %s", + parameters, + ).fetchall() + return [chat_dto(row) for row in rows] + + def list_messages( + self, *, chat_id: UUID, actor_id: UUID, after: Key | None, limit: int + ) -> list[Message]: + condition = "AND (m.created_at,m.id)>(%s,%s)" if after else "" + parameters = ( + [chat_id, *after, limit + 1] if after else [chat_id, limit + 1] + ) + with self.database.connection() as conn: + rows = conn.execute( + "SELECT " + + MESSAGE_COLUMNS + + " FROM messages m WHERE m.chat_id=%s " + + condition + + " ORDER BY m.created_at,m.id LIMIT %s", + parameters, + ).fetchall() + return [message_dto(row) for row in rows] diff --git a/src/schemas.py b/src/schemas.py new file mode 100644 index 00000000..744618c2 --- /dev/null +++ b/src/schemas.py @@ -0,0 +1,65 @@ +import re +from typing import Annotated +from uuid import UUID + +from pydantic import ( + BaseModel, + BeforeValidator, + ConfigDict, + Field, + StrictStr, + ValidationInfo, + field_validator, + model_validator, +) + +from src.cursor import parse_uuid + +UuidIn = Annotated[UUID, BeforeValidator(parse_uuid)] + + +class Input(BaseModel): + model_config = ConfigDict(extra="forbid") + + @field_validator("*", mode="after") + @classmethod + def nonblank(cls, value: object, info: ValidationInfo) -> object: + if isinstance(value, str) and ( + (not value.strip() and info.field_name != "text") + or any(0xD800 <= ord(c) <= 0xDFFF for c in value) + ): + raise ValueError("Invalid text") + return value + + +class CreateUser(Input): + username: StrictStr + display_name: StrictStr = Field(min_length=1, max_length=100) + password: StrictStr = Field(min_length=12, max_length=128) + + @field_validator("username") + @classmethod + def username_format(cls, value: str) -> str: + if not re.fullmatch(r"[a-z0-9_]{3,32}", value): + raise ValueError("Invalid username") + return value + + +class CreateChat(Input): + title: StrictStr = Field(min_length=1, max_length=100) + member_ids: list[UuidIn] = Field(min_length=1, max_length=100, strict=True) + + +class CreateMessage(Input): + text: StrictStr = Field(max_length=4000) + attachment_ids: list[UuidIn] = Field( + default_factory=list, max_length=10, strict=True + ) + + @model_validator(mode="after") + def content(self): + if len(set(self.attachment_ids)) != len(self.attachment_ids): + raise ValueError("Duplicate attachment") + if not self.text.strip() and not self.attachment_ids: + raise ValueError("Empty message") + return self diff --git a/src/service.py b/src/service.py new file mode 100644 index 00000000..6d5f5ace --- /dev/null +++ b/src/service.py @@ -0,0 +1,192 @@ +import base64 +import hashlib +import re +import secrets +import threading +from datetime import timedelta +from uuid import UUID, uuid4 + +from src.cursor import decode_cursor, encode_cursor +from src.domain import Chat, Message, Record, User, order_key, utcnow +from src.errors import DomainError, invalid, not_found +from src.password import Passwords +from src.repository import PostgresRepository + + +def page(*, rows: list[Record], limit: int, scope: str) -> dict: + items = rows[:limit] + cursor = ( + encode_cursor(scope=scope, key=order_key(items[-1])) + if len(rows) > limit + else None + ) + return {"items": items, "next_cursor": cursor} + + +def unauthorized(*, basic=False): + challenge = ( + 'Basic realm="messenger", charset="UTF-8"' + if basic + else 'Bearer realm="messenger"' + ) + return DomainError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": challenge}, + ) + + +def bearer(value): + if not value: + raise unauthorized() + scheme, _, token = value.partition(" ") + if scheme.lower() != "bearer" or not re.fullmatch( + r"[A-Za-z0-9_-]{32,256}", token + ): + raise unauthorized() + return token + + +def token_digest(value): + return hashlib.sha256(value.encode()).digest() + + +class MessengerService: + def __init__( + self, + *, + repository: PostgresRepository, + passwords: Passwords, + session_ttl: int, + ) -> None: + self.repository = repository + self.passwords = passwords + self.session_ttl = session_ttl + self.registration_locks = [threading.Lock() for _ in range(64)] + + def identify(self, *, value: str | None) -> User: + token = bearer(value) + user_id = self.repository.session_user(digest=token_digest(token)) + user = self.repository.get_user(user_id=user_id) if user_id else None + if user is None: + raise unauthorized() + return user + + def register( + self, *, username: str, display_name: str, password: str + ) -> User: + # Bound lock memory and avoid running Argon2 for every local duplicate. + # Different replicas still race safely through PostgreSQL UNIQUE. + stripe = hashlib.sha256(username.encode()).digest()[0] % 64 + lock = self.registration_locks[stripe] + if not lock.acquire(timeout=8): + raise DomainError(503, "unavailable", "Registration busy") + try: + if self.repository.username_exists(username=username): + raise DomainError(409, "conflict", "Username already exists") + encoded = self.passwords.hash(password=password) + return self.repository.add_user( + user=User(uuid4(), username, display_name, utcnow()), + password_hash=encoded, + ) + finally: + lock.release() + + def login(self, *, authorization: str | None): + try: + scheme, _, encoded = (authorization or "").partition(" ") + if scheme.lower() != "basic": + raise ValueError + plain = base64.b64decode(encoded, validate=True).decode("utf-8") + username, separator, password = plain.partition(":") + if ( + not separator + or not re.fullmatch(r"[a-z0-9_]{3,32}", username) + or not 12 <= len(password) <= 128 + or not password.strip() + ): + raise ValueError + except ValueError, UnicodeError: + raise unauthorized(basic=True) from None + user, password_hash = self.repository.credentials(username=username) + if not self.passwords.verify(password=password, encoded=password_hash): + raise unauthorized(basic=True) + now = utcnow() + expires = now + timedelta(seconds=self.session_ttl) + token = secrets.token_urlsafe(32) + self.repository.add_session( + digest=token_digest(token), + user_id=user.id, + created_at=now, + expires_at=expires, + ) + return { + "token": token, + "token_type": "Bearer", + "expires_at": expires, + "user": user, + } + + def logout(self, *, authorization: str | None): + self.identify(value=authorization) + self.repository.delete_session( + digest=token_digest(bearer(authorization)) + ) + + def create_chat( + self, *, actor_id: UUID, title: str, member_ids: list[UUID] + ) -> Chat: + return self.repository.add_chat( + chat=Chat(uuid4(), title, tuple(member_ids), utcnow()), + actor_id=actor_id, + ) + + def get_chat(self, *, actor_id: UUID, chat_id: UUID) -> Chat: + return self.repository.get_chat(actor_id=actor_id, chat_id=chat_id) + + def send( + self, + *, + actor_id: UUID, + chat_id: UUID, + text: str, + idempotency_key: str | None, + attachment_ids: list[UUID], + ) -> Message: + if not idempotency_key or not re.fullmatch( + r"[A-Za-z0-9._:-]{1,128}", idempotency_key + ): + raise invalid() + return self.repository.add_message( + message=Message( + uuid4(), + chat_id, + actor_id, + text, + utcnow(), + tuple(attachment_ids), + ), + idempotency_key=idempotency_key, + ) + + def chats(self, *, actor_id: UUID, limit: int, cursor: str | None) -> dict: + scope = f"chats:{actor_id}" + after = decode_cursor(value=cursor, scope=scope) + rows = self.repository.list_chats( + actor_id=actor_id, after=after, limit=limit + ) + return page(rows=rows, limit=limit, scope=scope) + + def history( + self, *, actor_id: UUID, chat: Chat, limit: int, cursor: str | None + ) -> dict: + if actor_id not in chat.member_ids: + raise not_found() + chat_id = chat.id + scope = f"messages:{chat_id}" + after = decode_cursor(value=cursor, scope=scope) + rows = self.repository.list_messages( + actor_id=actor_id, chat_id=chat_id, after=after, limit=limit + ) + return page(rows=rows, limit=limit, scope=scope) diff --git a/src/settings.py b/src/settings.py new file mode 100644 index 00000000..01ff9bfb --- /dev/null +++ b/src/settings.py @@ -0,0 +1,37 @@ +import os +from dataclasses import dataclass, field +from pathlib import Path + + +def secret(name: str) -> str: + file = os.environ.get(name + "_FILE") + if file: + return Path(file).read_text().strip() + value = os.environ.get(name) + if not value: + raise ValueError("Missing required secret setting: " + name) + return value + + +def integer(name: str, default: int, low: int, high: int) -> int: + value = int(os.environ.get(name, str(default))) + if not low <= value <= high: + raise ValueError("Invalid setting: " + name) + return value + + +@dataclass(frozen=True) +class Settings: + database_url: str = field(repr=False) + cache_url: str = field(repr=False) + session_ttl: int = 3600 + pool_size: int = 10 + + @classmethod + def load(cls): + return cls( + secret("DATABASE_URL"), + secret("CACHE_URL"), + integer("SESSION_TTL_SECONDS", 3600, 1, 86400), + integer("DB_POOL_SIZE", 10, 1, 32), + ) diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 00000000..aab146f7 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,428 @@ +"""Two bounded consumers; every decode runs in a killable isolated process.""" + +import hashlib +import io +import json +import multiprocessing +import resource +import threading +import time +from uuid import uuid4 + +import redis +from PIL import Image, ImageOps, UnidentifiedImageError + +from src.images import MAX_BYTES, MAX_PIXELS +from src.object_store import ObjectStore +from src.pipeline import ( + GROUP, + STREAM, + PermanentFailure, + broker, + database, + event, + fault, + retry, + validate_job, +) +from src.settings import integer + + +def sizes(width, height, bound): + longest = max(width, height) + return ( + (width, height) + if longest <= bound + else ( + max(1, width * bound // longest), + max(1, height * bound // longest), + ) + ) + + +def make_variants(raw, row): + if ( + len(raw) != row["size_bytes"] + or hashlib.sha256(raw).hexdigest() != row["sha256"] + ): + raise PermanentFailure("source_checksum_mismatch") + try: + with Image.open(io.BytesIO(raw)) as source: + if source.width * source.height > MAX_PIXELS: + raise PermanentFailure("pixel_limit") + if source.format not in {"PNG", "JPEG"}: + raise PermanentFailure("invalid_image") + source.load() + oriented = ImageOps.exif_transpose(source) + rgba = oriented.convert("RGBA") + rgb = Image.new("RGB", rgba.size, "white") + rgb.paste(rgba, mask=rgba.getchannel("A")) + rgb.info.clear() + yield "original", raw, row["content_type"], source.size + thumbnail = rgb.resize( + sizes(*rgb.size, 256), Image.Resampling.LANCZOS + ) + crop = ImageOps.fit( + rgb, (128, 128), Image.Resampling.LANCZOS, centering=(0.5, 0.5) + ) + optimized = rgb.resize( + sizes(*rgb.size, 1280), Image.Resampling.LANCZOS + ) + for name, image, fmt, mime, options in [ + ( + "thumbnail", + thumbnail, + "JPEG", + "image/jpeg", + {"quality": 85, "subsampling": 0}, + ), + ( + "crop", + crop, + "JPEG", + "image/jpeg", + {"quality": 85, "subsampling": 0}, + ), + ( + "optimized", + optimized, + "WEBP", + "image/webp", + { + "quality": 82 if row["pipeline_version"] == 1 else 76, + "method": 4, + }, + ), + ]: + output = io.BytesIO() + image.info.clear() + image.save(output, format=fmt, **options) + yield name, output.getvalue(), mime, image.size + except ( + UnidentifiedImageError, + OSError, + ValueError, + Image.DecompressionBombError, + ) as exc: + raise PermanentFailure("invalid_image") from exc + + +def process(job_id, fence, sender): + db = store = None + started = time.monotonic() + cpu = time.process_time() + try: + resource.setrlimit(resource.RLIMIT_AS, (768 * 1024**2, 768 * 1024**2)) + db, store = database(), ObjectStore() + with db.connection() as conn: + row = conn.execute( + "SELECT j.*,a.size_bytes,a.sha256,a.content_type FROM jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s AND j.fence=%s", + (job_id, fence), + ).fetchone() + if not row: + sender.send({"result": "stale"}) + return + fault(db, job_id, "before_effect") + upstream = store.get(key=row["source_key"]) + try: + raw = upstream.read(MAX_BYTES + 1) + finally: + upstream.close() + upstream.release_conn() + results = [] + for name, data, mime, (width, height) in make_variants(raw, row): + key = f"objects/{row['attachment_id']}/v{row['pipeline_version']}/{name}" + digest = hashlib.sha256(data).hexdigest() + store.put( + key=key, + stream=io.BytesIO(data), + size=len(data), + content_type=mime, + sha256=digest, + ) + results.append((name, key, mime, len(data), digest, width, height)) + fault(db, job_id, "after_effect") + with db.connection() as conn, conn.transaction(): + current = conn.execute( + "SELECT * FROM jobs WHERE id=%s FOR UPDATE", (job_id,) + ).fetchone() + allowed = conn.execute( + "SELECT pipeline_version FROM attachments WHERE id=%s FOR UPDATE", + (row["attachment_id"],), + ).fetchone() + valid_lease = conn.execute( + "SELECT %s::timestamptz>clock_timestamp() live", + (current["lease_until"],), + ).fetchone()["live"] + if ( + current["state"] != "processing" + or current["fence"] != fence + or not valid_lease + or allowed["pipeline_version"] != row["pipeline_version"] + ): + event( + conn, + job_id, + "stale_completion_rejected", + {"fence": fence, "version": row["pipeline_version"]}, + ) + sender.send({"result": "stale"}) + return + for result in results: + conn.execute( + "INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height) VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s) ON CONFLICT(attachment_id,pipeline_version,name) DO NOTHING", + (row["attachment_id"], row["pipeline_version"], *result), + ) + conn.execute( + "INSERT INTO effect_publications(attachment_id,pipeline_version,job_id,fence) VALUES(%s,%s,%s,%s)", + (row["attachment_id"], row["pipeline_version"], job_id, fence), + ) + conn.execute( + "UPDATE attachments SET state='ready',object_key=%s,published_version=pipeline_version,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s", + (results[0][1], row["attachment_id"]), + ) + conn.execute( + "UPDATE jobs SET state='ready',error_code=NULL,lease_owner=NULL,lease_until=NULL,updated_at=clock_timestamp() WHERE id=%s", + (job_id,), + ) + event( + conn, + job_id, + "effect_committed", + { + "fence": fence, + "version": row["pipeline_version"], + "variants": 4, + }, + ) + fault(db, job_id, "after_commit") + # Cleanup is independently retryable; it cannot undo the committed effect. + if row["source_key"].startswith("objects/staging/"): + try: + store.delete(key=row["source_key"]) + except Exception: + pass + sender.send( + { + "result": "ready", + "seconds": time.monotonic() - started, + "cpu_seconds": time.process_time() - cpu, + "rss_peak_mib": resource.getrusage( + resource.RUSAGE_SELF + ).ru_maxrss + / 1024, + } + ) + except PermanentFailure as exc: + sender.send({"result": "permanent", "code": str(exc)}) + except MemoryError: + sender.send({"result": "permanent", "code": "memory_limit"}) + except Exception: + sender.send({"result": "transient", "code": "processing_unavailable"}) + finally: + if store: + store.close() + if db: + db.close() + sender.close() + + +def claim(db, data, consumer): + with db.connection() as conn, conn.transaction(): + row = conn.execute( + "SELECT * FROM jobs WHERE id=%s FOR UPDATE", (data["job_id"],) + ).fetchone() + if ( + not row + or str(row["attachment_id"]) != data["attachment_id"] + or row["pipeline_version"] != data["pipeline_version"] + ): + return "obsolete", None + if row["state"] in {"ready", "failed", "obsolete"}: + return row["state"], row + if data["attempt"] < row["attempt"] or ( + row["state"] == "queued" and data["attempt"] <= row["attempt"] + ): + return "obsolete", row + if row["state"] == "processing": + return "busy", row + if data["attempt"] != row["attempt"] + 1: + return "obsolete", row + due = conn.execute( + "SELECT %s::timestamptz<=clock_timestamp() due", (row["next_at"],) + ).fetchone()["due"] + if not due: + return "busy", row + lease = integer("JOB_LEASE_SECONDS", 23, 3, 120) + updated = conn.execute( + "UPDATE jobs SET state='processing',attempt=attempt+1,fence=fence+1,lease_owner=%s,lease_until=clock_timestamp()+%s*interval '1 second',updated_at=clock_timestamp() WHERE id=%s RETURNING *", + (consumer, lease, row["id"]), + ).fetchone() + conn.execute( + "UPDATE attachments SET state='processing',updated_at=clock_timestamp() WHERE id=%s AND pipeline_version=%s", + (row["attachment_id"], row["pipeline_version"]), + ) + event( + conn, + row["id"], + "claimed", + { + "attempt": updated["attempt"], + "fence": updated["fence"], + "version": updated["pipeline_version"], + }, + ) + return "claimed", updated + + +def ready_effect(db, store, row): + with db.connection() as conn: + variants = conn.execute( + "SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s", + (row["attachment_id"], row["pipeline_version"]), + ).fetchall() + if len(variants) != 4: + return False + for variant in variants: + head = store.head(key=variant["object_key"]) + if ( + head is None + or head.size != variant["size_bytes"] + or head.metadata.get("x-amz-meta-sha256") != variant["sha256"] + ): + return False + return True + + +def consume(slot): + db, queue, store = database(), broker(), ObjectStore() + consumer = str(uuid4()) + f"-{slot}" + context = multiprocessing.get_context("spawn") + cursor = "0-0" + timeout = integer("JOB_TIMEOUT_SECONDS", 20, 2, 117) + if integer("JOB_LEASE_SECONDS", 23, 3, 120) <= timeout: + raise ValueError("Lease must exceed child timeout") + try: + while True: + try: + try: + queue.xgroup_create(STREAM, GROUP, id="0", mkstream=True) + except redis.ResponseError as exc: + if "BUSYGROUP" not in str(exc): + raise + reclaimed = queue.xautoclaim( + STREAM, + GROUP, + consumer, + min_idle_time=3000, + start_id=cursor, + count=1, + ) + cursor, messages = reclaimed[0], reclaimed[1] + if not messages: + batch = queue.xreadgroup( + GROUP, consumer, {STREAM: ">"}, count=1, block=500 + ) + messages = batch[0][1] if batch else [] + for entry, fields in messages: + try: + data = validate_job( + json.loads(fields.get("job", "null")) + ) + except ValueError, TypeError, KeyError: + # Malformed notifications cannot become jobs; preserve diagnostic count. + with db.connection() as conn: + event(conn, None, "invalid_envelope") + queue.xack(STREAM, GROUP, entry) + queue.xdel(STREAM, entry) + continue + state, row = claim(db, data, consumer) + if state == "busy": + continue + if state == "ready" and not ready_effect(db, store, row): + with db.connection() as conn: + event(conn, row["id"], "ready_effect_missing") + continue + if state == "claimed": + receiver, sender = context.Pipe(duplex=False) + child = context.Process( + target=process, + args=(str(row["id"]), row["fence"], sender), + ) + child.start() + sender.close() + child.join(timeout) + if child.is_alive(): + child.kill() + child.join(2) + result = { + "result": "transient", + "code": "worker_timeout", + } + else: + try: + result = ( + receiver.recv() + if receiver.poll() + else { + "result": "transient", + "code": "worker_crashed", + } + ) + except EOFError: + result = { + "result": "transient", + "code": "worker_crashed", + } + receiver.close() + child.close() + with db.connection() as conn, conn.transaction(): + current = conn.execute( + "SELECT * FROM jobs WHERE id=%s FOR UPDATE", + (row["id"],), + ).fetchone() + event(conn, row["id"], "child_finished", result) + if ( + current["state"] == "processing" + and current["fence"] == row["fence"] + ): + retry( + conn, + current, + result.get("code", "processing_failed"), + permanent=result["result"] == "permanent", + ) + # Commit result or a retry/terminal decision precedes ack. + queue.xack(STREAM, GROUP, entry) + queue.xdel(STREAM, entry) + except Exception: + print( + json.dumps( + { + "event": "worker_retry", + "code": "dependency_unavailable", + } + ), + flush=True, + ) + time.sleep(0.5) + finally: + queue.close() + store.close() + db.close() + + +def main(): + count = integer("WORKER_CONCURRENCY", 2, 1, 4) + threads = [ + threading.Thread(target=consume, args=(i,), daemon=False) + for i in range(count) + ] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + + +if __name__ == "__main__": + main() diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..b5bff444 --- /dev/null +++ b/uv.lock @@ -0,0 +1,1095 @@ +version = 1 +revision = 5 +requires-python = "==3.14.*" + +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "anyio" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/98/7e457a3b014976887a050561955c0b76e1e7e2d6e27b78ecef2ef80518f2/anyio-4.16.0.tar.gz", hash = "sha256:59bb03251907983d1603d27c3087ddb1e93299ea2163adcdba37899a4fb12ffb", size = 310971, upload-time = "2026-11-01T11:20:07.282Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/16/7757fb84c819accc7d7e9af5291b261e4abb8627ee040f99e7d35ef2457a/anyio-4.16.0-py3-none-any.whl", hash = "sha256:c66ae634e0b8228396b801741beb3294326d4b6102f7bba4041c0dfb58c29026", size = 134521, upload-time = "2026-11-01T11:20:05.59Z" }, +] + +[[package]] +name = "argon2-cffi" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi-bindings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, +] + +[[package]] +name = "argon2-cffi-bindings" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/43/bb8b6e8708d49a5ab36781333af092d9f483b198a2710d01281204640055/argon2_cffi_bindings-26.1.0.tar.gz", hash = "sha256:63505c71542a44b68b1e38060450fb006404170da375feb31af153e7f9c6205d", size = 1790807, upload-time = "2026-08-20T07:44:22.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/d2/0ae991f1b2181e5be49007c574710a800ad36c2978683addb3e67c474e55/argon2_cffi_bindings-26.1.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:21ca0396fe5ec995dd54431c32698189666f9224810acfa752e50d2bd94d9df2", size = 25521, upload-time = "2026-08-20T07:32:43.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e4/ad91d8297638aa2258aad4501c306aca99480dfe76ccd638173fa3702db9/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78de2d65e0b9ea7ce9d1b1c3e87297b2d7305a02c266ee2a2d6910daddd7ee69", size = 27177, upload-time = "2026-08-20T07:32:44.158Z" }, + { url = "https://files.pythonhosted.org/packages/6f/86/5363df11b86d02cf3662208e7406496327649cc90eb365bf6f4e8a54a41f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:27f1821903e2ceadcb88ec2b45ef190897b7682449c772f4d9b53e42c520cf29", size = 26597, upload-time = "2026-08-20T07:32:45.172Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b5/a14dcc592652347dad23ee93b278a4da5d2a25c9ed3ebd10d68eea823a4f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d88e5f7e60f28ae0b0cc6b2f16c43e87cd642a196a86f85e0d8bb6fe016fc16d", size = 27403, upload-time = "2026-08-20T07:32:46.13Z" }, + { url = "https://files.pythonhosted.org/packages/b3/81/b4a20d4902af7f796390bf9245ff83c5217dfa7367efa1d14986956c482b/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:34b7d9c24a4165a2c61cc8ae11d44d48c9ce2830fb536cb7914e11fdd9962728", size = 27132, upload-time = "2026-08-20T07:32:47.13Z" }, + { url = "https://files.pythonhosted.org/packages/7e/1b/c8de358af07b1c490e0fcb863ef98e46ddb486e45567aca5a60bd68d9daa/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:224865cbbcb7a2bd1356741dff12b0134df726b6d44bb7b500df8e303cbd9e81", size = 27588, upload-time = "2026-08-20T07:32:48.087Z" }, + { url = "https://files.pythonhosted.org/packages/48/2f/7ee62a6e79f9309f9d9982d301b22a00010adb580c05c8109b94d7b33de0/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ffff613aaa9ce6236766e2fc6dc560bb5abde7a2e2416e3db1f9ae395a2b4dd4", size = 26785, upload-time = "2026-08-20T07:32:48.977Z" }, + { url = "https://files.pythonhosted.org/packages/e9/10/960d0ee93d4897741bcaf4799c697dae2d81499f66fd1ed042a7dd54c1f4/argon2_cffi_bindings-26.1.0-cp310-abi3-win32.whl", hash = "sha256:a86c069c91a747a2c4e5c51473590aeb48172fff9b2130d23729a42d98665ecb", size = 23898, upload-time = "2026-08-20T07:32:50.114Z" }, + { url = "https://files.pythonhosted.org/packages/6d/3a/0cc14a05810e6add9bce5e87693334baa2222de5f647fa31781885b6573f/argon2_cffi_bindings-26.1.0-cp310-abi3-win_amd64.whl", hash = "sha256:2c36ff87b5dfaa477d0bd51e9d7f6abdae7c8955d2983c97419085d842154b3e", size = 25730, upload-time = "2026-08-20T07:32:51.091Z" }, + { url = "https://files.pythonhosted.org/packages/4e/db/d83cf2af140547f0b9cdaece05b2dc2dcbf991be4667331d073eff771435/argon2_cffi_bindings-26.1.0-cp310-abi3-win_arm64.whl", hash = "sha256:f9c4420a7a864fe1b86ce35befc95b8e39fb852493b81cf798671ddc265de638", size = 24478, upload-time = "2026-08-20T07:32:52.111Z" }, + { url = "https://files.pythonhosted.org/packages/76/38/de696045960f5b846d428c0fb6c130ed3da87aac2af209b05c193815404c/argon2_cffi_bindings-26.1.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:db0fcd827ca61622a01b220aadfbece01939acf53888f2cb98cd93e9b1e2c97e", size = 15449, upload-time = "2026-08-20T07:32:54.075Z" }, + { url = "https://files.pythonhosted.org/packages/91/0a/c25af768f6b75a5a71e31207f87c540656b2808c015260444a22763221ad/argon2_cffi_bindings-26.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:28524438cd3e723f25412f63d4fd516ff5bae9ae5aa56acbe2a1404398a0cf31", size = 25683, upload-time = "2026-08-20T07:32:55.05Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7e/be212c751ab0bcea7f646615f933bf262e8e50b3f7bef32f861d0a2d066b/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac82fc756a446b6ccd7139ce70efa9d8bbe541e7ad579a12dcb52764b7175c5f", size = 27311, upload-time = "2026-08-20T07:32:56.166Z" }, + { url = "https://files.pythonhosted.org/packages/a6/ee/f84b28e4afd13d3cac36c1d8fa8c239d2dc2c51cd978d02ee5d5ad98d9bb/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6a4e68eed961a8de6928d1c17ff3dc2a547e0e923c17f8f1cd79fb7bc9502f98", size = 26771, upload-time = "2026-08-20T07:32:57.206Z" }, + { url = "https://files.pythonhosted.org/packages/21/c3/95c07a023691ecd529da9cb6a8f0779e13ebc1bdfaa86d145fdc1c6e7e79/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:151dfaad9de753f4af2a7854e707e4784f2acc434340ade64239c5b104b2d605", size = 27568, upload-time = "2026-08-20T07:32:58.361Z" }, + { url = "https://files.pythonhosted.org/packages/e6/31/3a18e31406d8694b4d6a31573c3e572fff6bed318bb744453eb653766d22/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:061a6919145bbf282ebf1f9c59d3135d4833c25313c8595c0d68cf7712ddfce2", size = 27280, upload-time = "2026-08-20T07:32:59.343Z" }, + { url = "https://files.pythonhosted.org/packages/0b/39/d4be4577e178b2397aa5b5575c8a309bf0da2afe05fe0c72c8f398662d63/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:62ff20cd130c956c7c9144d5fe35228f98b51c579b2439e988b27ef93e16c02a", size = 27776, upload-time = "2026-08-20T07:33:00.325Z" }, + { url = "https://files.pythonhosted.org/packages/71/47/78f4dd96f7411339f723b96fe24039c1bd5835102b8a5ba71ac4ec712ac7/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:19423e5d7ac1cc354baab59eaabf18db2ec04ef6593b5abe5a34f323c4a8f87a", size = 26932, upload-time = "2026-08-20T07:33:01.272Z" }, + { url = "https://files.pythonhosted.org/packages/3b/cd/96bfd37434cc0a848a9066c291d84b28846c4c9ea289ed9866b1164d622b/argon2_cffi_bindings-26.1.0-cp314-cp314t-win32.whl", hash = "sha256:4f84cdd868978d7b7350a566c254042d44216d9e37f241f3a6d3b1dfebeede35", size = 24878, upload-time = "2026-08-20T07:33:02.189Z" }, + { url = "https://files.pythonhosted.org/packages/f1/42/d8b6810abd9b1bd2f47ebbccf460da59c9f32e94888bea4f7b137d998797/argon2_cffi_bindings-26.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2b741888c93147444fdfc851abd81cc207f37f7f7da42062a00deb3888e57da8", size = 26656, upload-time = "2026-08-20T07:33:03.222Z" }, + { url = "https://files.pythonhosted.org/packages/a9/d1/095d95eaf2ed1d9f77268cf3291bde148c6cd56121f8db2c74c1ba618a0e/argon2_cffi_bindings-26.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:6ab674f668d5962a3a4136ae0812519b0f1586874263723a32181d60d64137e1", size = 25378, upload-time = "2026-08-20T07:33:04.332Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "certifi" +version = "2026.10.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, +] + +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/d9/99/c4b0c17cacdc9c3b8f280026286a9826d6a208c0f047591a3c3ce99b91fd/cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54", size = 184964, upload-time = "2026-08-03T21:20:19.708Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a9/9db617d05d7367c1ad0ab00b3aa6e6f9281edd689b4ee9ea0e5a84e89c97/cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72", size = 184962, upload-time = "2026-08-03T21:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" }, + { url = "https://files.pythonhosted.org/packages/80/10/c5c0cbf0a657aecf59ef511409734230bf556f05a0d6c9eed7aa5c0a0166/cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062", size = 209985, upload-time = "2026-08-03T21:20:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/d5/6c/bfa0b87b03b9238148beca990292843c9396ba069b54496596594173de7b/cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03", size = 208530, upload-time = "2026-08-03T21:20:24.628Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" }, + { url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" }, + { url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e9/d0061c364cde06ee43168a0d076ac1da512cbc380d44767b844ba34fe2b6/cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c", size = 177682, upload-time = "2026-08-03T21:20:44.288Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1c3e01e3ba14c39f6d10bfbac52753b7e22259e38088e5cfe1d704918690/cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48", size = 187949, upload-time = "2026-08-03T21:20:45.623Z" }, + { url = "https://files.pythonhosted.org/packages/87/5b/da4e39efe18eeb89cf580ea9cfc66b6a7c3eadb808fc0cc1d3a295cb5a5d/cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836", size = 182947, upload-time = "2026-08-03T21:20:46.955Z" }, + { url = "https://files.pythonhosted.org/packages/23/59/40338bf421c5accea1d45158170c87006ef1cd371b05c077e76476949728/cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3", size = 188504, upload-time = "2026-08-03T21:20:29.495Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/5ecf1023850036e674c77ec4de86182d309ae344e39e7cba984b7df5d647/cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2", size = 188259, upload-time = "2026-08-03T21:20:31.291Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/4d/45/ba4c93527bc38616a8bd36488acb69a2212d60486794f0c1f318949bbb76/cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc", size = 211538, upload-time = "2026-08-03T21:20:33.808Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/b6ef565e452acb932fb0cb5443f44a78efbd1233e566f02b5a83855e9115/cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29", size = 210688, upload-time = "2026-08-03T21:20:34.974Z" }, + { url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/3b7176cb570a1d3e27faf67b72f591af508036e0d8b2be2ef9af9e8c84bb/cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4", size = 182868, upload-time = "2026-08-03T21:20:40.388Z" }, + { url = "https://files.pythonhosted.org/packages/56/78/31f00c1bcd97c9bbf55f1bfdf5bc809a5de8887473e90bb9960dca825e80/cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e", size = 194104, upload-time = "2026-08-03T21:20:41.725Z" }, + { url = "https://files.pythonhosted.org/packages/7b/1b/58496f2ed0a35de575250c02a43ab3cc2c04d494a88fed31c1cabc0fd176/cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5", size = 186402, upload-time = "2026-08-03T21:20:43.042Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/1c/f41d4e74c28ab327ff3acd36053f7ea506c55872d7a90b0fa71aa3ab0c89/charset_normalizer-3.5.2.tar.gz", hash = "sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef", size = 172659, upload-time = "2026-09-30T04:39:23.398Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/ca/7aa91362a2f77ac8e9e28a9b902a74f7d0e11a851ef0d27a74308da8cd90/charset_normalizer-3.5.2-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:1afb975bd5d68d5ce9f6b6d44fdf2f7e34b895a35e95708a7a91b20a3b51d187", size = 226010, upload-time = "2026-09-30T04:36:27.669Z" }, + { url = "https://files.pythonhosted.org/packages/a8/cf/ac8878d0322cf88a1aad4c7b147db32ca0bd806eb0060957b2e31486dbe6/charset_normalizer-3.5.2-cp314-cp314-android_24_x86_64.whl", hash = "sha256:bbbfc8e28816f19d7c0f1816664980c0a9875d01b27cdf8eedddb639d9e108ad", size = 239266, upload-time = "2026-09-30T04:36:29.434Z" }, + { url = "https://files.pythonhosted.org/packages/c9/6d/9a08d7e0b29b7208e2c6c01dc56c8e0520e7c7beadbbfb024b58fd69c8a5/charset_normalizer-3.5.2-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7967d08cf06dee78443b874f98c98036f624f3a4e73e11f9f64f5be4d25393cf", size = 206783, upload-time = "2026-09-30T04:36:30.872Z" }, + { url = "https://files.pythonhosted.org/packages/82/44/b0aa350280e6ff5a5492d17cf10460dd39d5ee848f872f7ba2df10607f60/charset_normalizer-3.5.2-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c2b5031f63e331e3839b40aed2dd6f191e9c07edbde303e7876846ea1946995", size = 210449, upload-time = "2026-09-30T04:36:32.625Z" }, + { url = "https://files.pythonhosted.org/packages/7c/8a/40db9aa9f5907bb0e6f8b6d64064bf8852fb33d4b813ff9414911df7647c/charset_normalizer-3.5.2-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fcff63213e8e6e47770541a4607175404f47cbb3ebea7b6058cc82d524a0e424", size = 367815, upload-time = "2026-09-30T04:36:34.197Z" }, + { url = "https://files.pythonhosted.org/packages/7f/72/9c5e7707b57c8ddfa9ddf7b0b1d009d7fbab9e9e887d5b721060f37e307d/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8d86d6fc60743dc916eb79e2eb1ec4818e21e427731543af40a3021851174a13", size = 247095, upload-time = "2026-09-30T04:36:35.803Z" }, + { url = "https://files.pythonhosted.org/packages/83/09/71e453691e927de4ddf792770cfaab3f49d494e222f66ea5e404bbd5e39c/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7a881931aa470808df94a8c380eed2bbbc76cd9dc622310f99665658c821eb6d", size = 233942, upload-time = "2026-09-30T04:36:37.407Z" }, + { url = "https://files.pythonhosted.org/packages/9f/86/85c84e4da8b27dd409577d9437926ff581c5f9d3c66038dc68c1a526de51/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8024d00c3faf3fc0c16e07a69f4405e8eac7cc0ab15f65fe6cf43827c4cf72b4", size = 273762, upload-time = "2026-09-30T04:36:38.904Z" }, + { url = "https://files.pythonhosted.org/packages/92/08/564955a4b5f2ccb410ab480bbe8c6a18063ff27f2d35458731c4a5335df9/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4d48f2d08b9de5864e2c8744d4461b862fb149a18274abc8b698c45975573438", size = 269514, upload-time = "2026-09-30T04:36:40.469Z" }, + { url = "https://files.pythonhosted.org/packages/18/24/bad3ac4271589df29cf5ce2f5ae490518a5739358052bd0d61209e6fea54/charset_normalizer-3.5.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34276fd796040bf0993ab33a369aa572e6979c7aab225a88893667ad8eac8f7a", size = 255056, upload-time = "2026-09-30T04:36:42.02Z" }, + { url = "https://files.pythonhosted.org/packages/d6/3e/350d89ad49916b86554d6f5f2d03ec1152148f87e5ff735106c6a03b1a36/charset_normalizer-3.5.2-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0521c5665880b33d603717defa76c094048900010897909952397feb3039da56", size = 251311, upload-time = "2026-09-30T04:36:43.577Z" }, + { url = "https://files.pythonhosted.org/packages/56/5b/4970a2d154df502e133402906dd04e3ae7cada7b3011283c88d0479a2585/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:eff0ac9dbe711a4aee69bf04a83896aa9b85f19641264053a9f6d48573abb7dd", size = 250190, upload-time = "2026-09-30T04:36:45.185Z" }, + { url = "https://files.pythonhosted.org/packages/88/8c/f1a91bddc8fb47c2889e29ea7ea49a194eb0d9868675d786806519c00d76/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:1503bccbeb36d5527790c3930327704c39af22de3112f1b1666a9f3ce15ee204", size = 237098, upload-time = "2026-09-30T04:36:46.689Z" }, + { url = "https://files.pythonhosted.org/packages/24/0e/bb5dace3cc7e79068425386a6589c19b5a2ab5fefc2a46abea6919683332/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:52aa6992700996af31f375de0c6bacd402b0097fe40b53c426b9f51a90ebabc7", size = 274228, upload-time = "2026-09-30T04:36:48.31Z" }, + { url = "https://files.pythonhosted.org/packages/9d/79/b849ad523017ea9f5a45581bbebed91439e0cf42fd2860a6f64e358eb5a6/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e09a3942ecbdee5cce73ea9d42da82b81b72ac1bf031ce069b93b5adf4eac8cd", size = 252401, upload-time = "2026-09-30T04:36:50.091Z" }, + { url = "https://files.pythonhosted.org/packages/89/8c/75469d690cf47200bce8f6cad7655724fc23148e147abfc5ce78b5f65863/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:c7c9ab723cde841fefb34efbad91e87f00a674b1fe1cd0784fde742bf2c154dc", size = 271846, upload-time = "2026-09-30T04:36:51.719Z" }, + { url = "https://files.pythonhosted.org/packages/26/cd/6d52d3c7437cdcf2e310ce9f28f282e733d4ef60ed19105d1819c356255f/charset_normalizer-3.5.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ddc7dacc8ece3a182e7f15cb862d1fd616b46d076cb1ae9dd232b2c38b655874", size = 258151, upload-time = "2026-09-30T04:36:53.234Z" }, + { url = "https://files.pythonhosted.org/packages/f7/4c/070b38bdb5f49a70199fce923ec0726a49536a63ab262abbfcaaf351110b/charset_normalizer-3.5.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:ee43c17b173d46a3212baa6ead3ae258eeabdae48c263a01ccf0218c366dd655", size = 143570, upload-time = "2026-09-30T04:36:54.816Z" }, + { url = "https://files.pythonhosted.org/packages/81/84/9ebfc8ed6c8c4fcd8e726ff6bf220cc8deb3966e31dce9be8dd8aa017e64/charset_normalizer-3.5.2-cp314-cp314-win32.whl", hash = "sha256:4f87960d57feabfb618e4e0af6e7371645fa26a277860739d6e5d6e0012c92f0", size = 186412, upload-time = "2026-09-30T04:36:56.643Z" }, + { url = "https://files.pythonhosted.org/packages/d1/78/5ed86f743d4bc350db307e7636419a0a5ee1d91806d30c7f667bd5c80dae/charset_normalizer-3.5.2-cp314-cp314-win_amd64.whl", hash = "sha256:e4e81e09c1578b8df602e3db08b0b3ea0a6947ad612f52bf8dc5ea8d47691f0c", size = 210965, upload-time = "2026-09-30T04:36:58.205Z" }, + { url = "https://files.pythonhosted.org/packages/53/94/a3a7698e9b1a395e1eb99ccd9a324be9347973bff4e72db2a06496d7cd27/charset_normalizer-3.5.2-cp314-cp314-win_arm64.whl", hash = "sha256:80d02b6f04e92601a081dd97b23d3128033098bff5d35d392ddcc0476ea11253", size = 200406, upload-time = "2026-09-30T04:36:59.764Z" }, + { url = "https://files.pythonhosted.org/packages/c1/48/c5dd00d5ef7791f02666de250a5bb6071e29b7e133cf4b835800b6d3bc27/charset_normalizer-3.5.2-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:dca9ab98072a5a54ebacebdc45f53e645336b320c667410b061be1ca588ae709", size = 396871, upload-time = "2026-09-30T04:37:01.543Z" }, + { url = "https://files.pythonhosted.org/packages/12/c8/8379554b42e8368161d898476686947a0fdbd3e8865170d7909dcabfdee8/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f0aa869112ef88429ae17820d99c3dd9504c9e9c671d3c246f3d7442cb051084", size = 249711, upload-time = "2026-09-30T04:37:03.111Z" }, + { url = "https://files.pythonhosted.org/packages/4a/eb/2ddb1035d17320caa9f41682935123a9a250277b261c3efc86b2d2a21343/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c0afc6800ba57ccc350374c5bd6150419915d95ce93cdbab2d783d75eaf30ecb", size = 231146, upload-time = "2026-09-30T04:37:04.721Z" }, + { url = "https://files.pythonhosted.org/packages/4a/24/2ecb4bde104322cd7859d6594fcfa74649f8d90b3221c9feecbef149875b/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7dcd882da75ef9adf94903b1e3b9419e8aa8fb4c7396822b834b9ef7fb96954f", size = 271552, upload-time = "2026-09-30T04:37:06.295Z" }, + { url = "https://files.pythonhosted.org/packages/3f/98/9d5f6ebc3aee9fef5d30b4aff11fb2ab7a1222b4064f8ef2c7c87cde217a/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2e06a3a98f916dd41d27f3105e02e7a40181c98c94b9158733d03a6f80506c09", size = 269810, upload-time = "2026-09-30T04:37:07.905Z" }, + { url = "https://files.pythonhosted.org/packages/09/e1/a3b06a10461b1b7628853c934c644e03bc28e42767116afb52f19a56519b/charset_normalizer-3.5.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bd128f206a7752ae1f2ab6c61bf8a24ba28913a10df8b14c2637b973ff97a80", size = 259845, upload-time = "2026-09-30T04:37:09.554Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d3/6f561f74a296cf27d61775a1dc665ad13f3bff6a798810ca05907f37a7c4/charset_normalizer-3.5.2-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c8f3d67aeaf55f017982b73683f0e7342ba2f6635a78f69ce89ebb26aa411e5c", size = 252825, upload-time = "2026-09-30T04:37:11.274Z" }, + { url = "https://files.pythonhosted.org/packages/26/9f/69e13ca3b18f43e0eafcd34c04a45b732ae22a43b54a5fc9e119103356eb/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:fe9753dfee015c570d73df76f899f18444d41388bffcde097deba51c4fadbb9f", size = 252659, upload-time = "2026-09-30T04:37:12.941Z" }, + { url = "https://files.pythonhosted.org/packages/73/a9/ace29806a0dae18939919c76ba526472d83214afa101105fabff2cf30625/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:92888bb3187c5ba50500b00b3b310c9f2c651709d28036077680cb5255450a03", size = 238819, upload-time = "2026-09-30T04:37:14.659Z" }, + { url = "https://files.pythonhosted.org/packages/f8/c1/6116d52a2e3311ec80f21f5fb5e17b27405f10b9608af8f6e69516841a1b/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:d008d90a7f2471519aef0c90dfbe73b3e6e4d5e66ac48e19154c17e89e98b604", size = 272973, upload-time = "2026-09-30T04:37:16.346Z" }, + { url = "https://files.pythonhosted.org/packages/19/aa/9955c7e93bba10a9c7e8f7a5031b7ced66f3a1883a55c00712b8d5850ff3/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:31f3930700408d211f13378ccbe1c40845d8da54bd0681fac3a9b5aae81c7aa8", size = 253692, upload-time = "2026-09-30T04:37:18.212Z" }, + { url = "https://files.pythonhosted.org/packages/bb/33/2a6ae7fdc1b10cb581cef91addd8cdfc5f40d50abb5702309369d5834579/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:2a925889534b3748302dae5dead07cc13480de1dac3aea80a941b729b471ef93", size = 268757, upload-time = "2026-09-30T04:37:19.877Z" }, + { url = "https://files.pythonhosted.org/packages/a2/22/80992720a0282cd39bba1db35868e6b9c22f41281160143a836544bc1d8a/charset_normalizer-3.5.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f5ec61164adcec446f8969a3358ec3f9b26bbda3b9213e5586d219afa8df2915", size = 262220, upload-time = "2026-09-30T04:37:21.583Z" }, + { url = "https://files.pythonhosted.org/packages/92/9f/181fd07e1bffea1d95cd80c84ac537354f50699c22cfc4d3c02b6fc16208/charset_normalizer-3.5.2-cp314-cp314t-win32.whl", hash = "sha256:598a11a2c7ebaa5334bf698bf29568c9c390abac6a154d8170fedecd1cea38c5", size = 197797, upload-time = "2026-09-30T04:37:23.235Z" }, + { url = "https://files.pythonhosted.org/packages/49/1c/25d8415ec1c4f2f41f1680435e4c87cfb378ff2f677d950946f2a45d0632/charset_normalizer-3.5.2-cp314-cp314t-win_amd64.whl", hash = "sha256:7fdde2c9fd9e3eca40631e024664cf2584272cc8f96308cbe5fdfc930f51d8bc", size = 222765, upload-time = "2026-09-30T04:37:24.891Z" }, + { url = "https://files.pythonhosted.org/packages/3e/b4/46b48f013dadfc0d0d33b375438e31bdf5a989dc68389c6bf627054d4df9/charset_normalizer-3.5.2-cp314-cp314t-win_arm64.whl", hash = "sha256:d1befeed746d247c81127bb14de9dc3d30edb6e5976d34f83f86ed262b1d9105", size = 210985, upload-time = "2026-09-30T04:37:26.634Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ab/176fbfd5b64939c55d652366aa5b9ef1d767af207a3aa6ebeb0d226c484d/charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd", size = 331815, upload-time = "2026-09-30T04:38:26.216Z" }, + { url = "https://files.pythonhosted.org/packages/7e/84/371eac6b30bdbcbf2d632a1a01809103459216fcaae61b8b8d922c1bfb8a/charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7", size = 253276, upload-time = "2026-09-30T04:38:28.032Z" }, + { url = "https://files.pythonhosted.org/packages/43/6f/c4fbae58febff71709c51bc7e18fdfa55341dc382704740f9f0cbf03817b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f", size = 241239, upload-time = "2026-09-30T04:38:29.732Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/458c3f42164a07d0c5210798e9e704b39e540a6793b05aba67f3a35243a9/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93", size = 231121, upload-time = "2026-09-30T04:38:31.462Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/ab9e89367076f6331bb6c65c4bf14a5361fa5191cb6561bf534f18504e1b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade", size = 260350, upload-time = "2026-09-30T04:38:33.239Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/061431ecc688d9d76602502cb57cc01e691e682c18f1beb45f9673b5bbd2/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0", size = 255430, upload-time = "2026-09-30T04:38:34.865Z" }, + { url = "https://files.pythonhosted.org/packages/8d/1f/20c8949f0676f7ab811abdeb7f4d7f1cbc6e61ff20bef08b44edeb092bc8/charset_normalizer-3.5.2-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26", size = 250612, upload-time = "2026-09-30T04:38:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9e/46f2fa4c431fc98c4ae76a8cb5bdca54e0341e3cfc3fcfd8e82740250818/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011", size = 242083, upload-time = "2026-09-30T04:38:38.26Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/559be29a0c0f086e0bba6922babd38916cc5e0b58ced4de13ee01ea05508/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621", size = 232738, upload-time = "2026-09-30T04:38:39.81Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6c/387b0e4f756a282831c1d9fc6aeb6c51ca4507ca202767c8de15ce9b12e2/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4", size = 260703, upload-time = "2026-09-30T04:38:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/96/92/1fdf015f09ef449f50d3ac4b67c90887c9c318b727daa95cc4f866e6521d/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e", size = 247622, upload-time = "2026-09-30T04:38:42.937Z" }, + { url = "https://files.pythonhosted.org/packages/dc/3c/8e7b8a5671ad5d433669fb2a76f1a0164df2d9b1718b0206bc2a16d840cc/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c", size = 257500, upload-time = "2026-09-30T04:38:44.604Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f0/45b579df5cabc1d5d53ea1cc35e8437d3ca768c0acccc7041517cb6fbb32/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0", size = 255100, upload-time = "2026-09-30T04:38:46.289Z" }, + { url = "https://files.pythonhosted.org/packages/31/68/fdec18a343f5fb3f310588dd478b09ac4799e0b187dbade3a8cd776f03ef/charset_normalizer-3.5.2-cp37-abi3-win32.whl", hash = "sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf", size = 174499, upload-time = "2026-09-30T04:38:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8a/b618149cc5207943a0242068d7a27897f56a62947b5a039085f2a22029f8/charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl", hash = "sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036", size = 200092, upload-time = "2026-09-30T04:38:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/03/cf/4c66866fa9e2b1c78e3c911516d1de497a677b7ac60f1eceda74ce777ca3/charset_normalizer-3.5.2-cp37-abi3-win_arm64.whl", hash = "sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e", size = 294363, upload-time = "2026-09-30T04:38:51.312Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ad/d07d7862a62ffa6d79d68074d14823243dd235a77c45262acbf6adeb28bf/charset_normalizer-3.5.2-py3-none-any.whl", hash = "sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685", size = 68872, upload-time = "2026-09-30T04:39:21.828Z" }, +] + +[[package]] +name = "click" +version = "8.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/37/edbf1b0707709013c0cf2e8317e03d97ed29bf4cac709fe328904ae92a38/click-8.5.1.tar.gz", hash = "sha256:9bbfada76703fd1d784b83eea012e3e8c6a001a876543ef104fa56aaaeb072b4", size = 399151, upload-time = "2026-11-06T17:37:01.021Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/7d/164b7ffc8624fe9efb4221bf9af485d5c1e41b9ae118045d2b6dc5457d8b/click-8.5.1-py3-none-any.whl", hash = "sha256:8bb5376d2006950386f8c6427613334a2acadc5f559cb80e638d00b4df957e37", size = 126163, upload-time = "2026-11-06T17:36:59.287Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "contourpy" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/5a/a55177dd22553a277388e8a1b3220e92de91bacb28356cdc73caa240121d/contourpy-1.4.0.tar.gz", hash = "sha256:20156f5a1ac4f8ce02656e39a61e82164a3d359796dc8026f75b062783d500e1", size = 13323726, upload-time = "2026-09-11T19:05:05.808Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/b0/b548628c8dddf6e5e0a981b7fa8e4c008024df09c7d17c6a9e271edfa0b3/contourpy-1.4.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:a3e67bc1a6a4618a1dac7c3053a9ffece5ddfa2046b2670b342cf430bb0b87d1", size = 297354, upload-time = "2026-09-11T19:03:35.017Z" }, + { url = "https://files.pythonhosted.org/packages/04/5a/513484208742f65af2648c519f0b7ab034616e183c4403538024c136a1ab/contourpy-1.4.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:072a702e2e178f4fcf96f04775d0c059e4c917925abf0d3208ebb6865df3c23d", size = 283729, upload-time = "2026-09-11T19:03:36.818Z" }, + { url = "https://files.pythonhosted.org/packages/35/da/5a6562febc994b2c4bf9d01e57a50458ef8a2051bb3e7b35a00333e0fc26/contourpy-1.4.0-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:404dbcd9233513dfd1323b66ea593fef90e899f658f6b8a9ed8e93bd0ca669db", size = 357819, upload-time = "2026-09-11T19:03:39.058Z" }, + { url = "https://files.pythonhosted.org/packages/6d/99/7b358ce888aaa426f755daa096c99c59d17c352037344e836f20e5d3b957/contourpy-1.4.0-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:61624f6722480aa7e168fd746164e7bfdf48f8ccb6542f1613741200dc37e2b1", size = 407795, upload-time = "2026-09-11T19:03:41.293Z" }, + { url = "https://files.pythonhosted.org/packages/be/02/34d7548adf08c60967435d79d7a90882eacc84f5093435287974c7e99943/contourpy-1.4.0-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2e741a39dfc96babe1722561e81351aeec104526c57dbe91c167c1db606c2d55", size = 408552, upload-time = "2026-09-11T19:03:42.728Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8c/9677dad226e5aaac5b823d5b09d881ebc32ff1648af417597649841e516c/contourpy-1.4.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e439ab450c93455feb0218532ded3e946ec7a9b5f459069f122cfa44b89229f6", size = 384850, upload-time = "2026-09-11T19:03:44.402Z" }, + { url = "https://files.pythonhosted.org/packages/08/f5/e3fbeaa489c223629fbb434737cb3de4ecaca9cae4825e77b92490f640f8/contourpy-1.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:417be048f7e122cefbe2a34c51a1f2b0410f8eb35796f794d45475ddb7872d9d", size = 1357601, upload-time = "2026-09-11T19:03:46.746Z" }, + { url = "https://files.pythonhosted.org/packages/88/be/55fec293d342c222ed58a5d4056df4362e192af2e59527a3d5d822de8828/contourpy-1.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:43d80072a32299bf945de0a6dd4e034ea0162e832261de2e07a274a5adbba9c9", size = 1427340, upload-time = "2026-09-11T19:03:48.745Z" }, + { url = "https://files.pythonhosted.org/packages/86/40/0faabf453edf59b0e3633381d7ca72c3414fb9b72b5cbb08b06ed792d3e5/contourpy-1.4.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:fa1b787362a3856e63dd2b89449f6c384d55ee1beb8f94f4bcc871b40f02462c", size = 118355, upload-time = "2026-09-11T19:03:50.047Z" }, + { url = "https://files.pythonhosted.org/packages/38/0c/7ea75aa3559ae8a6805015e15a19c92b6480752ecc92526361e5776e88c5/contourpy-1.4.0-cp314-cp314-win32.whl", hash = "sha256:738c44fa71735a617f36da58e32810512407d283d5d1bb5b1cecb00d7eeba7cf", size = 356977, upload-time = "2026-09-11T19:04:11.401Z" }, + { url = "https://files.pythonhosted.org/packages/15/41/3df8cc14bb572c8b447f9dfbc876703f0f2f2896058285b5c87a1d4a638d/contourpy-1.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:181bea01bc742734ae672fa00c717d855dd35e1f029536406538c52e7b0cd73d", size = 240101, upload-time = "2026-09-11T19:04:12.772Z" }, + { url = "https://files.pythonhosted.org/packages/66/01/ee6830a5aa4565662a345172f6f35eeeed625d20f1c9d4ecd35318fe4ade/contourpy-1.4.0-cp314-cp314-win_arm64.whl", hash = "sha256:64039341e2d8804f1a13bda8c69083eab935167bbd7e856cf096241f17d5bd45", size = 576359, upload-time = "2026-09-11T19:04:14.983Z" }, + { url = "https://files.pythonhosted.org/packages/51/af/49ee9c9cf012699e505c1b63c531cc99f19df50f205173092c4d541e3ec7/contourpy-1.4.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:ef47fef9a912c77e3d84b014f701e5a9340f25703e9ed3bcebe37f91fb69dc49", size = 313601, upload-time = "2026-09-11T19:03:51.452Z" }, + { url = "https://files.pythonhosted.org/packages/ad/59/ce411ab2be0f805038626d8f4ca9ce563487ff51ff6c363e7512271564ed/contourpy-1.4.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:96019f059bcb3774acc0104be3360a57b36d33eaaa2e1d7f77c800ca8e07618b", size = 302812, upload-time = "2026-09-11T19:03:53.022Z" }, + { url = "https://files.pythonhosted.org/packages/95/b0/390915f9af14c1e2d3b2a9ad99d85b98df4352346a51e368cbb825f813c3/contourpy-1.4.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ddf5a2596d716fd3793434844caf31abc7a40b1e8718431420c89858268fb909", size = 348996, upload-time = "2026-09-11T19:03:54.594Z" }, + { url = "https://files.pythonhosted.org/packages/c7/85/e0952576d54322f3e9983b1e049a7a31f42ce82c5c29d01821e6dd3780d7/contourpy-1.4.0-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:dd59df9e2fb0aff8bd7fd1adb9f349b7c835245a9d0f18c2f2deeb537190f2b1", size = 400272, upload-time = "2026-09-11T19:03:56.822Z" }, + { url = "https://files.pythonhosted.org/packages/d6/7d/f13543a5e1598e4a4ad01d9d5e2d65f9c09c37660616429913d606c5977d/contourpy-1.4.0-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b7794ea07cab575633daad8d6e963b662053391022294d3aacc1d9ba9ef54114", size = 412895, upload-time = "2026-09-11T19:03:58.487Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a8/6e5eaff53507dfd2dfc428240eecda32453067277d4b3e7ca71210b2d611/contourpy-1.4.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1c8a74744fa746eeaa12f0f9ed7f8b4add9d8f1b14d95e3b5e133675eac888f", size = 377060, upload-time = "2026-09-11T19:04:00.122Z" }, + { url = "https://files.pythonhosted.org/packages/16/a3/d00e44d35511b3cead5bd794ee672d4eecf30d9de4ceac57a49d6204064f/contourpy-1.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:107ec46f7aa1664266d69b1181aadc953d58d39f7d8d648fa5b795d4a060b0de", size = 1345688, upload-time = "2026-09-11T19:04:02.705Z" }, + { url = "https://files.pythonhosted.org/packages/1e/01/1e533b0cae32c8edd63fe40435ec3b29345dab87321afcd28722d7f957d1/contourpy-1.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5a4c89c7f38a0d7a94356d74e3391073c4325c4d47ad2fd065c3fe7dbae16c0c", size = 1416140, upload-time = "2026-09-11T19:04:04.77Z" }, + { url = "https://files.pythonhosted.org/packages/6d/0c/4ee66759f00d6a5d6909782f14f441d639fa18a4a43142e334351efc47f8/contourpy-1.4.0-cp314-cp314t-win32.whl", hash = "sha256:912c6afaa106e2f74ba22b30416b77ef7eb94e3bdc5a4df51ab147845dba9b6d", size = 371982, upload-time = "2026-09-11T19:04:06.233Z" }, + { url = "https://files.pythonhosted.org/packages/e6/ac/b3e5324138c3e3741f526663f4f265f4f69dfdf26415a39428940864a5f6/contourpy-1.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:ef9440f6f8506246269a82734f5ff9e2e4c5e775b3996fc883cc491c5257eca6", size = 262907, upload-time = "2026-09-11T19:04:07.882Z" }, + { url = "https://files.pythonhosted.org/packages/19/37/c9aa45e47819dc15a38fc5c81a2fb987fde55e9d3b991fbde514e3b6b5f5/contourpy-1.4.0-cp314-cp314t-win_arm64.whl", hash = "sha256:fc9feef8f1f001c5b87decadc67c4a5d1eebb62ca39c4763d1237ff62cf2b707", size = 587071, upload-time = "2026-09-11T19:04:09.898Z" }, +] + +[[package]] +name = "cycler" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/95/a3dbbb5028f35eafb79008e7522a75244477d2838f38cbb722248dabc2a8/cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c", size = 7615, upload-time = "2023-10-07T05:32:18.335Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/05/c19819d5e3d95294a6f5947fb9b9629efb316b96de511b418c53d245aae6/cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30", size = 8321, upload-time = "2023-10-07T05:32:16.783Z" }, +] + +[[package]] +name = "fastapi" +version = "0.143.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "opentelemetry-api" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/d5/b0aeb0b303650ac87b0411de4aaf3f901b06042dcc3fb00f30b97b7ff585/fastapi-0.143.3.tar.gz", hash = "sha256:b9ab567ee57afb4454004ce04c889f8f0160ac47946b99795887cd6e88484848", size = 469834, upload-time = "2026-10-16T14:14:13.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/1a/03d246befcf307219bb2161c97d8c5d314f84125bf7dd245730a28589828/fastapi-0.143.3-py3-none-any.whl", hash = "sha256:75e8bd68e2f52f77d29b2879749c257d64de2d85282d25270caf2bfccce5efe5", size = 144849, upload-time = "2026-10-16T14:14:11.763Z" }, +] + +[[package]] +name = "fonttools" +version = "4.68.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0c/b5/672312ae80ce8b8eb9779c56052a24c8f36652aa7293a1a420553bcf03e5/fonttools-4.68.0.tar.gz", hash = "sha256:73294e233070f71e2b3653b6bd8564af676d57f26d075761fb06d1ab2e4a7a5d", size = 3848583, upload-time = "2026-10-27T16:30:26.451Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/70/ee6655887166a111d0c4d7ca61f03c90b141cdc6f70c85138fd0b8bfa2e3/fonttools-4.68.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:b2ee7b9c1a5e1d637073c39b0978956febad7e862a1269719f527934c65535a4", size = 3145377, upload-time = "2026-10-27T16:29:03.757Z" }, + { url = "https://files.pythonhosted.org/packages/d6/37/66e7ab417660cb2c65889eb5ad8d80cc5a7cb2c09dd1becdbe6d56bac4fe/fonttools-4.68.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9d80813ae74b969ef2166a3040b591826e80ecd879bdeab7f13f4e8eaf08b345", size = 2634889, upload-time = "2026-10-27T16:29:05.651Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ad/da985167af1988b82112cf133e826cf075cecb28d120778589265cb3125d/fonttools-4.68.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6837def5de49772b82683dadd836df214def04b307cb98d0e7723d49e7e15bb", size = 5420416, upload-time = "2026-10-27T16:29:07.502Z" }, + { url = "https://files.pythonhosted.org/packages/b7/67/e9f2eb2aed41d4ed251a3c6f66e104685f0a03d8ded7ed87f1f0fc63898c/fonttools-4.68.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cbb63b26d33acc3eb61f544c22a0cadbe01682fc064b266bb0e33fa66cea853f", size = 5363969, upload-time = "2026-10-27T16:29:10.172Z" }, + { url = "https://files.pythonhosted.org/packages/e6/06/c4510ac6b43b374b1b707b99fdd0ba1d4059d2fb516584a412c0396fedef/fonttools-4.68.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:81ea963d4390bb9e79f91a905310d91ac7ff7675d8e1c0cfc0f1712521c41a43", size = 5362373, upload-time = "2026-10-27T16:29:12.298Z" }, + { url = "https://files.pythonhosted.org/packages/e0/ef/ea0bf283a30a88a84a47078ff39f7bdefa0346a8bd7737439fd33084031f/fonttools-4.68.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3b9b87645bcdbfc3f30240e2758a37bbef4fdd0f708683b2bfceb60975d63e59", size = 5495553, upload-time = "2026-10-27T16:29:14.614Z" }, + { url = "https://files.pythonhosted.org/packages/95/c0/d493f8c8974859e57920ce193ffbe277423da7eebccbb9241bc31e1b5f1d/fonttools-4.68.0-cp314-cp314-win32.whl", hash = "sha256:39913c1f576af86c69519dc351302332561ee62b39fed04dcfa8019d9c2d129f", size = 2484057, upload-time = "2026-10-27T16:29:16.45Z" }, + { url = "https://files.pythonhosted.org/packages/96/41/c9ca629ffe1b883c1422540a8a11a4ea09616366ab585ab4ee9833f2cd75/fonttools-4.68.0-cp314-cp314-win_amd64.whl", hash = "sha256:1782f2cf952eb262c4126b1f57b5f0e027e71bf2b632350a033dec526f448428", size = 2536130, upload-time = "2026-10-27T16:29:18.399Z" }, + { url = "https://files.pythonhosted.org/packages/17/40/f6264c988076393db3e0b00d6d0ea2a53ce40d445e8a196be241d16d362c/fonttools-4.68.0-cp314-cp314-win_arm64.whl", hash = "sha256:f0b39924698a85bee107184a3c2ee9ae27248144977160534f7d558831db50cf", size = 2508386, upload-time = "2026-10-27T16:29:20.318Z" }, + { url = "https://files.pythonhosted.org/packages/c5/d8/a4d710362f3f23aecc16015eeed1d602187189af7f63a63546233ad2fa73/fonttools-4.68.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:956230858b7a7bb9f5e0971ae07dc64b6155307304acd8c62aedca30e6aae874", size = 3219047, upload-time = "2026-10-27T16:29:22.337Z" }, + { url = "https://files.pythonhosted.org/packages/4d/4d/0d516a1d3d34e84c56237ba1d8dfed8575c350b0e8d39aa30b518d089daf/fonttools-4.68.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:b637d82a488116da2869afbddccfd93ca14e3c47b404c8ec5a9bad9d4381b08c", size = 2668060, upload-time = "2026-10-27T16:29:24.536Z" }, + { url = "https://files.pythonhosted.org/packages/4d/40/56893c685cb1ee7d7ee30d0ac94adf92020787cf5a8e15c2e4e6944f0bec/fonttools-4.68.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:419bf78474adaa6fb320a06822f4f941b19345d598a9de379d6e2f0e4d819df6", size = 5593080, upload-time = "2026-10-27T16:29:26.51Z" }, + { url = "https://files.pythonhosted.org/packages/41/b6/bc0bbad6f1aef22519c4e731c6dc8655e92c6f7204277ec36502ea7e76cd/fonttools-4.68.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fe80d05dd85e6ac58d0cd4fa10ddb96252a6f1046ee0fd00cd1d48f0c25921ac", size = 5399669, upload-time = "2026-10-27T16:29:28.969Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b4/4404a91dc699dd430732d366996cef6d8c8f61c758b5e544a76903db580a/fonttools-4.68.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:cea1969b9bfe230cb3fd66f1c66da5a63a9a917d9cd25a0839524d3d1a88b40e", size = 5460446, upload-time = "2026-10-27T16:29:31.15Z" }, + { url = "https://files.pythonhosted.org/packages/56/09/66876a0d1a01f7c16eb9f7c8ebcf064ebbd6b249f837691cb5a35780316a/fonttools-4.68.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fde43b9785c7f105aa563b6b5481edabea4ab6be7d9ef14ff4d76225950923e8", size = 5495792, upload-time = "2026-10-27T16:29:33.455Z" }, + { url = "https://files.pythonhosted.org/packages/15/7d/1d702dc628e6f5b0d63be906025afc5e5b487cee48128943bb0a71d2bcb3/fonttools-4.68.0-cp314-cp314t-win32.whl", hash = "sha256:a325e6d7f96f20b3004658c5ed17ccfce5b66277b43ccd883035c61be6945275", size = 2517805, upload-time = "2026-10-27T16:29:36.929Z" }, + { url = "https://files.pythonhosted.org/packages/f3/42/8fa0741e08ae16be66a0d2e6d67f918fa0f2961bad70ccef84f9af0767cc/fonttools-4.68.0-cp314-cp314t-win_amd64.whl", hash = "sha256:e80ac66627a01698675ce6e98d90ea9817d1e06644129ec8b0093f66ff1cbeac", size = 2567666, upload-time = "2026-10-27T16:29:40.081Z" }, + { url = "https://files.pythonhosted.org/packages/13/2f/b73cf496570e8d2c66a6099f825505283ef8e944bd588e0b63036d6b2412/fonttools-4.68.0-cp314-cp314t-win_arm64.whl", hash = "sha256:83bd3d558c15f4e1d87b8f3fcfb337c2bce63c7e5c05212abc8fdcf15e216c54", size = 2532338, upload-time = "2026-10-27T16:29:42.073Z" }, + { url = "https://files.pythonhosted.org/packages/35/74/4ea331ecc65ba05a58b1e4e8ccd604c8a15565f876dfbf93fc2a2ea2e22b/fonttools-4.68.0-py3-none-any.whl", hash = "sha256:d76f8fdcb6936f5e1045a0c9dd721a4bf11112e60d1e243bfa1d39f06a8b3104", size = 1248357, upload-time = "2026-10-27T16:30:24.389Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-path" +version = "0.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pathable" }, + { name = "pyyaml" }, + { name = "referencing" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6e/45/41ebc679c2a4fced6a722f624c18d658dee42612b83ea24c1caf7c0eb3a8/jsonschema_path-0.3.4.tar.gz", hash = "sha256:8365356039f16cc65fddffafda5f58766e34bebab7d6d105616ab52bc4297001", size = 11159, upload-time = "2025-01-24T14:33:16.547Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/58/3485da8cb93d2f393bce453adeef16896751f14ba3e2024bc21dc9597646/jsonschema_path-0.3.4-py3-none-any.whl", hash = "sha256:f502191fdc2b22050f9a81c9237be9d27145b9001c55842bece5e94e382e52f8", size = 14810, upload-time = "2025-01-24T14:33:14.652Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "kiwisolver" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ba/07/bd78e6a8fae171ea041ef5bba3ed21a003522fa088834b069b1909981f30/kiwisolver-1.5.1.tar.gz", hash = "sha256:f1303ef2eec81262a4b708c3e858afe58d7c75ad91c1c05266eda7673369859a", size = 104395, upload-time = "2026-08-28T10:28:27.153Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dd/87/2d5dfad0daf17dcc18d98c48ed2332fc3f051cf599e60be6182a30dd4cf1/kiwisolver-1.5.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0324cd2567259b7a095f6cf18a52b0ffc6f3de9e69528ff1bc0e7a37bd43ff1a", size = 62337, upload-time = "2026-08-28T10:26:18.778Z" }, + { url = "https://files.pythonhosted.org/packages/08/c8/83e1624f15d6262b470dbcc80b09979fd4d5b2ea3ddfc6b6e3327e235726/kiwisolver-1.5.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:74ea337e0ec3f6f342a36a4f1b5cd94dd9affddcd28ba9aae2905af932ee8c6b", size = 64513, upload-time = "2026-08-28T10:26:19.909Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2d/827ec30eb07f528c08d8459ffb318ae91a56d793ee8acbea8b491f0ff906/kiwisolver-1.5.1-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ee9df1f0d77b9c6e94f4ac0fec533fbddd5ea3a327807f18d7b069ae019ded80", size = 66287, upload-time = "2026-08-28T10:26:21.078Z" }, + { url = "https://files.pythonhosted.org/packages/53/11/5c43a562529dad8def4b81e5e1877c612a7e0298105a5939b3b409d2079c/kiwisolver-1.5.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fc271a6f0a2126958f4090e5507b9da5848927dae331f8f763bd4aa642b3d2cd", size = 123940, upload-time = "2026-08-28T10:26:22.475Z" }, + { url = "https://files.pythonhosted.org/packages/64/db/9bd6c505c95128c258a55236bfbb3a7a3fb6023f863316b6d7d9f3c69052/kiwisolver-1.5.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9b3092d8992a1d69b7a59c3e39f35e1b9be327a17f68a7c35fc17329e337d6f2", size = 66493, upload-time = "2026-08-28T10:26:23.743Z" }, + { url = "https://files.pythonhosted.org/packages/e1/f4/b3007a3ed5c9be73f81161140684cf7d9bdb9c4b632f5f484d2a1c713fb9/kiwisolver-1.5.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c2306e8bb53601979fcb3fa09cc65e031876d9ae01eff2fcbcd7a84ef94d5bc1", size = 64720, upload-time = "2026-08-28T10:26:24.95Z" }, + { url = "https://files.pythonhosted.org/packages/a1/13/08188f0cafa3a800403e4ff62b9aad4e7a17f9c4c7e080dc8f18c64794cf/kiwisolver-1.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:18a0cfb124546a4c2e6087c5f3029c7f44b37c85b142e0ced71f73a7599ac208", size = 1475867, upload-time = "2026-08-28T10:26:26.393Z" }, + { url = "https://files.pythonhosted.org/packages/8a/3e/053bdc3c9abdb8f2606225eda398adca25c0c91ab90add8222a69db65ee0/kiwisolver-1.5.1-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:34ec467940442c9943016fb2d4c81d1ba84351eeca2f1a78f8bc87f1ba0d414c", size = 1282865, upload-time = "2026-08-28T10:26:28.118Z" }, + { url = "https://files.pythonhosted.org/packages/5e/64/a44c341b36b610588cc2f1e89b3cae072a3119aa8be578e90987cd640751/kiwisolver-1.5.1-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a83ee7107df13abe42a54a6654670eef9bb39425cf2e27f65e0007465e1286ab", size = 1300865, upload-time = "2026-08-28T10:26:30.125Z" }, + { url = "https://files.pythonhosted.org/packages/60/5e/7e7d716dca38c714478b741257a5b4a321d9932b8d851551a136dcaf3984/kiwisolver-1.5.1-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:bebb89489b279b2f5661bbbb2abcc87bcd4a46607bb4a5c966f04f1db6b8df9a", size = 1348071, upload-time = "2026-08-28T10:26:31.829Z" }, + { url = "https://files.pythonhosted.org/packages/10/1a/2b98fdda8bf45b7be317e48ed12393d44334394d315c74b81f4a14c0e31b/kiwisolver-1.5.1-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:509735237ae0d849e8a843551d423d2500d2e0a9ac1611a145658b29c0fb9f85", size = 992191, upload-time = "2026-08-28T10:26:33.544Z" }, + { url = "https://files.pythonhosted.org/packages/b6/55/d893f5ede0e50f9e3fcf01f6015f42ec7d9cd221e26772701fe4a98745f9/kiwisolver-1.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:824c3d763a05ea9e9003610145186b0e9848c7584a5575c79bac5a8e7cd80bad", size = 2233854, upload-time = "2026-08-28T10:26:35.282Z" }, + { url = "https://files.pythonhosted.org/packages/b1/82/f85f6279555a6ee1639fef7bfe83adb037a03e11a6fc9eaa54b8d0380339/kiwisolver-1.5.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:1fff05e239575b1481b6ed1a782f6fad616efbf1f0b1f44e6e85c4dfe426e483", size = 2330621, upload-time = "2026-08-28T10:26:36.9Z" }, + { url = "https://files.pythonhosted.org/packages/56/31/e11aea078f66fc2fffcc179d38ca90d9da97652a241b64519169742ba46a/kiwisolver-1.5.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0627b9bceb9c3cdcf12b8a18655eedfed2692b038df27423383c120d0b7dc2d6", size = 1982848, upload-time = "2026-08-28T10:26:39.01Z" }, + { url = "https://files.pythonhosted.org/packages/af/ea/2956b63bf5140ca46aa2c2818e6aa03e2d5754dd2fa41db1c6b28922940c/kiwisolver-1.5.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:8a708a47ade1fe19e8371d5da076bac0dd4b0a5a7985ad6c637f7f7e361b6baa", size = 2494850, upload-time = "2026-08-28T10:26:40.837Z" }, + { url = "https://files.pythonhosted.org/packages/11/d1/3829542258d8b3fc0898d221e7ef0e2c83eca0d348709bb8dbe54f3d4005/kiwisolver-1.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:007a5553dfc4f4e8d184f588a0200e2cd4b63a59cc8796df3c39909e679dc7a0", size = 2298067, upload-time = "2026-08-28T10:26:42.803Z" }, + { url = "https://files.pythonhosted.org/packages/4e/0e/49522e1ab5788cbaf63a26fbd3b851f9028616828c961b8a31b35cb96df8/kiwisolver-1.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:f4167e87b397f273dc2356fcf1eaf50a6bac51e6105f45103ef7129c8efb0255", size = 72282, upload-time = "2026-08-28T10:26:44.268Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b6/22e7ca5315d363e6f81c9f37c9472e12e7b298731e77c0428e6a911a2c39/kiwisolver-1.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:5c490db2168a508088f59140dd392556a54b8bd1048fc6383c8baff13c359673", size = 69855, upload-time = "2026-08-28T10:26:45.725Z" }, + { url = "https://files.pythonhosted.org/packages/30/8c/03a9cfbe871964c8758a816eb03ac96c806da2795a9a7cd9bf9648bfb594/kiwisolver-1.5.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:4d4ca09bf13cff792b1884f64b98ee6c2467930d632233be25c56b442d99f10e", size = 126289, upload-time = "2026-08-28T10:26:47.023Z" }, + { url = "https://files.pythonhosted.org/packages/f2/e3/14ce3041ca79dff9c9d884ca00c7bf32374e76028a865a9ecd99b4f5a517/kiwisolver-1.5.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:44b8faef94f1857e77fa0238f3390ff1ac51d2ea20a487e2e452a59fd2b5f5ca", size = 67709, upload-time = "2026-08-28T10:26:48.268Z" }, + { url = "https://files.pythonhosted.org/packages/af/c4/45030471a66ec8ef042e9f96ffe1d522c9ab12da180186a0898966fc1385/kiwisolver-1.5.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2ae70bc59790d2af72a3f76f24b272403e135070340281108b447cb77ea70819", size = 65909, upload-time = "2026-08-28T10:26:49.523Z" }, + { url = "https://files.pythonhosted.org/packages/42/73/17dce073a6ae259bb32cf9d686c4079d2e538868bc45967462bf33df914a/kiwisolver-1.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:43844c1a7ad6d723d5b5b4c4fc7f5bd399c40e288120d16257c7c9e8765c6e85", size = 1584907, upload-time = "2026-08-28T10:26:50.933Z" }, + { url = "https://files.pythonhosted.org/packages/8c/84/ae3c75909f507283cbfcc7e916c7e822579ef962020b97e6882b27b4478f/kiwisolver-1.5.1-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22d5e5aaad6be121f2515765e3b1c444352cb8eb4c86510801db8f2e50757316", size = 1392474, upload-time = "2026-08-28T10:26:52.638Z" }, + { url = "https://files.pythonhosted.org/packages/34/31/8bcc83caad5bce8fa4577152389848bf6bc110e51e573a2b4e7c2aa34c89/kiwisolver-1.5.1-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3fa5855898f6d3d01b72ccd48a2d65cbdee301251603fefe34e2025bddba219c", size = 1405246, upload-time = "2026-08-28T10:26:54.248Z" }, + { url = "https://files.pythonhosted.org/packages/55/72/220345537d790cf4ae54f8acfff4b5cc2468e0702a384d651cf7a771c63e/kiwisolver-1.5.1-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d66a64dd5dec136040ec2ae94aa026a912ee60fdd45bc28d3db30037fd809e88", size = 1456099, upload-time = "2026-08-28T10:26:56.042Z" }, + { url = "https://files.pythonhosted.org/packages/cd/10/3725fd2398f66d18c34b4e0f81a8d03764cd4f4f089f58a527f0b4428086/kiwisolver-1.5.1-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:9e51c119992ea8820706871c30a4642ec76de20ae82f9b50b9a45517d8e9f810", size = 1073695, upload-time = "2026-08-28T10:26:57.658Z" }, + { url = "https://files.pythonhosted.org/packages/86/cb/28d6e09e66b93e4588b2e6b7d84d020ccefea09e2f4de788510a07efeab7/kiwisolver-1.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:70ed9a45c7484d2b30cdacf60d220f494a1763b9fec1ad03285c6553fa0889f2", size = 2335355, upload-time = "2026-08-28T10:26:59.202Z" }, + { url = "https://files.pythonhosted.org/packages/6a/b3/a0f31d5e4e40af7dc97c36b8a74fdd3a36cf3c8bbd098da9a23466ff6a94/kiwisolver-1.5.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:98b208a7cc42c803445ef551d6753cc42a5ea13e9cab1ee66cd8b9cb70195330", size = 2426524, upload-time = "2026-08-28T10:27:01.181Z" }, + { url = "https://files.pythonhosted.org/packages/2f/c9/728f63bd58c72cafdc79fc306abeeac7391bec03b757a48dadeb30906521/kiwisolver-1.5.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c6834b92dd2428e2dd85ef3d85f723d3c12f20aaf43a2ddd4f944ca25d833408", size = 2063430, upload-time = "2026-08-28T10:27:03.06Z" }, + { url = "https://files.pythonhosted.org/packages/84/df/ce188b96f92f9a2c958231da140768918cba53c9713dc887b82f85462118/kiwisolver-1.5.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:5d142e352eb13facc7dd047489aebdff6ba78576c239f1ea04931979caaf0567", size = 2597513, upload-time = "2026-08-28T10:27:05.072Z" }, + { url = "https://files.pythonhosted.org/packages/69/d6/76947c8203768968382e5bd74d9cc95654746703a61ea53015f2c74a2e06/kiwisolver-1.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:f9b1c4900736e489a812c529100de4b8fb617d4db075e931e213c57424b83d9b", size = 2394488, upload-time = "2026-08-28T10:27:07.423Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d4/14b21e4eb203c4d15425e8b6a2c625a320b4a1f2f7557eead63ffc30ffb7/kiwisolver-1.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:5978c3340f16a35c30f8ab2fa7bcf559973c55f1a5ef6970e1f621acf3c4db13", size = 75404, upload-time = "2026-08-28T10:27:08.892Z" }, + { url = "https://files.pythonhosted.org/packages/cb/f5/53157899fc7f45f76421b77b99eb1639dd0f83f26ff9d76300c96bb4a3b0/kiwisolver-1.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ca307d6c259e5c98d3cb9ade55342b47a6839762caf2536f3d7b46ee660cc82e", size = 72946, upload-time = "2026-08-28T10:27:10.944Z" }, +] + +[[package]] +name = "lazy-object-proxy" +version = "1.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d4/8e/f26b09778fcb26261fdeb4cc2b011975292316529bff9b541a5a02acb976/lazy_object_proxy-1.13.0.tar.gz", hash = "sha256:ce34046cb021d69c72d05ddb83821ecfd4d2d0a5d43a93c5cd01c9e34779cf14", size = 45649, upload-time = "2026-10-17T16:38:01.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/0b/410ea9a56ad59421d8969535099560f0bad7162d483869e1c07ff5880f35/lazy_object_proxy-1.13.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:62612150e2b67d1cb26c2201d020773e9bab9c596d783d55ada42885a695f749", size = 26856, upload-time = "2026-10-17T16:37:48.864Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/79ab37585905cf6841134806e73ace576d61a87c4eee333be4622161b23f/lazy_object_proxy-1.13.0-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ad7630266b5775463f2a1806165c1d9dff07780cf1bba3c53f311e1d026e8921", size = 70985, upload-time = "2026-10-20T01:05:49.274Z" }, + { url = "https://files.pythonhosted.org/packages/18/2e/66bbb45f20aab9e81139e2bf9fea1a29850b5af769cbf7f181221ce062dc/lazy_object_proxy-1.13.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73f05b09e37b0eb0ca7d6e80e2b0689c49bedac8c5d69667c1ebb9f02a4ed5c7", size = 70984, upload-time = "2026-10-20T01:05:50.548Z" }, + { url = "https://files.pythonhosted.org/packages/74/e0/63dc39750c7b266e9e25022962eba32ba80e4f17fdcd674fc4ad4143fd90/lazy_object_proxy-1.13.0-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:91a725d3c64dfe13d365cf78c469c2a8f95884859379aeca89cb8d2ed291649a", size = 65689, upload-time = "2026-10-20T01:05:51.765Z" }, + { url = "https://files.pythonhosted.org/packages/e7/6c/b94a8779eec1331477557ca3ea4742860d59dc9bcd4dcf21bcef3cd8c4f2/lazy_object_proxy-1.13.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e28a75d33378141927c08f3eb291cc98d0a22776670ae330f8347f898cd3b175", size = 69492, upload-time = "2026-10-20T01:05:53.116Z" }, + { url = "https://files.pythonhosted.org/packages/60/99/6c3ff4a32f15e807ffb868c47de136ebd4be31e7320af42b699dc43a7495/lazy_object_proxy-1.13.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:a785fe44b61fcbffc0e264fe6b735a909e0b5ef922f0e976e281bbe8477fb47a", size = 65019, upload-time = "2026-10-20T01:05:54.462Z" }, + { url = "https://files.pythonhosted.org/packages/c4/1a/447ff40629a11f879f194e839d59e5a026dd26f4d0eb8672de858644be85/lazy_object_proxy-1.13.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e321425f76451885c93083ecec0850b64de0f83bb94c113e930e8a0a630e45ab", size = 70334, upload-time = "2026-10-20T01:05:55.619Z" }, + { url = "https://files.pythonhosted.org/packages/70/52/2ca84fb914b17d4b02193cbf8b6ba9344b36459944a19711d3da1d14b03d/lazy_object_proxy-1.13.0-cp314-cp314-win_amd64.whl", hash = "sha256:0e52094b7fff1f26f4e227b88be3f350817078e4bb79b4c80846fca0d15efae4", size = 26122, upload-time = "2026-10-17T16:37:53.02Z" }, + { url = "https://files.pythonhosted.org/packages/84/c7/ebbeb84639e84fbf6f07a48373f76fe048d23fbcfd4557509b48f35d284d/lazy_object_proxy-1.13.0-cp314-cp314-win_arm64.whl", hash = "sha256:b2dd1c37d64cf9f3f4a2072971639408989b0803a3b7f05e75c1fe28d1a93c1f", size = 24746, upload-time = "2026-10-17T16:37:53.954Z" }, + { url = "https://files.pythonhosted.org/packages/38/ac/8ff35cc4d1316e056144edfb816dbff9b5db9ccd7e98c69f6b602c11bc29/lazy_object_proxy-1.13.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:348370676ea6d9ed08f9b74545c37e5be4cf009c45b9d50c6aff476fb7fcc7ec", size = 27524, upload-time = "2026-10-17T16:37:49.825Z" }, + { url = "https://files.pythonhosted.org/packages/14/c5/8afe34ead41fdd48a207c4535192a4e931c56c791d5f0a57c0abd4558c1c/lazy_object_proxy-1.13.0-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:dee2fb3326cd81384d2b752c3f6888d7ed9eba2cdbf0583097b5168b8bb4a703", size = 80523, upload-time = "2026-10-20T01:05:57.129Z" }, + { url = "https://files.pythonhosted.org/packages/38/fd/0e68a385daaf141b1b1d943b132b8a9887bd53abb88886dab286cca9b62e/lazy_object_proxy-1.13.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2b4c446cbc8e411b116c9ee9b475cb664b8bc5c3d912bcbf1bab5c785fc69a02", size = 82823, upload-time = "2026-10-20T01:05:58.56Z" }, + { url = "https://files.pythonhosted.org/packages/91/ce/648e09f90dc59cb6007d9093c2cc17d72e2d93c06553828037126205ea7c/lazy_object_proxy-1.13.0-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1e671e0e9f2793759f382a69a3853c8742474bc011865ed3595187340ac5956d", size = 74655, upload-time = "2026-10-20T01:05:59.717Z" }, + { url = "https://files.pythonhosted.org/packages/10/4d/811442390862dbf85e00520ecb0cb58c7dbdf824dc563ad933729f9e20f3/lazy_object_proxy-1.13.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a0155084a5dc8ce3e1a3d6e3f2fb4d84d5fc09e66ed84dc0e5f8a2697d421ea0", size = 80848, upload-time = "2026-10-20T01:06:00.884Z" }, + { url = "https://files.pythonhosted.org/packages/24/4e/c30ea3678b50a7ed7774760fbde917106cbb860edb88cc31a17a8385b3d4/lazy_object_proxy-1.13.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:2d07e0bde2738922a7117fccc1a82514aefbc9c07ee8ff0bb84a1c606dbb5ac8", size = 74127, upload-time = "2026-10-20T01:06:01.949Z" }, + { url = "https://files.pythonhosted.org/packages/19/d0/862f0f96b031dab7faa036679e2076bbd6a18ea06e34fb1400fca65b03f0/lazy_object_proxy-1.13.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:da9f1b8780678af93c4ddba46826166c53a25d8d71133fc818542bf7b481e033", size = 79497, upload-time = "2026-10-20T01:06:03.122Z" }, + { url = "https://files.pythonhosted.org/packages/d5/50/20454483d8f95a11710b951518f86aaaaaae9195982fe27c12eeed130660/lazy_object_proxy-1.13.0-cp314-cp314t-win_amd64.whl", hash = "sha256:f21b4711e6744d46a4672497ca2db390cbdb03241d9b1654bbf38dbbc4db2a5e", size = 26670, upload-time = "2026-10-17T16:37:50.977Z" }, + { url = "https://files.pythonhosted.org/packages/4a/df/c1ff843fd9c4978d9a50ef2bceda3897d71645d4127ec1413aa15204cb64/lazy_object_proxy-1.13.0-cp314-cp314t-win_arm64.whl", hash = "sha256:de295ae25e5c1fd712088547f251c4b2976d2f60df28057b6c416e771a99842b", size = 25297, upload-time = "2026-10-17T16:37:52.088Z" }, +] + +[[package]] +name = "matplotlib" +version = "3.11.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "contourpy" }, + { name = "cycler" }, + { name = "fonttools" }, + { name = "kiwisolver" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pillow" }, + { name = "pyparsing" }, + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/c8/9aa712a0afb882649424dd8de8ad9aa6235e796e84c6052e8f6dc1598d0d/matplotlib-3.11.2.tar.gz", hash = "sha256:cec596316640f2b394b8f0daa0ea61a8eae82d017b620b9f202befb972a59ea4", size = 32660610, upload-time = "2026-09-11T19:05:31.214Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/b4/4facdb700bc236e4c676495ca3798d8134ab8668cbfe0f6b2e9ebf962a38/matplotlib-3.11.2-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c9721f81275499da1feeb36a2cf8192ea086283b3bd16b7dc4c9d7aedb7396d6", size = 9478976, upload-time = "2026-09-11T19:04:00.12Z" }, + { url = "https://files.pythonhosted.org/packages/4c/a0/bba05f0a25bfeb0475c2f2a87958b31103aa6b7a5386017afb1a96212825/matplotlib-3.11.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1a3040b209f3968b4e84161df7b174f07a9fad33b0f2d7e48ea3bbd3075e2863", size = 9307932, upload-time = "2026-09-11T19:04:02.976Z" }, + { url = "https://files.pythonhosted.org/packages/d4/42/81d5cba4bb39b41b0998efc880e129f2d7cdb94256ca49678fdcd9c21be5/matplotlib-3.11.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf3fe71fbfb8ec0e310e0bc8537c3405a01f38f25f9394ed2135e6202fed542b", size = 10674370, upload-time = "2026-09-11T19:04:06.912Z" }, + { url = "https://files.pythonhosted.org/packages/29/5e/52f56f93d5b20815ee0ae352f7a38eff25580edb180b4fa7df87cfeba68f/matplotlib-3.11.2-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8c8255de28f986d935a64c9ca71c0ec2d2f41d355691f5ea684725dc91413f71", size = 10954597, upload-time = "2026-09-11T19:04:09.948Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ae/f8918a1b564db49c53cd603eb3c8748c5767e4ce0fb2cf1d7c38653e4c5d/matplotlib-3.11.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a8756cc73d9af9a7fe0deb54ea2e75ef73b01d9e575877e72acad5458e660943", size = 10812583, upload-time = "2026-09-11T19:04:12.887Z" }, + { url = "https://files.pythonhosted.org/packages/ea/61/4315ef0d63937fe56f451887b4a0be3b8e8b22cf1d92a8d157caac54570d/matplotlib-3.11.2-cp314-cp314-win_amd64.whl", hash = "sha256:ecea603dd2fbf8242fd31a305a8b12a4ece2de28096870c65fdd0d1e35b8d9a6", size = 9505972, upload-time = "2026-09-11T19:04:15.788Z" }, + { url = "https://files.pythonhosted.org/packages/d5/00/02398b0a1a62ef73d9af0a957fe1773363eec9a6ca7e433c72ae2790fbce/matplotlib-3.11.2-cp314-cp314-win_arm64.whl", hash = "sha256:01dc8eaaab5a9fce9ff615eca82345728f289e4715b186ee10c6d85272fc26bb", size = 9184534, upload-time = "2026-09-11T19:04:18.691Z" }, + { url = "https://files.pythonhosted.org/packages/ec/c5/1e6ca10baa7e04f113c37856aadb4adc3dbb1c55fc7da6dc830bba1292d7/matplotlib-3.11.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:79a258f58253dfa025af80a9e9bb228d75fced007f0e93ae7423fefbde81a74d", size = 9526415, upload-time = "2026-09-11T19:04:21.39Z" }, + { url = "https://files.pythonhosted.org/packages/e0/93/7561727af07ccc84747953c6a33eee3981cc436d04f465032e707ca0e429/matplotlib-3.11.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c0b83f044ce10a98027b105b3931548719a6e8c7ef986b4362651e0b5367c8dc", size = 9361064, upload-time = "2026-09-11T19:04:24.107Z" }, + { url = "https://files.pythonhosted.org/packages/c1/40/ce270ef2a6794d94a409fe28b8dabedffc07c07f9e5dbb89e0f605bd157d/matplotlib-3.11.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e43b188f0a5b75447bcc197728258166aa64365770ed1caa36595a5e1ca4bbba", size = 10684626, upload-time = "2026-09-11T19:04:27.201Z" }, + { url = "https://files.pythonhosted.org/packages/45/46/396e0307dbf1c1f2d0a3654caa6dbc780dd5379f5f11257369cf8d21d9fe/matplotlib-3.11.2-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d3304eb5a59442a8867f6920d484591c0fa09ffc29e9260be2feec3351e25869", size = 10960602, upload-time = "2026-09-11T19:04:30.254Z" }, + { url = "https://files.pythonhosted.org/packages/1c/97/4defb84695477340aa88b27c9482cc1ca4de11f36f66b6c8c0ddd685def1/matplotlib-3.11.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:02329432ae5c6af87cf208ee575b701d698bdf0b1a3bb28cc6d53c36e967e575", size = 10822603, upload-time = "2026-09-11T19:04:32.964Z" }, + { url = "https://files.pythonhosted.org/packages/7d/e7/eab54551c1ec6b72dcd678a078b6c6da8d4c85ec1ff0016e81fe52866304/matplotlib-3.11.2-cp314-cp314t-win_amd64.whl", hash = "sha256:f2ac30cf5eb5dff1b584627ae0b0e1186551a4f69ae3c75073911da497a29170", size = 9551206, upload-time = "2026-09-11T19:04:35.811Z" }, + { url = "https://files.pythonhosted.org/packages/4e/0e/04698032cb4d8fb1e30d8ffec5ded20f6f582b03c8c58b2385c95245f158/matplotlib-3.11.2-cp314-cp314t-win_arm64.whl", hash = "sha256:cf41ecd1b0c0b6f7177ed965a54c2afbe888715c7cf6054dc12d53bc1494002c", size = 9234977, upload-time = "2026-09-11T19:04:38.972Z" }, +] + +[[package]] +name = "minio" +version = "7.2.20" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi" }, + { name = "certifi" }, + { name = "pycryptodome" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/40/df/6dfc6540f96a74125a11653cce717603fd5b7d0001a8e847b3e54e72d238/minio-7.2.20.tar.gz", hash = "sha256:95898b7a023fbbfde375985aa77e2cd6a0762268db79cf886f002a9ea8e68598", size = 136113, upload-time = "2025-11-27T00:37:15.569Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/9a/b697530a882588a84db616580f2ba5d1d515c815e11c30d219145afeec87/minio-7.2.20-py3-none-any.whl", hash = "sha256:eb33dd2fb80e04c3726a76b13241c6be3c4c46f8d81e1d58e757786f6501897e", size = 93751, upload-time = "2025-11-27T00:37:13.993Z" }, +] + +[[package]] +name = "mtusi-messenger-lab5" +version = "1.0.0" +source = { virtual = "." } +dependencies = [ + { name = "argon2-cffi" }, + { name = "fastapi" }, + { name = "minio" }, + { name = "pillow" }, + { name = "psycopg", extra = ["binary", "pool"] }, + { name = "pydantic" }, + { name = "redis" }, + { name = "uvicorn" }, +] + +[package.dev-dependencies] +dev = [ + { name = "httpx" }, + { name = "matplotlib" }, + { name = "openapi-spec-validator" }, + { name = "psutil" }, + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "argon2-cffi", specifier = ">=25,<26" }, + { name = "fastapi", specifier = "==0.143.3" }, + { name = "minio", specifier = "==7.2.20" }, + { name = "pillow", specifier = "==13.0.0" }, + { name = "psycopg", extras = ["binary", "pool"], specifier = "==3.3.6" }, + { name = "pydantic", specifier = "==2.14.1" }, + { name = "redis", specifier = "==8.1.0" }, + { name = "uvicorn", specifier = "==0.54.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "httpx", specifier = ">=0.28,<1" }, + { name = "matplotlib", specifier = ">=3.10,<4" }, + { name = "openapi-spec-validator", specifier = "==0.7.2" }, + { name = "psutil", specifier = ">=7,<8" }, + { name = "pytest", specifier = ">=8,<10" }, + { name = "ruff", specifier = ">=0.14,<1" }, +] + +[[package]] +name = "numpy" +version = "2.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/b0/c7453d0b6e2073c3264468b106ee1563750cecc910965e67357e3698c83e/numpy-2.5.4.tar.gz", hash = "sha256:9a94cf751c9ad8ebaa835bcd3d40dacf8534ad086b88c38029b65123c7999d2a", size = 20866315, upload-time = "2026-10-10T20:05:31.422Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/ba/005cb5edd580d2f84d7ca3206b92dc17d4388e56e6f87ffe8f2762f83139/numpy-2.5.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:c668b2f0d651605b58892644b0e302c7157f7159544227758c896982ef384b18", size = 17005499, upload-time = "2026-10-10T20:03:37.961Z" }, + { url = "https://files.pythonhosted.org/packages/f3/49/fee7587c33ee35f7977f9051d7f2023d4e7246d62710c80f20c2361ea232/numpy-2.5.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ffa6ce09a1c6a08e9667dd9c97aa0b14184e8d18f2a14b78b2a2328c9147f076", size = 12019666, upload-time = "2026-10-10T20:03:40.606Z" }, + { url = "https://files.pythonhosted.org/packages/d5/b2/c6ce165acffceb15a82c07b9cc77d391f86b3f379ba62911908ae5d34b91/numpy-2.5.4-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:956555e0603a4d38019ae6925711cb9dc43195c076a928accf7ea5d50bddfe53", size = 5455617, upload-time = "2026-10-10T20:03:43.138Z" }, + { url = "https://files.pythonhosted.org/packages/77/7f/dd85ce260a669a89be06842cf355d7353a33e6cfbc590fb8ebb947d88dc9/numpy-2.5.4-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:2c2c4afffdeb7920e445028dd71eb932cac3e704792e964bc2a232426d4f1255", size = 6791932, upload-time = "2026-10-10T20:03:44.874Z" }, + { url = "https://files.pythonhosted.org/packages/63/d6/34b0a2b0741386a63025a65a2c09caaaaaad6d0ca95b66cd65c30dd7fcb5/numpy-2.5.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4054173604cd8658796053f1f3bc0befb68ec1c0762c57fdad61e199256a8617", size = 15710899, upload-time = "2026-10-10T20:03:46.839Z" }, + { url = "https://files.pythonhosted.org/packages/16/d5/928078d2b28f26829b138b4a6c3980045022fb409f570657a224ae60ef4e/numpy-2.5.4-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d549420b8858885cea8838a727842249218b9c1da24dd517e25c9c7a948310a3", size = 16721710, upload-time = "2026-10-10T20:03:49.489Z" }, + { url = "https://files.pythonhosted.org/packages/f9/cf/673fd1b8f4cd78eb6320e87ec4c90ac19c095644259e3749853a405c70f4/numpy-2.5.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:823874a507a84af050493b622affde94b6f7c3a0dc22cb2801381bc03b871c00", size = 17066182, upload-time = "2026-10-10T20:03:52.25Z" }, + { url = "https://files.pythonhosted.org/packages/f3/92/a77b5061b1b3e2643928c37976d79ee173e1b171ed158b7a3c61056b41bc/numpy-2.5.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4e263278bfb5ee6409db8aedbc4cc32973b1b82bc1e8d3c668551d04d83a7e37", size = 18480315, upload-time = "2026-10-10T20:03:55.39Z" }, + { url = "https://files.pythonhosted.org/packages/bb/1d/1486ef3d3fb2279fd93c4c43c1bbbf1ca389a19816696684409f71babaab/numpy-2.5.4-cp314-cp314-win32.whl", hash = "sha256:cfd73180400042a7c532d30c5e287bdd03c59ff9ee1b4c0316af0539e29dfe23", size = 6185739, upload-time = "2026-10-10T20:03:58.186Z" }, + { url = "https://files.pythonhosted.org/packages/52/9a/e1e512ebc948d5b9dd33b08736760f0ebbed2848fd4eda1f553088a6dcee/numpy-2.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:2ca144f15135b6212a5c47b1e2aeca6e412f102f95a2d5d88d8aec77eb255de3", size = 12703552, upload-time = "2026-10-10T20:04:00.28Z" }, + { url = "https://files.pythonhosted.org/packages/2c/05/de709a982d7bbcd688a3fad71f002e9ff80c2db39e03ee726609b610f1d1/numpy-2.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:468397ba3c64427474706e5c9123fe266395496714dc684294eac75cd4930d1e", size = 10803901, upload-time = "2026-10-10T20:04:02.659Z" }, + { url = "https://files.pythonhosted.org/packages/13/34/083570ada3bb2a30fbe5d77c8c6fef9141144a15d33e6f793a67e9749ab8/numpy-2.5.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1ef3aa6d7e29bb13677323114280b05acc57607fa2300e66432d665d5418a162", size = 12138695, upload-time = "2026-10-10T20:04:05.012Z" }, + { url = "https://files.pythonhosted.org/packages/94/06/1f9c24db48eef0c2d1207e3b11fffb0478e39dfd8c1e1be7476936885eed/numpy-2.5.4-cp314-cp314t-macosx_14_0_arm64.whl", hash = "sha256:98b053943e5a0474ec0da309d2cb9d3f18ea57f8a2067c2ab7b5f763d1068380", size = 5574615, upload-time = "2026-10-10T20:04:07.316Z" }, + { url = "https://files.pythonhosted.org/packages/da/0f/593fba2e1560e949123bc7d2fc48b5893d56e58cd4bd5a273d2fbf60b220/numpy-2.5.4-cp314-cp314t-macosx_14_0_x86_64.whl", hash = "sha256:b64a85f40e154983960a4167d4c1d57a50c7f109b3d3264a3a984154e90a8454", size = 6889383, upload-time = "2026-10-10T20:04:09.918Z" }, + { url = "https://files.pythonhosted.org/packages/eb/9f/b799dfdce4e05e80ed4bc815c71ff343a11533b2c0ffc221cae8538cda63/numpy-2.5.4-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a813ed7719bf45463c51779e6a98d0385fe905e48447526938a4b8337333d551", size = 15753763, upload-time = "2026-10-10T20:04:12.278Z" }, + { url = "https://files.pythonhosted.org/packages/34/88/16c5f12f86f5ad2817c4d103205131fc6c8acb3d1878af05a1a4f23ec859/numpy-2.5.4-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9b80cdf5cedba0e90d93fa5f9a333c4d65bd545cd669b71bb97ce2b703c9d73", size = 16757212, upload-time = "2026-10-10T20:04:14.799Z" }, + { url = "https://files.pythonhosted.org/packages/ff/4f/a1fe40e18a898e6a5089f4f0d891f0a493eb0574d5b34458f0fbe5aa3e5c/numpy-2.5.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2199ed071f460487c8db2c0e5c0b564494190edb4772fe80f9aad88b2604def5", size = 17116471, upload-time = "2026-10-10T20:04:17.58Z" }, + { url = "https://files.pythonhosted.org/packages/aa/46/e923a11c78e65c1722e7aaad817c06bd591324174b9d28ce5d31eee4d432/numpy-2.5.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:64f9c9878c1938476365e11ccfb6b770f3b9e5f045ccddc514235041e6959365", size = 18524063, upload-time = "2026-10-10T20:04:20.365Z" }, + { url = "https://files.pythonhosted.org/packages/5a/fa/84ab064514440c1f64a1b21088f2c82756defdd05e07c75ab233899565b2/numpy-2.5.4-cp314-cp314t-win32.whl", hash = "sha256:64d1c8ac28a4077cf987e0a71a7a0ef7e2df70722f07f0baa42dbb7eb6938647", size = 6340926, upload-time = "2026-10-10T20:04:22.865Z" }, + { url = "https://files.pythonhosted.org/packages/7e/7e/6cd886876f435b10685db9b9f7eeb70356f99e052116f4e5f11c5792c714/numpy-2.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:067374eb538c34c745436365cf7b0112595c1d326f21ce4ff340f61230239fbb", size = 12901584, upload-time = "2026-10-10T20:04:24.99Z" }, + { url = "https://files.pythonhosted.org/packages/38/1b/3c1684f6a06f7307f2335fca6e486cb162847fb97e91d65f8eb5cabad213/numpy-2.5.4-cp314-cp314t-win_arm64.whl", hash = "sha256:e94aef2c639da4a960ad0db8e06471208d8589974953d78b61d345b4eb99e394", size = 10891152, upload-time = "2026-10-10T20:04:27.52Z" }, +] + +[[package]] +name = "openapi-schema-validator" +version = "0.6.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonschema" }, + { name = "jsonschema-specifications" }, + { name = "rfc3339-validator" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8b/f3/5507ad3325169347cd8ced61c232ff3df70e2b250c49f0fe140edb4973c6/openapi_schema_validator-0.6.3.tar.gz", hash = "sha256:f37bace4fc2a5d96692f4f8b31dc0f8d7400fd04f3a937798eaf880d425de6ee", size = 11550, upload-time = "2025-01-10T18:08:22.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/21/c6/ad0fba32775ae749016829dace42ed80f4407b171da41313d1a3a5f102e4/openapi_schema_validator-0.6.3-py3-none-any.whl", hash = "sha256:f3b9870f4e556b5a62a1c39da72a6b4b16f3ad9c73dc80084b1b11e74ba148a3", size = 8755, upload-time = "2025-01-10T18:08:19.758Z" }, +] + +[[package]] +name = "openapi-spec-validator" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jsonschema" }, + { name = "jsonschema-path" }, + { name = "lazy-object-proxy" }, + { name = "openapi-schema-validator" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/82/af/fe2d7618d6eae6fb3a82766a44ed87cd8d6d82b4564ed1c7cfb0f6378e91/openapi_spec_validator-0.7.2.tar.gz", hash = "sha256:cc029309b5c5dbc7859df0372d55e9d1ff43e96d678b9ba087f7c56fc586f734", size = 36855, upload-time = "2025-06-07T14:48:56.299Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/dd/b3fd642260cb17532f66cc1e8250f3507d1e580483e209dc1e9d13bd980d/openapi_spec_validator-0.7.2-py3-none-any.whl", hash = "sha256:4bbdc0894ec85f1d1bea1d6d9c8b2c3c8d7ccaa13577ef40da9c006c9fd0eb60", size = 39713, upload-time = "2025-06-07T14:48:54.077Z" }, +] + +[[package]] +name = "opentelemetry-api" +version = "1.45.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/cf3a6b6448849656e1e5f4e2101f3988dc6789724050096aba1bc5ccf592/opentelemetry_api-1.45.2.tar.gz", hash = "sha256:d7b0008e21e5fd61cbe7a7f34ca3b6905a78ab677d9bc395e299359c75cda6b4", size = 73277, upload-time = "2026-10-29T17:29:37.888Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/2d/610db864d7513158cebb8f36cc88e67e73bd6c7aa1905ef18d89e1281c81/opentelemetry_api-1.45.2-py3-none-any.whl", hash = "sha256:2cbd737377e710ee740d261c94e93fa0f08e6d037c18f17513cc0a2d94479a34", size = 60335, upload-time = "2026-10-29T17:29:06.955Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pathable" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/67/93/8f2c2075b180c12c1e9f6a09d1a985bc2036906b13dff1d8917e395f2048/pathable-0.4.4.tar.gz", hash = "sha256:6905a3cd17804edfac7875b5f6c9142a218c7caef78693c2dbbbfbac186d88b2", size = 8124, upload-time = "2025-01-10T18:43:13.247Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/eb/b6260b31b1a96386c0a880edebe26f89669098acea8e0318bff6adb378fd/pathable-0.4.4-py3-none-any.whl", hash = "sha256:5ae9e94793b6ef5a4cbe0a7ce9dbbefc1eec38df253763fd0aeeacf2762dbbc2", size = 9592, upload-time = "2025-01-10T18:43:11.88Z" }, +] + +[[package]] +name = "pillow" +version = "13.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/a4/9cc30690a394a8305e8be4c9036aa16d0dfddadd09079860b582f562496d/pillow-13.0.0.tar.gz", hash = "sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510", size = 47058631, upload-time = "2026-10-16T22:18:09.718Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8e/45/fbad9c7f5e2f1745c373a434d46072bc84eddfefaa28b852425ac957fda6/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c", size = 4190721, upload-time = "2026-10-16T22:16:27.524Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bf/2074902a259a90692c5dbf132885d21671644cbbd3f690047516243a55db/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76", size = 4290236, upload-time = "2026-10-16T22:16:29.562Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a5/3d1bb93b8ea38d15d21356c1b8e9f50971ac592aaa248f7fb71739094c0e/pillow-13.0.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981", size = 3723093, upload-time = "2026-10-16T22:16:31.249Z" }, + { url = "https://files.pythonhosted.org/packages/84/a1/be060135d85187b25fa8b2573786c0e45c62ae27527fff5257675eb97f4a/pillow-13.0.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4", size = 5572253, upload-time = "2026-10-16T22:16:33.103Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f4/0bf6f1ac9ba7ee2ffe9914aeb410a1035a0c1a3c42f122388d7fc950bc4b/pillow-13.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d", size = 4852269, upload-time = "2026-10-16T22:16:35.102Z" }, + { url = "https://files.pythonhosted.org/packages/83/16/b603274617b3e6728da3a0133d4fba1ea8d7fbe0c4ea5fef9cbffa398b8b/pillow-13.0.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6", size = 6516058, upload-time = "2026-10-16T22:16:37.228Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f2/1371707ce438ba9a43f9d24089828494ec6f9396e28cd7903aa704dd6306/pillow-13.0.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d", size = 7142574, upload-time = "2026-10-16T22:16:39.172Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a9/62cf68587f910bc87c5951bcb6e9430c7643a65076adf5f18b4bf350e2bd/pillow-13.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c", size = 6614295, upload-time = "2026-10-16T22:16:41.117Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c9/9e9b48178649730299be0bafbf5f810a5b6977538f5a92d47e4ba5f5d608/pillow-13.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0", size = 7224841, upload-time = "2026-10-16T22:16:43.118Z" }, + { url = "https://files.pythonhosted.org/packages/8f/48/c44e91384b1b6c072a714a63d00608acc16a6a968735e478e4be4384e5ac/pillow-13.0.0-cp314-cp314-win32.whl", hash = "sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055", size = 6508694, upload-time = "2026-10-16T22:16:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/50/e2/7ad17747517e4c813dca78257d6bb593466f8231baf1384bc98d92745edb/pillow-13.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a", size = 7281830, upload-time = "2026-10-16T22:16:47.725Z" }, + { url = "https://files.pythonhosted.org/packages/41/9f/e5b51f2ce8ef4563215334035403ce61a244f93a2e65c24a940131618f87/pillow-13.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4", size = 2695468, upload-time = "2026-10-16T22:16:49.515Z" }, + { url = "https://files.pythonhosted.org/packages/97/73/a53502f731f6c84805a4ce87a8881e86d07a65aee64728b900d112dc703e/pillow-13.0.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa", size = 5578196, upload-time = "2026-10-16T22:16:51.291Z" }, + { url = "https://files.pythonhosted.org/packages/21/50/7e949959a56b5559c78ef172b70fc6025706acb289f3f648a15329e4fa6f/pillow-13.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f", size = 4857276, upload-time = "2026-10-16T22:16:53.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/52/43ded0ac08e77e39d17bd6d894162572201b5e8fdc4a022b14c55a1960a4/pillow-13.0.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1", size = 6607008, upload-time = "2026-10-16T22:16:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2e/4a57ed216d5bf877387cba4d44c02aa87abe51ad2e06404aa6b597c4da39/pillow-13.0.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683", size = 7224281, upload-time = "2026-10-16T22:16:57.738Z" }, + { url = "https://files.pythonhosted.org/packages/5c/3d/2929b248e135752d63aaeabf53f6de797e905b7b6263f9c1e2adade20aba/pillow-13.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058", size = 6695600, upload-time = "2026-10-16T22:16:59.842Z" }, + { url = "https://files.pythonhosted.org/packages/bc/33/fb0da992cf084ed1ee6ee05e96b6191e35c351eb707ca2df874feb24ce47/pillow-13.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e", size = 7300589, upload-time = "2026-10-16T22:17:01.686Z" }, + { url = "https://files.pythonhosted.org/packages/bb/af/c30dd9ef6a3dce84ecb1b2393cac629047a32e66279056c6cb46c3ffd65c/pillow-13.0.0-cp314-cp314t-win32.whl", hash = "sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15", size = 6517204, upload-time = "2026-10-16T22:17:03.722Z" }, + { url = "https://files.pythonhosted.org/packages/85/d5/ef3a8ae7609c8ab26570abf18194882534b480d245a9148b895a2262c004/pillow-13.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0", size = 7286695, upload-time = "2026-10-16T22:17:05.543Z" }, + { url = "https://files.pythonhosted.org/packages/84/3c/0861db0c69a03689ebe3e7599d35668a414deb5aaafd8255215e4e62e3d3/pillow-13.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692", size = 2698098, upload-time = "2026-10-16T22:17:07.674Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + +[[package]] +name = "psycopg" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/26/3ea4ca5eaea1c0debcdf7ee7c1613fbe721dc27a03c461c0817ffd8a0601/psycopg-3.3.6.tar.gz", hash = "sha256:c081f2250df751a943036e42db6df4571c66cd0aabe8291a7a506512b12007d2", size = 168171, upload-time = "2026-09-18T13:22:55.152Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/de/748bd7609c71cae5d737f0ba9192f19329f70180ecda8fff3cac02c5abe3/psycopg-3.3.6-py3-none-any.whl", hash = "sha256:a1db9f7148b06a28606767efaca51fa6f9398c5c0a3810519be69d7000bdb631", size = 215490, upload-time = "2026-09-18T13:15:29.374Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] +pool = [ + { name = "psycopg-pool" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/b9/60711317c284a442511644ea7185b56ebe627606d6741e732cd16108c47b/psycopg_binary-3.3.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:b3f75dee0f9afafabe4edc52c4842f1e1878ed2069bd05b22d6fe961e97e4dba", size = 4720512, upload-time = "2026-09-18T13:20:29.278Z" }, + { url = "https://files.pythonhosted.org/packages/63/da/28befc84454cbc6374550de7746f591f8fe1b6165c1fce249652cc8291c4/psycopg_binary-3.3.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5927b7ba63153cd8e9862987290a2b783a5c590daf2a4ef981700cc3569166d4", size = 4782318, upload-time = "2026-09-18T13:20:35.401Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8a/0d21c2c833cdc0d4244c77e858e0ed37fa2abec2623be4fd686f617109ce/psycopg_binary-3.3.6-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:0bf08b749cc144f33b44a91b78e3f71c60eb07963746a0df5a100b36ce3d7475", size = 5567460, upload-time = "2026-09-18T13:20:41.902Z" }, + { url = "https://files.pythonhosted.org/packages/49/6d/7692d0d4e656b6cc9868d8acc2e3b42f17a0db4a625400a6d093cb0533a1/psycopg_binary-3.3.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:31cd942c23f613276b81a6e6598cefa12960058b0f46e1e874b540c793f6aca5", size = 5246902, upload-time = "2026-09-18T13:20:47.661Z" }, + { url = "https://files.pythonhosted.org/packages/d4/c1/b8a1f18fb1b7558a17f57f7cb3fc8bc93189feea2958925950b3acb15743/psycopg_binary-3.3.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4690cf67738f0e0e49a32aeec99bf0e4595cc2b4f1af984a4345394b1dcff91a", size = 6847192, upload-time = "2026-09-18T13:20:56.874Z" }, + { url = "https://files.pythonhosted.org/packages/a5/76/404f33519167c65cca88ec4998776f1dbebccc301ee977f0e62c47fb0826/psycopg_binary-3.3.6-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ad1c785e784cfd87e8436c6b7702f2d321fc39601bbaf29bc63a41a867091638", size = 5079573, upload-time = "2026-09-18T13:21:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d9/79e8fbc8f37262a415f3550f0bcc5f98037442bf3d12ef6cbae2056655ae/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:79a2a1c3449f6c3409427078ed1cec10de79f3023cb5f2504f0597d350ad46c7", size = 4613633, upload-time = "2026-09-18T13:21:10.664Z" }, + { url = "https://files.pythonhosted.org/packages/d4/47/96225db74be7d2ce04b3a58678b53cda610225055edf5faa775c9f501d8b/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:86147cb5d140341c3363fb5bacce31f8d5543902a46699d3c536b101bbceaf9e", size = 4293375, upload-time = "2026-09-18T13:21:16.027Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d2/18e9c779a5efd565250329adaf529ecc2b8b2ed5be5cb0f6ccee208cbfd9/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:7308c93cf0b19bbaf8e6ff0a6ad50d3c442385739245fe15a8d593bf841734a6", size = 4019883, upload-time = "2026-09-18T13:21:21.587Z" }, + { url = "https://files.pythonhosted.org/packages/ef/28/0cc654afc6c2cda982767f5679d3646b30b1ec86545bdaa9402202d6776c/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:05a83ac9fd52b9bca7cb5ab04b3691163170bd16f53defa27216ea3aa07ee781", size = 4332607, upload-time = "2026-09-18T13:21:27.63Z" }, + { url = "https://files.pythonhosted.org/packages/f1/3e/0a753a74fbd7aef120f286c016e09d3cc3f1daf7688f4a145d27281260b2/psycopg_binary-3.3.6-cp314-cp314-win_amd64.whl", hash = "sha256:1fbd30e537dab22cafdf080608f10148fe2a5f3a61294ddb5113caac8a623840", size = 3755671, upload-time = "2026-09-18T13:21:33.855Z" }, +] + +[[package]] +name = "psycopg-pool" +version = "3.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/5e/c0664b968b102ff68b811d999c728546c48d5c1eec03e3bbaf88c0cb4472/psycopg_pool-3.3.3.tar.gz", hash = "sha256:df87b5d9d0ad7db37f6cdad4fa8ce113d250f5997f6db38e9a99192fb67f9e1d", size = 32006, upload-time = "2026-09-22T15:53:24.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/b4/452c6607a0f479465cd8a9b0d9956919fcb150050c1f83f9f11e6b8ee8dc/psycopg_pool-3.3.3-py3-none-any.whl", hash = "sha256:9b9cd6a4fcec47a410f7e82d408540e7f77b478509e91b44c1a5457a13e5ff37", size = 40304, upload-time = "2026-09-22T15:53:23.712Z" }, +] + +[[package]] +name = "pycparser" +version = "3.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/27/eeb9f307bf07060f3d9f9aa0fb8c15b01bdfded0a941e4b2719c7f94cacd/pycparser-3.12.tar.gz", hash = "sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6", size = 158785, upload-time = "2026-10-22T13:40:03.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/ae/315bbeee126a769cbde43cf4908acb4b40408d8bf8bbf334dd7368d8dadf/pycparser-3.12-py3-none-any.whl", hash = "sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e", size = 62600, upload-time = "2026-10-22T13:40:02.021Z" }, +] + +[[package]] +name = "pycryptodome" +version = "4.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/4d/dfbdfacc0579fa4394e3a442f034c2fb41a44d95d21bf9cd85cee89c3df6/pycryptodome-4.5.0.tar.gz", hash = "sha256:51e97a8d1cf101974bc40c20b3191cb1903891e5c172346943d5d4f3bc4f9d83", size = 1863771, upload-time = "2026-11-15T13:03:45.262Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/73/d1058461b29c5092cea78316a033918253223d8f64e928ba10b59e72a75b/pycryptodome-4.5.0-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:081af2cbd5d85ff282b0f22013b60bd87b7507d87790e0227e42050880adb1d4", size = 1828280, upload-time = "2026-11-15T13:03:07.867Z" }, + { url = "https://files.pythonhosted.org/packages/c1/9d/2f38ba3af1f024adc50eb113d1ff472603756e1e86b73b3c80e2ac958c83/pycryptodome-4.5.0-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:e031a0a430ce1d8a11ef37f2caaf51c5f1c0d9a037db8ffa2753b28a93fc5005", size = 1578107, upload-time = "2026-11-15T13:03:09.607Z" }, + { url = "https://files.pythonhosted.org/packages/e0/7a/47fcebddc746738a5ca6adb7fb0b19ea74188b83aac8ba03011755861742/pycryptodome-4.5.0-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:04c1b9feaf73db4f6aae714589bdabdf2c3d991b579976e1f734de6de133a099", size = 2591108, upload-time = "2026-11-15T13:03:11.352Z" }, + { url = "https://files.pythonhosted.org/packages/41/2d/e4d8e0ac2e1192c05ac7078fc37aaeb2bd21083af7c0561092ff3268e19f/pycryptodome-4.5.0-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3afbe7ecd62357203a588d451c604a61af7d7c9d3f0aa0b792de0612cffb8f8d", size = 3101721, upload-time = "2026-11-15T13:03:13.479Z" }, + { url = "https://files.pythonhosted.org/packages/75/46/e4e348fb1b4bc56704f7696a67f08a39afd7ea6bb751eca858bafbf459b7/pycryptodome-4.5.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:38229c875772dea2bbbe46061e2e5573b5810c52cd7bb5a9225f07cb2c9c00a9", size = 2610488, upload-time = "2026-11-15T13:03:15.134Z" }, + { url = "https://files.pythonhosted.org/packages/a5/c5/6cbd9d4394a94a3c0ee2081f3283736ce17df7e11dfc811ee8722a8a8ced/pycryptodome-4.5.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:33094d990610bf205f2b841599f514429d9ef61aca40e6c11ff1a08aeef5cbb9", size = 3663825, upload-time = "2026-11-15T13:03:16.913Z" }, + { url = "https://files.pythonhosted.org/packages/5a/ac/e0bf3f1d1bd55041051acb543b7a5f291848c62b3f73f2d3fc5badd753c2/pycryptodome-4.5.0-cp39-abi3-win32.whl", hash = "sha256:0a2f83187938665ffd0b80758d3e26e811d01523ba332495ea82fae0f5ca78d0", size = 1360703, upload-time = "2026-11-15T13:03:18.668Z" }, + { url = "https://files.pythonhosted.org/packages/61/04/471045179e76295f5da06e8d1f167054a360d418a90cd3218ed10f99975f/pycryptodome-4.5.0-cp39-abi3-win_amd64.whl", hash = "sha256:b05fed07d07d69c5205689293c3b71a21b8bb4c6f4bb964d1c2d24dfdf22433d", size = 1731036, upload-time = "2026-11-15T13:03:20.355Z" }, + { url = "https://files.pythonhosted.org/packages/0c/bb/997ee5870f70bc3f6d2c611bc4a74a33f7812347ad29799fdbb6bb93072d/pycryptodome-4.5.0-cp39-abi3-win_arm64.whl", hash = "sha256:ae9ad9b4a49f1fb5299b07fb52bf1f00dbe3df0fdce8dd38b9379e5e0b067f46", size = 1417666, upload-time = "2026-11-15T13:03:21.894Z" }, +] + +[[package]] +name = "pydantic" +version = "2.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7c/0b/8e10b2e693af8ec54346a14caf36221334775975a747c9ceaa3f8371d96d/pydantic-2.14.1.tar.gz", hash = "sha256:94f478203dd03404682a1ada216965651dd74b1d2d5ffd62e00e0837caab5c26", size = 900362, upload-time = "2026-10-11T18:37:55.396Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/ea/a56b9fe5066f3537b7882f77e9c5dfb26d8c2eefdaed9b5fc73d57b4dc22/pydantic-2.14.1-py3-none-any.whl", hash = "sha256:9195d967ec791692a04438115466764fb8b9a27b31f14a760437694f40d6b454", size = 488106, upload-time = "2026-10-11T18:37:53.437Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.50.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/24/af4ec4be49fbc810f35b0bdcacc3d433b56bbfa469fd3bfd4ae116cd9bf1/pydantic_core-2.50.1.tar.gz", hash = "sha256:e50d7b94baac6c7d09927fa5ca5800a0c7ee5015c7fcff65beb3a1931b5a6e09", size = 513515, upload-time = "2026-10-11T18:35:44.82Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/33/28b96e81677153715e3eafb9f26663a80841e859fde282a380359b0d3fa1/pydantic_core-2.50.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:1cf41f1ae3fa155cf167a72689ad044bcc1e3c97e064123677149bdfb5dafc4a", size = 2224428, upload-time = "2026-10-11T18:33:07.153Z" }, + { url = "https://files.pythonhosted.org/packages/94/40/15c06410c9b7b8da5805d27b64e09bd3f900e986728106db2689dbc51513/pydantic_core-2.50.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4df197990c15b5a37c5a277d131d9f2c67de6133f2e5dafd80d9bba4b99f46f9", size = 2023740, upload-time = "2026-10-11T18:33:08.763Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4e/5eb629f6efc2a27e421d789dd4bfacbb5f6d09c80c28b13d1e87d73163d5/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0036473f5583e6a60e50b8b21651511564277a3f05cc5dab8cf579f552cd5f6c", size = 2049435, upload-time = "2026-10-11T18:33:10.366Z" }, + { url = "https://files.pythonhosted.org/packages/ba/8e/f195aebec49ad12318876ac2368c197a7939f5d52f8e156d2238ef8a4588/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:992c3514ec891fa7858099183e4d64e6bd5a5d4ff452fae29df22faa77a006bb", size = 2172975, upload-time = "2026-10-11T18:33:12.368Z" }, + { url = "https://files.pythonhosted.org/packages/e0/f5/7ad9fb83010cd5ea0948409db105676ed779c4e709e2d3d89b9fe2558794/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:739dc730e6be3bd5ec2f4ab5cfc7eb047cc45fc1497b3bafec74ff2ed07df597", size = 2364197, upload-time = "2026-10-11T18:33:14.244Z" }, + { url = "https://files.pythonhosted.org/packages/ca/fb/bf0aab3e78301d202b82a0322ec968cd703b11fc0623fde9a28708936f62/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:32fad3a91e51b6d2039c572db04a5a873260b399f6bd62c3552671fa7a4a2899", size = 2444259, upload-time = "2026-10-11T18:33:15.79Z" }, + { url = "https://files.pythonhosted.org/packages/45/35/38f6d6564fae57d9b12e5676dfa947e0e7d5c46dbeaa7eb3352261d6d299/pydantic_core-2.50.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42b54c2c90ad348b5e3a85e03e715d572c1fde357ef104cdfe3b03b697a404ea", size = 2159857, upload-time = "2026-10-11T18:33:17.51Z" }, + { url = "https://files.pythonhosted.org/packages/65/a0/fb0a3ca10f139dcf765b2d312d10cf0f65c57c59993229d00ad12b14ceff/pydantic_core-2.50.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:2df1ff41884de2bc4b307bafd7c40a691094fad2ff8e767e5b45a319257bcf4e", size = 2233954, upload-time = "2026-10-11T18:33:19.591Z" }, + { url = "https://files.pythonhosted.org/packages/8c/6a/e63842252702aa4ec6e6b7178ca85e541a77459076592c23ebe2d9840b33/pydantic_core-2.50.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fe90228920fd8ff2be62622b6bb8a2b11acd65046d50c6b130614b5879605a20", size = 2243876, upload-time = "2026-10-11T18:33:21.393Z" }, + { url = "https://files.pythonhosted.org/packages/39/25/5991cf8318b37e0dfab47b87541619a1df8501a793cba0d978846cba37a7/pydantic_core-2.50.1-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:844b869f118e22a41a091bdcedda8a71bc1b0f62c38d1a0c3211cece47e1d8fc", size = 2277908, upload-time = "2026-10-11T18:33:23.324Z" }, + { url = "https://files.pythonhosted.org/packages/a7/3e/3ee8baaa6cc25a6961c69168cf9ff0f002d56f4e0d541ad6724c18fb61f3/pydantic_core-2.50.1-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:2eb75304506894a281d346220a4f7481a1b8729577c5ed2a05395991966a8396", size = 2449285, upload-time = "2026-10-11T18:33:24.942Z" }, + { url = "https://files.pythonhosted.org/packages/c0/c7/acbec6deac13fe697a80c275a9b6661db62c4d323bac8a47347b1f39c7cd/pydantic_core-2.50.1-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:6b20a4bffabdad0db2927ac034ae3b8a681b1f7a0182f3e60b479ad2fde21ebb", size = 2479013, upload-time = "2026-10-11T18:33:26.925Z" }, + { url = "https://files.pythonhosted.org/packages/10/08/21a3f237b264389f6219d053ee78fc1b5c2fd402c1b1cb2b6cb8b85f9834/pydantic_core-2.50.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:99ba9bc2b8062ea0c326a990f7f00e6530c23579de66dd246e72c4cafef950a5", size = 1484400, upload-time = "2026-10-11T18:33:28.693Z" }, + { url = "https://files.pythonhosted.org/packages/09/ce/077a6d262d12ef09108377ac0717f029f420d773ace63cafd6143af75568/pydantic_core-2.50.1-cp314-cp314-win32.whl", hash = "sha256:cf356f70551d40374eaffb1aa63f1eb6d2006681cbd7a9faea173ce0f4dd7cd2", size = 2103222, upload-time = "2026-10-11T18:33:30.326Z" }, + { url = "https://files.pythonhosted.org/packages/14/4c/350a2415209c43d670eb71d3c040f332c04a30583d39e311b05c7ac15762/pydantic_core-2.50.1-cp314-cp314-win_amd64.whl", hash = "sha256:d32f3acc081cc3923386d88f422cde8892335e95f034e0104bb4cf9310d9915f", size = 2171586, upload-time = "2026-10-11T18:33:32.139Z" }, + { url = "https://files.pythonhosted.org/packages/bf/92/9bea6ca96580a0902fed366f064f0829e404f41889b34543279a1162b888/pydantic_core-2.50.1-cp314-cp314-win_arm64.whl", hash = "sha256:bed5163e03b98bc1fa2eb05d74c63d9c5c95d8ed6254985481640fbf5e237dea", size = 2155773, upload-time = "2026-10-11T18:33:33.896Z" }, + { url = "https://files.pythonhosted.org/packages/86/8c/f121f073cf32bdd5cba7e6230b1ce0ac845b0cf43e44dd597d95af272db2/pydantic_core-2.50.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:9572c1369e9c9da2d64a7b7992c786d90ff295abc93964cfe3125e4290768070", size = 2234017, upload-time = "2026-10-11T18:33:35.588Z" }, + { url = "https://files.pythonhosted.org/packages/a4/69/2bb2bcd6146cffe98d760a46c42ae71efd5151d9b2f9c9bf6619a3b32083/pydantic_core-2.50.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2005207aafe1231315718bf6ed5d064a7300fb4772754af35ee72fc68159492e", size = 2001016, upload-time = "2026-10-11T18:33:37.57Z" }, + { url = "https://files.pythonhosted.org/packages/20/b3/fbf854c7d07ec114260c26e9e2071a4381740f9ae09641dbfcbdf2a18c45/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:64f6047f62a6c5ae08d0a6afb035667aa2d97c3d20d69762e034c5ea144d92a5", size = 2023015, upload-time = "2026-10-11T18:33:39.433Z" }, + { url = "https://files.pythonhosted.org/packages/65/20/6de55b2f92cdb614b745c6e9ced639fc4fd7e1e77604825a88bdece6fcbd/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1ef800dd7d85bcdadf4c3076e4c94e43939493558a3b69a1ea830c706d4617bb", size = 2192543, upload-time = "2026-10-11T18:33:41.381Z" }, + { url = "https://files.pythonhosted.org/packages/0c/d9/19e91c94bd5c405945ce2f15526808aea37e162c253160de8ed7bf70b406/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b0135bcdcaa0f23573f286e4cb5e0fd2962700964ed13df085b85f2b97aeab9e", size = 2388704, upload-time = "2026-10-11T18:33:43.167Z" }, + { url = "https://files.pythonhosted.org/packages/b6/bc/2e24c8415eae1a25ee5a5484946a917123bad2e9d01689a759236928175a/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0b3a6f334c6a2345ca15318ff894502a90012536404b37c844a976c76c846e0b", size = 2464118, upload-time = "2026-10-11T18:33:44.856Z" }, + { url = "https://files.pythonhosted.org/packages/bd/1f/945b8053cb061c64e102bcaf7bfb9ed740c0bd4349f9bb978a7d4ddff4ab/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:06e01fbbfdb9be777b316a71b6c49efaf4a08b615d0a98d678cda3023f79d019", size = 2150142, upload-time = "2026-10-11T18:33:46.661Z" }, + { url = "https://files.pythonhosted.org/packages/2b/78/96a3e50bf0d64aaae781107eb9335b7529d05a85793ed6e7241c4cd1d931/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:a29a061fec0b4e2d714f277e70a3a18125ecff803f2fea6eade2f2e53711d112", size = 2245577, upload-time = "2026-10-11T18:33:48.574Z" }, + { url = "https://files.pythonhosted.org/packages/7a/5d/a6038a0322232758a6ebfa709f4ca14a60bf5b93940cd0b345056a557da4/pydantic_core-2.50.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f5187624823423e1d1b82b1072ac41dc837389e18d3d0572cc19bbee46cd550a", size = 2263728, upload-time = "2026-10-11T18:33:50.527Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4b/76ded3333a457a9344c4f2d63f2d82d0101654b05178fa4ddfe7d3627674/pydantic_core-2.50.1-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:3e46a9eb0a0901dd6275e6b06ac3a464885ef350ec4121fe486869de8053e4bb", size = 2293630, upload-time = "2026-10-11T18:33:52.362Z" }, + { url = "https://files.pythonhosted.org/packages/a9/00/9eca378335c9c1b72bc779bf6e4c2a82ce784f14ec48a0e87102c9870e05/pydantic_core-2.50.1-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:756d669f04e62ec4148ecfe22be6a4484d9b1181a6ef32e205ebfd200540858b", size = 2468048, upload-time = "2026-10-11T18:33:54.118Z" }, + { url = "https://files.pythonhosted.org/packages/35/ca/e3832e9cf93651251de43c8c5c029ed680a3c1a0a1b17ee26c81bed08cbd/pydantic_core-2.50.1-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:c516cc5367ca3448995d42cb994bf3f4c9002d2a7c22eac9622551269ad1b807", size = 2512159, upload-time = "2026-10-11T18:33:55.99Z" }, + { url = "https://files.pythonhosted.org/packages/4b/f2/773469b5a10a39116a2cb17edaf6d722f017dd8da07161b371465311c542/pydantic_core-2.50.1-cp314-cp314t-win32.whl", hash = "sha256:9d1bed94af6a63835461f3cf7502058eb166c58c4778e11d0f433cfb1bd69e19", size = 2098406, upload-time = "2026-10-11T18:33:58.043Z" }, + { url = "https://files.pythonhosted.org/packages/ee/42/0bb74f8f25204b259b11ab7c12dc7f180893b6bd706118b385147fb6efd5/pydantic_core-2.50.1-cp314-cp314t-win_amd64.whl", hash = "sha256:c8dce1f1e0e5358b682a6ad3fa5e31b31d4560997b8e61417e9217c8d60f8a0c", size = 2148481, upload-time = "2026-10-11T18:33:59.913Z" }, + { url = "https://files.pythonhosted.org/packages/47/0d/d801646c9679a4e630e15cf521d4108b93854b42a6e6e02391bd4c6b1095/pydantic_core-2.50.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ceff0acc940be2715bd6ad17b24c0e5304abf44f6efd0f81ee8499e640f9dc86", size = 2174972, upload-time = "2026-10-11T18:34:01.875Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pyparsing" +version = "3.3.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/11/b213bebff182584360cb8d17c72c1677fec5c5c228de439e63bcf8ab1c8f/pyparsing-3.3.3.tar.gz", hash = "sha256:928ae7e20211f3b6f3915a72f06a0cfd29ab9d24279dd6346b6b1a7146397d36", size = 1050487, upload-time = "2026-09-20T20:59:05.609Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/bb/d215ee7c73b61497b28a5503f9f53523f294fcc936762b7caf90e0c1c2b5/pyparsing-3.3.3-py3-none-any.whl", hash = "sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4", size = 126420, upload-time = "2026-09-20T20:59:04.025Z" }, +] + +[[package]] +name = "pytest" +version = "9.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/79/42/bb4641652e58dedeac2ec51abc1f8cae4fee4ec63899a0999016229a0f2a/pytest-9.2.0.tar.gz", hash = "sha256:97c4f3b1f3c7414aa36465e09b8220f1d03627993bd51b1282e193753c80557b", size = 1797555, upload-time = "2026-10-27T21:30:20.728Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/39/1f5d8c8877a0fa327db959c19341d4be61660de1ab6b7391f9eb805bcd67/pytest-9.2.0-py3-none-any.whl", hash = "sha256:07acdcf88ec174f907dc01af70bf52d49b014bbab03faeeade31c96b18c2e588", size = 411776, upload-time = "2026-10-27T21:30:19.131Z" }, +] + +[[package]] +name = "python-dateutil" +version = "4.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/39/b3927f3616f6223832f9b760e4208e8a7c7e1dea2b4d9ad7d32a5d214c3a/python_dateutil-4.0.0.tar.gz", hash = "sha256:d7295ff14fdc0f92f662100bfb6c551d160ca090f2939f1ec5b9794265c03f06", size = 250419, upload-time = "2026-11-01T07:02:42.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/a9/52b89563d154433d79b51ed1eb4842bf7334cb2696891a7fe135618853d7/python_dateutil-4.0.0-py3-none-any.whl", hash = "sha256:a5e246f79eb80475e691bf93862cead98decd8a498cf53ba0516fcdc0f50dc1c", size = 92853, upload-time = "2026-11-01T07:02:40.797Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "redis" +version = "8.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a8/99/604f0b666d4c616d891cf77ebb9db6bb21601344c051aebf1b72b9ff915f/redis-8.1.0.tar.gz", hash = "sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25", size = 5254356, upload-time = "2026-07-30T08:51:00.269Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/66/9d/c5731f6e3608663d4d3656fd8d3aecee8b509c3082818f5a13eae925baea/redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb", size = 560618, upload-time = "2026-07-30T08:50:58.497Z" }, +] + +[[package]] +name = "referencing" +version = "0.36.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2f/db/98b5c277be99dd18bfd91dd04e1b759cad18d1a338188c936e92f921c7e2/referencing-0.36.2.tar.gz", hash = "sha256:df2e89862cd09deabbdba16944cc3f10feb6b3e6f18e902f7cc25609a34775aa", size = 74744, upload-time = "2025-01-25T08:48:16.138Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/b1/3baf80dc6d2b7bc27a95a67752d0208e410351e3feb4eb78de5f77454d8d/referencing-0.36.2-py3-none-any.whl", hash = "sha256:e8699adbbf8b5c7de96d8ffa0eb5c158b3beafce084968e2ea8bb08c6794dcd0", size = 26775, upload-time = "2025-01-25T08:48:14.241Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "rfc3339-validator" +version = "0.1.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/28/ea/a9387748e2d111c3c2b275ba970b735e04e15cdb1eb30693b6b5708c4dbd/rfc3339_validator-0.1.4.tar.gz", hash = "sha256:138a2abdf93304ad60530167e51d2dfb9549521a836871b88d7f4695d0022f6b", size = 5513, upload-time = "2021-05-12T16:37:54.178Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/44/4e421b96b67b2daff264473f7465db72fbdf36a07e05494f50300cc7b0c6/rfc3339_validator-0.1.4-py2.py3-none-any.whl", hash = "sha256:24f6ec1eda14ef823da9e36ec7113124b39c04d50a4d3d3a3c2859577e7791fa", size = 3490, upload-time = "2021-05-12T16:37:52.536Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/42/68/3bd46b8a5e01d3c2ebdf9c5e9497912e3fe0cde02bac21a7130ca866e403/rpds_py-2026.9.1.tar.gz", hash = "sha256:4793ef7f78268b124b73fa933440f01d258bbae01de9fa53e9080c9ab0425a12", size = 63948, upload-time = "2026-10-04T16:32:36.469Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/f2/67b94be1532767803415c1c5a1fd88ea487643d74a673cec1ba140af77bb/rpds_py-2026.9.1-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c9d1aca01f49170fdcf5c92761b1fafe97f554b721ca4570c5949fff778f0d4b", size = 347178, upload-time = "2026-10-04T16:30:00.865Z" }, + { url = "https://files.pythonhosted.org/packages/04/37/b751de2b59b0197a1d92a5dd491de88e8a5e928c2e6562581974f1e85263/rpds_py-2026.9.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:3f0e9ac28fc067d4d34b88ae43c48e9489455c97fee9633d851f7eeed5a05d35", size = 341878, upload-time = "2026-10-04T16:30:02.564Z" }, + { url = "https://files.pythonhosted.org/packages/72/e2/5873bc4643c250db9e05d48dc0c93763d4aa68bc3b81164cb1af3b45b284/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:07deecbfce94c78473018bc7d10b337cc651d12df87a1eb2cb3e4024bc9c33d0", size = 373963, upload-time = "2026-10-04T16:30:04.026Z" }, + { url = "https://files.pythonhosted.org/packages/51/03/5acf7632158247f3f6386ff0af3a1ee48167d575037e8d0920594b76d92b/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:821b2755db9194409254012f429c56643416fb96ef9be090be82ec8826b7f477", size = 377975, upload-time = "2026-10-04T16:30:05.555Z" }, + { url = "https://files.pythonhosted.org/packages/e6/00/63fda451b8bffa5808fc8bb311ee7c073b340974b09f273c7b2a145d3d62/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3c91c210ae7645626c608400e3519b4a642f837cce09ca830db3beb2e9f274d4", size = 490347, upload-time = "2026-10-04T16:30:07.156Z" }, + { url = "https://files.pythonhosted.org/packages/a1/ae/c093ffd070ba0fb02f76c565d06fecc65ad6e4afdbae78f7031076d3cdac/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:54ac2158a6f96cfbabff0b2eedaf94b90c5ec7ca8317fcadc61e1c2b2e0ff6ef", size = 392962, upload-time = "2026-10-04T16:30:08.77Z" }, + { url = "https://files.pythonhosted.org/packages/22/9d/d08a1128ab199b2f0cf25bfeb0639bd05119fff4b7c47bec24ef9a8ec23f/rpds_py-2026.9.1-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eac2f5dbafd585dfe31f86a23ebf0d3ba480a9d49ebc87947267b5608d4ea0cd", size = 371623, upload-time = "2026-10-04T16:30:10.501Z" }, + { url = "https://files.pythonhosted.org/packages/53/c7/4758ddcbb75609414bbccfcb11d612436f9b3ee821bd2f33f0f1604ee648/rpds_py-2026.9.1-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:8aa5dda18d39b6143eb24809d158f9252c88f402749b6f1b62a506cc7d96cc35", size = 376997, upload-time = "2026-10-04T16:30:12.124Z" }, + { url = "https://files.pythonhosted.org/packages/87/e4/947bd7f608ff60faf46dc9d389c3dffd0e3d767d78a0be19978448ef0ce7/rpds_py-2026.9.1-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5c90e7fa02e8f5de0d10c17595c568ada48c5302e749462c0ea1a4c362111a86", size = 400169, upload-time = "2026-10-04T16:30:13.804Z" }, + { url = "https://files.pythonhosted.org/packages/4b/35/fe93e020a0543b5670472c18d7e6af3197c08da571240ce1965c84f85c0f/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:e6d198bad4e49dd6732fbd636e2fc5c082f45c8cad0b4acb756b00c82c76072e", size = 551615, upload-time = "2026-10-04T16:30:15.332Z" }, + { url = "https://files.pythonhosted.org/packages/0d/4f/5d2a0136bb03b2a56a39dc6ff92d58a6e3e53a2e17079238b86228882f16/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:96beca19ec79de272e8668585380ff9092c47077c1d7a1e098e00bbd921f4785", size = 614655, upload-time = "2026-10-04T16:30:16.92Z" }, + { url = "https://files.pythonhosted.org/packages/09/1c/3f1025aaf70d9bf7272cc41f8b64ee76b48bf01726248138430e16f23b38/rpds_py-2026.9.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:a5cf77eb04f20b720be95265a3e00eb2a14814074255cc27069c551b2db53118", size = 579602, upload-time = "2026-10-04T16:30:18.555Z" }, + { url = "https://files.pythonhosted.org/packages/53/0d/5c72e6204f76610608706da32b6b7e11ef7e10317558a7bb15a122e008dc/rpds_py-2026.9.1-cp314-cp314-win32.whl", hash = "sha256:a03d57b86d2a51d0a66c92177e2be154ad015f357791d306e714569999cdb4cc", size = 206315, upload-time = "2026-10-04T16:30:20.05Z" }, + { url = "https://files.pythonhosted.org/packages/a4/0b/489d48abbcc7d70cf3fbf662d9d22abf1f4650761c0a9ae05260800800d3/rpds_py-2026.9.1-cp314-cp314-win_amd64.whl", hash = "sha256:837c6b305e26fe0f75b15c92cf3b2ba29e0ae19dc40b1c557b026cb426347d0c", size = 222852, upload-time = "2026-10-04T16:30:21.604Z" }, + { url = "https://files.pythonhosted.org/packages/91/16/bbb05a7e6a10cf79ba639be7f799d770ee15f64175cc61d081b218dd402a/rpds_py-2026.9.1-cp314-cp314-win_arm64.whl", hash = "sha256:fce4b85234a0cbad67bf8e6e1201ee815d172c9aebad75f25645bc4d834f8e31", size = 218709, upload-time = "2026-10-04T16:30:23.036Z" }, + { url = "https://files.pythonhosted.org/packages/22/ac/ac507a0a4ec478ca470440a09583db4be5259ba7670aeba0620822f1e57a/rpds_py-2026.9.1-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:3a72c11530d71abfb66c8d7696a2f86c43e63fca8b948f1a784ac490f4ec688e", size = 349596, upload-time = "2026-10-04T16:30:24.558Z" }, + { url = "https://files.pythonhosted.org/packages/e9/f2/817a46b658d5070f477f722c298ee9a24525b0e4017347964146ef5fdd0e/rpds_py-2026.9.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:068c37bba854ec2fe42f7365c640af11dd9895890ccbf2df5070d0c059bd7f96", size = 343114, upload-time = "2026-10-04T16:30:26.048Z" }, + { url = "https://files.pythonhosted.org/packages/6c/42/6ade976b13ac1b4cb3bf2eb603f1be2fe74df19a29988d4c2b386be59d6f/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fca4eb6df565e2a928f1c7dad92d27db8f9df0f449e76423ed5d7e713ed445", size = 374197, upload-time = "2026-10-04T16:30:27.699Z" }, + { url = "https://files.pythonhosted.org/packages/d9/70/77cdf1d3f1a07faabe936016ae623aec7981f73108a8fe7a203ed2e21998/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c933c6678c6f116ff8af47a4c6db0868b8ace74af0343016c0ef00f00272ea69", size = 378025, upload-time = "2026-10-04T16:30:29.451Z" }, + { url = "https://files.pythonhosted.org/packages/3f/6b/18a44a3beaa9b7931acb04af7bd9539836477c630a794452d4826d6185d4/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:028ad274ea951dac64491b5d1e65712a4aeabfdbdb9fccf797b57bd899b0c495", size = 491969, upload-time = "2026-10-04T16:30:30.995Z" }, + { url = "https://files.pythonhosted.org/packages/73/27/fb39cfd6bddaf741b024f813890374578ff8ac1f1adc473659c048b03b05/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:740d0a99cf9de0b17a3943388e9294a59becf75e7c43421f387bd3c7a9901f7c", size = 393905, upload-time = "2026-10-04T16:30:32.628Z" }, + { url = "https://files.pythonhosted.org/packages/ed/71/0fa7bb77b57af0d710273964180d11b503f62b8a5c358eb2d8c3f62feff6/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0da298fb372dc192610a4b9ecbc68a0cd8b675bbbd1fc519d01b41cfd658333e", size = 374569, upload-time = "2026-10-04T16:30:34.257Z" }, + { url = "https://files.pythonhosted.org/packages/54/22/f41cfac269af3b449513ef1bc3d7f32fde52abbbd2d01c7e76b47743acd9/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:eb61be926bb81567c1f48bdc8aa22b9855048dc2efd53871f9f7e6e9a5632346", size = 377482, upload-time = "2026-10-04T16:30:35.997Z" }, + { url = "https://files.pythonhosted.org/packages/b4/fc/312b49006e7f8f9ca5f96647577b8aa6f3df30519c46bd448f5c425af0b2/rpds_py-2026.9.1-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:42e75466f83cd43f6026c81eab74246efb2bdadafb307b85700632d06c68f299", size = 400040, upload-time = "2026-10-04T16:30:37.76Z" }, + { url = "https://files.pythonhosted.org/packages/cf/a6/18cca7a878dc7fa95165a83343fd4d7b65643fd22e54e47340a451121d5c/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:617f59cde379b4f648a09797b7f683d04b90a46344cddab85639da5aff0f5531", size = 551849, upload-time = "2026-10-04T16:30:39.443Z" }, + { url = "https://files.pythonhosted.org/packages/e4/6d/1f5685e20f39604691bdc3c05aaa6b8bd2f954e9e996477adf2376768e33/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:3edae8c5ddfdb6985d49ae9d150516e5076888879022f91a26c2de9276ce0bdb", size = 614842, upload-time = "2026-10-04T16:30:41.231Z" }, + { url = "https://files.pythonhosted.org/packages/c6/25/98652109fd9f7e10268dd4571aa52b81987001b806f37ef1a18260de714a/rpds_py-2026.9.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0f045bb053c9057720d72c56dffe30dffdc05997b2897a827b9325f0ab6623fa", size = 582606, upload-time = "2026-10-04T16:30:43.345Z" }, + { url = "https://files.pythonhosted.org/packages/19/03/11ca09099bab5f53373a80a334c424ec917c13d050760a59499d2af5171e/rpds_py-2026.9.1-cp314-cp314t-win32.whl", hash = "sha256:bf35d0568abda97233239ce32896d3ad53fccc537832c104e30c94aa5fb93569", size = 203341, upload-time = "2026-10-04T16:30:44.954Z" }, + { url = "https://files.pythonhosted.org/packages/6f/8a/88909e3ffd9f47f5b58211473875d8c3c09079f0058c46fb72c55a702a26/rpds_py-2026.9.1-cp314-cp314t-win_amd64.whl", hash = "sha256:1e8d4d79d828299bf44a55db22a9388ab967b49d17132c88eab0f4360b48da8e", size = 222843, upload-time = "2026-10-04T16:30:46.486Z" }, +] + +[[package]] +name = "ruff" +version = "0.17.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/84/0e/3d154514b5bad104f7e1c13169c2d09aa9959fccdc06eb47610ccdfa83ed/ruff-0.17.4.tar.gz", hash = "sha256:f32d178b50e826e00f6c6c1be3f56333068acb9b67fe167143527fa9b81e890d", size = 5357750, upload-time = "2026-11-05T18:52:17.638Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/61/c83c78bbb6108f6c6270a106cf12b54175f4f42da088b9bb880f18e31289/ruff-0.17.4-py3-none-linux_armv6l.whl", hash = "sha256:2b08c4ff98dc234f4a3871a3105353b922fa875ea1073ca31001eb8c06bb4d89", size = 10252195, upload-time = "2026-11-05T18:51:36.594Z" }, + { url = "https://files.pythonhosted.org/packages/0a/12/dd158708378866e8b53e145a465dbc51a7cccb9fa197a74d67eaedde7234/ruff-0.17.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:13e781c3c5f663e7fba737eaaeb70dd943dd55171daf316afb25cd5268cfe0c7", size = 10266044, upload-time = "2026-11-05T18:51:39.502Z" }, + { url = "https://files.pythonhosted.org/packages/28/32/89a94a8eb11fd505a381ae6f04e190d038d9958695209dc1786ce5cfcda3/ruff-0.17.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7ed2f416349944f908f9574215994602242b0836844eba48eec61473033afca0", size = 9432227, upload-time = "2026-11-05T18:51:41.968Z" }, + { url = "https://files.pythonhosted.org/packages/58/de/a392dc38b215798ae72e639d228b7f6c02d826a95d1c0ad7c4c0ee49ef5b/ruff-0.17.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8ea6e0017c785f7282092b11e46338e4ebed1b9e575ebf2e55a8a1eda968241a", size = 10136900, upload-time = "2026-11-05T18:51:44.562Z" }, + { url = "https://files.pythonhosted.org/packages/b7/8f/a880f60f42005f6ffdf58f77af34c7ee327000b5607f48a2f6a69d8b242c/ruff-0.17.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7e061dcc0933c7031a616ac8d66ac540309b2aa21ab558101a8f53e9ac7a1185", size = 10051245, upload-time = "2026-11-05T18:51:46.7Z" }, + { url = "https://files.pythonhosted.org/packages/09/55/98844264108af7d5ec27ca3a9cd5a989094ca657a14599c4b69a537a0e9a/ruff-0.17.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f99e95a1515161138be3cbd1b06c06e739d8feb38ade35564556999fffc3686e", size = 10704341, upload-time = "2026-11-05T18:51:49.228Z" }, + { url = "https://files.pythonhosted.org/packages/28/a2/5edf97659c3ac7c73f978003bd314936f671037fdac6b978e9de3e4b2b65/ruff-0.17.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5f28cf0dd37cb172fa5a8d2e92e6ada3a9dbd0609f094ed24039929eef0b7524", size = 11394465, upload-time = "2026-11-05T18:51:51.534Z" }, + { url = "https://files.pythonhosted.org/packages/f2/8f/51371ae856b2bef4d15a044a574695fbc0f6ae63aafa42a2a22428070fb6/ruff-0.17.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6b21c771b296888e8a35c6f48555f4b44e4483f61675d6ed51810a5203469ce6", size = 10988262, upload-time = "2026-11-05T18:51:53.91Z" }, + { url = "https://files.pythonhosted.org/packages/af/1b/62874f14c7873f27c7887d2b1ea41de990e14ba9518801ffe8eacb5d51a6/ruff-0.17.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2cd025509b2e15721b426fc70fd946d71f43b3db3b08e946dfec2d696d911690", size = 10572829, upload-time = "2026-11-05T18:51:56.129Z" }, + { url = "https://files.pythonhosted.org/packages/98/30/860b9dae67432b16b88558a9cb2d3194825fa62838f8b9cc49f5d9022095/ruff-0.17.4-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:49b77f7b3921669466882a4881168801bdb06ec2cd83fb936049016cf5fdece1", size = 10617545, upload-time = "2026-11-05T18:51:58.172Z" }, + { url = "https://files.pythonhosted.org/packages/0f/08/781a4b41a1c83fd6e30fea610d6502a6f94e49cf94693fd37bdcbc6bfe11/ruff-0.17.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:e3b84eb74f260d1c2dbd236fd9d4c011d4fdd0a52bdf8aea57a10e3a1b71fca5", size = 10111955, upload-time = "2026-11-05T18:52:00.59Z" }, + { url = "https://files.pythonhosted.org/packages/3c/e9/94b1c7b677120f30839254518924f73a8157e9bcfa200edcd5796fecdb99/ruff-0.17.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:20e43d7a08e712e33067ba887e6574546c79bd4831596ce8f16803ac7ff9ad90", size = 10058355, upload-time = "2026-11-05T18:52:03.61Z" }, + { url = "https://files.pythonhosted.org/packages/27/5f/a825910b9f309336dc33cb8c0774126662c1d2f765699b3e0a1bd538d01f/ruff-0.17.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:0d1f4848fd2ec0499be4046db9b7dfcc952666885f00233cc7033515282556a8", size = 10415932, upload-time = "2026-11-05T18:52:06.264Z" }, + { url = "https://files.pythonhosted.org/packages/8b/b8/0796468423413ea4df1a6e2442ae26cf49b8f7dda0eecf246e0d5d0dde54/ruff-0.17.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:6007e7f3a0e55b0fbdb947af69cf24c21b4928df6be8ef3e767c69b5ccccf496", size = 10826033, upload-time = "2026-11-05T18:52:08.675Z" }, + { url = "https://files.pythonhosted.org/packages/b9/ab/d8e407669ffe5014429baf780c2df47e001968a6e4c17f615e2e7b13a9ba/ruff-0.17.4-py3-none-win32.whl", hash = "sha256:3971a78cba76298b284c8321e6d40d93045ed348bec5c369d85099efc97fb7cf", size = 9931993, upload-time = "2026-11-05T18:52:11.169Z" }, + { url = "https://files.pythonhosted.org/packages/bd/15/42af8a72d8946e3c9efd2bbf6641e462b6832783b0d043d63ef5b280ba7f/ruff-0.17.4-py3-none-win_amd64.whl", hash = "sha256:67c5a894fc054cf41f75f7afa24815de5a74329d97634126c2a1015740ff4e8c", size = 10292922, upload-time = "2026-11-05T18:52:13.436Z" }, + { url = "https://files.pythonhosted.org/packages/68/9e/f1dbffcfb8ff25dadbc21f87a8b73c3848abe4abada9d4b08ccef8bec40c/ruff-0.17.4-py3-none-win_arm64.whl", hash = "sha256:181ae38a9a8fb94925bf7d0794327b48d7fd7f4b50cfd84c0e999cc09ef8d131", size = 9889396, upload-time = "2026-11-05T18:52:15.549Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "starlette" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/bb/3d8757f21e9a53d9035c9eb039a777047139a16bfdbedfc7a58241c3d914/starlette-1.10.0.tar.gz", hash = "sha256:b8399098d4c1e3522734f7235dc79b1fc967f96cf139c81e0fa30479e0685f9b", size = 2734356, upload-time = "2026-10-24T15:04:12.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/2e/947cfe02def9371c07cd5bf5b6b61c829e9aecb9904db02a66cfafbc1eec/starlette-1.10.0-py3-none-any.whl", hash = "sha256:478c0ab3b4ced6e82719849c118fa406f0278a3499c1a898cd15cd31bdc38ce4", size = 80571, upload-time = "2026-10-24T15:04:10.371Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/50/0195198eb1d75bab716bb0a813abea4cc394bfa17a92a189f021a34859af/typing_inspection-0.4.5.tar.gz", hash = "sha256:7f16c411e7930dc79596d42ab44672a3fe5f60eb34b781a1647e7872363120c7", size = 81194, upload-time = "2026-11-04T14:13:23.774Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/0c/ea2544850a7c135e15033a5c7a8e7211136184b7ae63e2239d557c2b3019/typing_inspection-0.4.5-py3-none-any.whl", hash = "sha256:bc210535442b39e1bf2805e0c7a17dfd6a4ace760c84c7f7d16245d23a6b1c39", size = 14874, upload-time = "2026-11-04T14:13:22.616Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7a/29/6a1391d31cd00ff9e927215a224a880e1e84e826b64ba80a499e51c074d8/tzdata-2026.6.tar.gz", hash = "sha256:85d2cd9c09f81f0bb29db27759fc83af4f1e620efb814f8c5bf8b4cc4feece6d", size = 201017, upload-time = "2026-10-28T09:57:09.137Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/66/9a06c2227fedfd4919f0eaaa8b94377d9730882a7e4f95823c9a331ab5e1/tzdata-2026.6-py2.py3-none-any.whl", hash = "sha256:0a10eec6e13ef16b627fabed9a5f2ba61a62b1c67f327305f4e7e412a70b4aec", size = 348210, upload-time = "2026-10-28T09:57:07.276Z" }, +] + +[[package]] +name = "urllib3" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.54.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/34/30e9280707135d2cfc589dfff3cb796bd07a3aeb1a3e415ba09dd89d7bb4/uvicorn-0.54.0.tar.gz", hash = "sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620", size = 112283, upload-time = "2026-09-25T06:52:37.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/0c/b54a4fdd7f90a3af8b02ebc9ce6712c2c208b7926a2f7bad95c33ebbe943/uvicorn-0.54.0-py3-none-any.whl", hash = "sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf", size = 87427, upload-time = "2026-09-25T06:52:35.829Z" }, +] -- 2.54.0 From d3e156e8a12925797e12eb7088b0da9126638d5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Sun, 15 Nov 2026 22:25:22 +0300 Subject: [PATCH 02/11] test(pipeline): add repeatable crash duplicate and stale version matrix --- pyproject.toml | 9 +- scripts/verify_clean.py | 215 +++++++++++ scripts/verify_cluster.py | 342 +++++++++++++++++ scripts/verify_pipeline.py | 526 +++++++++++++++++++++++++++ src/worker.py | 8 + student-tests/conftest.py | 119 ++++++ student-tests/test_api.py | 412 +++++++++++++++++++++ student-tests/test_attachments.py | 212 +++++++++++ student-tests/test_contract_edges.py | 26 ++ student-tests/test_history_acl.py | 23 ++ student-tests/test_idempotency.py | 75 ++++ student-tests/test_sessions.py | 114 ++++++ 12 files changed, 2077 insertions(+), 4 deletions(-) create mode 100644 scripts/verify_clean.py create mode 100644 scripts/verify_cluster.py create mode 100644 scripts/verify_pipeline.py create mode 100644 student-tests/conftest.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_contract_edges.py create mode 100644 student-tests/test_history_acl.py create mode 100644 student-tests/test_idempotency.py create mode 100644 student-tests/test_sessions.py diff --git a/pyproject.toml b/pyproject.toml index 2bd4436c..cd0d6b96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,13 +32,14 @@ select = ["E", "F", "I", "B", "UP"] "research/*.py" = ["E501"] "scripts/*.py" = ["E501"] -[tool.pytest.ini_options] -testpaths = ["student-tests"] -addopts = "-ra" - # Keep SQL statements together; the formatter controls executable statements. "src/attachments.py" = ["E501"] "src/pipeline.py" = ["E501"] "src/worker.py" = ["E501"] "src/operator.py" = ["E501"] "src/reconcile.py" = ["E501"] + +[tool.pytest.ini_options] +testpaths = ["student-tests"] +addopts = "-ra" + diff --git a/scripts/verify_clean.py b/scripts/verify_clean.py new file mode 100644 index 00000000..6a56626e --- /dev/null +++ b/scripts/verify_clean.py @@ -0,0 +1,215 @@ +import argparse +import atexit +import json +import os +import socket +import subprocess +import tempfile +from datetime import UTC, datetime +from pathlib import Path +from uuid import uuid4 + +ROOT = Path(__file__).resolve().parents[1] +PROGRESS = {} + + +def save_incomplete(): + receipt = PROGRESS.get("receipt") + if receipt and receipt.get("result") != "PASS": + target = PROGRESS["output"].with_name( + PROGRESS["output"].stem + "-failed.json" + ) + if target.exists(): + target = target.with_name( + target.stem + "-" + uuid4().hex[:8] + target.suffix + ) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + json.dumps( + {**receipt, "result": "INCOMPLETE"}, + ensure_ascii=False, + indent=2, + ) + + "\n" + ) + + +atexit.register(save_incomplete) + + +def run(args, *, cwd, env, expected=0): + result = subprocess.run( + args, cwd=cwd, env=env, capture_output=True, text=True + ) + + def redact(text): + return ( + text.replace(str(cwd), "") + .replace(str(ROOT), "") + .replace(str(Path.home()), "") + ) + + if result.returncode != expected: + if "receipt" in PROGRESS: + PROGRESS["receipt"]["failed_action"] = redact(" ".join(args)) + raise RuntimeError(redact(result.stdout + result.stderr)) + return redact(result.stdout.strip()) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", default="evidence/clean-clone.json") + args = parser.parse_args() + output = ROOT / args.output + if output.exists(): + raise SystemExit("Choose new output path") + assert not subprocess.check_output( + ["git", "status", "--porcelain"], cwd=ROOT + ) + sha = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True + ).strip() + identity = uuid4().hex[:10] + project = "mtusi-lab5-clean-" + identity + with socket.socket() as reservation: + reservation.bind(("127.0.0.1", 0)) + public_port = str(reservation.getsockname()[1]) + env = { + **os.environ, + "COMPOSE_PROJECT_NAME": project, + "API_IMAGE": "mtusi-messenger:lab5-clean-" + identity, + "PUBLIC_PORT": public_port, + "VCS_REF": sha, + "NO_PROXY": "localhost,127.0.0.1", + "no_proxy": "localhost,127.0.0.1", + } + receipt = { + "date": datetime.now(UTC).isoformat(), + "source_commit": sha, + "fresh_secrets_environment_and_volumes": True, + "command": "python scripts/verify_clean.py", + "fixed_host_port_for_restart": int(public_port), + } + PROGRESS.update(receipt=receipt, output=output) + with tempfile.TemporaryDirectory(prefix="mtusi-lab5-clean-") as temporary: + clone = Path(temporary) / "lab5" + run( + [ + "git", + "clone", + "--no-hardlinks", + "--branch", + "solution/bvt2301-maslov", + str(ROOT), + str(clone), + ], + cwd=ROOT, + env=env, + ) + for name in [".env", ".venv", "secrets"]: + assert not (clone / name).exists() + run(["uv", "sync", "--locked"], cwd=clone, env=env) + receipt["static_checks"] = run( + ["make", "check", "validate", "lint", "PYTHON=.venv/bin/python"], + cwd=clone, + env=env, + ) + run(["make", "bootstrap"], cwd=clone, env=env) + try: + run( + ["docker", "compose", "up", "-d", "--build", "--wait"], + cwd=clone, + env=env, + ) + env["BASE_URL"] = "http://127.0.0.1:" + public_port + receipt["smoke"] = run( + ["make", "test", "BASE_URL=" + env["BASE_URL"]], + cwd=clone, + env=env, + ) + receipt["http_tests"] = run( + [".venv/bin/pytest", "student-tests", "-q"], cwd=clone, env=env + ) + run( + [ + ".venv/bin/python", + "-m", + "scripts.verify_cluster", + "--output", + "evidence/clean-cluster.json", + ], + cwd=clone, + env=env, + ) + run( + [ + ".venv/bin/python", + "-m", + "scripts.verify_storage", + "--output", + "evidence/clean-storage.json", + ], + cwd=clone, + env=env, + ) + receipt["storage"] = json.loads( + (clone / "evidence/clean-storage.json").read_text() + ) + receipt["cluster"] = json.loads( + (clone / "evidence/clean-cluster.json").read_text() + ) + import hashlib + + expected = { + p.name: hashlib.sha256(p.read_bytes()).hexdigest() + for p in (clone / "src").glob("*.py") + } + ids = run( + ["docker", "compose", "ps", "-q", "api"], cwd=clone, env=env + ).split() + assert len(ids) == 2 + code = ( + "import hashlib,json,pathlib; print(json.dumps({p.name:" + "hashlib.sha256(p.read_bytes()).hexdigest() for p in pathlib.Path('/app/src').glob('*.py')}))" + ) + for cid in ids: + actual = json.loads( + run( + ["docker", "exec", cid, "python", "-c", code], + cwd=clone, + env=env, + ) + ) + assert actual == expected + info = json.loads( + run(["docker", "inspect", cid], cwd=clone, env=env) + )[0] + assert ( + info["Config"]["Labels"][ + "org.opencontainers.image.revision" + ] + == sha + ) + assert not info["HostConfig"]["PortBindings"] + receipt["both_replicas_runtime_hashes_and_revision_match"] = True + receipt["fsck"] = run( + ["git", "fsck", "--full"], cwd=clone, env=env + ) + receipt["checks_completed"] = True + finally: + assert project.startswith("mtusi-lab5-clean-") + run( + ["docker", "compose", "down", "--volumes", "--remove-orphans"], + cwd=clone, + env=env, + ) + receipt["result"] = "PASS" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(receipt, ensure_ascii=False, indent=2) + "\n") + print( + "PASS: fresh clone, build, HTTP tests, failover, rolling, persistence and runtime hashes" + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_cluster.py b/scripts/verify_cluster.py new file mode 100644 index 00000000..a0dbd639 --- /dev/null +++ b/scripts/verify_cluster.py @@ -0,0 +1,342 @@ +"""Проверки реальных процессов, failover, rolling и общей идемпотентности.""" + +import argparse +import json +import os +import subprocess +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + +import httpx + +from scripts import cluster +from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready, sql + + +def snapshot(): + ids = compose("ps", "-q").split() + result = [] + for c in json.loads(command(["docker", "inspect", *ids])): + role = c["Config"]["Labels"]["com.docker.compose.service"] + entry = { + "role": role, + "container": c["Id"][:12], + "pid": c["State"]["Pid"], + "started_at": c["State"]["StartedAt"], + "ports": c["HostConfig"]["PortBindings"], + "cpu_nano": c["HostConfig"]["NanoCpus"], + "memory_limit": c["HostConfig"]["Memory"], + } + if role == "api": + entry["instance"] = cluster.alias(c["Id"]) + assert not entry["ports"] + result.append(entry) + return result + + +def exercise(client, session, chat, action, *, seconds=15): + path = f"/api/v1/chats/{chat['id']}/messages" + observations, confirmed, errors, reads = [], {}, 0, 0 + started = time.monotonic() + finished = threading.Event() + action_result = {} + + def operate(): + time.sleep(0.5) + try: + action() + except BaseException as exc: + action_result["error"] = type(exc).__name__ + finally: + action_result["finished"] = round(time.monotonic() - started, 3) + finished.set() + + thread = threading.Thread(target=operate) + thread.start() + key = uuid4().hex + try: + while time.monotonic() - started < seconds or not finished.is_set(): + h = {**auth(session), "Idempotency-Key": key} + try: + r = client.post(path, headers=h, json={"text": key}) + if r.status_code != 201: + errors += 1 + continue + m = r.json() + confirmed[m["id"]] = m + observations.append( + { + "at": round(time.monotonic() - started, 3), + "instance": r.headers["x-instance-id"], + } + ) + key = uuid4().hex + read = client.get( + path, headers=auth(session), params={"limit": 100} + ) + reads += read.status_code == 200 + errors += read.status_code != 200 + except httpx.HTTPError: + errors += 1 + time.sleep(0.04) + finally: + thread.join(timeout=90) + assert not action_result.get("error"), action_result + found, cursor = [], None + while True: + r = client.get( + path, + headers=auth(session), + params={"limit": 100, **({"cursor": cursor} if cursor else {})}, + ) + assert r.status_code == 200 + page = r.json() + found.extend(page["items"]) + cursor = page["next_cursor"] + if not cursor: + break + by_id = {m["id"]: m for m in found} + assert all(by_id.get(k) == v for k, v in confirmed.items()) + texts = [m["text"] for m in found] + assert len(texts) == len(set(texts)) + recovery = next( + ( + x["at"] + for x in observations + if x["at"] >= action_result["finished"] + ), + None, + ) + assert recovery is not None and (seconds > 20 or recovery <= 10) + elapsed = time.monotonic() - started + return { + "confirmed_writes": len(confirmed), + "successful_reads": reads, + "errors": errors, + "successful_rps": round((len(confirmed) + reads) / elapsed, 2), + "first_success_after_action_seconds": recovery, + "action": action_result, + "observations": observations, + "confirmed_preserved": True, + "duplicate_texts": False, + } + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", default="evidence/cluster-repeat.json") + args = parser.parse_args() + report = { + "date": datetime.now(timezone.utc).isoformat(), + "source_commit": command(["git", "rev-parse", "HEAD"]), + "complete": False, + } + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + try: + ready() + initial = snapshot() + apis = [c for c in initial if c["role"] == "api"] + assert len(apis) == 2 and len({c["pid"] for c in apis}) == 2 + report["before"] = initial + with httpx.Client(base_url=BASE, trust_env=False, timeout=8) as client: + user, _, session = actor(client) + r = client.post( + "/api/v1/chats", + headers=auth(session), + json={ + "title": "Cluster verification", + "member_ids": [user["id"]], + }, + ) + assert r.status_code == 201 + chat = r.json() + path = f"/api/v1/chats/{chat['id']}/messages" + key = uuid4().hex + + def repeat(_): + with httpx.Client( + base_url=BASE, trust_env=False, timeout=8 + ) as c: + r = c.post( + path, + headers={**auth(session), "Idempotency-Key": key}, + json={"text": "Same concurrent request"}, + ) + assert r.status_code == 201 + return r.json(), r.headers["x-instance-id"] + + with ThreadPoolExecutor(max_workers=10) as pool: + repeated = list(pool.map(repeat, range(30))) + assert len({m[0]["id"] for m in repeated}) == 1 + assert len({m[1] for m in repeated}) == 2 + report["parallel_replay"] = { + "requests": 30, + "messages": 1, + "instances": sorted({m[1] for m in repeated}), + } + # Сбой второй записи транзакции не должен оставить сообщение. + fault_key = uuid4().hex + fault_text = "Atomic rollback " + fault_key + sql( + "CREATE FUNCTION student_idempo_fault() RETURNS trigger " + "LANGUAGE plpgsql AS $$ BEGIN IF NEW.key='" + + fault_key + + "' THEN RAISE EXCEPTION 'synthetic'; END IF; RETURN NEW; END $$; " + "CREATE TRIGGER student_idempo_fault BEFORE INSERT ON idempotency " + "FOR EACH ROW EXECUTE FUNCTION student_idempo_fault()" + ) + try: + failed = client.post( + path, + headers={**auth(session), "Idempotency-Key": fault_key}, + json={"text": fault_text}, + ) + assert failed.status_code == 500 + assert ( + sql( + "SELECT count(*) FROM messages WHERE text='" + + fault_text + + "'" + ) + == "0" + ) + finally: + sql( + "DROP TRIGGER student_idempo_fault ON idempotency; " + "DROP FUNCTION student_idempo_fault()" + ) + retried = client.post( + path, + headers={**auth(session), "Idempotency-Key": fault_key}, + json={"text": fault_text}, + ) + assert retried.status_code == 201 + report["atomic_rollback_and_5xx_retry"] = True + sql( + "UPDATE idempotency SET expires_at=clock_timestamp()-interval '1 second' " + "WHERE key='" + fault_key + "'" + ) + expired = client.post( + path, + headers={**auth(session), "Idempotency-Key": fault_key}, + json={"text": "New retention window"}, + ) + assert expired.status_code == 201 + assert expired.json()["id"] != retried.json()["id"] + report["expired_key_can_be_reused"] = True + # Отдельный чат каждой серии: проверки дубликатов не смешивают тексты. + report["failovers"] = [] + for api in apis: + + def stop(target=api["instance"]): + env = {**os.environ, "TARGET_INSTANCE_ID": target} + subprocess.run( + ["scripts/stop-one.sh"], + cwd=ROOT, + env=env, + check=True, + capture_output=True, + text=True, + ) + + r = client.post( + "/api/v1/chats", + headers=auth(session), + json={"title": "Failover", "member_ids": [user["id"]]}, + ) + result = exercise(client, session, r.json(), stop) + assert api["instance"] not in [ + o["instance"] for o in result["observations"][-5:] + ] + assert ( + command( + [ + "docker", + "inspect", + "-f", + "{{.State.Running}}", + api["container"], + ] + ) + == "false" + ) + result["stopped"] = api + report["failovers"].append(result) + compose("up", "-d", "--no-build", "--wait") + ready() + r = client.post( + "/api/v1/chats", + headers=auth(session), + json={"title": "Rolling", "member_ids": [user["id"]]}, + ) + before_roll = snapshot() + report["rolling"] = exercise( + client, + session, + r.json(), + lambda: command(["scripts/rolling-update.sh"]), + seconds=45, + ) + after_roll = snapshot() + assert not ( + {c["container"] for c in before_roll if c["role"] == "api"} + & {c["container"] for c in after_roll if c["role"] == "api"} + ) + report["rolling"]["before"] = before_roll + report["rolling"]["after"] = after_roll + replay = repeat(0)[0] + assert replay == repeated[0][0] + report["replay_after_replacement"] = True + before = snapshot() + env = { + **os.environ, + "NO_PROXY": "localhost,127.0.0.1", + "no_proxy": "localhost,127.0.0.1", + } + for mode, hook in [ + ("failover", "scripts/stop-one.sh"), + ("persistence", "scripts/restart.sh"), + ]: + if mode == "persistence": + compose("up", "-d", "--no-build", "--wait") + ready() + before = snapshot() + run = subprocess.run( + [ + ".venv/bin/python", + "tests/resilience.py", + "--mode", + mode, + "--action-script", + hook, + "--base-url", + BASE, + ], + cwd=ROOT, + env=env, + capture_output=True, + text=True, + ) + assert run.returncode == 0, run.stdout + run.stderr + report["public_" + mode] = run.stdout.strip() + after = snapshot() + for role in ["postgres", "cache", "s3", "broker", "lb"]: + old = next(c for c in before if c["role"] == role) + new = next(c for c in after if c["role"] == role) + assert old["started_at"] != new["started_at"] + report["after_persistence"] = after + report["complete"] = True + finally: + output.write_text( + json.dumps(report, ensure_ascii=False, indent=2) + "\n" + ) + print("PASS: process failover, rolling, replay and full persistence") + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_pipeline.py b/scripts/verify_pipeline.py new file mode 100644 index 00000000..1d4fd6f8 --- /dev/null +++ b/scripts/verify_pipeline.py @@ -0,0 +1,526 @@ +"""Repeatable fault injection for grade 5, track 2; synthetic data only.""" + +import argparse +import hashlib +import io +import json +import os +import time +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from uuid import uuid4 + +import httpx +from PIL import Image + +os.environ.setdefault("COMPOSE_FILE", "compose.yaml:compose.test.yaml") +os.environ.setdefault("NO_PROXY", "localhost,127.0.0.1") +os.environ.setdefault("no_proxy", "localhost,127.0.0.1") +from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready + +PREFIX = """import io,json,hashlib,time,urllib.request +from uuid import UUID,uuid4 +from psycopg.types.json import Jsonb +from src.pipeline import database,broker,envelope,STREAM,GROUP +from src.object_store import ObjectStore,BUCKET +from src.operator import change +from src.reconcile import reconcile +db,store,queue=database(),ObjectStore(),broker() +""" + + +def inside(code): + cid = compose("ps", "-q", "api").split()[0] + return json.loads( + command( + ["docker", "exec", "-i", cid, "python", "-"], input=PREFIX + code + ) + ) + + +def wait(fn, timeout=90): + end = time.monotonic() + timeout + while time.monotonic() < end: + value = fn() + if value: + return value + time.sleep(0.15) + raise TimeoutError("Fault scenario condition not reached") + + +def stop(role): + compose("stop", "--timeout", "1", role) + + +def start(role): + compose("up", "-d", "--no-deps", "--no-recreate", role) + + +def row(job): + return inside( + f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\nprint(json.dumps(r,default=str))" + ) + + +def inject(job, phase, action, remaining=1): + return inside( + f"with db.connection() as c: r=c.execute('INSERT INTO pipeline_faults(target,phase,action,remaining) VALUES(%s,%s,%s,%s) RETURNING id',({job!r},{phase!r},{action!r},{remaining})).fetchone()\nprint(json.dumps(r['id']))" + ) + + +def hit(identity): + return inside( + f"with db.connection() as c: r=c.execute('SELECT hit_at IS NOT NULL hit FROM pipeline_faults WHERE id=%s',({identity},)).fetchone()\nprint(json.dumps(r['hit']))" + ) + + +def operator(job, version=None): + return inside( + f"print(json.dumps(change(db,store,job_id=UUID({job!r}),operator='synthetic-verifier',reason='Explicit fault matrix replay',version={version!r})))" + ) + + +def duplicate(job, count=5): + return inside( + f"with db.connection() as c: r=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone()\ne=envelope(r)\nids=[queue.xadd(STREAM,{{'job':json.dumps(e)}}) for _ in range({count})]\nprint(json.dumps(ids))" + ) + + +def inspect_effect(job): + return inside(f"""with db.connection() as c: + j=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone() + v=c.execute('SELECT name,object_key,sha256,size_bytes FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',(j['attachment_id'],j['pipeline_version'])).fetchall() + p=c.execute('SELECT count(*) n FROM effect_publications WHERE attachment_id=%s AND pipeline_version=%s',(j['attachment_id'],j['pipeline_version'])).fetchone()['n'] + e=c.execute('SELECT event,detail,created_at FROM pipeline_events WHERE job_id=%s ORDER BY id',({job!r},)).fetchall() +keys=[o.object_name for o in store.objects() if o.object_name.startswith(f"objects/{{j['attachment_id']}}/v{{j['pipeline_version']}}/")] +print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e}},default=str))""") + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--output", default="evidence/pipeline-repeat.json") + parser.add_argument("--repeats", type=int, default=2) + args = parser.parse_args() + output = ROOT / args.output + if output.exists(): + raise ValueError("Choose a new output path") + report = { + "source_commit": command(["git", "rev-parse", "HEAD"]), + "date": datetime.now(timezone.utc).isoformat(), + "track": 2, + "complete": False, + "cases": [], + } + output.parent.mkdir(parents=True, exist_ok=True) + + def save(name, details): + report["cases"].append({"case": name, **details}) + output.write_text( + json.dumps(report, ensure_ascii=False, indent=2) + "\n" + ) + print("PASS:", name, flush=True) + + try: + ready() + # Check instrumentation is really enabled and production default is off. + assert inside( + "import os\nprint(json.dumps(os.getenv('PIPELINE_FAULTS')=='1'))" + ) + stop("reconciler") + with httpx.Client( + base_url=BASE, trust_env=False, timeout=25 + ) as client: + user, _, session = actor(client) + h = auth(session) + c = client.post( + "/api/v1/chats", + headers=h, + json={ + "title": "Pipeline synthetic faults", + "member_ids": [user["id"]], + }, + ) + assert c.status_code == 201 + chat = c.json()["id"] + raw_io = io.BytesIO() + Image.new("RGBA", (320, 200), (20, 50, 170, 120)).save( + raw_io, format="PNG" + ) + raw = raw_io.getvalue() + + def upload(): + r = client.post( + f"/api/v1/chats/{chat}/attachments", + headers={**h, "Content-Type": "image/png"}, + params={"filename": "synthetic.png"}, + content=raw, + ) + assert r.status_code == 202, r.text + a = r.json() + assert a["status"] == "queued" and not a["variants"] + j = inside( + f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(str(r['id'])))" + ) + return a, j + + def metadata(a): + r = client.get("/api/v1/attachments/" + a["id"], headers=h) + assert r.status_code == 200, r.text + return r.json() + + def until(a, state="ready", timeout=90): + return wait( + lambda: ( + r if (r := metadata(a))["status"] == state else None + ), + timeout, + ) + + def confirm(a, j): + until(a) + downloaded = client.get( + f"/api/v1/attachments/{a['id']}/content", headers=h + ) + assert ( + downloaded.status_code == 200 and downloaded.content == raw + ) + m = metadata(a) + assert ( + m["sha256"] == hashlib.sha256(raw).hexdigest() + and m["error_code"] is None + ) + assert {v["name"] for v in m["variants"]} == { + "thumbnail", + "crop", + "optimized", + } + for v in m["variants"]: + r = client.get( + f"/api/v1/attachments/{a['id']}/content", + headers=h, + params={"variant": v["name"]}, + ) + assert ( + r.status_code == 200 + and hashlib.sha256(r.content).hexdigest() + == v["sha256"] + ) + with Image.open(io.BytesIO(r.content)) as image: + assert ( + image.size + == { + "thumbnail": (256, 160), + "crop": (128, 128), + "optimized": (320, 200), + }[v["name"]] + ) + assert not image.getexif() + effect = inspect_effect(j) + assert ( + effect["publications"] == 1 + and len(effect["variants"]) == 4 + and len(effect["physical_keys"]) == 4 + ) + return effect + + # Source-only crash before the DB handoff must never be reported as accepted. + stop("worker") + stop("relay") + marker = inject(chat, "api_after_staging", "wait") + before = inside( + f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))" + ) + pool = ThreadPoolExecutor(max_workers=1) + future = pool.submit( + lambda: client.post( + f"/api/v1/chats/{chat}/attachments", + headers={**h, "Content-Type": "image/png"}, + params={"filename": "unconfirmed.png"}, + content=raw, + ) + ) + wait(lambda marker=marker: hit(marker)) + compose("kill", "-s", "SIGKILL", "api") + try: + response = future.result(15) + assert response.status_code != 202 + except httpx.HTTPError: + pass + pool.shutdown() + start("api") + ready() + after = inside( + f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))" + ) + assert before == after + save( + "staging_before_handoff_crash", + { + "no_202": True, + "no_attachment_or_job_created": True, + "orphan_retained_for_cleanup": True, + }, + ) + # API durable commit/outbox survives restart without a broker or worker. + a, j = upload() + linked = client.post( + f"/api/v1/chats/{chat}/messages", + headers={**h, "Idempotency-Key": uuid4().hex}, + json={"text": "", "attachment_ids": [a["id"]]}, + ) + assert linked.status_code == 201 + assert ( + client.get( + f"/api/v1/attachments/{a['id']}/content", headers=h + ).status_code + == 409 + ) + compose("restart", "api") + ready() + compose("kill", "-s", "SIGKILL", "broker") + start("broker") + start("relay") + start("worker") + effect = confirm(a, j) + save( + "commit_before_publish_and_worker_stopped", + { + "attachment": a["id"], + "effect": effect, + "queued_link_allowed": True, + "api_and_broker_restarted": True, + }, + ) + # Broker-volume survival with a confirmed, still-unconsumed stream entry. + stop("worker") + a, j = upload() + wait( + lambda: inside( + f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NOT NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']>0))" + ) + ) + ids = inside("print(json.dumps(queue.xrange(STREAM)))") + compose("kill", "-s", "SIGKILL", "broker") + start("broker") + wait(lambda: inside("print(json.dumps(queue.ping()))")) + restored = inside("print(json.dumps(queue.xrange(STREAM)))") + assert restored == ids + start("worker") + save( + "confirmed_broker_entry_survives_sigkill", + { + "effect": confirm(a, j), + "same_stream_entries": True, + "entry_count": len(ids), + }, + ) + # Confirm succeeded, PG marker did not: relay recovery may publish a duplicate. + stop("worker") + stop("relay") + a, j = upload() + marker = inject(j, "relay_after_publish", "crash") + start("relay") + wait(lambda marker=marker: hit(marker)) + time.sleep(0.5) + unpublished = inside( + f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']))" + ) + assert unpublished >= 1 + start("relay") + start("worker") + save( + "relay_crash_after_xadd_before_pg_marker", + { + "unpublished_after_crash": unpublished, + "effect": confirm(a, j), + }, + ) + # Repeated before/after effect and commit/ack windows. + for repeat in range(args.repeats): + for phase in ["before_effect", "after_effect", "after_commit"]: + stop("worker") + a, j = upload() + marker = inject(j, phase, "wait") + start("worker") + wait(lambda marker=marker: hit(marker)) + at_crash = inspect_effect(j) + assert at_crash["publications"] == ( + 1 if phase == "after_commit" else 0 + ) + assert len(at_crash["physical_keys"]) == ( + 0 if phase == "before_effect" else 4 + ) + compose("kill", "-s", "SIGKILL", "worker") + start("worker") + effect = confirm(a, j) + save( + f"{phase}_sigkill_repeat_{repeat + 1}", + {"at_crash": at_crash, "effect": effect}, + ) + stop("worker") + a, j = upload() + inject(j, "before_effect", "transient") + start("worker") + effect = confirm(a, j) + assert effect["job"]["attempt"] == 2 + save("transient_retry_with_jitter", {"effect": effect}) + stop("worker") + a, j = upload() + inject(j, "before_effect", "hang") + start("worker") + effect = confirm(a, j) + assert any( + e["event"] == "retry" + and e["detail"]["code"] == "worker_timeout" + for e in effect["events"] + ) + save("hung_child_killed_and_retried", {"effect": effect}) + stop("worker") + a, j = upload() + inject(j, "before_effect", "permanent") + start("worker") + failed = until(a, "failed") + assert failed["error_code"] == "invalid_image" + assert ( + client.get( + f"/api/v1/attachments/{a['id']}/content", headers=h + ).status_code + == 409 + ) + r = client.post( + f"/api/v1/chats/{chat}/messages", + headers={**h, "Idempotency-Key": uuid4().hex}, + json={"text": "", "attachment_ids": [a["id"]]}, + ) + assert r.status_code == 409 + assert operator(j) == j + effect = confirm(a, j) + assert effect["job"]["cycle"] == 2 + try: + operator(j) + except RuntimeError: + pass + else: + raise AssertionError("Replay of ready must be rejected") + save( + "permanent_failed_registry_and_audited_replay", + { + "failed": failed, + "effect": effect, + "ready_replay_rejected": True, + }, + ) + stop("worker") + a, j = upload() + inject(j, "before_effect", "transient", remaining=5) + start("worker") + failed = until(a, "failed", timeout=150) + assert ( + failed["error_code"] == "retry_exhausted" + and row(j)["attempt"] == 5 + ) + operator(j) + save( + "finite_retry_exhaustion_then_explicit_replay", + {"failed": failed, "effect": confirm(a, j)}, + ) + # Reverse notification order and add concurrent copies: one publication/key. + stop("worker") + stop("relay") + a, j = upload() + b, k = upload() + duplicate(k, 8) + duplicate(j, 8) + start("worker") + start("relay") + ea = confirm(a, j) + confirm(b, k) + duplicate(j, 10) + duplicate(k, 10) + wait( + lambda: any( + e["event"] == "duplicate_decision" + for e in inspect_effect(j)["events"] + ) + ) + assert inspect_effect(j)["job"]["attempt"] == ea["job"]["attempt"] + save( + "reordered_and_duplicate_notifications", + { + "effects": [inspect_effect(j), inspect_effect(k)], + "added_duplicates": 36, + "no_extra_attempt_after_ready": True, + }, + ) + # Hold v1 after its PUTs, complete v2 on the other bounded slot, release old CAS. + for repeat in range(args.repeats): + stop("worker") + a, j = upload() + marker = inject(j, "after_effect", "wait") + start("worker") + wait(lambda marker=marker: hit(marker)) + k = operator(j, version=2) + v2 = confirm(a, k) + hashes = {v["name"]: v["sha256"] for v in v2["variants"]} + inside( + f"with db.connection() as c: c.execute('UPDATE pipeline_faults SET released_at=clock_timestamp() WHERE id=%s',({marker},))\nprint('true')" + ) + wait( + lambda j=j: any( + e["event"] == "stale_completion_rejected" + for e in inspect_effect(j)["events"] + ) + ) + duplicate(j, 12) + wait( + lambda j=j: any( + e["event"] == "duplicate_decision" + for e in inspect_effect(j)["events"] + ) + ) + assert hashes == { + v["name"]: v["sha256"] for v in confirm(a, k)["variants"] + } + assert inspect_effect(j)["publications"] == 0 + save( + f"stale_v1_after_v2_repeat_{repeat + 1}", + { + "v1": inspect_effect(j), + "v2": inspect_effect(k), + "new_hashes_unchanged": True, + }, + ) + # No arbitrary URL/bytes fields in queue envelopes. + invalid = inside( + "bad={'schema_version':1,'url':'http://example.invalid/raw'}\nqueue.xadd(STREAM,{'job':json.dumps(bad)})\nprint('true')" + ) + assert invalid + wait( + lambda: inside( + "with db.connection() as c: r=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE event='invalid_envelope'\").fetchone()\nprint(json.dumps(r['n']>0))" + ) + ) + save( + "malformed_notification_rejected", {"durable_diagnostic": True} + ) + report["worker_metrics"] = inside( + "with db.connection() as c: r=c.execute(\"SELECT job_id,detail FROM pipeline_events WHERE event='child_finished' ORDER BY id\").fetchall()\nprint(json.dumps(r,default=str))" + ) + report["audit"] = inside( + "with db.connection() as c: r=c.execute('SELECT job_id,action,operator,reason,created_at FROM pipeline_audit ORDER BY id').fetchall()\nprint(json.dumps(r,default=str))" + ) + report["complete"] = True + finally: + for role in ["api", "broker", "relay", "worker", "reconciler"]: + try: + start(role) + except Exception: + pass + output.write_text( + json.dumps(report, ensure_ascii=False, indent=2) + "\n" + ) + print("PASS: complete versioned pipeline fault matrix", flush=True) + + +if __name__ == "__main__": + main() diff --git a/src/worker.py b/src/worker.py index aab146f7..70fc9fc2 100644 --- a/src/worker.py +++ b/src/worker.py @@ -343,6 +343,14 @@ def consume(slot): with db.connection() as conn: event(conn, row["id"], "ready_effect_missing") continue + if state in {"ready", "obsolete", "failed"}: + with db.connection() as conn: + event( + conn, + data["job_id"], + "duplicate_decision", + {"state": state, "entry": entry}, + ) if state == "claimed": receiver, sender = context.Pipe(duplex=False) child = context.Process( diff --git a/student-tests/conftest.py b/student-tests/conftest.py new file mode 100644 index 00000000..7c1f2dfb --- /dev/null +++ b/student-tests/conftest.py @@ -0,0 +1,119 @@ +import json +import os +from pathlib import Path +from uuid import uuid4 + +import httpx +import pytest +from jsonschema import Draft202012Validator, FormatChecker + +CONTRACT = json.loads( + (Path(__file__).parents[1] / "contracts/openapi.json").read_text() +) +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8084") +PASSWORD = "Synthetic-Password-🌍-123" +TOKENS = {} + + +def check(result: httpx.Response, status: int, schema: str) -> dict: + assert result.status_code == status, result.text + assert result.headers["x-request-id"] + assert result.headers["x-instance-id"] + assert result.headers["content-type"].startswith("application/json") + payload = result.json() + spec = {"$ref": f"#/components/schemas/{schema}", **CONTRACT} + Draft202012Validator(spec, format_checker=FormatChecker()).validate( + payload + ) + if schema == "Error": + assert payload["error"]["request_id"] == result.headers["x-request-id"] + assert ( + payload["error"]["code"] + == { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 500: "internal_error", + 503: "unavailable", + }[status] + ) + return payload + + +def register(client: httpx.Client, **overrides) -> dict: + data = { + "username": "u_" + uuid4().hex[:24], + "display_name": "Алиса 🌍", + "password": PASSWORD, + } + data.update(overrides) + user = check(client.post("/api/v1/users", json=data), 201, "User") + session = check( + client.post( + "/api/v1/auth/sessions", + auth=httpx.BasicAuth(data["username"], data["password"]), + ), + 201, + "Session", + ) + TOKENS[user["id"]] = session["token"] + return user + + +@pytest.fixture +def client(): + with httpx.Client( + base_url=BASE_URL, timeout=15, trust_env=False + ) as client: + yield client + + +@pytest.fixture +def actors(client): + return register(client), register(client), register(client) + + +@pytest.fixture +def chat(client, actors): + alice, bob, _ = actors + return check( + client.post( + "/api/v1/chats", + headers=headers(alice), + json={"title": "Учёба 🌍", "member_ids": [alice["id"], bob["id"]]}, + ), + 201, + "Chat", + ) + + +def headers(actor: dict) -> dict: + return { + "Authorization": "Bearer " + TOKENS[actor["id"]], + "Idempotency-Key": uuid4().hex, + } + + +def history(client, actor, chat_id, limit=7): + found, cursor = [], None + while True: + params = {"limit": limit} + if cursor: + params["cursor"] = cursor + page = check( + client.get( + f"/api/v1/chats/{chat_id}/messages", + headers=headers(actor), + params=params, + ), + 200, + "MessagePage", + ) + found.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return found diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..964c1425 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,412 @@ +import base64 +import json +import threading +from concurrent.futures import ThreadPoolExecutor +from uuid import UUID, uuid4 + +import httpx +import pytest +from conftest import BASE_URL, PASSWORD, check, headers, history, register + + +@pytest.mark.parametrize("path", ["/health/live", "/health/ready"]) +def test_health(client, path): + assert check(client.get(path), 200, "Health") == {"status": "ok"} + + +@pytest.mark.parametrize( + "value", [None, "not-a-uuid", str(uuid4()), uuid4().hex] +) +def test_identification(client, value): + check( + client.get( + "/api/v1/users/me", headers={"X-User-Id": value} if value else {} + ), + 401, + "Error", + ) + + +def test_identity(client, actors): + assert ( + check( + client.get("/api/v1/users/me", headers=headers(actors[0])), + 200, + "User", + ) + == actors[0] + ) + + +@pytest.mark.parametrize( + "data", + [ + {}, + [], + None, + {"username": "ABC", "display_name": "A"}, + {"username": "abc\n", "display_name": "A"}, + {"username": 123, "display_name": "A"}, + {"username": "abc", "display_name": " "}, + {"username": "abc", "display_name": "x" * 101}, + {"username": "abc", "display_name": "A", "extra": True}, + {"username": "abc", "display_name": "\ud800"}, + ], +) +def test_invalid_user(client, data): + if isinstance(data, dict): + data = {"password": PASSWORD, **data} + check(client.post("/api/v1/users", content=json.dumps(data)), 400, "Error") + + +@pytest.mark.parametrize( + "raw", [b"{", b"NaN", b'{"username":"abc", "display_name":NaN}', b"\xff"] +) +def test_malformed_json(client, raw): + check(client.post("/api/v1/users", content=raw), 400, "Error") + + +def test_auth_before_json(client): + check(client.post("/api/v1/chats", content=b"{"), 401, "Error") + + +def test_duplicate_username(client, actors): + check( + client.post( + "/api/v1/users", + json={ + "username": actors[0]["username"], + "display_name": "B", + "password": PASSWORD, + }, + ), + 409, + "Error", + ) + + +def test_unicode_codepoints(client, actors, chat): + user = register(client, display_name="🌍" * 100) + assert user["display_name"] == "🌍" * 100 + text = "🌍" * 4000 + result = check( + client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers=headers(actors[0]), + json={"text": text}, + ), + 201, + "Message", + ) + assert result["text"] == text + assert result["sender_id"] == actors[0]["id"] + + +@pytest.mark.parametrize( + "method,suffix,body", + [ + ("GET", "", None), + ("GET", "/messages", None), + ("POST", "/messages", {"text": "hello"}), + ], +) +def test_acl(client, actors, chat, method, suffix, body): + outsider = headers(actors[2]) + actual = check( + client.request( + method, + f"/api/v1/chats/{chat['id']}{suffix}", + headers=outsider, + json=body, + ), + 404, + "Error", + ) + missing = check( + client.request( + method, + f"/api/v1/chats/{uuid4()}{suffix}", + headers=outsider, + json=body, + ), + 404, + "Error", + ) + assert actual["error"]["message"] == missing["error"]["message"] + + +@pytest.mark.parametrize("value", ["", " \t\n", 42, True, None, "a" * 4001]) +def test_bad_message(client, actors, chat, value): + check( + client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers=headers(actors[0]), + json={"text": value}, + ), + 400, + "Error", + ) + + +def test_sender_not_client_controlled(client, actors, chat): + check( + client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers=headers(actors[0]), + json={"text": "Hi", "sender_id": actors[1]["id"]}, + ), + 400, + "Error", + ) + + +@pytest.mark.parametrize( + "case", + [ + "duplicate", + "unknown", + "no_caller", + "empty", + "wrong_type", + "bad_uuid", + "too_many", + "blank_title", + ], +) +def test_invalid_chat(client, actors, case): + a, b, _ = actors + members = { + "duplicate": [a["id"], a["id"]], + "unknown": [a["id"], str(uuid4())], + "no_caller": [b["id"]], + "empty": [], + "wrong_type": a["id"], + "bad_uuid": ["broken"], + "too_many": [str(uuid4()) for _ in range(101)], + "blank_title": [a["id"]], + }[case] + check( + client.post( + "/api/v1/chats", + headers=headers(a), + json={ + "title": " " if case == "blank_title" else "Title", + "member_ids": members, + }, + ), + 400, + "Error", + ) + + +@pytest.mark.parametrize( + "limit", ["0", "101", "-1", "1.5", "true", "1e1", "", "12"] +) +def test_bad_limit(client, actors, chat, limit): + check( + client.get( + f"/api/v1/chats/{chat['id']}/messages", + headers=headers(actors[0]), + params={"limit": limit}, + ), + 400, + "Error", + ) + + +@pytest.mark.parametrize( + "cursor", ["", "broken", "W10", "e30", "*", "x" * 2049] +) +def test_bad_cursor(client, actors, chat, cursor): + check( + client.get( + f"/api/v1/chats/{chat['id']}/messages", + headers=headers(actors[0]), + params={"cursor": cursor}, + ), + 400, + "Error", + ) + + +def test_pagination(client, actors, chat): + a, b, outsider = actors + sent = [] + path = f"/api/v1/chats/{chat['id']}/messages" + for i in range(23): + sent.append( + check( + client.post( + path, + headers=headers(a if i % 2 else b), + json={"text": f"Msg {i}"}, + ), + 201, + "Message", + ) + ) + got = history(client, a, chat["id"]) + assert got == sorted( + sent, key=lambda m: (m["created_at"], UUID(m["id"]).int) + ) + first = check( + client.get(path, headers=headers(a), params={"limit": 1}), + 200, + "MessagePage", + ) + second = check( + client.get( + path, + headers=headers(b), + params={"limit": 22, "cursor": first["next_cursor"]}, + ), + 200, + "MessagePage", + ) + assert second["items"] == got[1:] + assert second["next_cursor"] is None + chats = check( + client.get("/api/v1/chats", headers=headers(outsider)), 200, "ChatPage" + ) + assert chats["items"] == [] + other = check( + client.post( + "/api/v1/chats", + headers=headers(a), + json={"title": "Other", "member_ids": [a["id"]]}, + ), + 201, + "Chat", + ) + check( + client.get( + f"/api/v1/chats/{other['id']}/messages", + headers=headers(a), + params={"cursor": first["next_cursor"]}, + ), + 400, + "Error", + ) + # Валидный base64/JSON с неверным типом ключа тоже даёт 400. + data = json.loads( + base64.urlsafe_b64decode( + first["next_cursor"] + "=" * (-len(first["next_cursor"]) % 4) + ) + ) + data["at"] = 17 + forged = ( + base64.urlsafe_b64encode(json.dumps(data).encode()) + .decode() + .rstrip("=") + ) + check( + client.get(path, headers=headers(a), params={"cursor": forged}), + 400, + "Error", + ) + + +def test_chat_pagination_scope(client, actors): + a, b, _ = actors + created = [] + for i in range(4): + created.append( + check( + client.post( + "/api/v1/chats", + headers=headers(a), + json={"title": str(i), "member_ids": [a["id"], b["id"]]}, + ), + 201, + "Chat", + ) + ) + page = check( + client.get("/api/v1/chats", headers=headers(a), params={"limit": 2}), + 200, + "ChatPage", + ) + assert page["items"] == created[:2] + last = check( + client.get( + "/api/v1/chats", + headers=headers(a), + params={"limit": 2, "cursor": page["next_cursor"]}, + ), + 200, + "ChatPage", + ) + assert last["items"] == created[2:] + assert last["next_cursor"] is None + check( + client.get( + "/api/v1/chats", + headers=headers(b), + params={"cursor": page["next_cursor"]}, + ), + 400, + "Error", + ) + + +def test_concurrent_messages(client, actors, chat): + barrier = threading.Barrier(10) + a = actors[0] + path = f"/api/v1/chats/{chat['id']}/messages" + + def worker(i): + with httpx.Client( + base_url=BASE_URL, timeout=15, trust_env=False + ) as conn: + barrier.wait(timeout=15) + return [ + check( + conn.post( + path, headers=headers(a), json={"text": f"{i}:{j}"} + ), + 201, + "Message", + )["id"] + for j in range(10) + ] + + with ThreadPoolExecutor(max_workers=10) as pool: + ids = [item for batch in pool.map(worker, range(10)) for item in batch] + rows = history(client, a, chat["id"]) + assert len(rows) == len(ids) == len(set(ids)) == 100 + assert set(ids) == {row["id"] for row in rows} + assert len({row["text"] for row in rows}) == 100 + + +def test_registration_race(client): + username = "race_" + uuid4().hex[:20] + barrier = threading.Barrier(10) + + def worker(_): + with httpx.Client( + base_url=BASE_URL, timeout=15, trust_env=False + ) as conn: + barrier.wait(timeout=15) + result = conn.post( + "/api/v1/users", + json={ + "username": username, + "display_name": "Test", + "password": PASSWORD, + }, + ) + check( + result, + result.status_code, + "User" if result.status_code == 201 else "Error", + ) + return result.status_code + + with ThreadPoolExecutor(max_workers=10) as pool: + statuses = list(pool.map(worker, range(10))) + assert statuses.count(201) == 1 + assert statuses.count(409) == 9 + + +def test_unknown_path(client): + check(client.get("/does-not-exist"), 404, "Error") diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..b1ce48c5 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,212 @@ +import hashlib +import io +import time +from pathlib import Path +from uuid import uuid4 + +import pytest +from conftest import check, headers +from PIL import Image + +PNG = (Path(__file__).parents[1] / "tests/fixtures/sample.png").read_bytes() + + +def upload( + client, actor, chat, *, data=PNG, filename="Фото 🌍.png", mime="image/png" +): + return client.post( + f"/api/v1/chats/{chat['id']}/attachments", + headers={**headers(actor), "Content-Type": mime}, + params={"filename": filename}, + content=data, + ) + + +def wait_ready(client, actor, attachment): + end = time.monotonic() + 60 + while time.monotonic() < end: + row = check( + client.get( + f"/api/v1/attachments/{attachment['id']}", + headers=headers(actor), + ), + 200, + "Attachment", + ) + if row["status"] == "ready": + return row + assert row["status"] != "failed", row + time.sleep(0.2) + raise TimeoutError("Pipeline readiness") + + +def test_attachment_roundtrip_acl_and_message(client, actors, chat): + attachment = check(upload(client, actors[0], chat), 202, "Attachment") + assert attachment["status"] == "queued" and attachment["variants"] == [] + attachment = wait_ready(client, actors[0], attachment) + assert attachment["sha256"] == hashlib.sha256(PNG).hexdigest() + assert attachment["size_bytes"] == len(PNG) + path = f"/api/v1/attachments/{attachment['id']}" + assert ( + check(client.get(path, headers=headers(actors[1])), 200, "Attachment") + == attachment + ) + downloaded = client.get(path + "/content", headers=headers(actors[1])) + assert downloaded.status_code == 200 and downloaded.content == PNG + assert downloaded.headers["Content-Type"] == "image/png" + assert not downloaded.history + for suffix in ["", "/content"]: + check( + client.get(path + suffix, headers=headers(actors[2])), 404, "Error" + ) + check(client.get(path + suffix), 401, "Error") + thumb = client.get( + path + "/content", + headers=headers(actors[1]), + params={"variant": "thumbnail"}, + ) + assert thumb.status_code == 200 + with Image.open(io.BytesIO(thumb.content)) as image: + assert image.format == "JPEG" and image.size == (256, 160) + sent = check( + client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers=headers(actors[1]), + json={"text": "", "attachment_ids": [attachment["id"]]}, + ), + 201, + "Message", + ) + assert sent["attachment_ids"] == [attachment["id"]] + assert sent["sender_id"] == actors[1]["id"] + + +@pytest.mark.parametrize( + "name", + [ + "../x.png", + "x/y.png", + "x\\y.png", + "x\x00.png", + "x\x7f.png", + "x\n.png", + "a" * 256, + "", + ], +) +def test_filename_guard(client, actors, chat, name): + check(upload(client, actors[0], chat, filename=name), 400, "Error") + + +@pytest.mark.parametrize( + "data,mime,status", + [ + (b"not an image", "image/png", 415), + (PNG, "image/jpeg", 415), + (PNG, "application/octet-stream", 415), + (PNG[:25], "image/png", 415), + (b"", "image/png", 415), + (b"x" * (10 * 1024 * 1024 + 1), "image/png", 413), + ], +) +def test_image_guards(client, actors, chat, data, mime, status): + r = upload(client, actors[0], chat, data=data, mime=mime) + check(r, status, "Error") + + +def test_jpeg_original_unchanged(client, actors, chat): + stream = io.BytesIO() + Image.new("RGB", (320, 200), (30, 60, 90)).save(stream, format="JPEG") + jpeg = stream.getvalue() + attachment = check( + upload( + client, + actors[0], + chat, + data=jpeg, + filename="camera.jpg", + mime="image/jpeg", + ), + 202, + "Attachment", + ) + attachment = wait_ready(client, actors[0], attachment) + r = client.get( + f"/api/v1/attachments/{attachment['id']}/content", + headers=headers(actors[1]), + ) + assert r.content == jpeg + + +def test_cross_chat_link_and_unknown(client, actors, chat): + a = check(upload(client, actors[0], chat), 202, "Attachment") + other = check( + client.post( + "/api/v1/chats", + headers=headers(actors[0]), + json={"title": "Other", "member_ids": [actors[0]["id"]]}, + ), + 201, + "Chat", + ) + for identity in [a["id"], str(uuid4())]: + check( + client.post( + f"/api/v1/chats/{other['id']}/messages", + headers=headers(actors[0]), + json={"text": "", "attachment_ids": [identity]}, + ), + 404, + "Error", + ) + check( + client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers=headers(actors[0]), + json={"text": "", "attachment_ids": [a["id"], a["id"]]}, + ), + 400, + "Error", + ) + + +def test_default_attachments_are_semantically_equal(client, actors, chat): + h = headers(actors[0]) + path = f"/api/v1/chats/{chat['id']}/messages" + first = check( + client.post(path, headers=h, json={"text": "Semantic default"}), + 201, + "Message", + ) + second = check( + client.post( + path, + headers=h, + json={"attachment_ids": [], "text": "Semantic default"}, + ), + 201, + "Message", + ) + assert first == second and first["attachment_ids"] == [] + + +def test_pixel_limit_before_decode(client, actors, chat): + # Compressible synthetic image stays below10MiB but exceeds40M pixels. + stream = io.BytesIO() + Image.new("1", (8000, 5001)).save(stream, format="PNG") + assert len(stream.getvalue()) < 10 * 1024 * 1024 + r = upload(client, actors[0], chat, data=stream.getvalue()) + assert r.status_code == 413 + + +def test_chunked_body_limit_without_content_length(client, actors, chat): + r = client.post( + f"/api/v1/chats/{chat['id']}/attachments", + headers={**headers(actors[0]), "Content-Type": "image/png"}, + params={"filename": "chunked.png"}, + content=iter([b"x" * (10 * 1024 * 1024 + 1)]), + ) + assert r.request.headers.get("content-length") is None + assert r.status_code == 413 + assert r.json()["error"]["code"] == "payload_too_large" + assert r.json()["error"]["request_id"] == r.headers["x-request-id"] diff --git a/student-tests/test_contract_edges.py b/student-tests/test_contract_edges.py new file mode 100644 index 00000000..81d39edf --- /dev/null +++ b/student-tests/test_contract_edges.py @@ -0,0 +1,26 @@ +import json +from uuid import uuid4 + +import pytest +from conftest import check, headers + + +@pytest.mark.parametrize("encoding", ["utf-16", "utf-32"]) +def test_only_utf8_json(client, encoding): + payload = {"username": "u_" + uuid4().hex[:20], "display_name": "UTF"} + payload["password"] = "Synthetic-encoding-test-123" + result = client.post( + "/api/v1/users", + content=json.dumps(payload).encode(encoding), + headers={"Content-Type": "application/json"}, + ) + check(result, 400, "Error") + + +def test_leading_zero_limit(client, actors, chat): + result = client.get( + "/api/v1/chats/" + chat["id"] + "/messages", + headers=headers(actors[0]), + params={"limit": "0001"}, + ) + check(result, 200, "MessagePage") diff --git a/student-tests/test_history_acl.py b/student-tests/test_history_acl.py new file mode 100644 index 00000000..61bd461c --- /dev/null +++ b/student-tests/test_history_acl.py @@ -0,0 +1,23 @@ +from conftest import check, headers + + +def test_history_acl_precedes_query_validation(client, actors, chat): + path = f"/api/v1/chats/{chat['id']}/messages" + check( + client.get( + path, + headers=headers(actors[2]), + params={"limit": 0, "cursor": "broken"}, + ), + 404, + "Error", + ) + check( + client.get( + path, + headers=headers(actors[0]), + params={"limit": 0, "cursor": "broken"}, + ), + 400, + "Error", + ) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..bbf61fc4 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,75 @@ +from concurrent.futures import ThreadPoolExecutor +from uuid import uuid4 + +import httpx +import pytest +from conftest import BASE_URL, check, headers, history + + +@pytest.mark.parametrize("key", [None, "", "a b", "x" * 129, "a/b"]) +def test_key_required_and_valid(client, actors, chat, key): + h = headers(actors[0]) + h.pop("Idempotency-Key") + if key is not None: + h["Idempotency-Key"] = key + check( + client.post( + f"/api/v1/chats/{chat['id']}/messages", + headers=h, + json={"text": "Key"}, + ), + 400, + "Error", + ) + + +def test_parallel_replay_across_instances(client, actors, chat): + h = headers(actors[0]) + path = f"/api/v1/chats/{chat['id']}/messages" + + def send(_): + with httpx.Client(base_url=BASE_URL, trust_env=False, timeout=15) as c: + r = c.post(path, headers=h, json={"text": "Один результат 🌍"}) + return check(r, 201, "Message"), r.headers["x-instance-id"] + + with ThreadPoolExecutor(max_workers=10) as pool: + results = list(pool.map(send, range(30))) + assert len({r[0]["id"] for r in results}) == 1 + assert len({r[1] for r in results}) >= 2 + assert history(client, actors[0], chat["id"]) == [results[0][0]] + check( + client.post(path, headers=h, json={"text": "Другой текст"}), + 409, + "Error", + ) + # Другой пользователь может независимо использовать тот же ключ. + other = headers(actors[1]) + other["Idempotency-Key"] = h["Idempotency-Key"] + second = check( + client.post(path, headers=other, json={"text": "Боб"}), 201, "Message" + ) + assert second["id"] != results[0][0]["id"] + + +def test_replay_rechecks_acl(client, actors, chat): + h = headers(actors[0]) + path = f"/api/v1/chats/{chat['id']}/messages" + check( + client.post(path, headers=h, json={"text": "Existing"}), 201, "Message" + ) + outsider = headers(actors[2]) + outsider["Idempotency-Key"] = h["Idempotency-Key"] + check( + client.post(path, headers=outsider, json={"text": "Existing"}), + 404, + "Error", + ) + check( + client.post( + path, + headers={"Idempotency-Key": uuid4().hex}, + json={"text": "Existing"}, + ), + 401, + "Error", + ) diff --git a/student-tests/test_sessions.py b/student-tests/test_sessions.py new file mode 100644 index 00000000..78a5ce4d --- /dev/null +++ b/student-tests/test_sessions.py @@ -0,0 +1,114 @@ +from uuid import uuid4 + +import httpx +import pytest +from conftest import PASSWORD, TOKENS, check, headers, register + + +@pytest.mark.parametrize( + "password", ["", "x" * 11, "x" * 129, " " * 12, 42, True, None] +) +def test_password_validation(client, password): + check( + client.post( + "/api/v1/users", + json={ + "username": "pwd_" + uuid4().hex[:20], + "display_name": "Test", + "password": password, + }, + ), + 400, + "Error", + ) + + +def test_full_utf8_password(client): + password = "🌍" * 127 + ":" + user = register(client, password=password) + assert ( + check( + client.get("/api/v1/users/me", headers=headers(user)), 200, "User" + ) + == user + ) + + +@pytest.mark.parametrize( + "authorization", [None, "Basic broken", "Bearer unknown", "Basic /w=="] +) +def test_bad_basic(client, authorization): + result = client.post( + "/api/v1/auth/sessions", + headers={"Authorization": authorization} if authorization else {}, + ) + check(result, 401, "Error") + assert ( + result.headers["www-authenticate"] + == 'Basic realm="messenger", charset="UTF-8"' + ) + + +def test_wrong_password_and_unknown_user(client): + user = register(client) + for username in [user["username"], "missing_" + uuid4().hex[:20]]: + result = client.post( + "/api/v1/auth/sessions", + auth=httpx.BasicAuth(username, PASSWORD + "wrong"), + ) + data = check(result, 401, "Error") + assert data["error"]["message"] == "Authentication required" + + +def test_logout_is_not_revived_by_cache(client): + user = register(client) + auth = headers(user) + check(client.get("/api/v1/users/me", headers=auth), 200, "User") + result = client.delete("/api/v1/auth/sessions/current", headers=auth) + assert result.status_code == 204 and result.content == b"" + assert result.headers["x-request-id"] + check(client.get("/api/v1/users/me", headers=auth), 401, "Error") + check( + client.delete("/api/v1/auth/sessions/current", headers=auth), + 401, + "Error", + ) + + +def test_only_current_session_revoked(client): + user = register(client) + another = check( + client.post( + "/api/v1/auth/sessions", + auth=httpx.BasicAuth(user["username"], PASSWORD), + ), + 201, + "Session", + ) + assert another["token"] != TOKENS[user["id"]] + assert len(another["token"]) >= 32 + assert ( + client.delete( + "/api/v1/auth/sessions/current", headers=headers(user) + ).status_code + == 204 + ) + result = client.get( + "/api/v1/users/me", + headers={"Authorization": "Bearer " + another["token"]}, + ) + assert check(result, 200, "User") == user + + +def test_bearer_source_and_ignored_user_header(client, actors): + alice, bob, _ = actors + auth = {**headers(alice), "X-User-Id": bob["id"]} + assert ( + check(client.get("/api/v1/users/me", headers=auth), 200, "User")["id"] + == alice["id"] + ) + result = client.get( + "/api/v1/users/me", auth=httpx.BasicAuth(alice["username"], PASSWORD) + ) + check(result, 401, "Error") + assert result.headers["www-authenticate"] == 'Bearer realm="messenger"' -- 2.54.0 From bb4e8a225476474e0b6b909db0eadf25f3956d33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Sun, 15 Nov 2026 22:33:09 +0300 Subject: [PATCH 03/11] test(pipeline): cover variant metadata and correct lab endpoint --- evidence/http-initial-environment.json | 1 + src/pipeline.py | 13 ++++-- src/reconcile.py | 13 ++++++ student-tests/conftest.py | 2 +- student-tests/test_attachments.py | 65 ++++++++++++++++++++++++++ 5 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 evidence/http-initial-environment.json diff --git a/evidence/http-initial-environment.json b/evidence/http-initial-environment.json new file mode 100644 index 00000000..26160da2 --- /dev/null +++ b/evidence/http-initial-environment.json @@ -0,0 +1 @@ +{"result":"INVALID_ENVIRONMENT","cause":"Inherited student fixture default pointed to stopped lab4 port8084, not lab5 port8085; all connections refused; no API correctness conclusion.","failed":40,"errors":72,"seconds":8.77} diff --git a/src/pipeline.py b/src/pipeline.py index 2b152a75..5439384d 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -3,6 +3,7 @@ import json import os import random +import re import time from datetime import UTC, datetime from uuid import UUID @@ -68,14 +69,20 @@ def validate_job(data): ): raise ValueError("Invalid job type") for k in ["job_id", "attachment_id"]: + if not isinstance(data[k], str): + raise ValueError("Invalid job UUID") UUID(data[k]) for k in ["pipeline_version", "attempt"]: if type(data[k]) is not int or data[k] < 1: raise ValueError("Invalid job number") - datetime.fromisoformat(data["created_at"].replace("Z", "+00:00")) + timestamp = data["created_at"] + if not isinstance(timestamp, str) or not re.fullmatch( + r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})", + timestamp, + ): + raise ValueError("Invalid job timestamp") + datetime.fromisoformat(timestamp.replace("Z", "+00:00")) if "traceparent" in data: - import re - if not isinstance(data["traceparent"], str) or not re.fullmatch( r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", data["traceparent"] ): diff --git a/src/reconcile.py b/src/reconcile.py index 5a4e9c15..e48d77b0 100644 --- a/src/reconcile.py +++ b/src/reconcile.py @@ -69,6 +69,19 @@ def reconcile(db, store, *, apply=False, grace=None): (reservation["id"],), ) store.delete(key=obj.object_name) + for job in jobs: + if ( + job["state"] == "failed" + and job["source_key"] == obj.object_name + ): + conn.execute( + "UPDATE jobs SET error_code='source_expired' WHERE id=%s", + (job["id"],), + ) + conn.execute( + "UPDATE attachments SET error_code='source_expired' WHERE id=%s AND pipeline_version=%s AND state='failed'", + (job["attachment_id"], job["pipeline_version"]), + ) actions.append(action) return {"apply": apply, "retention_seconds": grace, "actions": actions} diff --git a/student-tests/conftest.py b/student-tests/conftest.py index 7c1f2dfb..c672d63a 100644 --- a/student-tests/conftest.py +++ b/student-tests/conftest.py @@ -10,7 +10,7 @@ from jsonschema import Draft202012Validator, FormatChecker CONTRACT = json.loads( (Path(__file__).parents[1] / "contracts/openapi.json").read_text() ) -BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8084") +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8085") PASSWORD = "Synthetic-Password-🌍-123" TOKENS = {} diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py index b1ce48c5..6aacd2e9 100644 --- a/student-tests/test_attachments.py +++ b/student-tests/test_attachments.py @@ -210,3 +210,68 @@ def test_chunked_body_limit_without_content_length(client, actors, chat): assert r.status_code == 413 assert r.json()["error"]["code"] == "payload_too_large" assert r.json()["error"]["request_id"] == r.headers["x-request-id"] + + +@pytest.mark.parametrize("size", [(1, 17), (320, 200), (1600, 900)]) +def test_variant_geometry_and_alpha(client, actors, chat, size): + stream = io.BytesIO() + Image.new("RGBA", size, (0, 0, 0, 0)).save(stream, format="PNG") + a = check( + upload(client, actors[0], chat, data=stream.getvalue()), + 202, + "Attachment", + ) + a = wait_ready(client, actors[0], a) + for v in a["variants"]: + r = client.get( + f"/api/v1/attachments/{a['id']}/content", + headers=headers(actors[1]), + params={"variant": v["name"]}, + ) + assert len(r.content) == v["size_bytes"] + assert hashlib.sha256(r.content).hexdigest() == v["sha256"] + with Image.open(io.BytesIO(r.content)) as image: + assert image.size == (v["width"], v["height"]) + assert not image.getexif() + assert min(image.convert("RGB").getpixel((0, 0))) >= 250 + if v["name"] == "crop": + assert image.size == (128, 128) + else: + bound = 256 if v["name"] == "thumbnail" else 1280 + assert max(image.size) <= bound + assert image.width <= size[0] and image.height <= size[1] + + +def test_exif_orientation_applied_but_original_preserved(client, actors, chat): + stream = io.BytesIO() + exif = Image.Exif() + exif[274] = 6 + exif[270] = "Synthetic metadata must be removed" + Image.new("RGB", (320, 200), "red").save(stream, format="JPEG", exif=exif) + raw = stream.getvalue() + a = check( + upload( + client, + actors[0], + chat, + data=raw, + mime="image/jpeg", + filename="exif.jpg", + ), + 202, + "Attachment", + ) + a = wait_ready(client, actors[0], a) + r = client.get( + f"/api/v1/attachments/{a['id']}/content", headers=headers(actors[1]) + ) + assert r.content == raw + v = next(v for v in a["variants"] if v["name"] == "thumbnail") + assert (v["width"], v["height"]) == (160, 256) + r = client.get( + f"/api/v1/attachments/{a['id']}/content", + headers=headers(actors[1]), + params={"variant": "thumbnail"}, + ) + with Image.open(io.BytesIO(r.content)) as image: + assert image.size == (160, 256) and not image.getexif() -- 2.54.0 From a275cf2632d3bf3d96409408d9cb313cdb2824c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Sun, 15 Nov 2026 22:45:10 +0300 Subject: [PATCH 04/11] test(pipeline): verify durable cleanup and retain successful HTTP evidence --- .dockerignore | 2 + Makefile | 10 +- REPORT.md | 243 +++++++++++++++++++++++++++--- compose.test.yaml | 17 +++ evidence/http-first.json | 12 ++ pyproject.toml | 2 +- scripts/verify_clean.py | 57 +++++-- scripts/verify_cleanup.py | 223 +++++++++++++++++++++++++++ scripts/verify_pipeline.py | 82 +++++++++- student-unit/test_job_envelope.py | 68 +++++++++ 10 files changed, 678 insertions(+), 38 deletions(-) create mode 100644 evidence/http-first.json create mode 100644 scripts/verify_cleanup.py create mode 100644 student-unit/test_job_envelope.py diff --git a/.dockerignore b/.dockerignore index 8e5d3661..93056db6 100644 --- a/.dockerignore +++ b/.dockerignore @@ -33,3 +33,5 @@ Makefile compose.yaml .work* + +student-unit diff --git a/Makefile b/Makefile index 788ce574..6e04c7dd 100644 --- a/Makefile +++ b/Makefile @@ -39,9 +39,15 @@ up: bootstrap down: docker compose down lint: - $(UV) run --locked ruff format --check src student-tests scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py) - $(UV) run --locked ruff check src student-tests scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py) + $(UV) run --locked ruff format --check src student-tests student-unit scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py) + $(UV) run --locked ruff check src student-tests student-unit scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py) test-student: BASE_URL="$(BASE_URL)" $(UV) run --locked pytest student-tests verify: $(UV) run --locked python -m scripts.verify_pipeline --output evidence/pipeline-repeat.json + +.PHONY: unit reconcile +unit: + PYTHONPATH=. $(UV) run --locked pytest student-unit -q +reconcile: + docker compose run --rm --no-deps reconciler python -m src.reconcile diff --git a/REPORT.md b/REPORT.md index e98f75e9..0d440c08 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,234 @@ -# Отчёт — лабораторная 5 +# Лабораторная 5 — Маслов, БВТ2301 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +Цель: **5, трек 2 — идемпотентность versioned pipeline**. Собственная ЛР4 +перенесена в независимую историю текущего teacher skeleton; обязательные +contracts/tests/check/validate сохранены. API принимает исходник, а отдельный +worker публикует original/thumbnail/crop/optimized. Исследование проверяет +дубли, reorder и реальные SIGKILL, а не моделирует их ответами mock. -## Запуск из чистого clone +Результаты полного fault-injection и clean clone будут внесены после запуска. +Промежуточный public smoke уже PASS; итоговые evidence ниже обновляются +только фактическими командами. Первый запуск HTTP suite ошибочно обращался +к порту предыдущей лабы; `http-initial-environment.json` честно отмечает +INVALID_ENVIRONMENT, это не проверка корректности API. -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +## Воспроизведение -## Архитектура и решения +Нужны Git, Docker Desktop с Compose v2, uv0.13.7. Версии Python3.14.7, +PostgreSQL18.3, Valkey8.1.3, HAProxy3.2.1, Pillow13.0.0 закреплены; +MinIO/mc собираются из точных upstream commits в infra/s3.Dockerfile, +Go module checksum проверяется GoSumDB. Сборка требует доступа к registries, +Go module proxy и PyPI; наличие сети нельзя заменить уже установленной БД. -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```sh +uv sync --locked +make bootstrap +VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait +make check validate PYTHON=.venv/bin/python +make lint unit +NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1 make test PYTHON=.venv/bin/python +BASE_URL=http://127.0.0.1:8085 make test-student +``` -## Доказательства критериев +URL: http://127.0.0.1:8085. API не имеет persistent volume/host port. +Bootstrap создаёт только отсутствующие secrets и .env; не печатает значения, +не перезаписывает существующие. `secrets/`0700, файлы0644 нужны UID10001 +через readonly Docker secrets mounts; сам каталог недоступен другим users. +`docker compose down` сохраняет volumes. Удаление volumes — отдельное +осознанное действие над своим синтетическим стендом. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +Для автоматических отказов включается только явный test override: + +```sh +export COMPOSE_FILE=compose.yaml:compose.test.yaml +export NO_PROXY=localhost,127.0.0.1 no_proxy=localhost,127.0.0.1 +VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait +uv run --locked python -m scripts.verify_cleanup seed --seed evidence/gc-seed-repeat.json +uv run --locked python -m scripts.verify_pipeline --output evidence/pipeline-repeat.json --repeats 2 +uv run --locked python -m scripts.verify_cleanup verify --seed evidence/gc-seed-repeat.json --output evidence/gc-repeat.json +uv run --locked python -m scripts.verify_cluster --output evidence/cluster-repeat.json +``` + +Проверки, меняющие состояние контейнеров, выполняются последовательно. +Seed и результат — разные файлы; нельзя использовать seed с уже удалённым +orphan. Verify ожидает настоящий возраст S3 не менее302s; часы/LastModified +не подменяются. Для нового полного чистого повторения: + +```sh +# Перед стартом весь проверяемый source должен быть закоммичен и git clean. +uv run --locked python scripts/verify_clean.py --output evidence/clean-clone-repeat.json +``` + +Этот verifier создаёт отдельные secrets/env/volumes/image/project, собирает, +запускает static/unit/public smoke/HTTP, cluster, всю fault matrix и cleanup, +сверяет source hashes и OCI revision обоих API, worker, relay, reconciler и +тестового controller; после удаляет только временный проект и его volumes. + +## Границы устойчивости + +1. API проверяет фактические bytes≤10MiB, MIME PNG/JPEG, декодирование, + filename и ≤40M pixels до полной распаковки. Временный spool≤1MiB в RAM, + затем bounded tmpfs; upload slots2/API, decode slot1/API, срок запроса20s. +2. Устойчивый upload reservation120s COMMIT предшествует PUT в приватный + `objects/staging//original`. После PUT одна PG transaction проверяет + reservation с FOR UPDATE, создаёт Attachment queued + logical Job + Outbox, + удаляет reservation и COMMIT. Только затем202 с захваченным queued-снимком. + Быстрый worker не превращает upload-ответ в ready. Сам upload не имеет + Idempotency-Key: потерянный HTTP-ответ нельзя безопасно повторять вслепую. +3. API зависит от PG и S3 для ready, cache используется как необязательный + read-through user cache. Broker не критичен для синхронного приёма: + работа остаётся в outbox. Relay имеет PG+broker secrets, не S3/root keys. +4. Relay читает due outbox FOR UPDATE SKIP LOCKED, XADD и WAITAOF1/0/2000 + **на одной выделенной connection**. Подтверждён fsync локального AOF; + только затем published_at/COMMIT в PG. Crash в этом окне даёт дубль. +5. Отдельный Valkey Streams broker с persistent volume, AOF appendfsync always, + noeviction. Disposable cache — другой сервис/volume/policy. Нет Pub/Sub. +6. Worker consumer group, COUNT=1 на каждый из2 слотов; XAUTOCLAIM возвращает + зависшие PEL entries. PG атомарно переводит queued→processing, увеличивает + attempt/fence, сохраняет owner/lease. Duplicate active оставляется pending; + ready duplicate делает HEAD всех4вариантов/size/SHA metadata и ACK без resize. +7. Child получает ID/fence, trusted source берётся из PG, не из произвольного + URL или filename. До обработки проверяется исходный size/SHA. Все4 PUT идут + в `objects//v/`. Original побайтно неизменён. +8. В одной PG transaction worker проверяет state/version/fence/live lease, + публикует metadata всех вариантов, unique effect_publication и ready. + Старый child может дописать свой version-prefix, но не подтвердить новый + результат. Стабильный алгоритм нельзя менять под тем же pipeline_version. +9. ACK только после устойчивой ready, failed/obsolete или решения queued retry. + Watchdog relay восстанавливает expired leases; pending outbox сохраняет retry. + Потерянные wakeups queued jobs переиздаются через15s без создания logical job. + +## Состояния и варианты + +GET показывает queued/processing/ready/failed. Queued/processing можно связать +с Message; failed —409. Все metadata/content проходят Bearer+membership, +чужой/неизвестный UUID —404, content до ready —409. У ready все3 производных +варианта и error_code=null; size/content_type/SHA описывают original. +Прямого redirect/public S3 URL нет. Atomic Message+Idempotency-Key сохранены. + +EXIF orientation применяется; RGBA компонуется на белом. Thumbnail JPEG +вписан≤256×256 без увеличения, floor/min1;320×200→256×160. Crop — центральный +JPEG128×128 с допустимым увеличением маленького исходника. JPEGquality85, +subsampling0,LANCZOS. Optimized — WebP≤1280×1280 без увеличения, quality82(v1) +или76(v2), method4. EXIF/info удаляются из производных; metadata original +сохраняются в исходных bytes. Маленький WebP не обязательно меньше original. + +Job соответствует contracts/jobs.schema.json: schema_version1, +type attachment.process.v1, UUID job/attachment, version, attempt>=1, RFC3339 +created_at; возможен W3C carrier. Bytes/credentials/paths/URL запрещены. +Broker redelivery не меняет job_id; finite replay сохраняет job_id и увеличивает +cycle; новая pipeline version получает новый job_id. Idempotency effect key — +(attachment_id,pipeline_version,variant), PK в PG и детерминированный S3 key. + +## Ограничение ресурсов и повторы + +| Роль | CPU | RAM | Ограничение работы | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| API каждая из2 |0.5|768MiB|2uploads,1decode,128MiB tmpfs| +| Worker |2|1536MiB|2child,COUNT1/slot,child RLIMIT_AS768MiB| +| Relay |0.25|192MiB|один bounded publish/transaction| +| Broker |0.5|256MiB|192MiB maxmemory,noeviction,AOF volume| +| PostgreSQL |1|512MiB|pool10/API,4/worker-slot| +| S3 |1|1024MiB|persistent volume,private bucket| +| Reconciler |0.25|256MiB|inventory проход/60s| -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Normal timeout20s / lease23s; test override12s/15s даёт время контроллеру +завершить v2 в другой slot пока v1 удерживается после PUT. Child действительно +killable: parent убивает процесс по timeout, а не лишь отменяет future/thread. +Весь container ограничен cgroup; лимит40M pixels не обещает, что всякий такой +файл уложится в память и срок — memory_limit/timeout приводят к safe failed/retry. -## Эксперименты и отказы +Максимум5attempts; временные ошибки/смерть worker/timeout → persisted next_at +с jitter uniform0.25…min(30,2^attempt)s. Постоянные decode/checksum/memory ошибки +сразу failed; временные после5→retry_exhausted. Публичные error_code безопасны, +без stack/SQL/key. Реестр failed и audited replay доступны только оператору: -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +```sh +docker compose exec worker python -m src.operator failed +docker compose exec worker python -m src.operator replay --job-id UUID --operator maslov --reason 'Synthetic recovery demonstration' +docker compose exec worker python -m src.operator version --job-id UUID --version 2 --operator maslov --reason 'Upgrade pipeline' +docker compose run --rm --no-deps reconciler python -m src.reconcile +docker compose run --rm --no-deps reconciler python -m src.reconcile --apply +``` -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Ready replay отвергается. Failed replay проверяет наличие source, оставляет +audit и новый finite cycle. После source_expired нельзя обещать обработку +отсутствующих bytes. New-version поддерживает реализованный переход1→2. -## Ограничения и следующие шаги +## Retention -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Staging удаляется после безопасного COMMIT ready; сбой удаления не отменяет +результат, последующий GC уберёт остаток. Failed/unconfirmed source и orphan +живут минимум retention86400s, конфигурационный hard minimum300s. Cleanup +по умолчанию dry-run; --apply явный. Active queued/processing source и prefix, +действующий reservation и current published variants защищены. Перед delete +повторяются PG lock/recheck и HEAD реального возраста. Expired reservation +удаляется под lock; API не сможет поздно подтвердить удалённый source. +Старые версии после успешной новой могут удаляться; metadata/audit сохраняются. +Failed без source остаётся failed/source_expired и replay отвергается. +Текущий original не удаляется лишь из-за отсутствия Message links. + +## Исследование: гипотеза и инварианты + +Гипотеза: durable intent/outbox + PG version/fence/lease CAS + immutable +version keys сохраняют подтверждённую работу и не допускают publication старой +версии поверх новой при at-least-once delivery. + +Проверяемые инварианты: каждый202 имеет durable Job; все принятые matrix jobs +доходят до ready или наблюдаемого failed; финальный ready имеет ровно4 variant +rows/keys и1effect_publication на версию; original SHA равен загрузке; +old version не меняет hashes текущей; конечный retry/replay наблюдаемы; +cleanup не удаляет current/active/fresh. До handoff crash не получает202. + +`scripts/verify_pipeline.py --repeats2` использует отдельный test controller +с PG/S3/broker secrets (API не получает broker secrets), single-use hooks в PG +с durable hit_at, реальные container SIGKILL и повторный запуск. Hooks по +умолчанию выключены, публичного endpoint для них нет. Проверяются: +source-before-handoff, COMMIT-before-publish, confirmed broker SIGKILL/volume, +relay XADD-before-PG-marker, worker before_effect/after_effect/after_commit +каждый дважды; transient, timeout, permanent/replay,5attempt exhaustion/replay, +duplicates/reorder, held stalev1 послеv2 и старые notifications; malformed +notification, private IAM/anonymous denial и другая API после uploader stop. + +Raw evidence сохраняет IDs, attempts, fences, events,4keys/checksums и +publications. Worker child_finished даёт elapsed/CPU/peakRSS; это длительность +child, не полный queue lag. Исследование про корректность, не максимальный RPS. + +## Вопросы защиты + +**Publisher confirm и consumer ACK?** Первый подтверждает durable приём брокером +(здесь локальный AOF fsync через WAITAOF), второй — устойчивое завершение или +сохранённое решение о повторе у consumer. Между PG/Valkey/S3 нет общей transaction. + +**Что уже сохранено к202?** Полный проверенный original в private persistent +S3 staging плюс attachment/job/outbox в одной подтверждённой PG transaction. +Смерть API/worker не удаляет намерение. Перед202 тяжёлого resize нет. + +**Почему дубли нормальны?** Producer/consumer могут потерять ответ после +успешного действия. Отправитель повторяет, поскольку не знает исход. Logical +job_id и effect tuple/version/fence позволяют повторить доставку безопасно. +Ровно одна publication в PG не означает exactly-once delivery в broker. + +**Как отличить зависший worker от долгого?** Совершенно точно нельзя: lease — +выбранный бюджет. Parent timeout убивает child; watchdog восстанавливает работу +после lease, новый fence делает старое завершение недействительным. Ограничение +slots/COUNT сдерживает конкуренцию. Возможен лишний PUT, но не старый COMMIT. + +## Источники, помощь, ограничения + +Использованы текущие teacher contracts, собственная ЛР4, pinned dependencies, +[Valkey WAITAOF](https://valkey.io/commands/waitaof/), +[Valkey XAUTOCLAIM](https://valkey.io/commands/xautoclaim/), +[Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html). +При проектировании, реализации, тестах и отчёте использована помощь ИИ. +Код/данные/история других студентов не использовались; результаты — реальные +локальные запуски, а remote CI отдельно не считается прошедшим. + +Один host/LB/PG/S3/broker — SPOF, AOF/fsync/volume не заменяет backup и защиту +от потери диска. HTTP допустим для локальных синтетических данных; удалённый +доступ требует TLS. Строгого production exactly-once нет. HEAD metadata не +заменяет полный periodic checksum audit. Удалённые bytes не восстановить по SHA. +Весь inventory GC хранится в RAM, outbox/events/audit требуют retention при +долгой эксплуатации. Нет global admission quota: при долгой недоступности +worker/broker PG outbox может расти; исследование перегрузки — другой трек. +Short synthetic fixtures и повторы ограничивают обобщение на большие файлы. diff --git a/compose.test.yaml b/compose.test.yaml index 094c4c70..850a9ff8 100644 --- a/compose.test.yaml +++ b/compose.test.yaml @@ -11,3 +11,20 @@ services: PIPELINE_FAULTS: "1" JOB_TIMEOUT_SECONDS: "12" JOB_LEASE_SECONDS: "15" + controller: + image: ${API_IMAGE:-mtusi-messenger:lab5} + pull_policy: never + command: [python, -c, "import time; time.sleep(86400)"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + secrets: [database_url, broker_url, s3_access_key, s3_secret_key] + cpus: 0.5 + mem_limit: 192m + memswap_limit: 192m + read_only: true + healthcheck: {disable: true} + cap_drop: [ALL] + security_opt: [no-new-privileges:true] diff --git a/evidence/http-first.json b/evidence/http-first.json new file mode 100644 index 00000000..f89baa75 --- /dev/null +++ b/evidence/http-first.json @@ -0,0 +1,12 @@ +{ + "source_commit": "bb4e8a2", + "runtime_revision": "bb4e8a2", + "public_smoke": "PASS", + "http_tests": 112, + "passed": 112, + "failed": 0, + "seconds": 404.42, + "unit_tests": 19, + "unit_result": "PASS", + "output": "........................................................................ [ 64%]\n........................................ [100%]\n112 passed in 404.42s (0:06:44)\n" +} diff --git a/pyproject.toml b/pyproject.toml index cd0d6b96..ea928755 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ dev = ["pytest>=8,<10", "httpx>=0.28,<1", "ruff>=0.14,<1", "openapi-spec-validat [tool.ruff] line-length = 79 target-version = "py314" -include = ["src/**/*.py", "student-tests/**/*.py", "research/**/*.py", "scripts/verify_*.py"] +include = ["src/**/*.py", "student-tests/**/*.py", "student-unit/**/*.py", "research/**/*.py", "scripts/**/*.py"] [tool.ruff.lint] select = ["E", "F", "I", "B", "UP"] diff --git a/scripts/verify_clean.py b/scripts/verify_clean.py index 6a56626e..deb961b2 100644 --- a/scripts/verify_clean.py +++ b/scripts/verify_clean.py @@ -80,6 +80,7 @@ def main(): "API_IMAGE": "mtusi-messenger:lab5-clean-" + identity, "PUBLIC_PORT": public_port, "VCS_REF": sha, + "COMPOSE_FILE": "compose.yaml:compose.test.yaml", "NO_PROXY": "localhost,127.0.0.1", "no_proxy": "localhost,127.0.0.1", } @@ -114,6 +115,7 @@ def main(): cwd=clone, env=env, ) + receipt["unit_tests"] = run(["make", "unit"], cwd=clone, env=env) run(["make", "bootstrap"], cwd=clone, env=env) try: run( @@ -141,19 +143,39 @@ def main(): cwd=clone, env=env, ) - run( + for command_args in [ [ ".venv/bin/python", "-m", - "scripts.verify_storage", - "--output", - "evidence/clean-storage.json", + "scripts.verify_cleanup", + "seed", + "--seed", + "evidence/clean-gc-seed.json", ], - cwd=clone, - env=env, + [ + ".venv/bin/python", + "-m", + "scripts.verify_pipeline", + "--output", + "evidence/clean-pipeline.json", + ], + [ + ".venv/bin/python", + "-m", + "scripts.verify_cleanup", + "verify", + "--seed", + "evidence/clean-gc-seed.json", + "--output", + "evidence/clean-cleanup.json", + ], + ]: + run(command_args, cwd=clone, env=env) + receipt["pipeline"] = json.loads( + (clone / "evidence/clean-pipeline.json").read_text() ) - receipt["storage"] = json.loads( - (clone / "evidence/clean-storage.json").read_text() + receipt["cleanup"] = json.loads( + (clone / "evidence/clean-cleanup.json").read_text() ) receipt["cluster"] = json.loads( (clone / "evidence/clean-cluster.json").read_text() @@ -172,6 +194,21 @@ def main(): "import hashlib,json,pathlib; print(json.dumps({p.name:" "hashlib.sha256(p.read_bytes()).hexdigest() for p in pathlib.Path('/app/src').glob('*.py')}))" ) + ids += run( + [ + "docker", + "compose", + "ps", + "-q", + "worker", + "relay", + "reconciler", + "controller", + ], + cwd=clone, + env=env, + ).split() + assert len(ids) == 6 for cid in ids: actual = json.loads( run( @@ -191,7 +228,9 @@ def main(): == sha ) assert not info["HostConfig"]["PortBindings"] - receipt["both_replicas_runtime_hashes_and_revision_match"] = True + receipt[ + "api_worker_relay_reconciler_runtime_hashes_and_revision_match" + ] = True receipt["fsck"] = run( ["git", "fsck", "--full"], cwd=clone, env=env ) diff --git a/scripts/verify_cleanup.py b/scripts/verify_cleanup.py new file mode 100644 index 00000000..e4e622c0 --- /dev/null +++ b/scripts/verify_cleanup.py @@ -0,0 +1,223 @@ +"""Real S3 retention test: seed is separate from verification output.""" + +import argparse +import json +import time +from datetime import datetime, timezone +from pathlib import Path +from uuid import uuid4 + +import httpx + +from scripts.stand import BASE, ROOT, actor, auth, command, compose, ready +from scripts.verify_pipeline import inside, operator, wait + + +def seed(path): + if path.exists(): + raise ValueError("Choose a new seed file") + ready() + compose("stop", "--timeout", "1", "reconciler") + raw = (ROOT / "tests/fixtures/sample.png").read_bytes() + with httpx.Client(base_url=BASE, trust_env=False, timeout=20) as c: + user, _, session = actor(c) + headers = auth(session) + result = c.post( + "/api/v1/chats", + headers=headers, + json={ + "title": "Cleanup synthetic seed", + "member_ids": [user["id"]], + }, + ) + assert result.status_code == 201 + chat = result.json()["id"] + r = c.post( + f"/api/v1/chats/{chat}/attachments", + headers={**headers, "Content-Type": "image/png"}, + params={"filename": "retained.png"}, + content=raw, + ) + assert r.status_code == 202 + attachment = r.json()["id"] + wait( + lambda: ( + c.get( + "/api/v1/attachments/" + attachment, headers=headers + ).json()["status"] + == "ready" + ) + ) + job = inside( + f"with db.connection() as c: r=c.execute('SELECT id FROM jobs WHERE attachment_id=%s',({attachment!r},)).fetchone()\nprint(json.dumps(str(r['id'])))" + ) + data = { + "seed_id": uuid4().hex, + "source_commit": command(["git", "rev-parse", "HEAD"]), + "date": datetime.now(timezone.utc).isoformat(), + "ready_attachment": attachment, + "ready_job": job, + "chat": chat, + "owner": user["id"], + } + # Real old v1 objects will become obsolete after v2 is ready. + second = operator(job, version=2) + wait( + lambda: inside( + f"with db.connection() as c: r=c.execute('SELECT state FROM jobs WHERE id=%s',({second!r},)).fetchone()\nprint(json.dumps(r['state']=='ready'))" + ) + ) + data["version2_job"] = second + data["orphan"] = f"objects/cleanup-{data['seed_id']}/old-orphan" + data["failed_id"] = str(uuid4()) + data["active_id"] = str(uuid4()) + data["reservation_id"] = str(uuid4()) + data["expired_reservation_id"] = str(uuid4()) + data["failed_source"] = f"objects/staging/{data['failed_id']}/original" + data["active_source"] = f"objects/staging/{data['active_id']}/original" + data["reservation_key"] = ( + f"objects/staging/{data['reservation_id']}/original" + ) + data["expired_reservation_key"] = ( + f"objects/staging/{data['expired_reservation_id']}/original" + ) + inside(f"""from pathlib import Path +raw=Path('/app/src/__init__.py').read_bytes() or b'fixture' +keys={json.dumps([data[k] for k in ["orphan", "failed_source", "active_source", "reservation_key", "expired_reservation_key"]])} +for key in keys: store.put(key=key,stream=io.BytesIO(raw),size=len(raw),content_type='image/png',sha256=hashlib.sha256(raw).hexdigest()) +with db.connection() as c, c.transaction(): + for identity,state in [({data["failed_id"]!r},'failed'),({data["active_id"]!r},'processing')]: + key='objects/staging/'+identity+'/original' + c.execute(\"INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,error_code,uploader_instance,expires_at) VALUES(%s,%s,%s,'synthetic.png','image/png',%s,%s,%s,%s,%s,'cleanup-fixture',clock_timestamp())\",(identity,{chat!r},{user["id"]!r},len(raw),hashlib.sha256(raw).hexdigest(),key,state,'invalid_image' if state=='failed' else None)) + c.execute(\"INSERT INTO jobs(id,attachment_id,pipeline_version,source_key,state,attempt,lease_until,updated_at,error_code) VALUES(%s,%s,1,%s,%s,1,clock_timestamp()+interval '1 hour',clock_timestamp()-interval '2 days',%s)\",(uuid4(),identity,key,state,'invalid_image' if state=='failed' else None)) + c.execute(\"INSERT INTO upload_reservations(id,object_key,expires_at) VALUES(%s,%s,clock_timestamp()+interval '1 hour'),(%s,%s,clock_timestamp()-interval '1 second')\",({data["reservation_id"]!r},{data["reservation_key"]!r},{data["expired_reservation_id"]!r},{data["expired_reservation_key"]!r})) +print('true')""") + data["seeded_unix"] = time.time() + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n") + print( + "PASS: retention seed created; real S3 age must reach300s", flush=True + ) + + +def verify(seed_path, output): + if output.exists(): + raise ValueError("Choose a new output path") + data = json.loads(seed_path.read_text()) + report = { + "source_commit": command(["git", "rev-parse", "HEAD"]), + "seed": seed_path.name, + "complete": False, + } + try: + compose("stop", "--timeout", "1", "reconciler") + + def age(): + return inside( + f"h=store.head(key={data['orphan']!r})\nassert h is not None\nwith db.connection() as c: now=c.execute('SELECT clock_timestamp() now').fetchone()['now']\nprint(json.dumps((now-h.last_modified).total_seconds()))" + ) + + while age() < 302: + time.sleep(min(5, 303 - age())) + report["real_s3_age_seconds"] = age() + fresh = f"objects/cleanup-{data['seed_id']}/fresh-orphan" + inside( + f"b=b'fresh'\nstore.put(key={fresh!r},stream=io.BytesIO(b),size=len(b),content_type='image/png',sha256=hashlib.sha256(b).hexdigest())\nprint('true')" + ) + oldkeys = [ + data["orphan"], + data["failed_source"], + data["expired_reservation_key"], + ] + [ + f"objects/{data['ready_attachment']}/v1/{v}" + for v in ["original", "thumbnail", "crop", "optimized"] + ] + protected = [data["active_source"], data["reservation_key"], fresh] + [ + f"objects/{data['ready_attachment']}/v2/{v}" + for v in ["original", "thumbnail", "crop", "optimized"] + ] + before = inside( + f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))" + ) + assert all(before.values()) + dry = inside( + "print(json.dumps(reconcile(db,store,apply=False,grace=300)))" + ) + candidates = {a["key"] for a in dry["actions"]} + assert set(oldkeys) <= candidates and not set(protected) & candidates + afterdry = inside( + f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))" + ) + assert afterdry == before + applied = inside( + "print(json.dumps(reconcile(db,store,apply=True,grace=300)))" + ) + repeat = inside( + "print(json.dumps(reconcile(db,store,apply=True,grace=300)))" + ) + assert not repeat["actions"] + after = inside( + f"print(json.dumps({{k:store.head(key=k) is not None for k in {oldkeys + protected!r}}}))" + ) + assert all(not after[k] for k in oldkeys) and all( + after[k] for k in protected + ) + failed = inside( + f"with db.connection() as c: j=c.execute('SELECT id,error_code FROM jobs WHERE attachment_id=%s',({data['failed_id']!r},)).fetchone()\nprint(json.dumps(j,default=str))" + ) + assert failed["error_code"] == "source_expired" + try: + operator(failed["id"]) + except RuntimeError: + replay_refused = True + else: + raise AssertionError("Expired source replay must fail") + # Expired reservation was removed, so a delayed API cannot commit its handoff. + fenced = inside( + f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM upload_reservations WHERE id=%s AND expires_at>clock_timestamp()',({data['expired_reservation_id']!r},)).fetchone()\nprint(json.dumps(r['n']==0))" + ) + assert fenced + report.update( + dry_run=dry, + apply=applied, + repeat=repeat, + before=before, + after=after, + dry_run_unchanged=True, + active_current_and_fresh_protected=True, + expired_source_replay_refused=replay_refused, + expired_writer_fenced=fenced, + complete=True, + ) + finally: + # Remove only this fixture's artificial active reservation/job; never real jobs. + inside( + f"with db.connection() as c, c.transaction():\n c.execute(\"UPDATE jobs SET state='failed',error_code='fixture_complete',lease_until=NULL WHERE attachment_id=%s\",({data['active_id']!r},))\n c.execute(\"UPDATE attachments SET state='failed',error_code='fixture_complete' WHERE id=%s\",({data['active_id']!r},))\n c.execute('DELETE FROM upload_reservations WHERE id=%s',({data['reservation_id']!r},))\nprint('true')" + ) + compose("up", "-d", "--no-deps", "--no-recreate", "reconciler") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text( + json.dumps(report, ensure_ascii=False, indent=2) + "\n" + ) + print( + "PASS: real retention, dry-run/apply/repeat, current/active/fresh protection", + flush=True, + ) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("action", choices=["seed", "verify"]) + p.add_argument("--seed", required=True, type=Path) + p.add_argument("--output", type=Path) + a = p.parse_args() + if a.action == "seed": + seed(a.seed) + else: + if a.output is None: + p.error("output required") + verify(a.seed, a.output) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_pipeline.py b/scripts/verify_pipeline.py index 1d4fd6f8..78281d38 100644 --- a/scripts/verify_pipeline.py +++ b/scripts/verify_pipeline.py @@ -30,7 +30,7 @@ db,store,queue=database(),ObjectStore(),broker() def inside(code): - cid = compose("ps", "-q", "api").split()[0] + cid = compose("ps", "-q", "controller").split()[0] return json.loads( command( ["docker", "exec", "-i", cid, "python", "-"], input=PREFIX + code @@ -123,8 +123,19 @@ def main(): try: ready() # Check instrumentation is really enabled and production default is off. - assert inside( - "import os\nprint(json.dumps(os.getenv('PIPELINE_FAULTS')=='1'))" + cid = compose("ps", "-q", "api").split()[0] + assert ( + command( + [ + "docker", + "exec", + cid, + "python", + "-c", + "import os;print(os.getenv('PIPELINE_FAULTS'))", + ] + ) + == "1" ) stop("reconciler") with httpx.Client( @@ -503,6 +514,71 @@ def main(): save( "malformed_notification_rejected", {"durable_diagnostic": True} ) + # Private bucket, bounded IAM, and download after uploader process dies. + a, j = upload() + confirm(a, j) + key = f"objects/{a['id']}/v1/original" + security = inside(f"""from minio.error import S3Error +from urllib.error import HTTPError +try: urllib.request.urlopen('http://s3:9000/'+BUCKET+'/' + {key!r},timeout=3) +except HTTPError as e: anonymous=e.code +else: raise AssertionError('Anonymous access allowed') +denials={{}} +for name,fn in [('create_bucket',lambda:store.client.make_bucket('forbidden-'+uuid4().hex)),('write_outside_prefix',lambda:store.client.put_object(BUCKET,'outside-'+uuid4().hex,io.BytesIO(b'x'),1)),('list_other_bucket',lambda:list(store.client.list_objects('forbidden-'+uuid4().hex)))]: + try: fn() + except S3Error as e: denials[name]=e.code + else: raise AssertionError('IAM allowed forbidden action') +print(json.dumps({{'anonymous':anonymous,'iam':denials}}))""") + assert security["anonymous"] == 403 and set( + security["iam"].values() + ) == {"AccessDenied"} + selected = None + for cid in compose("ps", "-q", "api").split(): + alias = command( + [ + "docker", + "exec", + cid, + "python", + "-c", + "import urllib.request;print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])", + ] + ) + uploader = inside( + f"with db.connection() as c: r=c.execute('SELECT uploader_instance FROM attachments WHERE id=%s',({a['id']!r},)).fetchone()\nprint(json.dumps(r['uploader_instance']))" + ) + if alias == uploader: + selected = cid + assert selected + command(["docker", "stop", "--time", "1", selected]) + ready() + wait( + lambda: ( + client.get( + f"/api/v1/attachments/{a['id']}/content", headers=h + ).status_code + == 200 + ) + ) + downloaded = client.get( + f"/api/v1/attachments/{a['id']}/content", headers=h + ) + assert ( + downloaded.content == raw + and downloaded.headers["X-Instance-Id"] != uploader + ) + start("api") + ready() + save( + "private_iam_and_other_api_download", + { + "security": security, + "attachment": a["id"], + "uploader": uploader, + "downloader": downloaded.headers["X-Instance-Id"], + "sha256": hashlib.sha256(raw).hexdigest(), + }, + ) report["worker_metrics"] = inside( "with db.connection() as c: r=c.execute(\"SELECT job_id,detail FROM pipeline_events WHERE event='child_finished' ORDER BY id\").fetchall()\nprint(json.dumps(r,default=str))" ) diff --git a/student-unit/test_job_envelope.py b/student-unit/test_job_envelope.py new file mode 100644 index 00000000..6f3444e9 --- /dev/null +++ b/student-unit/test_job_envelope.py @@ -0,0 +1,68 @@ +import copy +from datetime import UTC, datetime +from uuid import uuid4 + +import pytest +from jsonschema import Draft202012Validator, FormatChecker + +from src.pipeline import validate_job + + +def example(): + return dict( + schema_version=1, + job_id=str(uuid4()), + type="attachment.process.v1", + attachment_id=str(uuid4()), + pipeline_version=1, + attempt=1, + created_at=datetime.now(UTC).isoformat(), + ) + + +def test_schema_and_runtime_agree_on_valid_job(): + import json + from pathlib import Path + + schema = json.loads(Path("contracts/jobs.schema.json").read_text()) + for version in [1, 2]: + data = example() + data["pipeline_version"] = version + data.update( + traceparent="00-" + "a" * 32 + "-" + "b" * 16 + "-01", + tracestate="vendor=opaque", + ) + Draft202012Validator(schema, format_checker=FormatChecker()).validate( + data + ) + assert validate_job(data) == data + + +@pytest.mark.parametrize( + "field,value", + [ + ("schema_version", True), + ("schema_version", 2), + ("type", "other"), + ("job_id", 123), + ("job_id", "broken"), + ("attachment_id", None), + ("pipeline_version", 0), + ("pipeline_version", True), + ("attempt", 0), + ("attempt", 1.0), + ("created_at", "2026-01-01"), + ("created_at", "2026-01-01T12:00:00"), + ("created_at", None), + ("traceparent", "not-w3c"), + ("tracestate", "x" * 513), + ("url", "http://invalid.example"), + ("bytes", "secret-contents"), + ("filename", "../../bad"), + ], +) +def test_invalid_notification_is_rejected(field, value): + data = copy.deepcopy(example()) + data[field] = value + with pytest.raises((ValueError, TypeError)): + validate_job(data) -- 2.54.0 From 2fa1b45da2caf58f3f3439dcd4e23b68f772d2b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Sun, 15 Nov 2026 22:49:40 +0300 Subject: [PATCH 05/11] test(pipeline): isolate durable stream snapshot from concurrent relay --- .github/workflows/application.yml | 27 +++++++ SCHEMA.md | 48 ++++++++++++ evidence/pipeline-first-failed.json | 109 ++++++++++++++++++++++++++++ evidence/pipeline-first-failure.txt | 12 +++ infra/README.md | 17 ++++- scripts/stop-worker.sh | 3 + scripts/verify_pipeline.py | 4 +- 7 files changed, 215 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/application.yml create mode 100644 SCHEMA.md create mode 100644 evidence/pipeline-first-failed.json create mode 100644 evidence/pipeline-first-failure.txt create mode 100755 scripts/stop-worker.sh diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..cb469576 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,27 @@ +name: Application integration +on: [push, pull_request, workflow_dispatch] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + COMPOSE_FILE: compose.yaml:compose.test.yaml + NO_PROXY: localhost,127.0.0.1 + no_proxy: localhost,127.0.0.1 + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v6 + with: + version: 0.13.7 + - run: uv sync --locked + - run: make bootstrap check validate lint unit PYTHON=.venv/bin/python + - run: VCS_REF=$(git rev-parse HEAD) docker compose up -d --build --wait + - run: make test test-student PYTHON=.venv/bin/python BASE_URL=http://127.0.0.1:8085 + - run: uv run --locked python -m scripts.verify_cleanup seed --seed evidence/gc-ci-seed.json + - run: uv run --locked python -m scripts.verify_pipeline --output evidence/pipeline-ci.json + - run: uv run --locked python -m scripts.verify_cleanup verify --seed evidence/gc-ci-seed.json --output evidence/gc-ci.json + - run: uv run --locked python -m scripts.verify_cluster --output evidence/cluster-ci.json + - if: always() + run: docker compose down --volumes --remove-orphans diff --git a/SCHEMA.md b/SCHEMA.md new file mode 100644 index 00000000..62848eae --- /dev/null +++ b/SCHEMA.md @@ -0,0 +1,48 @@ +# PostgreSQL / S3 / broker — ЛР5 + +```mermaid +flowchart LR + C[Client] --> LB[HAProxy] + LB --> A[API 1/2] + A --> S[Private S3 staging] + A --> P[(PG attachment + job + outbox)] + P --> R[Relay + watchdog] + R --> B[(Valkey Streams + AOF)] + B --> W[Worker 2 bounded child slots] + W --> V[S3 versioned original + 3 variants] + W --> E[(PG fenced atomic publication)] + E --> A +``` + +Core: users→sessions; chats↔members; chats→messages; message_attachments +имеет составные FK chat/message и chat/attachment. Сессии и idempotency +сохраняются в PG; cache не является источником истины. Row locks и unique +(user,method,path,key) сохраняют Message+result атомарно. + +Attachment хранит свойства original, target pipeline_version и +published_version. Jobs unique(attachment_id,pipeline_version), UUID logical ID, +state,attempt/cycle,fence,lease/owner,next_at,safeerror. Outbox содержит только +schema-valid envelope, due/published. Consumer group хранит pending entries, +но окончательные решения находятся в PG. Failed registry — jobs.state=failed. + +Результаты: attachment_variants PK(attachment_id,pipeline_version,name), +unique object_key; effect_publications PK(attachment_id,pipeline_version). +Worker в одной транзакции проверяет target version, fence, live lease и +processing, записывает4variants+effect_publication+ready. Object bytes +предшествуют COMMIT; дубли могут повторить PUT тех же bytes/keys. +Публикацию новой версии нельзя перезаписать старым fence/version. + +Upload reservation120s предотвращает поздний COMMIT после GC: row lock плюс +проверка expiry нужны и при handoff, и при удалении просроченного reservation. +Unconfirmed bytes не становятся accepted jobs сами по себе. + +S3: staging `objects/staging//original`, final +`objects//v/{original,thumbnail,crop,optimized}`. +Filename — только отображаемое поле, не путь. IAM ограничен messenger/objects/*, +корневые credentials получает только S3/bootstrap. PG содержит metadata/SHA, +а не bytes. Broker envelope содержит ID/version/attempt/time, не storage paths. + +pipeline_audit сохраняет explicit operator replay/version; pipeline_events — +наблюдаемые решения без secrets. pipeline_faults — test-only single-use hooks, +выключены без PIPELINE_FAULTS=1. В production API не имеет broker credentials; +test controller появляется только в compose.test.yaml и не публикует портов. diff --git a/evidence/pipeline-first-failed.json b/evidence/pipeline-first-failed.json new file mode 100644 index 00000000..5550214c --- /dev/null +++ b/evidence/pipeline-first-failed.json @@ -0,0 +1,109 @@ +{ + "source_commit": "a275cf2632d3bf3d96409408d9cb313cdb2824c2", + "date": "2026-11-15T19:46:36.154413+00:00", + "track": 2, + "complete": false, + "cases": [ + { + "case": "staging_before_handoff_crash", + "no_202": true, + "no_attachment_or_job_created": true, + "orphan_retained_for_cleanup": true + }, + { + "case": "commit_before_publish_and_worker_stopped", + "attachment": "f554e320-4881-4087-b813-9f28607fc3b1", + "effect": { + "job": { + "id": "1016e737-7ec1-4fde-9946-a342d70b2a71", + "attachment_id": "f554e320-4881-4087-b813-9f28607fc3b1", + "pipeline_version": 1, + "source_key": "objects/staging/f554e320-4881-4087-b813-9f28607fc3b1/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:47:10.665115+00:00", + "error_code": null, + "created_at": "2026-11-15 19:47:10.665115+00:00", + "updated_at": "2026-11-15 19:47:36.689673+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/crop", + "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/optimized", + "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/original", + "objects/f554e320-4881-4087-b813-9f28607fc3b1/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:47:34.529681+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772054509-0", + "outbox": 11 + }, + "created_at": "2026-11-15 19:47:34.584052+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:47:36.690043+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2740797910000765, + "cpu_seconds": 0.2354738329999997, + "rss_peak_mib": 79.3125 + }, + "created_at": "2026-11-15 19:47:36.847178+00:00" + } + ] + }, + "queued_link_allowed": true, + "api_and_broker_restarted": true + } + ] +} diff --git a/evidence/pipeline-first-failure.txt b/evidence/pipeline-first-failure.txt new file mode 100644 index 00000000..eea202b5 --- /dev/null +++ b/evidence/pipeline-first-failure.txt @@ -0,0 +1,12 @@ +PASS: staging_before_handoff_crash +PASS: commit_before_publish_and_worker_stopped +Traceback (most recent call last): + File "", line 203, in _run_module_as_main + File "", line 88, in _run_code + File "/scripts/verify_pipeline.py", line 602, in + main() + ~~~~^^ + File "/scripts/verify_pipeline.py", line 318, in main + assert restored == ids + ^^^^^^^^^^^^^^^ +AssertionError diff --git a/infra/README.md b/infra/README.md index 5a39e862..c1306536 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,16 @@ -# Инфраструктура студента +# Инфраструктура ЛР5 -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +Топология и реальные pinned versions — compose.yaml; сборка MinIO/mc из +закреплённых commits — s3.Dockerfile; bucket/IAM — s3-bootstrap.sh и s3-policy.json. +Credentials создаёт scripts/bootstrap.py, значения не входят в Git/image. +API/worker/reconciler получают scoped application keys, relay только PG+broker, +root keys доступны только S3/bootstrap. Консоль S3 слушает loopback контейнера. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. +Сервисы: postgres persistent volume; cache disposable; s3 persistent volume; +broker отдельный Valkey Streams persistent AOF volume; relay/outbox/watchdog; +worker отдельный2CPU1536MiB2slots; api2replicas; lb один localhost:8085. +S3/broker/PG/API host ports не публикуются. Сеть — private Compose default; +production TLS/изоляция от потери host не заявлены. Детали — REPORT.md/SCHEMA.md. -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +compose.test.yaml включает fault instrumentation и отдельный controller, +без публичных endpoints; обычный запуск не имеет этого controller/hooks. diff --git a/scripts/stop-worker.sh b/scripts/stop-worker.sh new file mode 100755 index 00000000..2f4ef642 --- /dev/null +++ b/scripts/stop-worker.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +docker compose stop --timeout 1 worker diff --git a/scripts/verify_pipeline.py b/scripts/verify_pipeline.py index 78281d38..6f850068 100644 --- a/scripts/verify_pipeline.py +++ b/scripts/verify_pipeline.py @@ -310,12 +310,14 @@ def main(): f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NOT NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']>0))" ) ) + stop("relay") ids = inside("print(json.dumps(queue.xrange(STREAM)))") compose("kill", "-s", "SIGKILL", "broker") start("broker") wait(lambda: inside("print(json.dumps(queue.ping()))")) restored = inside("print(json.dumps(queue.xrange(STREAM)))") - assert restored == ids + assert restored == ids, {"before": ids, "after": restored} + start("relay") start("worker") save( "confirmed_broker_entry_survives_sigkill", -- 2.54.0 From 972cef8f7f3938913beb56d32551a88f5e40bc88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Sun, 15 Nov 2026 23:03:53 +0300 Subject: [PATCH 06/11] test(pipeline): coordinate stale writers within the live lease window --- .env.example | 1 - REPORT.md | 11 + evidence/pipeline-second-failed.json | 2488 ++++++++++++++++++++++++++ evidence/pipeline-second-failure.txt | 30 + research/analyze_pipeline.py | 126 ++ scripts/verify_pipeline.py | 558 +++--- student-unit/test_job_envelope.py | 51 + 7 files changed, 3024 insertions(+), 241 deletions(-) create mode 100644 evidence/pipeline-second-failed.json create mode 100644 evidence/pipeline-second-failure.txt create mode 100644 research/analyze_pipeline.py diff --git a/.env.example b/.env.example index b385b8fc..2e9ee126 100644 --- a/.env.example +++ b/.env.example @@ -23,7 +23,6 @@ PUBLIC_PORT=8085 API_CPUS=0.5 API_MEMORY=768m DB_POOL_SIZE=10 -RECONCILE_GRACE_SECONDS=300 BROKER_URL_FILE=/run/secrets/broker_url WORKER_CONCURRENCY=2 diff --git a/REPORT.md b/REPORT.md index 0d440c08..7c635047 100644 --- a/REPORT.md +++ b/REPORT.md @@ -232,3 +232,14 @@ slots/COUNT сдерживает конкуренцию. Возможен лиш долгой эксплуатации. Нет global admission quota: при долгой недоступности worker/broker PG outbox может расти; исследование перегрузки — другой трек. Short synthetic fixtures и повторы ограничивают обобщение на большие файлы. +Успешно завершившиеся child samples не включают RSS убитых процессов. + +## Настройки + +Compose использует PUBLIC_PORT, API_REPLICAS/CPUS/MEMORY, DB_POOL_SIZE, +SESSION_TTL_SECONDS, WORKER_CONCURRENCY, JOB_MAX_ATTEMPTS/TIMEOUT/LEASE и +STAGING_RETENTION_SECONDS. Значения .env.example для S3 endpoint/region/bucket, +MAX_UPLOAD_BYTES/MAX_IMAGE_PIXELS и idempotency TTL документируют фиксированный +контракт реализации: s3:9000/us-east-1/messenger,10MiB/40M/24h. Их нельзя изменить +одним редактированием env: соответствующие constants/SQL надо менять согласованно. +Secret *_FILE paths заданы в compose; значения создаёт bootstrap. diff --git a/evidence/pipeline-second-failed.json b/evidence/pipeline-second-failed.json new file mode 100644 index 00000000..4c1d95a5 --- /dev/null +++ b/evidence/pipeline-second-failed.json @@ -0,0 +1,2488 @@ +{ + "source_commit": "2fa1b45da2caf58f3f3439dcd4e23b68f772d2b3", + "date": "2026-11-15T19:49:40.865980+00:00", + "track": 2, + "complete": false, + "cases": [ + { + "case": "staging_before_handoff_crash", + "no_202": true, + "no_attachment_or_job_created": true, + "orphan_retained_for_cleanup": true + }, + { + "case": "commit_before_publish_and_worker_stopped", + "attachment": "b8d37d7a-dca2-48ef-b4b9-0b86253c5fca", + "effect": { + "job": { + "id": "817ac0d3-2ae3-4721-a1c4-0d517f624f3a", + "attachment_id": "b8d37d7a-dca2-48ef-b4b9-0b86253c5fca", + "pipeline_version": 1, + "source_key": "objects/staging/b8d37d7a-dca2-48ef-b4b9-0b86253c5fca/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:50:15.964915+00:00", + "error_code": null, + "created_at": "2026-11-15 19:50:15.964915+00:00", + "updated_at": "2026-11-15 19:50:42.384949+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/b8d37d7a-dca2-48ef-b4b9-0b86253c5fca/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/b8d37d7a-dca2-48ef-b4b9-0b86253c5fca/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/b8d37d7a-dca2-48ef-b4b9-0b86253c5fca/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/b8d37d7a-dca2-48ef-b4b9-0b86253c5fca/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/b8d37d7a-dca2-48ef-b4b9-0b86253c5fca/v1/crop", + "objects/b8d37d7a-dca2-48ef-b4b9-0b86253c5fca/v1/optimized", + "objects/b8d37d7a-dca2-48ef-b4b9-0b86253c5fca/v1/original", + "objects/b8d37d7a-dca2-48ef-b4b9-0b86253c5fca/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772240105-0", + "outbox": 14 + }, + "created_at": "2026-11-15 19:50:40.129072+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:50:40.135328+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:50:42.385442+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2753011250001691, + "cpu_seconds": 0.24031520900000003, + "rss_peak_mib": 79.31640625 + }, + "created_at": "2026-11-15 19:50:42.489496+00:00" + } + ] + }, + "queued_link_allowed": true, + "api_and_broker_restarted": true + }, + { + "case": "confirmed_broker_entry_survives_sigkill", + "effect": { + "job": { + "id": "a68327bb-1d99-4ca1-a541-c197ad0683ea", + "attachment_id": "c5c25a87-8648-46ac-987f-1e8c70b5a031", + "pipeline_version": 1, + "source_key": "objects/staging/c5c25a87-8648-46ac-987f-1e8c70b5a031/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:50:48.722481+00:00", + "error_code": null, + "created_at": "2026-11-15 19:50:48.722481+00:00", + "updated_at": "2026-11-15 19:51:16.786267+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/c5c25a87-8648-46ac-987f-1e8c70b5a031/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/c5c25a87-8648-46ac-987f-1e8c70b5a031/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/c5c25a87-8648-46ac-987f-1e8c70b5a031/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/c5c25a87-8648-46ac-987f-1e8c70b5a031/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/c5c25a87-8648-46ac-987f-1e8c70b5a031/v1/crop", + "objects/c5c25a87-8648-46ac-987f-1e8c70b5a031/v1/optimized", + "objects/c5c25a87-8648-46ac-987f-1e8c70b5a031/v1/original", + "objects/c5c25a87-8648-46ac-987f-1e8c70b5a031/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772248812-0", + "outbox": 15 + }, + "created_at": "2026-11-15 19:50:48.818743+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:51:14.646984+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:51:16.786535+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2575967509997099, + "cpu_seconds": 0.23045399499999997, + "rss_peak_mib": 79.84765625 + }, + "created_at": "2026-11-15 19:51:16.936565+00:00" + } + ] + }, + "same_stream_entries": true, + "entry_count": 1 + }, + { + "case": "relay_crash_after_xadd_before_pg_marker", + "unpublished_after_crash": 1, + "effect": { + "job": { + "id": "2b40b034-2ea5-4612-8d4e-f099e1e3f65d", + "attachment_id": "a26e3328-0c54-492c-9390-c70a6bde55f0", + "pipeline_version": 1, + "source_key": "objects/staging/a26e3328-0c54-492c-9390-c70a6bde55f0/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:51:24.018468+00:00", + "error_code": null, + "created_at": "2026-11-15 19:51:24.018468+00:00", + "updated_at": "2026-11-15 19:51:51.221904+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/a26e3328-0c54-492c-9390-c70a6bde55f0/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/a26e3328-0c54-492c-9390-c70a6bde55f0/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/a26e3328-0c54-492c-9390-c70a6bde55f0/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/a26e3328-0c54-492c-9390-c70a6bde55f0/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/a26e3328-0c54-492c-9390-c70a6bde55f0/v1/crop", + "objects/a26e3328-0c54-492c-9390-c70a6bde55f0/v1/optimized", + "objects/a26e3328-0c54-492c-9390-c70a6bde55f0/v1/original", + "objects/a26e3328-0c54-492c-9390-c70a6bde55f0/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:51:49.054434+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:51:51.222245+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2825338749998991, + "cpu_seconds": 0.24960387800000006, + "rss_peak_mib": 79.30859375 + }, + "created_at": "2026-11-15 19:51:51.344923+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772312170-0", + "outbox": 16 + }, + "created_at": "2026-11-15 19:51:52.191334+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772312170-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:51:52.212691+00:00" + } + ] + } + }, + { + "case": "before_effect_sigkill_repeat_1", + "at_crash": { + "job": { + "id": "681235f5-558b-44b8-af12-e61d2f6c65bf", + "attachment_id": "1953015b-b257-4faf-baae-088595d3cb94", + "pipeline_version": 1, + "source_key": "objects/staging/1953015b-b257-4faf-baae-088595d3cb94/original", + "state": "processing", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": "6da99350-39af-4443-bd18-0ace582dc8c4-0", + "lease_until": "2026-11-15 19:52:23.166586+00:00", + "next_at": "2026-11-15 19:51:57.004335+00:00", + "error_code": null, + "created_at": "2026-11-15 19:51:57.004335+00:00", + "updated_at": "2026-11-15 19:52:08.166587+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772317162-0", + "outbox": 17 + }, + "created_at": "2026-11-15 19:51:57.168914+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:52:08.168710+00:00" + } + ] + }, + "effect": { + "job": { + "id": "681235f5-558b-44b8-af12-e61d2f6c65bf", + "attachment_id": "1953015b-b257-4faf-baae-088595d3cb94", + "pipeline_version": 1, + "source_key": "objects/staging/1953015b-b257-4faf-baae-088595d3cb94/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:52:25.107302+00:00", + "error_code": null, + "created_at": "2026-11-15 19:51:57.004335+00:00", + "updated_at": "2026-11-15 19:52:27.449434+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/1953015b-b257-4faf-baae-088595d3cb94/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/1953015b-b257-4faf-baae-088595d3cb94/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/1953015b-b257-4faf-baae-088595d3cb94/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/1953015b-b257-4faf-baae-088595d3cb94/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/1953015b-b257-4faf-baae-088595d3cb94/v1/crop", + "objects/1953015b-b257-4faf-baae-088595d3cb94/v1/optimized", + "objects/1953015b-b257-4faf-baae-088595d3cb94/v1/original", + "objects/1953015b-b257-4faf-baae-088595d3cb94/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772317162-0", + "outbox": 17 + }, + "created_at": "2026-11-15 19:51:57.168914+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:52:08.168710+00:00" + }, + { + "event": "retry", + "detail": { + "code": "lease_expired", + "attempt": 1, + "backoff_seconds": 1.8055802565230599 + }, + "created_at": "2026-11-15 19:52:23.304490+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772317162-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 19:52:23.945422+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772345183-0", + "outbox": 18 + }, + "created_at": "2026-11-15 19:52:25.192037+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 19:52:25.199196+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:52:27.449886+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2647618330001933, + "cpu_seconds": 0.23189429199999978, + "rss_peak_mib": 79.6796875 + }, + "created_at": "2026-11-15 19:52:27.582283+00:00" + } + ] + } + }, + { + "case": "after_effect_sigkill_repeat_1", + "at_crash": { + "job": { + "id": "e4b36262-0688-4b3c-90ce-9252e9633353", + "attachment_id": "f3a006b9-2325-4c58-8680-0019c1eb3d51", + "pipeline_version": 1, + "source_key": "objects/staging/f3a006b9-2325-4c58-8680-0019c1eb3d51/original", + "state": "processing", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": "9e7b8a57-6aa3-4d92-8fee-74f29c6daab8-1", + "lease_until": "2026-11-15 19:52:59.402691+00:00", + "next_at": "2026-11-15 19:52:33.289892+00:00", + "error_code": null, + "created_at": "2026-11-15 19:52:33.289893+00:00", + "updated_at": "2026-11-15 19:52:44.402692+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/f3a006b9-2325-4c58-8680-0019c1eb3d51/v1/crop", + "objects/f3a006b9-2325-4c58-8680-0019c1eb3d51/v1/optimized", + "objects/f3a006b9-2325-4c58-8680-0019c1eb3d51/v1/original", + "objects/f3a006b9-2325-4c58-8680-0019c1eb3d51/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772353422-0", + "outbox": 19 + }, + "created_at": "2026-11-15 19:52:33.427928+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:52:44.404616+00:00" + } + ] + }, + "effect": { + "job": { + "id": "e4b36262-0688-4b3c-90ce-9252e9633353", + "attachment_id": "f3a006b9-2325-4c58-8680-0019c1eb3d51", + "pipeline_version": 1, + "source_key": "objects/staging/f3a006b9-2325-4c58-8680-0019c1eb3d51/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:53:00.020142+00:00", + "error_code": null, + "created_at": "2026-11-15 19:52:33.289893+00:00", + "updated_at": "2026-11-15 19:53:02.386447+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/f3a006b9-2325-4c58-8680-0019c1eb3d51/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/f3a006b9-2325-4c58-8680-0019c1eb3d51/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/f3a006b9-2325-4c58-8680-0019c1eb3d51/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/f3a006b9-2325-4c58-8680-0019c1eb3d51/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/f3a006b9-2325-4c58-8680-0019c1eb3d51/v1/crop", + "objects/f3a006b9-2325-4c58-8680-0019c1eb3d51/v1/optimized", + "objects/f3a006b9-2325-4c58-8680-0019c1eb3d51/v1/original", + "objects/f3a006b9-2325-4c58-8680-0019c1eb3d51/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772353422-0", + "outbox": 19 + }, + "created_at": "2026-11-15 19:52:33.427928+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:52:44.404616+00:00" + }, + { + "event": "retry", + "detail": { + "code": "lease_expired", + "attempt": 1, + "backoff_seconds": 0.4311944608844672 + }, + "created_at": "2026-11-15 19:52:59.593089+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772380024-0", + "outbox": 20 + }, + "created_at": "2026-11-15 19:53:00.028594+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 19:53:00.032406+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772353422-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 19:53:01.476187+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:53:02.387128+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2914882079999188, + "cpu_seconds": 0.24316999600000022, + "rss_peak_mib": 79.80078125 + }, + "created_at": "2026-11-15 19:53:02.497162+00:00" + } + ] + } + }, + { + "case": "after_commit_sigkill_repeat_1", + "at_crash": { + "job": { + "id": "1de536e0-3904-460e-910d-3887d01993de", + "attachment_id": "cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8", + "pipeline_version": 1, + "source_key": "objects/staging/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:53:08.018084+00:00", + "error_code": null, + "created_at": "2026-11-15 19:53:08.018084+00:00", + "updated_at": "2026-11-15 19:53:21.314939+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/v1/crop", + "objects/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/v1/optimized", + "objects/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/v1/original", + "objects/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772388045-0", + "outbox": 21 + }, + "created_at": "2026-11-15 19:53:08.058435+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:53:19.071014+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:53:21.315358+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772388045-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:53:22.296452+00:00" + } + ] + }, + "effect": { + "job": { + "id": "1de536e0-3904-460e-910d-3887d01993de", + "attachment_id": "cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8", + "pipeline_version": 1, + "source_key": "objects/staging/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:53:08.018084+00:00", + "error_code": null, + "created_at": "2026-11-15 19:53:08.018084+00:00", + "updated_at": "2026-11-15 19:53:21.314939+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/v1/crop", + "objects/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/v1/optimized", + "objects/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/v1/original", + "objects/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772388045-0", + "outbox": 21 + }, + "created_at": "2026-11-15 19:53:08.058435+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:53:19.071014+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:53:21.315358+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772388045-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:53:22.296452+00:00" + } + ] + } + }, + { + "case": "before_effect_sigkill_repeat_2", + "at_crash": { + "job": { + "id": "e1ebdd4b-996f-41d2-aa54-246cc9f4c8d3", + "attachment_id": "673b3c28-9161-4d4c-b8a5-0f23791d8b22", + "pipeline_version": 1, + "source_key": "objects/staging/673b3c28-9161-4d4c-b8a5-0f23791d8b22/original", + "state": "processing", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": "db6677b1-a7f2-4ca1-9aec-06ca877b18f5-0", + "lease_until": "2026-11-15 19:54:04.373219+00:00", + "next_at": "2026-11-15 19:53:36.291768+00:00", + "error_code": null, + "created_at": "2026-11-15 19:53:36.291768+00:00", + "updated_at": "2026-11-15 19:53:49.373220+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772416353-0", + "outbox": 22 + }, + "created_at": "2026-11-15 19:53:36.357384+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:53:49.375300+00:00" + } + ] + }, + "effect": { + "job": { + "id": "e1ebdd4b-996f-41d2-aa54-246cc9f4c8d3", + "attachment_id": "673b3c28-9161-4d4c-b8a5-0f23791d8b22", + "pipeline_version": 1, + "source_key": "objects/staging/673b3c28-9161-4d4c-b8a5-0f23791d8b22/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:54:04.974226+00:00", + "error_code": null, + "created_at": "2026-11-15 19:53:36.291768+00:00", + "updated_at": "2026-11-15 19:54:07.246028+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/673b3c28-9161-4d4c-b8a5-0f23791d8b22/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/673b3c28-9161-4d4c-b8a5-0f23791d8b22/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/673b3c28-9161-4d4c-b8a5-0f23791d8b22/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/673b3c28-9161-4d4c-b8a5-0f23791d8b22/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/673b3c28-9161-4d4c-b8a5-0f23791d8b22/v1/crop", + "objects/673b3c28-9161-4d4c-b8a5-0f23791d8b22/v1/optimized", + "objects/673b3c28-9161-4d4c-b8a5-0f23791d8b22/v1/original", + "objects/673b3c28-9161-4d4c-b8a5-0f23791d8b22/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772416353-0", + "outbox": 22 + }, + "created_at": "2026-11-15 19:53:36.357384+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:53:49.375300+00:00" + }, + { + "event": "retry", + "detail": { + "code": "lease_expired", + "attempt": 1, + "backoff_seconds": 0.414707157589379 + }, + "created_at": "2026-11-15 19:54:04.565323+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772444982-0", + "outbox": 23 + }, + "created_at": "2026-11-15 19:54:04.991746+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 19:54:04.994610+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772416353-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 19:54:05.579928+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:54:07.246428+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.259311875000094, + "cpu_seconds": 0.23085762499999984, + "rss_peak_mib": 79.45703125 + }, + "created_at": "2026-11-15 19:54:07.365349+00:00" + } + ] + } + }, + { + "case": "after_effect_sigkill_repeat_2", + "at_crash": { + "job": { + "id": "fbedc7f8-97a6-4c59-848c-d4ab8045d4d2", + "attachment_id": "6357ee24-c57e-4663-927f-ab5acd7b5d25", + "pipeline_version": 1, + "source_key": "objects/staging/6357ee24-c57e-4663-927f-ab5acd7b5d25/original", + "state": "processing", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": "0f632cf8-4d02-442a-b8ff-671eec2c8248-0", + "lease_until": "2026-11-15 19:54:40.083125+00:00", + "next_at": "2026-11-15 19:54:13.213302+00:00", + "error_code": null, + "created_at": "2026-11-15 19:54:13.213302+00:00", + "updated_at": "2026-11-15 19:54:25.083126+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/6357ee24-c57e-4663-927f-ab5acd7b5d25/v1/crop", + "objects/6357ee24-c57e-4663-927f-ab5acd7b5d25/v1/optimized", + "objects/6357ee24-c57e-4663-927f-ab5acd7b5d25/v1/original", + "objects/6357ee24-c57e-4663-927f-ab5acd7b5d25/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772453305-0", + "outbox": 24 + }, + "created_at": "2026-11-15 19:54:13.312119+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:54:25.085248+00:00" + } + ] + }, + "effect": { + "job": { + "id": "fbedc7f8-97a6-4c59-848c-d4ab8045d4d2", + "attachment_id": "6357ee24-c57e-4663-927f-ab5acd7b5d25", + "pipeline_version": 1, + "source_key": "objects/staging/6357ee24-c57e-4663-927f-ab5acd7b5d25/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:54:40.850838+00:00", + "error_code": null, + "created_at": "2026-11-15 19:54:13.213302+00:00", + "updated_at": "2026-11-15 19:54:43.243188+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/6357ee24-c57e-4663-927f-ab5acd7b5d25/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/6357ee24-c57e-4663-927f-ab5acd7b5d25/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/6357ee24-c57e-4663-927f-ab5acd7b5d25/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/6357ee24-c57e-4663-927f-ab5acd7b5d25/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/6357ee24-c57e-4663-927f-ab5acd7b5d25/v1/crop", + "objects/6357ee24-c57e-4663-927f-ab5acd7b5d25/v1/optimized", + "objects/6357ee24-c57e-4663-927f-ab5acd7b5d25/v1/original", + "objects/6357ee24-c57e-4663-927f-ab5acd7b5d25/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772453305-0", + "outbox": 24 + }, + "created_at": "2026-11-15 19:54:13.312119+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:54:25.085248+00:00" + }, + { + "event": "retry", + "detail": { + "code": "lease_expired", + "attempt": 1, + "backoff_seconds": 0.6629327164251897 + }, + "created_at": "2026-11-15 19:54:40.189480+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772481015-0", + "outbox": 25 + }, + "created_at": "2026-11-15 19:54:41.022164+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 19:54:41.027219+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772453305-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 19:54:42.535933+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:54:43.243574+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.26234775000011723, + "cpu_seconds": 0.22806161800000013, + "rss_peak_mib": 79.328125 + }, + "created_at": "2026-11-15 19:54:43.414923+00:00" + } + ] + } + }, + { + "case": "after_commit_sigkill_repeat_2", + "at_crash": { + "job": { + "id": "df35ac8a-d73e-4972-893f-8321e4343fca", + "attachment_id": "4d806c50-751a-4ea0-b4e0-06fd832a82ca", + "pipeline_version": 1, + "source_key": "objects/staging/4d806c50-751a-4ea0-b4e0-06fd832a82ca/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:54:48.867494+00:00", + "error_code": null, + "created_at": "2026-11-15 19:54:48.867495+00:00", + "updated_at": "2026-11-15 19:55:03.022925+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/4d806c50-751a-4ea0-b4e0-06fd832a82ca/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/4d806c50-751a-4ea0-b4e0-06fd832a82ca/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/4d806c50-751a-4ea0-b4e0-06fd832a82ca/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/4d806c50-751a-4ea0-b4e0-06fd832a82ca/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/4d806c50-751a-4ea0-b4e0-06fd832a82ca/v1/crop", + "objects/4d806c50-751a-4ea0-b4e0-06fd832a82ca/v1/optimized", + "objects/4d806c50-751a-4ea0-b4e0-06fd832a82ca/v1/original", + "objects/4d806c50-751a-4ea0-b4e0-06fd832a82ca/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772489028-0", + "outbox": 26 + }, + "created_at": "2026-11-15 19:54:49.032563+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:55:00.824089+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:55:03.023281+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772489028-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:55:03.976606+00:00" + } + ] + }, + "effect": { + "job": { + "id": "df35ac8a-d73e-4972-893f-8321e4343fca", + "attachment_id": "4d806c50-751a-4ea0-b4e0-06fd832a82ca", + "pipeline_version": 1, + "source_key": "objects/staging/4d806c50-751a-4ea0-b4e0-06fd832a82ca/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:54:48.867494+00:00", + "error_code": null, + "created_at": "2026-11-15 19:54:48.867495+00:00", + "updated_at": "2026-11-15 19:55:03.022925+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/4d806c50-751a-4ea0-b4e0-06fd832a82ca/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/4d806c50-751a-4ea0-b4e0-06fd832a82ca/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/4d806c50-751a-4ea0-b4e0-06fd832a82ca/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/4d806c50-751a-4ea0-b4e0-06fd832a82ca/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/4d806c50-751a-4ea0-b4e0-06fd832a82ca/v1/crop", + "objects/4d806c50-751a-4ea0-b4e0-06fd832a82ca/v1/optimized", + "objects/4d806c50-751a-4ea0-b4e0-06fd832a82ca/v1/original", + "objects/4d806c50-751a-4ea0-b4e0-06fd832a82ca/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772489028-0", + "outbox": 26 + }, + "created_at": "2026-11-15 19:54:49.032563+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:55:00.824089+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:55:03.023281+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772489028-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:55:03.976606+00:00" + } + ] + } + }, + { + "case": "transient_retry_with_jitter", + "effect": { + "job": { + "id": "be6bd110-e0e9-4dbf-92cb-2ee8e18d407d", + "attachment_id": "021c526b-3ee1-4bf0-806a-300bfdb54c90", + "pipeline_version": 1, + "source_key": "objects/staging/021c526b-3ee1-4bf0-806a-300bfdb54c90/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:55:33.880597+00:00", + "error_code": null, + "created_at": "2026-11-15 19:55:19.724112+00:00", + "updated_at": "2026-11-15 19:55:36.546033+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/021c526b-3ee1-4bf0-806a-300bfdb54c90/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/021c526b-3ee1-4bf0-806a-300bfdb54c90/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/021c526b-3ee1-4bf0-806a-300bfdb54c90/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/021c526b-3ee1-4bf0-806a-300bfdb54c90/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/021c526b-3ee1-4bf0-806a-300bfdb54c90/v1/crop", + "objects/021c526b-3ee1-4bf0-806a-300bfdb54c90/v1/optimized", + "objects/021c526b-3ee1-4bf0-806a-300bfdb54c90/v1/original", + "objects/021c526b-3ee1-4bf0-806a-300bfdb54c90/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772519797-0", + "outbox": 27 + }, + "created_at": "2026-11-15 19:55:19.808772+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:55:31.440834+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 19:55:33.429636+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 1, + "backoff_seconds": 0.4505134832470537 + }, + "created_at": "2026-11-15 19:55:33.430761+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772534021-0", + "outbox": 28 + }, + "created_at": "2026-11-15 19:55:34.026968+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 19:55:34.037749+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:55:36.546474+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.30820225099978416, + "cpu_seconds": 0.27197879400000025, + "rss_peak_mib": 79.34765625 + }, + "created_at": "2026-11-15 19:55:36.658571+00:00" + } + ] + } + }, + { + "case": "hung_child_killed_and_retried", + "effect": { + "job": { + "id": "5d2588f5-72b5-41ce-95ce-c78327ceddeb", + "attachment_id": "b6b567e2-9db5-44a2-9227-a612bf870a4a", + "pipeline_version": 1, + "source_key": "objects/staging/b6b567e2-9db5-44a2-9227-a612bf870a4a/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:56:06.802539+00:00", + "error_code": null, + "created_at": "2026-11-15 19:55:42.749908+00:00", + "updated_at": "2026-11-15 19:56:09.315964+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/b6b567e2-9db5-44a2-9227-a612bf870a4a/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/b6b567e2-9db5-44a2-9227-a612bf870a4a/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/b6b567e2-9db5-44a2-9227-a612bf870a4a/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/b6b567e2-9db5-44a2-9227-a612bf870a4a/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/b6b567e2-9db5-44a2-9227-a612bf870a4a/v1/crop", + "objects/b6b567e2-9db5-44a2-9227-a612bf870a4a/v1/optimized", + "objects/b6b567e2-9db5-44a2-9227-a612bf870a4a/v1/original", + "objects/b6b567e2-9db5-44a2-9227-a612bf870a4a/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772542919-0", + "outbox": 29 + }, + "created_at": "2026-11-15 19:55:42.924245+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:55:54.099837+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "worker_timeout", + "result": "transient" + }, + "created_at": "2026-11-15 19:56:06.157298+00:00" + }, + { + "event": "retry", + "detail": { + "code": "worker_timeout", + "attempt": 1, + "backoff_seconds": 0.6421558595933988 + }, + "created_at": "2026-11-15 19:56:06.161579+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772567013-0", + "outbox": 30 + }, + "created_at": "2026-11-15 19:56:07.019328+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 19:56:07.024895+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:56:09.316371+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.26163899999983187, + "cpu_seconds": 0.22942266700000014, + "rss_peak_mib": 79.33203125 + }, + "created_at": "2026-11-15 19:56:09.453981+00:00" + } + ] + } + }, + { + "case": "permanent_failed_registry_and_audited_replay", + "failed": { + "id": "d4dba512-1288-4b46-a156-850f12e8ae30", + "chat_id": "bd910135-1b1e-4d6a-b20a-5480cbc0f84e", + "owner_id": "ecd9e718-026e-4f48-9fb6-0e075177b749", + "filename": "synthetic.png", + "content_type": "image/png", + "size_bytes": 742, + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "created_at": "2026-11-15T19:56:15.666529Z", + "updated_at": "2026-11-15T19:56:29.413836Z", + "status": "failed", + "variants": [], + "error_code": "invalid_image" + }, + "effect": { + "job": { + "id": "a9ce36d7-5c35-4b59-8f4e-a696a0a6b4c1", + "attachment_id": "d4dba512-1288-4b46-a156-850f12e8ae30", + "pipeline_version": 1, + "source_key": "objects/staging/d4dba512-1288-4b46-a156-850f12e8ae30/original", + "state": "ready", + "attempt": 1, + "cycle": 2, + "fence": 4, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:56:33.962628+00:00", + "error_code": null, + "created_at": "2026-11-15 19:56:15.668386+00:00", + "updated_at": "2026-11-15 19:56:36.351192+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/d4dba512-1288-4b46-a156-850f12e8ae30/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/d4dba512-1288-4b46-a156-850f12e8ae30/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/d4dba512-1288-4b46-a156-850f12e8ae30/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/d4dba512-1288-4b46-a156-850f12e8ae30/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/d4dba512-1288-4b46-a156-850f12e8ae30/v1/crop", + "objects/d4dba512-1288-4b46-a156-850f12e8ae30/v1/optimized", + "objects/d4dba512-1288-4b46-a156-850f12e8ae30/v1/original", + "objects/d4dba512-1288-4b46-a156-850f12e8ae30/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772575869-0", + "outbox": 31 + }, + "created_at": "2026-11-15 19:56:15.875563+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:56:27.025527+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "invalid_image", + "result": "permanent" + }, + "created_at": "2026-11-15 19:56:29.412139+00:00" + }, + { + "event": "failed", + "detail": { + "code": "invalid_image", + "attempt": 1, + "backoff_seconds": 1.594934405434164 + }, + "created_at": "2026-11-15 19:56:29.414422+00:00" + }, + { + "event": "replay", + "detail": { + "previous_job": "a9ce36d7-5c35-4b59-8f4e-a696a0a6b4c1" + }, + "created_at": "2026-11-15 19:56:33.964625+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772594157-0", + "outbox": 32 + }, + "created_at": "2026-11-15 19:56:34.164732+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 4, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:56:34.178604+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 4, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:56:36.351601+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.27349454200020773, + "cpu_seconds": 0.23941921199999983, + "rss_peak_mib": 79.28125 + }, + "created_at": "2026-11-15 19:56:36.488687+00:00" + } + ] + }, + "ready_replay_rejected": true + }, + { + "case": "finite_retry_exhaustion_then_explicit_replay", + "failed": { + "id": "59a8cb4b-ae61-4d13-8571-30b439549aeb", + "chat_id": "bd910135-1b1e-4d6a-b20a-5480cbc0f84e", + "owner_id": "ecd9e718-026e-4f48-9fb6-0e075177b749", + "filename": "synthetic.png", + "content_type": "image/png", + "size_bytes": 742, + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "created_at": "2026-11-15T19:56:46.654033Z", + "updated_at": "2026-11-15T19:57:29.334556Z", + "status": "failed", + "variants": [], + "error_code": "retry_exhausted" + }, + "effect": { + "job": { + "id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9", + "attachment_id": "59a8cb4b-ae61-4d13-8571-30b439549aeb", + "pipeline_version": 1, + "source_key": "objects/staging/59a8cb4b-ae61-4d13-8571-30b439549aeb/original", + "state": "ready", + "attempt": 1, + "cycle": 2, + "fence": 12, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:57:37.395785+00:00", + "error_code": null, + "created_at": "2026-11-15 19:56:46.657236+00:00", + "updated_at": "2026-11-15 19:57:40.120970+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/59a8cb4b-ae61-4d13-8571-30b439549aeb/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/59a8cb4b-ae61-4d13-8571-30b439549aeb/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/59a8cb4b-ae61-4d13-8571-30b439549aeb/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/59a8cb4b-ae61-4d13-8571-30b439549aeb/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/59a8cb4b-ae61-4d13-8571-30b439549aeb/v1/crop", + "objects/59a8cb4b-ae61-4d13-8571-30b439549aeb/v1/optimized", + "objects/59a8cb4b-ae61-4d13-8571-30b439549aeb/v1/original", + "objects/59a8cb4b-ae61-4d13-8571-30b439549aeb/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794772606789-0", + "outbox": 33 + }, + "created_at": "2026-11-15 19:56:46.795123+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:56:57.378530+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 19:56:59.430873+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 1, + "backoff_seconds": 0.5013925580381666 + }, + "created_at": "2026-11-15 19:56:59.432081+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772620134-0", + "outbox": 34 + }, + "created_at": "2026-11-15 19:57:00.141998+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 19:57:00.147402+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 19:57:02.733197+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 2, + "backoff_seconds": 2.4550537805315966 + }, + "created_at": "2026-11-15 19:57:02.734863+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772625344-0", + "outbox": 35 + }, + "created_at": "2026-11-15 19:57:05.353430+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 5, + "attempt": 3, + "version": 1 + }, + "created_at": "2026-11-15 19:57:05.358049+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 19:57:07.694467+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 3, + "backoff_seconds": 5.50086367785347 + }, + "created_at": "2026-11-15 19:57:07.698911+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772633226-0", + "outbox": 36 + }, + "created_at": "2026-11-15 19:57:13.231545+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 7, + "attempt": 4, + "version": 1 + }, + "created_at": "2026-11-15 19:57:13.242052+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 19:57:15.367437+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 4, + "backoff_seconds": 11.883846279026603 + }, + "created_at": "2026-11-15 19:57:15.369367+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772647261-0", + "outbox": 37 + }, + "created_at": "2026-11-15 19:57:27.266880+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 9, + "attempt": 5, + "version": 1 + }, + "created_at": "2026-11-15 19:57:27.272088+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 19:57:29.333699+00:00" + }, + { + "event": "failed", + "detail": { + "code": "retry_exhausted", + "attempt": 5, + "backoff_seconds": 11.244043439111428 + }, + "created_at": "2026-11-15 19:57:29.335016+00:00" + }, + { + "event": "replay", + "detail": { + "previous_job": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9" + }, + "created_at": "2026-11-15 19:57:37.397576+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772657558-0", + "outbox": 38 + }, + "created_at": "2026-11-15 19:57:37.562630+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 12, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:57:37.565713+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 12, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:57:40.121395+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2960601250001673, + "cpu_seconds": 0.26317221, + "rss_peak_mib": 79.25 + }, + "created_at": "2026-11-15 19:57:40.233260+00:00" + } + ] + } + }, + { + "case": "reordered_and_duplicate_notifications", + "effects": [ + { + "job": { + "id": "c013ea06-4dea-4e65-b715-8a3f5976fa57", + "attachment_id": "8a783984-ce20-4c16-b889-b586fde7e423", + "pipeline_version": 1, + "source_key": "objects/staging/8a783984-ce20-4c16-b889-b586fde7e423/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:57:46.982473+00:00", + "error_code": null, + "created_at": "2026-11-15 19:57:46.982473+00:00", + "updated_at": "2026-11-15 19:58:08.722945+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/8a783984-ce20-4c16-b889-b586fde7e423/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/8a783984-ce20-4c16-b889-b586fde7e423/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/8a783984-ce20-4c16-b889-b586fde7e423/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/8a783984-ce20-4c16-b889-b586fde7e423/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/8a783984-ce20-4c16-b889-b586fde7e423/v1/crop", + "objects/8a783984-ce20-4c16-b889-b586fde7e423/v1/optimized", + "objects/8a783984-ce20-4c16-b889-b586fde7e423/v1/original", + "objects/8a783984-ce20-4c16-b889-b586fde7e423/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:58:06.163185+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:58:08.723492+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.3056550410001364, + "cpu_seconds": 0.26324616899999986, + "rss_peak_mib": 79.65234375 + }, + "created_at": "2026-11-15 19:58:08.900932+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772683589-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:08.959940+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772683590-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:08.960334+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772683591-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:08.982919+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772683592-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:08.983689+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772683593-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:09.007104+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772683593-1", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:09.007496+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772683594-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:09.029504+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772690624-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:10.637124+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772690624-0", + "outbox": 39 + }, + "created_at": "2026-11-15 19:58:10.640193+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772701694-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:21.725022+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772701695-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:21.725323+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772701696-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:21.735316+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772701697-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:21.735756+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772701698-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:21.743926+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772701700-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:21.745132+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772701701-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:21.756397+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772701703-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:21.756779+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772701704-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:21.771997+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772701703-1", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:21.772085+00:00" + } + ] + }, + { + "job": { + "id": "20c34eb3-0582-4144-847c-fa037647f863", + "attachment_id": "8e7ceb2a-b80a-417e-ab36-92db623ef1b0", + "pipeline_version": 1, + "source_key": "objects/staging/8e7ceb2a-b80a-417e-ab36-92db623ef1b0/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 19:57:51.131606+00:00", + "error_code": null, + "created_at": "2026-11-15 19:57:51.131606+00:00", + "updated_at": "2026-11-15 19:58:08.721578+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/8e7ceb2a-b80a-417e-ab36-92db623ef1b0/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/8e7ceb2a-b80a-417e-ab36-92db623ef1b0/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/8e7ceb2a-b80a-417e-ab36-92db623ef1b0/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/8e7ceb2a-b80a-417e-ab36-92db623ef1b0/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/8e7ceb2a-b80a-417e-ab36-92db623ef1b0/v1/crop", + "objects/8e7ceb2a-b80a-417e-ab36-92db623ef1b0/v1/optimized", + "objects/8e7ceb2a-b80a-417e-ab36-92db623ef1b0/v1/original", + "objects/8e7ceb2a-b80a-417e-ab36-92db623ef1b0/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 19:58:06.122059+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 19:58:08.722131+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.3045327089998864, + "cpu_seconds": 0.2630334579999998, + "rss_peak_mib": 79.578125 + }, + "created_at": "2026-11-15 19:58:08.897055+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772679199-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:09.549785+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772679203-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:09.550099+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772679204-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:09.560954+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772679205-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:09.561657+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772679205-1", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:09.572384+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772679206-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:09.572788+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772679207-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:09.585767+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794772690642-0", + "outbox": 40 + }, + "created_at": "2026-11-15 19:58:10.646195+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772690642-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:10.654864+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772705889-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:25.918154+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772705891-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:25.918500+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772705893-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:25.928851+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772705894-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:25.929202+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772705895-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:25.938175+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772705896-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:25.939546+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772705898-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:25.947318+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772705898-1", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:25.948583+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772705899-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:25.960119+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794772705900-0", + "state": "ready" + }, + "created_at": "2026-11-15 19:58:25.962620+00:00" + } + ] + } + ], + "added_duplicates": 36, + "no_extra_attempt_after_ready": true + } + ] +} diff --git a/evidence/pipeline-second-failure.txt b/evidence/pipeline-second-failure.txt new file mode 100644 index 00000000..933a250a --- /dev/null +++ b/evidence/pipeline-second-failure.txt @@ -0,0 +1,30 @@ +PASS: staging_before_handoff_crash +PASS: commit_before_publish_and_worker_stopped +PASS: confirmed_broker_entry_survives_sigkill +PASS: relay_crash_after_xadd_before_pg_marker +PASS: before_effect_sigkill_repeat_1 +PASS: after_effect_sigkill_repeat_1 +PASS: after_commit_sigkill_repeat_1 +PASS: before_effect_sigkill_repeat_2 +PASS: after_effect_sigkill_repeat_2 +PASS: after_commit_sigkill_repeat_2 +PASS: transient_retry_with_jitter +PASS: hung_child_killed_and_retried +PASS: permanent_failed_registry_and_audited_replay +PASS: finite_retry_exhaustion_then_explicit_replay +PASS: reordered_and_duplicate_notifications +Traceback (most recent call last): + File "", line 203, in _run_module_as_main + File "", line 88, in _run_code + File "/scripts/verify_pipeline.py", line 604, in + report["worker_metrics"] = inside( + + File "/scripts/verify_pipeline.py", line 481, in main + "reordered_and_duplicate_notifications", + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ...<4 lines>... + }, + ^ + File "/scripts/verify_pipeline.py", line 48, in wait + raise TimeoutError("Fault scenario condition not reached") +TimeoutError: Fault scenario condition not reached diff --git a/research/analyze_pipeline.py b/research/analyze_pipeline.py new file mode 100644 index 00000000..2436dc16 --- /dev/null +++ b/research/analyze_pipeline.py @@ -0,0 +1,126 @@ +"""Summarize measured fault invariants and child resource samples.""" + +import argparse +import csv +import json +from pathlib import Path + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt + + +def effects(case): + for key in ["effect", "v1", "v2"]: + if key in case: + yield key, case[key] + for i, item in enumerate(case.get("effects", [])): + yield str(i + 1), item + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("input", type=Path) + p.add_argument("--output", required=True, type=Path) + a = p.parse_args() + d = json.loads(a.input.read_text()) + assert d["complete"] and d.get("scope", "all") == "all" + rows = [] + for case in d["cases"]: + for role, e in effects(case): + j = e["job"] + rows.append( + dict( + case=case["case"], + role=role, + job_id=j["id"], + version=j["pipeline_version"], + attempt=j["attempt"], + cycle=j["cycle"], + fence=j["fence"], + state=j["state"], + publications=e["publications"], + variant_rows=len(e["variants"]), + physical_keys=len(e["physical_keys"]), + ) + ) + a.output.mkdir(parents=True, exist_ok=True) + with (a.output / "effects.csv").open("w") as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0])) + writer.writeheader() + writer.writerows(rows) + ready = [r for r in rows if r["state"] == "ready"] + assert all( + r["publications"] == 1 + and r["variant_rows"] == 4 + and r["physical_keys"] == 4 + for r in ready + ) + stale = [r for r in rows if r["state"] == "obsolete"] + assert all(r["publications"] == 0 for r in stale) + samples = [ + x["detail"] + for x in d["worker_metrics"] + if x["detail"].get("result") == "ready" + and "rss_peak_mib" in x["detail"] + ] + stats = { + "source_commit": d["source_commit"], + "cases": len(d["cases"]), + "observed_ready_effects": len(ready), + "observed_obsolete_effects": len(stale), + "all_ready_have_one_publication_and_four_variants": True, + "all_obsolete_have_zero_publications": True, + "successful_child_samples": len(samples), + "child_elapsed_range_seconds": [ + min(x["seconds"] for x in samples), + max(x["seconds"] for x in samples), + ], + "child_cpu_range_seconds": [ + min(x["cpu_seconds"] for x in samples), + max(x["cpu_seconds"] for x in samples), + ], + "child_peak_rss_range_mib": [ + min(x["rss_peak_mib"] for x in samples), + max(x["rss_peak_mib"] for x in samples), + ], + "scope": "process self metrics, no queue lag or throughput claim; fault-free and retry children only where a result was returned", + } + (a.output / "summary.json").write_text(json.dumps(stats, indent=2) + "\n") + fig, axes = plt.subplots(1, 2, figsize=(13, 5), layout="constrained") + axes[0].bar( + range(1, len(rows) + 1), + [r["publications"] for r in rows], + color=[ + "#246b8e" if r["state"] == "ready" else "#d17d20" for r in rows + ], + ) + axes[0].axhline(1, color="#5b6870", linestyle="--", linewidth=1) + axes[0].set( + xlabel="Effect observation (CSV row)", + ylabel="Committed publications", + title="One publication per ready version; stale v1 = 0", + yticks=[0, 1], + ) + axes[1].scatter( + [x["seconds"] for x in samples], + [x["rss_peak_mib"] for x in samples], + color="#246b8e", + s=35, + alpha=0.8, + ) + axes[1].set( + xlabel="Child elapsed, seconds", + ylabel="Child peak RSS, MiB", + title=f"{len(samples)} completed child samples (synthetic320×200)", + ) + for ax in axes: + ax.grid(axis="y", alpha=0.2) + fig.savefig(a.output / "pipeline-results.png", dpi=150) + plt.close(fig) + print(json.dumps(stats, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_pipeline.py b/scripts/verify_pipeline.py index 6f850068..c92da95d 100644 --- a/scripts/verify_pipeline.py +++ b/scripts/verify_pipeline.py @@ -100,6 +100,9 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument("--output", default="evidence/pipeline-repeat.json") parser.add_argument("--repeats", type=int, default=2) + parser.add_argument( + "--start-at", choices=["all", "versions"], default="all" + ) args = parser.parse_args() output = ROOT / args.output if output.exists(): @@ -108,6 +111,7 @@ def main(): "source_commit": command(["git", "rev-parse", "HEAD"]), "date": datetime.now(timezone.utc).isoformat(), "track": 2, + "scope": args.start_at, "complete": False, "cases": [], } @@ -234,256 +238,330 @@ def main(): ) return effect - # Source-only crash before the DB handoff must never be reported as accepted. - stop("worker") - stop("relay") - marker = inject(chat, "api_after_staging", "wait") - before = inside( - f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))" - ) - pool = ThreadPoolExecutor(max_workers=1) - future = pool.submit( - lambda: client.post( - f"/api/v1/chats/{chat}/attachments", - headers={**h, "Content-Type": "image/png"}, - params={"filename": "unconfirmed.png"}, - content=raw, + if args.start_at == "all": + # Source-only crash before the DB handoff must never be reported as accepted. + stop("worker") + stop("relay") + marker = inject(chat, "api_after_staging", "wait") + before = inside( + f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))" ) - ) - wait(lambda marker=marker: hit(marker)) - compose("kill", "-s", "SIGKILL", "api") - try: - response = future.result(15) - assert response.status_code != 202 - except httpx.HTTPError: - pass - pool.shutdown() - start("api") - ready() - after = inside( - f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))" - ) - assert before == after - save( - "staging_before_handoff_crash", - { - "no_202": True, - "no_attachment_or_job_created": True, - "orphan_retained_for_cleanup": True, - }, - ) - # API durable commit/outbox survives restart without a broker or worker. - a, j = upload() - linked = client.post( - f"/api/v1/chats/{chat}/messages", - headers={**h, "Idempotency-Key": uuid4().hex}, - json={"text": "", "attachment_ids": [a["id"]]}, - ) - assert linked.status_code == 201 - assert ( - client.get( - f"/api/v1/attachments/{a['id']}/content", headers=h - ).status_code - == 409 - ) - compose("restart", "api") - ready() - compose("kill", "-s", "SIGKILL", "broker") - start("broker") - start("relay") - start("worker") - effect = confirm(a, j) - save( - "commit_before_publish_and_worker_stopped", - { - "attachment": a["id"], - "effect": effect, - "queued_link_allowed": True, - "api_and_broker_restarted": True, - }, - ) - # Broker-volume survival with a confirmed, still-unconsumed stream entry. - stop("worker") - a, j = upload() - wait( - lambda: inside( - f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NOT NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']>0))" + pool = ThreadPoolExecutor(max_workers=1) + future = pool.submit( + lambda: client.post( + f"/api/v1/chats/{chat}/attachments", + headers={**h, "Content-Type": "image/png"}, + params={"filename": "unconfirmed.png"}, + content=raw, + ) ) - ) - stop("relay") - ids = inside("print(json.dumps(queue.xrange(STREAM)))") - compose("kill", "-s", "SIGKILL", "broker") - start("broker") - wait(lambda: inside("print(json.dumps(queue.ping()))")) - restored = inside("print(json.dumps(queue.xrange(STREAM)))") - assert restored == ids, {"before": ids, "after": restored} - start("relay") - start("worker") - save( - "confirmed_broker_entry_survives_sigkill", - { - "effect": confirm(a, j), - "same_stream_entries": True, - "entry_count": len(ids), - }, - ) - # Confirm succeeded, PG marker did not: relay recovery may publish a duplicate. - stop("worker") - stop("relay") - a, j = upload() - marker = inject(j, "relay_after_publish", "crash") - start("relay") - wait(lambda marker=marker: hit(marker)) - time.sleep(0.5) - unpublished = inside( - f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']))" - ) - assert unpublished >= 1 - start("relay") - start("worker") - save( - "relay_crash_after_xadd_before_pg_marker", - { - "unpublished_after_crash": unpublished, - "effect": confirm(a, j), - }, - ) - # Repeated before/after effect and commit/ack windows. - for repeat in range(args.repeats): - for phase in ["before_effect", "after_effect", "after_commit"]: - stop("worker") - a, j = upload() - marker = inject(j, phase, "wait") - start("worker") - wait(lambda marker=marker: hit(marker)) - at_crash = inspect_effect(j) - assert at_crash["publications"] == ( - 1 if phase == "after_commit" else 0 + wait(lambda marker=marker: hit(marker)) + compose("kill", "-s", "SIGKILL", "api") + try: + response = future.result(15) + assert response.status_code != 202 + except httpx.HTTPError: + pass + pool.shutdown() + start("api") + ready() + after = inside( + f"with db.connection() as c: n=c.execute('SELECT count(*) n FROM attachments WHERE chat_id=%s',({chat!r},)).fetchone()['n']\nprint(json.dumps(n))" + ) + assert before == after + save( + "staging_before_handoff_crash", + { + "no_202": True, + "no_attachment_or_job_created": True, + "orphan_retained_for_cleanup": True, + }, + ) + # API durable commit/outbox survives restart without a broker or worker. + stop("broker") + a, j = upload() + linked = client.post( + f"/api/v1/chats/{chat}/messages", + headers={**h, "Idempotency-Key": uuid4().hex}, + json={"text": "", "attachment_ids": [a["id"]]}, + ) + assert linked.status_code == 201 + assert ( + client.get( + f"/api/v1/attachments/{a['id']}/content", headers=h + ).status_code + == 409 + ) + compose("restart", "api") + ready() + start("broker") + start("relay") + start("worker") + effect = confirm(a, j) + save( + "commit_before_publish_and_worker_stopped", + { + "attachment": a["id"], + "effect": effect, + "queued_link_allowed": True, + "api_and_broker_restarted": True, + "broker_stopped_during_upload": True, + }, + ) + # Broker-volume survival with a confirmed, still-unconsumed stream entry. + stop("worker") + a, j = upload() + wait( + lambda: inside( + f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NOT NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']>0))" ) - assert len(at_crash["physical_keys"]) == ( - 0 if phase == "before_effect" else 4 - ) - compose("kill", "-s", "SIGKILL", "worker") - start("worker") - effect = confirm(a, j) - save( - f"{phase}_sigkill_repeat_{repeat + 1}", - {"at_crash": at_crash, "effect": effect}, - ) - stop("worker") - a, j = upload() - inject(j, "before_effect", "transient") - start("worker") - effect = confirm(a, j) - assert effect["job"]["attempt"] == 2 - save("transient_retry_with_jitter", {"effect": effect}) - stop("worker") - a, j = upload() - inject(j, "before_effect", "hang") - start("worker") - effect = confirm(a, j) - assert any( - e["event"] == "retry" - and e["detail"]["code"] == "worker_timeout" - for e in effect["events"] - ) - save("hung_child_killed_and_retried", {"effect": effect}) - stop("worker") - a, j = upload() - inject(j, "before_effect", "permanent") - start("worker") - failed = until(a, "failed") - assert failed["error_code"] == "invalid_image" - assert ( - client.get( - f"/api/v1/attachments/{a['id']}/content", headers=h - ).status_code - == 409 - ) - r = client.post( - f"/api/v1/chats/{chat}/messages", - headers={**h, "Idempotency-Key": uuid4().hex}, - json={"text": "", "attachment_ids": [a["id"]]}, - ) - assert r.status_code == 409 - assert operator(j) == j - effect = confirm(a, j) - assert effect["job"]["cycle"] == 2 - try: + ) + stop("relay") + ids = inside("print(json.dumps(queue.xrange(STREAM)))") + compose("kill", "-s", "SIGKILL", "broker") + start("broker") + wait(lambda: inside("print(json.dumps(queue.ping()))")) + restored = inside("print(json.dumps(queue.xrange(STREAM)))") + assert restored == ids, {"before": ids, "after": restored} + start("relay") + start("worker") + save( + "confirmed_broker_entry_survives_sigkill", + { + "effect": confirm(a, j), + "same_stream_entries": True, + "entry_count": len(ids), + }, + ) + # Confirm succeeded, PG marker did not: relay recovery may publish a duplicate. + stop("worker") + stop("relay") + a, j = upload() + marker = inject(j, "relay_after_publish", "crash") + start("relay") + wait(lambda marker=marker: hit(marker)) + time.sleep(0.5) + unpublished = inside( + f"with db.connection() as c: r=c.execute('SELECT count(*) n FROM outbox WHERE job_id=%s AND published_at IS NULL',({j!r},)).fetchone()\nprint(json.dumps(r['n']))" + ) + assert unpublished >= 1 + start("relay") + start("worker") + save( + "relay_crash_after_xadd_before_pg_marker", + { + "unpublished_after_crash": unpublished, + "effect": confirm(a, j), + }, + ) + # Repeated before/after effect and commit/ack windows. + for repeat in range(args.repeats): + for phase in [ + "before_effect", + "after_effect", + "after_commit", + ]: + stop("worker") + a, j = upload() + marker = inject(j, phase, "wait") + start("worker") + wait(lambda marker=marker: hit(marker)) + at_crash = inspect_effect(j) + assert at_crash["job"]["state"] == ( + "ready" + if phase == "after_commit" + else "processing" + ) + assert at_crash["publications"] == ( + 1 if phase == "after_commit" else 0 + ) + assert len(at_crash["physical_keys"]) == ( + 0 if phase == "before_effect" else 4 + ) + compose("kill", "-s", "SIGKILL", "worker") + start("worker") + effect = confirm(a, j) + if phase == "after_commit": + wait( + lambda j=j: any( + e["event"] == "duplicate_decision" + and e["detail"]["state"] == "ready" + for e in inspect_effect(j)["events"] + ) + ) + wait( + lambda j=j: inside( + f"print(json.dumps(not any(json.loads(fields.get('job','{{}}')).get('job_id')=={j!r} for _,fields in queue.xrange(STREAM))))" + ) + ) + effect = inspect_effect(j) + else: + assert effect["job"]["attempt"] >= 2 + save( + f"{phase}_sigkill_repeat_{repeat + 1}", + {"at_crash": at_crash, "effect": effect}, + ) + stop("worker") + a, j = upload() + inject(j, "before_effect", "transient") + start("worker") + effect = confirm(a, j) + assert effect["job"]["attempt"] == 2 + save("transient_retry_with_jitter", {"effect": effect}) + stop("worker") + a, j = upload() + marker = inject(j, "before_effect", "hang") + start("worker") + wait(lambda: hit(marker)) + cid = compose("ps", "-q", "worker").split()[0] + code = """import json,pathlib,re + limits=[] + for p in pathlib.Path('/proc').glob('[0-9]*/limits'): + try: + line=next(x for x in p.read_text().splitlines() if x.startswith('Max address space')) + if re.search(r'805306368\\s+805306368',line): limits.append({'pid':p.parent.name,'limit':line}) + except (FileNotFoundError,PermissionError,StopIteration,ProcessLookupError): pass + root=pathlib.Path('/sys/fs/cgroup') + print(json.dumps({'bounded_children':limits,'cgroup':{n:(root/n).read_text().strip() for n in ['memory.max','memory.current','memory.peak','cpu.max','pids.current']}})) + """ + limits = json.loads( + command(["docker", "exec", cid, "python", "-c", code]) + ) + assert 1 <= len(limits["bounded_children"]) <= 2 + assert limits["cgroup"]["memory.max"] == "1610612736" + effect = confirm(a, j) + assert any( + e["event"] == "retry" + and e["detail"]["code"] == "worker_timeout" + for e in effect["events"] + ) + save( + "hung_child_killed_and_retried", + {"effect": effect, "observed_limits": limits}, + ) + stop("worker") + a, j = upload() + inject(j, "before_effect", "permanent") + start("worker") + failed = until(a, "failed") + assert failed["error_code"] == "invalid_image" + assert ( + client.get( + f"/api/v1/attachments/{a['id']}/content", headers=h + ).status_code + == 409 + ) + r = client.post( + f"/api/v1/chats/{chat}/messages", + headers={**h, "Idempotency-Key": uuid4().hex}, + json={"text": "", "attachment_ids": [a["id"]]}, + ) + assert r.status_code == 409 + assert operator(j) == j + effect = confirm(a, j) + assert effect["job"]["cycle"] == 2 + try: + operator(j) + except RuntimeError: + pass + else: + raise AssertionError("Replay of ready must be rejected") + save( + "permanent_failed_registry_and_audited_replay", + { + "failed": failed, + "effect": effect, + "ready_replay_rejected": True, + }, + ) + stop("worker") + a, j = upload() + inject(j, "before_effect", "transient", remaining=5) + start("worker") + failed = until(a, "failed", timeout=150) + assert ( + failed["error_code"] == "retry_exhausted" + and row(j)["attempt"] == 5 + ) operator(j) - except RuntimeError: - pass - else: - raise AssertionError("Replay of ready must be rejected") - save( - "permanent_failed_registry_and_audited_replay", - { - "failed": failed, - "effect": effect, - "ready_replay_rejected": True, - }, - ) - stop("worker") - a, j = upload() - inject(j, "before_effect", "transient", remaining=5) - start("worker") - failed = until(a, "failed", timeout=150) - assert ( - failed["error_code"] == "retry_exhausted" - and row(j)["attempt"] == 5 - ) - operator(j) - save( - "finite_retry_exhaustion_then_explicit_replay", - {"failed": failed, "effect": confirm(a, j)}, - ) - # Reverse notification order and add concurrent copies: one publication/key. - stop("worker") - stop("relay") - a, j = upload() - b, k = upload() - duplicate(k, 8) - duplicate(j, 8) - start("worker") - start("relay") - ea = confirm(a, j) - confirm(b, k) - duplicate(j, 10) - duplicate(k, 10) - wait( - lambda: any( - e["event"] == "duplicate_decision" - for e in inspect_effect(j)["events"] + save( + "finite_retry_exhaustion_then_explicit_replay", + {"failed": failed, "effect": confirm(a, j)}, + ) + # Reverse notification order and add concurrent copies: one publication/key. + stop("worker") + stop("relay") + a, j = upload() + b, k = upload() + duplicate(k, 8) + duplicate(j, 8) + start("worker") + start("relay") + ea = confirm(a, j) + confirm(b, k) + duplicate(j, 10) + duplicate(k, 10) + wait( + lambda: any( + e["event"] == "duplicate_decision" + for e in inspect_effect(j)["events"] + ) + ) + assert ( + inspect_effect(j)["job"]["attempt"] == ea["job"]["attempt"] + ) + save( + "reordered_and_duplicate_notifications", + { + "effects": [inspect_effect(j), inspect_effect(k)], + "added_duplicates": 36, + "no_extra_attempt_after_ready": True, + }, ) - ) - assert inspect_effect(j)["job"]["attempt"] == ea["job"]["attempt"] - save( - "reordered_and_duplicate_notifications", - { - "effects": [inspect_effect(j), inspect_effect(k)], - "added_duplicates": 36, - "no_extra_attempt_after_ready": True, - }, - ) # Hold v1 after its PUTs, complete v2 on the other bounded slot, release old CAS. for repeat in range(args.repeats): stop("worker") a, j = upload() marker = inject(j, "after_effect", "wait") - start("worker") - wait(lambda marker=marker: hit(marker)) - k = operator(j, version=2) - v2 = confirm(a, k) - hashes = {v["name"]: v["sha256"] for v in v2["variants"]} - inside( - f"with db.connection() as c: c.execute('UPDATE pipeline_faults SET released_at=clock_timestamp() WHERE id=%s',({marker},))\nprint('true')" - ) - wait( - lambda j=j: any( - e["event"] == "stale_completion_rejected" - for e in inspect_effect(j)["events"] + # Keep one controller process hot throughout the narrow stale-writer window. + code = f"""from src.pipeline import event +with db.connection() as c: event(c,UUID({j!r}),'version_controller_waiting') +end=time.monotonic()+45 +while True: + with db.connection() as c: hit=c.execute('SELECT hit_at FROM pipeline_faults WHERE id=%s',({marker},)).fetchone()['hit_at'] + if hit: break + if time.monotonic()>end: raise TimeoutError('v1 hook never hit') + time.sleep(.05) +k=change(db,store,job_id=UUID({j!r}),operator='synthetic-verifier',reason='Held stale writer race',version=2) +end=time.monotonic()+10 +while True: + with db.connection() as c: ready=c.execute('SELECT state FROM jobs WHERE id=%s',(k,)).fetchone()['state']=='ready' + if ready: break + if time.monotonic()>end: raise TimeoutError('v2 did not finish inside live v1 budget') + time.sleep(.05) +with db.connection() as c: + hashes=c.execute('SELECT name,sha256 FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=2 ORDER BY name',({a["id"]!r},)).fetchall() + c.execute('UPDATE pipeline_faults SET released_at=clock_timestamp() WHERE id=%s',({marker},)) +end=time.monotonic()+10 +while True: + with db.connection() as c: stale=c.execute("SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='stale_completion_rejected'",({j!r},)).fetchone()['n'] + if stale: break + if time.monotonic()>end: raise TimeoutError('Old completion was not rejected') + time.sleep(.05) +print(json.dumps({{'job_id':k,'hashes':{{r['name']:r['sha256'] for r in hashes}}}})) +""" + with ThreadPoolExecutor(max_workers=1) as hot: + future = hot.submit(inside, code) + wait( + lambda j=j: inside( + f"with db.connection() as c: n=c.execute(\"SELECT count(*) n FROM pipeline_events WHERE job_id=%s AND event='version_controller_waiting'\",({j!r},)).fetchone()['n']\nprint(json.dumps(n>0))" + ) ) - ) + start("worker") + race = future.result(timeout=45) + k = race["job_id"] + hashes = race["hashes"] + confirm(a, k) duplicate(j, 12) wait( lambda j=j: any( @@ -597,7 +675,7 @@ print(json.dumps({{'anonymous':anonymous,'iam':denials}}))""") output.write_text( json.dumps(report, ensure_ascii=False, indent=2) + "\n" ) - print("PASS: complete versioned pipeline fault matrix", flush=True) + print("PASS: versioned pipeline scope=" + args.start_at, flush=True) if __name__ == "__main__": diff --git a/student-unit/test_job_envelope.py b/student-unit/test_job_envelope.py index 6f3444e9..895dfb52 100644 --- a/student-unit/test_job_envelope.py +++ b/student-unit/test_job_envelope.py @@ -66,3 +66,54 @@ def test_invalid_notification_is_rejected(field, value): data[field] = value with pytest.raises((ValueError, TypeError)): validate_job(data) + + +def test_worker_rejects_oversized_pixels_before_decode(): + import hashlib + import io + + from PIL import Image + + from src.pipeline import PermanentFailure + from src.worker import make_variants + + stream = io.BytesIO() + Image.new("1", (8000, 5001)).save(stream, format="PNG") + raw = stream.getvalue() + row = dict( + size_bytes=len(raw), + sha256=hashlib.sha256(raw).hexdigest(), + content_type="image/png", + pipeline_version=1, + ) + with ( + pytest.warns(Image.DecompressionBombWarning), + pytest.raises(PermanentFailure, match="pixel_limit"), + ): + list(make_variants(raw, row)) + + +def test_worker_rejects_changed_source_checksum(): + from src.pipeline import PermanentFailure + from src.worker import make_variants + + with pytest.raises(PermanentFailure, match="source_checksum_mismatch"): + list(make_variants(b"changed", dict(size_bytes=7, sha256="0" * 64))) + + +def test_worker_rejects_invalid_image_as_permanent(): + import hashlib + + from src.pipeline import PermanentFailure + from src.worker import make_variants + + raw = b"not an image" + with pytest.raises(PermanentFailure, match="invalid_image"): + list( + make_variants( + raw, + dict( + size_bytes=len(raw), sha256=hashlib.sha256(raw).hexdigest() + ), + ) + ) -- 2.54.0 From ef619490f23877adbce37c9012496d01128bae4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Sun, 15 Nov 2026 23:13:14 +0300 Subject: [PATCH 07/11] test(cluster): observe confirmed writes after longer rolling actions --- .github/pull_request_template.md | 31 +- Makefile | 4 +- REPORT.md | 43 + evidence/cluster-first-failed.json | 1839 +++++++++++++++++++++++++ evidence/cluster-first-failure.txt | 20 + evidence/gc-first.json | 209 +++ evidence/gc-seed-first.json | 20 + evidence/pipeline-versions-first.json | 980 +++++++++++++ research/analyze_pipeline.py | 4 +- scripts/verify_cluster.py | 7 +- 10 files changed, 3138 insertions(+), 19 deletions(-) create mode 100644 evidence/cluster-first-failed.json create mode 100644 evidence/cluster-first-failure.txt create mode 100644 evidence/gc-first.json create mode 100644 evidence/gc-seed-first.json create mode 100644 evidence/pipeline-versions-first.json diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a4..2a2dcabd 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,19 @@ -## Лабораторная 5 +## Лабораторная5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Маслов, БВТ2301. Цель5, трек2: идемпотентность versioned image pipeline. +Собственная ЛР4 перенесена с объявленным переходом upload201→202queued. +Реализованы durable private S3 staging/reservation, atomic PG job+outbox, +persistent Valkey Streams/AOF/WAITAOF, relay/watchdog, отдельный bounded worker, +4variants, version/fence/live-lease CAS, finite retry/jitter/failed registry, +audited replay/newversion и safe retention cleanup. -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Контракт и публичные tests сохранены. Запуск, схема, fault methodology, +результаты и ответы защиты — REPORT.md/SCHEMA.md.112HTTP и22unit прошли; +полный clean-clone repeat будет внесён перед сдачей. Промежуточные failed +harness runs сохранены и не выданы за полный PASS. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Ограничения: одинhost/SPOFs, нет production TLS/backup/global admission quota; +HEAD не periodic byte audit; at-least-once с идемпотентным эффектом, +не exactly-once delivery; удалённый CI не подтверждён. +При разработке кода, проверок и отчёта использована помощь ИИ. Источники — +teacher contracts, собственная ЛР4, pinned libraries и официальные Valkey/Pillow docs. diff --git a/Makefile b/Makefile index 6e04c7dd..1edbd7ad 100644 --- a/Makefile +++ b/Makefile @@ -39,8 +39,8 @@ up: bootstrap down: docker compose down lint: - $(UV) run --locked ruff format --check src student-tests student-unit scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py) - $(UV) run --locked ruff check src student-tests student-unit scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py) + $(UV) run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py) + $(UV) run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py $(wildcard scripts/verify_*.py) test-student: BASE_URL="$(BASE_URL)" $(UV) run --locked pytest student-tests verify: diff --git a/REPORT.md b/REPORT.md index 7c635047..8c0318b2 100644 --- a/REPORT.md +++ b/REPORT.md @@ -243,3 +243,46 @@ MAX_UPLOAD_BYTES/MAX_IMAGE_PIXELS и idempotency TTL документируют контракт реализации: s3:9000/us-east-1/messenger,10MiB/40M/24h. Их нельзя изменить одним редактированием env: соответствующие constants/SQL надо менять согласованно. Secret *_FILE paths заданы в compose; значения создаёт bootstrap. + +## Карта критериев и промежуточные результаты + +| Уровень / требование | Реализация / автоматическая проверка | +| --- | --- | +| 3: минимум ЛР4 +202queued | API/ACL/byte/pixel/MIME guards;112 HTTP tests, public smoke | +| 3: durable broker и отдельный worker | persistent Valkey Streams/AOF/WAITAOF; broker SIGKILL с volume | +| 3:4states, original/thumbnail, retry | src/worker.py, schema-valid metadata; failed/retry scenarios | +| 3:worker stop→queued link→restart→ready | commit_before_publish_and_worker_stopped | +| 4:crop/optimized/metadata removal | HTTP geometry/EXIF/alpha +22 unit tests | +| 4:outbox gap и publisher crash | API COMMIT-before-publish и relay XADD-before-PG-marker | +| 4:finite backoff/permanent/transient/replay |5attempt exhaustion, timeout, failed registry, audit, ready replay rejection | +| 4:crash после PUT доACK, lease, bounded slots | before_effect/after_effect/after_commit ×2; live state/actual limits проверяются в полном повторе | +| 4:cleanup после фиксации/retention | verify_cleanup seed/verify, реальныйS3возраст, dry/apply/repeat | +| 5:versioned idempotency track | duplicates/reorder и held stalev1→v2×2, no old publication/hash change | + +[evidence/http-first.json](evidence/http-first.json):112 HTTP PASS404.42s, +source/runtime bb4e8a2. Отдельные22unit PASS после добавления real pixel, +checksum и malformed image worker tests. Public smoke PASS. + +Первые полные matrix runs сохранены честно как незавершённые: +[pipeline-first-failed.json](evidence/pipeline-first-failed.json) —2cases, +затем тест сравнил очередь при работающем relay; snapshot изолирован остановкой +relay, повтор подтвердил сохранность exact entries после broker SIGKILL. +[pipeline-second-failed.json](evidence/pipeline-second-failed.json) —15cases +прошли, затем отдельные controller invocations съели12s бюджет старого child. +V2 была ready, но parent успел убитьv1; это ещё не доказательство CAS. + +Новая координация использует один заранее запущенный controller: ждёт phase, +создаётv2, ждёт ready, отпускаетv1 и видит stale_completion_rejected внутри +живого окна. [pipeline-versions-first.json](evidence/pipeline-versions-first.json) +— **scope=versions**,4cases PASS:2version races, malformed notification, +private/IAM/otherAPI. Это целевой повтор, а не19cases полного matrix. +Приложение в этих исправлениях harness не менялось; полный clean clone обязан +завершить scope=all, включая более строгие ACK/live-state/resource assertions. + +[evidence/gc-first.json](evidence/gc-first.json): реальный возраст orphan +1279.103s (выше300s); dry-run сохранил bytes, apply удалил expired objects, +повтор не изменил ничего. Active/current/fresh и неистёкший reservation +защищены; source_expired replay отвергнут; expired reservation больше не +проходит handoff predicate. Fixture metadata failed состарены SQL отдельно, +но возраст S3 LastModified не подменялся. Cleanup fixtures синтетические и +не обещают восстановление bytes из checksum. diff --git a/evidence/cluster-first-failed.json b/evidence/cluster-first-failed.json new file mode 100644 index 00000000..f13f4058 --- /dev/null +++ b/evidence/cluster-first-failed.json @@ -0,0 +1,1839 @@ +{ + "date": "2026-11-15T20:08:42.652330+00:00", + "source_commit": "972cef8f7f3938913beb56d32551a88f5e40bc88", + "complete": false, + "before": [ + { + "role": "api", + "container": "0c7b704276e3", + "pid": 75153, + "started_at": "2026-11-15T19:50:21.154879093Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "role": "api", + "container": "a90645b411d6", + "pid": 22120, + "started_at": "2026-11-15T20:06:45.480384299Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "role": "broker", + "container": "5d3ebf6f87ff", + "pid": 77503, + "started_at": "2026-11-15T19:51:03.049505001Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "6ad17613e620", + "pid": 60587, + "started_at": "2026-11-15T19:17:14.204239507Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "c6fb99c3ddbc", + "pid": 60326, + "started_at": "2026-11-15T19:45:43.876610632Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "c60ecc6c94f4", + "pid": 63147, + "started_at": "2026-11-15T19:17:36.952605129Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8085" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "76efa23ba942", + "pid": 60650, + "started_at": "2026-11-15T19:17:14.476492674Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "a990e3899324", + "pid": 27505, + "started_at": "2026-11-15T20:08:42.302428089Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "c8273b4552c9", + "pid": 98443, + "started_at": "2026-11-15T19:58:04.386027252Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "de0c05e2e665", + "pid": 60843, + "started_at": "2026-11-15T19:17:15.006133424Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "a89e2cf493d9", + "pid": 18369, + "started_at": "2026-11-15T20:05:23.317701469Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ], + "parallel_replay": { + "requests": 30, + "messages": 1, + "instances": [ + "0024aba8755f4f3d92a85c4dbb751f5a", + "ee20d98fc7ac4ef08a41c2bc8fe82534" + ] + }, + "atomic_rollback_and_5xx_retry": true, + "expired_key_can_be_reused": true, + "failovers": [ + { + "confirmed_writes": 212, + "successful_reads": 200, + "errors": 13, + "successful_rps": 27.28, + "first_success_after_action_seconds": 5.028, + "action": { + "finished": 3.996 + }, + "observations": [ + { + "at": 0.008, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.067, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.124, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.205, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.265, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.321, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.392, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.46, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.517, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.576, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.639, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.701, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.759, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.816, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.871, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.936, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 0.996, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.063, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.134, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.192, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.25, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.312, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.37, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.426, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.495, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.557, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.623, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.685, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.745, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.809, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.871, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 1.933, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.035, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.138, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.236, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.337, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.456, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.519, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.577, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.64, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.701, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.795, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.895, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 2.995, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 3.101, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 3.196, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 3.252, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + }, + { + "at": 3.322, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.388, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.445, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.506, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.563, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.616, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.667, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.72, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.778, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.831, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.888, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 3.95, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.028, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.092, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.149, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.209, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.269, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.329, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.389, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.454, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.515, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.581, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.641, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.703, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.76, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.817, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.881, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 5.94, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.005, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.073, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.138, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.201, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.268, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.33, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.392, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.45, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.512, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.582, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.639, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.697, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.757, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.847, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 6.944, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.049, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.147, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.246, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.348, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.443, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.503, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.563, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.624, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.686, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.748, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.81, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.872, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 7.933, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.0, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.063, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.134, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.196, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.264, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.33, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.401, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.464, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.532, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.604, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.669, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.732, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.795, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.865, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.928, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 8.99, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.078, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.146, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.208, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.27, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.334, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.4, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.465, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.527, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.594, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.661, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.724, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.783, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.844, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.903, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 9.968, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.032, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.1, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.159, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.22, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.289, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.355, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.424, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.493, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.56, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.634, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.714, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.774, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.838, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.905, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 10.973, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.035, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.1, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.172, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.233, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.294, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.354, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.415, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.479, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.541, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.603, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.662, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.728, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.791, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.849, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.907, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 11.966, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.03, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.103, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.169, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.236, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.303, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.372, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.436, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.501, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.566, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.643, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.712, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.78, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.847, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.915, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 12.981, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.049, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.119, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.188, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.249, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.31, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.37, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.437, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.499, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.559, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.62, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.682, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.742, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.801, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.858, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.915, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 13.973, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.033, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.093, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.152, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.216, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.277, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.345, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.41, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.475, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.538, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.601, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.667, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.737, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.812, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.876, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 14.941, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 15.008, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "0c7b704276e3", + "pid": 75153, + "started_at": "2026-11-15T19:50:21.154879093Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "0024aba8755f4f3d92a85c4dbb751f5a" + } + }, + { + "confirmed_writes": 204, + "successful_reads": 191, + "errors": 14, + "successful_rps": 26.15, + "first_success_after_action_seconds": 4.344, + "action": { + "finished": 3.311 + }, + "observations": [ + { + "at": 0.021, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.091, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.157, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.216, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.276, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.343, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.433, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.554, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.628, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.717, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.784, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.864, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 0.94, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.011, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.106, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.178, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.237, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.341, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.436, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.54, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.647, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.748, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.843, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.904, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 1.978, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 2.082, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 2.179, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 2.282, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 2.381, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 2.484, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 2.543, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + }, + { + "at": 2.606, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 2.667, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 2.725, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 2.787, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 2.844, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 2.899, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 2.953, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 3.011, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 3.063, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 3.113, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 3.165, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 3.216, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 3.268, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.344, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.409, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.477, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.552, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.628, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.687, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.744, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.798, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.856, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.918, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 4.983, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.045, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.106, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.182, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.262, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.329, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.389, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.453, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.521, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.59, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.651, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.713, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.825, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.891, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 5.975, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.034, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.106, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.171, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.232, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.288, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.347, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.411, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.481, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.545, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.611, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.686, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.758, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.817, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.878, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 6.943, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.007, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.064, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.119, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.18, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.263, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.329, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.386, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.442, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.501, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.558, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.617, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.685, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.755, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.82, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.885, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 7.945, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.007, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.072, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.139, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.202, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.269, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.333, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.392, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.451, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.515, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.574, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.644, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.709, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.775, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.842, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.9, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 8.96, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.034, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.093, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.156, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.219, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.3, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.491, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.594, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.693, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.798, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.887, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 9.991, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.089, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.152, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.213, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.271, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.333, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.397, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.454, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.522, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.591, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.658, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.728, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.797, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.864, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.924, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 10.986, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.041, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.101, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.16, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.217, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.276, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.356, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.432, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.492, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.556, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.621, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.691, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.755, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.82, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.894, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 11.956, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.016, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.076, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.137, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.198, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.258, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.327, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.392, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.46, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.539, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.604, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.669, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.737, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.81, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.873, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 12.938, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.003, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.061, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.122, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.184, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.25, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.307, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.367, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.44, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.506, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.568, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.634, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.697, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.765, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.831, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.903, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 13.978, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.04, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.101, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.16, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.222, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.285, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.355, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.415, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.472, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.537, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.6, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.664, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.73, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.797, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.862, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.93, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + }, + { + "at": 14.998, + "instance": "4dd2c4628eb443f2b562bc8800b39dc0" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "a90645b411d6", + "pid": 22120, + "started_at": "2026-11-15T20:06:45.480384299Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "ee20d98fc7ac4ef08a41c2bc8fe82534" + } + } + ] +} diff --git a/evidence/cluster-first-failure.txt b/evidence/cluster-first-failure.txt new file mode 100644 index 00000000..bb88731c --- /dev/null +++ b/evidence/cluster-first-failure.txt @@ -0,0 +1,20 @@ +Traceback (most recent call last): + File "", line 203, in _run_module_as_main + File "", line 88, in _run_code + File "/scripts/verify_cluster.py", line 342, in + main() + ~~~~^^ + File "/scripts/verify_cluster.py", line 278, in main + report["rolling"] = exercise( + ~~~~~~~~^ + client, + ^^^^^^^ + ...<3 lines>... + seconds=45, + ^^^^^^^^^^^ + ) + ^ + File "/scripts/verify_cluster.py", line 114, in exercise + assert recovery is not None and (seconds > 20 or recovery <= 10) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +AssertionError diff --git a/evidence/gc-first.json b/evidence/gc-first.json new file mode 100644 index 00000000..fed9a9e6 --- /dev/null +++ b/evidence/gc-first.json @@ -0,0 +1,209 @@ +{ + "source_commit": "972cef8f7f3938913beb56d32551a88f5e40bc88", + "seed": "gc-seed-first.json", + "complete": true, + "real_s3_age_seconds": 1279.102705, + "dry_run": { + "apply": false, + "retention_seconds": 300, + "actions": [ + { + "action": "delete_expired_object", + "key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/crop", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/optimized", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/thumbnail", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/crop", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/optimized", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/thumbnail", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/4d806c50-751a-4ea0-b4e0-06fd832a82ca/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/667404a0-d3d8-4d0e-b1b9-b362cea129a7/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/8f12140f-f650-415e-8881-e5707ed58127/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/original", + "applied": false + } + ] + }, + "apply": { + "apply": true, + "retention_seconds": 300, + "actions": [ + { + "action": "delete_expired_object", + "key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/crop", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/optimized", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/7c6169d7-dd48-4cce-9e16-b994e1dbb037/v1/thumbnail", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/crop", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/optimized", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/thumbnail", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/4d806c50-751a-4ea0-b4e0-06fd832a82ca/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/667404a0-d3d8-4d0e-b1b9-b362cea129a7/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/8f12140f-f650-415e-8881-e5707ed58127/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/cd130b51-0b2c-4285-a6a3-89ddfc3a4bd8/original", + "applied": true + } + ] + }, + "repeat": { + "apply": true, + "retention_seconds": 300, + "actions": [] + }, + "before": { + "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan": true, + "objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original": true, + "objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/original": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/thumbnail": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/crop": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/optimized": true, + "objects/staging/69150761-a8a1-437b-91b0-b363d7a1507a/original": true, + "objects/staging/ec7a6893-8d3b-4488-b0cd-06e53499cf9d/original": true, + "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/fresh-orphan": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/original": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/thumbnail": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/crop": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/optimized": true + }, + "after": { + "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan": false, + "objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original": false, + "objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original": false, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/original": false, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/thumbnail": false, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/crop": false, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v1/optimized": false, + "objects/staging/69150761-a8a1-437b-91b0-b363d7a1507a/original": true, + "objects/staging/ec7a6893-8d3b-4488-b0cd-06e53499cf9d/original": true, + "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/fresh-orphan": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/original": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/thumbnail": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/crop": true, + "objects/d6f10e70-74e9-4394-be76-5c0b14d3f91f/v2/optimized": true + }, + "dry_run_unchanged": true, + "active_current_and_fresh_protected": true, + "expired_source_replay_refused": true, + "expired_writer_fenced": true +} diff --git a/evidence/gc-seed-first.json b/evidence/gc-seed-first.json new file mode 100644 index 00000000..229ed850 --- /dev/null +++ b/evidence/gc-seed-first.json @@ -0,0 +1,20 @@ +{ + "seed_id": "ae4b883f7b784f40ae8f5e1c78ac3091", + "source_commit": "a275cf2632d3bf3d96409408d9cb313cdb2824c2", + "date": "2026-11-15T19:46:23.318659+00:00", + "ready_attachment": "d6f10e70-74e9-4394-be76-5c0b14d3f91f", + "ready_job": "c0936959-f7ec-4fa2-a9ec-ee0de8d58d35", + "chat": "3428fad7-2178-4730-b56a-4b579bdd4966", + "owner": "36007eaa-ceba-4f80-98fa-13c912f73873", + "version2_job": "cc18c75e-8886-4333-bae8-5da93d626c73", + "orphan": "objects/cleanup-ae4b883f7b784f40ae8f5e1c78ac3091/old-orphan", + "failed_id": "b98bc768-f36a-4615-9097-c93a78bfed05", + "active_id": "69150761-a8a1-437b-91b0-b363d7a1507a", + "reservation_id": "ec7a6893-8d3b-4488-b0cd-06e53499cf9d", + "expired_reservation_id": "402b3515-91ad-40e0-8e36-6627feb18be7", + "failed_source": "objects/staging/b98bc768-f36a-4615-9097-c93a78bfed05/original", + "active_source": "objects/staging/69150761-a8a1-437b-91b0-b363d7a1507a/original", + "reservation_key": "objects/staging/ec7a6893-8d3b-4488-b0cd-06e53499cf9d/original", + "expired_reservation_key": "objects/staging/402b3515-91ad-40e0-8e36-6627feb18be7/original", + "seeded_unix": 1794771995.922627 +} diff --git a/evidence/pipeline-versions-first.json b/evidence/pipeline-versions-first.json new file mode 100644 index 00000000..abecfe1f --- /dev/null +++ b/evidence/pipeline-versions-first.json @@ -0,0 +1,980 @@ +{ + "source_commit": "972cef8f7f3938913beb56d32551a88f5e40bc88", + "date": "2026-11-15T20:03:53.705513+00:00", + "track": 2, + "scope": "versions", + "complete": true, + "cases": [ + { + "case": "stale_v1_after_v2_repeat_1", + "v1": { + "job": { + "id": "51c2387d-8bc4-4f37-9ad9-b722464c1f46", + "attachment_id": "4567437f-228f-4cd6-be3b-0609664645cd", + "pipeline_version": 1, + "source_key": "objects/staging/4567437f-228f-4cd6-be3b-0609664645cd/original", + "state": "obsolete", + "attempt": 1, + "cycle": 1, + "fence": 2, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:03:59.131799+00:00", + "error_code": null, + "created_at": "2026-11-15 20:03:59.131799+00:00", + "updated_at": "2026-11-15 20:04:25.401001+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/4567437f-228f-4cd6-be3b-0609664645cd/v1/crop", + "objects/4567437f-228f-4cd6-be3b-0609664645cd/v1/optimized", + "objects/4567437f-228f-4cd6-be3b-0609664645cd/v1/original", + "objects/4567437f-228f-4cd6-be3b-0609664645cd/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794773039190-0", + "outbox": 43 + }, + "created_at": "2026-11-15 20:03:59.200524+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794773054245-0", + "outbox": 44 + }, + "created_at": "2026-11-15 20:04:14.251008+00:00" + }, + { + "event": "version_controller_waiting", + "detail": {}, + "created_at": "2026-11-15 20:04:18.270502+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:04:25.403107+00:00" + }, + { + "event": "stale_completion_rejected", + "detail": { + "fence": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:04:30.862701+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773039190-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:30.980982+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773054245-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:30.994675+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "stale" + }, + "created_at": "2026-11-15 20:04:31.083095+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079900-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.913386+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079902-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.914038+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079904-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.920903+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079905-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.922028+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079906-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.925707+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079908-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.926905+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079909-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.930906+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079910-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.931649+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079911-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.935325+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079912-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.935997+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079913-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.940184+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773079914-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:04:39.940743+00:00" + } + ] + }, + "v2": { + "job": { + "id": "49ab086a-9b0e-4185-a4a7-1c95c546aed9", + "attachment_id": "4567437f-228f-4cd6-be3b-0609664645cd", + "pipeline_version": 2, + "source_key": "objects/staging/4567437f-228f-4cd6-be3b-0609664645cd/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:04:28.320656+00:00", + "error_code": null, + "created_at": "2026-11-15 20:04:28.320656+00:00", + "updated_at": "2026-11-15 20:04:30.796918+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/optimized", + "sha256": "f2b2ece82b03105a787a0a9b75ce63787364fa39329cd6c21e23f44ec2ef2ced", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/crop", + "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/optimized", + "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/original", + "objects/4567437f-228f-4cd6-be3b-0609664645cd/v2/thumbnail" + ], + "events": [ + { + "event": "new_version", + "detail": { + "previous_job": "51c2387d-8bc4-4f37-9ad9-b722464c1f46" + }, + "created_at": "2026-11-15 20:04:28.323557+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794773068363-0", + "outbox": 45 + }, + "created_at": "2026-11-15 20:04:28.367667+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 2 + }, + "created_at": "2026-11-15 20:04:28.373092+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 2, + "variants": 4 + }, + "created_at": "2026-11-15 20:04:30.798523+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.29709320799975103, + "cpu_seconds": 0.24198554100000003, + "rss_peak_mib": 79.61328125 + }, + "created_at": "2026-11-15 20:04:30.965510+00:00" + } + ] + }, + "new_hashes_unchanged": true + }, + { + "case": "stale_v1_after_v2_repeat_2", + "v1": { + "job": { + "id": "8679296c-7a76-46b4-a6a4-02b57d0dedaf", + "attachment_id": "0a837770-bc8b-40c0-b155-a1d29956e589", + "pipeline_version": 1, + "source_key": "objects/staging/0a837770-bc8b-40c0-b155-a1d29956e589/original", + "state": "obsolete", + "attempt": 1, + "cycle": 1, + "fence": 2, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:05:03.917398+00:00", + "error_code": null, + "created_at": "2026-11-15 20:05:03.917399+00:00", + "updated_at": "2026-11-15 20:05:25.442589+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v1/crop", + "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v1/optimized", + "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v1/original", + "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794773103958-0", + "outbox": 46 + }, + "created_at": "2026-11-15 20:05:03.964545+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794773119171-0", + "outbox": 47 + }, + "created_at": "2026-11-15 20:05:19.176271+00:00" + }, + { + "event": "version_controller_waiting", + "detail": {}, + "created_at": "2026-11-15 20:05:22.667065+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:05:25.444410+00:00" + }, + { + "event": "stale_completion_rejected", + "detail": { + "fence": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:05:30.563412+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773103958-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:30.764786+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773119171-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:30.775876+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "stale" + }, + "created_at": "2026-11-15 20:05:30.815598+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139795-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.811610+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139792-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.811671+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139797-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.818733+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139798-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.819185+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139799-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.824008+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139800-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.824598+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139801-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.829008+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139803-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.829676+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139804-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.833617+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139805-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.834470+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139805-1", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.838465+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794773139806-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:05:39.839569+00:00" + } + ] + }, + "v2": { + "job": { + "id": "3c673acd-d8c5-44c7-8366-eed0480acdd8", + "attachment_id": "0a837770-bc8b-40c0-b155-a1d29956e589", + "pipeline_version": 2, + "source_key": "objects/staging/0a837770-bc8b-40c0-b155-a1d29956e589/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:05:28.099769+00:00", + "error_code": null, + "created_at": "2026-11-15 20:05:28.099769+00:00", + "updated_at": "2026-11-15 20:05:30.544288+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/optimized", + "sha256": "f2b2ece82b03105a787a0a9b75ce63787364fa39329cd6c21e23f44ec2ef2ced", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/crop", + "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/optimized", + "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/original", + "objects/0a837770-bc8b-40c0-b155-a1d29956e589/v2/thumbnail" + ], + "events": [ + { + "event": "new_version", + "detail": { + "previous_job": "8679296c-7a76-46b4-a6a4-02b57d0dedaf" + }, + "created_at": "2026-11-15 20:05:28.102749+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794773128278-0", + "outbox": 48 + }, + "created_at": "2026-11-15 20:05:28.288692+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 2 + }, + "created_at": "2026-11-15 20:05:28.307421+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 2, + "variants": 4 + }, + "created_at": "2026-11-15 20:05:30.544711+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.28706112500003655, + "cpu_seconds": 0.24108328799999978, + "rss_peak_mib": 79.671875 + }, + "created_at": "2026-11-15 20:05:30.752666+00:00" + } + ] + }, + "new_hashes_unchanged": true + }, + { + "case": "malformed_notification_rejected", + "durable_diagnostic": true + }, + { + "case": "private_iam_and_other_api_download", + "security": { + "anonymous": 403, + "iam": { + "create_bucket": "AccessDenied", + "write_outside_prefix": "AccessDenied", + "list_other_bucket": "AccessDenied" + } + }, + "attachment": "5875dfe8-09d9-4ded-9e31-a9329f0974ac", + "uploader": "c12dfe7a14a648809e07d709180d70e3", + "downloader": "0024aba8755f4f3d92a85c4dbb751f5a", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df" + } + ], + "worker_metrics": [ + { + "job_id": "947fae60-fb2b-4000-ac82-76c0aa946f17", + "detail": { + "result": "ready", + "seconds": 0.5540742509999745, + "cpu_seconds": 0.47041774699999994, + "rss_peak_mib": 79.33203125 + } + }, + { + "job_id": "ac292276-9fc0-4d20-b9d0-1c784017f555", + "detail": { + "result": "ready", + "seconds": 0.26326079199998276, + "cpu_seconds": 0.23446212599999994, + "rss_peak_mib": 79.6796875 + } + }, + { + "job_id": "4d1ee9b7-32e0-44c4-8264-a831b9bc3bdb", + "detail": { + "result": "ready", + "seconds": 0.2615961250003238, + "cpu_seconds": 0.22782362500000009, + "rss_peak_mib": 79.44140625 + } + }, + { + "job_id": "1174230c-fb4f-4cdc-9db8-896d002f90a4", + "detail": { + "result": "ready", + "seconds": 0.2803810010000234, + "cpu_seconds": 0.24079599299999987, + "rss_peak_mib": 79.34765625 + } + }, + { + "job_id": "4b28f16f-162d-4f60-b1fd-3a907ee834e1", + "detail": { + "result": "ready", + "seconds": 0.2567913339998995, + "cpu_seconds": 0.228462414, + "rss_peak_mib": 79.47265625 + } + }, + { + "job_id": "12fa59f7-ed8a-4810-95cf-8b9be4c020e8", + "detail": { + "result": "ready", + "seconds": 0.28690104199995403, + "cpu_seconds": 0.23826428900000018, + "rss_peak_mib": 79.3046875 + } + }, + { + "job_id": "6247813c-df74-4eeb-9326-cbb13f860336", + "detail": { + "result": "ready", + "seconds": 0.4403786249999939, + "cpu_seconds": 0.4114635, + "rss_peak_mib": 108.4140625 + } + }, + { + "job_id": "9a3fad2f-b3cc-4a28-b996-10c84a4c4b94", + "detail": { + "result": "ready", + "seconds": 0.2648974580001777, + "cpu_seconds": 0.23192129000000028, + "rss_peak_mib": 79.3515625 + } + }, + { + "job_id": "c0936959-f7ec-4fa2-a9ec-ee0de8d58d35", + "detail": { + "result": "ready", + "seconds": 0.30638329200019143, + "cpu_seconds": 0.27590770600000014, + "rss_peak_mib": 79.4296875 + } + }, + { + "job_id": "cc18c75e-8886-4333-bae8-5da93d626c73", + "detail": { + "result": "ready", + "seconds": 0.2798381249999693, + "cpu_seconds": 0.24821178899999996, + "rss_peak_mib": 79.6640625 + } + }, + { + "job_id": "1016e737-7ec1-4fde-9946-a342d70b2a71", + "detail": { + "result": "ready", + "seconds": 0.2740797910000765, + "cpu_seconds": 0.2354738329999997, + "rss_peak_mib": 79.3125 + } + }, + { + "job_id": "6e004e09-d91d-42b9-907b-9fda074afeb3", + "detail": { + "result": "ready", + "seconds": 0.2604502920003142, + "cpu_seconds": 0.22423492300000003, + "rss_peak_mib": 79.328125 + } + }, + { + "job_id": "817ac0d3-2ae3-4721-a1c4-0d517f624f3a", + "detail": { + "result": "ready", + "seconds": 0.2753011250001691, + "cpu_seconds": 0.24031520900000003, + "rss_peak_mib": 79.31640625 + } + }, + { + "job_id": "a68327bb-1d99-4ca1-a541-c197ad0683ea", + "detail": { + "result": "ready", + "seconds": 0.2575967509997099, + "cpu_seconds": 0.23045399499999997, + "rss_peak_mib": 79.84765625 + } + }, + { + "job_id": "2b40b034-2ea5-4612-8d4e-f099e1e3f65d", + "detail": { + "result": "ready", + "seconds": 0.2825338749998991, + "cpu_seconds": 0.24960387800000006, + "rss_peak_mib": 79.30859375 + } + }, + { + "job_id": "681235f5-558b-44b8-af12-e61d2f6c65bf", + "detail": { + "result": "ready", + "seconds": 0.2647618330001933, + "cpu_seconds": 0.23189429199999978, + "rss_peak_mib": 79.6796875 + } + }, + { + "job_id": "e4b36262-0688-4b3c-90ce-9252e9633353", + "detail": { + "result": "ready", + "seconds": 0.2914882079999188, + "cpu_seconds": 0.24316999600000022, + "rss_peak_mib": 79.80078125 + } + }, + { + "job_id": "e1ebdd4b-996f-41d2-aa54-246cc9f4c8d3", + "detail": { + "result": "ready", + "seconds": 0.259311875000094, + "cpu_seconds": 0.23085762499999984, + "rss_peak_mib": 79.45703125 + } + }, + { + "job_id": "fbedc7f8-97a6-4c59-848c-d4ab8045d4d2", + "detail": { + "result": "ready", + "seconds": 0.26234775000011723, + "cpu_seconds": 0.22806161800000013, + "rss_peak_mib": 79.328125 + } + }, + { + "job_id": "be6bd110-e0e9-4dbf-92cb-2ee8e18d407d", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "be6bd110-e0e9-4dbf-92cb-2ee8e18d407d", + "detail": { + "result": "ready", + "seconds": 0.30820225099978416, + "cpu_seconds": 0.27197879400000025, + "rss_peak_mib": 79.34765625 + } + }, + { + "job_id": "5d2588f5-72b5-41ce-95ce-c78327ceddeb", + "detail": { + "code": "worker_timeout", + "result": "transient" + } + }, + { + "job_id": "5d2588f5-72b5-41ce-95ce-c78327ceddeb", + "detail": { + "result": "ready", + "seconds": 0.26163899999983187, + "cpu_seconds": 0.22942266700000014, + "rss_peak_mib": 79.33203125 + } + }, + { + "job_id": "a9ce36d7-5c35-4b59-8f4e-a696a0a6b4c1", + "detail": { + "code": "invalid_image", + "result": "permanent" + } + }, + { + "job_id": "a9ce36d7-5c35-4b59-8f4e-a696a0a6b4c1", + "detail": { + "result": "ready", + "seconds": 0.27349454200020773, + "cpu_seconds": 0.23941921199999983, + "rss_peak_mib": 79.28125 + } + }, + { + "job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9", + "detail": { + "result": "ready", + "seconds": 0.2960601250001673, + "cpu_seconds": 0.26317221, + "rss_peak_mib": 79.25 + } + }, + { + "job_id": "20c34eb3-0582-4144-847c-fa037647f863", + "detail": { + "result": "ready", + "seconds": 0.3045327089998864, + "cpu_seconds": 0.2630334579999998, + "rss_peak_mib": 79.578125 + } + }, + { + "job_id": "c013ea06-4dea-4e65-b715-8a3f5976fa57", + "detail": { + "result": "ready", + "seconds": 0.3056550410001364, + "cpu_seconds": 0.26324616899999986, + "rss_peak_mib": 79.65234375 + } + }, + { + "job_id": "44de0dbe-f2b8-40bd-8b65-479e8f7243b5", + "detail": { + "code": "worker_timeout", + "result": "transient" + } + }, + { + "job_id": "fe00834b-4f3b-4d9b-bf47-c9cec70caac9", + "detail": { + "result": "ready", + "seconds": 0.26621149999982663, + "cpu_seconds": 0.23299866699999994, + "rss_peak_mib": 79.67578125 + } + }, + { + "job_id": "49ab086a-9b0e-4185-a4a7-1c95c546aed9", + "detail": { + "result": "ready", + "seconds": 0.29709320799975103, + "cpu_seconds": 0.24198554100000003, + "rss_peak_mib": 79.61328125 + } + }, + { + "job_id": "51c2387d-8bc4-4f37-9ad9-b722464c1f46", + "detail": { + "result": "stale" + } + }, + { + "job_id": "3c673acd-d8c5-44c7-8366-eed0480acdd8", + "detail": { + "result": "ready", + "seconds": 0.28706112500003655, + "cpu_seconds": 0.24108328799999978, + "rss_peak_mib": 79.671875 + } + }, + { + "job_id": "8679296c-7a76-46b4-a6a4-02b57d0dedaf", + "detail": { + "result": "stale" + } + }, + { + "job_id": "8eedc435-5509-4724-9a18-e478d920f101", + "detail": { + "result": "ready", + "seconds": 0.32150650000039604, + "cpu_seconds": 0.28338829200000015, + "rss_peak_mib": 79.37109375 + } + } + ], + "audit": [ + { + "job_id": "cc18c75e-8886-4333-bae8-5da93d626c73", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 19:46:27.434253+00:00" + }, + { + "job_id": "a9ce36d7-5c35-4b59-8f4e-a696a0a6b4c1", + "action": "replay", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 19:56:33.964020+00:00" + }, + { + "job_id": "3c76bbf8-cd39-4c10-8b99-23f7d4ccf0c9", + "action": "replay", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 19:57:37.397099+00:00" + }, + { + "job_id": "fe00834b-4f3b-4d9b-bf47-c9cec70caac9", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 19:59:07.104714+00:00" + }, + { + "job_id": "49ab086a-9b0e-4185-a4a7-1c95c546aed9", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Held stale writer race", + "created_at": "2026-11-15 20:04:28.322925+00:00" + }, + { + "job_id": "3c673acd-d8c5-44c7-8366-eed0480acdd8", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Held stale writer race", + "created_at": "2026-11-15 20:05:28.102142+00:00" + } + ] +} diff --git a/research/analyze_pipeline.py b/research/analyze_pipeline.py index 2436dc16..7a9e0d84 100644 --- a/research/analyze_pipeline.py +++ b/research/analyze_pipeline.py @@ -59,10 +59,12 @@ def main(): ) stale = [r for r in rows if r["state"] == "obsolete"] assert all(r["publications"] == 0 for r in stale) + job_ids = {r["job_id"] for r in rows} samples = [ x["detail"] for x in d["worker_metrics"] - if x["detail"].get("result") == "ready" + if str(x["job_id"]) in job_ids + and x["detail"].get("result") == "ready" and "rss_peak_mib" in x["detail"] ] stats = { diff --git a/scripts/verify_cluster.py b/scripts/verify_cluster.py index a0dbd639..19be4197 100644 --- a/scripts/verify_cluster.py +++ b/scripts/verify_cluster.py @@ -59,7 +59,12 @@ def exercise(client, session, chat, action, *, seconds=15): thread.start() key = uuid4().hex try: - while time.monotonic() - started < seconds or not finished.is_set(): + while ( + time.monotonic() - started < seconds + or not finished.is_set() + or time.monotonic() - started + < action_result.get("finished", 0) + 2 + ): h = {**auth(session), "Idempotency-Key": key} try: r = client.post(path, headers=h, json={"text": key}) -- 2.54.0 From eb6d229afb1437cd0f8b5e2ee0ca384d584ddd8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Sun, 15 Nov 2026 23:18:33 +0300 Subject: [PATCH 08/11] test(pipeline): record verified limits and complete cluster recovery evidence --- REPORT.md | 18 + evidence/cluster-first.json | 5162 +++++++++++++++++++++++++++++++++++ scripts/verify_pipeline.py | 19 +- src/worker.py | 15 + 4 files changed, 5199 insertions(+), 15 deletions(-) create mode 100644 evidence/cluster-first.json diff --git a/REPORT.md b/REPORT.md index 8c0318b2..83821142 100644 --- a/REPORT.md +++ b/REPORT.md @@ -286,3 +286,21 @@ private/IAM/otherAPI. Это целевой повтор, а не19cases пол проходит handoff predicate. Fixture metadata failed состарены SQL отдельно, но возраст S3 LastModified не подменялся. Cleanup fixtures синтетические и не обещают восстановление bytes из checksum. + +Для финальной проверки worker перед fault hook сохраняет diagnostic event +worker_resource_limits с фактическим RLIMIT_AS и cgroup memory.max/cpu.max. +Сбор показаний самим child исключает влияние медленного внешнего DockerCLI +на12s test budget. Это диагностическое добавление также проверяется новым +образом из чистого clone; предыдущие HTTP/source evidence не переименованы. + +Первый cluster receipt [cluster-first-failed.json](evidence/cluster-first-failed.json) +сохраняет успешные отказы двухAPI. Rolling занял дольше45s окна, поэтому +генератор завершился сразу с action и не получил post-action observation. +Verifier теперь обязательно наблюдает ещё2s после action, сохраняя все +проверки данных и failover≤10s. Это исправление наблюдения, не ослабление SLA. + +[evidence/cluster-first.json](evidence/cluster-first.json): complete PASS; +API failovers (writes/errors/recovery seconds): 214/6/3.086, 216/8/2.762. +Rolling: 730 confirmed writes, 18 transient errors; все исходныеAPI заменены, данные/replay сохранены. +Public failover/persistence PASS; StartedAt PG/cache/S3/broker/LB изменились, +прежний token и file сохранились. Это не zero-error rolling. diff --git a/evidence/cluster-first.json b/evidence/cluster-first.json new file mode 100644 index 00000000..e642d431 --- /dev/null +++ b/evidence/cluster-first.json @@ -0,0 +1,5162 @@ +{ + "date": "2026-11-15T20:13:14.393195+00:00", + "source_commit": "ef619490f23877adbce37c9012496d01128bae4a", + "complete": true, + "before": [ + { + "role": "api", + "container": "638206e980b8", + "pid": 31709, + "started_at": "2026-11-15T20:09:56.153775471Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "c5f6207268fe429cb5ead0adc052bc01" + }, + { + "role": "api", + "container": "b5d2b8c25029", + "pid": 33126, + "started_at": "2026-11-15T20:10:27.099626888Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "role": "broker", + "container": "5d3ebf6f87ff", + "pid": 77503, + "started_at": "2026-11-15T19:51:03.049505001Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "6ad17613e620", + "pid": 60587, + "started_at": "2026-11-15T19:17:14.204239507Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "c6fb99c3ddbc", + "pid": 60326, + "started_at": "2026-11-15T19:45:43.876610632Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "c60ecc6c94f4", + "pid": 63147, + "started_at": "2026-11-15T19:17:36.952605129Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8085" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "76efa23ba942", + "pid": 60650, + "started_at": "2026-11-15T19:17:14.476492674Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "a990e3899324", + "pid": 27505, + "started_at": "2026-11-15T20:08:42.302428089Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "c8273b4552c9", + "pid": 98443, + "started_at": "2026-11-15T19:58:04.386027252Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "de0c05e2e665", + "pid": 60843, + "started_at": "2026-11-15T19:17:15.006133424Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "a89e2cf493d9", + "pid": 18369, + "started_at": "2026-11-15T20:05:23.317701469Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ], + "parallel_replay": { + "requests": 30, + "messages": 1, + "instances": [ + "71696073e468446ea13d821b111ad508", + "c5f6207268fe429cb5ead0adc052bc01" + ] + }, + "atomic_rollback_and_5xx_retry": true, + "expired_key_can_be_reused": true, + "failovers": [ + { + "confirmed_writes": 214, + "successful_reads": 208, + "errors": 6, + "successful_rps": 27.94, + "first_success_after_action_seconds": 3.086, + "action": { + "finished": 3.081 + }, + "observations": [ + { + "at": 0.008, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.065, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.184, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.256, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.368, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.469, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.57, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.663, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.723, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.778, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.843, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.907, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.968, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.059, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.162, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.264, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.358, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.469, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.527, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.617, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.707, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.818, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.918, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.014, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.111, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.212, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.283, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.34, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.394, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.448, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.51, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.571, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.63, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.691, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.75, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.808, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.876, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.945, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.007, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.086, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.164, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.232, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.296, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.365, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.447, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.506, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.57, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.634, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.699, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.759, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.814, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.877, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 3.944, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.008, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.073, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.152, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.232, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.293, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.351, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.413, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.474, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.542, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.608, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.677, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.745, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.808, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.862, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.93, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 4.988, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.054, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.112, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.174, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.243, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.308, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.379, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.452, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.519, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.595, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.669, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.742, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.807, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.871, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.936, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 5.999, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.057, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.122, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.18, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.244, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.312, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.375, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.441, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.508, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.574, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.642, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.758, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.828, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.888, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 6.962, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.023, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.092, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.152, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.213, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.28, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.341, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.409, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.471, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.543, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.609, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.678, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.747, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.809, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.878, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 7.963, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.047, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.125, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.194, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.258, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.32, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.395, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.458, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.528, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.592, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.662, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.726, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.795, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.859, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.929, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 8.999, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.072, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.133, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.198, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.256, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.326, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.392, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.457, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.523, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.608, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.711, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.815, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 9.915, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.016, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.11, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.208, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.319, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.39, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.456, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.569, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.63, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.69, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.757, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.814, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.874, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.935, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 10.997, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.064, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.131, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.199, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.263, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.339, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.407, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.469, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.536, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.594, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.663, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.733, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.806, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.873, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 11.945, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.011, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.085, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.154, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.217, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.289, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.355, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.419, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.486, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.547, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.607, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.668, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.736, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.797, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.857, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.922, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 12.984, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.049, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.126, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.189, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.248, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.307, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.365, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.428, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.496, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.558, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.624, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.696, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.771, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.846, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.918, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 13.988, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.062, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.142, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.21, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.276, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.342, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.416, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.483, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.552, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.611, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.67, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.734, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.794, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.857, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.925, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 14.994, + "instance": "71696073e468446ea13d821b111ad508" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "638206e980b8", + "pid": 31709, + "started_at": "2026-11-15T20:09:56.153775471Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "c5f6207268fe429cb5ead0adc052bc01" + } + }, + { + "confirmed_writes": 216, + "successful_reads": 209, + "errors": 8, + "successful_rps": 28.18, + "first_success_after_action_seconds": 2.762, + "action": { + "finished": 2.701 + }, + "observations": [ + { + "at": 0.018, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.084, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.15, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.215, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.28, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.343, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.403, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.459, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.525, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.582, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.636, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.694, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.751, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.817, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.873, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 0.935, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.005, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.108, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.204, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.316, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.41, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.468, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.574, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.677, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.779, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.87, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 1.97, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.031, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.1, + "instance": "71696073e468446ea13d821b111ad508" + }, + { + "at": 2.164, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.223, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.278, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.339, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.394, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.447, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.505, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.561, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.619, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.687, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.762, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.832, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.888, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.946, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.009, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.072, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.139, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.209, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.275, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.338, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.431, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.487, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.542, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.608, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.673, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.738, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.796, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.854, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.914, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.973, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.045, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.107, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.169, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.241, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.312, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.38, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.446, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.504, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.563, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.628, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.69, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.759, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.83, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.89, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.953, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.014, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.071, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.139, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.204, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.272, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.343, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.407, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.489, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.556, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.624, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.693, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.767, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.838, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.904, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.96, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.025, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.092, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.157, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.234, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.301, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.367, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.435, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.499, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.566, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.635, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.702, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.765, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.837, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.931, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.0, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.084, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.156, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.22, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.285, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.347, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.41, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.473, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.568, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.634, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.701, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.767, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.842, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.914, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.987, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.054, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.123, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.19, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.261, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.32, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.386, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.446, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.51, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.581, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.648, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.72, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.787, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.856, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.923, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.986, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.047, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.116, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.181, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.248, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.311, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.371, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.459, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.659, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.765, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.864, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.954, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.055, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.164, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.26, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.363, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.425, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.487, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.557, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.625, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.698, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.767, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.833, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.904, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.974, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.033, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.091, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.155, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.225, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.286, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.355, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.426, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.49, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.554, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.613, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.68, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.748, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.815, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.878, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.945, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.009, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.073, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.137, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.198, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.255, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.315, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.382, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.443, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.513, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.577, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.653, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.723, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.795, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.866, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.935, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.01, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.135, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.201, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.269, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.339, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.409, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.475, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.545, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.615, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.681, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.752, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.822, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.885, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.958, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.024, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.101, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.185, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.253, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.314, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.381, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.445, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.513, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.572, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.64, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.704, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.765, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.829, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.891, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.959, + "instance": "5877164629d44631b753484921940317" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "b5d2b8c25029", + "pid": 33126, + "started_at": "2026-11-15T20:10:27.099626888Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "71696073e468446ea13d821b111ad508" + } + } + ], + "rolling": { + "confirmed_writes": 730, + "successful_reads": 728, + "errors": 18, + "successful_rps": 27.16, + "first_success_after_action_seconds": 51.552, + "action": { + "finished": 51.507 + }, + "observations": [ + { + "at": 0.023, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.087, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.156, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.22, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.278, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.343, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.402, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.466, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.53, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.586, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.646, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.702, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.758, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.827, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 0.909, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.002, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.068, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.152, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.232, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.327, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.422, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.501, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.595, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.684, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.76, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.86, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 1.952, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.478, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.561, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.671, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.748, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.808, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.88, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 2.945, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.007, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.067, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.129, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.196, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.259, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.317, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.384, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.443, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.498, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.558, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.644, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.699, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.763, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.819, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.883, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 3.948, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.013, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.083, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.152, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.222, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.281, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.341, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.4, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.463, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.54, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.626, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.708, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.769, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.835, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.895, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 4.964, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.022, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.087, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.153, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.22, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.285, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.347, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.412, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.475, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.535, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.604, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.675, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.744, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.808, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.872, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.934, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 5.992, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.052, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.118, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.175, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.234, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.296, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.361, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.427, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.492, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.548, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.615, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.684, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.749, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 6.899, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.033, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.16, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.622, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 7.931, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.048, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.153, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.24, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.347, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.443, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.537, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.638, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.709, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.778, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.846, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.913, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 8.973, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.035, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.103, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.172, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.235, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.298, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.367, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.433, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.49, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.546, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.613, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.68, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.741, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.81, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.873, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 9.943, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.005, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.067, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.136, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.202, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.272, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.338, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.404, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.469, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.534, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.602, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.675, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.737, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.802, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.862, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.919, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 10.984, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.05, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.118, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.188, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.253, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.33, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.408, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.473, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.539, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.601, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.669, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.737, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.816, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 11.92, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.01, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.088, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.155, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.226, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.349, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.438, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.503, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.568, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.636, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.705, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.778, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.844, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.903, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 12.964, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.03, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.095, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.167, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.239, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.306, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.376, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.441, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.513, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.585, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.643, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.705, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.774, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.836, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.903, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 13.974, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.044, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.104, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.175, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.236, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.294, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.351, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.412, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.48, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.547, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.618, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.688, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.758, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.834, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 14.921, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 14.998, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 15.061, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 15.13, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 15.196, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 15.263, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 15.337, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 15.406, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 15.475, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 15.539, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 15.608, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 15.665, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 15.723, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 15.783, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 15.851, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 15.931, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 16.002, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 16.071, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 16.139, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 16.202, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 16.27, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 16.34, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 16.4, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 16.468, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 16.539, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 16.604, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 16.679, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 16.746, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 16.809, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 16.87, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 16.929, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 16.994, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 17.061, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 17.124, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 17.191, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 17.26, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 17.33, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 17.4, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 17.465, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 17.528, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 17.588, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 17.649, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 17.712, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 17.778, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 17.836, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 17.903, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 17.986, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 18.052, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 18.133, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 18.202, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 18.278, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 18.344, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 18.411, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 18.479, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 18.554, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 18.689, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 18.82, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 18.953, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 19.08, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 19.144, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 19.249, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 19.382, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 19.446, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 19.518, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 19.593, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 19.665, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 19.732, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 19.793, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 19.862, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 19.929, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 20.006, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 20.088, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 20.158, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 20.24, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 20.308, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 20.369, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 20.428, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 20.492, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 20.552, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 20.613, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 20.678, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 20.74, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 20.805, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 20.872, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 20.937, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 20.997, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 21.054, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 21.112, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 21.177, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 21.236, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 21.304, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 21.365, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 21.428, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 21.485, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 21.548, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 21.607, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 21.666, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 21.737, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 21.804, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 21.862, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 21.92, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 21.987, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 22.055, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 22.139, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 22.211, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 22.276, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 22.334, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 22.399, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 22.456, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 22.517, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 22.584, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 22.653, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 22.723, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 22.796, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 22.87, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 22.936, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 22.998, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 23.065, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 23.131, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 23.196, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 23.264, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 23.33, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 23.388, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 23.448, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 23.514, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 23.587, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 23.713, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 23.776, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 23.88, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 23.942, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 24.01, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 24.068, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 24.135, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 24.203, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 24.274, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 24.343, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 24.412, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 24.469, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 24.533, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 24.6, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 24.656, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 24.714, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 24.78, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 24.844, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 24.911, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 24.976, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 25.045, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 25.113, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 25.188, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 25.264, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 25.34, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 25.424, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 25.489, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 25.548, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 25.617, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 25.687, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 25.753, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 25.819, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 25.885, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 25.953, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 26.02, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 26.083, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 26.156, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 26.23, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 26.302, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 26.369, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 26.435, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 26.5, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 26.564, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 26.624, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 26.69, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 26.756, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 26.818, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 26.884, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 26.947, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 27.012, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 27.073, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 27.136, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 27.197, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 27.257, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 27.322, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 27.395, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 27.476, + "instance": "5877164629d44631b753484921940317" + }, + { + "at": 27.556, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 27.62, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 27.695, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 27.851, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 27.926, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 27.998, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.062, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.14, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.248, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.314, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.399, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.476, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.557, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.637, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.717, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.791, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.865, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 28.937, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.004, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.062, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.131, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.195, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.266, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.331, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.404, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.484, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.581, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.685, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.788, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.895, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 29.99, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.089, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.155, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.219, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.301, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.369, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.437, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.501, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.564, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.631, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.698, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.767, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.838, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.918, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 30.99, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.068, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.143, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.221, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.285, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.352, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.418, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.478, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.54, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.604, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.676, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.742, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.808, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.873, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 31.935, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.009, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.073, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.147, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.226, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.322, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.428, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.511, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.595, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.686, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.768, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.839, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.897, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 32.96, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.022, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.092, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.152, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.255, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.324, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.392, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.467, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.532, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.615, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.693, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.781, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.856, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.922, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 33.984, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.053, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.174, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.276, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.383, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.48, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.577, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.672, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.767, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.864, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 34.98, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.047, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.118, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.201, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.271, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.336, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.417, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.482, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.548, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.618, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.678, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.74, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.811, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.896, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 35.982, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.051, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.12, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.189, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.265, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.325, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.392, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.489, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.551, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.623, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.69, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.754, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.821, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.888, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 36.966, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.023, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.09, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.153, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.218, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.276, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.337, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.397, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.466, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.534, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.602, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.666, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.724, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.785, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.843, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.906, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 37.974, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.039, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.11, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.184, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.25, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.317, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.383, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.443, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.512, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.597, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.664, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.741, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.809, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.879, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 38.946, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.015, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.081, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.137, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.195, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.26, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.318, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.378, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.439, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.503, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.571, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.643, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.718, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.777, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.836, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.896, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 39.96, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.025, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.082, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.144, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.284, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.383, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.489, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.585, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.689, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.78, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.84, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.917, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 40.991, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.062, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.124, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.182, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.25, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.32, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.387, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.453, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.519, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.582, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.643, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.738, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.801, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.862, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.929, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 41.994, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.066, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.125, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.189, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.252, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.316, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.382, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.445, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.514, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.585, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.663, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.723, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.787, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.854, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.919, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 42.981, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.052, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.126, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.186, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.251, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.315, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.38, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.452, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.514, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.579, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.646, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.705, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.764, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.831, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.892, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 43.958, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.016, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.076, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.149, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.215, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.283, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.342, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.43, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 44.492, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 44.56, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.62, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 44.69, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 44.753, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 44.824, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 44.893, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 44.959, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 45.028, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 45.121, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 45.181, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 45.272, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 45.331, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 45.394, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 45.476, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 45.547, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 45.613, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 45.698, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 45.768, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 45.835, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 45.902, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 45.96, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 46.029, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 46.09, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 46.149, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 46.208, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 46.274, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 46.341, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 46.404, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 46.467, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 46.534, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 46.595, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 46.657, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 46.723, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 46.786, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 46.847, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 46.915, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 46.977, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 47.043, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 47.104, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 47.161, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 47.228, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 47.294, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 47.356, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 47.429, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 47.497, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 47.564, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "at": 47.635, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "at": 47.691, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 47.772, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 47.843, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 47.915, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 47.982, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 48.044, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 48.107, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.229, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.301, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.371, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.43, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.487, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.557, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.626, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.687, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.755, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.828, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.892, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 49.965, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.036, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.097, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.164, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.227, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.297, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.371, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.44, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.512, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.576, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.635, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.7, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.773, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.84, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.904, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 50.97, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.037, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.095, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.153, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.219, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.281, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.35, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.416, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.483, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.552, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.625, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.692, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.761, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.834, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.896, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 51.961, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.024, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.089, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.154, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.421, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.514, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.582, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.643, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.704, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.772, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.846, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.905, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 52.971, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 53.04, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 53.104, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 53.17, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 53.237, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 53.304, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 53.37, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 53.438, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "at": 53.507, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "before": [ + { + "role": "api", + "container": "638206e980b8", + "pid": 41688, + "started_at": "2026-11-15T20:13:39.959219713Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "5877164629d44631b753484921940317" + }, + { + "role": "api", + "container": "b5d2b8c25029", + "pid": 43471, + "started_at": "2026-11-15T20:14:12.673332131Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "b0488e03a2ca4039bc9467fead666372" + }, + { + "role": "broker", + "container": "5d3ebf6f87ff", + "pid": 77503, + "started_at": "2026-11-15T19:51:03.049505001Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "6ad17613e620", + "pid": 60587, + "started_at": "2026-11-15T19:17:14.204239507Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "c6fb99c3ddbc", + "pid": 60326, + "started_at": "2026-11-15T19:45:43.876610632Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "c60ecc6c94f4", + "pid": 63147, + "started_at": "2026-11-15T19:17:36.952605129Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8085" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "76efa23ba942", + "pid": 60650, + "started_at": "2026-11-15T19:17:14.476492674Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "a990e3899324", + "pid": 27505, + "started_at": "2026-11-15T20:08:42.302428089Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "c8273b4552c9", + "pid": 98443, + "started_at": "2026-11-15T19:58:04.386027252Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "de0c05e2e665", + "pid": 60843, + "started_at": "2026-11-15T19:17:15.006133424Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "a89e2cf493d9", + "pid": 18369, + "started_at": "2026-11-15T20:05:23.317701469Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ], + "after": [ + { + "role": "api", + "container": "6ca6b7ae13ea", + "pid": 44302, + "started_at": "2026-11-15T20:14:28.278164513Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "359cf22faa7d4f44b1b8da28db00d30c" + }, + { + "role": "api", + "container": "a6c104cad48f", + "pid": 45744, + "started_at": "2026-11-15T20:14:59.195657555Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "67a67b95e6874afea14457ec00d09992" + }, + { + "role": "broker", + "container": "5d3ebf6f87ff", + "pid": 77503, + "started_at": "2026-11-15T19:51:03.049505001Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "6ad17613e620", + "pid": 60587, + "started_at": "2026-11-15T19:17:14.204239507Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "c6fb99c3ddbc", + "pid": 60326, + "started_at": "2026-11-15T19:45:43.876610632Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "c60ecc6c94f4", + "pid": 63147, + "started_at": "2026-11-15T19:17:36.952605129Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8085" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "76efa23ba942", + "pid": 60650, + "started_at": "2026-11-15T19:17:14.476492674Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "a990e3899324", + "pid": 27505, + "started_at": "2026-11-15T20:08:42.302428089Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "c8273b4552c9", + "pid": 98443, + "started_at": "2026-11-15T19:58:04.386027252Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "de0c05e2e665", + "pid": 60843, + "started_at": "2026-11-15T19:17:15.006133424Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "a89e2cf493d9", + "pid": 18369, + "started_at": "2026-11-15T20:05:23.317701469Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ] + }, + "replay_after_replacement": true, + "public_failover": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 112, \"failed_attempts\": 4, \"first_recovery_seconds\": 3.812}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.", + "public_persistence": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.", + "after_persistence": [ + { + "role": "api", + "container": "6ca6b7ae13ea", + "pid": 51295, + "started_at": "2026-11-15T20:16:29.70121475Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "f9d06f9380aa472987e637ad3b18840e" + }, + { + "role": "api", + "container": "a6c104cad48f", + "pid": 51341, + "started_at": "2026-11-15T20:16:29.821670166Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "8c9b45de14704aa59c25b46076ac749a" + }, + { + "role": "broker", + "container": "5d3ebf6f87ff", + "pid": 50906, + "started_at": "2026-11-15T20:16:28.119800305Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "6ad17613e620", + "pid": 51018, + "started_at": "2026-11-15T20:16:28.220991221Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "c6fb99c3ddbc", + "pid": 60326, + "started_at": "2026-11-15T19:45:43.876610632Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "c60ecc6c94f4", + "pid": 51053, + "started_at": "2026-11-15T20:16:28.347761055Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8085" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "76efa23ba942", + "pid": 51061, + "started_at": "2026-11-15T20:16:28.35964443Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "a990e3899324", + "pid": 27505, + "started_at": "2026-11-15T20:08:42.302428089Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "c8273b4552c9", + "pid": 51509, + "started_at": "2026-11-15T20:16:30.976766958Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "de0c05e2e665", + "pid": 50945, + "started_at": "2026-11-15T20:16:28.170668388Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "a89e2cf493d9", + "pid": 52672, + "started_at": "2026-11-15T20:16:52.907797677Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ] +} diff --git a/scripts/verify_pipeline.py b/scripts/verify_pipeline.py index c92da95d..e327f5fc 100644 --- a/scripts/verify_pipeline.py +++ b/scripts/verify_pipeline.py @@ -413,22 +413,11 @@ def main(): marker = inject(j, "before_effect", "hang") start("worker") wait(lambda: hit(marker)) - cid = compose("ps", "-q", "worker").split()[0] - code = """import json,pathlib,re - limits=[] - for p in pathlib.Path('/proc').glob('[0-9]*/limits'): - try: - line=next(x for x in p.read_text().splitlines() if x.startswith('Max address space')) - if re.search(r'805306368\\s+805306368',line): limits.append({'pid':p.parent.name,'limit':line}) - except (FileNotFoundError,PermissionError,StopIteration,ProcessLookupError): pass - root=pathlib.Path('/sys/fs/cgroup') - print(json.dumps({'bounded_children':limits,'cgroup':{n:(root/n).read_text().strip() for n in ['memory.max','memory.current','memory.peak','cpu.max','pids.current']}})) - """ - limits = json.loads( - command(["docker", "exec", cid, "python", "-c", code]) + limits = inside( + f"with db.connection() as c: r=c.execute(\"SELECT detail FROM pipeline_events WHERE job_id=%s AND event='worker_resource_limits' ORDER BY id DESC LIMIT 1\",({j!r},)).fetchone()\nprint(json.dumps(r['detail']))" ) - assert 1 <= len(limits["bounded_children"]) <= 2 - assert limits["cgroup"]["memory.max"] == "1610612736" + assert limits["address_space_bytes"] == [805306368, 805306368] + assert limits["memory.max"] == "1610612736" effect = confirm(a, j) assert any( e["event"] == "retry" diff --git a/src/worker.py b/src/worker.py index 70fc9fc2..274684e3 100644 --- a/src/worker.py +++ b/src/worker.py @@ -7,6 +7,7 @@ import multiprocessing import resource import threading import time +from pathlib import Path from uuid import uuid4 import redis @@ -122,6 +123,20 @@ def process(job_id, fence, sender): if not row: sender.send({"result": "stale"}) return + # Record limits before fault hooks: a slow external inspector must not + # consume the child timeout while attempting to observe the limit. + cgroup = Path("/sys/fs/cgroup") + limits = { + name: (cgroup / name).read_text().strip() + if (cgroup / name).exists() + else "unavailable" + for name in ["memory.max", "cpu.max"] + } + limits["address_space_bytes"] = list( + resource.getrlimit(resource.RLIMIT_AS) + ) + with db.connection() as conn: + event(conn, job_id, "worker_resource_limits", limits) fault(db, job_id, "before_effect") upstream = store.get(key=row["source_key"]) try: -- 2.54.0 From 442163b0f0a549ef065cb13ee39b5ca2f8580c18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Sun, 15 Nov 2026 23:47:51 +0300 Subject: [PATCH 09/11] fix(pipeline): reclaim pending entries after the live lease budget --- REPORT.md | 16 + evidence/clean-clone-first.json | 8796 +++++++++++++++++++++++++++++++ scripts/verify_pipeline.py | 22 +- src/worker.py | 6 +- 4 files changed, 8835 insertions(+), 5 deletions(-) create mode 100644 evidence/clean-clone-first.json diff --git a/REPORT.md b/REPORT.md index 83821142..d2d9c12e 100644 --- a/REPORT.md +++ b/REPORT.md @@ -304,3 +304,19 @@ API failovers (writes/errors/recovery seconds): 214/6/3.086, 216/8/2.762. Rolling: 730 confirmed writes, 18 transient errors; все исходныеAPI заменены, данные/replay сохранены. Public failover/persistence PASS; StartedAt PG/cache/S3/broker/LB изменились, прежний token и file сохранились. Это не zero-error rolling. + +## Уточнение строгого окна COMMIT→ACK + +Первый чистый повтор [clean-clone-first.json](evidence/clean-clone-first.json) +sourceeb6d229 прошёл112HTTP379.34s/22unit/19cases/cluster/cleanup/hashes. +Однако в after_commit snapshot уже был duplicate_decision: второйconsumer +XAUTOCLAIM после3s успевал ACK раньшеSIGKILL. Это безопасно для данных, +но не доказывает конкретное окно COMMIT-before-ACK. Первый receipt сохранён +как результат прежней версии проверок; строгая граница требует нового повтора. + +Reclaim idle теперь равен lease(23s normal/15s test), превышая child timeout +20s/12s. Это предотвращает ранний захват чужого live entry. Новый тест +снимком XPENDING+XRANGE подтверждает конкретные job entries доSIGKILL и +отсутствие child_finished; после restart требует ready duplicate decision +и исчезновение notifications. Scope=all, данные и все прежние assertions +сохранены. Финальным доказательством станет clean-clone-final.json. diff --git a/evidence/clean-clone-first.json b/evidence/clean-clone-first.json new file mode 100644 index 00000000..535b6246 --- /dev/null +++ b/evidence/clean-clone-first.json @@ -0,0 +1,8796 @@ +{ + "date": "2026-11-15T20:20:18.627769+00:00", + "source_commit": "eb6d229afb1437cd0f8b5e2ee0ca384d584ddd8c", + "fresh_secrets_environment_and_volumes": true, + "command": "python scripts/verify_clean.py", + "fixed_host_port_for_restart": 61281, + "static_checks": ".venv/bin/python scripts/check_contract.py\nPASS: offline skeleton structure, local refs and Python syntax; application NOT tested.\n.venv/bin/python scripts/validate_openapi.py\nPASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.\n/.local/bin/uv run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/verify_clean.py scripts/verify_cleanup.py scripts/verify_cluster.py scripts/verify_pipeline.py\n38 files already formatted\n/.local/bin/uv run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/verify_clean.py scripts/verify_cleanup.py scripts/verify_cluster.py scripts/verify_pipeline.py\nAll checks passed!", + "unit_tests": "PYTHONPATH=. /.local/bin/uv run --locked pytest student-unit -q\n...................... [100%]\n22 passed in 14.11s", + "smoke": "python3 tests/smoke.py --base-url \"http://127.0.0.1:61281\" \nPASS: lab5 public API scenario; infrastructure and grading evidence remain separate.", + "http_tests": "........................................................................ [ 64%]\n........................................ [100%]\n112 passed in 379.34s (0:06:19)", + "pipeline": { + "source_commit": "eb6d229afb1437cd0f8b5e2ee0ca384d584ddd8c", + "date": "2026-11-15T20:31:54.110235+00:00", + "track": 2, + "scope": "all", + "complete": true, + "cases": [ + { + "case": "staging_before_handoff_crash", + "no_202": true, + "no_attachment_or_job_created": true, + "orphan_retained_for_cleanup": true + }, + { + "case": "commit_before_publish_and_worker_stopped", + "attachment": "b93c8166-bdaa-4645-aa38-f9565856d888", + "effect": { + "job": { + "id": "7e6614be-b097-46d0-8398-47ba5ce3444a", + "attachment_id": "b93c8166-bdaa-4645-aa38-f9565856d888", + "pipeline_version": 1, + "source_key": "objects/staging/b93c8166-bdaa-4645-aa38-f9565856d888/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:32:28.373080+00:00", + "error_code": null, + "created_at": "2026-11-15 20:32:28.373080+00:00", + "updated_at": "2026-11-15 20:32:54.127104+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/b93c8166-bdaa-4645-aa38-f9565856d888/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/b93c8166-bdaa-4645-aa38-f9565856d888/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/b93c8166-bdaa-4645-aa38-f9565856d888/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/b93c8166-bdaa-4645-aa38-f9565856d888/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/b93c8166-bdaa-4645-aa38-f9565856d888/v1/crop", + "objects/b93c8166-bdaa-4645-aa38-f9565856d888/v1/optimized", + "objects/b93c8166-bdaa-4645-aa38-f9565856d888/v1/original", + "objects/b93c8166-bdaa-4645-aa38-f9565856d888/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:32:51.920425+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794774771894-0", + "outbox": 13 + }, + "created_at": "2026-11-15 20:32:51.963453+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:32:53.891667+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:32:54.127296+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2639559589997589, + "cpu_seconds": 0.23261253799999992, + "rss_peak_mib": 79.73046875 + }, + "created_at": "2026-11-15 20:32:54.259195+00:00" + } + ] + }, + "queued_link_allowed": true, + "api_and_broker_restarted": true, + "broker_stopped_during_upload": true + }, + { + "case": "confirmed_broker_entry_survives_sigkill", + "effect": { + "job": { + "id": "c7761c1b-92ce-40cd-9058-bdad8f5f189c", + "attachment_id": "6ad9c0b0-4b93-4129-b066-270b361e9256", + "pipeline_version": 1, + "source_key": "objects/staging/6ad9c0b0-4b93-4129-b066-270b361e9256/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:33:00.555147+00:00", + "error_code": null, + "created_at": "2026-11-15 20:33:00.555148+00:00", + "updated_at": "2026-11-15 20:33:28.308832+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/6ad9c0b0-4b93-4129-b066-270b361e9256/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/6ad9c0b0-4b93-4129-b066-270b361e9256/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/6ad9c0b0-4b93-4129-b066-270b361e9256/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/6ad9c0b0-4b93-4129-b066-270b361e9256/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/6ad9c0b0-4b93-4129-b066-270b361e9256/v1/crop", + "objects/6ad9c0b0-4b93-4129-b066-270b361e9256/v1/optimized", + "objects/6ad9c0b0-4b93-4129-b066-270b361e9256/v1/original", + "objects/6ad9c0b0-4b93-4129-b066-270b361e9256/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774780583-0", + "outbox": 14 + }, + "created_at": "2026-11-15 20:33:00.588385+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:33:26.112048+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:33:28.043378+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:33:28.309061+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2951308749998134, + "cpu_seconds": 0.2648634940000001, + "rss_peak_mib": 79.609375 + }, + "created_at": "2026-11-15 20:33:28.419122+00:00" + } + ] + }, + "same_stream_entries": true, + "entry_count": 1 + }, + { + "case": "relay_crash_after_xadd_before_pg_marker", + "unpublished_after_crash": 1, + "effect": { + "job": { + "id": "0048f677-7bca-4f87-8cdf-1a60d6e3bd2c", + "attachment_id": "c660e20a-a0cd-402e-9062-8e4aa3631ef7", + "pipeline_version": 1, + "source_key": "objects/staging/c660e20a-a0cd-402e-9062-8e4aa3631ef7/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:33:35.232787+00:00", + "error_code": null, + "created_at": "2026-11-15 20:33:35.232787+00:00", + "updated_at": "2026-11-15 20:34:04.233613+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/c660e20a-a0cd-402e-9062-8e4aa3631ef7/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/c660e20a-a0cd-402e-9062-8e4aa3631ef7/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/c660e20a-a0cd-402e-9062-8e4aa3631ef7/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/c660e20a-a0cd-402e-9062-8e4aa3631ef7/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/c660e20a-a0cd-402e-9062-8e4aa3631ef7/v1/crop", + "objects/c660e20a-a0cd-402e-9062-8e4aa3631ef7/v1/optimized", + "objects/c660e20a-a0cd-402e-9062-8e4aa3631ef7/v1/original", + "objects/c660e20a-a0cd-402e-9062-8e4aa3631ef7/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:34:02.035998+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:34:03.981197+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:34:04.233799+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2813682500000141, + "cpu_seconds": 0.23825850199999987, + "rss_peak_mib": 79.734375 + }, + "created_at": "2026-11-15 20:34:04.343185+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794774845269-0", + "outbox": 15 + }, + "created_at": "2026-11-15 20:34:05.343360+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794774845269-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:34:05.346880+00:00" + } + ] + } + }, + { + "case": "before_effect_sigkill_repeat_1", + "at_crash": { + "job": { + "id": "6cc8e812-e4ae-4d36-a230-d5235fa45b4b", + "attachment_id": "420109c5-01d0-42de-82ba-06d85d0df610", + "pipeline_version": 1, + "source_key": "objects/staging/420109c5-01d0-42de-82ba-06d85d0df610/original", + "state": "processing", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": "f7bc3cf0-c12c-49aa-bb37-ec0f0fb4937d-1", + "lease_until": "2026-11-15 20:34:36.244033+00:00", + "next_at": "2026-11-15 20:34:10.365420+00:00", + "error_code": null, + "created_at": "2026-11-15 20:34:10.365420+00:00", + "updated_at": "2026-11-15 20:34:21.244033+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774850500-0", + "outbox": 16 + }, + "created_at": "2026-11-15 20:34:10.505316+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:34:21.245618+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:34:23.191221+00:00" + } + ] + }, + "effect": { + "job": { + "id": "6cc8e812-e4ae-4d36-a230-d5235fa45b4b", + "attachment_id": "420109c5-01d0-42de-82ba-06d85d0df610", + "pipeline_version": 1, + "source_key": "objects/staging/420109c5-01d0-42de-82ba-06d85d0df610/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:34:37.016501+00:00", + "error_code": null, + "created_at": "2026-11-15 20:34:10.365420+00:00", + "updated_at": "2026-11-15 20:34:39.412593+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/420109c5-01d0-42de-82ba-06d85d0df610/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/420109c5-01d0-42de-82ba-06d85d0df610/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/420109c5-01d0-42de-82ba-06d85d0df610/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/420109c5-01d0-42de-82ba-06d85d0df610/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/420109c5-01d0-42de-82ba-06d85d0df610/v1/crop", + "objects/420109c5-01d0-42de-82ba-06d85d0df610/v1/optimized", + "objects/420109c5-01d0-42de-82ba-06d85d0df610/v1/original", + "objects/420109c5-01d0-42de-82ba-06d85d0df610/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774850500-0", + "outbox": 16 + }, + "created_at": "2026-11-15 20:34:10.505316+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:34:21.245618+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:34:23.191221+00:00" + }, + { + "event": "retry", + "detail": { + "code": "lease_expired", + "attempt": 1, + "backoff_seconds": 0.7349758269229827 + }, + "created_at": "2026-11-15 20:34:36.286227+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794774877116-0", + "outbox": 17 + }, + "created_at": "2026-11-15 20:34:37.122203+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 20:34:37.129858+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794774850500-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:34:37.408734+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:34:39.170954+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:34:39.412973+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.27222879200053285, + "cpu_seconds": 0.2380743289999998, + "rss_peak_mib": 79.97265625 + }, + "created_at": "2026-11-15 20:34:39.527386+00:00" + } + ] + } + }, + { + "case": "after_effect_sigkill_repeat_1", + "at_crash": { + "job": { + "id": "beeb7929-64e8-4ed5-9990-16f0dd1d5aa8", + "attachment_id": "83cc6e0a-1397-4939-a0af-5527f3975b56", + "pipeline_version": 1, + "source_key": "objects/staging/83cc6e0a-1397-4939-a0af-5527f3975b56/original", + "state": "processing", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": "455238d9-616c-4663-8081-b75644c34445-1", + "lease_until": "2026-11-15 20:35:12.872510+00:00", + "next_at": "2026-11-15 20:34:45.852662+00:00", + "error_code": null, + "created_at": "2026-11-15 20:34:45.852662+00:00", + "updated_at": "2026-11-15 20:34:57.872510+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/crop", + "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/optimized", + "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/original", + "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774885985-0", + "outbox": 18 + }, + "created_at": "2026-11-15 20:34:45.991777+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:34:57.875075+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:35:00.481122+00:00" + } + ] + }, + "effect": { + "job": { + "id": "beeb7929-64e8-4ed5-9990-16f0dd1d5aa8", + "attachment_id": "83cc6e0a-1397-4939-a0af-5527f3975b56", + "pipeline_version": 1, + "source_key": "objects/staging/83cc6e0a-1397-4939-a0af-5527f3975b56/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:35:11.577038+00:00", + "error_code": null, + "created_at": "2026-11-15 20:34:45.852662+00:00", + "updated_at": "2026-11-15 20:35:14.926277+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/crop", + "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/optimized", + "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/original", + "objects/83cc6e0a-1397-4939-a0af-5527f3975b56/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774885985-0", + "outbox": 18 + }, + "created_at": "2026-11-15 20:34:45.991777+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:34:57.875075+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:35:00.481122+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "worker_timeout", + "result": "transient" + }, + "created_at": "2026-11-15 20:35:09.925522+00:00" + }, + { + "event": "retry", + "detail": { + "code": "worker_timeout", + "attempt": 1, + "backoff_seconds": 1.650536463275796 + }, + "created_at": "2026-11-15 20:35:09.927630+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794774911683-0", + "outbox": 19 + }, + "created_at": "2026-11-15 20:35:11.687860+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 20:35:12.689706+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:35:14.686531+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:35:14.926555+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2685703340002874, + "cpu_seconds": 0.23196675100000008, + "rss_peak_mib": 79.7109375 + }, + "created_at": "2026-11-15 20:35:15.204431+00:00" + } + ] + } + }, + { + "case": "after_commit_sigkill_repeat_1", + "at_crash": { + "job": { + "id": "08f2419e-cb14-41d5-b2b2-5e0e36b023a6", + "attachment_id": "42e402e7-8c37-4c43-a0cd-b456575b21b6", + "pipeline_version": 1, + "source_key": "objects/staging/42e402e7-8c37-4c43-a0cd-b456575b21b6/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:35:21.265720+00:00", + "error_code": null, + "created_at": "2026-11-15 20:35:21.265720+00:00", + "updated_at": "2026-11-15 20:35:34.120048+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/crop", + "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/optimized", + "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/original", + "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774921333-0", + "outbox": 20 + }, + "created_at": "2026-11-15 20:35:21.338179+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:35:31.994723+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:35:33.882444+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:35:34.120287+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794774921333-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:35:35.241237+00:00" + } + ] + }, + "effect": { + "job": { + "id": "08f2419e-cb14-41d5-b2b2-5e0e36b023a6", + "attachment_id": "42e402e7-8c37-4c43-a0cd-b456575b21b6", + "pipeline_version": 1, + "source_key": "objects/staging/42e402e7-8c37-4c43-a0cd-b456575b21b6/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:35:21.265720+00:00", + "error_code": null, + "created_at": "2026-11-15 20:35:21.265720+00:00", + "updated_at": "2026-11-15 20:35:34.120048+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/crop", + "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/optimized", + "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/original", + "objects/42e402e7-8c37-4c43-a0cd-b456575b21b6/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774921333-0", + "outbox": 20 + }, + "created_at": "2026-11-15 20:35:21.338179+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:35:31.994723+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:35:33.882444+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:35:34.120287+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794774921333-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:35:35.241237+00:00" + } + ] + } + }, + { + "case": "before_effect_sigkill_repeat_2", + "at_crash": { + "job": { + "id": "347bd84b-610b-427f-98e0-b6a290923456", + "attachment_id": "aeb85ccc-0787-49a8-910d-54a4c083316d", + "pipeline_version": 1, + "source_key": "objects/staging/aeb85ccc-0787-49a8-910d-54a4c083316d/original", + "state": "processing", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": "402beccc-189b-41d8-9bcf-641b86b96677-0", + "lease_until": "2026-11-15 20:36:29.671877+00:00", + "next_at": "2026-11-15 20:36:03.390772+00:00", + "error_code": null, + "created_at": "2026-11-15 20:36:03.390772+00:00", + "updated_at": "2026-11-15 20:36:14.671878+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774963487-0", + "outbox": 21 + }, + "created_at": "2026-11-15 20:36:03.493220+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:36:14.673658+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:36:16.642679+00:00" + } + ] + }, + "effect": { + "job": { + "id": "347bd84b-610b-427f-98e0-b6a290923456", + "attachment_id": "aeb85ccc-0787-49a8-910d-54a4c083316d", + "pipeline_version": 1, + "source_key": "objects/staging/aeb85ccc-0787-49a8-910d-54a4c083316d/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:36:31.201421+00:00", + "error_code": null, + "created_at": "2026-11-15 20:36:03.390772+00:00", + "updated_at": "2026-11-15 20:36:33.481230+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/aeb85ccc-0787-49a8-910d-54a4c083316d/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/aeb85ccc-0787-49a8-910d-54a4c083316d/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/aeb85ccc-0787-49a8-910d-54a4c083316d/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/aeb85ccc-0787-49a8-910d-54a4c083316d/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/aeb85ccc-0787-49a8-910d-54a4c083316d/v1/crop", + "objects/aeb85ccc-0787-49a8-910d-54a4c083316d/v1/optimized", + "objects/aeb85ccc-0787-49a8-910d-54a4c083316d/v1/original", + "objects/aeb85ccc-0787-49a8-910d-54a4c083316d/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774963487-0", + "outbox": 21 + }, + "created_at": "2026-11-15 20:36:03.493220+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:36:14.673658+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:36:16.642679+00:00" + }, + { + "event": "retry", + "detail": { + "code": "lease_expired", + "attempt": 1, + "backoff_seconds": 1.32552866661181 + }, + "created_at": "2026-11-15 20:36:29.877852+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794774963487-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:36:30.532350+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794774991334-0", + "outbox": 22 + }, + "created_at": "2026-11-15 20:36:31.340026+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 20:36:31.343727+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:36:33.246295+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:36:33.481412+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2644206660006603, + "cpu_seconds": 0.23357183000000004, + "rss_peak_mib": 79.86328125 + }, + "created_at": "2026-11-15 20:36:33.606213+00:00" + } + ] + } + }, + { + "case": "after_effect_sigkill_repeat_2", + "at_crash": { + "job": { + "id": "ea0a2fc7-4484-46fa-8c13-b4523eebd673", + "attachment_id": "590de76f-eca1-4cac-8623-81d980778c96", + "pipeline_version": 1, + "source_key": "objects/staging/590de76f-eca1-4cac-8623-81d980778c96/original", + "state": "processing", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": "b37ae4d8-b8e2-414f-9051-5a71c065b53d-0", + "lease_until": "2026-11-15 20:37:05.257675+00:00", + "next_at": "2026-11-15 20:36:39.256083+00:00", + "error_code": null, + "created_at": "2026-11-15 20:36:39.256083+00:00", + "updated_at": "2026-11-15 20:36:50.257676+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/crop", + "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/optimized", + "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/original", + "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774999364-0", + "outbox": 23 + }, + "created_at": "2026-11-15 20:36:39.425553+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:36:50.267695+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:36:52.972919+00:00" + } + ] + }, + "effect": { + "job": { + "id": "ea0a2fc7-4484-46fa-8c13-b4523eebd673", + "attachment_id": "590de76f-eca1-4cac-8623-81d980778c96", + "pipeline_version": 1, + "source_key": "objects/staging/590de76f-eca1-4cac-8623-81d980778c96/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:37:03.206829+00:00", + "error_code": null, + "created_at": "2026-11-15 20:36:39.256083+00:00", + "updated_at": "2026-11-15 20:37:07.376667+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/crop", + "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/optimized", + "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/original", + "objects/590de76f-eca1-4cac-8623-81d980778c96/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794774999364-0", + "outbox": 23 + }, + "created_at": "2026-11-15 20:36:39.425553+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:36:50.267695+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:36:52.972919+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "worker_timeout", + "result": "transient" + }, + "created_at": "2026-11-15 20:37:02.332522+00:00" + }, + { + "event": "retry", + "detail": { + "code": "worker_timeout", + "attempt": 1, + "backoff_seconds": 0.8732405781066988 + }, + "created_at": "2026-11-15 20:37:02.337632+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775023321-0", + "outbox": 24 + }, + "created_at": "2026-11-15 20:37:03.327334+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 20:37:05.140639+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:37:07.142690+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:37:07.376943+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2620389579997209, + "cpu_seconds": 0.22754557900000028, + "rss_peak_mib": 79.73046875 + }, + "created_at": "2026-11-15 20:37:07.477626+00:00" + } + ] + } + }, + { + "case": "after_commit_sigkill_repeat_2", + "at_crash": { + "job": { + "id": "760d739e-0946-468a-a5a9-12b2894d6c27", + "attachment_id": "30baebd5-f3c3-44f7-91b5-57fe4503861c", + "pipeline_version": 1, + "source_key": "objects/staging/30baebd5-f3c3-44f7-91b5-57fe4503861c/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:37:13.467787+00:00", + "error_code": null, + "created_at": "2026-11-15 20:37:13.467787+00:00", + "updated_at": "2026-11-15 20:37:27.022215+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/crop", + "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/optimized", + "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/original", + "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794775033601-0", + "outbox": 25 + }, + "created_at": "2026-11-15 20:37:13.607336+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:37:24.631354+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:37:26.772981+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:37:27.022580+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775033601-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:37:27.861083+00:00" + } + ] + }, + "effect": { + "job": { + "id": "760d739e-0946-468a-a5a9-12b2894d6c27", + "attachment_id": "30baebd5-f3c3-44f7-91b5-57fe4503861c", + "pipeline_version": 1, + "source_key": "objects/staging/30baebd5-f3c3-44f7-91b5-57fe4503861c/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:37:13.467787+00:00", + "error_code": null, + "created_at": "2026-11-15 20:37:13.467787+00:00", + "updated_at": "2026-11-15 20:37:27.022215+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/crop", + "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/optimized", + "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/original", + "objects/30baebd5-f3c3-44f7-91b5-57fe4503861c/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794775033601-0", + "outbox": 25 + }, + "created_at": "2026-11-15 20:37:13.607336+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:37:24.631354+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:37:26.772981+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:37:27.022580+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775033601-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:37:27.861083+00:00" + } + ] + } + }, + { + "case": "transient_retry_with_jitter", + "effect": { + "job": { + "id": "bce86e16-6c52-4947-8b94-ed17b69df55b", + "attachment_id": "07f472a7-9b00-4153-8307-b06bd0a39a8d", + "pipeline_version": 1, + "source_key": "objects/staging/07f472a7-9b00-4153-8307-b06bd0a39a8d/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:38:09.399222+00:00", + "error_code": null, + "created_at": "2026-11-15 20:37:55.139843+00:00", + "updated_at": "2026-11-15 20:38:11.677100+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/07f472a7-9b00-4153-8307-b06bd0a39a8d/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/07f472a7-9b00-4153-8307-b06bd0a39a8d/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/07f472a7-9b00-4153-8307-b06bd0a39a8d/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/07f472a7-9b00-4153-8307-b06bd0a39a8d/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/07f472a7-9b00-4153-8307-b06bd0a39a8d/v1/crop", + "objects/07f472a7-9b00-4153-8307-b06bd0a39a8d/v1/optimized", + "objects/07f472a7-9b00-4153-8307-b06bd0a39a8d/v1/original", + "objects/07f472a7-9b00-4153-8307-b06bd0a39a8d/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794775075318-0", + "outbox": 26 + }, + "created_at": "2026-11-15 20:37:55.323741+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:38:06.421657+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:38:08.356592+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 20:38:08.456980+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 1, + "backoff_seconds": 0.941461107889648 + }, + "created_at": "2026-11-15 20:38:08.458651+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775089510-0", + "outbox": 27 + }, + "created_at": "2026-11-15 20:38:09.515268+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 20:38:09.518861+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:38:11.430068+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:38:11.677376+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.28160174999993615, + "cpu_seconds": 0.24880712500000013, + "rss_peak_mib": 79.8515625 + }, + "created_at": "2026-11-15 20:38:11.832807+00:00" + } + ] + } + }, + { + "case": "hung_child_killed_and_retried", + "effect": { + "job": { + "id": "0c0fa874-bbb1-4e6e-8c4d-d2123250f72e", + "attachment_id": "06b673c6-800b-4688-8ea8-4e42e396dc93", + "pipeline_version": 1, + "source_key": "objects/staging/06b673c6-800b-4688-8ea8-4e42e396dc93/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:38:41.110764+00:00", + "error_code": null, + "created_at": "2026-11-15 20:38:17.573851+00:00", + "updated_at": "2026-11-15 20:38:43.355872+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/06b673c6-800b-4688-8ea8-4e42e396dc93/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/06b673c6-800b-4688-8ea8-4e42e396dc93/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/06b673c6-800b-4688-8ea8-4e42e396dc93/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/06b673c6-800b-4688-8ea8-4e42e396dc93/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/06b673c6-800b-4688-8ea8-4e42e396dc93/v1/crop", + "objects/06b673c6-800b-4688-8ea8-4e42e396dc93/v1/optimized", + "objects/06b673c6-800b-4688-8ea8-4e42e396dc93/v1/original", + "objects/06b673c6-800b-4688-8ea8-4e42e396dc93/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794775097743-0", + "outbox": 28 + }, + "created_at": "2026-11-15 20:38:17.757424+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:38:28.429239+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:38:30.339378+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "worker_timeout", + "result": "transient" + }, + "created_at": "2026-11-15 20:38:40.544843+00:00" + }, + { + "event": "retry", + "detail": { + "code": "worker_timeout", + "attempt": 1, + "backoff_seconds": 0.5645840246252455 + }, + "created_at": "2026-11-15 20:38:40.547793+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775121151-0", + "outbox": 29 + }, + "created_at": "2026-11-15 20:38:41.159193+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 20:38:41.164202+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:38:43.119014+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:38:43.356116+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.26547958299943275, + "cpu_seconds": 0.23297874699999999, + "rss_peak_mib": 79.734375 + }, + "created_at": "2026-11-15 20:38:43.455687+00:00" + } + ] + }, + "observed_limits": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + } + }, + { + "case": "permanent_failed_registry_and_audited_replay", + "failed": { + "id": "4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8", + "chat_id": "98d8f122-f8cf-435c-be59-583b97667d1e", + "owner_id": "fa26fdd2-4267-4706-964c-ebd760e343a1", + "filename": "synthetic.png", + "content_type": "image/png", + "size_bytes": 742, + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "created_at": "2026-11-15T20:38:48.980408Z", + "updated_at": "2026-11-15T20:39:03.438755Z", + "status": "failed", + "variants": [], + "error_code": "invalid_image" + }, + "effect": { + "job": { + "id": "441e0f5a-cf4e-4d34-b08d-8bf63a5ace71", + "attachment_id": "4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8", + "pipeline_version": 1, + "source_key": "objects/staging/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/original", + "state": "ready", + "attempt": 1, + "cycle": 2, + "fence": 4, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:39:07.659023+00:00", + "error_code": null, + "created_at": "2026-11-15 20:38:48.982040+00:00", + "updated_at": "2026-11-15 20:39:09.979880+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/v1/crop", + "objects/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/v1/optimized", + "objects/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/v1/original", + "objects/4aa1d3a5-7a4d-4c6b-88ce-33f09dfe01c8/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794775129172-0", + "outbox": 30 + }, + "created_at": "2026-11-15 20:38:49.176239+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:39:00.596129+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:39:02.599916+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "invalid_image", + "result": "permanent" + }, + "created_at": "2026-11-15 20:39:03.430340+00:00" + }, + { + "event": "failed", + "detail": { + "code": "invalid_image", + "attempt": 1, + "backoff_seconds": 0.7867716366060661 + }, + "created_at": "2026-11-15 20:39:03.439770+00:00" + }, + { + "event": "replay", + "detail": { + "previous_job": "441e0f5a-cf4e-4d34-b08d-8bf63a5ace71" + }, + "created_at": "2026-11-15 20:39:07.661357+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775147735-0", + "outbox": 31 + }, + "created_at": "2026-11-15 20:39:07.740726+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 4, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:39:07.750907+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:39:09.734619+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 4, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:39:09.980287+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.28228149999995367, + "cpu_seconds": 0.24383666199999987, + "rss_peak_mib": 79.75 + }, + "created_at": "2026-11-15 20:39:10.113396+00:00" + } + ] + }, + "ready_replay_rejected": true + }, + { + "case": "finite_retry_exhaustion_then_explicit_replay", + "failed": { + "id": "6e40e091-61d8-4b8f-b4dc-2e26dbe1b019", + "chat_id": "98d8f122-f8cf-435c-be59-583b97667d1e", + "owner_id": "fa26fdd2-4267-4706-964c-ebd760e343a1", + "filename": "synthetic.png", + "content_type": "image/png", + "size_bytes": 742, + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "created_at": "2026-11-15T20:39:20.743932Z", + "updated_at": "2026-11-15T20:39:50.591547Z", + "status": "failed", + "variants": [], + "error_code": "retry_exhausted" + }, + "effect": { + "job": { + "id": "030165d4-972d-4456-a8d9-955a85120a09", + "attachment_id": "6e40e091-61d8-4b8f-b4dc-2e26dbe1b019", + "pipeline_version": 1, + "source_key": "objects/staging/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/original", + "state": "ready", + "attempt": 1, + "cycle": 2, + "fence": 12, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:39:59.841431+00:00", + "error_code": null, + "created_at": "2026-11-15 20:39:20.745669+00:00", + "updated_at": "2026-11-15 20:40:02.325527+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/v1/crop", + "objects/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/v1/optimized", + "objects/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/v1/original", + "objects/6e40e091-61d8-4b8f-b4dc-2e26dbe1b019/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794775160881-0", + "outbox": 32 + }, + "created_at": "2026-11-15 20:39:20.885407+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:39:32.235402+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:39:34.125168+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 20:39:34.211760+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 1, + "backoff_seconds": 1.2866648375283887 + }, + "created_at": "2026-11-15 20:39:34.212727+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775175536-0", + "outbox": 33 + }, + "created_at": "2026-11-15 20:39:35.541877+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 20:39:35.545139+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:39:37.402676+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 20:39:37.495151+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 2, + "backoff_seconds": 1.0989196801088283 + }, + "created_at": "2026-11-15 20:39:37.496314+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775178620-0", + "outbox": 34 + }, + "created_at": "2026-11-15 20:39:38.628009+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 5, + "attempt": 3, + "version": 1 + }, + "created_at": "2026-11-15 20:39:38.630658+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:39:40.657197+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 20:39:40.765673+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 3, + "backoff_seconds": 2.912731852995912 + }, + "created_at": "2026-11-15 20:39:40.766978+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775183797-0", + "outbox": 35 + }, + "created_at": "2026-11-15 20:39:43.801337+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 7, + "attempt": 4, + "version": 1 + }, + "created_at": "2026-11-15 20:39:43.802540+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:39:45.671882+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 20:39:45.781450+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 4, + "backoff_seconds": 2.594216138460152 + }, + "created_at": "2026-11-15 20:39:45.782611+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775188560-0", + "outbox": 36 + }, + "created_at": "2026-11-15 20:39:48.563912+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 9, + "attempt": 5, + "version": 1 + }, + "created_at": "2026-11-15 20:39:48.566032+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:39:50.487885+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 20:39:50.590927+00:00" + }, + { + "event": "failed", + "detail": { + "code": "retry_exhausted", + "attempt": 5, + "backoff_seconds": 21.98370831486453 + }, + "created_at": "2026-11-15 20:39:50.592015+00:00" + }, + { + "event": "replay", + "detail": { + "previous_job": "030165d4-972d-4456-a8d9-955a85120a09" + }, + "created_at": "2026-11-15 20:39:59.843878+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775200050-0", + "outbox": 37 + }, + "created_at": "2026-11-15 20:40:00.056086+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 12, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:40:00.064203+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:40:02.013527+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 12, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:40:02.325858+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.3446279999998296, + "cpu_seconds": 0.3051448689999998, + "rss_peak_mib": 79.73046875 + }, + "created_at": "2026-11-15 20:40:02.476674+00:00" + } + ] + } + }, + { + "case": "reordered_and_duplicate_notifications", + "effects": [ + { + "job": { + "id": "a3878536-8835-4ff8-b051-3af704a36b8f", + "attachment_id": "aae81ab4-ced7-4b50-a3e2-1bd38ed5a234", + "pipeline_version": 1, + "source_key": "objects/staging/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:40:10.338027+00:00", + "error_code": null, + "created_at": "2026-11-15 20:40:10.338027+00:00", + "updated_at": "2026-11-15 20:40:32.822839+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/v1/crop", + "objects/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/v1/optimized", + "objects/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/v1/original", + "objects/aae81ab4-ced7-4b50-a3e2-1bd38ed5a234/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:40:30.495056+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:40:32.568451+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:40:32.823097+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.28610912500062113, + "cpu_seconds": 0.2469917960000001, + "rss_peak_mib": 79.7890625 + }, + "created_at": "2026-11-15 20:40:32.991116+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775227777-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.078142+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775227778-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.078392+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775227779-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.090334+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775227781-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.090559+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775227781-1", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.103713+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775227820-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.104109+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775227822-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.119854+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775234392-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:34.400641+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775234392-0", + "outbox": 38 + }, + "created_at": "2026-11-15 20:40:34.468800+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244937-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:44.976495+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244934-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:44.976628+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244939-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:44.990009+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244940-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:44.990346+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244940-1", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:44.999297+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244941-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:45.000627+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244942-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:45.008765+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244943-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:45.010144+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244944-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:45.022585+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775244945-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:45.022915+00:00" + } + ] + }, + { + "job": { + "id": "06e45874-a726-4e60-9b00-2fa39f91462a", + "attachment_id": "c11f5af4-44f2-47df-93d9-9248fe267bec", + "pipeline_version": 1, + "source_key": "objects/staging/c11f5af4-44f2-47df-93d9-9248fe267bec/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:40:15.202254+00:00", + "error_code": null, + "created_at": "2026-11-15 20:40:15.202254+00:00", + "updated_at": "2026-11-15 20:40:32.822963+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/c11f5af4-44f2-47df-93d9-9248fe267bec/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/c11f5af4-44f2-47df-93d9-9248fe267bec/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/c11f5af4-44f2-47df-93d9-9248fe267bec/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/c11f5af4-44f2-47df-93d9-9248fe267bec/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/c11f5af4-44f2-47df-93d9-9248fe267bec/v1/crop", + "objects/c11f5af4-44f2-47df-93d9-9248fe267bec/v1/optimized", + "objects/c11f5af4-44f2-47df-93d9-9248fe267bec/v1/original", + "objects/c11f5af4-44f2-47df-93d9-9248fe267bec/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:40:30.446987+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:40:32.567586+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 20:40:32.823310+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.28661216700038494, + "cpu_seconds": 0.24859508499999983, + "rss_peak_mib": 79.33203125 + }, + "created_at": "2026-11-15 20:40:32.991322+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775223439-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.695830+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775223440-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.696231+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775223442-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.705671+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775223441-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.705689+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775223443-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.714961+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775223444-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.715387+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775223444-1", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:33.726372+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775234472-0", + "outbox": 39 + }, + "created_at": "2026-11-15 20:40:34.474328+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775234472-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:34.479232+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249028-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.043428+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249025-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.043807+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249029-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.054945+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249031-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.056420+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249032-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.067328+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249033-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.067641+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249034-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.077663+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249035-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.077980+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249036-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.087385+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775249038-0", + "state": "ready" + }, + "created_at": "2026-11-15 20:40:49.087714+00:00" + } + ] + } + ], + "added_duplicates": 36, + "no_extra_attempt_after_ready": true + }, + { + "case": "stale_v1_after_v2_repeat_1", + "v1": { + "job": { + "id": "90ce943c-1a07-4c16-80d4-8d8f25423f6e", + "attachment_id": "d93fddc0-ac9a-40bf-8f43-45f56f976d25", + "pipeline_version": 1, + "source_key": "objects/staging/d93fddc0-ac9a-40bf-8f43-45f56f976d25/original", + "state": "obsolete", + "attempt": 1, + "cycle": 1, + "fence": 2, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:41:08.452978+00:00", + "error_code": null, + "created_at": "2026-11-15 20:41:08.452978+00:00", + "updated_at": "2026-11-15 20:41:34.395893+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v1/crop", + "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v1/optimized", + "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v1/original", + "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794775268516-0", + "outbox": 40 + }, + "created_at": "2026-11-15 20:41:08.526228+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775283705-0", + "outbox": 41 + }, + "created_at": "2026-11-15 20:41:23.709433+00:00" + }, + { + "event": "version_controller_waiting", + "detail": {}, + "created_at": "2026-11-15 20:41:27.122131+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:41:34.397674+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:41:36.282725+00:00" + }, + { + "event": "stale_completion_rejected", + "detail": { + "fence": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:41:38.834601+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775268516-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:38.887075+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775283705-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:38.892943+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "stale" + }, + "created_at": "2026-11-15 20:41:38.992209+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308761-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.770748+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308764-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.771391+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308765-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.781251+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308766-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.782276+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308768-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.789264+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308769-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.789785+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308769-1", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.794573+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308770-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.795175+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308771-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.799185+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308773-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.799791+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308774-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.804072+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775308774-1", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:41:48.804819+00:00" + } + ] + }, + "v2": { + "job": { + "id": "a131fd82-344a-4a6a-9b20-06e70b896d0b", + "attachment_id": "d93fddc0-ac9a-40bf-8f43-45f56f976d25", + "pipeline_version": 2, + "source_key": "objects/staging/d93fddc0-ac9a-40bf-8f43-45f56f976d25/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:41:36.591313+00:00", + "error_code": null, + "created_at": "2026-11-15 20:41:36.591313+00:00", + "updated_at": "2026-11-15 20:41:38.753737+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v2/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v2/optimized", + "sha256": "f2b2ece82b03105a787a0a9b75ce63787364fa39329cd6c21e23f44ec2ef2ced", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v2/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v2/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v2/crop", + "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v2/optimized", + "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v2/original", + "objects/d93fddc0-ac9a-40bf-8f43-45f56f976d25/v2/thumbnail" + ], + "events": [ + { + "event": "new_version", + "detail": { + "previous_job": "90ce943c-1a07-4c16-80d4-8d8f25423f6e" + }, + "created_at": "2026-11-15 20:41:36.593847+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775296653-0", + "outbox": 42 + }, + "created_at": "2026-11-15 20:41:36.658428+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 2 + }, + "created_at": "2026-11-15 20:41:36.659959+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:41:38.510633+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 2, + "variants": 4 + }, + "created_at": "2026-11-15 20:41:38.754008+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.27137612600017746, + "cpu_seconds": 0.24279383700000023, + "rss_peak_mib": 79.78125 + }, + "created_at": "2026-11-15 20:41:38.874944+00:00" + } + ] + }, + "new_hashes_unchanged": true + }, + { + "case": "stale_v1_after_v2_repeat_2", + "v1": { + "job": { + "id": "c9ff05d2-7fe8-4b1f-934a-e83572707614", + "attachment_id": "4a818886-a36d-492f-a356-8f119eb36457", + "pipeline_version": 1, + "source_key": "objects/staging/4a818886-a36d-492f-a356-8f119eb36457/original", + "state": "obsolete", + "attempt": 1, + "cycle": 1, + "fence": 2, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:42:12.041598+00:00", + "error_code": null, + "created_at": "2026-11-15 20:42:12.041598+00:00", + "updated_at": "2026-11-15 20:42:37.014637+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/4a818886-a36d-492f-a356-8f119eb36457/v1/crop", + "objects/4a818886-a36d-492f-a356-8f119eb36457/v1/optimized", + "objects/4a818886-a36d-492f-a356-8f119eb36457/v1/original", + "objects/4a818886-a36d-492f-a356-8f119eb36457/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794775332158-0", + "outbox": 43 + }, + "created_at": "2026-11-15 20:42:12.165123+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775347185-0", + "outbox": 44 + }, + "created_at": "2026-11-15 20:42:27.194548+00:00" + }, + { + "event": "version_controller_waiting", + "detail": {}, + "created_at": "2026-11-15 20:42:30.145665+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:42:37.016243+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:42:39.040876+00:00" + }, + { + "event": "stale_completion_rejected", + "detail": { + "fence": 1, + "version": 1 + }, + "created_at": "2026-11-15 20:42:41.648675+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775332158-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:41.820730+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775347185-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:41.836455+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "stale" + }, + "created_at": "2026-11-15 20:42:41.907292+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370021-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.030566+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370018-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.030569+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370023-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.041619+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370025-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.042919+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370026-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.048086+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370028-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.048296+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370029-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.053246+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370030-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.053925+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370031-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.059020+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370033-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.059407+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370035-1", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.065127+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794775370035-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 20:42:50.065845+00:00" + } + ] + }, + "v2": { + "job": { + "id": "d3a419fa-e9ca-4ca8-9e62-4fe224ef6ccf", + "attachment_id": "4a818886-a36d-492f-a356-8f119eb36457", + "pipeline_version": 2, + "source_key": "objects/staging/4a818886-a36d-492f-a356-8f119eb36457/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 20:42:39.348499+00:00", + "error_code": null, + "created_at": "2026-11-15 20:42:39.348499+00:00", + "updated_at": "2026-11-15 20:42:41.599185+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/4a818886-a36d-492f-a356-8f119eb36457/v2/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/4a818886-a36d-492f-a356-8f119eb36457/v2/optimized", + "sha256": "f2b2ece82b03105a787a0a9b75ce63787364fa39329cd6c21e23f44ec2ef2ced", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/4a818886-a36d-492f-a356-8f119eb36457/v2/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/4a818886-a36d-492f-a356-8f119eb36457/v2/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/4a818886-a36d-492f-a356-8f119eb36457/v2/crop", + "objects/4a818886-a36d-492f-a356-8f119eb36457/v2/optimized", + "objects/4a818886-a36d-492f-a356-8f119eb36457/v2/original", + "objects/4a818886-a36d-492f-a356-8f119eb36457/v2/thumbnail" + ], + "events": [ + { + "event": "new_version", + "detail": { + "previous_job": "c9ff05d2-7fe8-4b1f-934a-e83572707614" + }, + "created_at": "2026-11-15 20:42:39.351053+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794775359398-0", + "outbox": 45 + }, + "created_at": "2026-11-15 20:42:39.403387+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 2 + }, + "created_at": "2026-11-15 20:42:39.412390+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 20:42:41.344884+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 2, + "variants": 4 + }, + "created_at": "2026-11-15 20:42:41.599440+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.28745358400010446, + "cpu_seconds": 0.2529595779999998, + "rss_peak_mib": 79.85546875 + }, + "created_at": "2026-11-15 20:42:41.808234+00:00" + } + ] + }, + "new_hashes_unchanged": true + }, + { + "case": "malformed_notification_rejected", + "durable_diagnostic": true + }, + { + "case": "private_iam_and_other_api_download", + "security": { + "anonymous": 403, + "iam": { + "create_bucket": "AccessDenied", + "write_outside_prefix": "AccessDenied", + "list_other_bucket": "AccessDenied" + } + }, + "attachment": "5a62a72b-e2b0-424f-926b-9e4a22c9f53d", + "uploader": "ea289153015c4784a656d9b1d302dbde", + "downloader": "5f8f83d988fc4479b4865a203c766f1f", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df" + } + ], + "worker_metrics": [ + { + "job_id": "5b31c9b5-e415-4916-b879-f863098efb3f", + "detail": { + "result": "ready", + "seconds": 0.2819997910000893, + "cpu_seconds": 0.24735475399999984, + "rss_peak_mib": 79.609375 + } + }, + { + "job_id": "ebfdafb4-422c-4dab-84fc-8cac7b6690a5", + "detail": { + "result": "ready", + "seconds": 0.2749179169995841, + "cpu_seconds": 0.24024937599999996, + "rss_peak_mib": 79.61328125 + } + }, + { + "job_id": "fb8e6292-7c89-4985-9379-d6129d94b8bc", + "detail": { + "result": "ready", + "seconds": 0.2542630829993868, + "cpu_seconds": 0.22826758300000005, + "rss_peak_mib": 79.72265625 + } + }, + { + "job_id": "082b41be-2f81-4127-9340-bc9276fa01b1", + "detail": { + "result": "ready", + "seconds": 0.2787759169996207, + "cpu_seconds": 0.24001637300000023, + "rss_peak_mib": 79.48828125 + } + }, + { + "job_id": "52e9a948-1efd-4914-95f9-5d3a3fc7d2fc", + "detail": { + "result": "ready", + "seconds": 0.29629000000022643, + "cpu_seconds": 0.2464472950000003, + "rss_peak_mib": 79.72265625 + } + }, + { + "job_id": "f60bb029-9f97-485f-a301-ef3d7ad509c4", + "detail": { + "result": "ready", + "seconds": 0.2661684159993456, + "cpu_seconds": 0.23351224500000023, + "rss_peak_mib": 79.859375 + } + }, + { + "job_id": "60afcdf4-d40f-4c90-999f-7c668eb2390f", + "detail": { + "result": "ready", + "seconds": 0.485136500000408, + "cpu_seconds": 0.4508282039999998, + "rss_peak_mib": 108.73828125 + } + }, + { + "job_id": "246bb511-bdc1-4a45-a0ee-fd20ab22f2f9", + "detail": { + "result": "ready", + "seconds": 0.2561037919995215, + "cpu_seconds": 0.2209544109999999, + "rss_peak_mib": 79.50390625 + } + }, + { + "job_id": "89c40d3f-711a-430c-a175-6a1a0c48ef9e", + "detail": { + "result": "ready", + "seconds": 0.3143088329998136, + "cpu_seconds": 0.27227425500000013, + "rss_peak_mib": 79.75 + } + }, + { + "job_id": "dad2632b-589e-4fa6-9f7f-d416444590b9", + "detail": { + "result": "ready", + "seconds": 0.2876035839999531, + "cpu_seconds": 0.24817545600000024, + "rss_peak_mib": 79.7265625 + } + }, + { + "job_id": "5c269448-69fd-47bc-8eae-e066be853c8e", + "detail": { + "result": "ready", + "seconds": 0.29481404200032557, + "cpu_seconds": 0.263315376, + "rss_peak_mib": 79.734375 + } + }, + { + "job_id": "fe1c5591-95fc-4ba3-88fb-2fa26badcbde", + "detail": { + "result": "ready", + "seconds": 0.31842683299964847, + "cpu_seconds": 0.26803228499999987, + "rss_peak_mib": 79.734375 + } + }, + { + "job_id": "7e6614be-b097-46d0-8398-47ba5ce3444a", + "detail": { + "result": "ready", + "seconds": 0.2639559589997589, + "cpu_seconds": 0.23261253799999992, + "rss_peak_mib": 79.73046875 + } + }, + { + "job_id": "c7761c1b-92ce-40cd-9058-bdad8f5f189c", + "detail": { + "result": "ready", + "seconds": 0.2951308749998134, + "cpu_seconds": 0.2648634940000001, + "rss_peak_mib": 79.609375 + } + }, + { + "job_id": "0048f677-7bca-4f87-8cdf-1a60d6e3bd2c", + "detail": { + "result": "ready", + "seconds": 0.2813682500000141, + "cpu_seconds": 0.23825850199999987, + "rss_peak_mib": 79.734375 + } + }, + { + "job_id": "6cc8e812-e4ae-4d36-a230-d5235fa45b4b", + "detail": { + "result": "ready", + "seconds": 0.27222879200053285, + "cpu_seconds": 0.2380743289999998, + "rss_peak_mib": 79.97265625 + } + }, + { + "job_id": "beeb7929-64e8-4ed5-9990-16f0dd1d5aa8", + "detail": { + "code": "worker_timeout", + "result": "transient" + } + }, + { + "job_id": "beeb7929-64e8-4ed5-9990-16f0dd1d5aa8", + "detail": { + "result": "ready", + "seconds": 0.2685703340002874, + "cpu_seconds": 0.23196675100000008, + "rss_peak_mib": 79.7109375 + } + }, + { + "job_id": "347bd84b-610b-427f-98e0-b6a290923456", + "detail": { + "result": "ready", + "seconds": 0.2644206660006603, + "cpu_seconds": 0.23357183000000004, + "rss_peak_mib": 79.86328125 + } + }, + { + "job_id": "ea0a2fc7-4484-46fa-8c13-b4523eebd673", + "detail": { + "code": "worker_timeout", + "result": "transient" + } + }, + { + "job_id": "ea0a2fc7-4484-46fa-8c13-b4523eebd673", + "detail": { + "result": "ready", + "seconds": 0.2620389579997209, + "cpu_seconds": 0.22754557900000028, + "rss_peak_mib": 79.73046875 + } + }, + { + "job_id": "bce86e16-6c52-4947-8b94-ed17b69df55b", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "bce86e16-6c52-4947-8b94-ed17b69df55b", + "detail": { + "result": "ready", + "seconds": 0.28160174999993615, + "cpu_seconds": 0.24880712500000013, + "rss_peak_mib": 79.8515625 + } + }, + { + "job_id": "0c0fa874-bbb1-4e6e-8c4d-d2123250f72e", + "detail": { + "code": "worker_timeout", + "result": "transient" + } + }, + { + "job_id": "0c0fa874-bbb1-4e6e-8c4d-d2123250f72e", + "detail": { + "result": "ready", + "seconds": 0.26547958299943275, + "cpu_seconds": 0.23297874699999999, + "rss_peak_mib": 79.734375 + } + }, + { + "job_id": "441e0f5a-cf4e-4d34-b08d-8bf63a5ace71", + "detail": { + "code": "invalid_image", + "result": "permanent" + } + }, + { + "job_id": "441e0f5a-cf4e-4d34-b08d-8bf63a5ace71", + "detail": { + "result": "ready", + "seconds": 0.28228149999995367, + "cpu_seconds": 0.24383666199999987, + "rss_peak_mib": 79.75 + } + }, + { + "job_id": "030165d4-972d-4456-a8d9-955a85120a09", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "030165d4-972d-4456-a8d9-955a85120a09", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "030165d4-972d-4456-a8d9-955a85120a09", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "030165d4-972d-4456-a8d9-955a85120a09", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "030165d4-972d-4456-a8d9-955a85120a09", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "030165d4-972d-4456-a8d9-955a85120a09", + "detail": { + "result": "ready", + "seconds": 0.3446279999998296, + "cpu_seconds": 0.3051448689999998, + "rss_peak_mib": 79.73046875 + } + }, + { + "job_id": "a3878536-8835-4ff8-b051-3af704a36b8f", + "detail": { + "result": "ready", + "seconds": 0.28610912500062113, + "cpu_seconds": 0.2469917960000001, + "rss_peak_mib": 79.7890625 + } + }, + { + "job_id": "06e45874-a726-4e60-9b00-2fa39f91462a", + "detail": { + "result": "ready", + "seconds": 0.28661216700038494, + "cpu_seconds": 0.24859508499999983, + "rss_peak_mib": 79.33203125 + } + }, + { + "job_id": "a131fd82-344a-4a6a-9b20-06e70b896d0b", + "detail": { + "result": "ready", + "seconds": 0.27137612600017746, + "cpu_seconds": 0.24279383700000023, + "rss_peak_mib": 79.78125 + } + }, + { + "job_id": "90ce943c-1a07-4c16-80d4-8d8f25423f6e", + "detail": { + "result": "stale" + } + }, + { + "job_id": "d3a419fa-e9ca-4ca8-9e62-4fe224ef6ccf", + "detail": { + "result": "ready", + "seconds": 0.28745358400010446, + "cpu_seconds": 0.2529595779999998, + "rss_peak_mib": 79.85546875 + } + }, + { + "job_id": "c9ff05d2-7fe8-4b1f-934a-e83572707614", + "detail": { + "result": "stale" + } + }, + { + "job_id": "dda2169e-2083-40ef-8a3e-0c385a3f03d4", + "detail": { + "result": "ready", + "seconds": 0.30348837499968795, + "cpu_seconds": 0.2656605380000001, + "rss_peak_mib": 79.73828125 + } + } + ], + "audit": [ + { + "job_id": "fe1c5591-95fc-4ba3-88fb-2fa26badcbde", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 20:31:45.094363+00:00" + }, + { + "job_id": "441e0f5a-cf4e-4d34-b08d-8bf63a5ace71", + "action": "replay", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 20:39:07.660718+00:00" + }, + { + "job_id": "030165d4-972d-4456-a8d9-955a85120a09", + "action": "replay", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 20:39:59.843247+00:00" + }, + { + "job_id": "a131fd82-344a-4a6a-9b20-06e70b896d0b", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Held stale writer race", + "created_at": "2026-11-15 20:41:36.593403+00:00" + }, + { + "job_id": "d3a419fa-e9ca-4ca8-9e62-4fe224ef6ccf", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Held stale writer race", + "created_at": "2026-11-15 20:42:39.350444+00:00" + } + ] + }, + "cleanup": { + "source_commit": "eb6d229afb1437cd0f8b5e2ee0ca384d584ddd8c", + "seed": "clean-gc-seed.json", + "complete": true, + "real_s3_age_seconds": 733.85265, + "dry_run": { + "apply": false, + "retention_seconds": 300, + "actions": [ + { + "action": "delete_expired_object", + "key": "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/crop", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/optimized", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/thumbnail", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/cleanup-3174841402a74a1fb5b6e7241a5e8d2a/old-orphan", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/30baebd5-f3c3-44f7-91b5-57fe4503861c/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/42e402e7-8c37-4c43-a0cd-b456575b21b6/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/6df2b381-f693-4cc3-be87-ceced86eecb8/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/ef82bc3b-0c79-482f-b73f-156b7f378015/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/f973cbdb-f407-4a48-914e-fdf85f1a258d/original", + "applied": false + } + ] + }, + "apply": { + "apply": true, + "retention_seconds": 300, + "actions": [ + { + "action": "delete_expired_object", + "key": "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/crop", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/optimized", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/thumbnail", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/cleanup-3174841402a74a1fb5b6e7241a5e8d2a/old-orphan", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/30baebd5-f3c3-44f7-91b5-57fe4503861c/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/42e402e7-8c37-4c43-a0cd-b456575b21b6/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/6df2b381-f693-4cc3-be87-ceced86eecb8/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/ef82bc3b-0c79-482f-b73f-156b7f378015/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/f973cbdb-f407-4a48-914e-fdf85f1a258d/original", + "applied": true + } + ] + }, + "repeat": { + "apply": true, + "retention_seconds": 300, + "actions": [] + }, + "before": { + "objects/cleanup-3174841402a74a1fb5b6e7241a5e8d2a/old-orphan": true, + "objects/staging/6df2b381-f693-4cc3-be87-ceced86eecb8/original": true, + "objects/staging/f973cbdb-f407-4a48-914e-fdf85f1a258d/original": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/original": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/thumbnail": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/crop": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/optimized": true, + "objects/staging/d86d3319-0b87-443b-a8e7-b9d9e803c2e0/original": true, + "objects/staging/6ca87961-0147-4da6-a1d5-6207025ecde6/original": true, + "objects/cleanup-3174841402a74a1fb5b6e7241a5e8d2a/fresh-orphan": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v2/original": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v2/thumbnail": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v2/crop": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v2/optimized": true + }, + "after": { + "objects/cleanup-3174841402a74a1fb5b6e7241a5e8d2a/old-orphan": false, + "objects/staging/6df2b381-f693-4cc3-be87-ceced86eecb8/original": false, + "objects/staging/f973cbdb-f407-4a48-914e-fdf85f1a258d/original": false, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/original": false, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/thumbnail": false, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/crop": false, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v1/optimized": false, + "objects/staging/d86d3319-0b87-443b-a8e7-b9d9e803c2e0/original": true, + "objects/staging/6ca87961-0147-4da6-a1d5-6207025ecde6/original": true, + "objects/cleanup-3174841402a74a1fb5b6e7241a5e8d2a/fresh-orphan": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v2/original": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v2/thumbnail": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v2/crop": true, + "objects/8c00cc03-e1cc-4d0b-a5de-c5af7d1c818d/v2/optimized": true + }, + "dry_run_unchanged": true, + "active_current_and_fresh_protected": true, + "expired_source_replay_refused": true, + "expired_writer_fenced": true + }, + "cluster": { + "date": "2026-11-15T20:28:04.394923+00:00", + "source_commit": "eb6d229afb1437cd0f8b5e2ee0ca384d584ddd8c", + "complete": true, + "before": [ + { + "role": "api", + "container": "8bbd861211fd", + "pid": 59296, + "started_at": "2026-11-15T20:21:21.104054635Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "079d9eff1d0c4c8cbc0fc0d4b89cb455" + }, + { + "role": "api", + "container": "ebc7e4c3838d", + "pid": 59353, + "started_at": "2026-11-15T20:21:21.27865926Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "role": "broker", + "container": "55ab9f986bae", + "pid": 58562, + "started_at": "2026-11-15T20:21:12.353588672Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "7014391ecf89", + "pid": 58493, + "started_at": "2026-11-15T20:21:12.04480938Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "fa3c50f83a1f", + "pid": 58431, + "started_at": "2026-11-15T20:21:11.739337255Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "242a49094f0d", + "pid": 60206, + "started_at": "2026-11-15T20:21:33.497731001Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "61281" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "932bb6635829", + "pid": 58277, + "started_at": "2026-11-15T20:21:11.12347638Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "6f6af6bdcd00", + "pid": 59239, + "started_at": "2026-11-15T20:21:20.927591635Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "a4dba713c111", + "pid": 58990, + "started_at": "2026-11-15T20:21:17.651360383Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "ca00d9061bb8", + "pid": 58361, + "started_at": "2026-11-15T20:21:11.435409755Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "1963232f6596", + "pid": 59409, + "started_at": "2026-11-15T20:21:21.488607385Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ], + "parallel_replay": { + "requests": 30, + "messages": 1, + "instances": [ + "079d9eff1d0c4c8cbc0fc0d4b89cb455", + "6e400225cb214074bf1ffd479b9a828e" + ] + }, + "atomic_rollback_and_5xx_retry": true, + "expired_key_can_be_reused": true, + "failovers": [ + { + "confirmed_writes": 217, + "successful_reads": 209, + "errors": 8, + "successful_rps": 28.29, + "first_success_after_action_seconds": 3.881, + "action": { + "finished": 3.821 + }, + "observations": [ + { + "at": 0.007, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.062, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.129, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.183, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.271, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.333, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.392, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.454, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.52, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.586, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.649, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.713, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.766, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.829, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.888, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.959, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.047, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.116, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.181, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.246, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.312, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.376, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.431, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.487, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.545, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.604, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.66, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.721, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.793, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.897, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.996, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.098, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.198, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.291, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.35, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.408, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.492, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.605, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.694, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.796, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.902, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.963, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.034, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.105, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.163, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.223, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.287, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.348, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.406, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.464, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.517, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.577, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.65, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.726, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.799, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.881, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 3.95, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.017, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.089, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.153, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.221, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.283, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.351, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.416, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.488, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.558, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.633, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.718, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.784, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.863, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.93, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 4.987, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.083, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.179, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.247, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.309, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.374, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.45, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.526, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.592, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.655, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.713, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.78, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.853, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 5.94, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.013, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.091, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.196, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.296, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.397, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.502, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.599, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.701, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.763, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.82, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.884, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 6.951, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.025, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.092, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.164, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.237, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.296, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.357, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.424, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.488, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.554, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.615, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.681, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.746, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.813, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.879, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 7.947, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.013, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.08, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.142, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.228, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.294, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.355, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.42, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.487, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.549, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.614, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.69, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.765, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.842, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.912, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 8.986, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.05, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.115, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.195, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.271, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.331, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.397, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.463, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.533, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.596, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.656, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.718, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.78, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.846, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.913, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 9.974, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.046, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.104, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.164, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.229, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.296, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.357, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.416, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.477, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.543, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.617, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.691, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.763, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.833, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.903, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 10.969, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.044, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.12, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.194, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.258, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.328, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.386, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.441, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.508, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.572, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.637, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.696, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.761, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.826, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.892, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 11.961, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.027, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.088, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.168, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.231, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.298, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.361, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.42, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.483, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.553, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.613, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.672, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.735, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.804, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.872, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 12.941, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.016, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.083, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.149, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.222, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.293, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.365, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.429, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.488, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.548, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.612, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.679, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.746, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.81, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.867, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.93, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 13.998, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.064, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.151, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.216, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.292, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.356, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.425, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.49, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.567, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.635, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.701, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.768, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.836, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.902, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 14.974, + "instance": "6e400225cb214074bf1ffd479b9a828e" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "8bbd861211fd", + "pid": 59296, + "started_at": "2026-11-15T20:21:21.104054635Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "079d9eff1d0c4c8cbc0fc0d4b89cb455" + } + }, + { + "confirmed_writes": 214, + "successful_reads": 208, + "errors": 7, + "successful_rps": 28.03, + "first_success_after_action_seconds": 3.725, + "action": { + "finished": 3.682 + }, + "observations": [ + { + "at": 0.017, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.088, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.155, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.215, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.276, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.337, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.406, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.473, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.529, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.589, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.642, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.7, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.754, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.812, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.865, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 0.939, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.046, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.133, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.239, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.35, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.454, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.54, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.659, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.775, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.853, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 1.954, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.053, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.151, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.242, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.347, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.444, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.547, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.645, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.702, + "instance": "6e400225cb214074bf1ffd479b9a828e" + }, + { + "at": 2.771, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.833, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.896, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.969, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.035, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.097, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.161, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.237, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.294, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.355, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.412, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.473, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.543, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.606, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.725, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.814, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.902, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.977, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.038, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.098, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.163, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.221, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.279, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.346, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.407, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.476, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.546, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.61, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.675, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.739, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.804, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.861, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.919, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.979, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.048, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.115, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.181, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.249, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.324, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.393, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.465, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.531, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.598, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.665, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.732, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.814, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.884, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.947, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.011, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.073, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.136, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.204, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.273, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.34, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.399, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.458, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.518, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.581, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.645, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.711, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.77, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.832, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.898, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.972, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.039, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.113, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.18, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.25, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.316, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.383, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.45, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.522, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.588, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.662, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.732, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.81, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.878, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.951, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.019, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.079, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.145, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.212, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.281, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.347, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.415, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.481, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.55, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.615, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.679, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.744, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.81, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.876, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.94, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.002, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.066, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.182, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.288, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.379, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.49, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.581, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.68, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.746, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.849, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.92, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.979, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.043, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.101, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.16, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.223, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.287, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.35, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.416, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.484, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.556, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.626, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.697, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.762, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.828, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.885, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.956, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.02, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.086, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.148, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.212, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.278, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.337, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.403, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.461, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.525, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.598, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.672, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.742, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.817, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.889, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.978, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.046, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.112, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.177, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.244, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.313, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.38, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.45, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.524, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.594, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.666, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.74, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.813, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.874, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.934, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.995, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.053, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.12, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.188, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.255, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.319, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.385, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.445, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.511, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.569, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.63, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.687, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.752, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.822, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.89, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.971, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.038, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.099, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.164, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.228, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.294, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.364, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.43, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.498, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.564, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.631, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.694, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.759, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.829, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.897, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.967, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "ebc7e4c3838d", + "pid": 59353, + "started_at": "2026-11-15T20:21:21.27865926Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "6e400225cb214074bf1ffd479b9a828e" + } + } + ], + "rolling": { + "confirmed_writes": 618, + "successful_reads": 617, + "errors": 35, + "successful_rps": 27.33, + "first_success_after_action_seconds": 40.902, + "action": { + "finished": 40.842 + }, + "observations": [ + { + "at": 0.022, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 0.102, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 0.159, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 0.225, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 0.291, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 0.363, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 0.456, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 0.989, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.098, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.157, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.224, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.286, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.349, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.407, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.463, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.528, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.588, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.65, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.719, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.786, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.843, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.898, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 1.955, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.009, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.074, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.131, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.198, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.259, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.333, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.432, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.503, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.578, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.641, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.73, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.794, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.858, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.916, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 2.976, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.106, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.175, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.24, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.303, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.368, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.434, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.504, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.578, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.645, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.705, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.769, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.835, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.902, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 3.969, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.034, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.096, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.168, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.236, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.3, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.366, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.426, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.482, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.547, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.619, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.686, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.752, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.814, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.873, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.928, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 4.984, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.049, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.113, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.176, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.238, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.297, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.357, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.424, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.483, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.543, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.612, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.673, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.745, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.814, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.879, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 5.946, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.014, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.076, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.149, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.21, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.273, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.33, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.395, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.453, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.521, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.582, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.654, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.724, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.79, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.853, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.925, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 6.997, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.163, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.262, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.361, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.463, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.573, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.689, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.746, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.808, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.868, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 7.934, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.001, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.074, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.143, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.208, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.272, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.352, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.414, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.475, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.531, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.598, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.687, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.772, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.84, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.897, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 8.959, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.022, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.083, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.149, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.21, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.28, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.347, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.405, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.462, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.525, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.587, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.653, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.718, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.784, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.851, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.913, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 9.982, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.049, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.107, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.173, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.234, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.296, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.358, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.421, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.49, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.558, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.624, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.684, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.754, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.824, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.884, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 10.943, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.007, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.076, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.139, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.198, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.276, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.349, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.412, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.475, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.541, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.605, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.665, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.736, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.794, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.851, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.915, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 11.984, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.046, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.116, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.174, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.24, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.311, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.382, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.453, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.511, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.568, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.628, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.695, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.801, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.863, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.932, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 12.997, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.06, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.123, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.187, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.251, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.315, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.387, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.45, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.52, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.585, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.648, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.734, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 13.814, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 13.874, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 13.941, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 14.01, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 14.075, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.139, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 14.2, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 14.265, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.328, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 14.394, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 14.465, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.528, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 14.609, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 14.689, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.757, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 14.842, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 14.928, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 14.995, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 15.059, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 15.128, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 15.19, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 15.251, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 15.321, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 15.39, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 15.458, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 15.528, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 15.596, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 15.664, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 15.725, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 15.79, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 15.908, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 15.99, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 16.051, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 16.122, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 16.185, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 16.253, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 16.317, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 16.382, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 16.45, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 16.51, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 16.579, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 16.65, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 16.711, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 16.789, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "at": 16.862, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 16.932, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 16.995, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 17.07, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 17.138, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 17.207, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 17.278, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 17.343, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 17.432, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 18.585, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 18.663, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 18.737, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 18.81, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 18.879, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 18.956, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.028, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.092, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.163, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.229, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.303, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.365, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.433, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.493, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.553, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.611, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.679, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.744, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.815, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.882, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 19.945, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.012, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.075, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.134, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.201, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.267, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.337, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.399, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.46, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.523, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.59, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.661, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.731, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.795, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.863, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.924, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 20.996, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.061, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.125, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.193, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.263, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.332, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.401, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.475, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.548, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.605, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.663, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.729, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.79, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.854, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.921, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 21.981, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 22.04, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 22.102, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 22.159, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.232, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.305, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.371, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.444, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.514, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.573, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.639, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.71, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.779, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.844, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.91, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 22.968, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.031, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.096, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.158, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.224, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.281, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.351, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.446, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.547, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.645, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.747, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.845, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 23.947, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.01, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.096, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.182, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.25, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.309, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.48, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.548, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.608, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.683, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.75, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.818, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.881, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 24.952, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.019, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.083, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.146, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.21, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.274, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.342, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.412, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.481, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.55, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.619, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.684, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.752, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.816, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.876, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 25.943, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.009, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.075, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.136, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.206, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.269, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.339, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.401, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.459, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.518, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.589, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.659, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.718, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.776, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.844, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.91, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 26.994, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.054, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.118, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.182, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.249, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.308, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.374, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.432, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.492, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.551, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.618, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.685, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.755, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.817, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.885, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 27.964, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.045, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.116, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.196, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.266, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.351, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.427, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.504, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.625, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.714, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.816, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 28.909, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 29.004, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 29.171, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 29.406, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 29.522, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 29.705, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 29.909, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.121, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.213, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.31, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.383, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.444, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.51, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.578, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.639, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.698, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.775, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.84, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.907, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 30.968, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.031, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.094, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.156, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.238, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.309, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.384, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.464, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.538, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.612, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.693, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.763, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.836, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.911, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 31.981, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.05, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.107, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.182, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.253, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.313, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.385, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.443, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.512, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.578, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.648, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.71, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.773, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.845, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.929, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 32.993, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 33.179, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 33.277, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 33.36, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 33.433, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 33.496, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 33.578, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 33.643, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 33.706, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 33.769, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 33.837, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 33.9, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 33.968, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 34.041, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 34.117, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 34.245, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 34.31, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 34.402, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 34.47, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 34.533, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 34.6, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 34.671, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 34.737, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 34.803, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 34.871, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 34.943, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 35.008, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 35.076, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 35.142, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 35.209, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 35.267, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 35.333, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 35.401, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 35.46, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 35.532, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 35.599, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 35.66, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 35.721, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 35.788, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 35.854, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 35.916, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 35.984, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 36.045, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 36.105, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 36.175, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 36.237, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 36.301, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 36.359, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 36.424, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 36.492, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 36.56, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "at": 36.628, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "at": 36.691, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 36.753, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 36.813, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 36.892, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 36.963, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.031, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.1, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.171, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.241, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.311, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.383, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.45, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.529, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.609, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.689, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.77, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.843, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.911, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 37.977, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.043, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.105, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.168, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.231, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.3, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.368, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.433, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.494, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.568, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.628, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.694, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.763, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.829, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.895, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 38.961, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.03, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.096, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.175, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.241, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.301, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.366, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.434, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.493, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.56, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.627, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.692, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.758, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.823, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.893, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 39.96, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.024, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.093, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.159, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.218, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.284, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.35, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.434, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.497, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.564, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.627, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.69, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.758, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.833, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.902, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 40.978, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.048, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.11, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.178, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.248, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.311, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.372, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.438, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.511, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.58, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.642, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.712, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.782, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.848, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.913, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 41.977, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.05, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.108, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.168, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.226, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.284, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.355, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.419, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.484, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.549, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.609, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.677, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.743, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.807, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.878, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 42.952, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.029, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.095, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.159, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.227, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.297, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.377, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.486, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.584, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.679, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.779, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.876, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 43.943, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.009, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.077, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.144, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.213, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.277, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.343, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.405, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.476, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.546, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.61, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.687, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.754, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.812, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "at": 44.904, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "before": [ + { + "role": "api", + "container": "8bbd861211fd", + "pid": 99583, + "started_at": "2026-11-15T20:28:29.115127097Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "a59964a4bea444e08b62548bbeb29bb2" + }, + { + "role": "api", + "container": "ebc7e4c3838d", + "pid": 1593, + "started_at": "2026-11-15T20:29:01.248059875Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "792629b652374561a6a278f92e7e84ab" + }, + { + "role": "broker", + "container": "55ab9f986bae", + "pid": 58562, + "started_at": "2026-11-15T20:21:12.353588672Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "7014391ecf89", + "pid": 58493, + "started_at": "2026-11-15T20:21:12.04480938Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "fa3c50f83a1f", + "pid": 58431, + "started_at": "2026-11-15T20:21:11.739337255Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "242a49094f0d", + "pid": 60206, + "started_at": "2026-11-15T20:21:33.497731001Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "61281" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "932bb6635829", + "pid": 58277, + "started_at": "2026-11-15T20:21:11.12347638Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "6f6af6bdcd00", + "pid": 59239, + "started_at": "2026-11-15T20:21:20.927591635Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "a4dba713c111", + "pid": 58990, + "started_at": "2026-11-15T20:21:17.651360383Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "ca00d9061bb8", + "pid": 58361, + "started_at": "2026-11-15T20:21:11.435409755Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "1963232f6596", + "pid": 59409, + "started_at": "2026-11-15T20:21:21.488607385Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ], + "after": [ + { + "role": "api", + "container": "b8473993e183", + "pid": 2444, + "started_at": "2026-11-15T20:29:17.8060713Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "4a7cf90bb93c4e7a8fb5ded631682004" + }, + { + "role": "api", + "container": "5afc95b85319", + "pid": 3409, + "started_at": "2026-11-15T20:29:36.574748503Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "7391d2eaedad448fb52f3e4ce9d34529" + }, + { + "role": "broker", + "container": "55ab9f986bae", + "pid": 58562, + "started_at": "2026-11-15T20:21:12.353588672Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "7014391ecf89", + "pid": 58493, + "started_at": "2026-11-15T20:21:12.04480938Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "fa3c50f83a1f", + "pid": 58431, + "started_at": "2026-11-15T20:21:11.739337255Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "242a49094f0d", + "pid": 60206, + "started_at": "2026-11-15T20:21:33.497731001Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "61281" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "932bb6635829", + "pid": 58277, + "started_at": "2026-11-15T20:21:11.12347638Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "6f6af6bdcd00", + "pid": 59239, + "started_at": "2026-11-15T20:21:20.927591635Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "a4dba713c111", + "pid": 58990, + "started_at": "2026-11-15T20:21:17.651360383Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "ca00d9061bb8", + "pid": 58361, + "started_at": "2026-11-15T20:21:11.435409755Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "1963232f6596", + "pid": 59409, + "started_at": "2026-11-15T20:21:21.488607385Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ] + }, + "replay_after_replacement": true, + "public_failover": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 123, \"failed_attempts\": 2, \"first_recovery_seconds\": 2.6}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.", + "public_persistence": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.", + "after_persistence": [ + { + "role": "api", + "container": "b8473993e183", + "pid": 8683, + "started_at": "2026-11-15T20:30:59.216457638Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "e61d9c18db8d4153912643f82ea3fd9e" + }, + { + "role": "api", + "container": "5afc95b85319", + "pid": 8607, + "started_at": "2026-11-15T20:30:58.937054013Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "e528375076554fa4b5c9293a17332121" + }, + { + "role": "broker", + "container": "55ab9f986bae", + "pid": 8273, + "started_at": "2026-11-15T20:30:57.728498804Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "7014391ecf89", + "pid": 8292, + "started_at": "2026-11-15T20:30:57.796289054Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "fa3c50f83a1f", + "pid": 58431, + "started_at": "2026-11-15T20:21:11.739337255Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "242a49094f0d", + "pid": 8371, + "started_at": "2026-11-15T20:30:57.957485721Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "61281" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "932bb6635829", + "pid": 8411, + "started_at": "2026-11-15T20:30:57.988178888Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "6f6af6bdcd00", + "pid": 59239, + "started_at": "2026-11-15T20:21:20.927591635Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "a4dba713c111", + "pid": 8870, + "started_at": "2026-11-15T20:31:00.647972291Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "ca00d9061bb8", + "pid": 8267, + "started_at": "2026-11-15T20:30:57.780225846Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "1963232f6596", + "pid": 10016, + "started_at": "2026-11-15T20:31:22.563564969Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ] + }, + "api_worker_relay_reconciler_runtime_hashes_and_revision_match": true, + "fsck": "", + "checks_completed": true, + "result": "PASS" +} diff --git a/scripts/verify_pipeline.py b/scripts/verify_pipeline.py index e327f5fc..28e93854 100644 --- a/scripts/verify_pipeline.py +++ b/scripts/verify_pipeline.py @@ -86,14 +86,20 @@ def duplicate(job, count=5): ) -def inspect_effect(job): +def inspect_effect(job, *, pending=False): return inside(f"""with db.connection() as c: j=c.execute('SELECT * FROM jobs WHERE id=%s',({job!r},)).fetchone() v=c.execute('SELECT name,object_key,sha256,size_bytes FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY name',(j['attachment_id'],j['pipeline_version'])).fetchall() p=c.execute('SELECT count(*) n FROM effect_publications WHERE attachment_id=%s AND pipeline_version=%s',(j['attachment_id'],j['pipeline_version'])).fetchone()['n'] e=c.execute('SELECT event,detail,created_at FROM pipeline_events WHERE job_id=%s ORDER BY id',({job!r},)).fetchall() keys=[o.object_name for o in store.objects() if o.object_name.startswith(f"objects/{{j['attachment_id']}}/v{{j['pipeline_version']}}/")] -print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e}},default=str))""") +pending_entries=[] +if {pending!r}: + for item in queue.xpending_range(STREAM,GROUP,'-','+',1000): + entry=item['message_id'] + messages=queue.xrange(STREAM,min=entry,max=entry) + if messages and json.loads(messages[0][1].get('job','{{}}')).get('job_id')==str(j['id']): pending_entries.append(entry) +print(json.dumps({{'job':j,'variants':v,'publications':p,'physical_keys':keys,'events':e,'pending_stream_entries':pending_entries}},default=str))""") def main(): @@ -366,7 +372,9 @@ def main(): marker = inject(j, phase, "wait") start("worker") wait(lambda marker=marker: hit(marker)) - at_crash = inspect_effect(j) + at_crash = inspect_effect( + j, pending=phase == "after_commit" + ) assert at_crash["job"]["state"] == ( "ready" if phase == "after_commit" @@ -378,6 +386,14 @@ def main(): assert len(at_crash["physical_keys"]) == ( 0 if phase == "before_effect" else 4 ) + if phase == "after_commit": + assert at_crash["pending_stream_entries"], ( + "Result must be committed while notification is still unacknowledged" + ) + assert not any( + e["event"] == "child_finished" + for e in at_crash["events"] + ) compose("kill", "-s", "SIGKILL", "worker") start("worker") effect = confirm(a, j) diff --git a/src/worker.py b/src/worker.py index 274684e3..e9530911 100644 --- a/src/worker.py +++ b/src/worker.py @@ -315,7 +315,8 @@ def consume(slot): context = multiprocessing.get_context("spawn") cursor = "0-0" timeout = integer("JOB_TIMEOUT_SECONDS", 20, 2, 117) - if integer("JOB_LEASE_SECONDS", 23, 3, 120) <= timeout: + lease = integer("JOB_LEASE_SECONDS", 23, 3, 120) + if lease <= timeout: raise ValueError("Lease must exceed child timeout") try: while True: @@ -329,7 +330,8 @@ def consume(slot): STREAM, GROUP, consumer, - min_idle_time=3000, + # Do not steal an entry from a consumer inside its live budget. + min_idle_time=lease * 1000, start_id=cursor, count=1, ) -- 2.54.0 From 8c3d4bcaa5af0a8f733c17a6f4d6d261efd72b9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=81=D0=BB=D0=BE=D0=B2?= Date: Mon, 16 Nov 2026 00:15:46 +0300 Subject: [PATCH 10/11] docs(report): record final clean fault matrix and measured pipeline invariants --- .github/pull_request_template.md | 5 +- REPORT.md | 102 +- evidence/clean-clone-final.json | 9211 +++++++++++++++++ evidence/cleanup-final.json | 159 + evidence/pipeline-analysis-final/effects.csv | 20 + .../pipeline-results.png | Bin 0 -> 67940 bytes evidence/pipeline-analysis-final/summary.json | 22 + evidence/pipeline-final.json | 3879 +++++++ 8 files changed, 13386 insertions(+), 12 deletions(-) create mode 100644 evidence/clean-clone-final.json create mode 100644 evidence/cleanup-final.json create mode 100644 evidence/pipeline-analysis-final/effects.csv create mode 100644 evidence/pipeline-analysis-final/pipeline-results.png create mode 100644 evidence/pipeline-analysis-final/summary.json create mode 100644 evidence/pipeline-final.json diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 2a2dcabd..7a8557ab 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -9,8 +9,9 @@ audited replay/newversion и safe retention cleanup. Контракт и публичные tests сохранены. Запуск, схема, fault methodology, результаты и ответы защиты — REPORT.md/SCHEMA.md.112HTTP и22unit прошли; -полный clean-clone repeat будет внесён перед сдачей. Промежуточные failed -harness runs сохранены и не выданы за полный PASS. +полный clean-clone-final PASS:19fault cases, cluster/rolling/persistence, +real-age cleanup,6runtime source/OCI checks и fsck. Строгое pending-before-SIGKILL +окно COMMIT→ACK подтверждено дважды. Промежуточные failed runs сохранены. Ограничения: одинhost/SPOFs, нет production TLS/backup/global admission quota; HEAD не periodic byte audit; at-least-once с идемпотентным эффектом, diff --git a/REPORT.md b/REPORT.md index d2d9c12e..40754f8f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -6,11 +6,13 @@ contracts/tests/check/validate сохранены. API принимает исх worker публикует original/thumbnail/crop/optimized. Исследование проверяет дубли, reorder и реальные SIGKILL, а не моделирует их ответами mock. -Результаты полного fault-injection и clean clone будут внесены после запуска. -Промежуточный public smoke уже PASS; итоговые evidence ниже обновляются -только фактическими командами. Первый запуск HTTP suite ошибочно обращался -к порту предыдущей лабы; `http-initial-environment.json` честно отмечает -INVALID_ENVIRONMENT, это не проверка корректности API. +Полный повтор из чистого clone — **PASS**, проверенный runtime source +`442163b0f0a549ef065cb13ee39b5ca2f8580c18`. Прошли112HTTP,22unit, public smoke, +19fault cases, cluster/rolling/persistence и cleanup с настоящим возрастом S3. +Raw receipt: [clean-clone-final.json](evidence/clean-clone-final.json). +Предыдущие неполные запуски сохранены и разобраны ниже; ошибочный ранний +HTTP-порт отмечен INVALID_ENVIRONMENT и не считается проверкой API. +Удалённый CI отдельно не подтверждён. ## Воспроизведение @@ -244,7 +246,7 @@ MAX_UPLOAD_BYTES/MAX_IMAGE_PIXELS и idempotency TTL документируют одним редактированием env: соответствующие constants/SQL надо менять согласованно. Secret *_FILE paths заданы в compose; значения создаёт bootstrap. -## Карта критериев и промежуточные результаты +## Карта критериев и история проверок | Уровень / требование | Реализация / автоматическая проверка | | --- | --- | @@ -255,7 +257,7 @@ Secret *_FILE paths заданы в compose; значения создаёт boo | 4:crop/optimized/metadata removal | HTTP geometry/EXIF/alpha +22 unit tests | | 4:outbox gap и publisher crash | API COMMIT-before-publish и relay XADD-before-PG-marker | | 4:finite backoff/permanent/transient/replay |5attempt exhaustion, timeout, failed registry, audit, ready replay rejection | -| 4:crash после PUT доACK, lease, bounded slots | before_effect/after_effect/after_commit ×2; live state/actual limits проверяются в полном повторе | +| 4:crash после PUT доACK, lease, bounded slots | before_effect/after_effect/after_commit ×2; live state/actual limits подтверждены clean-clone-final | | 4:cleanup после фиксации/retention | verify_cleanup seed/verify, реальныйS3возраст, dry/apply/repeat | | 5:versioned idempotency track | duplicates/reorder и held stalev1→v2×2, no old publication/hash change | @@ -276,8 +278,8 @@ V2 была ready, но parent успел убитьv1; это ещё не до живого окна. [pipeline-versions-first.json](evidence/pipeline-versions-first.json) — **scope=versions**,4cases PASS:2version races, malformed notification, private/IAM/otherAPI. Это целевой повтор, а не19cases полного matrix. -Приложение в этих исправлениях harness не менялось; полный clean clone обязан -завершить scope=all, включая более строгие ACK/live-state/resource assertions. +Приложение в этих исправлениях harness не менялось. Финальный clean clone +завершил scope=all, включая строгие ACK/live-state/resource assertions. [evidence/gc-first.json](evidence/gc-first.json): реальный возраст orphan 1279.103s (выше300s); dry-run сохранил bytes, apply удалил expired objects, @@ -319,4 +321,84 @@ Reclaim idle теперь равен lease(23s normal/15s test), превыша снимком XPENDING+XRANGE подтверждает конкретные job entries доSIGKILL и отсутствие child_finished; после restart требует ready duplicate decision и исчезновение notifications. Scope=all, данные и все прежние assertions -сохранены. Финальным доказательством станет clean-clone-final.json. +сохранены. Финальное доказательство — clean-clone-final.json (PASS). + +## Финальный результат из чистого clone + +Дата запуска UTC: 2026-11-15T20:48:55.825623+00:00. Source: `442163b0f0a549ef065cb13ee39b5ca2f8580c18`. +Последующие изменения отчёта/evidence не меняют исполняемые исходники. +Host: Apple M1, 8CPU, 8GiB RAM; Docker VM около6GiB RAM. Лимиты +контейнеров перечислены выше и сохранены в cluster runtime snapshots. +Новые secrets/env/volumes, отдельный image/project/port; после проверки +удалён только временный проект. Основные данные сохранены. + +HTTP: `112 passed in 397.50s (0:06:37)`. Unit:22PASS; public smoke, +check/validate/lint, git fsck и сверка SHA каждого src/*.py с OCI revision +всех6контейнеров (2API/worker/relay/reconciler/controller) — PASS. + +| Сценарий | Итоговые jobs: version/state/attempt/cycle/fence | Publications | Rows/keys | +| --- | --- | --- | --- | +| staging_before_handoff_crash | см. raw assertions | — | — | +| commit_before_publish_and_worker_stopped | v1/ready/1/1/1 | 1 | 4/4 | +| confirmed_broker_entry_survives_sigkill | v1/ready/1/1/1 | 1 | 4/4 | +| relay_crash_after_xadd_before_pg_marker | v1/ready/1/1/1 | 1 | 4/4 | +| before_effect_sigkill_repeat_1 | v1/ready/2/1/3 | 1 | 4/4 | +| after_effect_sigkill_repeat_1 | v1/ready/2/1/3 | 1 | 4/4 | +| after_commit_sigkill_repeat_1 | v1/ready/1/1/1 | 1 | 4/4 | +| before_effect_sigkill_repeat_2 | v1/ready/2/1/3 | 1 | 4/4 | +| after_effect_sigkill_repeat_2 | v1/ready/2/1/3 | 1 | 4/4 | +| after_commit_sigkill_repeat_2 | v1/ready/1/1/1 | 1 | 4/4 | +| transient_retry_with_jitter | v1/ready/2/1/3 | 1 | 4/4 | +| hung_child_killed_and_retried | v1/ready/2/1/3 | 1 | 4/4 | +| permanent_failed_registry_and_audited_replay | v1/ready/1/2/4 | 1 | 4/4 | +| finite_retry_exhaustion_then_explicit_replay | v1/ready/1/2/12 | 1 | 4/4 | +| reordered_and_duplicate_notifications | v1/ready/1/1/1; v1/ready/1/1/1 | 1; 1 | 4/4; 4/4 | +| stale_v1_after_v2_repeat_1 | v1/obsolete/1/1/2; v2/ready/1/1/1 | 0; 1 | 0/4; 4/4 | +| stale_v1_after_v2_repeat_2 | v1/obsolete/1/1/2; v2/ready/1/1/1 | 0; 1 | 0/4; 4/4 | +| malformed_notification_rejected | см. raw assertions | — | — | +| private_iam_and_other_api_download | см. raw assertions | — | — | + +Полные UUID/attempts/fences/events/checksums — в +[pipeline-final.json](evidence/pipeline-final.json) и +[effects.csv](evidence/pipeline-analysis-final/effects.csv). + +Строгие COMMIT→ACK snapshots перед SIGKILL: + +- `after_commit_sigkill_repeat_1`: job `efd4aa67-b47e-462f-8e86-614c5dbb8a17`, pending entries `1794776649599-0`; ready/publication1, child_finished отсутствует. +- `after_commit_sigkill_repeat_2`: job `c422c6de-4c05-43ea-b748-e4ff856a70c8`, pending entries `1794776766791-0`; ready/publication1, child_finished отсутствует. + +После restart оба сценария дождались duplicate_decision(state=ready) и +отсутствия job notifications в stream (ACK+XDEL), без нового resize. +Два held stalev1 действительно дошли до stale_completion_rejected после +readyv2; уv1 ноль publications, hashes всех4currentv2 не изменились. + +Агрегация: 17 ready observations — каждое с1publication,4rows/4keys; +2 obsolete observations — с0publications. Это конечные наблюдения +синтетической матрицы, а не формальное доказательство для всех возможных расписаний. + +![Measured pipeline invariants and resource samples](evidence/pipeline-analysis-final/pipeline-results.png) + +15 successful child samples: callback elapsed 0.2563–0.4592s, +CPU 0.2274–0.3770s, peakRSS 79.12–79.59MiB. +Elapsed/CPU измеряют тело callback и не включают spawn/import до него; RSS +учитывает startup процесса. Убитые child не входят в successful samples. +Это не queue lag, p95 до ready или throughput/SLA. Фактические лимиты +наблюдал сам child: RLIMIT_AS soft/hard805306368bytes, cgroup memory.max +1610612736bytes; CPU quota — в observed_limits raw case. + +Cluster failovers (confirmed writes / transient errors / recovery seconds): + +- 202 / 10 / 4.776s; confirmed data preserved, duplicate texts отсутствуют. +- 218 / 8 / 2.971s; confirmed data preserved, duplicate texts отсутствуют. + +Rolling: 739 confirmed writes, 45 transient errors; +исходные API CID заменены, replay и данные сохранены. Public failover и +persistence с прежними token/file прошли; StartedAt хранилищ изменился. +Кратковременные ошибки учитываются, zero-error rolling не заявлен. + +Cleanup: реальный возраст S3 orphan 754.765s; +dry-run 10actions без изменения bytes; apply 10actions; +repeat0actions. Active/current/fresh и действующий reservation защищены, +expired source replay отвергнут, expired writer fenced. Возраст fixture +metadata менялся SQL отдельно; S3 LastModified не подменялся. +Raw: [cleanup-final.json](evidence/cleanup-final.json). diff --git a/evidence/clean-clone-final.json b/evidence/clean-clone-final.json new file mode 100644 index 00000000..989f6fb7 --- /dev/null +++ b/evidence/clean-clone-final.json @@ -0,0 +1,9211 @@ +{ + "date": "2026-11-15T20:48:55.825623+00:00", + "source_commit": "442163b0f0a549ef065cb13ee39b5ca2f8580c18", + "fresh_secrets_environment_and_volumes": true, + "command": "python scripts/verify_clean.py", + "fixed_host_port_for_restart": 64543, + "static_checks": ".venv/bin/python scripts/check_contract.py\nPASS: offline skeleton structure, local refs and Python syntax; application NOT tested.\n.venv/bin/python scripts/validate_openapi.py\nPASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.\n/.local/bin/uv run --locked ruff format --check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/verify_clean.py scripts/verify_cleanup.py scripts/verify_cluster.py scripts/verify_pipeline.py\n38 files already formatted\n/.local/bin/uv run --locked ruff check src student-tests student-unit research scripts/bootstrap.py scripts/stand.py scripts/cluster.py scripts/verify_clean.py scripts/verify_cleanup.py scripts/verify_cluster.py scripts/verify_pipeline.py\nAll checks passed!", + "unit_tests": "PYTHONPATH=. /.local/bin/uv run --locked pytest student-unit -q\n...................... [100%]\n22 passed in 9.36s", + "smoke": "python3 tests/smoke.py --base-url \"http://127.0.0.1:64543\" \nPASS: lab5 public API scenario; infrastructure and grading evidence remain separate.", + "http_tests": "........................................................................ [ 64%]\n........................................ [100%]\n112 passed in 397.50s (0:06:37)", + "pipeline": { + "source_commit": "442163b0f0a549ef065cb13ee39b5ca2f8580c18", + "date": "2026-11-15T21:00:37.007710+00:00", + "track": 2, + "scope": "all", + "complete": true, + "cases": [ + { + "case": "staging_before_handoff_crash", + "no_202": true, + "no_attachment_or_job_created": true, + "orphan_retained_for_cleanup": true + }, + { + "case": "commit_before_publish_and_worker_stopped", + "attachment": "9b846ca0-dd72-412c-8856-c79588f2db54", + "effect": { + "job": { + "id": "3e2eccd9-713d-4982-a025-5537b66c114c", + "attachment_id": "9b846ca0-dd72-412c-8856-c79588f2db54", + "pipeline_version": 1, + "source_key": "objects/staging/9b846ca0-dd72-412c-8856-c79588f2db54/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:01:12.134125+00:00", + "error_code": null, + "created_at": "2026-11-15 21:01:12.134125+00:00", + "updated_at": "2026-11-15 21:01:38.701930+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/9b846ca0-dd72-412c-8856-c79588f2db54/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/9b846ca0-dd72-412c-8856-c79588f2db54/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/9b846ca0-dd72-412c-8856-c79588f2db54/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/9b846ca0-dd72-412c-8856-c79588f2db54/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/9b846ca0-dd72-412c-8856-c79588f2db54/v1/crop", + "objects/9b846ca0-dd72-412c-8856-c79588f2db54/v1/optimized", + "objects/9b846ca0-dd72-412c-8856-c79588f2db54/v1/original", + "objects/9b846ca0-dd72-412c-8856-c79588f2db54/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:01:36.273141+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794776496251-0", + "outbox": 13 + }, + "created_at": "2026-11-15 21:01:36.274685+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:01:38.300623+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:01:38.702709+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.4591874589996223, + "cpu_seconds": 0.37699009000000006, + "rss_peak_mib": 79.5859375 + }, + "created_at": "2026-11-15 21:01:39.115396+00:00" + } + ], + "pending_stream_entries": [] + }, + "queued_link_allowed": true, + "api_and_broker_restarted": true, + "broker_stopped_during_upload": true + }, + { + "case": "confirmed_broker_entry_survives_sigkill", + "effect": { + "job": { + "id": "5747edc1-9de6-40ba-a143-85ded56744d1", + "attachment_id": "cce6836d-ea56-4387-9426-dc6662844702", + "pipeline_version": 1, + "source_key": "objects/staging/cce6836d-ea56-4387-9426-dc6662844702/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:01:46.873471+00:00", + "error_code": null, + "created_at": "2026-11-15 21:01:46.873472+00:00", + "updated_at": "2026-11-15 21:02:16.668530+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/cce6836d-ea56-4387-9426-dc6662844702/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/cce6836d-ea56-4387-9426-dc6662844702/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/cce6836d-ea56-4387-9426-dc6662844702/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/cce6836d-ea56-4387-9426-dc6662844702/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/cce6836d-ea56-4387-9426-dc6662844702/v1/crop", + "objects/cce6836d-ea56-4387-9426-dc6662844702/v1/optimized", + "objects/cce6836d-ea56-4387-9426-dc6662844702/v1/original", + "objects/cce6836d-ea56-4387-9426-dc6662844702/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776506972-0", + "outbox": 14 + }, + "created_at": "2026-11-15 21:01:46.978632+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:02:14.438581+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:02:16.433487+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:02:16.668713+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2649612499999421, + "cpu_seconds": 0.23184441700000025, + "rss_peak_mib": 79.40234375 + }, + "created_at": "2026-11-15 21:02:16.824413+00:00" + } + ], + "pending_stream_entries": [] + }, + "same_stream_entries": true, + "entry_count": 1 + }, + { + "case": "relay_crash_after_xadd_before_pg_marker", + "unpublished_after_crash": 1, + "effect": { + "job": { + "id": "2177d978-9a49-4856-a9e9-6211e94e9246", + "attachment_id": "05cc42d4-cab3-4ae2-8b56-cc1d95d07248", + "pipeline_version": 1, + "source_key": "objects/staging/05cc42d4-cab3-4ae2-8b56-cc1d95d07248/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:02:23.886230+00:00", + "error_code": null, + "created_at": "2026-11-15 21:02:23.886230+00:00", + "updated_at": "2026-11-15 21:02:51.753609+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/05cc42d4-cab3-4ae2-8b56-cc1d95d07248/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/05cc42d4-cab3-4ae2-8b56-cc1d95d07248/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/05cc42d4-cab3-4ae2-8b56-cc1d95d07248/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/05cc42d4-cab3-4ae2-8b56-cc1d95d07248/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/05cc42d4-cab3-4ae2-8b56-cc1d95d07248/v1/crop", + "objects/05cc42d4-cab3-4ae2-8b56-cc1d95d07248/v1/optimized", + "objects/05cc42d4-cab3-4ae2-8b56-cc1d95d07248/v1/original", + "objects/05cc42d4-cab3-4ae2-8b56-cc1d95d07248/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:02:49.621588+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:02:51.514692+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:02:51.753791+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.27407174999916606, + "cpu_seconds": 0.23973883399999996, + "rss_peak_mib": 79.4765625 + }, + "created_at": "2026-11-15 21:02:51.851871+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794776572771-0", + "outbox": 15 + }, + "created_at": "2026-11-15 21:02:52.787722+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776572771-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:02:52.802564+00:00" + } + ], + "pending_stream_entries": [] + } + }, + { + "case": "before_effect_sigkill_repeat_1", + "at_crash": { + "job": { + "id": "3f046f7c-825f-446f-bf52-07e4cd7e330b", + "attachment_id": "ff5d02a3-079c-49d0-97ba-6e999eea4bfd", + "pipeline_version": 1, + "source_key": "objects/staging/ff5d02a3-079c-49d0-97ba-6e999eea4bfd/original", + "state": "processing", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": "ba79028a-c6ab-475b-9409-197b3745a723-1", + "lease_until": "2026-11-15 21:03:22.732682+00:00", + "next_at": "2026-11-15 21:02:57.236876+00:00", + "error_code": null, + "created_at": "2026-11-15 21:02:57.236876+00:00", + "updated_at": "2026-11-15 21:03:07.732682+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776577314-0", + "outbox": 16 + }, + "created_at": "2026-11-15 21:02:57.323182+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:03:07.734338+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:03:09.616434+00:00" + } + ], + "pending_stream_entries": [] + }, + "effect": { + "job": { + "id": "3f046f7c-825f-446f-bf52-07e4cd7e330b", + "attachment_id": "ff5d02a3-079c-49d0-97ba-6e999eea4bfd", + "pipeline_version": 1, + "source_key": "objects/staging/ff5d02a3-079c-49d0-97ba-6e999eea4bfd/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:03:23.545562+00:00", + "error_code": null, + "created_at": "2026-11-15 21:02:57.236876+00:00", + "updated_at": "2026-11-15 21:03:25.818692+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/ff5d02a3-079c-49d0-97ba-6e999eea4bfd/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/ff5d02a3-079c-49d0-97ba-6e999eea4bfd/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/ff5d02a3-079c-49d0-97ba-6e999eea4bfd/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/ff5d02a3-079c-49d0-97ba-6e999eea4bfd/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/ff5d02a3-079c-49d0-97ba-6e999eea4bfd/v1/crop", + "objects/ff5d02a3-079c-49d0-97ba-6e999eea4bfd/v1/optimized", + "objects/ff5d02a3-079c-49d0-97ba-6e999eea4bfd/v1/original", + "objects/ff5d02a3-079c-49d0-97ba-6e999eea4bfd/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776577314-0", + "outbox": 16 + }, + "created_at": "2026-11-15 21:02:57.323182+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:03:07.734338+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:03:09.616434+00:00" + }, + { + "event": "retry", + "detail": { + "code": "lease_expired", + "attempt": 1, + "backoff_seconds": 0.7795288826695932 + }, + "created_at": "2026-11-15 21:03:22.769892+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776577314-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:03:22.910830+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794776603607-0", + "outbox": 17 + }, + "created_at": "2026-11-15 21:03:23.615198+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 21:03:23.623659+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:03:25.580833+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:03:25.818946+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2666690420001032, + "cpu_seconds": 0.2308113759999999, + "rss_peak_mib": 79.35546875 + }, + "created_at": "2026-11-15 21:03:25.945922+00:00" + } + ], + "pending_stream_entries": [] + } + }, + { + "case": "after_effect_sigkill_repeat_1", + "at_crash": { + "job": { + "id": "64d6ff0c-6a0d-4596-8e70-ffbd66a38fc0", + "attachment_id": "db693326-2e5b-4a48-9008-51cfd1ada11f", + "pipeline_version": 1, + "source_key": "objects/staging/db693326-2e5b-4a48-9008-51cfd1ada11f/original", + "state": "processing", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": "040d968a-0acb-4fce-a35d-d920cf5029b7-0", + "lease_until": "2026-11-15 21:03:59.053911+00:00", + "next_at": "2026-11-15 21:03:31.903675+00:00", + "error_code": null, + "created_at": "2026-11-15 21:03:31.903675+00:00", + "updated_at": "2026-11-15 21:03:44.053912+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/db693326-2e5b-4a48-9008-51cfd1ada11f/v1/crop", + "objects/db693326-2e5b-4a48-9008-51cfd1ada11f/v1/optimized", + "objects/db693326-2e5b-4a48-9008-51cfd1ada11f/v1/original", + "objects/db693326-2e5b-4a48-9008-51cfd1ada11f/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776612008-0", + "outbox": 18 + }, + "created_at": "2026-11-15 21:03:32.014713+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:03:44.055816+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:03:46.213806+00:00" + } + ], + "pending_stream_entries": [] + }, + "effect": { + "job": { + "id": "64d6ff0c-6a0d-4596-8e70-ffbd66a38fc0", + "attachment_id": "db693326-2e5b-4a48-9008-51cfd1ada11f", + "pipeline_version": 1, + "source_key": "objects/staging/db693326-2e5b-4a48-9008-51cfd1ada11f/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:04:01.032228+00:00", + "error_code": null, + "created_at": "2026-11-15 21:03:31.903675+00:00", + "updated_at": "2026-11-15 21:04:03.430438+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/db693326-2e5b-4a48-9008-51cfd1ada11f/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/db693326-2e5b-4a48-9008-51cfd1ada11f/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/db693326-2e5b-4a48-9008-51cfd1ada11f/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/db693326-2e5b-4a48-9008-51cfd1ada11f/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/db693326-2e5b-4a48-9008-51cfd1ada11f/v1/crop", + "objects/db693326-2e5b-4a48-9008-51cfd1ada11f/v1/optimized", + "objects/db693326-2e5b-4a48-9008-51cfd1ada11f/v1/original", + "objects/db693326-2e5b-4a48-9008-51cfd1ada11f/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776612008-0", + "outbox": 18 + }, + "created_at": "2026-11-15 21:03:32.014713+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:03:44.055816+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:03:46.213806+00:00" + }, + { + "event": "retry", + "detail": { + "code": "lease_expired", + "attempt": 1, + "backoff_seconds": 1.7745594232426813 + }, + "created_at": "2026-11-15 21:03:59.259715+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794776641131-0", + "outbox": 19 + }, + "created_at": "2026-11-15 21:04:01.140047+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 21:04:01.147961+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:04:03.169103+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:04:03.430714+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2976695839997774, + "cpu_seconds": 0.2562047089999999, + "rss_peak_mib": 79.33984375 + }, + "created_at": "2026-11-15 21:04:03.540441+00:00" + } + ], + "pending_stream_entries": [] + } + }, + { + "case": "after_commit_sigkill_repeat_1", + "at_crash": { + "job": { + "id": "efd4aa67-b47e-462f-8e86-614c5dbb8a17", + "attachment_id": "a705c027-cdd8-4c13-a75a-8dd8057d4f46", + "pipeline_version": 1, + "source_key": "objects/staging/a705c027-cdd8-4c13-a75a-8dd8057d4f46/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:04:09.418799+00:00", + "error_code": null, + "created_at": "2026-11-15 21:04:09.418799+00:00", + "updated_at": "2026-11-15 21:04:24.745507+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/a705c027-cdd8-4c13-a75a-8dd8057d4f46/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/a705c027-cdd8-4c13-a75a-8dd8057d4f46/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/a705c027-cdd8-4c13-a75a-8dd8057d4f46/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/a705c027-cdd8-4c13-a75a-8dd8057d4f46/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/a705c027-cdd8-4c13-a75a-8dd8057d4f46/v1/crop", + "objects/a705c027-cdd8-4c13-a75a-8dd8057d4f46/v1/optimized", + "objects/a705c027-cdd8-4c13-a75a-8dd8057d4f46/v1/original", + "objects/a705c027-cdd8-4c13-a75a-8dd8057d4f46/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776649599-0", + "outbox": 20 + }, + "created_at": "2026-11-15 21:04:09.605196+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:04:22.204332+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:04:24.439669+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:04:24.746598+00:00" + } + ], + "pending_stream_entries": [ + "1794776649599-0" + ] + }, + "effect": { + "job": { + "id": "efd4aa67-b47e-462f-8e86-614c5dbb8a17", + "attachment_id": "a705c027-cdd8-4c13-a75a-8dd8057d4f46", + "pipeline_version": 1, + "source_key": "objects/staging/a705c027-cdd8-4c13-a75a-8dd8057d4f46/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:04:09.418799+00:00", + "error_code": null, + "created_at": "2026-11-15 21:04:09.418799+00:00", + "updated_at": "2026-11-15 21:04:24.745507+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/a705c027-cdd8-4c13-a75a-8dd8057d4f46/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/a705c027-cdd8-4c13-a75a-8dd8057d4f46/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/a705c027-cdd8-4c13-a75a-8dd8057d4f46/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/a705c027-cdd8-4c13-a75a-8dd8057d4f46/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/a705c027-cdd8-4c13-a75a-8dd8057d4f46/v1/crop", + "objects/a705c027-cdd8-4c13-a75a-8dd8057d4f46/v1/optimized", + "objects/a705c027-cdd8-4c13-a75a-8dd8057d4f46/v1/original", + "objects/a705c027-cdd8-4c13-a75a-8dd8057d4f46/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776649599-0", + "outbox": 20 + }, + "created_at": "2026-11-15 21:04:09.605196+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:04:22.204332+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:04:24.439669+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:04:24.746598+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776649599-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:04:37.680800+00:00" + } + ], + "pending_stream_entries": [] + } + }, + { + "case": "before_effect_sigkill_repeat_2", + "at_crash": { + "job": { + "id": "6eacee3c-6b6b-4435-bd05-bb50c29df24f", + "attachment_id": "41d11ee2-c51e-4a39-bfde-8de3eb5318d9", + "pipeline_version": 1, + "source_key": "objects/staging/41d11ee2-c51e-4a39-bfde-8de3eb5318d9/original", + "state": "processing", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": "5d3848a5-ac5f-4471-8b8a-9b7c000ac95a-1", + "lease_until": "2026-11-15 21:05:21.104605+00:00", + "next_at": "2026-11-15 21:04:53.277964+00:00", + "error_code": null, + "created_at": "2026-11-15 21:04:53.277964+00:00", + "updated_at": "2026-11-15 21:05:06.104605+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776693288-0", + "outbox": 21 + }, + "created_at": "2026-11-15 21:04:53.294351+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:05:06.106430+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:05:08.092072+00:00" + } + ], + "pending_stream_entries": [] + }, + "effect": { + "job": { + "id": "6eacee3c-6b6b-4435-bd05-bb50c29df24f", + "attachment_id": "41d11ee2-c51e-4a39-bfde-8de3eb5318d9", + "pipeline_version": 1, + "source_key": "objects/staging/41d11ee2-c51e-4a39-bfde-8de3eb5318d9/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:05:22.075698+00:00", + "error_code": null, + "created_at": "2026-11-15 21:04:53.277964+00:00", + "updated_at": "2026-11-15 21:05:24.719240+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/41d11ee2-c51e-4a39-bfde-8de3eb5318d9/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/41d11ee2-c51e-4a39-bfde-8de3eb5318d9/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/41d11ee2-c51e-4a39-bfde-8de3eb5318d9/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/41d11ee2-c51e-4a39-bfde-8de3eb5318d9/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/41d11ee2-c51e-4a39-bfde-8de3eb5318d9/v1/crop", + "objects/41d11ee2-c51e-4a39-bfde-8de3eb5318d9/v1/optimized", + "objects/41d11ee2-c51e-4a39-bfde-8de3eb5318d9/v1/original", + "objects/41d11ee2-c51e-4a39-bfde-8de3eb5318d9/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776693288-0", + "outbox": 21 + }, + "created_at": "2026-11-15 21:04:53.294351+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:05:06.106430+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:05:08.092072+00:00" + }, + { + "event": "retry", + "detail": { + "code": "lease_expired", + "attempt": 1, + "backoff_seconds": 0.8577338543216321 + }, + "created_at": "2026-11-15 21:05:21.229423+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776693288-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:05:21.245895+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794776722265-0", + "outbox": 22 + }, + "created_at": "2026-11-15 21:05:22.269465+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 21:05:22.275045+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:05:24.458089+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:05:24.719520+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.3038405419993069, + "cpu_seconds": 0.26212937199999997, + "rss_peak_mib": 79.4765625 + }, + "created_at": "2026-11-15 21:05:24.856191+00:00" + } + ], + "pending_stream_entries": [] + } + }, + { + "case": "after_effect_sigkill_repeat_2", + "at_crash": { + "job": { + "id": "0d40fa0a-e6b9-49cb-907d-e82faa346906", + "attachment_id": "009cd1f4-e486-40df-b570-b0fa866afecb", + "pipeline_version": 1, + "source_key": "objects/staging/009cd1f4-e486-40df-b570-b0fa866afecb/original", + "state": "processing", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": "6bfb8691-1798-408d-94b8-eda438402c44-1", + "lease_until": "2026-11-15 21:05:57.555606+00:00", + "next_at": "2026-11-15 21:05:31.212662+00:00", + "error_code": null, + "created_at": "2026-11-15 21:05:31.212662+00:00", + "updated_at": "2026-11-15 21:05:42.555607+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/009cd1f4-e486-40df-b570-b0fa866afecb/v1/crop", + "objects/009cd1f4-e486-40df-b570-b0fa866afecb/v1/optimized", + "objects/009cd1f4-e486-40df-b570-b0fa866afecb/v1/original", + "objects/009cd1f4-e486-40df-b570-b0fa866afecb/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776731297-0", + "outbox": 23 + }, + "created_at": "2026-11-15 21:05:31.308530+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:05:42.557360+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:05:44.483008+00:00" + } + ], + "pending_stream_entries": [] + }, + "effect": { + "job": { + "id": "0d40fa0a-e6b9-49cb-907d-e82faa346906", + "attachment_id": "009cd1f4-e486-40df-b570-b0fa866afecb", + "pipeline_version": 1, + "source_key": "objects/staging/009cd1f4-e486-40df-b570-b0fa866afecb/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:05:58.547354+00:00", + "error_code": null, + "created_at": "2026-11-15 21:05:31.212662+00:00", + "updated_at": "2026-11-15 21:06:00.920587+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/009cd1f4-e486-40df-b570-b0fa866afecb/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/009cd1f4-e486-40df-b570-b0fa866afecb/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/009cd1f4-e486-40df-b570-b0fa866afecb/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/009cd1f4-e486-40df-b570-b0fa866afecb/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/009cd1f4-e486-40df-b570-b0fa866afecb/v1/crop", + "objects/009cd1f4-e486-40df-b570-b0fa866afecb/v1/optimized", + "objects/009cd1f4-e486-40df-b570-b0fa866afecb/v1/original", + "objects/009cd1f4-e486-40df-b570-b0fa866afecb/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776731297-0", + "outbox": 23 + }, + "created_at": "2026-11-15 21:05:31.308530+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:05:42.557360+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:05:44.483008+00:00" + }, + { + "event": "retry", + "detail": { + "code": "lease_expired", + "attempt": 1, + "backoff_seconds": 0.8218544726326943 + }, + "created_at": "2026-11-15 21:05:57.735279+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776731297-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:05:57.794557+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794776758586-0", + "outbox": 24 + }, + "created_at": "2026-11-15 21:05:58.593691+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 21:05:58.607480+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:06:00.658771+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:06:00.920946+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.30515825000020413, + "cpu_seconds": 0.2600924969999998, + "rss_peak_mib": 79.46484375 + }, + "created_at": "2026-11-15 21:06:01.087475+00:00" + } + ], + "pending_stream_entries": [] + } + }, + { + "case": "after_commit_sigkill_repeat_2", + "at_crash": { + "job": { + "id": "c422c6de-4c05-43ea-b748-e4ff856a70c8", + "attachment_id": "dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6", + "pipeline_version": 1, + "source_key": "objects/staging/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:06:06.598511+00:00", + "error_code": null, + "created_at": "2026-11-15 21:06:06.598511+00:00", + "updated_at": "2026-11-15 21:06:20.170245+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/v1/crop", + "objects/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/v1/optimized", + "objects/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/v1/original", + "objects/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776766791-0", + "outbox": 25 + }, + "created_at": "2026-11-15 21:06:06.797094+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:06:17.703932+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:06:19.911384+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:06:20.170542+00:00" + } + ], + "pending_stream_entries": [ + "1794776766791-0" + ] + }, + "effect": { + "job": { + "id": "c422c6de-4c05-43ea-b748-e4ff856a70c8", + "attachment_id": "dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6", + "pipeline_version": 1, + "source_key": "objects/staging/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:06:06.598511+00:00", + "error_code": null, + "created_at": "2026-11-15 21:06:06.598511+00:00", + "updated_at": "2026-11-15 21:06:20.170245+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/v1/crop", + "objects/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/v1/optimized", + "objects/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/v1/original", + "objects/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776766791-0", + "outbox": 25 + }, + "created_at": "2026-11-15 21:06:06.797094+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:06:17.703932+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:06:19.911384+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:06:20.170542+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776766791-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:06:33.168249+00:00" + } + ], + "pending_stream_entries": [] + } + }, + { + "case": "transient_retry_with_jitter", + "effect": { + "job": { + "id": "00893137-7e67-434b-8764-5c85ed5353d2", + "attachment_id": "1785f2aa-95d9-4a1c-9aa2-2b2ab0e15942", + "pipeline_version": 1, + "source_key": "objects/staging/1785f2aa-95d9-4a1c-9aa2-2b2ab0e15942/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:07:02.310533+00:00", + "error_code": null, + "created_at": "2026-11-15 21:06:49.046950+00:00", + "updated_at": "2026-11-15 21:07:04.469249+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/1785f2aa-95d9-4a1c-9aa2-2b2ab0e15942/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/1785f2aa-95d9-4a1c-9aa2-2b2ab0e15942/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/1785f2aa-95d9-4a1c-9aa2-2b2ab0e15942/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/1785f2aa-95d9-4a1c-9aa2-2b2ab0e15942/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/1785f2aa-95d9-4a1c-9aa2-2b2ab0e15942/v1/crop", + "objects/1785f2aa-95d9-4a1c-9aa2-2b2ab0e15942/v1/optimized", + "objects/1785f2aa-95d9-4a1c-9aa2-2b2ab0e15942/v1/original", + "objects/1785f2aa-95d9-4a1c-9aa2-2b2ab0e15942/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776809186-0", + "outbox": 26 + }, + "created_at": "2026-11-15 21:06:49.194896+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:06:59.973892+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:07:01.871654+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 21:07:01.958826+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 1, + "backoff_seconds": 0.3511529054902287 + }, + "created_at": "2026-11-15 21:07:01.960412+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794776822327-0", + "outbox": 27 + }, + "created_at": "2026-11-15 21:07:02.331263+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 21:07:02.331841+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:07:04.238170+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:07:04.469494+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2563110829996731, + "cpu_seconds": 0.22742345900000016, + "rss_peak_mib": 79.35546875 + }, + "created_at": "2026-11-15 21:07:04.567907+00:00" + } + ], + "pending_stream_entries": [] + } + }, + { + "case": "hung_child_killed_and_retried", + "effect": { + "job": { + "id": "eefeceb7-6e35-4358-bbac-772170b964fa", + "attachment_id": "0ca0fee0-80d3-4e61-97e0-2ce0c80a5c44", + "pipeline_version": 1, + "source_key": "objects/staging/0ca0fee0-80d3-4e61-97e0-2ce0c80a5c44/original", + "state": "ready", + "attempt": 2, + "cycle": 1, + "fence": 3, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:07:34.420927+00:00", + "error_code": null, + "created_at": "2026-11-15 21:07:10.206988+00:00", + "updated_at": "2026-11-15 21:07:36.908514+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/0ca0fee0-80d3-4e61-97e0-2ce0c80a5c44/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/0ca0fee0-80d3-4e61-97e0-2ce0c80a5c44/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/0ca0fee0-80d3-4e61-97e0-2ce0c80a5c44/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/0ca0fee0-80d3-4e61-97e0-2ce0c80a5c44/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/0ca0fee0-80d3-4e61-97e0-2ce0c80a5c44/v1/crop", + "objects/0ca0fee0-80d3-4e61-97e0-2ce0c80a5c44/v1/optimized", + "objects/0ca0fee0-80d3-4e61-97e0-2ce0c80a5c44/v1/original", + "objects/0ca0fee0-80d3-4e61-97e0-2ce0c80a5c44/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776830310-0", + "outbox": 28 + }, + "created_at": "2026-11-15 21:07:10.317655+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:07:21.224560+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:07:24.486029+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "worker_timeout", + "result": "transient" + }, + "created_at": "2026-11-15 21:07:33.302366+00:00" + }, + { + "event": "retry", + "detail": { + "code": "worker_timeout", + "attempt": 1, + "backoff_seconds": 1.1168498939777787 + }, + "created_at": "2026-11-15 21:07:33.305793+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794776854566-0", + "outbox": 29 + }, + "created_at": "2026-11-15 21:07:34.578812+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 21:07:34.590076+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:07:36.640418+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 3, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:07:36.908759+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.30338325100001384, + "cpu_seconds": 0.2594866250000001, + "rss_peak_mib": 79.484375 + }, + "created_at": "2026-11-15 21:07:37.023632+00:00" + } + ], + "pending_stream_entries": [] + }, + "observed_limits": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + } + }, + { + "case": "permanent_failed_registry_and_audited_replay", + "failed": { + "id": "e10c9771-0ec9-413d-a614-49ee56630a54", + "chat_id": "ac2fc940-248f-45be-a850-c0c0ecf2716f", + "owner_id": "005794fb-230d-4bf9-ae8f-314210ada7b9", + "filename": "synthetic.png", + "content_type": "image/png", + "size_bytes": 742, + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "created_at": "2026-11-15T21:07:43.308484Z", + "updated_at": "2026-11-15T21:07:57.681037Z", + "status": "failed", + "variants": [], + "error_code": "invalid_image" + }, + "effect": { + "job": { + "id": "88929609-e34a-4b5f-ad36-ffce561ebde8", + "attachment_id": "e10c9771-0ec9-413d-a614-49ee56630a54", + "pipeline_version": 1, + "source_key": "objects/staging/e10c9771-0ec9-413d-a614-49ee56630a54/original", + "state": "ready", + "attempt": 1, + "cycle": 2, + "fence": 4, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:08:02.550595+00:00", + "error_code": null, + "created_at": "2026-11-15 21:07:43.311092+00:00", + "updated_at": "2026-11-15 21:08:05.060029+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/e10c9771-0ec9-413d-a614-49ee56630a54/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/e10c9771-0ec9-413d-a614-49ee56630a54/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/e10c9771-0ec9-413d-a614-49ee56630a54/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/e10c9771-0ec9-413d-a614-49ee56630a54/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/e10c9771-0ec9-413d-a614-49ee56630a54/v1/crop", + "objects/e10c9771-0ec9-413d-a614-49ee56630a54/v1/optimized", + "objects/e10c9771-0ec9-413d-a614-49ee56630a54/v1/original", + "objects/e10c9771-0ec9-413d-a614-49ee56630a54/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776863521-0", + "outbox": 30 + }, + "created_at": "2026-11-15 21:07:43.527690+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:07:54.618558+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:07:57.403308+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "invalid_image", + "result": "permanent" + }, + "created_at": "2026-11-15 21:07:57.674457+00:00" + }, + { + "event": "failed", + "detail": { + "code": "invalid_image", + "attempt": 1, + "backoff_seconds": 0.7190645315074458 + }, + "created_at": "2026-11-15 21:07:57.681770+00:00" + }, + { + "event": "replay", + "detail": { + "previous_job": "88929609-e34a-4b5f-ad36-ffce561ebde8" + }, + "created_at": "2026-11-15 21:08:02.552597+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794776882684-0", + "outbox": 31 + }, + "created_at": "2026-11-15 21:08:02.694338+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 4, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:08:02.698560+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:08:04.807935+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 4, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:08:05.060316+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2860605830001077, + "cpu_seconds": 0.24953375300000014, + "rss_peak_mib": 79.171875 + }, + "created_at": "2026-11-15 21:08:05.192888+00:00" + } + ], + "pending_stream_entries": [] + }, + "ready_replay_rejected": true + }, + { + "case": "finite_retry_exhaustion_then_explicit_replay", + "failed": { + "id": "0b1d6d7e-7be9-4563-ae0e-823c4f96c63d", + "chat_id": "ac2fc940-248f-45be-a850-c0c0ecf2716f", + "owner_id": "005794fb-230d-4bf9-ae8f-314210ada7b9", + "filename": "synthetic.png", + "content_type": "image/png", + "size_bytes": 742, + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "created_at": "2026-11-15T21:08:15.484975Z", + "updated_at": "2026-11-15T21:08:56.505915Z", + "status": "failed", + "variants": [], + "error_code": "retry_exhausted" + }, + "effect": { + "job": { + "id": "b6e323c3-3e4a-45fd-a4a5-709094e97485", + "attachment_id": "0b1d6d7e-7be9-4563-ae0e-823c4f96c63d", + "pipeline_version": 1, + "source_key": "objects/staging/0b1d6d7e-7be9-4563-ae0e-823c4f96c63d/original", + "state": "ready", + "attempt": 1, + "cycle": 2, + "fence": 12, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:09:05.556919+00:00", + "error_code": null, + "created_at": "2026-11-15 21:08:15.487089+00:00", + "updated_at": "2026-11-15 21:09:07.784436+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/0b1d6d7e-7be9-4563-ae0e-823c4f96c63d/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/0b1d6d7e-7be9-4563-ae0e-823c4f96c63d/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/0b1d6d7e-7be9-4563-ae0e-823c4f96c63d/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/0b1d6d7e-7be9-4563-ae0e-823c4f96c63d/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/0b1d6d7e-7be9-4563-ae0e-823c4f96c63d/v1/crop", + "objects/0b1d6d7e-7be9-4563-ae0e-823c4f96c63d/v1/optimized", + "objects/0b1d6d7e-7be9-4563-ae0e-823c4f96c63d/v1/original", + "objects/0b1d6d7e-7be9-4563-ae0e-823c4f96c63d/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794776895637-0", + "outbox": 32 + }, + "created_at": "2026-11-15 21:08:15.641729+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:08:26.875963+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:08:28.888163+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 21:08:28.990615+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 1, + "backoff_seconds": 1.0323181550842835 + }, + "created_at": "2026-11-15 21:08:28.992048+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794776910184-0", + "outbox": 33 + }, + "created_at": "2026-11-15 21:08:30.194045+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 3, + "attempt": 2, + "version": 1 + }, + "created_at": "2026-11-15 21:08:30.196216+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:08:32.246984+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 21:08:32.344779+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 2, + "backoff_seconds": 1.4475073727137884 + }, + "created_at": "2026-11-15 21:08:32.346554+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794776913871-0", + "outbox": 34 + }, + "created_at": "2026-11-15 21:08:33.877192+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 5, + "attempt": 3, + "version": 1 + }, + "created_at": "2026-11-15 21:08:33.879017+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:08:35.762479+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 21:08:35.853391+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 3, + "backoff_seconds": 7.658626209275384 + }, + "created_at": "2026-11-15 21:08:35.854346+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794776923558-0", + "outbox": 35 + }, + "created_at": "2026-11-15 21:08:43.574973+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 7, + "attempt": 4, + "version": 1 + }, + "created_at": "2026-11-15 21:08:43.586328+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:08:45.569399+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 21:08:45.662144+00:00" + }, + { + "event": "retry", + "detail": { + "code": "processing_unavailable", + "attempt": 4, + "backoff_seconds": 8.675384166106461 + }, + "created_at": "2026-11-15 21:08:45.663111+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794776934462-0", + "outbox": 36 + }, + "created_at": "2026-11-15 21:08:54.467772+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 9, + "attempt": 5, + "version": 1 + }, + "created_at": "2026-11-15 21:08:54.472314+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:08:56.362775+00:00" + }, + { + "event": "child_finished", + "detail": { + "code": "processing_unavailable", + "result": "transient" + }, + "created_at": "2026-11-15 21:08:56.505197+00:00" + }, + { + "event": "failed", + "detail": { + "code": "retry_exhausted", + "attempt": 5, + "backoff_seconds": 21.631642529772595 + }, + "created_at": "2026-11-15 21:08:56.506180+00:00" + }, + { + "event": "replay", + "detail": { + "previous_job": "b6e323c3-3e4a-45fd-a4a5-709094e97485" + }, + "created_at": "2026-11-15 21:09:05.558943+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794776945581-0", + "outbox": 37 + }, + "created_at": "2026-11-15 21:09:05.590354+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 12, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:09:05.599491+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:09:07.541668+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 12, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:09:07.784730+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.2706124579999596, + "cpu_seconds": 0.23715611999999986, + "rss_peak_mib": 79.3359375 + }, + "created_at": "2026-11-15 21:09:07.890702+00:00" + } + ], + "pending_stream_entries": [] + } + }, + { + "case": "reordered_and_duplicate_notifications", + "effects": [ + { + "job": { + "id": "e7f4c924-cb49-40a5-b15b-b5bd8b3bd401", + "attachment_id": "2b2d8110-9c6c-43e7-984f-7e161ea107a8", + "pipeline_version": 1, + "source_key": "objects/staging/2b2d8110-9c6c-43e7-984f-7e161ea107a8/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:09:15.003320+00:00", + "error_code": null, + "created_at": "2026-11-15 21:09:15.003320+00:00", + "updated_at": "2026-11-15 21:09:39.656944+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/2b2d8110-9c6c-43e7-984f-7e161ea107a8/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/2b2d8110-9c6c-43e7-984f-7e161ea107a8/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/2b2d8110-9c6c-43e7-984f-7e161ea107a8/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/2b2d8110-9c6c-43e7-984f-7e161ea107a8/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/2b2d8110-9c6c-43e7-984f-7e161ea107a8/v1/crop", + "objects/2b2d8110-9c6c-43e7-984f-7e161ea107a8/v1/optimized", + "objects/2b2d8110-9c6c-43e7-984f-7e161ea107a8/v1/original", + "objects/2b2d8110-9c6c-43e7-984f-7e161ea107a8/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:09:37.034199+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:09:39.325428+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:09:39.657331+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.3865292509999563, + "cpu_seconds": 0.3280367870000003, + "rss_peak_mib": 79.31640625 + }, + "created_at": "2026-11-15 21:09:39.975413+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776973967-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:40.064029+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776973969-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:40.080707+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776973970-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:40.092604+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776973971-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:40.144118+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776973971-1", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:40.145441+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776973973-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:40.177926+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776973974-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:40.180915+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776982460-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:42.492331+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794776982460-0", + "outbox": 38 + }, + "created_at": "2026-11-15 21:09:42.534060+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776993258-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:53.267430+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776993260-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:53.267786+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776993261-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:53.281363+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776993262-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:53.281641+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776993263-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:53.290979+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776993264-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:53.291468+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776993265-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:53.301679+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776993266-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:53.302033+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776993266-1", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:53.310292+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776993267-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:53.311363+00:00" + } + ], + "pending_stream_entries": [] + }, + { + "job": { + "id": "108ade45-7571-4362-92d5-bde3c6232fe9", + "attachment_id": "b2f6bb54-8d35-4810-b756-b8418c8abec6", + "pipeline_version": 1, + "source_key": "objects/staging/b2f6bb54-8d35-4810-b756-b8418c8abec6/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:09:19.400523+00:00", + "error_code": null, + "created_at": "2026-11-15 21:09:19.400523+00:00", + "updated_at": "2026-11-15 21:09:39.657060+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/b2f6bb54-8d35-4810-b756-b8418c8abec6/v1/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/b2f6bb54-8d35-4810-b756-b8418c8abec6/v1/optimized", + "sha256": "646916c35748e49b6d5a71921ec6bdf772c3a8729c661992391a99f5bd5289a8", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/b2f6bb54-8d35-4810-b756-b8418c8abec6/v1/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/b2f6bb54-8d35-4810-b756-b8418c8abec6/v1/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/b2f6bb54-8d35-4810-b756-b8418c8abec6/v1/crop", + "objects/b2f6bb54-8d35-4810-b756-b8418c8abec6/v1/optimized", + "objects/b2f6bb54-8d35-4810-b756-b8418c8abec6/v1/original", + "objects/b2f6bb54-8d35-4810-b756-b8418c8abec6/v1/thumbnail" + ], + "events": [ + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:09:36.987974+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:09:39.325412+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 1, + "variants": 4 + }, + "created_at": "2026-11-15 21:09:39.657458+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.4013020000002143, + "cpu_seconds": 0.3315808340000004, + "rss_peak_mib": 79.125 + }, + "created_at": "2026-11-15 21:09:39.940590+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794776982541-0", + "outbox": 39 + }, + "created_at": "2026-11-15 21:09:42.549013+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776982541-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:42.565822+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776968841-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:52.442047+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776968842-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:52.442335+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776968843-1", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:52.455893+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776968843-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:52.455941+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776968844-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:52.466753+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776968845-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:52.467092+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776968845-1", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:52.476031+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776998174-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:58.209393+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776998169-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:58.209943+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776998175-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:58.221815+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776998176-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:58.222159+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776998177-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:58.237946+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776998179-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:58.241326+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776998180-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:58.252021+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776998181-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:58.254664+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776998182-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:58.265579+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794776998183-0", + "state": "ready" + }, + "created_at": "2026-11-15 21:09:58.270049+00:00" + } + ], + "pending_stream_entries": [] + } + ], + "added_duplicates": 36, + "no_extra_attempt_after_ready": true + }, + { + "case": "stale_v1_after_v2_repeat_1", + "v1": { + "job": { + "id": "bc562c08-8221-4f9d-b5ff-890d16987495", + "attachment_id": "1611ef24-62d2-447f-bd7c-7f42164ec9aa", + "pipeline_version": 1, + "source_key": "objects/staging/1611ef24-62d2-447f-bd7c-7f42164ec9aa/original", + "state": "obsolete", + "attempt": 1, + "cycle": 1, + "fence": 2, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:10:18.834209+00:00", + "error_code": null, + "created_at": "2026-11-15 21:10:18.834209+00:00", + "updated_at": "2026-11-15 21:10:40.073074+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/1611ef24-62d2-447f-bd7c-7f42164ec9aa/v1/crop", + "objects/1611ef24-62d2-447f-bd7c-7f42164ec9aa/v1/optimized", + "objects/1611ef24-62d2-447f-bd7c-7f42164ec9aa/v1/original", + "objects/1611ef24-62d2-447f-bd7c-7f42164ec9aa/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794777018864-0", + "outbox": 40 + }, + "created_at": "2026-11-15 21:10:18.878696+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794777033930-0", + "outbox": 41 + }, + "created_at": "2026-11-15 21:10:33.933699+00:00" + }, + { + "event": "version_controller_waiting", + "detail": {}, + "created_at": "2026-11-15 21:10:37.128496+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:10:40.074867+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:10:42.557367+00:00" + }, + { + "event": "stale_completion_rejected", + "detail": { + "fence": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:10:45.177951+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "stale" + }, + "created_at": "2026-11-15 21:10:45.322248+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777053864-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:10:53.874402+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777053868-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:10:53.875053+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777053870-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:10:53.884009+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777053871-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:10:53.884649+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777053872-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:10:53.889841+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777053873-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:10:53.890661+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777053873-1", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:10:53.895303+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777053874-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:10:53.895712+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777053876-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:10:53.900375+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777053877-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:10:53.900998+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777053877-1", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:10:53.905965+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777053878-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:10:53.906887+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777033930-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:10:55.492042+00:00" + } + ], + "pending_stream_entries": [] + }, + "v2": { + "job": { + "id": "75ceffdc-919b-4f92-96c4-2e0c775850d7", + "attachment_id": "1611ef24-62d2-447f-bd7c-7f42164ec9aa", + "pipeline_version": 2, + "source_key": "objects/staging/1611ef24-62d2-447f-bd7c-7f42164ec9aa/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:10:42.871051+00:00", + "error_code": null, + "created_at": "2026-11-15 21:10:42.871051+00:00", + "updated_at": "2026-11-15 21:10:45.091378+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/1611ef24-62d2-447f-bd7c-7f42164ec9aa/v2/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/1611ef24-62d2-447f-bd7c-7f42164ec9aa/v2/optimized", + "sha256": "f2b2ece82b03105a787a0a9b75ce63787364fa39329cd6c21e23f44ec2ef2ced", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/1611ef24-62d2-447f-bd7c-7f42164ec9aa/v2/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/1611ef24-62d2-447f-bd7c-7f42164ec9aa/v2/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/1611ef24-62d2-447f-bd7c-7f42164ec9aa/v2/crop", + "objects/1611ef24-62d2-447f-bd7c-7f42164ec9aa/v2/optimized", + "objects/1611ef24-62d2-447f-bd7c-7f42164ec9aa/v2/original", + "objects/1611ef24-62d2-447f-bd7c-7f42164ec9aa/v2/thumbnail" + ], + "events": [ + { + "event": "new_version", + "detail": { + "previous_job": "bc562c08-8221-4f9d-b5ff-890d16987495" + }, + "created_at": "2026-11-15 21:10:42.874336+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794777042975-0", + "outbox": 42 + }, + "created_at": "2026-11-15 21:10:42.981699+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 2 + }, + "created_at": "2026-11-15 21:10:42.991618+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:10:44.860034+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 2, + "variants": 4 + }, + "created_at": "2026-11-15 21:10:45.091591+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.25812733300062973, + "cpu_seconds": 0.22754957700000022, + "rss_peak_mib": 79.46484375 + }, + "created_at": "2026-11-15 21:10:45.197357+00:00" + } + ], + "pending_stream_entries": [] + }, + "new_hashes_unchanged": true + }, + { + "case": "stale_v1_after_v2_repeat_2", + "v1": { + "job": { + "id": "61e16816-661d-46bd-882e-80cc1cab86dd", + "attachment_id": "1b50482e-6808-434e-afaa-165e4c9cba8c", + "pipeline_version": 1, + "source_key": "objects/staging/1b50482e-6808-434e-afaa-165e4c9cba8c/original", + "state": "obsolete", + "attempt": 1, + "cycle": 1, + "fence": 2, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:11:18.213312+00:00", + "error_code": null, + "created_at": "2026-11-15 21:11:18.213312+00:00", + "updated_at": "2026-11-15 21:11:38.910232+00:00" + }, + "variants": [], + "publications": 0, + "physical_keys": [ + "objects/1b50482e-6808-434e-afaa-165e4c9cba8c/v1/crop", + "objects/1b50482e-6808-434e-afaa-165e4c9cba8c/v1/optimized", + "objects/1b50482e-6808-434e-afaa-165e4c9cba8c/v1/original", + "objects/1b50482e-6808-434e-afaa-165e4c9cba8c/v1/thumbnail" + ], + "events": [ + { + "event": "published", + "detail": { + "entry": "1794777078414-0", + "outbox": 43 + }, + "created_at": "2026-11-15 21:11:18.418482+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794777093457-0", + "outbox": 44 + }, + "created_at": "2026-11-15 21:11:33.460682+00:00" + }, + { + "event": "version_controller_waiting", + "detail": {}, + "created_at": "2026-11-15 21:11:36.132010+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:11:38.912132+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:11:40.865539+00:00" + }, + { + "event": "stale_completion_rejected", + "detail": { + "fence": 1, + "version": 1 + }, + "created_at": "2026-11-15 21:11:43.972563+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "stale" + }, + "created_at": "2026-11-15 21:11:44.160883+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777112661-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:11:52.673420+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777112662-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:11:52.674082+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777112663-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:11:52.680144+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777112665-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:11:52.680608+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777112667-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:11:52.685819+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777112669-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:11:52.686713+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777112669-1", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:11:52.691674+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777112670-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:11:52.692365+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777112671-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:11:52.696793+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777112672-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:11:52.697563+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777112673-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:11:52.701833+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777112674-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:11:52.702558+00:00" + }, + { + "event": "duplicate_decision", + "detail": { + "entry": "1794777078414-0", + "state": "obsolete" + }, + "created_at": "2026-11-15 21:11:54.294784+00:00" + } + ], + "pending_stream_entries": [] + }, + "v2": { + "job": { + "id": "6d569c1e-c44c-4b65-a49d-ae731029270a", + "attachment_id": "1b50482e-6808-434e-afaa-165e4c9cba8c", + "pipeline_version": 2, + "source_key": "objects/staging/1b50482e-6808-434e-afaa-165e4c9cba8c/original", + "state": "ready", + "attempt": 1, + "cycle": 1, + "fence": 1, + "lease_owner": null, + "lease_until": null, + "next_at": "2026-11-15 21:11:41.210214+00:00", + "error_code": null, + "created_at": "2026-11-15 21:11:41.210215+00:00", + "updated_at": "2026-11-15 21:11:43.872127+00:00" + }, + "variants": [ + { + "name": "crop", + "object_key": "objects/1b50482e-6808-434e-afaa-165e4c9cba8c/v2/crop", + "sha256": "3ffc6696b681aa9a794eb707c1845586d7b21bfa5359a17e55fd1bdb7818a8ef", + "size_bytes": 1077 + }, + { + "name": "optimized", + "object_key": "objects/1b50482e-6808-434e-afaa-165e4c9cba8c/v2/optimized", + "sha256": "f2b2ece82b03105a787a0a9b75ce63787364fa39329cd6c21e23f44ec2ef2ced", + "size_bytes": 194 + }, + { + "name": "original", + "object_key": "objects/1b50482e-6808-434e-afaa-165e4c9cba8c/v2/original", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df", + "size_bytes": 742 + }, + { + "name": "thumbnail", + "object_key": "objects/1b50482e-6808-434e-afaa-165e4c9cba8c/v2/thumbnail", + "sha256": "734d3760e748c054f8c7ae069e9c44b189b5e4e647fee7bbafe3b308ffab2199", + "size_bytes": 1749 + } + ], + "publications": 1, + "physical_keys": [ + "objects/1b50482e-6808-434e-afaa-165e4c9cba8c/v2/crop", + "objects/1b50482e-6808-434e-afaa-165e4c9cba8c/v2/optimized", + "objects/1b50482e-6808-434e-afaa-165e4c9cba8c/v2/original", + "objects/1b50482e-6808-434e-afaa-165e4c9cba8c/v2/thumbnail" + ], + "events": [ + { + "event": "new_version", + "detail": { + "previous_job": "61e16816-661d-46bd-882e-80cc1cab86dd" + }, + "created_at": "2026-11-15 21:11:41.212988+00:00" + }, + { + "event": "published", + "detail": { + "entry": "1794777101326-0", + "outbox": 45 + }, + "created_at": "2026-11-15 21:11:41.335998+00:00" + }, + { + "event": "claimed", + "detail": { + "fence": 1, + "attempt": 1, + "version": 2 + }, + "created_at": "2026-11-15 21:11:41.346278+00:00" + }, + { + "event": "worker_resource_limits", + "detail": { + "cpu.max": "200000 100000", + "memory.max": "1610612736", + "address_space_bytes": [ + 805306368, + 805306368 + ] + }, + "created_at": "2026-11-15 21:11:43.640701+00:00" + }, + { + "event": "effect_committed", + "detail": { + "fence": 1, + "version": 2, + "variants": 4 + }, + "created_at": "2026-11-15 21:11:43.872394+00:00" + }, + { + "event": "child_finished", + "detail": { + "result": "ready", + "seconds": 0.25968787600049836, + "cpu_seconds": 0.2296233299999999, + "rss_peak_mib": 79.46484375 + }, + "created_at": "2026-11-15 21:11:43.975975+00:00" + } + ], + "pending_stream_entries": [] + }, + "new_hashes_unchanged": true + }, + { + "case": "malformed_notification_rejected", + "durable_diagnostic": true + }, + { + "case": "private_iam_and_other_api_download", + "security": { + "anonymous": 403, + "iam": { + "create_bucket": "AccessDenied", + "write_outside_prefix": "AccessDenied", + "list_other_bucket": "AccessDenied" + } + }, + "attachment": "439e8cec-078a-4b71-b836-6ca71a33d3ad", + "uploader": "6194c06e63d84f6b9262fe3a46b18495", + "downloader": "4e2acedc6f6f458a934dbd310445f914", + "sha256": "3e32f9fe6c30bc812cc1bb29e4f02f600440e886a6be18af53d3775938e916df" + } + ], + "worker_metrics": [ + { + "job_id": "cedd8181-ddd0-4312-a1f4-845c1ebde09b", + "detail": { + "result": "ready", + "seconds": 0.32190941699991527, + "cpu_seconds": 0.2808402049999996, + "rss_peak_mib": 79.21484375 + } + }, + { + "job_id": "a20bfaa6-deb4-4f07-bb26-3bdab33ecd78", + "detail": { + "result": "ready", + "seconds": 0.328002166999795, + "cpu_seconds": 0.2833926269999998, + "rss_peak_mib": 79.52734375 + } + }, + { + "job_id": "6bcd6f21-25e0-474d-a8b9-2a3d3529fa39", + "detail": { + "result": "ready", + "seconds": 0.2745089589998315, + "cpu_seconds": 0.24005962699999994, + "rss_peak_mib": 79.328125 + } + }, + { + "job_id": "a9edfa4a-5cd3-4051-b9ea-4eae2f397f43", + "detail": { + "result": "ready", + "seconds": 0.2622620000001916, + "cpu_seconds": 0.23187154500000018, + "rss_peak_mib": 79.33984375 + } + }, + { + "job_id": "d0d30cf2-42ef-4a6d-9993-e4efcaca5536", + "detail": { + "result": "ready", + "seconds": 0.2670629160002136, + "cpu_seconds": 0.23448512099999985, + "rss_peak_mib": 79.3515625 + } + }, + { + "job_id": "96087c3c-b072-4153-b848-8845b9cc3471", + "detail": { + "result": "ready", + "seconds": 0.25213508300021203, + "cpu_seconds": 0.22260241599999997, + "rss_peak_mib": 79.4609375 + } + }, + { + "job_id": "c3a72d12-66c4-4e7d-b44d-dc5b9c58617c", + "detail": { + "result": "ready", + "seconds": 0.47884837499987043, + "cpu_seconds": 0.43823329099999997, + "rss_peak_mib": 108.4375 + } + }, + { + "job_id": "cd0c7233-9ea8-4c0d-acdb-3c334c717031", + "detail": { + "result": "ready", + "seconds": 0.295458165999662, + "cpu_seconds": 0.2460942530000001, + "rss_peak_mib": 78.98046875 + } + }, + { + "job_id": "585b77ee-f960-4226-b07d-d1147986fc41", + "detail": { + "result": "ready", + "seconds": 0.271344792000491, + "cpu_seconds": 0.23713863300000004, + "rss_peak_mib": 79.2265625 + } + }, + { + "job_id": "14311a68-9802-4b57-80b2-f336ea55816d", + "detail": { + "result": "ready", + "seconds": 0.29587733399966964, + "cpu_seconds": 0.25321979000000017, + "rss_peak_mib": 79.21484375 + } + }, + { + "job_id": "7c2ef8f9-d167-451d-902f-32d73e909d3e", + "detail": { + "result": "ready", + "seconds": 0.29315970900006505, + "cpu_seconds": 0.2512351220000002, + "rss_peak_mib": 79.44921875 + } + }, + { + "job_id": "626a8509-a930-4cee-a6f6-63c0daac594f", + "detail": { + "result": "ready", + "seconds": 0.2957551669996974, + "cpu_seconds": 0.253827797, + "rss_peak_mib": 79.46484375 + } + }, + { + "job_id": "3e2eccd9-713d-4982-a025-5537b66c114c", + "detail": { + "result": "ready", + "seconds": 0.4591874589996223, + "cpu_seconds": 0.37699009000000006, + "rss_peak_mib": 79.5859375 + } + }, + { + "job_id": "5747edc1-9de6-40ba-a143-85ded56744d1", + "detail": { + "result": "ready", + "seconds": 0.2649612499999421, + "cpu_seconds": 0.23184441700000025, + "rss_peak_mib": 79.40234375 + } + }, + { + "job_id": "2177d978-9a49-4856-a9e9-6211e94e9246", + "detail": { + "result": "ready", + "seconds": 0.27407174999916606, + "cpu_seconds": 0.23973883399999996, + "rss_peak_mib": 79.4765625 + } + }, + { + "job_id": "3f046f7c-825f-446f-bf52-07e4cd7e330b", + "detail": { + "result": "ready", + "seconds": 0.2666690420001032, + "cpu_seconds": 0.2308113759999999, + "rss_peak_mib": 79.35546875 + } + }, + { + "job_id": "64d6ff0c-6a0d-4596-8e70-ffbd66a38fc0", + "detail": { + "result": "ready", + "seconds": 0.2976695839997774, + "cpu_seconds": 0.2562047089999999, + "rss_peak_mib": 79.33984375 + } + }, + { + "job_id": "6eacee3c-6b6b-4435-bd05-bb50c29df24f", + "detail": { + "result": "ready", + "seconds": 0.3038405419993069, + "cpu_seconds": 0.26212937199999997, + "rss_peak_mib": 79.4765625 + } + }, + { + "job_id": "0d40fa0a-e6b9-49cb-907d-e82faa346906", + "detail": { + "result": "ready", + "seconds": 0.30515825000020413, + "cpu_seconds": 0.2600924969999998, + "rss_peak_mib": 79.46484375 + } + }, + { + "job_id": "00893137-7e67-434b-8764-5c85ed5353d2", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "00893137-7e67-434b-8764-5c85ed5353d2", + "detail": { + "result": "ready", + "seconds": 0.2563110829996731, + "cpu_seconds": 0.22742345900000016, + "rss_peak_mib": 79.35546875 + } + }, + { + "job_id": "eefeceb7-6e35-4358-bbac-772170b964fa", + "detail": { + "code": "worker_timeout", + "result": "transient" + } + }, + { + "job_id": "eefeceb7-6e35-4358-bbac-772170b964fa", + "detail": { + "result": "ready", + "seconds": 0.30338325100001384, + "cpu_seconds": 0.2594866250000001, + "rss_peak_mib": 79.484375 + } + }, + { + "job_id": "88929609-e34a-4b5f-ad36-ffce561ebde8", + "detail": { + "code": "invalid_image", + "result": "permanent" + } + }, + { + "job_id": "88929609-e34a-4b5f-ad36-ffce561ebde8", + "detail": { + "result": "ready", + "seconds": 0.2860605830001077, + "cpu_seconds": 0.24953375300000014, + "rss_peak_mib": 79.171875 + } + }, + { + "job_id": "b6e323c3-3e4a-45fd-a4a5-709094e97485", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "b6e323c3-3e4a-45fd-a4a5-709094e97485", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "b6e323c3-3e4a-45fd-a4a5-709094e97485", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "b6e323c3-3e4a-45fd-a4a5-709094e97485", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "b6e323c3-3e4a-45fd-a4a5-709094e97485", + "detail": { + "code": "processing_unavailable", + "result": "transient" + } + }, + { + "job_id": "b6e323c3-3e4a-45fd-a4a5-709094e97485", + "detail": { + "result": "ready", + "seconds": 0.2706124579999596, + "cpu_seconds": 0.23715611999999986, + "rss_peak_mib": 79.3359375 + } + }, + { + "job_id": "108ade45-7571-4362-92d5-bde3c6232fe9", + "detail": { + "result": "ready", + "seconds": 0.4013020000002143, + "cpu_seconds": 0.3315808340000004, + "rss_peak_mib": 79.125 + } + }, + { + "job_id": "e7f4c924-cb49-40a5-b15b-b5bd8b3bd401", + "detail": { + "result": "ready", + "seconds": 0.3865292509999563, + "cpu_seconds": 0.3280367870000003, + "rss_peak_mib": 79.31640625 + } + }, + { + "job_id": "75ceffdc-919b-4f92-96c4-2e0c775850d7", + "detail": { + "result": "ready", + "seconds": 0.25812733300062973, + "cpu_seconds": 0.22754957700000022, + "rss_peak_mib": 79.46484375 + } + }, + { + "job_id": "bc562c08-8221-4f9d-b5ff-890d16987495", + "detail": { + "result": "stale" + } + }, + { + "job_id": "6d569c1e-c44c-4b65-a49d-ae731029270a", + "detail": { + "result": "ready", + "seconds": 0.25968787600049836, + "cpu_seconds": 0.2296233299999999, + "rss_peak_mib": 79.46484375 + } + }, + { + "job_id": "61e16816-661d-46bd-882e-80cc1cab86dd", + "detail": { + "result": "stale" + } + }, + { + "job_id": "c365c2f6-21a0-4958-9a8d-fbef517b2438", + "detail": { + "result": "ready", + "seconds": 0.3136757089996536, + "cpu_seconds": 0.26488450100000005, + "rss_peak_mib": 79.4609375 + } + } + ], + "audit": [ + { + "job_id": "626a8509-a930-4cee-a6f6-63c0daac594f", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 21:00:27.998177+00:00" + }, + { + "job_id": "88929609-e34a-4b5f-ad36-ffce561ebde8", + "action": "replay", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 21:08:02.552049+00:00" + }, + { + "job_id": "b6e323c3-3e4a-45fd-a4a5-709094e97485", + "action": "replay", + "operator": "synthetic-verifier", + "reason": "Explicit fault matrix replay", + "created_at": "2026-11-15 21:09:05.558417+00:00" + }, + { + "job_id": "75ceffdc-919b-4f92-96c4-2e0c775850d7", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Held stale writer race", + "created_at": "2026-11-15 21:10:42.873685+00:00" + }, + { + "job_id": "6d569c1e-c44c-4b65-a49d-ae731029270a", + "action": "new_version", + "operator": "synthetic-verifier", + "reason": "Held stale writer race", + "created_at": "2026-11-15 21:11:41.212454+00:00" + } + ] + }, + "cleanup": { + "source_commit": "442163b0f0a549ef065cb13ee39b5ca2f8580c18", + "seed": "clean-gc-seed.json", + "complete": true, + "real_s3_age_seconds": 754.765394, + "dry_run": { + "apply": false, + "retention_seconds": 300, + "actions": [ + { + "action": "delete_expired_object", + "key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/crop", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/optimized", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/thumbnail", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/old-orphan", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/651d13cc-9ebf-43f9-96bb-4da43041232a/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/7156ee96-6a61-431a-8aea-6e44402aca80/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/a705c027-cdd8-4c13-a75a-8dd8057d4f46/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/c55c3fd6-958b-4baf-b436-8cee3f228e18/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/original", + "applied": false + } + ] + }, + "apply": { + "apply": true, + "retention_seconds": 300, + "actions": [ + { + "action": "delete_expired_object", + "key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/crop", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/optimized", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/thumbnail", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/old-orphan", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/651d13cc-9ebf-43f9-96bb-4da43041232a/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/7156ee96-6a61-431a-8aea-6e44402aca80/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/a705c027-cdd8-4c13-a75a-8dd8057d4f46/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/c55c3fd6-958b-4baf-b436-8cee3f228e18/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/original", + "applied": true + } + ] + }, + "repeat": { + "apply": true, + "retention_seconds": 300, + "actions": [] + }, + "before": { + "objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/old-orphan": true, + "objects/staging/7156ee96-6a61-431a-8aea-6e44402aca80/original": true, + "objects/staging/c55c3fd6-958b-4baf-b436-8cee3f228e18/original": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/original": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/thumbnail": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/crop": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/optimized": true, + "objects/staging/8f2a0358-fe4e-4992-af51-9a95ad638e80/original": true, + "objects/staging/6e60a745-ad14-4270-97f1-b74b91d5e365/original": true, + "objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/fresh-orphan": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/original": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/thumbnail": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/crop": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/optimized": true + }, + "after": { + "objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/old-orphan": false, + "objects/staging/7156ee96-6a61-431a-8aea-6e44402aca80/original": false, + "objects/staging/c55c3fd6-958b-4baf-b436-8cee3f228e18/original": false, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/original": false, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/thumbnail": false, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/crop": false, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/optimized": false, + "objects/staging/8f2a0358-fe4e-4992-af51-9a95ad638e80/original": true, + "objects/staging/6e60a745-ad14-4270-97f1-b74b91d5e365/original": true, + "objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/fresh-orphan": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/original": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/thumbnail": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/crop": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/optimized": true + }, + "dry_run_unchanged": true, + "active_current_and_fresh_protected": true, + "expired_source_replay_refused": true, + "expired_writer_fenced": true + }, + "cluster": { + "date": "2026-11-15T20:56:37.768517+00:00", + "source_commit": "442163b0f0a549ef065cb13ee39b5ca2f8580c18", + "complete": true, + "before": [ + { + "role": "api", + "container": "eab0cd802b15", + "pid": 53575, + "started_at": "2026-11-15T20:49:36.429341461Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "6c3bf4efeb37426a9a8b0b5af3c847c8" + }, + { + "role": "api", + "container": "2336759a5f8d", + "pid": 53710, + "started_at": "2026-11-15T20:49:36.845556836Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "role": "broker", + "container": "4d137f9082a6", + "pid": 52638, + "started_at": "2026-11-15T20:49:25.670241428Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "bf556d707dbe", + "pid": 52699, + "started_at": "2026-11-15T20:49:25.902509928Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "14c7f8cd111f", + "pid": 52579, + "started_at": "2026-11-15T20:49:25.376057678Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "1417ad16e4fa", + "pid": 54468, + "started_at": "2026-11-15T20:49:48.562425633Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "64543" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "b754a512ac12", + "pid": 52773, + "started_at": "2026-11-15T20:49:26.185891512Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "84ee67d8a61d", + "pid": 53502, + "started_at": "2026-11-15T20:49:36.266057252Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "81ea8da3a6d6", + "pid": 53427, + "started_at": "2026-11-15T20:49:35.873595252Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "9886f90f9e58", + "pid": 52517, + "started_at": "2026-11-15T20:49:25.085013095Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "14282fbd37d4", + "pid": 53652, + "started_at": "2026-11-15T20:49:36.629893127Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ], + "parallel_replay": { + "requests": 30, + "messages": 1, + "instances": [ + "4aab3f4814b3449880a75a43bcb6101f", + "6c3bf4efeb37426a9a8b0b5af3c847c8" + ] + }, + "atomic_rollback_and_5xx_retry": true, + "expired_key_can_be_reused": true, + "failovers": [ + { + "confirmed_writes": 202, + "successful_reads": 192, + "errors": 10, + "successful_rps": 26.11, + "first_success_after_action_seconds": 4.776, + "action": { + "finished": 3.744 + }, + "observations": [ + { + "at": 0.008, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.073, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.127, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.194, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.291, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.362, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.433, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.512, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.575, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.636, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.698, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.759, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.817, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.876, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.945, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.008, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.073, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.14, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.2, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.269, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.334, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.399, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.459, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.52, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.576, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.635, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.7, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.765, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.836, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.942, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 2.051, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 2.153, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 2.246, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 2.342, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 2.4, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 2.483, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 2.584, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 2.68, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 2.78, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 2.88, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 2.984, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 3.076, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 3.141, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 3.206, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 3.267, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 3.327, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 3.386, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 3.439, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 3.491, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 3.54, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 3.591, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 3.643, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 3.704, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 4.776, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 4.842, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 4.913, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 4.98, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 5.054, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 5.125, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 5.198, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 5.258, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 5.32, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 5.376, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 5.442, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 5.508, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 5.569, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 5.641, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 5.713, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 5.779, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 5.84, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 5.907, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 5.977, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 6.041, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 6.103, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 6.165, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 6.226, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 6.291, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 6.358, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 6.427, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 6.494, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 6.561, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 6.635, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 6.703, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 6.774, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 6.846, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 6.911, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 6.987, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 7.06, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 7.136, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 7.199, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 7.265, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 7.337, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 7.404, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 7.479, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 7.556, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 7.623, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 7.689, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 7.748, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 7.818, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 7.882, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 7.953, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 8.023, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 8.083, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 8.149, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 8.207, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 8.275, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 8.342, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 8.413, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 8.477, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 8.543, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 8.615, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 8.683, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 8.757, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 8.828, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 8.902, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 8.966, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 9.035, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 9.113, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 9.179, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 9.243, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 9.311, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 9.375, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 9.446, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 9.521, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 9.597, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 9.676, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 9.786, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 9.882, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 9.976, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 10.077, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 10.175, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 10.285, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 10.377, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 10.436, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 10.495, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 10.558, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 10.625, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 10.692, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 10.758, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 10.827, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 10.89, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 10.959, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 11.027, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 11.101, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 11.175, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 11.245, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 11.316, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 11.387, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 11.46, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 11.529, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 11.595, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 11.666, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 11.737, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 11.811, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 11.88, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 11.953, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 12.015, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 12.082, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 12.148, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 12.211, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 12.272, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 12.34, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 12.41, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 12.474, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 12.54, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 12.607, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 12.671, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 12.74, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 12.806, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 12.876, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 12.948, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 13.024, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 13.092, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 13.167, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 13.248, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 13.319, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 13.384, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 13.45, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 13.515, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 13.577, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 13.644, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 13.711, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 13.776, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 13.845, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 13.913, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 13.987, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 14.056, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 14.115, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 14.177, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 14.239, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 14.303, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 14.365, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 14.425, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 14.493, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 14.555, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 14.618, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 14.685, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 14.747, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 14.814, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 14.874, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 14.94, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 15.007, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "eab0cd802b15", + "pid": 53575, + "started_at": "2026-11-15T20:49:36.429341461Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "6c3bf4efeb37426a9a8b0b5af3c847c8" + } + }, + { + "confirmed_writes": 218, + "successful_reads": 211, + "errors": 8, + "successful_rps": 28.4, + "first_success_after_action_seconds": 2.971, + "action": { + "finished": 2.947 + }, + "observations": [ + { + "at": 0.018, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.083, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.147, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.21, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.276, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.337, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.398, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.453, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.52, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.578, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.634, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.697, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.754, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.815, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.882, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 0.996, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.091, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.198, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.293, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.394, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.489, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.571, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.671, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.773, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.883, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 1.972, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 2.075, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 2.175, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 2.234, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + }, + { + "at": 2.295, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.356, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.425, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.479, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.531, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.588, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.645, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.705, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.769, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.832, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.904, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.971, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.034, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.094, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.158, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.239, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.304, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.371, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.43, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.488, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.553, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.613, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.677, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.739, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.804, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.873, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.943, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.016, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.075, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.137, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.202, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.261, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.326, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.388, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.453, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.517, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.572, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.638, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.694, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.758, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.83, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.898, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.965, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.031, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.095, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.164, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.233, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.303, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.373, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.45, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.518, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.587, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.649, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.716, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.774, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.838, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.897, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.959, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.021, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.086, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.153, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.21, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.27, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.336, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.399, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.469, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.538, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.603, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.669, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.736, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.805, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.874, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.944, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.013, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.082, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.154, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.224, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.295, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.372, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.446, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.519, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.592, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.656, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.73, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.806, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.879, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.944, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.003, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.07, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.144, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.215, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.283, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.35, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.409, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.471, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.536, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.602, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.658, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.719, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.786, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.854, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.925, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.986, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 9.046, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 9.134, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 9.236, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 9.336, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 9.538, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 9.637, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 9.742, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 9.81, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 9.875, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 9.937, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.005, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.07, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.147, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.21, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.274, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.344, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.403, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.46, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.522, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.581, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.644, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.703, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.765, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.821, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.886, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.943, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.002, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.076, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.151, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.222, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.299, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.385, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.462, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.523, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.588, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.655, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.73, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.806, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.872, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.946, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.011, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.083, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.148, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.214, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.276, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.335, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.402, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.467, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.526, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.598, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.662, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.732, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.794, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.857, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.919, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.985, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.047, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.114, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.181, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.237, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.304, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.372, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.458, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.528, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.595, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.666, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.731, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.797, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.862, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.926, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.01, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.082, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.161, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.229, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.288, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.356, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.418, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.48, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.541, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.604, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.671, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.738, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.81, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.868, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.925, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.998, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "stopped": { + "role": "api", + "container": "2336759a5f8d", + "pid": 53710, + "started_at": "2026-11-15T20:49:36.845556836Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "4aab3f4814b3449880a75a43bcb6101f" + } + } + ], + "rolling": { + "confirmed_writes": 739, + "successful_reads": 738, + "errors": 45, + "successful_rps": 27.1, + "first_success_after_action_seconds": 52.399, + "action": { + "finished": 52.347 + }, + "observations": [ + { + "at": 0.014, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 0.082, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 0.141, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 0.203, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 0.272, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 0.335, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 0.401, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 0.461, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 0.524, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 0.594, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 0.657, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 0.719, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 0.78, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 0.843, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 0.905, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 0.966, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 1.023, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 1.078, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 1.141, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 1.202, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 1.265, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 1.325, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 1.386, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 1.443, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 1.499, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 1.557, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 1.623, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 1.687, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 1.761, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 1.839, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 1.916, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 1.983, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.051, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.104, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.166, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.222, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.277, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.334, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.394, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.456, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.521, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.598, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.664, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.721, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.784, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.841, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.903, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 2.962, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.023, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.091, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.155, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.223, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.301, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.368, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.43, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.485, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.551, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.618, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.68, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.744, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.807, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.864, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.922, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 3.982, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.048, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.107, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.171, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.227, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.284, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.347, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.41, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.478, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.545, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.637, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.705, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.775, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.84, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.899, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 4.967, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.028, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.088, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.149, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.213, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.27, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.329, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.391, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.461, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.526, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.583, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.651, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.717, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.781, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.846, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.911, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 5.98, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.041, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.11, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.168, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.229, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.293, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.362, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.428, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.495, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.562, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.627, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.693, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.763, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.82, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.891, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 6.955, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.021, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.087, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.166, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.234, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.306, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.37, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.557, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.731, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.827, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 7.924, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.006, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.216, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 8.859, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 9.435, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 9.559, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 9.65, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 9.747, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 9.935, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.023, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.09, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.16, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.22, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.296, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.362, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.434, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.496, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.563, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.626, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.697, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.777, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.852, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 10.92, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.002, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.066, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.135, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.203, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.284, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.423, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.524, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.611, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.673, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.738, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.799, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.864, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.927, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 11.989, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.052, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.115, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.18, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.256, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.32, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.388, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.452, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.515, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.579, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.65, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.713, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.779, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.851, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 12.983, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.044, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.104, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.17, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.238, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.309, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.379, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.449, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.518, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.606, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.692, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.768, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.832, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.894, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 13.961, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.032, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.114, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.18, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.255, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.32, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.397, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.455, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.514, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.576, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.646, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.706, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.765, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.83, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.897, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 14.969, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 15.04, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 15.116, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 15.178, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 15.245, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 15.311, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 15.37, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 15.44, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 15.507, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 15.573, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 15.635, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 15.709, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 15.777, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 15.858, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 15.931, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 16.0, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 16.068, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 16.137, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 16.211, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 16.28, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 16.353, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 16.417, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 16.488, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 16.557, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 16.622, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 16.685, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 16.747, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 16.83, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 16.898, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 16.962, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 17.03, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 17.093, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 17.156, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 17.22, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 17.277, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 17.334, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 17.394, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 17.466, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 17.528, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 17.595, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 17.658, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 17.725, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 17.807, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 17.868, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 17.933, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 17.989, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 18.048, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 18.118, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 18.178, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 18.24, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 18.302, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 18.362, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 18.421, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 18.486, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 18.548, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 18.604, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 18.67, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 18.73, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 18.786, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 18.846, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 18.911, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 18.98, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 19.069, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 19.142, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 19.207, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 19.274, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 19.332, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 19.397, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 19.465, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 19.532, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 19.6, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 19.662, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 19.738, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 19.814, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 19.886, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 20.003, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 20.07, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 20.177, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 20.245, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 20.332, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 20.401, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 20.47, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 20.535, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 20.605, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 20.672, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 20.736, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 20.8, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 20.857, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 20.918, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 20.988, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 21.054, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 21.12, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 21.182, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 21.24, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 21.306, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 21.363, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 21.421, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 21.487, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 21.555, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 21.619, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 21.684, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 21.764, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 21.834, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 21.927, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 22.017, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 22.097, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 22.174, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 22.25, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 22.337, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 22.404, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 22.484, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 22.553, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 22.619, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 22.681, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 22.756, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 22.832, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 22.895, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 22.962, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 23.032, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 23.097, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 23.171, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 23.24, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 23.303, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 23.365, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 23.432, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 23.498, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 23.554, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 23.616, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 23.679, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 23.736, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 23.805, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 23.873, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 23.95, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 24.015, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 24.086, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 24.146, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 24.213, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 24.28, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 24.345, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 24.413, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 24.488, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 24.547, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 24.619, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 24.706, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 24.78, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 24.85, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 24.914, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 24.984, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 25.046, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 25.114, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 25.205, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 25.28, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 25.345, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 25.407, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 25.474, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 25.534, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 25.598, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 25.668, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 25.744, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 25.834, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 25.905, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 25.974, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 26.036, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 26.098, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 26.164, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 26.234, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 26.3, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 26.371, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 26.437, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 26.5, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 26.563, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 26.63, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 26.696, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 26.767, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 26.839, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 26.902, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 26.967, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 27.042, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 27.113, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 27.183, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 27.256, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 27.333, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 27.402, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 27.464, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 27.529, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 27.591, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 27.651, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 27.716, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 27.78, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 27.847, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "at": 27.907, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 27.965, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 28.042, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 28.125, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 28.192, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 28.274, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 28.345, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 28.415, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 28.481, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 28.55, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 28.628, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 29.795, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 29.867, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 29.932, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 29.99, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 30.05, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 30.111, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 30.173, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 30.238, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 30.309, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 30.386, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 30.449, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 30.512, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 30.571, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 30.64, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 30.715, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 30.779, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 30.876, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 30.978, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 31.078, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 31.182, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 31.28, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 31.352, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 31.41, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 31.477, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 31.544, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 31.606, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 31.675, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 31.746, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 31.813, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 31.879, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 31.948, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 32.012, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 32.079, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 32.157, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 32.222, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 32.3, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 32.371, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 32.442, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 32.527, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 32.607, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 32.687, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 32.819, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 32.911, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 32.997, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 33.089, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 33.169, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 33.238, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 33.311, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 33.387, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 33.456, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 33.522, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 33.587, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 33.646, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 33.716, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 33.794, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 33.863, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 33.924, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 33.991, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 34.059, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 34.12, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 34.218, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 34.294, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 34.357, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 34.425, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 34.497, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 34.564, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 34.66, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 34.739, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 34.859, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 34.947, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 35.047, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 35.147, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 35.247, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 35.339, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 35.4, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 35.463, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 35.528, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 35.592, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 35.655, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 35.713, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 35.779, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 35.849, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 35.917, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 35.983, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 36.049, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 36.115, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 36.174, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 36.25, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 36.313, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 36.379, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 36.45, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 36.516, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 36.58, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 36.642, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 36.704, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 36.772, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 36.834, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 36.901, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 36.972, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 37.049, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 37.116, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 37.192, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 37.256, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 37.316, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 37.373, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 37.436, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 37.495, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 37.563, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 37.632, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 37.698, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 37.768, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 37.832, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 37.897, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 37.955, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 38.022, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 38.082, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 38.141, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 38.199, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 38.262, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 38.326, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 38.388, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 38.453, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 38.518, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 38.596, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 38.692, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 38.775, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 38.854, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 38.947, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 39.046, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 39.126, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 39.2, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 39.291, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 39.38, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 39.458, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 39.534, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 39.6, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 39.665, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 39.732, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 39.795, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 39.862, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 39.933, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 40.006, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 40.08, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 40.146, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 40.203, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 40.262, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 40.327, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 40.395, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 40.458, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 40.522, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 40.582, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 40.644, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 40.708, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 40.781, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 40.845, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 40.904, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 40.971, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 41.039, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 41.109, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 41.167, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 41.232, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 41.299, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 41.392, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 41.486, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 41.586, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 41.681, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 41.792, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 41.879, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 41.987, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 42.056, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 42.116, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 42.182, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 42.243, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 42.311, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 42.376, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 42.437, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 42.502, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 42.579, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 42.646, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 42.714, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 42.788, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 42.854, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 42.922, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 42.982, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 43.041, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 43.103, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 43.166, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 43.23, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 43.289, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 43.354, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 43.419, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 43.491, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 43.552, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 43.613, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 43.683, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 43.751, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 43.813, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 43.886, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 43.947, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 44.014, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 44.088, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 44.152, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 44.223, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 44.289, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 44.355, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 44.42, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 44.486, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 44.552, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 44.619, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 44.686, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 44.746, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 44.805, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 44.905, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 44.997, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 45.109, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 45.18, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 45.24, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 45.303, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 45.379, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 45.437, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 45.505, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 45.576, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 45.661, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 45.788, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 45.856, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 45.951, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 46.017, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 46.079, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 46.147, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 46.213, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 46.278, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 46.347, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 46.416, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 46.482, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 46.544, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 46.604, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 46.671, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 46.736, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 46.796, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 46.855, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 46.918, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 46.982, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 47.051, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 47.123, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 47.195, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 47.255, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 47.318, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 47.381, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 47.445, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 47.516, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 47.582, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 47.652, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 47.726, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 47.797, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 47.865, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 47.926, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "at": 47.991, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "at": 48.056, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 48.126, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 48.187, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 48.251, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 48.327, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 48.39, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 48.459, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 48.526, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 48.601, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 48.668, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 48.754, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 48.826, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 48.923, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 49.02, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 49.156, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 49.275, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 49.396, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 49.478, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 49.543, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 49.611, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 49.674, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 49.753, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 49.815, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 49.884, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 49.949, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 50.015, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 50.087, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 50.158, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 50.221, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 50.285, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 50.357, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 50.432, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 50.502, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 50.566, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 50.633, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 50.696, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 50.764, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 50.83, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 50.899, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 50.97, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 51.043, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 51.119, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 51.217, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 51.29, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 51.385, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 51.469, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 51.545, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 51.635, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 51.711, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 52.055, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 52.171, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 52.259, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 52.325, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 52.399, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 52.468, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 52.531, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 52.602, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 52.673, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 52.748, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 52.813, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 52.881, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 52.946, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 53.014, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 53.083, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 53.147, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 53.216, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 53.284, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 53.358, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 53.423, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 53.495, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 53.565, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 53.624, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 53.685, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 53.745, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 53.81, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 53.872, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 53.937, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 54.009, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 54.081, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 54.146, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 54.207, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 54.275, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "at": 54.35, + "instance": "00527ff4190343f78b6ada02dc5074bd" + } + ], + "confirmed_preserved": true, + "duplicate_texts": false, + "before": [ + { + "role": "api", + "container": "eab0cd802b15", + "pid": 94499, + "started_at": "2026-11-15T20:57:03.027731251Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "7b172c7a284c4540b43f1c2207ae103f" + }, + { + "role": "api", + "container": "2336759a5f8d", + "pid": 96286, + "started_at": "2026-11-15T20:57:35.466686919Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "b086988c0ed8457fb08ac362bb19e420" + }, + { + "role": "broker", + "container": "4d137f9082a6", + "pid": 52638, + "started_at": "2026-11-15T20:49:25.670241428Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "bf556d707dbe", + "pid": 52699, + "started_at": "2026-11-15T20:49:25.902509928Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "14c7f8cd111f", + "pid": 52579, + "started_at": "2026-11-15T20:49:25.376057678Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "1417ad16e4fa", + "pid": 54468, + "started_at": "2026-11-15T20:49:48.562425633Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "64543" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "b754a512ac12", + "pid": 52773, + "started_at": "2026-11-15T20:49:26.185891512Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "84ee67d8a61d", + "pid": 53502, + "started_at": "2026-11-15T20:49:36.266057252Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "81ea8da3a6d6", + "pid": 53427, + "started_at": "2026-11-15T20:49:35.873595252Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "9886f90f9e58", + "pid": 52517, + "started_at": "2026-11-15T20:49:25.085013095Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "14282fbd37d4", + "pid": 53652, + "started_at": "2026-11-15T20:49:36.629893127Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ], + "after": [ + { + "role": "api", + "container": "e2a8f2c18e86", + "pid": 97114, + "started_at": "2026-11-15T20:57:50.896257176Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "cfbf76eaea374f7f8482cbae030020a9" + }, + { + "role": "api", + "container": "6750838fa191", + "pid": 98565, + "started_at": "2026-11-15T20:58:21.798847676Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "00527ff4190343f78b6ada02dc5074bd" + }, + { + "role": "broker", + "container": "4d137f9082a6", + "pid": 52638, + "started_at": "2026-11-15T20:49:25.670241428Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "bf556d707dbe", + "pid": 52699, + "started_at": "2026-11-15T20:49:25.902509928Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "14c7f8cd111f", + "pid": 52579, + "started_at": "2026-11-15T20:49:25.376057678Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "1417ad16e4fa", + "pid": 54468, + "started_at": "2026-11-15T20:49:48.562425633Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "64543" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "b754a512ac12", + "pid": 52773, + "started_at": "2026-11-15T20:49:26.185891512Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "84ee67d8a61d", + "pid": 53502, + "started_at": "2026-11-15T20:49:36.266057252Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "81ea8da3a6d6", + "pid": 53427, + "started_at": "2026-11-15T20:49:35.873595252Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "9886f90f9e58", + "pid": 52517, + "started_at": "2026-11-15T20:49:25.085013095Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "14282fbd37d4", + "pid": 53652, + "started_at": "2026-11-15T20:49:36.629893127Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ] + }, + "replay_after_replacement": true, + "public_failover": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 112, \"failed_attempts\": 3, \"first_recovery_seconds\": 3.423}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.", + "public_persistence": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.", + "after_persistence": [ + { + "role": "api", + "container": "e2a8f2c18e86", + "pid": 4007, + "started_at": "2026-11-15T20:59:40.230514379Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "1ac6f01b5ace4c0493fa2701d35e5ddc" + }, + { + "role": "api", + "container": "6750838fa191", + "pid": 3951, + "started_at": "2026-11-15T20:59:40.110009296Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 805306368, + "instance": "c75b73f0a1e64957af139c8c13ad140d" + }, + { + "role": "broker", + "container": "4d137f9082a6", + "pid": 3566, + "started_at": "2026-11-15T20:59:38.820680753Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 268435456 + }, + { + "role": "cache", + "container": "bf556d707dbe", + "pid": 3560, + "started_at": "2026-11-15T20:59:38.812745212Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 134217728 + }, + { + "role": "controller", + "container": "14c7f8cd111f", + "pid": 52579, + "started_at": "2026-11-15T20:49:25.376057678Z", + "ports": {}, + "cpu_nano": 500000000, + "memory_limit": 201326592 + }, + { + "role": "lb", + "container": "1417ad16e4fa", + "pid": 3662, + "started_at": "2026-11-15T20:59:38.969360462Z", + "ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "64543" + } + ] + }, + "cpu_nano": 500000000, + "memory_limit": 134217728 + }, + { + "role": "postgres", + "container": "b754a512ac12", + "pid": 3690, + "started_at": "2026-11-15T20:59:39.01175442Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 536870912 + }, + { + "role": "reconciler", + "container": "84ee67d8a61d", + "pid": 53502, + "started_at": "2026-11-15T20:49:36.266057252Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 268435456 + }, + { + "role": "relay", + "container": "81ea8da3a6d6", + "pid": 4199, + "started_at": "2026-11-15T20:59:41.82030238Z", + "ports": {}, + "cpu_nano": 250000000, + "memory_limit": 201326592 + }, + { + "role": "s3", + "container": "9886f90f9e58", + "pid": 3689, + "started_at": "2026-11-15T20:59:39.013327962Z", + "ports": {}, + "cpu_nano": 1000000000, + "memory_limit": 1073741824 + }, + { + "role": "worker", + "container": "14282fbd37d4", + "pid": 5322, + "started_at": "2026-11-15T21:00:03.732398709Z", + "ports": {}, + "cpu_nano": 2000000000, + "memory_limit": 1610612736 + } + ] + }, + "api_worker_relay_reconciler_runtime_hashes_and_revision_match": true, + "fsck": "", + "checks_completed": true, + "result": "PASS" +} diff --git a/evidence/cleanup-final.json b/evidence/cleanup-final.json new file mode 100644 index 00000000..209c939b --- /dev/null +++ b/evidence/cleanup-final.json @@ -0,0 +1,159 @@ +{ + "source_commit": "442163b0f0a549ef065cb13ee39b5ca2f8580c18", + "seed": "clean-gc-seed.json", + "complete": true, + "real_s3_age_seconds": 754.765394, + "dry_run": { + "apply": false, + "retention_seconds": 300, + "actions": [ + { + "action": "delete_expired_object", + "key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/crop", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/optimized", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/thumbnail", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/old-orphan", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/651d13cc-9ebf-43f9-96bb-4da43041232a/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/7156ee96-6a61-431a-8aea-6e44402aca80/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/a705c027-cdd8-4c13-a75a-8dd8057d4f46/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/c55c3fd6-958b-4baf-b436-8cee3f228e18/original", + "applied": false + }, + { + "action": "delete_expired_object", + "key": "objects/staging/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/original", + "applied": false + } + ] + }, + "apply": { + "apply": true, + "retention_seconds": 300, + "actions": [ + { + "action": "delete_expired_object", + "key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/crop", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/optimized", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/thumbnail", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/old-orphan", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/651d13cc-9ebf-43f9-96bb-4da43041232a/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/7156ee96-6a61-431a-8aea-6e44402aca80/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/a705c027-cdd8-4c13-a75a-8dd8057d4f46/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/c55c3fd6-958b-4baf-b436-8cee3f228e18/original", + "applied": true + }, + { + "action": "delete_expired_object", + "key": "objects/staging/dd1bf2f6-52ee-4f4c-b1ae-92b69ea682e6/original", + "applied": true + } + ] + }, + "repeat": { + "apply": true, + "retention_seconds": 300, + "actions": [] + }, + "before": { + "objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/old-orphan": true, + "objects/staging/7156ee96-6a61-431a-8aea-6e44402aca80/original": true, + "objects/staging/c55c3fd6-958b-4baf-b436-8cee3f228e18/original": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/original": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/thumbnail": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/crop": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/optimized": true, + "objects/staging/8f2a0358-fe4e-4992-af51-9a95ad638e80/original": true, + "objects/staging/6e60a745-ad14-4270-97f1-b74b91d5e365/original": true, + "objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/fresh-orphan": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/original": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/thumbnail": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/crop": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/optimized": true + }, + "after": { + "objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/old-orphan": false, + "objects/staging/7156ee96-6a61-431a-8aea-6e44402aca80/original": false, + "objects/staging/c55c3fd6-958b-4baf-b436-8cee3f228e18/original": false, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/original": false, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/thumbnail": false, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/crop": false, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v1/optimized": false, + "objects/staging/8f2a0358-fe4e-4992-af51-9a95ad638e80/original": true, + "objects/staging/6e60a745-ad14-4270-97f1-b74b91d5e365/original": true, + "objects/cleanup-7ae8cd5f0fbb42a9a73b5dc31f301db7/fresh-orphan": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/original": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/thumbnail": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/crop": true, + "objects/9b3d09db-c441-4a25-b976-567cb8c8e75e/v2/optimized": true + }, + "dry_run_unchanged": true, + "active_current_and_fresh_protected": true, + "expired_source_replay_refused": true, + "expired_writer_fenced": true +} diff --git a/evidence/pipeline-analysis-final/effects.csv b/evidence/pipeline-analysis-final/effects.csv new file mode 100644 index 00000000..c1f4677f --- /dev/null +++ b/evidence/pipeline-analysis-final/effects.csv @@ -0,0 +1,20 @@ +case,role,job_id,version,attempt,cycle,fence,state,publications,variant_rows,physical_keys +commit_before_publish_and_worker_stopped,effect,3e2eccd9-713d-4982-a025-5537b66c114c,1,1,1,1,ready,1,4,4 +confirmed_broker_entry_survives_sigkill,effect,5747edc1-9de6-40ba-a143-85ded56744d1,1,1,1,1,ready,1,4,4 +relay_crash_after_xadd_before_pg_marker,effect,2177d978-9a49-4856-a9e9-6211e94e9246,1,1,1,1,ready,1,4,4 +before_effect_sigkill_repeat_1,effect,3f046f7c-825f-446f-bf52-07e4cd7e330b,1,2,1,3,ready,1,4,4 +after_effect_sigkill_repeat_1,effect,64d6ff0c-6a0d-4596-8e70-ffbd66a38fc0,1,2,1,3,ready,1,4,4 +after_commit_sigkill_repeat_1,effect,efd4aa67-b47e-462f-8e86-614c5dbb8a17,1,1,1,1,ready,1,4,4 +before_effect_sigkill_repeat_2,effect,6eacee3c-6b6b-4435-bd05-bb50c29df24f,1,2,1,3,ready,1,4,4 +after_effect_sigkill_repeat_2,effect,0d40fa0a-e6b9-49cb-907d-e82faa346906,1,2,1,3,ready,1,4,4 +after_commit_sigkill_repeat_2,effect,c422c6de-4c05-43ea-b748-e4ff856a70c8,1,1,1,1,ready,1,4,4 +transient_retry_with_jitter,effect,00893137-7e67-434b-8764-5c85ed5353d2,1,2,1,3,ready,1,4,4 +hung_child_killed_and_retried,effect,eefeceb7-6e35-4358-bbac-772170b964fa,1,2,1,3,ready,1,4,4 +permanent_failed_registry_and_audited_replay,effect,88929609-e34a-4b5f-ad36-ffce561ebde8,1,1,2,4,ready,1,4,4 +finite_retry_exhaustion_then_explicit_replay,effect,b6e323c3-3e4a-45fd-a4a5-709094e97485,1,1,2,12,ready,1,4,4 +reordered_and_duplicate_notifications,1,e7f4c924-cb49-40a5-b15b-b5bd8b3bd401,1,1,1,1,ready,1,4,4 +reordered_and_duplicate_notifications,2,108ade45-7571-4362-92d5-bde3c6232fe9,1,1,1,1,ready,1,4,4 +stale_v1_after_v2_repeat_1,v1,bc562c08-8221-4f9d-b5ff-890d16987495,1,1,1,2,obsolete,0,0,4 +stale_v1_after_v2_repeat_1,v2,75ceffdc-919b-4f92-96c4-2e0c775850d7,2,1,1,1,ready,1,4,4 +stale_v1_after_v2_repeat_2,v1,61e16816-661d-46bd-882e-80cc1cab86dd,1,1,1,2,obsolete,0,0,4 +stale_v1_after_v2_repeat_2,v2,6d569c1e-c44c-4b65-a49d-ae731029270a,2,1,1,1,ready,1,4,4 diff --git a/evidence/pipeline-analysis-final/pipeline-results.png b/evidence/pipeline-analysis-final/pipeline-results.png new file mode 100644 index 0000000000000000000000000000000000000000..dcff5c3b5928baa6835d6209a0a6b73b1deb7be3 GIT binary patch literal 67940 zcmeFZcT|(>_BD#4SP-!RDpKqs0wTR*L8YTe3r#_KlWyo@K|#SnFQNBdLJJ9^n<9aP z9t4u0KmvpiAV3K9K5?J(+vj}$e0SV&|M~8?dyEaf5Z>~XwdR_0uJ@gR-aW2^rw_8R zv2khMzw?ldZ6Az{ZTI|s4)7B{_QoUNo1%|~sgI$Dy^sGBFFQ8fCqBsBvpAh%(a*zo7Fb__`cHr54GjBFFzGJNacR8f@ zJ!IR(#-@4a=A-ASbNGEhCk98p{*p@(6h0DgFJNEXrTvjtc$2f<3fr`nU3p&lq~Ypn zIM>mm8ZCsqoNv!*J;=F&Jb#~d;jGF}8bh%8CqmOtZuE2ar5!TC_j^~;&+2|ZOjG`y zVX7a3b@6}wH-0&#nREBQkJ#AM$4~upaO65fnf;%`-M9Yhmq&JE|M^{5*pcTPM`ZnI z1TkbFL$*A4W0lzy6cEs8%lyDE7obuu&4~-j_Idv6HD6aHeQhEpLQ&Mt-duHkIy)1i z)$nx{jP4cuC4HFdd<+diQx!oIX)pE(Ua7)MD9WliGw4D3g4NE7q+&6TdZc5jpVROiqfkSSWD|!U(q7BV|O%n=P zyNOZT32I;v0;8s+zCi;$@Qp^iVvwg+>nd}8ftBvXpR7L9! zFl+`6Txcx@L}#>s@_}cD#hIK8+81?($(7oBW=0ld zJ{G@&H)C!+xGs{d9|I*Y8>&@vxu)!NA6TRVB;%o z+dE0!DVCVf-|sF@#v1B@=i@8*l7nvvbN*}>C%WZ`(l^O1FZZPgo7ZzjP;{=)*X9wl zPB6S4Fv~{H$T}CCp6d3RtrNb{Hv?tR!Dpbvx+jypMg=w~)5m%u1Bg@i8Q(5UaK`)_d~c0-t_Zm@ss{fjkMW zC0iPF?D+AiPSNsMm`b45!-t;>S90NB386c|{dTQp4rv$r^Ys&ceUGT#(tfwL_B=@~ z6vaf?q2rH~Zhogh6C&eAI(gY+x|PvcbzvVPmdZSK}oI z*XhJiO2u1Z=nhhZv&^izt;-tWR|=K2s<@HlJW%k-B48%Buhd+QjworKSsz8&wjb_~ zG&k=<@MCmhWJJFfQF_Rdke|%ra@6NKLIApE?Zbx;asCA8J*`_#YkDoFR){X#xcm!; zJ&|B7?XeDF=Dz2b!)%sXYK1Je@;xjzE)8;8eo`{zK1*zFo|*kUTU;IwR+=^$llF68 zZkXRrmRRpM-Fg!L6&Fa2;}feeB}l(H=Wh&S-w3;w{06m-sI$x1>Fj!DgrHX!Vmd3T z>wBQsBTBsu&eA_EHDI>zY?l}qteKZ%DuJu_(-eKr^w$hVs|Bpg@)%nV?{3Mm9G~Uf zpJ!HWpHvpVGTg@@e`SM=RP7%UPqOsQEv4IcB|bU*)akXP8#NFuRyV1gecOKK`NDe1 zScAj0VEZyBtP)L`2^L`Y@J|qg5Ca7d&J&lwV|m5qu0Wgoj=G(acKlf5JlH9HxEf-Q zP!fleA5sOI^wU)}LL9gPbmS(Lx4dhg`bF-4^E}IWGE?L>v}SOe{nByf2(WCy=$XZPnkOl>D~2^(#V1D_1K{ zC+aM9_IFR_?Y#Izj?*ggo>IKD`kK$mbaJ?DcH!4|)!>i^6Zfp2 zwSb^3VwhL&rV_L;ev>)*MZfF2U+J&>jeGQh^x}!4`Xvy>k|8UjwQf?Q26ZVz1334w zWhu^RW-yqZB-+zncP-IJ1>+ry90_9dN zVu4rXof%3J&6`yXB&WTpS*u1~UR4L~I;r|h@6x}0czV4i#R{JeQL29YqAi_T_?iu` zw!SrAKXKUbi=o=R_FuhCDyze`(X`?Xlfc3rY8mddK(S8y#LNi1n0a__OrT59MSK~i zr7oOk9dlI7bsByLOR04mP4;{?TIWIYN(zTsH9fCXAjn`F1y)jVbc0Zn$f&x}Z;fr{ zc|h)>1S#tMvsSL8-6SL`GzW4qtW3w>ri|XuGHLQzIkW+79B1sNwqgG z?_cM)+w$ARHZF=JF656LJveecE`F!x8}#noHw_+rfil#^F3W@h#iq}M%d=aBs~@wt zzrjcK1in0opm-g-k?ulO3qQfWiFTn-X~sf;M-F$} zN@i8Ul_8A?lH0BuK^=SOyW6ms+uZti^I49jsw{&c;M2lj{-j{o*b8boM=`GKHQynPY#$2&8i1^Dsb7uZEP5H?C zGqjiS$P)2aob&7+*f{kvsqNG+7d^$6SDP49kVgY>O{|)V+HzgD5x4bv>VW0m>-Yo+ zX-LZk@&-q*eoQeJvvvMyf%P{jjaX?Y3h_`OFXIQj#Z(Tj$Qjp_8pC>qe$G$J;+O$r z&S>{wW3v&I;q}^gswsoyzedcH^3E>24;$XKXA(rXT1sI~Ik)QC*J3-t;N=JHH$Z;& zMnO~LJp8&gHtAR@*JXUA)-(i4>!@y$}GK0^9x> zLU%>x@z=0h=v1#x5%+!yWw~F!PDuWlF!6>%dT?#KoucpTq%TV9#*H}>D`@VhFG%57 zlw)t!E%aWBywByH;8Pu*^P4|u%{wThaVdlu?=n_ji6v$hthnME2Sa8H^BeKpAVAj$ zDf$%jIOwFS;stqlru-DP=3!+@jcyYY6PDtEKkX8JmLujB>5R8W&o7DPw^zlreVNB2 zQhem_K40+WEO?8y2woW!>|dGbZ_nXZ@OqjF-FC-vN;LI!HUE0UKX*IguvpC)xM|}_ zzkU-No2+0O0m&Q+p7MF}hEAH4IKRi<$@NGleUhvHtr+$l&vNC#? z6jr(%XBF5zFKS);+V=xvIaw_<#2}iMRa@8Ge&16Fu@~+(I+TN)Eh?V-)uEW@fm|`J zb6OrAl=h_ziS;(^V(Ub?;~SHRiQ3`~IUO|+jsvyDGGxu`x*6Wff z4&3~*T@$hu%hXX^W$8=LvbBJ;1Iww`-nl$^(Eg>l*%Ed}HE1}aSGi;h)` z#^T8Hd((XSG&SP0cj)OmiPCYRCbi_LDW2r?v@|!N8&BU(1f?qZm!VfSK_-s&Dku1# zQ41L((+rQXvg%)oCZ*D4um`&c=N1m_={RWhsOZrTyC)R|LMegs4ef5%wLN0LeEvMg zt-KHxygK)DRbOpu;DlnS5Dh%+*wlJ@$ofnWH;(vxf{V^x)FLUG0k)&sW=}p1a@}M7 zw31)Rk5NABpeIXtyN;xl%Z0X{dcI1gvma1aTk1F;%{+ajM|c@v1DDkw)mT-R-Nyp5 zO$`7N(OFmq(Y3C5V;->zb_PM!w(Zby>B?{ga;mo;%5Hm;uHVj$MAcN}aGx*10o>k% zVSwZ-K!;c4Y*;Y9AJx#ct&+l_VA;wb$rG^;R4PLo; z^L6Hs4E?Wn$KA_64Q=DTp{mtu^H;upT_*&%IxWyy31dU2Tn#c^MRK_q1ENLiTz@%g zz4HVwT)mjw1+oV6@T=L%$2@H^YpY}4*reb^LLluY78Hc-egomoH}}#@UTYxVOfv>I z2gUYAA_&r zeLfW7I#iiU{v^k<9K^NC&Sr4tg#%KqC-54QyT9svJk^gn!qd|>lo3WCn{m&;zh%VV zUWU;?#ZVI?s4^1Uyd07<&Q7hbm#?pW|`$FpGacnCMcOOt_V-3|zaZndYGTgJPGCk`)76Wc+qGMQZQ>k!6!4cJ%%6=B z;-+m9&gsvSO;Sf~iKyU99mQzB>7JHH;-cVTn&bNqrWU zT^zam@XOuMsFgsaeh16JA|omKN>X|AP4e$;y_T^{n5iItu4*<9KyK_F$k9CREvZT= zsSbi*4)QuujQv}#A>R>d-OMxXWY`MmD{X2D3JSy~E`C{g_b@zMy>CX*TBwWf+GFVN zR5@hB#!PyU(pXkZ^g>KR;#7cJjS8mqS3;|gh!2>j~oe{O9Q8enP+deJkBR< zHjEx7kj!5?4K?i(3hJ5a&QKkD#S106JyM3_%pyCoK9bwrv!*MPM>S{ z^YOLKf6YyizcTD>#K}4De3C>WjqKwVbdB$aB=0qQ?It$N)1)wDy9PF%|77#yaWkA7 zPsnm+xTtO0dr3ONkLt{LEhuW7%UOD1JW};p^}+SzgvW0B*(wI zC*7^fU%fQ$COk09ROhsWOp8y)o4b4R|K*RV=Fqj=YgTH!6QiSbel2mimB{!L=5+o9&3>_|HLY zv^2ziiNqM?DIScC=VVnD2~IUS4;N4tq9PQp^>J%+B##e3=1NQT&wkxXU|5u0tr6v# zl@GmF+GN@zg;k!|A1QBg zogvA0Up^|EXO9}8y$HGsHe%N@;B68+<1ou78^Br@7;Q`cTHV|pMOW=*i!0u6*+R`n z9|&c~wl=6p-*<=`#qTZf34lN#D5qp@f2@7Y#dP=%e9YVcs_egs$7n#;enoT#z<*aL zZHM@Q@~zA3_$8fZ4ezb0@xgu>@_UfRZBvKA%n+WwQ98h2eeSuYaF;bMV*s<^MZCNEEP{kvPP$T0rhAo3H(Uz{B5 z*Vb(dOTR|0-@MwP-~nqq0&hGo_T2Dz*S{96nK^OR#|=40(S81)B;)hY(?fpaW**x< z0sVQpcn*K7&@JEgB~0l>c9p|Xe`gH;?YRwFGci91v2l2QD0_6EsAjxS+ttlC(`nfr z&u7f19UwL53SdEWQv>HE1>9jj->s@QdmM3PY3T3>Bf7Cm%X+8tyU>=se9sTH&AhCA z?5gO$=$Jn`jC~>MmJhiUpe5FX^EImt$Bx$_NDZBd66gu?n9fCeQ0$cyuK6sn2 zc!(Y^W*)Sz2Ues9SeJrCSgpTjy7#sq;^NWgLzGlH`weSr!eKs}-jVU4D##TE>1K0z ziyIqT_^IbZ8)C?ao?rKB?6Pa42O@(p-w!s-zR0_RQGwML*UUt6oQj-Pxx;R3{JJNc z!8xRoV0_8XBy)_J*&qseFw%LuB|c$jeTATfRxL9vJ!Tnq55HHb=3Ji9yXUh5g?|3h z!;hX7h^x&o>y(;HPj>_T* z@I^vAYaV~hSB5FI!qe`3IBhpnjCCJdWK_#c+o>=_hQ%t3pI4NDQZQQ`%VP~5IY{sC z2>HSyDs@JxJ={)PU{x#dxi>DY`tpbrq~%vns?)`5E~xx@Jg(f@dJ>M#!tF1 z3ax-$c2-yE$?*0J&4281y!dR?s?LEmO6d81_62QF=s{yXd{}X6sIRXNLLaip)7h;17i*LZQjWX?5OL`;jYyos5s-l?3Gl5QCRyk}twt2DLhZe$Pjk@H z>uKlComU;2+`;G}NeTj5kK1-E;H3fYMNo0w8`<2Fz0jTKC%-azcBqblPST|JTQm1G)B1A$(Ew$9N^43t9Wr~F5^}2jOxFn2v zH-O|y+5(hCJRmc*oSlpId>i2lNNq${lk9pjZZiA$eA-Sq)T_ks1EzUZd4N8uXa$Ib zSwq50q?pAbd2RKAY5V<(82slRWS~DiaEew8XdcyID?orwc;E^1ckbOoSa7g~eI5Xn znjt9wK<_C45@e_LBX7`j6lQ(Z4*Ka!C#DyFm)wgH zbZ0c=SG`b+3=A~2JFG&31ubc6>glZpm?2|u8#@5D>Weo_eh#ubpn^35dcvsN0~jVv zy|()DzQbp((~OFZeN)L{ zv&fu0WhUOA_+#?N7rU=vyYgI>ZT{L%@9IssF%G6goH6Dd!`xUM_3&qonx2_5u*8C( z<*`&e`h2Y3(+hA{3R}zl-IaM*zAYnw`iBAPPcNnOFGOsdKN@E8`Sa(l*K45QRdtix z+9f(mD)E}?6wjyVz93l4#AFh3iiIuhy(4HD_U+3r!8KgIw`Wc0P~X)8DQ}wGepbkzm?OAJ&34>2tT)K zYt^-J_69meN#GB8NzhlAI^hHuID@);xed#h%ca6g0LvzR%~$JVdbDx|yUui07fH77kK#pNn;)^EvJ zIQ>a5Hy1B7Agp|}wJ2wvGc+c_AdU+xO~;?NUaEljRXG;Avz2$_K8xh@;i^vqm0k42 z@J|qrblJ$D`l1(I%WPR(3YuR-q(#{T$LHH^ZMmwbu~4QVrqAIR-~{|K9ga0x>60I5 zriuO?jiWXoIjQM9g(>chn)X;hu)CUc6Yqr<+ZFdjdsb<0V2UZ(_5rqPm;Qcb_Fa7`9$d=irKwGs@JLw?!knq9 z>GGw&8qfZ$_soX!v=W7m>bG!^c?(X*UW@+ftr?7R?`IYLl$AF_s|afCXczHvdQerv zlOgpC!bb=*boHmh)Tl@EN?~hS9kbTmIqKjcGa}lMvhjP8f;vROIE(R6*1j@Z-?vpbv%)M^*62UUzrlS4~cM<)2kiM9N7{RaQHiP89YtqyxCvvHlwE9BxN6@2!oCcq{3<12XM(E``_OyM7{SKSl7fvp`lV7) zU+U}*deCf310|?WG+#a-EWlq|3#T6~sd`iaqK3#kHFFvE(42Qx8<%Yt{hE_6`9?&% z1`ROy#|>()Hi?_pc0DleIs44Qmrm85__*XXzt)(B%-p5uu^YP`QtS5wj|vVn^cx^t zkq#@|r5x;6fYh1Ud}eeE_szg6Y8m>816Jqy^S2}(@bPY^XC1?g(S5Wgdi3a%w%Ap@ ztoF!lV;6-1f#Neo*esQmJ%!5wSy=ZNAvJfHq_f68xBEHm$;OX++fKsX95}S%e{fZ{ z65~MG)%SD&#)Qt-T8|v*fgNyQTl7**VLm_uP z{B9uvS_D`}W{7F&yj_iK{&phb;6axKRK2R&N4KK5z00|H`#|(8iCKjf_lC0Px4%cY zMuPIDg;KhMuB=r{XwS-X$I7%V1NF1(c_->Ur>%fm zfYkKWD`b)D4!)e_a%+%+e%>>Cohk|n7Aec}17ClGD_IV} zhMn_uimAS_No&A&+AN3MN9v43X-$UZof&O<0Y1xz1HqKX&7`^2?h z4x;z4v9;b5jWxT;7CNt1qhu{czd? zf>~XgKnR?cU4U!E$uW{yz<&#|`nA8}$?XI?XmQD`C@TShrS#xqyd~V>La@r>U&rk& zoVdhYlD;3*g4iACjzU!CZi#5eu6VcOsB>Io2kzFDpX+lsUxM}UlSPqABJ1}|Bw09t zkg^1MYE-l&J@A_2IXuNvMPFLI_`B7Kl`|&k-Hyo#=AQVh%0#Yj4m+VncEf5P>5c%jaD8 z#^mdzIXhFJO9O`bNkY4G<@8$SYrmc5W62DD?{zKC0H4QPm6C8NkDW>wuFNQvMY~2f z<9W1vkgY4s?W#tHK3=!ePZ#5TZGX96D+90%E19Qt6C-E89lcW`w`j}|pKr__n)iwm zj8H`B{m+W26&w(bU4528MoE-R+%q3BpFqp^XKb+8rkh!S3@Kb2>pRfJpf-fJ_4Bx z2~(B#$DtF99S5r-=xfC=M@|i{le1ocRY}A$7fDu9AF(}KHb$RuOEHLFjx26}F#<4f z1t%D*3wO|mj5WN;HzJJUd$d6O?d~UYj@*7w5Lf$WrMO9a9P%3a7~Zn&DrP&gpaZFW z4yL=CE{1n+HCtUGDIPEQR~AA_%+iWh6w0Q9#Lmwulvq9pJf0y2R>}N`rOb4%?P7`t zN~G#c5!$7~>?LNTyEdLuy7)mR(Z%RZPv#++wv+ApFGJj)371Kz{(V zF$@NqPpoqrmE8cop(GD}AX~{16+v3iYnO3an*716#j?^xZE^kt7U*1-_I(-}8c7Uu z59FZWp}^HSNoK#Vm1ipt$udTSOgmjdEffa;>>ORqclJ?rNihYtG=-+FP;hdTdSI?u z>P9TK@pt>+jrRhN7v>Wg@a|)c(v5OAq!r8RqFo+MRL;(07Tew#1Y7PDuG!PkS9b zdUO@6Kvklrlmq5WyKH{~p*&$=@prc>Zc17g?1bQv%Po;V2=J^r>k^|jhlG8a=A`I6zs{s)by@E+sBTg}DSEoQgv@B~s9-LQ* z)%Ss_gbbCKeW5{bZLCtem4h!Or>4%WCcRp*(5d@S*3YBjP)g=rKQs=|p;walAE(J3 zU)$@yFc4!~2KsYl6YrrFWmmu-ps>Jw=agy3&ZX?~adiEV z*t7L-M7Kh7XVF#YW@Le`FE#X*^N zdyqmyftA3aIe7G0tzet;lwv4gSc0Q{XM*3xBbNzKRti=qok(6&*cG?FcVmc6qow=hRrJXQqgQg?2O zcj=86^Zb?xJnsig;Ld|{K=JCR8v~v^vuU$1)%42RG;gzWk6w`bd{}im@`;hoNnoc! zGr_hI6IDt;9)wZ${^!0i9ax;i!aKCe$*AqTS!R|*@nY19CGZ@!-b&ApD<~m8-BPBQ zM4l@mdx)cD5kKO5&<>578eh%|j+Y~9oEaMf%KIzK&R)>EM6;Mz=gF~|AT zm$^gQq0cailxyDf32n+Xa+6bWUW8!46(W03uoETEqj_s+B7o%|yvM0VzJ((6xS$`g zDvh8J;TI0YdFi$1d{K+lAzi+KjU#+fykcH8UA}~E5SLH~bj;zhmV39C+w6c%H9Ccl z{Oz3tb4WOo@E9U@dPtj7HQ-Qv(QA{>FUacA7ek}z276#}LrcnlO6!JaJhqbA89A6e z{!-lSY=K-I=B7`u(=K(FL*?dm7`Mk5#Wyusw=66yP*dZs6YjaZi98A{NVMBH@Ag(RU|sKpxl>>G+Ll zBIKignG+YAy6|!)Qm#3>Y`3*X)8ZQ>u**mRi#ud3_|+v44+D}98L zn#;T<+$Az=n{(Az-mB*+D(^%1ke9u3bHW`OL)@r&jART^n`R5lw(GaVDv?}%rc-u! z3FC{h5kFJQr(qNsE&{d7oA3Tz8UJL;RZfSv0C6m=@|sux9D)JWW-zGrc3lS_HS zyXn={1&h?t+qCoWeHX~WIV*j1-?fEbd2m?q<9he=xUcM&nqg2DIK8k1a9Ev>lKCD6xm-IMD&(oR{sQ?Q?@;Y(4at-q; z?=0uWi7lJuAsk`@-an_CyCZ8JHYlxc6ktu#(o<&rd&soC=1zz#?1jjKib~%8nD{)} ziH@apSHu3<5MNxNzZsTuhh1`}uP2-`D< zTwLFU^8%w?TOyEoEI&=$p^d<`fi=Rm&7r_BKYF0Z2>l*AQw%ig`yc{$6M=P2&Q{h% z&?oh6_Pv%jmkS>iem0>W>|4C=YYto>rmW16cnxEF1Ccc$Fr(Vz1IWowfBkhw%CfG7 z#p@9O9;EuYp4f_3bdLh&K;R*CL9LjMHTo8xD$WM6Y!I*-@1&aecYk0u$^rrd?&zmD%mdf@%R`eUi`O zqHXg>yZRUNeH6BI1oQ*fmuI{NjZ8EvrWe;hBuQ%aC8l|{og(4QQLg#|=V`!s`LNtd zrG1h7PB1FKVx}RR;tmM;QI~wUnD1ztW7E0k|VzMqw@vSzN#?z#Sy{+9T<&K=sm5nyXv45Vy* z%EOVb3~fPL0`HnIk^_jD8~u&A0j#J0&M&NkDWeDy-{+PCY@&5^2HmG=@5b{*`*ey5 zB3^EJcn0_+a2tP(ZW*PHP5>7F(oSQ{vRM?2eZI3Ev*8PPb<)Y6Ku49KQcqaIH8Y#- z`JvJjh5Tz%TVsn3N_-ma@3%l>L-+6qU^GPHu|rF}0DyI^4TCbgxV`2DOq5{d5Wb?& z3Kd(PdcL$$r`>PXT(~}VU+p&blnMs#88T1d5;K`CE+F$Xa!IQt4>5zT9mri&Vpeb7 z=2m&y#@`I!_7w0QF=pst;ehAef5mkF+DluJrXWze0Sk9RV)!or`RP0(4;t|R0^b0V zL6W>@))*9b~^Gc3b9kqImfIlyVg3S_Sp zMOpnIVK?rZH$;PCq$XsL_eV=kX(+9cB2KR}esy;G&>kYooP>b`2p*Fm6@Kojd@*Qw zvHQ_LhPgD}7$^fIj1BdM6A)W6F}y)6SBRnGLY+4BPoqiX(!zIcZP0HK&U+FCc;W5} zLl<0`%q~fRs(Ap49qzHUj%P)Q-@xtyJ|xVlh5>MGu(7=ctr8OU7T0no+*}v|sP_{e z`hEd&>yR(+pOf09`n|-1&u5hCw|5N+4#!4UtAJEn2&n6xA@1?eMn7sNN!>uQO^U5N zH$NW@JXG%6s??f($D8|_J+HmKH~!PTPP-HW&jIf4V>}v`GT^isO{UEu$T?y&aT(TM zWK?Y6ux~$qs!|K5GI?_HKJfE&OTC+(o)#`W#bz)DCi2gn{Q0umhSSyapF?%%z5h+m z%|D;r^XjYAKS#pC4<70bmRXwXrOKZL^d@Ko`14DR;Rf&gM+F8qfeEU;_Ui2`DXgoj z{}f4280E#D_Q7BWkA2<%HGE|kQ6N8n;W5B3${rhx|qqSnA z1&6X+S<8#fE8XK|w!F!bJ6KjKma+s|S0ZaX zh*p9D(`Pkxbyt+sfCDuikPK#!T2~QGP0c=_YBtQ|r9A!qhHF=dA?vE@W)IxkS#u3W zzNYiXk564)UFZ5&rhCs5n}H^N5ttIta@4iOi7McgIsS1Q2(~!@_Nz|Wfj$&_B&)T= z2^3miKzu@jmXnKtjQJI(VAkovs@|2=f)xsylk94IJ%Rp+#sS6Jqt9oiPeajXdfb#W z5u+x1libm9Kk53DaMzzz8i$2cud!5#R46Iwl0gu#d2YnU0Apgupoz6xIEK&S*^6ry z^#)*zT;18E$22iXDSbd_%Lw&nT}SnERe1W@V}qfc)+au<6r1)`1J?F@d$4 zPN{`7*#-ayrX9$ZeKk&o%<1fy$-;ctXOLu}aWUrZ4H}>X4 zWl|EXYp|UVgZ+7^{l|ebC>FG!SX_NnAPfc+E$6|Q$CH4^b^djHv1w)2DGFHL=n}K) zru7mawEqA%3fiS^vHEN7_59r`W^zU5PXpPJON5XOw|**|BTdw(DAofe<42F=5j`lD z@afZ&@D7kEjI4s=qxZS_&Xkts{<;3btACB|e)bUvbatSSM0=2ZmnT`xu}7E**q2EV zUg9Be=O4h4m$TxwpYHPQCrqJqNDV&y+Ka7#PF)(dx@_uu2euC~3FeSpp7 z2&1N7PgL5v-JIq3(utZLbw}uejeH5XO#8|#m6CuDltm$4vV=gAP0|xUr#N zIeA>+*?1TWSj2-Ap*q15Xwq5xhmGxjF2W)LXn=@8qM4_k!hoo8DuyLW%ZR1<29ZD+ z+6juOi>&r2O_Pc!cmLnNe|K#FZ}22wnPkKYmHg%$A(ulz2VCCXW#AAM1{B(7z;r#g zz=0MtmD*Q&Iy%c&^;uU8>+!;k{Ol6#O=TUiv86jPh5@-5_g7n6C#dc5dHv&bfL@*g z8~bPAafOIVfQ1>bIwumkxfr3@54udsRe`@OIhw7;?-xE_U#LG4f@m93r=O2ewjqUr zMxwa~e@^_u;m!W+~Aa@oO6-5j&O@ z@i7uKsj(tSmweYOSXEMZUnLiSpJwy|w>O#0JeQB_tOl%sfkzD>+302{T)y011A_te z?Y)p1A&39Q)B8Qmf3~Xd=AGAye+TieNYJX1$N=P}9}g3!A`J}mZ~gfCzCP@aA1)7xhsu&}-KLjL~iLf6m(T`_E1b z>(gr2hXKmuj)uk$5KJtevi^{`CxL=1_t^cHqTn)cUOe61&gvJ$rkA57%&P9pqc+@~ zK#SD@<$L$;p#iA7cKotlEU3lxfU9G3(~mW6VO1Bt&082XoB;xUDHOD3xfN9Z+0j>S zLB+m1MNO5ITsyuXHmp%AhjuO|XF0k!1WRY^n#}48MCf_M;kH=4vTVOP4HcsQ9V~OD zJBlYHHV`FpK?fZ7NlhPOed3Dw70Lf-HQjwH>hD#<`W@T;%&xgJOy8};W=QYm90z;t zAlI524^!;`xci05-d{)i#NZ8GAhd%fGD{X*;pDyc5lqbtt~_1&PZR0sKd1TgWzRJU zthkVbl~{GWTkLlcr_afsE<#t8D>a6nVmiBT1aBWa)l-;zh}vuATKo1PvQ^t|oCzWf zXka)FtS1;bBp@2kqL2T_38ki$Jy*{q)j1cRkq@~Amgu9%n7EVXv6G~qQ(qS!DI^Ac z+mqujq!};Ct|u|rLVkGaIY^}OH8s*Rz1b1KhHi3LQ29zZaurfgz4xE<-dtS#J$Pxl ziQyRes5n+Q?zhc_bH77hOywN({QUWo9m%CJ!05Q0DfV!#A0?slt2fxEy2WHroX}|? zPst3J{WF|l866#+l6V&vZ3iyJ%WdtvTG=-pT^jRJkU`mp=7;hIqM5E=9$vY6wezUY zpG^7ZF2bsw{(2*j$LZFO^s{*1UZZH;FQBFO>z zvITbh5->R*-hO0pa((vpc1c0?S@4rKTRJImx$H3rz2e_Cwp-dMX<|?)^rmJp6W++# zfx&zlYZ}YQ8QjX-(4PYMN6-CjKZ%sQ44g_Dn!vY(b@KN1zBqatQu_7i)IA8^p^-M0 z_u;+t%|7-|!Gx0&?^)EM${S-*na(YB!$AZk^s}Kw{ z@nZ=FH}B%1<&-IiOoDlN!?#76l2GFK>MmpYrOQ7O`O7^wH|Fo9#}6EL67QESu{N1U zZryEf7JBZSbw(ke%Wgzb#--z>Kc2M4u*EiC4dg!KlT_>ckSFOG?PPm%dVpQz+!(yw zZX}ZD*kn?~yys@%@7Ov!dTleFpfqq{cj zDo1O$-!~Y|4wX4=!?wN$G5rWwci@>BNiizQcvL#9Tia=iOq%&^*dBz zR3>DpPHB4aq7jGvlnn&v;}hV|`cNM%LR;;kLy2D4@9zCxAJ+JAZ+Cv3UX&<1YH>c* z^TYV*2S;%G%Z|RVI4^z0&1fD-v}|m`W4R!@z*O{YTnVpT&buA`oR%W$lr%3o76D1( zR*t9^h=|r~ZYTO~BHU*sJ#jtTUn>WKoG3Yh@!wQhb|}vwcTqKP{kbpla!yiJ_`1)& ztx%*)-|CO%OdFhd2r0Po-sZ-5LBRU1!s)Zrg?B>QFe;I^!Kj~Gt3uJHnfamoqS6wv z6moIm?W4tytC+gxPbD3pb&qUF4+>|`Qjt<*t-Ft!n23i%o`t+Qo2rY<-I^)zvCXuP zX;*jNbXlgzyL?iXb$~OKR5xA?EPY&sD`-#6F!{TQo(eM$mF_cxcM_T>;ki4B&Dq}7 zlf(+7OZrx=F=5lYnW=PD!hAOHzEa@NyHJ1XQF@)2gvsfEBq0^jX=q|O-Z1Ew&!P0m zvs58-Ke7^m-=17AfOxzT9n9>jrJpM|USAg}be9;o==ovIZ!^w{Y&57dDth2-1 zg;1U_WU)AD?0RXF1)*R5wC{%m`;7Cs?OtE1`X+G5AHLGQ+dV zG)t>|3!h3_kRCH7;&H0?kit}%QiusTL^GX%VJ;7RaPKKM46!toAi4f-%!b3mwP#!(_{;WnwNhF6H+(LO|cCzX_GxcCA)OuRch@)}D zFEnjwyNSC^=G&leQH>8AKQmz1y#Zh*KJVt*>ch4WRx&dnbeY4AU>nYvXXIg5xr$9z zhw-(l3B%Co`hgOQ2W^(aQaP4xfmhq(D7@H4_B0`%=fB-RrcqS;y0|RdG1#oqyH9V` z=00UJtfZi6=H;#6$J3UA;a_UUYvs6gcXrO^AcEdpZ7z#GPVX5wIDZ1(eE!^a@Hq7Y zTVH@8Y$dZ0!vH-ZFE)9h^UNwMA*_OR-@4a_D`m=SK7_z}y|_1YlrYzkJBT%?C>$q} zPbU~!1$W=Rj!`gaFY+Y>b~=E0(S_)fV^_?DmMO96fH!8ff3(XqB&**7>nOyF=g}gv%_HD%AhA`}kaA|a$ajx8dz$k;eTmaVi z32DmW-Gu;DiO$Q@&RsKoSEG?AnP%80rK?R=)O^=hciZp_O8Wf5A6)kL1uu5lM|}kF zC4mJwhN?c~*?EK1`IJnG6#HfmQE*|5jPwD8KN%%03V%TsEpK|rGG%S6K3~Xyvf4Xy zSkj?1i`m85_WuF(f6lA!WK*rtQ2`Tp`m9d2>e;{Wot?e*tv~n<{eH$G8G-XNcJ?m3 zNoKW~gWS?_lwNgWj(p~yk@%+T>Jyua&n$$#E`G@W942Y|hULt*O)e;SnDVDm|NFC7 z zFkIW=13E&!0N#m|!s>-#Rn~yN)V^`!Mh=kfy$DdUCMZ0EG3`Jux&nAoZ>1g93sk=7 zDgGd@{0Xz2g8!ymzI|nJLl2)eg5*HvRNeuGDKF3?-T~alOSw@oF$t`$cz4jQp3aU1 zMAkEbOm+F=_7s``;U3vLuFJe0uekHSn zu~Vl+x>Mv5RD;*=1I=t}TafikVO8MONq-*h|6=dW!*X7~_TkKA9zqnMXi#a^AR#o1 zD9vb|X--9Kvye1TLfuJ|W(~@e#=9t*Cz_;*qNM0Oug_h3+t2eH-{bfG^ZxPvp8Gg< zd+)kG)AhNob**)-bDit>3DmI*=t{@kWS1`aSa7|gV3naC2u`a$Te{=>odY;{Y&>h$ zjQ4<}0zlK!Bd=g*9tA~2ZbeZbIgQ-vH?}GHKR@#+Dk%6sTxeAYDiF-w0AQPp4t7+= zYm2aMKN-o%#YIryc4Q>|fi@Qt36AY=^2{rvt~&NqtF93>=Ky-_k79{~0H-%)HaUm3 zx3>o+KD}OvO4-1w^zY7-7;MZW2D?d6@PM^=%>&8h3=E%R0cua^Gf(fo$;Ja>&c#k- z2910RxXf=LmjpoDrr=Ddf?#bB4^D_VEn@GpMJcF(Ak~C97W9prS&ce;Ve@VElbj& zGl;_uWG0nTlLIG=jH1s@{O;^f$tlu|3A0yHUniPo+f?7)jl}%4q{ZX~EcxAdIf}Yb zx!7%lc&#hFiseajn#CP?dHJB^yMMi%?hcT?b6|~_<;+}J>Ye?VbmHWAGR!L01@Car zc=hU4(AUr%qaby0Nk~AcRbrJ1dS`FksnEXyiWu(r&b9n*lq4pnT&z5^0e}@GQZb1)|P^v$jtxXTo;mSp3=nxX#GKlw89LR59w`hm>;Cs-LB1Hd&8!SM3L9cq?Ht=5-Tm6{`^w(wdx! z>>MwWTYivyD##7v!qfzdsU3C5Ur$fwZ*EL3k?40w7j)EHC5bgz-}M}>+e;$|x(PCk zo7KmW0SQ`JX%=?sQkr#LWuGug^!zbtJrcsf(fzB;^-X=pFy_2UoOz#44%j2Ky-pq; z9_o3U$bCjyb4;!up6Di%o*$)o=5Hd;5i`{DZH=q*7Nko+L^xqUzRTo#Vrcs+>c6NT zUK{`AL;lwt+WYIIgYG91Tqil$^OHU0G-AApvVb48<`ylF1HM@{DV%uVW;_>XYdN)Q z9^=e@%iY|ZM_s>K2B%v;AaTPx@2$nDU03;EY2p^s-;(7_r^`Hi_vLvF+U8oTJZMP^je1uVg>eYv1bnh)qTI`1H!0Tb!jq(_)r; zz^ZxG{O#Ozcx5q7-U#eZv&((kg$t}B%YNsc7nBZc^kXDoSg zajkQZK$_}E{VAb!Ri8anUAQGxS0=U`JmCfdE$8}oaN~C>&ZR#PLm)zPcpgwbtJ_00 zrrSvpr+K#a?}ru^*dd}j*z2~;Vu;N#!(2-BZa}bpuN&@BuI_J5MDDhaMh^;B9>3El zlcUuw7EU{G<%Ym8)K>nP16U%b`qozaUOOe^>l|ZrA9CE`xehu zoNC2q)_1MJ_O)ALL=eXgT?uBVYMgw@X6~hoQ$KrzZDslW}lSOq3!)Kqs_v zPDG^Ir>fk|-#y#Wl4p;5+N#h%qqz@^EegEOh3l5wekWSzE?S^cT3+nm)*b8{8oDLR zIj450_r$FqnFS7dj(+bR=^ebkX;a-1T;>b&q?KcGT_ANJ?5RR$LOt)Cn|HVEDpvZM zyv*pja2wP5?GctbZ!P6ckK8t96=`)mUlHx6x!{dw)R=wOy*A%Fp?sQ24TvdqW#Z;u z-8Sajw{wi1XLc73#Xs+*H2~8|E;_?EL6c_}Z$UwjZ4kdA%SV+OI`D({aUt9adn3%Bmx%1aql?$r(H`>tF+IqIQZ1wjfNs6Sp+yhjyg2`); z^PkxkKTWYKGPUw-k7IaJx2a%@`Rnuhy8Zm`SWPc^ds{qKQ)t&CP4B>xdC9ctyApU= zq7UBSH8s>!yJAG$?3SrX)LQvsp|r}XdYXIRlhwC$6)|)76kCv(H^ZE#CFYT42eT;~ z!f*ew98Hcl0-5~18n~w)=cf|ndO9TaVQL&K6h1%0Q9yw)aqc;uAeV`JGBx$Rl)HoL ze9bEED1_p!_f}c_Oe?d-6$)groy(p2JMIkpW3uWh;AgA)F#(%Z0Iev}LnGMfHu^`_U`LR^V9*F8=*2 z*KXYO#k^9Ym$0;-R0`qDF&~YesP2}^H|@A49I2t4%BHksmfR&%4xQd1SPJ_B7{sTxHkzdZJkkZ;^gwJ|t^d(q#6A3$HZ!Dx1v z!1qU+OX5w|s+M*Vt5;5E(Y-)Vv#;(?PmfNWEh@G{NrG|N8B;TO1?xk$>eRoGB-sb^ zv4I#145;Fj8v(-$t?(|o4Wx4)A0SK2dg@{xKEdIe@_(QGA_lARabo9>`gH9b+Sw*9 z_qv~!sHqU&E!)Ra+{69PQD_!@wk5H!#vu;Ym5&GVT%b3Ku+5%+rmL z8L>X>$2zvX!d9Y5y+S)nn^s}COXA(!9-}ZLvLiN6?lVlSd*y6D_U;u1Y$|zj`~oCh zV^^%_d%N_u{Z@XBN?|d%E%qBXDla=>#2LFg?9UX{R{{(faf!?pe$l=9R`T9cTVddy z@l8F}wa5fHF^Z$imP3YF=WO-@T(Rw~SSzY+; zUD?*+V*1#~w9<(=-SIBr5h1s`72-;}2QT|p8Grc6Zf$p=K&YR&P#8DT_|G{$hmUcI zZ7TJ4>1kvZ-N6T&?<``PI`mCJ_doxY)_;;3SC*;3blt1Meb=%1 zX00)IKJ)PYuI#zXjRdxSJlkZe>*%I`V7Iks!`kDgymMYAlkHw>YBP0xlXsjycJ-=F zJl(;j3-pvdyl3}pR4Fy<>R#h_-TR8qk09AV5x60zQe&W;;m*rr)dCH8f z6c5GC%mocre7rW2yesBmfLmNf$+@;I?dF!*$fk-t@A&jk0qfO2ELWy5A9)#jPO&CO z{rCih9!@T$mPM-c(_SZ~UVlGY-QOn3m}pZEy7zJ2M&mNCs%V4v^2Ft_U{BP`0qA zZr5xVt4;H327E{wQO7ic^VL4(u}kSu%BiOZ3j>-i(t3i*9g}XPob+~}4qrZXKeBH) z28jXC+SN~+VbZaB4xYO)IV&8CZR+E-YU(XhzxjWI8*wU~=h(Dq52=B#OEtTt=lFK7 zJ?c^B7^P309HX34{ZUdxM1ydqz-kI4bXg0q4JyIm$RaGl+xyquCQOs@(V^>n`F|XE za*@ngBH(LLwOJDGHYX2HvC0?}0F0psJa~6vd_O9Jb{ChIi%&EE=P0?+Rt>F&>#GFC z2jK3zsDvU62?JGi5Tt0ON%qFdtw)dP$m>40_~Wp;4vp(w;nFT4BVbBO(LnT7IX%*E z^ZVwlTN3)G;-GslOM;mB$&c5U#WGtsIfF*p+l_}J5bJ@6=8ur~x7|hYP$QW?Z>X+I zqcsci!=0YxVUnG{pzR@IBsJ1oS9QA+EFtlME#Z!bPC77r0qX2?C!P&~5051ZkE#rZ{=gL(A3=!_jN`E=p}(rGEoAZ!WboI(3<6UgK8ez zkQt;2n)iFJUOFDeF#FVTO3~bgtDb+`bkj)mk_0I*o zQdYQXf)Oj4biO{#V!1aXU}pTi?4Q_Z_NvX;h*}vI8P6JMPlgzJRTQ=`oZ|ymmOci ze0+{ITfKYuq4dPm_c2?A%VM4lk&?Bf4hEpX_?zsk2=L}l6Ix0x2FU)?n++Kh^1=AC z+}8gD>U_K2dD|!Hcn!y@WSNN@}AL7twW(W|R@k$d45Y5o& zwmV-Ae!h-9Kolhdg9tLwmO5(IGyjBz5I+lnbTRoXDzf8Ppd(`?VFW3XCq(-JdhK65 zh6p7MbBoju$x`@5aZ2R$D#x@~5l!a8bJ z&&^59=&HDjC(yYPA?5Ul$jA@kpHP8J3EM_)0EyYLZ&6M9>0{}?Y=f>0A;@+9LJ64< zSdAT_F$s4uK6@;Hlx;owSdK7)ASg7~)|PRZa}bU#0us^tV+2z{U(F?g>>EsSZXY#T z-B6g?`3qy^ENNKy#PaR^i}Ct7)hp8GF{rjB)BApvu5m62?IuSs`AJ~qB%FF=Z*Sj7 zOYY2C(?uCNuakse+g+Rdz_7F$l@OmssS8J{%#27E8-jX?l>yKtKGCx%wg4!!a=9bQ z3ADEj%_HZOyIOUe#=J<3f()6Mhhnt{L7AeDd9#X8#M9nh>%v^vf}-d^175PzBpL1` zu&rR+WWY|{*yU#qfSSlO7d_j8MRX|L@`z-IpGJGs9W`HKQ6NBS^4bZ7X@URVmJYq{ zIv2D>EsUE~wngUeS7qCkmhP2ma6ru`5H2z_5U+nx6p9mu&5-^1Q z&KN*uigTZiq;qc&(W$R|eEel=7aB0Y2rV0awfum>;lqA1z@9JC+=lE8C{D9iiU64>|DAk@Ho#`jkQP8r&E;8Ty#KhqyjH?jd8Jd$y<5jgX9cJ7Fr6 zfDaF<4qY0uCl_RVyi!VSTHRpXQ;t2K-d&d(v&qZ)zkbDjwd*=SoqDLut;CDjU6uPG zmysO(U_SBtSBlGkL4Lsd*g78Foo@zMYvtdsZWP|OY*-MHNla(vsJmnttkuEOqaPq2(kf?b0uL zR(C)OQUqs80|n3?0&{$mZie;k3m0ZuporyE2LpXvrUjW|R>-cp@q@lRe+^4!f zlR%JJy(y(-G7A-`$5U@!!RmSj(!OOu^l}XJ=Vb9803kVLWo5TZQl=`>uN#G?Zlo4= z5YkU2&>l0=ktK{(zpK;=X>18yg!Jfs?J$s$E*bD=senu8x}Vk&d$?Kh0r|X<)nLEasdr2iZDOy44T(~ zHIEJP-ccw?;Ujl+c7%jo^Hwc=Cj#~s!K4eq{G@j~_Y&_IJlP^hI&;7Y^+(N%77C0E zJs5jb>H3<#w6yo+7_ZJfFX_07{ONQrB=S}$P@lN<|a`9Zr2HyER9hp829u@D}l zC6Xw|{+?t)U^x#l%vD6)t2t0@6dpj;<(a(h>eUtXYqGJ?529K{0ekF!v!te`G&e|` zgt*_KP*Ctw=uSSqQjd(h2d?JZ+<=P=mRInZyzKvrfYtK#8#x7}N^*PhYX+nsb1QY8 z@(*4Je2IVFPO=8r`+Q=JcA%C*S0~e!A0>u?c=HM{s~vFg_@|cKYt;xD?T>A3*F98& zg}GYnby&wqpO77X1qB6)*fT`U-^f8Ql>hSL^^3L}>I~PI&G=b|)*$m-Jx}(v6~vh| zKT1d;z)I|rEnr9mJYs2!w%;A*R>J@tMR$n|eL(W6>4Eyr!5(lnZdyQ1AO+$OZXNzw#xQIt=5IH~ z>SiXkjAkS1W50XjM&SX4bftN|s||l_fAh;ywZ;)I$M1Pl{kBfx5_x^G z_}Z(8l8t!`V*8SrR4GM&I>`^-p_x@gR6>2OP#CE<(OqTJDSq)?x|UkZCHp1| zo>uIPbMK@4Cfjt!yvc}3?4^-KX*Y^4gWa4yzq^g-01OM0>Xl;LHh)1zX*Ar$b<2NGAJ8|G zt!Ihsxh`UFH52uCh^vV7DN6`RJ(WU)qyvQf1*Vg3=6n&;G9To_LaU1~3m5tjDnhw5XL%+)5fP-MNin5Z&;K7Lj1c ztvje_jn|nk&*%JgNVdGR)ZyDU&BviOPH3i5+FO^}fz)CkSm>;>9#3LLp)@Z=3X)id zwAr_xeAZJT1ML)6yKHDXLZ2@4NB9V0jO?&Hryfld_9b&}*)sODKZRz~SCW!uAI7Gr zdZ*;=(selv)nCJ_Hrhv9cFeTU$PsQJJAhnrRZ1+~bbf9Daoz^s-*+wuiMa?lR(wkC z;E9QjBVGM){h%9)Wl`=J6&bIqyUt2mH>cA1efTg+{CUuR#<#Ii&#m>`oGADN#N#Ax z7KvDF(Mjo!z>XbFJr(b~(lxi|;CaZl_EO zR}PNAoDDr+5z9z{w9g;6QtkqMP%_MSvy<<#4(~<$QK#R(*mIxET)D?tR(;kG+qy5H zpB*$p`Z;urqvj@9AcvHA0b%(jc9+~NlOKCWt+C>#-dL7y~^_88~6bqvjZexYJ@$ z04jY$%>uo9*@hA)Z1wXm>AfE_&T^1je24PtUp$KIL)tgQHp4de>pl?!|C<``q0@;P zkpRRQA)@sVRk*d&KyQ>FU!gGRt+oHnOP3^;C*xd;70@9{s%i z_zvl!R$Ci_Y*RVl(y{>LbEq7!HOT>co{py`VL?gPvTM$D(M=b2e0es5wr>PPjN>%0 zH?foWK{ur`slJBs{`UbC3?MaqAdC#D*0lC`O`Bm{8o+PrVbV8Snu}{MNGZz!9N2po zvvK#GaEHT6Y(7~$g=wqw9dCyLeITw4a`88v>T z?{$?Vx5}0EA#?g}sOWN`)b2e2PL?lm;SvxicZ^F&$as3MMWZUG=Nc8>sLKkiCjA6~ z#gGZ!`4s!z#m--ViA=rQ5GIJUd_zbSEOZx|PAk7SzZXb45Vrk;(@(;$K=YOMDzhK; z`Wc#XE;S1&2S0M$o9^4)Sf$}Rv+84w)r2&q8RsO?gYfGOv8*x2N<~Xoxjj|fj`YV zL;pu_3TFKKaUgXxc~qR3Q9`Xh^Msqlit1Evx0?;_TBp$ zj*d{9p?-B6CUbjsHx)2O>Sazc4K@<(sV3{QaAQp4yC7U$^z7UW`nU6C{{8zZkF^aWfL}FJn=uo&UE}NsvcZFG z_r<*cU1>NOBogi+N4+j$$3Vb}MoD=kf@yU#3WD*fT>du>a3<*q1X0ci&L@v2XOVoO zfLS}!Hr(;^J4Lv$;dafKA|?<;Ar;SsX_zaALE~eGp7!Eh47iEgr6FSM~<(5U!Za>?#9GaRie4%cPc-6fGEsCJ)L=Y0m8zM+IMaL1Rq$^w;>!736) z)h}saMKT9OP@ou5EZaC6H7%BGgk`$JSd$<_<++ha&5gKiq>>x(XFGvT;TS5S`WNUL z$0VE>xxy*EuN+wpB3^UR(w1ng(bgtoa6^;hp&0d7@B!-Pl^RiRYt+0JQd;KCm}+R? zDP-RnY}Ogtj7EVXokQ?pl(sT<&I?DboAI9n4P&8{OK>YXRZu`7f=6_8){IfFSKaopE#CUu{-esyK(=B5X?WO>16)gaT5&!2@!5CEJ4&Ga)gQ6HR} zA)GSgTsV(BI(!>m`Rpy1sn>Gh4QPfom%@}36>mer{V-~(H?r^^_F)wjNo;Y;63vf7Z(ek8Fv+@z8%>Z zS;4?ju6+_7Lqy_-CsVq}%mN6@0L--qWZK;G`aGcJOQY1>J9 zXr;M=aO*JFVKU)i_tgWlFC?U?IhO#zA4?gQg6kN9aw2ImG$K}`XEeS1XJvI6q#d*;$I z55`9R(Xu9yAMpG)t2*>dn7Lyo<62jr4S9|<5tcc3%CTS2^qCdHn0>o#QRLsrfUM2~ z@rDGO1zk^FUj70)X(>W%gM&DoMqy^ud8i+9B+csvXQv==`5(VMnle?6PyN!%{@uqq zdsGz>Dl@Po5-EVt3!89b!CG;Wab_Raqwbk;juN$2Hz)uIqXC5^{>kemY91=V!h0_* zUM~Re<0d*R%n@Pi1&j-%ym>#2{bLhiP2pW#goKTd}oU?x!gC`(T^I6?bt&_2SC)-1s%PEfVMgP*`?lNxlq!8*;M{wPc2os)kx`^ zu70%bPM}%ftyM_@+NI(pAN)iW=|#SvDAjJX7B=fJ4VFE7alxl|hb|493ySN`WF+Fy z{igZ}9bl|eC=Jf)<}bax+ZfYsI}tPO2y-aEjoWus$%H37ih2OnYrJLW%a@x8`c5)H zUO_;I)TkxzRkV%JU&MC6|F7aLDpS0{Z_5_TUfx4tBz}gnH@-HKMzMKtYXXN-Q&Nl} z?LcFK7HDZ}JLO4x9op?avIfoC0O>waO+D4NkJubeNy#kvp-a!qQ62xOe@Dp1BfPH^ zUw-zFwk&Xs4O?Q?{GDQ&N~s!FzSux5iW0%Q(<=yq6$Wi$Y9B#FH$*SR{)U7v!lG+3 z=)1OT)v8s!499iJArkKJ2RqH>)q)(;dP$^ZJCF*pM>h{Y*Wayo2^Z>VM#jcO#PQ48 zjJP&#Jga>CSEgFcXo&6b;0@ET4&P;qs4-Q~8-4eF7;nz{^{F$^-j46=NCGtY!{c*_33*o^fSVsXN9u5v@`57h&PL}0s28p5~pq^!OG ztZEiIyGdqig29@s&2@#kiPT4zOrZL`-~2|`JO#LrQQ*1zhJ^6KoQUY3Ld0@#-%U2w&(<&I zOc8Mn?4N7Cnk3?8`7+B)yWCakh#lGGmCV1tW z`tQTUiV&R=sBswd=MI&tn=O@90yP@*^;K=IyUH?)>ps1WdWxlG0?9V)e~;`6*J|4g z%64)OJUSe^k*N7S?*ay*49~IAu}W3=_=^kZ#C_(~IQHti?k~8WMNC;`7$$2mYi1yG_x6{ryfBZNlRB8k{1g6NTORkVJ91 zl>~zRtC^!j-2#c!GKm4~#l|_8t{Nn)RRmOPYTXXh=avP~s5l1mcOx}ZTMtZ;^H+ff zf`QjD&PLi^5k0!HWM1r4#u8DasS z2L8%5)*GU})AVJr4Np zgJa*}_5?I$Qh$h?2{d(o#M$3&SFc?okyk0ZOXG1GDc2zT6_GjuUQ8{lBB0J?3^&|w zt?i5oQuP2QB->-}{{5k%aI9&wGlznX)Q1PrP`<+*Y)4%{ZXWSY0J-ijmmRZ4aAlQ8 zq&y;AKhCqLPmA?lBUX~es7sP7ac?CeME*GOiqi{MZ<9kVMz!Wd{l!{9uPQeENS5p+ z&9N;#33`2zxs|$r-^G;`NEB<{JeFE{EbaRM@%@S77z8YB<~lG(#up)_MC|;ly6YWR zD^>=%z~YfXd~w3tObe6|lk6310VB(xuk&ap03u}~=G*<|-d}pqejY@|c8+usb)^f3 z;b7C<;c~cTo@PSrSkR^$H^<^(YVZ2AFO{qF`9I&em8UxfxRe0{>=?r(+5!-mE93o)f?2f>x^=ct)K~Yuf7kA)>N>#AuRP zYN80hg^PzrkqBaDSIM?{i;#v=$RTo~E5^gav4{2oEaoE+$GRP-Nzuj&zyDHdAymVn zeCXnK#o{92Au;`@kFE;Gn_c$lB=h~G^y^eUD(OcAu1ZJiZ`1-Hptk_qFzumSpAx^? z+M`qh*IR}l@&zCpF2d+v2%0QZEMqozp8?a z(g{l^tIQMdE%>GqfF)E5qgfYT);_Plk(%-FeXA+zD%$IYUZEaDl24-a+bEy0Tugv4 zIP9+%ELxhdAaX3)_xeBeCX{WP%XK&IaUQ@g5-GYN$}vU;hVa@7Ys>dsT5ubXwN8kr z#UVB~h94})w5#iC>Z)L&bLsA;RrkeZHBVR{B#`xtLXD1NgY%aAG=neydcpSaR)@Lf z^Bu`m3)bGwm`jMXX=S}9Do6;qBuywHA;gb(Gu=7_$0eZ{u%#R5U#mw@-t*;T^u2@$ zp~{{vRzvOLSO9})wjR>Wr2 zPP;tmIyp|dPV;TpcSIy~Q*M*+6Wz71Jp_(RRcAQ6C-T3!!~4pu*H$-aCrHzWQtkJq z68~X|CVvD!@fBEmMv>L^NBlHLLW?{zlizM^n!@(&3d^ka{s}67n=s)?L&Fc z8eG!84%Soc2)5m6>m zIQ%yQ1kPRUQ!Z{Y^H5;Ay z-S?>WNGY$Rc(2-()RtWk7iJqKrLG7k*5c4 zYks#3(xhh}5KeI@%lVz-^lOd%k%2oM<2cGY!7VTW zEM%fH0mH zZ=jw%TK=xAr;2hXsT>uye{TO3u!ySQF@ktuW92g*oyq+9efI6#UscSI9NT?JIE7P4 zwz_X$<>+Z^tph6Co~vF6m>pQr|K=S*nZ1Oo-aDiFxZB9P8^g#p(@BTj>2wNIVv1J6 zW^01b+5Y3Jc?4t}ha!$C$t-l_jrHFFo)LVu1uixe-~E?#tt?uf%f^^xyNK6lf0|}d z*Qhb)=c6n%5f8^{sv;BCS^9|b zzUR&-`xd zH6r=1yO-hIHteLkKr5b>Is+T}R(_Ch+4Y9{W!-=Banp4vK=X@#1=qZUxBB^KJPE=_DzQyNps;!P;W3?EJHNlv@b&R& zb=XHupSco5D-|s0(BxAGz=0-owuh-RUHATxV_UM|%#q_y5fYaXMexm^;`#5QLiw}# zG)By>jd1gBG2Xk`^osH{YnkIOX0nP~A1xZBf*rZo#WEzFmvVOE44E^e1mE)A!{Y`K zmDCNW2Ygy`7~4e7XbxPv%DksHFIwG!cUX+rneciD_)UC_-Qh>5?<;wODo&`rtQBss z_0t%tpd4&{O*vYaiDtjoiI{r#HfCuM3N{gQ#P+SZb1ye(>7BPB=Gy?-SqYd4Yd;7xd`XnyRr*E!5B$R2Btg;n z6eJe`KE3_&@(L($9*kxrd1bimy}O9);@cfT-xI6AY|HfM2GN$#-iQSuwe#@HS86fx zBXh3;KMx&2{yEbk-fiI;QSM|9YWk~xo>4)#tRz>_l7p%&HBB5v!TG!!3J;|1FKfe`39kuKm!0JnrNeRE~XF~`ckZxWebe2Tzw7o+>AEJ@_ zAFafs4|(ANCRH?Y0|kLb)Zp)ypHV_u8tR?yj;_op#l`DnYQJW#ypJi!L6XX_EUR3asZ(+vF>w7b%MDo>P86RI z-<7cEK7BG55tNp0$So`=C}z>pJ7``95K0+OkhiRYkd2s_qBnl&1*uZt4pK<&Y%xkX zYa2kbC%TCUuUNdP*px|~7Z9`L>ytNc+@MQT{LE~!b&;H%CIDC7t5Q%zdjA>y{5_fj zje;nTTm}U&5`UA>**fjOaTX?SEsyNovd9ve=5~z^b>%GNb-zQ+X|I!x15Zan2sU35 z@!uY?iqAgGT%NT7-CgG0L!jWY>%_`W8!&dKSQ9Aed$oQ%S|Uj3-u zZ6P|>=HEQ9AItv5Y!!d%oV)`T0o#_K<$H1t>a!vJ7c)>ONin+40wb8@bhY;` zQdPzbT{3*GOS=CFCMWBO-)F$zP`TcTtz8*l_E7Y7Z*RY-C{99>Mb`A;4@ThwIrS3G zDa44D>M&A|9#mB6HeBge@TyVVaWfxC(ScYLta!uRO1bpwy}Qr2Dg2Dqne<&=YZ@25 zx{|h6!Pz!3%EWY)dovS9n5zrRul(Wu>1vNYPfYx%;|j2~u#i6#6}@1|!6ma;c5PE% zaWMZ{_?4@_x2mj|(fdqp)6#UmT z!9vaL7aeX z$m2Pdm9FAFe1$OP<$OOiWh857+60c;KDSSG>oQ37tT>~QdorXrFyus`qvl9GvpnA? z57RsK&S#k9E}ZK^hJWuj28fe&5SAHg!A_N)c_6_%fC(206gC*)c`%G{2+Zyf3W91k zJOWkOjD)fX=|b5QC2%|U!J<21+uvM%m`CG<_k>fdO|Xt%r@)4mb}oxIcv>H? zlU`E&&DjBsVuG+-U#AeKm5FItB@zmBLUw~%$sK`DTM2=T)Ub>qt3yW_)-ko6zrL|4 zZozj?#I)MSY)F8txlBdvS`! zGr&mQ87w{0I*RjTvYI)qC7wjE$LOGmehKvq3fs_~^lCd^h`nkr=>=u^qt&rB*O4cKp)1OXSo?c@abkYi9y8X;a1SsS20bNU0P%dY_#> zYPLxEC21!j2{%j?Gs{S4J-IZ8%QfCFji8H*fW|LkQnDhHLB!@rLe|rpzN!bFqG2%H zqPpa}D1|#tpt-7O|2JoNJrUpUp zYEu8!iGF7yHF^%J@)ot&t4U@@6)VxY<^w>bjA_H9kU(r(kXB(^;kYvW zs4eLGL|lj1M^zwoBEJM?#WbBSzt7=}DyK4}{Cc-DeBh{ksq=_D{#r*;M@g+um?CSh zy1r;8Mk2e|60Y#(v%klM^4}R9%~eJJIqcG}XZx-dHD4l1@4nFs_cO6n>TVoMsuq^Z z<6T{Zg$S0Ty8&Z@-AZ@;4P3W(3}2+9xkum;3r2WT7#r6O&Hd9lvdw^RI_m4SA^2lU zx?}^`<{?B6aCcZI(nwcf@KtJlKg9xlm_LH>D*Oty zVXwyl@rbuHV|r_#J6S4O?^tIV2xn4Vzn(xN!^DUAq`iRz7a#Ah*1eX0Y-=s!RSLRj_~*3y@&ew`8f_jPOAzyFjvM3#v|tgl&eilMEGV2VE(qn z2Cvgw9ZvH*kd_m_Q4Mw|*h@z?j%65*;av*`uf%5z?~rhN%HRU@wvZ}avK?OvR%fJM zyo|*>ew=Jb|3IRPALls?VvvzP4C16xjc3GsW-@FXxzjg$4`vMbB z>Iq~(>qXhL3IN7UA(@~sB-0qYH8w4jFK8s6*gLlFr*>3n;Y&PHUo9OLLL5B(kd zCkS5|dg&8pB-@?^UUXjzcz_b5LQ-d6o9F--je11J>cVohFJ}|4brD&wpWx1sM)YGY zfKhi$P*Jely7@AO0E@K)sj@z%Wkp!j=7zzZ0q=>FoC6pe4yZy60kC1ReE=B`8{NHD z@n*AbfCXqu;w5TCdi3`Cn_?4fiIJl%%;aLrOylGmh@@~^xfALmb)kqvC) zC=~SvjseSQ_LPypj_gHi5l0%JUn$X^BE3TH>QC@8O+h?iS_1SRj=%5XV zgxP^!<(Nsu#6|voTYdv6E(2al2dD5xMVGR~0pgkA5<5I^WoNEKuY|*;emoklArZ&_ zaWpCW^P>eZH3_;kdDLTF(QPFJ{)`5JbXR1XG7>)MVaC+B0eNPlGXf$7X2IMq=d(cU z>U&bmN+`!&lLePq!e7Q=l^Y@*u+^85sx#b2pcBI`a6BjA-Fin-E?9=rR7+j196~nF zh*?ao>=iXv#tzLy^9)g2Imyp)D+hvM^;bCpdJ(uGB6v%Z;hfZcF57J>Dx#I3^FGh4 zI(#+ttE{)U;a>O4i`eH*R?1klX2A#zd*VgD-O7MX64kM4BAG;~Bd%=t0$9(?fmCYO z7`W4OG1t?Gj2079J5N`b!_8Q|tQ0D~M|f{9#c=UC&04iV%Q7W4WcI=}#@ANzTY_I> zgST*}Ukq9U@rpv!*(IY+et6LdD<+1j?ewST6ZCTe7*ruIq0MJSb&ta>5dV~ml>XQR z;Q=F2`lLJuc#ilsay*B{j-fIUQ*2PjhjPJz%JTH*KKNnn3Y4C!N)h0aZ>8=|R8 zacZ}YLTE5;R~JeRgMcgq!+Hm!lWJDE?&D+6U84yLg~`j>1^Ep){t=I>swykT{Zqc% z>r&{ASMEma$gn@gEf_va5IA$=m8jG!lVm^IsHO-~ms!#!6fx9MYSv-}D1d%~`W^Md zOKs2cs?da7>+XIuUV2`WI!e*#$SXqs}K)4>j=5Ge0(+nvE&4{8##w&_h#*p-`#>Tmnu8V1P zw2E4va=VXaSO2Wmw{84nSIov^)(p4L zSUa2gwSs7@B;`FK+ISDs(&w(Dqw}hC1@Gaayn(vZEv1#zVwXFV>ADDHr41r>%`&L~ zPeWr94fFG2kwN#+pj-mEMZCHh@t%fYXTK^XW$Jl>d|CoihgZ&tz%sR5=lSg*CR#-h zvH-JHty=P)vVt^C8h^=DF`|o`+tKW_cZ}$2g5{`sZu%@?x(JZJk>LU2-zl%5UVuVL zVnz~@i6WrKz~e$Sq$FwB1NYS?t_v*DAn z2&*oCgjjj%AR|a=XTcVH#U2qDweqN-xgh(Le_s<1`SgDe$rqC?}U!UBnO>J1r-bew(vvIf~96WhE|(;gw{o zI4G=i)8NLH5m5#4PkLy8<4R*pbT(9A@l*kq9HYjIX=3@2)b>2jv3eILo?);&L$UJE zIS_tbCu#bIYBMonJn7yH=ZG3H`+NklBvb_Olzi5zjMr&{Gl_)YpalW&4+0R_a4H;T zZZuwrVQi?wIG)SzSTY3iT^V{AQ)U*Mrx7q{Ga_d}c2=TNbLQ7~EwWzY75S;77>8yR zx;m4r@^CuJ;>h~Yc&NUCywR~m|9q=R_XM+yG=iTKhX4H1E0?b-G2{J>VI;)LVETpi z>j)>ks|k^0ZY8y}7++TCd^QBeNtxo7aN`g-e-q_ps4g@Aj4nV1z;az_)XQRg*C+m5 z7m2_CqF03hv7o_04^E7Si$44dZuV6}W@;HRzFfT4m9#hV#P;PW&qsP8@f?b}0=9?Y z#?K)P>qvdkr*uXIyz|Ec5DvNNdJ&$Ouy1`4kK${g+% zZt}-<+_rzw!>Eh_vuVDf^{K39D;2Uc4ljCnt_xLF&3P4fckaCJZ=u(PxU)VVi*U`Y zZ-1P`SMrMoDu3!nP?A%~CTY)E+bGC&x7?LL}ojg}i4DXz=BRx<^$o{)mYp@)@HL;wZy(prc;W zx~Bf~C3kQgV3fp75iWQ7X{ZUI=)5fNk9W&dbZ}H8^d2@rQ$liuB*->Ydp=G&2}X|D zj4H&p-Y!yqJX~?@$GTI12^s?ET6(AUW+#VgXlST{TUUnMl_^vPfKLhDJz_zbWX8R6 z8eKOlyt*aAoB;QQZU$&TSce{^Jcl3pu+PXwm7;d2(m$giH#jGPea%1?>0~p z8QZCGx{?XMH5d+^iIm~hWn3z^H}2JoAtly%ShXyHqNGCdhli|2uItw)@l-hDlQY#S zZYIGVH2_Vi7}`PV`;9%sw1rTKYC<%3skM9XPOQBaB!EYzK(aqFIGe4+-D!LjAOQ-nAlQ58kQLg?b z3y}vw#fK;0K{@Y0^ej$T^T9H1DZ%Jo7Ylev-a`G&C@AsYCh}pKh|M+yeoAbkDO^BP z#DY|uKkiQ1#Nu0%-6A-Hrp=fg1a@V1p268)PSO?dq-_xf^%X|aKneL)vr;SZQ>j+AOQrbr-!{@tr zvWjxjsca=ZTp|}$S3o%HO?W*F{O0;f)_Y`8A zg&aBM*ifmlitabHz_CV8USwQXUIZALV<;Z9PYP9kicqmLs= zYM;yxZJUWD7s_8(YG&O4K|@>1>UfL7`_vUUWsh7^ zQ+2XyP?>C6;ronm`hb~Zf3#770#c=_R%8}PUm0UP5Y8)czN`=LsOqV^ODb4t4lfr< z)Q+MH^%93Q#ObPu5y*##-f`5Ec6Uoqv%!9`4x|3fXC^RCmW=e&howEl3HU~8OaV}s ze*IW5>@e%mWyH%dir^O+{rpWx5d`KK2jR05KI6!olub($-R)wy9I>=cFv3m__ls72 zTPLG3g_A%ZM2XfNc)1b?7EWCnGJw|NFpJVOqgbrUcP;s-E^w&UHVel79fWoBy3u2Z zelUreM?N`A83m*q&z}z0TRF2OK)gQ)!SUXw;|S!-5ZCmz;Q1{Nms+xQ0{p6N9YLzf zIuYM~m`|ZOK4Vj-IL9vQQA^O&-|#H8%(f?;ue#S1VNuoLQR7s&xDEa;Koox6tw6*}raf^q zWyX3_otu>QKLl@7po!tvxJ?nmm(@Nz_@H+T88VS@lJ)5X4?t}sOGh{^6mFHQ^v41A z9jDXkz?%D|%TN1m|3Ki$dtU~qmN_*ZyJDe5dG@!(&7Hqp5h6^ebZNzP1R9(!_CS0{ zP%awk5onMf>;y0*0i}>r`ZNID8EgB~($XR;lmoU}Kn=mC+oiIeN+eEo2~ki-Y%c^< z^p@+HS+}7mv}^3zJAgI%JGE9dou+2rlB@5obTT3m%1Co8m?EuH|9jwHvYs+iy0g^Z zzWl2}^RvI@LjQktAZLDN0mG5?dgb!v%Tf`~ua>DN>6=q0T90dNPw*o7?QZq_Y(+k~ zrDW_|Sq{9X27sw3{U-ZRaB}L(kgxdyV!AE^BJPXWgbO_iRV=2@y;)3*i;?N${M?_m{>+!HaimLp z$c%R0O68c|PHED6>6yLueV;Mkrs1jcz4NT)aor&UE&5|+-UV@~{3Hn~XZyFum{tBl zEVL|Q#$_an@9~H4vB>VM8rLDgq};cu$;MGrclOI;$xdn0!Dv%9M`v?cdj3G`2f{C4 zV)9u#xonu^63H7-R8XWZ*|_q% z%Bb&NIv4|G0)YV)WPrX6a$OJZr}xx#;3k;lZ*5_y$}zPoJ#7#d@5$&BZs9*_D?ZR@ zl`&9fcWsrL$mk&r48>&B+uwQsTF*?X13~ zuK0lD$s#t{)9<2aQT3XPqgfoQ$H_%c?D_l21I`zF)=LOMRezhv$|UjmHP*soi`I$n ztz-^*{hqg$h=G$K{c>{>duFoVF)L77EY^mK^*#!U&9A0X9^|x4OhtR9UJ&)a{})<= z{~Ijc|M$=MkB!B=^08jDPWA*%(7CJnX;;>1 z{S7{O$$YF8y2~nUb&qwT=TNHCh-@lhy_C6>VP%9U<_n70sfAl*OC*_{&a|FS>$DAxs(gK6K|5x7#^GB~g>`dQp>tt7~C{OK3{L^&l{bLj3&DB1Awx&8N<@)h9f0^pPLW@tmPkhg!1qy1H zIocVZE{W-OD+3D012Or``DWWt8{d#o>XCUYt1j0esV>hfpT}T3y77jS*QN{~wx4xA|B*qZD@h%#`9^zah8*&2f8$E+OPM#1ShT$QX5I9x z>}YQNRx5AfPnq3J4;-B}iy0y-Ewchzb@! zRGL8Oy@eV|s5VeK1PHx|5Fj)OMf$mx`+d*zjB~~~pUxQn{ly_$B_wOz>nd~p=0jGU z?1%Nvpi10T@ZaxM{k5`z%=@+*d!D)}1UF)km;OKhxu5+1>ZSh=kEnia*V15l(G2ns zN3nk;Ykyz%QxgUWwP?W~h}`xH4^ML#uR&k+ez5SuwjzRb|NnlccI~9G2K^I%J zUsq&ye6x}n(9w`h@v-~lqi}X6@av~)1;&||$S?o--6ZGaSzg~bdp~o!7Yxi?CP>>X zoL>o$><#w&iLz+t`q|>7wCwq>bv}yhGugjt|KGoG@9QgbxLB{nTk9Sx(sSzdbMN=3 zdk!PN{O5P>4}&N$aq3+KVHZ)=Rmznb@ZZaTD!AkZQtlt$kOC4s8D9oNZZ^Tc4eY;H z$vMg;laV!GJrU-i`{xHq6=U>D`#p{*L~{mwML=)#{Zjw$J;Xsaw>cO=z)mr14cxL# zpl~vuXo@n1y}c5vt&=_lR)&j9lxt|Bt4{_Zvqwgul;QTbli_n%As9p#eq z|5l2~SKs}y5dN!s{P&Oi|JB}wZVsJnWrH`xL<+d7NCS?nkVs$!BY`aqu0+2V4>*E= zO!00FXu?g1vZxzxV2!M$Abjs3fXE*qa+0nqV!%}@2XG8Ojskmu8lB!YPXN1qw~w-@_o0F0SlmBZz3=;WGfHCR!q-YDOfO9sqpE}x^z|B0;pqZpuFfu zB%hJ?@D@ztVn9*zGS(2*`%e9~rE-LpfoWtK1{02gjcFJR6ttRvA>XEFjRPdVFr^^o zlMq2R33z)Mqyj{P22>Md*I6-77zXLknPm^hB6vIcoDgX0-xU(fFjtrxeH=ko-U0k3 z`C?zNX|LszVApTK?l$75Ob$JTDCi7Vk*hKup%__08l9D@$mk13NI9HGhK8*RAg3!P zH-a2Pa>Dtmr}uXZaeG?S2b|1^NNr`^6{bgW-aAe$WY|^r7~3P>VAyTmLWZyiC!V^A zctolWzaSNvtHng*sJbTofmM}_k4o!haE0Z!y1_czv6Cw5v6L}*7#a3}K3o_SD+r2X z0=ehNP3oanRQN&2(&$`;Xp|A$`sNOq)&#_t&n>7iYWvwUSvJpK(4J-Xwq16yp==v5ypQ#D!e z-RYj0actP;Ztn^ZY)nmT!CN@HYBm8v;o|X6sy_2XD!$Qv8<@-kVO*$&OJ~MkYul~D zjnHSCPMKxkEvpUV)@h*;2`%758{GB=d05}3Wo5^^@TnF~t}pFXGUS8doEbQ6cht8l zl^*bvv=MzMKb}TxL4qpd?!n`-kTxln>=qi+)RGD-`KUl%;`d9*Ib3?}v`$1yH_t+h z*csm0GSY7hJUQOQf8L0F6{e6I@VJ>HrTyb2WH-62I>bgM2ng@e6GILr2Tu@Tm~l?D*vh zx8fR{Jh5|8=P$z9-elhow0Ws%=K(msr{12bLXu;KA(hnIeGo)*7ud#^xjZognl~IN;f?oiLio_s)U(5Snk`O(4wpbs`bON{? zxP^XSjsald26!1G=1Na+YJQa(lauYwhDRj6U!ME}K8*fl!_!KvKcnXE-o#F8BV_l; zxm!~{w?Olo(VaS04YvkO-{6W zB6e+(q>fk~0z&~dL9|^I7%YXp%!TfuV%A@L<~Q` z)TAZ}GzJGHS#+9sx)+F9s*hb(OxaNB3xLCKp?D6Mp=vwTP0oRWLBPTf zgUq=j_g0b9eNw0fIwH3cc$nMED0p_^$K_lnM)?yZ27l`25_!vi9qY$T0e_i5DAOL( z;CYQ853KVn*a`Vfua`$_v^{kQ_fTK-34Pj5Iy}Zw>ysC0@Duf;qq06_uy(hL%$1Eb z$sU&cC8k2qJ0c!?AS?9@SJBCviQ3j1@J5sh4Wg0}g+s{-&gzBbaCSsBM*3y8G@~o= zWH~|p9U><<>6gVh4A*Uja3UrlyCgpb2u1ng{0Z!A&aounJP(ywt^(^~J zVGM#?L>Ooj;GM9y?k?Su~wM&V3;Bkd?Gl!N5YPJ&p*BHx8;i|c#us&M( zcz6j)iD(23H5@I||A6P2;0_{!oEOS^&ynetcUV`cz8;Y&g)kF>$XlvB^3F`!jqU`W z_BGan1l~4*tT>h8VJu@7HVJ>qN^$`nBYkVmw%Fu|BCA*y#i|e!Q{r)de2ij^$HNM^ zEM|++Fo{*5=gO#vA*=n>9 z5AbZ!;EqFIgou9K?Xvm>{Z0l6TrzHSD2QDv*FH zBq^c)%lnoyuo_EJr8-eDNqr2ma zuIsgLt%IDbxSy+NvC5(`(6EVWDpM^n15mI6Ax1a}J|sjZoVjJU1bdess#?KAI1I-& z`LnW+jBKW<9I{vvqEG(Ufh6p3lHi8$0LKcex$`PJ@bgd`EV04zYiAAajukVr}L z^6c;YEJsC{p>4%bX@2TD~W!%mZ%kHTI6ZQBlos4AA50;b#c1XE`=tkvqhzQ;F*sIuK!fs(VK zJDZf-Q*W=P-^vE%|J`CdY>C%^i1|~@7CQ6oh`rIRx!;@wtNwfy7<_$)rx?_>HpChT z&PuR^$0{M7%FLJ$EtHmVMKsRr(EAS>7ku4E~h`}&tI_{_F-}|Nv^!_%)E`j$LWwB1hF3Z%l?-xVv zR#wBIRbGP8?9~Dx=InaFA-3rq{nbVUa6yn;2jx^CL>Fom-i>y2Q;7qzeW-m#d2Ny9C~nj=#|hq&)by zumdqz{!};3WTYU=oIAb#y@lr^ED{{@bdxy3Kebs-hx$8j|bJE{1$o&G_1)2tiEnfA2)K7M~h zH~gbu2>rdm3~Gr5`?ImC>dFfrYMtcwGoF4JSpyu83x)x;PD_p#7IK0Tx>3*xk@ti{ zqod*8;2&8_gGOhj5G?N=+KoruR^(f|rVVxarSdC8nKQ)w!|eQjBb`7j-tPBBXv5B1 zXC^&cdHL+B9a$R{*WpaZHTG$W*A+UUxIL}Cv2!yC=C@MIp>oHQKqSImO20TA*K_d7 zUCg=kkwSU~R5@SifjHC%ewGH*meiQE%E&!ZTxxIX^Nm>Xio>Z}R3WC`DEL>-z0!Kh z=F$8Pc}A&%@l8X&dL9&}Z-^ZkTtn6@bdHWrBcc`avs~gG>)KQON+Y7Tq(U>2{5uA~ zr9dt$huXGx3K8N`<~99Ua+P+gm(usy(Wr(AmHHORDFdh7&~oghG^3MzhBJ8!Q^H>r z_nVzv1$7q+BGlphF1aLiM)MVe?(M( zwHwu#S)gEdNW$E-_YJ9S0u82*0F z#EK(Wl@mNzYjIeU3qVR*zua3QN|QXfplTqKsZ@$=>s`h^=$1y=bxH=Vyd*YhxQdOo zaAH@}=1NCniPuo2H>XPs4G!L;f|aQxI|a7xy&wQ)E7@Sd7QFbQwQ8;E+Gv+lML}B0 zbsx=HKER-D%F^ts1! z27Bvd1}`F3_!XRHWYM&YcI(Rb&7nN&5MtkYaIYDw z<~@VQp1%==4n9yo?0~UoWan~L97=nLALPl1bu2WfkTxU*W~&OduG~zb7?uX*(nn}o z1_!|AQWEGpUg{_|*n2sdrZ1~^P3?Hz@GnO0G_2KNRD1prE}E9kz$mds=hjd|spxnG zGS_4~d7*a_Gi_sB>Ye!Uc^r1h^NVET3mU+^d#wFSwB)}q*L?`cQFc|0u%=>i9{tmW z@{(w!0mY)lWpxKyYNn*v7hJM=L7XZ>){&x1|jtBGtw(rKE1U<}3eY2rG z6cJ0l!>qvlD#d{gPUIP}*1d-%-pvUF3uT-$dsr=T%dnz|c1v<=iKH=p`N9v+40R!A zA{Q~^Ls4C*>PdLi(S@?iGSEFq9z4}o?;)?T_G-@>mjzn)WnC=OfP6v9d2UiHiSnY( z+m+i~o#lB0tF}ui#LI|Z@kpr#DF2yC?QaBRh=@FU&w@f`j)U3eo{aH`*w_hIZ~}j+ zH8@Atjv6|qCY3w#8cGv5ywi>m$oo)Exb4= zdO6!Cw1tk&1$QpwnL8*&3YaSg8khzxY$#20%vtYFM8D+56lYZN$%*k|FQ|WE3eV=; zVVTqKu5KgT-;go3xc3-*v%=@{e!p3+JTZ@)ILFs;_H*aCLeZCBD%2k%_cQ@YFV))z zdqzu2GwV_ncn9^~*V&#mH^2T<9U{~UnC6j5+2Z>0eP&j|JkDCVvM~@n@!3VYljM*4S7acW*7Et|NgzP93|UdeN! zAIc!_H+AglJ6CMxbt#Nf9C11$=8D94b{kZMUX(-^D|%Po5Bu7Rrk2C8XO2ykBM%hX zY_4Y{Mk8ZHK~ns#Y!s{MY1EN4CpE%qxLht%|}oUR>@~yztewPm#Y-uq2wLc%2oe39hto%S)G5tnX!Mo!ubB*4*D*p zWkitmb-R^VJPs6@5zr0 zA!EaGsm$va@+p#Zx(Fq*U$IvOrg=8*0eiOTlfZ7>Wv3-+Y$eXE5XJ_hD&6In6K;p9 zq&RC*Jp=I6sCFOqC$F*|=GI*wZE!^u$nF+N_gV#wRna^m>a04%74lAbLz4m&mb!+-US#pU{=5p=ld zU>2;>lrn>BdsD;mdE;qSg;NJiZT52Px|kUAoS&E|q^d0-$Fj$u7$l>rds=Un{raFZ zQ$i9xl+yH$Um7mGBI%QZ76M;H!>e#jPuUG0m&<1|9L2^TS1^dkLZ{l*!ly*YO$<$Z z%FANn;&a!t=Jo3OkvMiP=qI;xCSY%Hb?$}f~B5BUK!L4=Du9~N)^}R%*PQ#f=;COpLnR? zqF|KTH=B0n-5y`RGPQBr_jA5{ttY3gx-+k5D&p-1ZL%M~cqU+%d#qJ_UPA9d%DG~$ zOXfPnB0ZOZkj~tqXU+7C8Tq$AJX5|ppZJX2R%!e&@!_PmuDc7t1?~P6Q<&X#|5A9g z6Xszf#@4BnUHkFiDe&YM8c15X6AHVQb*p}6*y~aPNFBmZ;g}Mg6wwg^88?h_Q&GE( zf?1uVCdC9@W*&Iv9z6TuLKuDi^7Xuve$&C9caXpJw4zKQVRHFS_ zriOCWtU`M*QFWrMQ%c=hE%rD(kbQ`}uVnLtlF)ug+?YE|L1)&rrEg}FVzmmVZ%wV^ zsiD^r@)}q??6d=Q=;CEPn*vMB84W_U*TjmSWy~>K{Yj|((2(MMy=>Y3-6_19l}YND zp(*8kv19OMrt*SKVQ!yLuExxcWV~pBCiTG}cjxn`E4f_FWhg;)QJYCgJ;|huZ$&M# z2FFnW@u%|Ci1(;{P968J_VN}LWmR6gd}@e0DcsXn_X-X}HyzO!Y;`$930Lc0++F^yO~s6qZ}0Em-~VbjxY$_upr5 z$IP2fws1ai_ST$Gx>aFONBHMg1gM=u+5artXGeSg;Gfgs|30&6FBP&h@M?Aa;b*&h z1fr&N7Q4Ff)P+JQRq?BmU1iz}qS4gr{55YmixTWYJ7wSO!ErSO@1tn1Xh=5z;mUS5 z`ubo~_K_aj>pFuKxadZl)+cQ}!`xhF9!+TvL@aB2drr$;FJ;4M_m>A4y^IlMX%^}3uNVu5ds(tqlVti^UswH(y-zOA{fYtYVhE;+*Tz4MvOo9wX-5-h0Gjum{| ze0h(&mLyWp^2oB~$DQ0-+@2A)d4@q?c&SOd*`AR!&r;XheQSIc_pJGU8l?AO0$BqfMpRVhN+NRbs^(`_^lR7L)*T$IuQm`S&xPrneuP; z=z9FoO4O;WIMN~eT8p9Upw$JbrFR_7F4z`M(<9EmT^eyyiSh8f z_d!Qlxm!3_%>&?e96oe3A&8*Yri&aXy6#Nc3A%I{T$38E^P|sN3UI^azx!gtYv7^y zX#2A;UF=0RKJ{Fi;z87ViGea#77bHf?8_MCRtYh~SYuxCw|wC-lVVj?U0nUogu4Yz zT!}@`otiSS`9$?YsprBmA2MaC$iF{ebF3KeU!5G0??;huGR^;ny6@Y-mKDELkqJde zhkP~tgLj$tXn?2O=BgY_ux4-ODJrm_*fpQ9irwq_qH579_FlcWg0y%;w}ru=bXgCz zo%`tWfCxqD(uPcKGjodexuMKmBb7M(By(-&4x!3eMYfU@%SgUTar$=T0*flDRhf9z zV&lae1L}zVo}~2}k1)nmd;xaO!Z7$~uHn+-_vCrztm2IGNyWq3oSgQzGp@2}WIIy? zw%A@*I_AAX$1zD{>q*kFFjs2cGDy_UR{R`%{(To!_C}YPPN@v>T84ZGZ=r5`YB!j- zY^}}Ba~^*UsSAB&R>2jTe$&WV%Rc)xv`bc_ztEu!uQe?48g{GBYPMwlaiG_&58-&? zC1unmxMjq%E4du$!2U^V@-K;H7wOs~H@I-=Gxp?rjs2KGo_j^Mu}+G+kCn=(#4(J? z8U%ALq`Y}GEp>B9B>h0MDz=aQ?^LaosB{@w$Z@LviiZ|I|+rc|v2mvt`>HOmn{xLBcv#j1%)LbTBrVC5y%Yh1oyRgH^4$LwzCPNY#K8Urc*Trf-KtjVM!AAq%A?Z8ti*@rrzHNpnmfPbWLG@9&8P& zp@1+YA_mMEpAKs%2Bk>|&2kt5|6=W;KmxH|lYT$+%1rb%TbL-Wf{CTamSY+`ge%Ov z`aO32@Xf?6`oQxr>*E0;91@Q!ECTvWS%GG<0X64VJYshRn{a)vrCsBB$C}N>z`?gL zG;CcOAi}bKrn;m2*d**GSGK@zJT);~>0Xov!zPz4fUYcqG(ODWxF5WhLAxR--(SX> z!^(KuX#^M%2cuVD=ib)}H4?QSV%ge?!71D|_odDHVGgyC3JwhO(q>4sJ>tSHx!0n9 z0>0%cOwi8i`A}5ciJ;4@-kSlF)rm$TZ>xsA%cGjup|n| z_cvQrZVj502A5c0!cktpn`|b6EE8vZSjC(zh?YQmRV_7=tN#Ma(I;eIK+8<54LZZ@ zvESAK%Ka42L;s}1Y|({f{5=?0jxFhb{P5vtWBhGj;zm2zSuIz)z$NjsQ0f;BW?{Ws zsjIb{pTKG14GX<|k`Ls5Ig{_r?z}#JHBE0A@yxC|!7APxe3cDL8>e7wv@sF$q(jno z+I=-MJ+5##FLI7#NQ(Wg&6{X0lDYve>TvW5Kx6t~Ign)j=?>A=(kKTQLW3(*QhJEN z92g}IbFIGm`q`ghwh;DAE3?Bjw`aTnwUrxtD{S!vxJU{N?m?S=f$NOI39*2%IT?TY zHM7>Z*&Y4azH@E^!Y;CLPtkL4gn6M=JomDs2mbz*-(Ki8f;)W7_~GPANViMv z2*1AtW{PTPG;a=WidDl-omaf3BOm(l)+tpJ@Jy6a%6v%hfluvhf*7M38IbHSX6%TA zoN9*8m8t3^toL~bw`850kxYx(bzaijM{5#8gE4?rcl0(}DHt@5*8S zz9t~X^}w0kW{(m5D>}`@O0ADa*3Tu!WU=pK)<d4fUk8J zy=sFkk`V|kW=p!iWESzMN-lRMymX|NlP^jjiCF_Nzt&F1Trs)!e3KGr|bX>l6ORQ!W8fvH;-&7B01A zxx>js@!09$E`DkOzC4#Gz+$8;J*VA)V?xge4y;)M4Z?O8=dUs+(3!niu%Td?A2jWHo4B;v(>6iC+Z!rfZPYBhqK`^UB`;zpzSqAeRqQR-N(zs#Z9hhyVGFN{D(Y_-o>g!JRTc2iW|-`XNVQ|Ba4>4ao`f^F0Q?koN?hT%`4BA z)z0Vzg90cemR;7`Ha*MUD0tvCwq-1(fSa*JS0y$-$h!9u5mB0?8Ik$7c-f3pFGCFR z{*%IiB{f>_I26OR!*7#l#KpsX9AdZkl(xf>J}=5tz#IB~DF{!~o8ki+op)NduyY9{ ztT%Ts_fd(z_F3-XoE*=o=RDB9zLbNfedWA(a6q13L2O*juT>&3ua=^eej#6vV|Y3eXqL8yDE8 zRGn4*oziDiYcfHovu$0?dUl`Y_<3`m@B{lbb+b$S7w-^P51qbIYo|l$HQ+;?W2$#& z|0nkZhhpp8I{WzfB}vsayp~HUO*#C~hu3>%Ra4o=HI`;_X@i+3l5I2I?0G+d<>zM5 zmV4`#az4WJ*dPED#_|S?z{s}Y>1-ojABkrfjdNOJ=cM!s7JrUiENk(;l;6a;#Qear z)>1#kx(66eX7PGm=M>1_YB%)@U#Hq!PA-hL4@V2tyzNylh&UD8I%X>4F1BzzYxL}& zgo(<|!tPu-4S7GxgYK72qKCs-s+`+H1Q#|zt=hLMs^25*(?2@0$5^xyWN!(kCtO)b z#Z>OBwfgEaoa3jL2&BIm+n3^(2XODeh{x=+;GL_ffsNiVrx9K0ad@EhF5^1Ng@I$1 zcfWZV_n8o6Xk~k38aPIZt}UD+9C$jrzu!5Mobm2L(N+pI+!&VWQ7w6q7}a~MZs_yn z^d&On#61qVSM$5%4JS?WqWo`NS)ke-G zbKRkLZMqvXXR-jIjSX&pF+NEeK(jlr+aPF7Cpeh#l~Rr_oSh3xq>H|eQ)Pa9>Xcs{ao=3RZqC`kM(&76x9}C!m}wdrDK?W z)M|=-*h@5PF;ADH=tbiE%Z)gILIm)vDmX)#`{`5LaLRC2cXQ%THn+Ezk^68`dplFn zKC$LLv#)kqbhw|nd5zf$D{Q5FSq5vgb(^L>om)Uvx;YGk0(`bl_8$MM5}V6$aH%y0 zE_y_%zuCU&j$3~mk6;Pm-c|t);>-hHIQ|`DT|xBaB5{6snFBxn#{#ZQORrZ5>qkTl z-pl88TWuoeS#L=rA&ugPKK8raMdoT_%3;@aI|PBosdIQT&OlK38(C`NmlKujsiFczCtfYJp0#8W|S9z9#95}ivBAm}? z2nSg_k6KFsJJtAB#}TqXVgPMOY&qV%zQ`)cFFJNCAlbGXR)ZU*C4>@pGv|3x(8D~N zQ)8_tgt7|BsGpi3mkASF0L0hf^KvCMPAG}R=w@jQAe9i7j{xqT@kM{Bc*`6PNy?N# zt3B(N5}4+JwXh4z#(Ka4YJUW?lm_Y!Jk5f`l2OFq3 zxH2HA*sr~Mc9tW!k@d%xYO5zSM=G}r#%x6?VR%2W5FvDGi}YO=DO2b84k4x0aL|(0 z&pn40rQa6U?zDp(JE|b8BfBT*ayZ7tu?dyLxA2Ys5WJGVc)(6@Xd1M4wnK?fsBgDo z@l;%EQtqg}>ejd8FcWOus#IRZ^3P+uKO(W&PCg6kal}NxdC}uC7glk3+ z)TCFUrrbKqqFM7W^(l3`ojtqTXnzpUS_m2Z_+?Tt(|^!X%Jsy$*3OlKu3JHunGE3$ zWUX)lpxnxOg83W!H64m!hIJ5LC{gfGiq@f|*<_kLnYC0RvgH$^fFl)W4kUw`0g6qh zFPu*_q!IvAb0z8c+bU*0Jz8lLE+= zr$GxA4c_-2idgu87ly5|6m#>lW|G+z;Vc#MKf7ae}6_RE+iijfJk&XV>OJEjq2hvYHzs*dEaMZBBJbF%5N zs5t_0b?P)}>j8P7jiOQyJ;w~;?sy`R{q9f1tN~fVHxa|Qx~N}~kwCI%!P~Vq^J)Y( zUo7(fs?KXef8nG50KrGnn4r0wG29Q*AO4XLD}5CsK+QL=aA%}o#)d{XLJdopCi^%j;6K7KMCZNIV;mColHKDwf)d`Yf$uLYZ~+vN zg-G_1O`9&VK-zo1JlMww-~!Km#4qR|SSKJ<#r#mz20=rDcQc5=0h%Z{EJ^boVH&43^T8 z8b_cWZd=gZ7LEU->Mm^-Z~hAs!~{X_e*+;|Ai4qN|HnHd3&aM6UGy_KdZoNy+#yEV z7U`H1M!F?e2aI&5z+Co^lU-rjB5vzmn#b_X2j|P{;eifj>k2S#|`~(3WB)mDpmEXpnm2ebu zKKap?fQe@ey!n)O5jy}UXv%-|1p!Gm1F$*lkp~={<`pWaYae<=W50uz_c3A#ia|8D zIAA~^UY>tpjpxX~AoO!zYQNVEqHbG6BrK>BZnbRV1NlZD!Cm3D&b}Pr0wBAPOz@8= zgn+T_(mRMOcwu|K7XrZ&c958}6$73)U=y+m+Cs{%@Z@Iv&SpIlRec3ia_dtX|Ng{u z*2klswa{OI;N)C92}Dko{e?#L6&cil{CTY zYXb4LF*LYYI8}sm`Yt`My$x@1Z$vXpu}XLHz_o)g_-wMD!-dw0 zp0rPXxQcY#J1tCpnB}ht!(xDw)&K1YujuLBREb6e%d%4qb5UE@23A$BHkd~=!OA*r z>V^UQ5z*Jkiw15?cTYhaZln-nmW6$s0MDqky%y6}x?s21r&aDap0L(QkLDvZX{QN) z_-)a({2PuN6#(WSaQKC*&Az{-zAh0c!P7Z?Y@R@2h~k8ZEdj@ zJuu!Bj;(`iB?&Ufj4*8-Fs_71m}~>B1<55DQiKHdrsivGa*Wi(}nZ3 zZ8n@L8y`mS(X`!cV$QF)avPrW8dmm9oF~k|jO$EZ*a6LI4XUYJN*sT7+*b5W8AX&t626 z-|&E&bc>2C+`T}Pe>mC@jC#Y+A5HfI`;5A36$L-*0UltXNDn{Vz1It&*3l3-@uQj= zs5g$<2Eqero!xiS!3gC2&m$1BFF?iPly2?7PKA|wd70xJzC0D0k7&6GjMpY{N{%vYZ0lSc~?OfRo;^9 zojis3pYBqdW<(q|fwo-@idkd;w?)D-I?oOCmcnx;0qe8*yWV9T2DB%Z%DYFei`(?% zCs;xYU9^RSN2^8F!Gx#ZQj8sOjed<^jP{t|MP-FGU%ac-n5Fy(WG~3HM*Lb0u)uAH z!Q|D!lk8;(HQ>eV&F}9oRwIze3FBC7BoJjMSfFY$-tmPgrhK^FI1NJYj)gsG0;t{J z89elC^0*T4a5a$u_3OO$3z)ZmKq3-0Dy#=rdmdER(v6%daFZwzO(9)&3h;m!BEKI5 zKdhm19*nN7ntl^LbT!*Iu&4=ARL>SG!!vf)6<$*vA*k+$lXxEV8*2z+dl;4gKOG0^ z3xksf(YtoV|9uXqkGO-Pwi-$0sYZ~G6mNK)Hcyofk6_ZELs;4I%Y)~1uO*bnOTH^G zJnI=>Zkv<1ZY@P;#jK9rr5O*{>hGn{fg~(Ru1_X!n=U=YGvfH!;O;v_&!`HNOb#zC zb+H>NfZ%;=p@(E$N=Gr$y=6|oN-;2g-M@c+Ne@E6t)4r9ZuHsKY(u9Ai<{}S!DoeF zqFN$YlIr${$2Qd;;P5)K#MW*na8-rg1}3w=g=NvuoK>(PURm*081z1-v~N2n+vZHs zr-`3P=rlKN2>UGrG4694KfeWTJfHCM8&p`8E30+N*^sU&M`rf@*0--EPub|Gzz!jg zPqWVqV+S?(R2rv6jwRr*mgKBq7`Z)*WHb(gX6PbjuB6|tsL=o(@l$C?UYc;S9?ZmF zF$7)^RdpV)@)c+Cq+7t|h}FA5W#$cvs^57|Q0;85?WqC@%yv-q+toWE5{s{5(-^TC zzzOZK1ZD24$-l6tS1rghExvqXY3evBjZ{eAz!g98ixY_vYt7@no$mDv#IB%inr&s^ z63N(1T)d$AZNlPtQiG?6pUn6@y%HPC#?!b;W4JcGA=LBLHQRg_(Jc&ZP zR?C_m5)(!}ss_&+-jp{>WVR`9D`}Yh!9gF`DmjSGB@foM+i~xz^#yPK(|z<3=1ZDN zlsl*N?ftgApfR87PJijB0;+fYHi{KnrgHc*{X`xUusr^x_7yj{Bj9)~n*Z>mioEJRWnhNe>QP~Z|6jZb@6McRYu zZ%jNF4Km4Ia}vP{rPK%hLbpCRpELum-?%ZUZn{TBr*T9*D=OO(M&}Ico8bRAv_6}3 zl7tkqu{)5Fp81L#3G1BePV8%fr?7ML!+_-Z4i_D>?q=vgqwFf~jX`qU zYPwF1R@A+&2R_hHZH7z8Sl+hsDUFy$X0WkoPrXy%V!#=r{5k!?uWA?daX!>&q}-vu zPNmgieqloB)p#pr_%}TF3XDLlUrakNCr}SE`58geiGRXwLzoo9Ho z&Bq3`*=*LjV`>N9c5T8cvV5*cp`7h1Ys?IAcMIzWVW`C+*!=)pRif=6s_H4rj zocFJ{cc94m!N|gYDa-3)3>^M_tJe7PD;7bP<3${vGX)8oPM3xtyx8f_!Xqc(zV;Vj z&6_@swiF(=&#gt zW>`GfDn`9d0`9(^-yITf*~lKf-_kXtgXou)LJLp|JzZlBW`Pqj)S4{xZ-dd)D^o>4@)6uf=5Pemf`(jZ7bwNLuGFzMK_0 z`cJ)Xd=ZZSJ!E8Mh(O#84O;5{lO@q=JY|UYz5kg=x9vm*_)P<9bO<(37g`Bu(4WXM z&3@Km>vrr`BZv<`!&?DHXp4cBJ3wNXCZ&UJJx%pcLLm4#XIgN54Lxo3tHs1!%37pb7dJH3xU~zGC8$@IDH<#1>Pm!$~+u8#9|Xh z;JAC7o>`cyHzcG#BwEZUy*7`SJ?XzU$PPNSN|q!&T{J*4MZf^N8X5I_7LFbsA_qB+Ia9=x8OO!>0|eaf6@rlhIgL zZp_8>fGh!{a|96-`D@~*M|4Bm5IgC-TZ*ou^KztNAz&r?n_E}`&kAcvC;o_}1|mo}$ML6JoA&j{1Sg#QMIO8Tm7<+VvD6)_jG-<>NKb^uns zO}?RlLiQCR*Z!}8^t@pjT`q9`*s9lQPiZQd6JDYH{g73A5XbEy7ja+MI!C)XmgI5W zzs8gG@rZ{F_wlnLc^M0L!%sNtXC}z8m~{Pd!KiS5&uL)Bph?9;_69FR4cxLT_>nu{ zce+=?VycIW&aFCCHTl<{e;nuDNvPDkc2(`z?0Ogl8(^~7VU2_6s6ft;iKU(sr;l_w z-)dNW(pSuBP>ueiIR0r5X?ow4F3)H~kU(DvJr}c^vFihM!Sda z(7&nc2#=&K?7%6T)_68+oC@8>89^h933W@x)Q5!_0q%u~Vnvq?wk(#d+P#N+Y9LK> z#4<+i%fTDmV@W{~L^-tW9=!*{R3QboUB7Mb52;eeL4 z)SXr1GP$YF7@aRDj-BaguLfPeG$ zml5Dp7W5vb2!|UAh!E3h)21#YA6{OCg#Ez8)wWU7^a0q34hw+Fc;g6c_sM;x+D;F0 zpGIESeOw=nYBb_WMu$fm2?cfS(IFJ?t+=RpWElzg;NfH5>~uBh=0fqjVID!rC3E zxZ5OnG&iC_HG30Iu!{!P7FXMHhO^!r^)s-vK5NCcSM`Rt{;mEb+jzEcge&v3QaKkJ zUZcJ(=YE1NNoX@(-&_TcVZvP=f0o$vT|zN zmw^mrS@*lP{V@CPtz8Dylr6hr=jX1eLR}JpeVjYI9-V4T_n0{aEv4W8rExwGd&Uk~56}sM;jkn?NW+ zXbhBfgFaPomor5B0{F=V1L<~Nr2~$ER7aSPFCgoN8fAz~{Jg!5D14AmD7Ah_EPQ`j z%sIo?0Q6gn;D+QGl-Swc0O3ch7=CyMe6N49bn@(Wq+ragy&+SNf*oOgftU*+jpBW5=IQcPvG)@RI>_? z^VL?pT#V2#AaA_e-~1A3cMu28bp8~ASNP)N3fLOz~HI4xS(f%elT9Slk1((#czdfhspu5zZm z+|Xb90s&d&6HJJxZ>Jh2xfi!{8?3P~FQ0j$tfI0U%p!1wRE#JV$Pj)_#ty(3w;G`& z)RaCMo+CETfnk7b3OCH-VW`qKY39X7VQFxc)Y0YxP0!qheai|Bh{$&jP}kW#V1v_p0{Sx zQ46LU32R*)S~Vyhd(?5nAzei9S+2@r&K^GA!>WfknGjl4{RAr4A^2jBdsh_}*P^aaP8U-N zhF$~#%2|tN%a?hus5iUuB%h=x#$!%SG81|BPKjt(@5LTZi|AHW`D1a{5qFI#%ONlB zfOhBri?y;)Eows|`S)1=)Q?3~m1;Vuly z{dhRjHNNDygidNC=F7Br*zMr3#bKB<6-nxIwkCO%FQOZaxofg2-wL4ZRwg4S$gZ1t5x>ETi|Ygio6|oIC|ylcMiAof6YA= zuI<0HU;VmB_PAAB=sbZa zy#?>DeTdBBU0Zl=s{Qqiz@Aui^s^^n??jPOpSJ78=l}l$o2m{@fo`EQ>q-aq-5x04KV(I Date: Mon, 16 Nov 2026 00:16:37 +0300 Subject: [PATCH 11/11] docs(submission): link the public lab 5 pull request --- REPORT.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/REPORT.md b/REPORT.md index 40754f8f..274a3590 100644 --- a/REPORT.md +++ b/REPORT.md @@ -402,3 +402,11 @@ repeat0actions. Active/current/fresh и действующий reservation за expired source replay отвергнут, expired writer fenced. Возраст fixture metadata менялся SQL отдельно; S3 LastModified не подменялся. Raw: [cleanup-final.json](evidence/cleanup-final.json). + +## Публикация + +Публичный fork: [wh1tet3a/mtusi2026lab5](https://gitea.privcloud.tech/wh1tet3a/mtusi2026lab5). +PR: [ЛР 5 — БВТ2301 — Маслов — на 5, №70](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/70), +ветка `solution/bvt2301-maslov` → преподавательская `main`. +При публикации remote CI имел pending/0checks; локальный clean PASS не выдаётся +за результат удалённого CI. Commit со ссылкой на PR меняет только документацию. -- 2.54.0