Prepare verified laboratory 6 implementation and evidence

This commit is contained in:
rey
2026-11-17 17:32:06 +03:00
parent 470d9d6f1e
commit 7d56e52fd0
335 changed files with 31206 additions and 72 deletions
+9 -20
View File
@@ -1,20 +1,9 @@
.git
.github
.env
.env.*
!.env.example
secrets
certs
volumes
data
backups
*.key
*.pem
*.p12
*.pfx
*.jks
.venv
__pycache__
node_modules
evidence
.DS_Store
**/.git
**/.local
**/.env
**/secrets
**/evidence
**/node_modules
**/.venv
**/submission
+4 -29
View File
@@ -1,32 +1,7 @@
# Local/private state
.local/
.env
.env.*
!.env.example
secrets/
certs/private/
*.key
*.p12
*.pfx
*.jks
*.pem
*.tfstate*
volumes/
data/
backups/
# Runtime/build artifacts
.venv/
__pycache__/
*.py[cod]
node_modules/
dist/
build/
target/
bin/
obj/
coverage/
*.log
.DS_Store
.idea/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
*.pyc
submission/
+110
View File
@@ -0,0 +1,110 @@
{
"checkedAt": "2026-11-15T19:04:33.198Z",
"validationScope": "Go1.27.2 Linux unit/race/vet/lint/build; native OTLP protobuf fields; real PG immutable SLO and native pool wait; core public smoke/persistence/both API failures while Collector unavailable",
"fullObservabilityRuntime": false,
"sourceHashes": {
"cmd/app/main.go": "00e7cd80cd07e9696a2eb761890b87fde26a09ed96efe01e181e62c98799509a",
"cmd/worker/main.go": "405bdcfead49e30c988629db8478e4aee3e0ee6ffba94d7517c8676d5aff9dc4",
"go.mod": "7f2b2a2764494f7be5f395d78bbf42f04196d3696363f89a0f38308e3a2f1cad",
"integration/object_metadata_test.go": "598e973575d57c6ee563f0b15d0111d37b27de6774c6f486c998dbc70594f15d",
"internal/config/config.go": "01fe90ef662e1bfe164b858818f68dc8e73d1ae3b1c3a07f83a8a7a00bb9d883",
"internal/config/config_test.go": "aa9f1c670f525e4e895beccb807108e926cc9950bbf509ed71ecc96d6f64891d",
"internal/adapter/handler/handler.go": "1a99601365ed52aa36f5ef2b6be4dc760e28e1ad5cc38eb6790d69c5559751ce",
"internal/adapter/handler/auth.go": "3889f58ac4c58b798192776ea05455d52735a2c024ba664ab9af5eb405187557",
"internal/adapter/handler/api.gen.go": "4034501f5c661c87af3b81d0d1ecb9996050828569bea7e85b3604a2a8c8bdbe",
"internal/adapter/handler/utf8_test.go": "eac6a14b5f3b0ea9aa5329337afc6db27e17f5c17999874da9ce3592dc7a4205",
"internal/adapter/handler/message_json_test.go": "8dd976c4cad81ee531c76165787b2c8b6052057d934ae4b09b60d8a261cf30e5",
"internal/adapter/handler/gate_test.go": "f7938bdeb5617ed3611303c54818000de9861dde5915fa2c6fa0f74e1fa33268",
"internal/adapter/handler/stats.go": "5cd375e5a0db8cc3596293d323e635b86fef22d8ec680f310ab5a460f8176f64",
"internal/adapter/handler/gate.go": "b662058ebc59010f30d21da12404837aa2329fe5e5590761efdd9140c1d24e9c",
"internal/adapter/handler/api.go": "9d9f109e13c9322287b29ed18e7913e0e8d5a9ef2dc4be23f10c7686de3f70fe",
"internal/adapter/handler/attachments.go": "608827653c4d3e4252dff863d30007e93b1bf82579bbe71d8b647bf22128b1b2",
"internal/adapter/handler/middleware.go": "41ce83f0b7eaf25aecdbeb99d5292066f25e2043e08c0f681db2cfc80c01c528",
"internal/adapter/handler/json.go": "d6458b8e8e4426816ec8c855ff6c8d88a151161bc1560e4b98df0ea4b0dbf6be",
"internal/adapter/handler/errors.go": "39a5b916112fd9cc2be9979b6e78d6305e7c3c8ab5fa462f1b8e9f18c88c1a4a",
"internal/adapter/repository/postgres/pipeline_integration_test.go": "58274475a02f1098a0f79a65a9e8d7608d05dfa26213f90c706a9239fabb586c",
"internal/adapter/repository/postgres/observability.go": "b6001827364a226026e7a651da8a57918ec426e34cd64d3b6ef709814f0a5667",
"internal/adapter/repository/postgres/queries/models.go": "dab26f6c8a5542ce223c234faac1e8ec8c7066507a84989ca3f620583956d0cd",
"internal/adapter/repository/postgres/queries/core.sql.go": "9a0e082263cbea39535de9c7c72c41a39a551efc9ee57fdd7b08856831d074b3",
"internal/adapter/repository/postgres/queries/db.go": "4863732830879d4162be215ff245375b953359de23bcd821ed7abc5c32a36c0f",
"internal/adapter/repository/postgres/queries/attachments.sql.go": "407ed2864afad2b039a33c571228e6ee6a9acdc515d957bb55dc600308b5eacc",
"internal/adapter/repository/postgres/queries/idempotency.sql.go": "cf6fbc316e93339e0f938c7ca82007e29c3e32f1a21ee1433a2ce8784697f002",
"internal/adapter/repository/postgres/queries/pipeline.sql.go": "43009cbd2754d478727cce5d4b263dcdcafe7ad8000de011f22795531e1fb3ec",
"internal/adapter/repository/postgres/queries/slo.sql.go": "bcaa6b4515507e555caff58a0448861ffaeac2f21771a4bde587cf296f6c685e",
"internal/adapter/repository/postgres/queries/observability.sql.go": "c2971da00171c126b129ca2010a1cd1a6223b11ed1b1b20632af19fe8a66f313",
"internal/adapter/repository/postgres/pipeline.go": "9a367d7a50a9418e96851c253f61ace4a54753044e7fa672c73a3b4526a22245",
"internal/adapter/repository/postgres/attachments.go": "88f8b8744d95407421eafc111ecc83bcb4b145f1683618d69cac19a8d1c39500",
"internal/adapter/repository/postgres/repository.go": "e380d7c0c52a2447653db9e4d13db432f4c1330e3205a0396510d957441bb978",
"internal/adapter/repository/postgres/slo_integration_test.go": "e396c722fe8ea7ade19e2de5c9a070352efbde8e8712ae6e910f628a11c5c2b0",
"internal/adapter/password/argon.go": "94643bf4b61ca390f8dd56a241e02e1325a994fc40887ffa0e2b8f8e5aebb4c7",
"internal/adapter/storage/bootstrap.go": "44c8f422826b29f85a918029bb71a62cd3018190f8efe889cbeee63d19d7039e",
"internal/adapter/storage/readiness_test.go": "a6c79cba7af74ce67e69678085e5bb267a47f9dd7be57daf003ba03951274a5c",
"internal/adapter/storage/s3.go": "99a717604c71da5c181bef74612a603abe3cfc913ccb3da84ed581a57a11ccd8",
"internal/adapter/imagefile/variants.go": "50762d1e2af635c9a9344d2721ee145bb9d9d43408f680fc280cdd493aeef886",
"internal/adapter/imagefile/variants_integration_test.go": "d25e3371432128e58073ff61e1b87fa741bc32ee7d9bc6e8cb40ae3f3a5b4434",
"internal/adapter/imagefile/stress_test.go": "3d7b0671f57ce5e01fa7e175a86c16a51d56d8f11ec6d3bfb09b2c8bfe8c04d8",
"internal/adapter/imagefile/variants_test.go": "79c24a3d5c0cb339a32fdea28eb56689703f35ad3c668c464ea7a4f5e82322e5",
"internal/adapter/imagefile/image_test.go": "a2dc0d3797a5c2ac7ad46a7453a34ea13aba94c3385d2692032705610026dc71",
"internal/adapter/imagefile/image.go": "8a9b399b22573992fd7fd4c87096f953b46c1bc7cad4e09c023923919d755375",
"internal/adapter/queue/streams.go": "c9f0864c264c23bb6927bb20de06c6a659c709b3228255059a45edaca953c360",
"internal/adapter/queue/publish_trace_test.go": "057552f74a6af595439f7e68c7b0c8bbdb888093767f46dd559e8c656adec049",
"internal/adapter/telemetry/telemetry_test.go": "c727d6901ac3b8a5cc3e704645f8a7715a44d88f8bdf9bd990ac176c017fbba2",
"internal/adapter/telemetry/gauge_failure_test.go": "4d8d3aaff9e471a36c0f65db0a23d799101c72f804b6d7270bc9a9570181b225",
"internal/adapter/telemetry/telemetry.go": "db9ec2b20d504e7c50424414618e00fe1b81ce68b997c70ae65040260d31b890",
"internal/adapter/telemetry/gauges.go": "80782d0bdb1bbaa9132c6c3a471887e4a6d48839b448a5c3c558edf217a01642",
"internal/adapter/telemetry/dependencies.go": "09d5bcf97bfff77d42dbdca9637e015a2eb73e03ea3ff801d710e1340c85ea44",
"internal/adapter/telemetry/dependencies_test.go": "d7738b7fe098b2d72967243dce8ab7a84135e570e4697371755b73245d908981",
"internal/adapter/telemetry/pool_integration_test.go": "b8e8d24ee46549c0898726f770f206701f147ea7f9cbb4271b24dcaa63f5a0dc",
"internal/application/usecase/pipeline_retry_test.go": "96a569aa26e23909c109dda14c8828901abe2914bbad2ec3506a10441a264dce",
"internal/application/usecase/auth.go": "93802954db2c8eda3782acc71bca817947fd556986d3f6552c78c4b346d988ca",
"internal/application/usecase/messenger.go": "c58aba0d3f5d0ab92fab5d2771b90a3af5ad06e6afe0c7cc46f481a429e89f48",
"internal/application/usecase/pipeline.go": "05b4a6e87c2bfd86c06f5b8cc3e7b960a26c0a69726a366a3137158b1e9e2e45",
"internal/application/usecase/attachments.go": "0c835f88c72b90753b5458960f7b345e6e51b964070ce37eaae129a4c296b0ca",
"internal/application/usecase/reconcile_pipeline_test.go": "75e1fb5fcc5ffb2188a2368688e5505a4bf54453c2800b5e46c8e9e23de89b7c",
"internal/application/usecase/reconcile_test.go": "fdbcfad86851fc97a3cf635b6d884c41b43084eedc42b3c4847790c5eaad4a6f",
"internal/application/usecase/reconcile.go": "239edee16f767dd16b116791900994540a8f99ae3403fd0308f5e2c0797a892d",
"internal/domain/repository/repository.go": "0008747c69b32df9e4b7f36fd1ac157dee537d5e1e415bd48bd55b0b05852362",
"internal/domain/model/model_test.go": "77bb8767ac1352bd3551e724eaaeaad4fe93eae248cc8c3008ee8b1c08adddca",
"internal/domain/model/user.go": "c20c8b8d8a9c0327a3a24afd74f3051d78caed1d2f0851baf340ef61ac502a92",
"internal/domain/model/trace.go": "9ff395bce67142f6808fa7c652e50db2a68cf4e34c128fb04af408566a14220a",
"internal/domain/model/page.go": "cd6a07579bff01aa0d8a11b34fd000a804142b321cef302d386bdb95e9c7f953",
"internal/domain/model/message.go": "0e0126663d2de04c9d14c8c7427f42a9fb733e3b170ceee7ed86c2138f51b23c",
"internal/domain/model/postgres_text_test.go": "8c245eef99a97261daf0aa16fec9edd7bcd3253ec4144c50b4f8e56e45e2186e",
"internal/domain/model/job.go": "564c219e526b99e06d2c882d7184fc6ee0b58d29c326be5969a38cc6b92b8c0a",
"internal/domain/model/text.go": "c9410cacd9d1b2fd7f3f2cb7e4112e637ecc494f6e4b6da649f936ce0c9a0457",
"internal/domain/model/attachment.go": "e532f2c8e8c123d0fda6101bff4b21168527a0a446e53f566f5e5919b972133b",
"internal/domain/model/chat.go": "3f7454f48ef442c277fd2741ce5aeeccf03d590bd2d31b9b89fa39f0ba05de7f",
"internal/domain/model/errors.go": "d043547d742e94d9ef9f0799dced7c43dc2adc44d7d90906e891e5afee8de04d",
"internal/domain/port/auth.go": "465c1a1f19075b6f8301eef8dbd7fd9a8d20255716d72c97e946db04c2850dda",
"internal/domain/port/pipeline.go": "5fe7c6909cbaf6a21a4be5d9ed0c2b1551681894d0054463cdba13ea09c21aac",
"internal/domain/port/objects.go": "4d22b5ef5ff6fa13404884c160fe52bd7cef4099f84b29268137c27034a0f3ec",
"go.sum": "8d58fbc798eb94634668719ac9ea68535b390621026a9097d0a3bbd0d122fbd2",
"codegen.yaml": "e4d37ce5c5a9f969f696833cd59a6e1b86f99fb2b0de69101a1aa5f4a539aebe",
".golangci.yaml": "215578dee14bcef1b66759a68543083e0a624611efcdb785197307b50cb7725d",
"generator-overlay.yaml": "e666da2959b31befed713a8366237de53070a565f93850d6daae7a582a038819",
"db/migrations/00003_attachments.sql": "6b4f707dde32f0c3f531741c108bcf0f22be05c9e2ede327f27a1d9ef0cfcc7f",
"db/migrations/00006_slo_facts.sql": "2cdb8cdb1618a90eea76e59ac99f3d89a3665b2d6ade17502abe4261d5ffcf3b",
"db/migrations/00001_core.sql": "b6bd8c70d51aed6c966e07eebffd262af2e694c8b3479c70bbf9a9eacf5a7aad",
"db/migrations/00005_replay_audit.sql": "234a04177bca08621fa38b135cb8086bcdd55ae9084008b25814751e1dd742a4",
"db/migrations/00002_idempotency.sql": "97c98785547db8391c1fa0a685e4d4bcc810b8f79055c0e2fc6ef473cd38b37b",
"db/sqlc/sqlc.yaml": "803b161e063692398c93152b11a08f9e42e1bf3c72a1e64553b82c494545fd10",
"db/sqlc/queries/idempotency.sql": "5bdec86e85705f7805fb603e2c792bb87c11cf3156952fa48e7bcee35fd683ca",
"db/sqlc/queries/pipeline.sql": "9551971f39bced93c8690022495f15a87f7e14fa865d050a200a1b5d52c1acc1",
"db/sqlc/queries/slo.sql": "6f9b0092fca3ee0dda312d421bdb9ea587e3f580983372af47dc54f3d79a1873",
"db/sqlc/queries/core.sql": "1aed3d0d2fc04650351d86d15837cf3239636844727e799aabf5ddd534429f33",
"db/sqlc/queries/observability.sql": "65c5dd26008542a80b7673d25631d1e7faeb0ae4caba661aee52d25fca26e4f3",
"db/sqlc/queries/attachments.sql": "8e64f0e6169fab7664b0f4a97f2be353553b93c0c928d10678cb930e44de37da",
"db/migrate.go": "33044d905354f24fff194fefd35ee9327d63d6e5062ded8496e518b6ce38dbea",
"pkg/logger/logger.go": "d839f31015eee692065c31d2538a4cb4c841b0d89e690f0ab8baf80168510556",
"pkg/server/server.go": "2916acc42708dd493068452ac4eed5208cd166f910c517f16196d001aac5d269",
"pkg/server/server_test.go": "7eccb3acd7f9e9ac0b724b17fa50f1f1ff94be043554fed0a5e0fad1e2cb9ee3"
},
"evidence": [
"evidence/final-correlation-check.txt",
"evidence/green-slo-cohort.txt",
"evidence/green-native-pool-pending.txt",
"evidence/final-core-runtime.txt",
"evidence/core-resilience-collector-unavailable.txt",
"evidence/both-replicas-collector-unavailable.txt"
]
}
+14
View File
@@ -0,0 +1,14 @@
# syntax=docker/dockerfile:1
FROM golang:1.27.2-alpine@sha256:f92b6ef800e499660581efdabdf25d9d817a9d124eaf900924f0504e7e27e12d AS build
WORKDIR /src
ENV CGO_ENABLED=0 GOFLAGS=-mod=readonly
COPY backend/ ./
RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build go build -trimpath -ldflags='-s -w' -o /out/messenger ./cmd/app && go build -trimpath -ldflags='-s -w' -o /out/worker ./cmd/worker
FROM debian@sha256:a29215f6a35e51e22adffa17f89e9d2ef06214e64a2bad10d765c46aea49f11f
RUN apt-get update && apt-get install -y --no-install-recommends imagemagick ca-certificates && apt-get clean
COPY --from=build /out/messenger /messenger
COPY --from=build /out/worker /worker
USER 65532:65532
ENV APP_HOST=0.0.0.0 APP_PORT=8080 GOMEMLIMIT=384MiB
ENTRYPOINT ["/messenger"]
+17
View File
@@ -0,0 +1,17 @@
# Проверяемые результаты
| Что проверено | Основные артефакты | Область вывода |
|---|---|---|
| Source checks | [final checks](evidence/grafana/final-source-checks.json), [backend95](evidence/grafana/backend95-final.json), [review guards](evidence/grafana/reviewed-source-guards.json) | Native TypeScript60, unit41/136, immutable Go95 и original contract |
| Grafana dashboard | [18 panels](evidence/grafana/dashboard-panels.json), [representative screenshot](evidence/grafana/dashboard-panel18.png), [strict metadata](evidence/grafana/provisioning-strict-native-green.json) | Actual finite values/rendering отдельно от metadata200 |
| Canonical trace | [ns→UI readback](evidence/grafana/trace-ui-first-readback.json), [overview](evidence/grafana/trace-first.png), [worker tree](evidence/grafana/linked-worker-tree.png) |42 unique spans,41parents,2services, действительные durations |
| Оба UI links | [clicks/window](evidence/grafana/both-native-links.json), [returned log](evidence/grafana/both-native-links.png) | Same trace, span-specific window и отдельный HTTP request_id |
| Retry | [same job](evidence/grafana/retry-owned-job.json), [58 spans/logs/metrics](evidence/grafana/retry-telemetry.json), [error child screenshot](evidence/grafana/retry-storage-child.png) | Failed processing attempt→retry→ready, exact restored bytes |
| Реальные alerts | [original command FAIL](evidence/grafana/incidents-original-command-fail.json), [same-data causal readback](evidence/grafana/incidents-causal-readback.json), [app503→DB trace](evidence/grafana/incident-database-trace.json) | DB short/long и worker faults, firing/recovery; process start не равен backlog recovery |
| Response/query boundaries | [native14](evidence/grafana/response-boundary-green.json), [browser before](evidence/grafana/browser-cap-red-sdk-cap-green.json), [after](evidence/grafana/browser-cap-green.json), [histogram9](evidence/grafana/histogram-native-green.json) | Known normalized Content-Length≤1MiB; size не hard rowcap |
| Чистые volumes | [first identity guard FAIL](evidence/grafana/fresh-volume-identity-guard-fail.json), [120s deadline FAIL](evidence/grafana/fresh-volume-deadline-red.json), [new600s run PASS](evidence/grafana/fresh-volume-green.json) | New explicit project/volumes, original smoke, all3 signals, native GF18/5 |
| UI diagnostic limits | [strict browser FAIL](evidence/grafana/browser-three-functional-icon-aborted.json), [single icon leaf](evidence/grafana/icon-cancellation-diagnostic.json), [normal history state](evidence/grafana/history-normal-ui.json), [shipped source binding](evidence/grafana/history-source-binding.json) | Functional3/3 и console0; static canceled fetch не объявляется networkclean |
| Lifecycle source fix | [delta](evidence/grafana/browser-lifecycle-delta.json), [cases](evidence/grafana/BROWSER-LIFECYCLE-CASES.md), [native GREEN](evidence/grafana/browser-lifecycle-green-native.txt) | Required fake ports; resource closed before result I/O; no browser/workload rerun |
| Остановка | [final owned resources](evidence/grafana/final-stop.json) | Running containers/browser/listeners0; volumes/private receipts/cache сохранены |
Независимые source/config reviews скопированы byte-identical: [response-limit advisory](evidence/grafana/reviews/browser-response-limit-advisory.md), [native boundary](evidence/grafana/reviews/response-boundary-native.md), [histogram request](evidence/grafana/reviews/histogram-request.md), [startup/strict provisioning](evidence/grafana/reviews/startup-lab6.md). Они относятся к указанным immutable slices; final combined review координируется отдельно. Корректные FAIL и прошлые limitations не переписывались.
+47
View File
@@ -0,0 +1,47 @@
# OpenObserve helper entrypoints and callers
All maintained new automation is TypeScript/Bun or thin POSIX wrappers. Python check_contract.py/validate_openapi.py and tests/ remain unchanged upstream tools. The application Go backend is a separate preserved95-file manifest; migration helpers do not edit it.
| Entrypoint or family | Actual consumer and purpose |
|---|---|
| scripts/up.sh | Operator start; preflight images → application/bootstrap owner → build → start stores/OO → wait → provision strict accounts/render → start core/Collector/query gateway → wait both APIs → Grafana → bounded native readiness |
| scripts/compose.sh | Every lifecycle wrapper; pins absolute compose.yaml and project mtuci-vp6-medvedev |
| scripts/stop.sh, restart.sh, stop-one.sh | Operator/upstream resilience callers; only pinned own project; stop preserves volumes |
| scripts/bootstrap.ts | up.sh; preserved application generated private connection files |
| scripts/bootstrap-openobserve.ts | up.sh; own owner initializer, generated password-policy-safe private files; no Grafana owner credential |
| scripts/preflight-openobserve-images.ts, wait-openobserve.ts, wait-grafana.ts | up.sh; exact cached image/plugin hashes, compatible10s scheduler groups, native health/version; bounded cold Grafana wait600s |
| scripts/bootstrap-grafana.ts → lib/grafana-credential.ts | up.sh; required ports preserve existing owner or encrypt/verify a new generated password before runtime publication; age identities/ciphertext remain private |
| scripts/provision-openobserve-accounts.ts | up.sh; native fixed-localhost account HTTP adapter and required bootstrap factory; raw/partial capture → per-account checkpoint → strict capability proof → canonical/derived publication |
| scripts/lib/openobserve-account-http.ts | Provisioner and HTTP tests; mandatory request/capture ports, bounded response, no automatic POST retry |
| scripts/lib/openobserve-bootstrap.ts | Provisioner and bootstrap tests; mandatory persistence/create/verify/publication ports; no production process or Docker side effects inside tests |
| scripts/render-openobserve-credentials.ts | Provisioner only after both principals verify; idempotent exact-value files, mismatch fails closed |
| scripts/check-backend-unchanged.ts | Author/operator verification; hashes all95 current backend files against BACKEND-READY.json |
| scripts/image-manifests.ts | Explicit metadata-only provenance operation; pinned registry manifest/config descriptor discovery, no layer pull |
| scripts/discover-openobserve-cache.ts, stat-openobserve-cache.ts, copy-openobserve-cache.ts, complete-openobserve-cache.ts, prepare-cache-import.ts | Explicit one-time author cache provenance/reuse operations; public exact blobs from stopped owned-source node, own destination and raw SHA/size checks; not required for ordinary startup |
| scripts/inspect-openobserve-start.ts | Explicit author startup diagnostic; own container metadata and private logs, allowlisted safe outcome |
| student-tests/openobserve-account-http.test.ts, openobserve-bootstrap.test.ts | bun test; actual factory behavior with mandatory fake ports; never production entrypoint with optional fakes |
| student-tests/wait.ts, both-replicas.ts | up.sh/readiness and explicit fault acceptance; actual own API replicas/history |
| student-tests/check-openobserve-config.ts | Explicit cached Collector native validate against historical and current configs; no network; config validation does not prove export |
| student-tests/openobserve-readback.ts, openobserve-signals.ts | Explicit native schema/all3 application signal acceptance; creates isolated author receipts, queries actual own storage |
| student-tests/openobserve-gateway.ts, openobserve-gateway-boundaries.ts | Actual HAProxy/nativeOO allow/deny, duplicate/parser/window/chunked cases; intended small requests |
| student-tests/openobserve-provisioned-queries.ts | Explicit23 actual query syntax/content checks; HTTP200 alone is not positive-value/Grafana acceptance |
| student-tests/openobserve-historical-queries.ts, openobserve-cohort-query.ts | Retained-generation diagnostic readbacks for idle/active windows and native instrument labels; original evidence preserved |
| student-tests/openobserve-positive-queries.ts | Actual provisioned expressions + labels + matured/timely controls on the retained known workload;17 bounded positive cases, no new producer |
| student-tests/openobserve-network-run.ts → openobserve-network.ts | Actual cached unprivileged query-SA client on configured Grafana network; query positive and rawOO/hostloopback/Collector negative; Grafana binary not run |
| student-tests/openobserve-ui-port.ts → openobserve-ui-listener.ts | Owned cached listener on actual future Grafana network/28306; native publication check and exact labelled container stop; not Grafana acceptance |
| student-tests/openobserve-collector-snapshot.ts | Explicit before/down/recovered bounded telemetry outage snapshots; queue/cgroup/RSS counters kept distinct |
| student-tests/openobserve-persisted.ts | Explicit before/after retained trace/log/metric fingerprint; no replacement producer; fresh volume cannot satisfy historical fixture |
| student-tests/grafana-credential.test.ts, grafana-scheduler.test.ts, incident-readback.test.ts, owned-fault.test.ts | Bun unit tests of actual factories/oracles with explicit fake ports and retained RED/GREEN |
| student-tests/grafana-preflight.ts | Actual server/datasource health, provisioned18-panel dashboard and5 native rules; metadata only, not visual or firing acceptance |
| student-tests/grafana-response-boundary.ts | Explicit owned native HAProxy + synthetic raw upstream;14 framing classes, distinct ACL/no-marker HTTP outcomes |
| student-tests/grafana-native-histogram.ts | Actual browser omitted-size histogram query compared through native OO and gateway;9 paired cases, no query rewriting |
| student-tests/grafana-incidents.ts | Explicit native owned worker/DB fault experiment; original CLI FAIL retained separately from corrected causal readback |
| student-tests/grafana-incident-readback.ts | Read-only same captured snapshots/jobs with immutable hashes; requires owned accepted cohort and finite queue samples |
| student-tests/grafana-incident-trace-readback.ts | Read-only actual captured app503 request_id→HTTP log→DB error trace; no producer/fault replay |
| student-tests/grafana-retry.ts → grafana-retry-readback.ts | Exactly one owned synthetic staged-object visibility fault, durable retry→ready, restoration/session revocation, then read-only same-trace parents/logs/counter evidence |
| scripts/lib/browser-lifecycle.ts, student-tests/browser-lifecycle.test.ts | Required-port lifecycle; setup/run protected immediately after acquire, close precedes result reporting, primary/secondary failures preserved;10 native fake-port cases |
| student-tests/grafana-browser.ts | Explicit3-scenario authenticated native UI; rejects partial filters, captures statuses/raw query responses before decoding, preserves static cancellation separately; no producer |
| scripts/package.ts | Explicit --build creates a new source-only ZIP/staging/manifest; excludes private/cache/build state and refuses symlinks; separate native unzip verifier checks every hash |
| student-tests/openobserve-instant-time.ts | Native read-only vector(1) grammar provenance; no timestamp shim |
Generated runtime inputs are `.local/*`, never archive/build inputs. Grafana datasource, dashboard and rules are committed source under config/grafana, not self-modifying generators. Its environment file is rendered only after strict account verification. Config/raw cache historical copies and legacy RED extraction text are evidence, not active imports. Source manifests enumerate the exact reviewed bytes; later edits need their own delta receipt.
+65
View File
@@ -0,0 +1,65 @@
{
"checkedAt": "2026-11-16T00:09:42.321Z",
"scope": "official immutable public OCI manifest metadata; no new layers",
"images": [
{
"id": "openobserve",
"sourceRef": "public.ecr.aws/zinclabs/openobserve:v1.1.2@sha256:4a154849432f75c4fb024de32fe34163c02406c905dab22bf8d213d2e3386392",
"localAlias": "docker.io/library/mtuci-vp6-openobserve-cache:v1.1.2",
"linuxAmd64Manifest": "sha256:26618f379c86b253c0d91624c487dd14b71293699845ecbc8457c94dca14819f",
"configDigest": "sha256:51138955364cbef4d1d47aa90d5396c0517c9e53bb71fcd133ad41e5d923add5",
"configBytes": 5184,
"compressedLayerBytes": 161805957,
"compressedDescriptorBytes": 161811141,
"platform": "linux/amd64",
"dockerCachePresent": true,
"layerDownloadPerformed": false
},
{
"id": "collector",
"sourceRef": "otel/opentelemetry-collector-k8s:0.163.2@sha256:cce4142919331aa26dd9e2900b0583dac12c44d2f87c270d3a2110912d4785a9",
"localAlias": "docker.io/library/mtuci-vp6-collector-cache:0.163.2",
"linuxAmd64Manifest": "sha256:a65df9129a5b3f99ee4434c5e53aee1965ff0129eda6a2da75ea917165b7fa6b",
"configDigest": "sha256:bd290dfd15ea1639718a22236307688e62063659f383371d26b6785421efe465",
"configBytes": 2024,
"compressedLayerBytes": 51498969,
"compressedDescriptorBytes": 51500993,
"platform": "linux/amd64",
"dockerCachePresent": true,
"layerDownloadPerformed": false
},
{
"id": "grafana",
"sourceRef": "grafana/grafana:13.3.1@sha256:1c88285f1dd391a62232c010e6c224846e6a73bdc583b3365be2eba614b7919e",
"localAlias": "docker.io/library/mtuci-vp6-grafana-cache:13.3.1",
"linuxAmd64Manifest": "sha256:1c88285f1dd391a62232c010e6c224846e6a73bdc583b3365be2eba614b7919e",
"configDigest": "sha256:bc5ca66188ced0ed137172d30a53944c7f13e183c099cae842da777103feedaa",
"configBytes": 6672,
"compressedLayerBytes": 506200855,
"compressedDescriptorBytes": 506207527,
"platform": "linux/amd64",
"dockerCachePresent": false,
"layerDownloadPerformed": false
}
],
"plugin": {
"repository": "Windforce17/openobserve-grafana-plugin",
"version": "1.0.1",
"commit": "e10214739827dda9318705d49db262b28d67bcfe",
"assetBytes": 52781711,
"assetSHA256": "e0c2cbcc7cf6d05bcfaa606db39e52249eb0dd234376e45a3f896353831c2fd5",
"downloadedByThisCheck": false
},
"metadataReceipt": "/mnt/data/home/bench-saves/mtuci-vp-remaining-20261115/lab6/.local/image-manifests-dea5f79c-4f7a-4f4b-9fb8-a8b18d18362a",
"currentRuntimeEvidence": {
"metadataFieldsAbove": "Historical metadata-only observation; no layer download occurred in that original check",
"allThreeCachedAndNativeVerified": [
"OPENOBSERVE-IDENTITIES.json",
"evidence/grafana/cache-ready.json",
"evidence/grafana/final-source-checks.json"
],
"approvedGrafanaPluginNetworkBytes": 559001001,
"approvedBudgetBytes": 600000000,
"pluginUnchanged": true
}
}
+3
View File
@@ -0,0 +1,3 @@
# OpenObserve migration
The completed migration specification and its historical case table are archived in [docs/archive/openobserve-migration.md](docs/archive/openobserve-migration.md). Current measured results, limits and acceptance are in [REPORT.md](REPORT.md) and [RUNBOOK.md](RUNBOOK.md). Earlier gates in the archived specification are historical; they are not current status.
+142
View File
@@ -0,0 +1,142 @@
{
"status": "Native SDK/Collector/OpenObserve, actual Grafana18-panel UI/canonical+retry traces/links, real DB+worker alert transitions and fresh-volume startup verified; source/artifact independent final review in progress",
"checkedAt": "2026-11-16T04:59:01.166Z",
"backendSourceChanged": false,
"openobserve": {
"version": "1.1.2",
"manifest": "sha256:26618f379c86b253c0d91624c487dd14b71293699845ecbc8457c94dca14819f",
"organization": "default",
"storage": "isolated persistent local disk"
},
"collector": {
"version": "0.163.2",
"distribution": "opentelemetry-collector-k8s",
"manifest": "sha256:a65df9129a5b3f99ee4434c5e53aee1965ff0129eda6a2da75ea917165b7fa6b",
"receiver": "http/protobuf :4318",
"exportEndpoint": "http://openobserve:5080/api/default",
"authorization": "file provider Basic scalar, exact bytes without CR/LF; no root/admin in Grafana",
"logsAndTracesHeader": {
"stream-name": "messenger"
},
"metricsHeaderStreamName": false,
"queueBytesPerSignal": 8388608,
"maxRetryElapsedSeconds": 5,
"timeoutSeconds": 2,
"memoryLimiterMiB": 128,
"containerLimitMiB": 256
},
"logs": {
"stream": "messenger",
"timestamp": "_timestamp",
"body": "body",
"event": "event",
"requestId": "request_id",
"traceId": "trace_id",
"spanId": "span_id",
"service": "service_name",
"severity": "severity",
"outcome": "outcome"
},
"traces": {
"stream": "messenger",
"traceId": "trace_id",
"spanId": "span_id",
"parentSpanId": "reference_parent_span_id",
"parentTraceId": "reference_parent_trace_id",
"operation": "operation_name",
"service": "service_name",
"status": "span_status",
"start": "start_time",
"end": "end_time",
"duration": "duration",
"timestamp": "_timestamp",
"units": {
"start_time": "nanoseconds",
"end_time": "nanoseconds",
"duration": "microseconds",
"_timestamp": "microseconds"
}
},
"metrics": {
"nativePrometheusEndpoint": "/api/default/prometheus",
"counterNames": [
"http_requests",
"jobs_processed",
"database_observer_failures",
"process_cpu"
],
"histogramBases": [
"http_duration",
"job_duration",
"time_to_ready",
"db_pool_acquire_duration"
],
"histogramSuffixes": [
"_bucket",
"_count",
"_sum",
"_min",
"_max"
],
"gauges": [
"queue_depth",
"queue_oldest_age_seconds",
"db_pool_used",
"db_pool_capacity",
"db_pool_pending_acquires",
"worker_busy",
"worker_capacity",
"go_heap_bytes",
"process_resident_memory_bytes",
"database_observer_up",
"ready_slo_matured_10s",
"ready_slo_matured_30s",
"ready_slo_matured_300s",
"ready_slo_timely_10s",
"ready_slo_timely_30s",
"ready_slo_timely_300s"
],
"resourceLabels": [
"service_name",
"service_instance_id",
"service_version",
"deployment_environment_name"
],
"noLegacyExporterNamespace": true,
"noAutomaticTotalCounterSuffix": true,
"instrumentStartTimeLabel": {
"label": "start_time",
"semantics": "native per-instrument OTLP start time; differs between matured/timely/observer instruments",
"arithmeticMatching": [
"service_name",
"service_instance_id"
],
"positiveEvidence": "evidence/openobserve/positive-queries-green.json"
}
},
"evidence": "evidence/openobserve/schema-discovery.json",
"runtimeGrafana": true,
"runtimeAlerts": true,
"runtimePersistence": true,
"independentReview": false,
"nativeSignalEvidence": "evidence/openobserve/native-signals.json",
"exampleTraceId": "ca31fa667ee8a9b15a40d3cc904f6683",
"observedSpanCount": 42,
"observedParentRelations": 41,
"persistenceEvidence": "evidence/openobserve/persistence-after.json",
"queryScopeEvidence": [
"evidence/openobserve/parser-green.json",
"evidence/openobserve/network-published-green.json"
],
"runtimeFreshVolumes": true,
"nativeGrafanaEvidence": [
"evidence/grafana/dashboard-panels.json",
"evidence/grafana/both-native-links.json",
"evidence/grafana/provisioning-strict-native-green.json",
"evidence/grafana/retry-telemetry.json",
"evidence/grafana/incidents-causal-readback.json",
"evidence/grafana/fresh-volume-green.json"
],
"browserNetworkStatus": "Three functional scenarios and console0; two aggregate browser commands remain FAIL due to nonblocking canceled sync.svg static fetch. Single target CDP identifies CacheAPI fetchFromPersistentCache; exact cleanup trigger unknown, no ERR_ABORTED whitelist.",
"independentReviewScope": "Partial source/config reviews completed; final combined checkpoint pending"
}
+125
View File
@@ -0,0 +1,125 @@
{
"status": "Native dedicated service accounts and gateway scope verified; actual Grafana runtime/datasources/strict provisioning and fresh-volume path verified",
"checkedAt": "2026-11-15T23:13:48.522Z",
"sourceCommit": "d16d693ba99bc20621a58b948ecbcad7ec7fa882",
"ownerBootstrap": {
"environmentKeys": [
"ZO_ROOT_USER_EMAIL",
"ZO_ROOT_USER_PASSWORD"
],
"passwordPolicy": {
"minCharacters": 8,
"maxCharacters": 128,
"requires": [
"ASCII lowercase",
"ASCII uppercase",
"ASCII digit",
"one other character"
]
},
"generator": "Aa1! plus two random UUIDs; actual credential private, never printed"
},
"nativeCreate": {
"method": "POST",
"path": "/api/{org}/service_accounts",
"request": {
"email": "unique synthetic email",
"first_name": "label",
"last_name": "label"
},
"responseFields": [
"code",
"message",
"token",
"user"
],
"verifiedHTTP": 200,
"verifiedCode": 200,
"tokenCapture": "same process, exclusive file creation, private parent0700/file0600 before readback",
"persistedRole": "service_account",
"revokeRoute": "DELETE /api/{org}/service_accounts/{email}",
"adminRotateRoute": "PUT /api/{org}/service_accounts/{email}?rotateToken=true"
},
"doNotUseHumanUserCreate": {
"path": "POST /api/{org}/users",
"reason": "OSS handler forcibly assigns Admin even if viewer/user requested"
},
"measuredRights": {
"ingestPrincipal": {
"role": "service_account",
"searchHTTP": 200,
"ingestHTTP": 200,
"adminRotateHTTP": 403
},
"queryPrincipal": {
"role": "service_account",
"searchHTTP": 200,
"ingestHTTP": 200,
"adminRotateHTTP": 403
},
"nativeReadOnly": false,
"nativeIngestOnly": false
},
"collectorCredential": {
"principal": "dedicated ingest service_account",
"header": "Authorization",
"format": "Basic base64(user:token), exact raw file bytes without terminal newline or CR/LF",
"fileSubstitution": "${file:/run/secrets/openobserve_ingest_auth}",
"fileOutsideGitAndBuildContext": true,
"reload": "restart Collector after credential update; config parse does not prove export succeeds"
},
"grafanaCredential": {
"principal": "dedicated query service_account, never root/admin",
"backendSettings": [
"basicAuth=true",
"basicAuthUser",
"secureJsonData.basicAuthPassword"
],
"queryGatewayRequired": true,
"doNotLabelNativeRoleReadOnly": true,
"nativeEvidence": "evidence/grafana/provisioning-strict-native-green.json"
},
"images": {
"openobserve": {
"alias": "docker.io/library/mtuci-vp6-openobserve-cache:v1.1.2",
"imageId": "sha256:26618f379c86b253c0d91624c487dd14b71293699845ecbc8457c94dca14819f",
"version": "1.1.2",
"architecture": "linux/amd64"
},
"collector": {
"alias": "docker.io/library/mtuci-vp6-collector-cache:0.163.2",
"imageId": "sha256:a65df9129a5b3f99ee4434c5e53aee1965ff0129eda6a2da75ea917165b7fa6b",
"version": "0.163.2",
"distribution": "k8s",
"architecture": "linux/amd64"
},
"probePullPolicy": "never; normalized alias/digest verified through native inspect before run"
},
"evidence": "evidence/openobserve/native-identities.json",
"sourceFiles": [
"src/config/src/utils/password.rs",
"src/api/management/src/request/users/mod.rs",
"src/api/management/src/request/service_accounts/mod.rs",
"src/common/src/meta/service_account.rs",
"src/common/src/meta/user.rs"
],
"bootstrapRecovery": {
"perAccountCheckpoint": ".local/openobserve-account-{ingest,query}.json",
"intent": ".local/openobserve-account-{ingest,query}.intent.json",
"rawResponse": "claimed operation/create-{ingest,query}.raw.private",
"uncertainPOST": "never retried automatically; recover captured native JSON and reverify live, otherwise fail closed",
"requiredReadback": {
"searchHTTP": 200,
"ingestHTTP": 200,
"adminRotateHTTP": 403,
"role": "service_account"
},
"firstBoot": "owned identity_probe ingest precedes exact-event search; no application stream needed",
"actualCaller": "scripts/up.sh -> scripts/provision-openobserve-accounts.ts -> required-port createAccountBootstrap -> renderer only after both identities pass"
},
"gatewayEvidence": [
"evidence/openobserve/gateway-green.json",
"evidence/openobserve/parser-green.json",
"evidence/openobserve/network-published-green.json"
]
}
+122 -23
View File
@@ -1,37 +1,136 @@
# Отчёт — лабораторная 6
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
Медведев Андрей Михайлович, БПИ2303. Цель — уровни3–4 и трек2 уровня5 «SLO и обнаружение инцидентов». Текущий checkpoint содержит приложение, SDK, Collector/OpenObserve и настоящий Grafana13.3.1:18 панелей, trace42 spans/41 parents, оба UI links и реальные переходы алертов при отказах PostgreSQL/worker. Fresh-volume startup, исходный smoke, реальные faults/retry и native source checks выполнены; final artifact и независимый итоговый обзор готовятся. Оценка и принятие преподавателем не заявляются.
## Запуск из чистого clone
## Архитектура и сохранённая основа
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
По решению владельца Prometheus/Loki/Tempo заменены OpenObserve1.1.2. Две API, worker, PostgreSQL18.6, Valkey9, RustFS и HAProxy сохраняются. API/worker передают native OTLP HTTP/protobuf в отдельный Collector-k8s0.163.2, затем все три сигнала идут в изолированную организацию default OpenObserve. Его локальный persistent volume независим от БД приложения; это временный инженерный default, а не заимствование Azur S3/Litestream.
## Архитектура и решения
Backend95 остался побайтно прежним на Mac и Linux. Исходный manifest — BACKEND-READY.json; текущая проверка95/95 находится в evidence/openobserve/backend-unchanged.json. Ранее проверенные Go1.27.2 race/vet/lint/build/format, настоящий OTLP protobuf, PostgreSQL SLO cohort/pool regressions остаются исторической основой. Новые запросы к storage проверены отдельно. Архив5 и source112 лабораторной7 не менялись.
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
Перед миграцией49 config/scripts/docs/evidence файлов сохранены и проверены в private `.local/history/pre-openobserve-2026-11-15T224455797Z`. Прежние storage-конфиги перенесены в docs/archive/pre-openobserve и не используются текущими callers. Общий контракт включён внутрь пакета в docs/reference/OPENOBSERVE-BOUNDARY.md с проверенной provenance.
## Доказательства критериев
## Фактические результаты
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Все runtime операции выполнялись на Linux omarchy в `/mnt/data/home/bench-saves/mtuci-vp-remaining-20261115/lab6`, project `mtuci-vp6-medvedev`, в background.slice с низкими CPU/IO weights. Это проверки поведения, не бенчмарки.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Проверка | Наблюдение | Доказательство |
|---|---|---|
| Настоящие API/worker → Collector → OpenObserve | Все3 сигнала; одна цепочка42 spans/41 parents и3 связанных лога API/worker с request_id | evidence/openobserve/native-signals.json, native-signals-sa.json |
| Native schema/PromQL | Реальные имена без messenger_ и автоматических _total/_seconds; histogram p95 вычислен из buckets | schema-discovery.json, OPENOBSERVE-CONTRACT.json |
| Query identity | Два service_account, search/ingest200, административная rotation403; native роль не read-only | native-identities.json, OPENOBSERVE-IDENTITIES.json |
| Штатный HAProxy gateway | Query/search/editor/PromQL доступны; ingest/admin/иная организация запрещены; лимиты запроса и parser cases проверены | gateway-green.json, parser-green.json |
| Подготовленные панели/правила |23 native parse/content checks; отдельные17 positive controls проверяют реальные labels, конечные histogram percentiles, error/burn0 и зрелую когорту1/1 | provisioned-queries.json, positive-queries-red/green.json; это не Grafana evaluation |
| Отказ OpenObserve | Процесс не работал примерно114,541с; полный публичный API smoke с async image/original/thumbnail прошёл | runtime-checks.json, collector-before/down/recovered.json |
| Очередь/память Collector |177781 байт во время отказа, после восстановления0; capacity25165824 байт суммарно по3 сигналам; working memory36,78 →36,88 →41,36МиБ при hard limit256МиБ, OOMfalse | collector snapshots; RSS является отдельной метрикой |
| Persistent history | Прежние42 spans,3 лога и point-in-time HTTP counter127 сохранились с тем же fingerprint bf0e595e… | persistence-before.json, persistence-after.json |
| Обе API реплики | Каждая отдельно остановлена/восстановлена; сессия и4 подтверждённых сообщения сохранены, дублей нет | both-replicas.json |
## Эксперименты и отказы
Observed checks27/35ms при остановке реплик относятся только к конкретному fault-сценарию после stop, а не к performance SLO. У отказа telemetry зафиксированы exporter send_failed counters; они не объявляются точным числом уникальных потерянных событий. После восстановления очередь пуста, новые сигналы доставляются. Конечный тест и hard container limit не доказывают отсутствие OOM при произвольной нагрузке.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
## Имена, единицы и корреляция
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
OPENOBSERVE-CONTRACT.json и TELEMETRY-SCHEMA.json описывают actual mapping. Counters называются http_requests/jobs_processed/process_cpu; histogram families — http_duration, job_duration, time_to_ready, db_pool_acquire_duration, их _bucket/_count/_sum. Duration instruments измеряются в секундах. Native trace start_time/end_time — наносекунды, duration/_timestamp — микросекунды; проверены численно по настоящим строкам. Parent — reference_parent_span_id. Logs имеют body/event/outcome/service_name/request_id/trace_id/span_id.
## Ограничения и следующие шаги
Пример сохранённой цепочки — ca31fa667ee8a9b15a40d3cc904f6683. В ней есть API upload, postgres.acquire/query, s3.put/get/stat, attachment.publish и attachment.process. Секреты и тела сообщений не являются telemetry attributes. request_id присутствует в HTTP-логе и не становится metric label. Resources нормализованы в service_name/service_instance_id/service_version/deployment_environment_name. Native start_time различается между OTLP instruments: SLO subtraction/division и availability joins явно сопоставляют service_name/service_instance_id. Исходный empty-series RED сохранён;17 positive controls на прежнем workload прошли, включая finite p50/p95/p99 и зрелую когорту1/1. Idle NaN/NoData не объявляются успешным percentile.
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Native instant PromQL проверен с отсутствующим time, integer/fractional Unix seconds, milliseconds/microseconds и RFC3339: все200/status success. Дробные Unix seconds parser этой версии усекает до целой секунды. Это отдельно от SQL start_time/end_time, которые должны быть целыми микросекундами; для persisted readback они получены через BigInt из сохранённых span timestamps. evidence/openobserve/instant-time-grammar.json.
## Доступ и ограничения
Начальный owner нужен только bootstrap/operator reader. POST `/api/{org}/users` в Community принудительно назначает Admin, поэтому Grafana так не создаётся. Используется native POST `/api/{org}/service_accounts` с email/first_name/last_name; token/user сохраняются вне git в том же процессе. Native SA позволяет и read, и ingest, поэтому права query principal дополнительно ограничивает HAProxy по identity, методам и exact paths. Root/admin не передаются Grafana. Grafana состоит только в отдельной сети grafana-query-v2 с HAProxy; raw OpenObserve и Collector к ней не подключены. Сетевой RED воспроизвёл raw read/write и Collector bypass; final GREEN сохранил query200 с реальной строкой, raw-IP/read/write/host-loopback-route/Collector недоступны из этой сети. Operator edge остаётся для опубликованных loopback endpoints. Доступность UI28306 теперь проверена настоящим Grafana и authenticated Chromium. Anonymous Viewer сохранён; обычная форма owner login временно включена как default, без расширения ролей.
Collector получает dedicated ingest SA через file-provider Authorization; файл содержит точные Basic bytes без CR/LF. Есть memory_limiter128МиБ, batch64/128,8МиБ queue на каждый сигнал и retry максимум5с. SDK queues128, batches32, export timeout1с, retry выключен, metric interval2с. Эти SDK параметры не менялись.
Gateway допускает native search requested size0–1000 либо отсутствие size, SQL time window до3600с, _values size1–1000 и окно до3600с, request inspection64КиБ. Это лимиты параметров запроса, не универсальный предел SQL результата: native SQL LIMIT2 при query.size1 возвращает2 строки. Provisioned SQL должен иметь явный LIMIT; отдельно подготовлен Grafana data-proxy response cap1МиБ, OpenObserve query timeout5с, DataFusion budget256МиБ, metrics series/points bounds и gateway concurrency/queue limits. Plugin Go SDK отклонил2МиБ response при1048576, однако browser proxy вернул2097625байт. Поэтому существующий HAProxy принудительно запрашивает identity и пропускает только распознанный одиночный normalized Content-Length≤1048576 без Content-Encoding/Transfer-Encoding. После reload тот же browser request получил502/35байт, ordinary metadata/search/Prom controls остались успешными.14 native framing classes проверены отдельно; это bounded known-Content-Length контракт, не универсальный streaming/decompression counter. Public tag и deployed image имеют разные commit, поэтому исходная причина browser setting остаётся unknown.
Prometheus range gateway консервативно допускает floor(end)-floor(start) от1 до3599с: дробные timestamps работают, реальное окно строго меньше3600с, same-second/inverted ranges отклоняются. Duplicate protected URL parameters отклоняются400, duplicate JSON keys native backend отклоняет422. Complete chunked JSON проходит; oversized body отклонён413, incomplete408, delayed prefix+padding после2,2с закрылся408. Первая узкая oracle ожидала только413; её FAIL сохранён отдельно, bypass200 не наблюдался.
Фабрика account HTTP с обязательными fake ports воспроизвела parse-before-capture RED4fail/1pass и дала GREEN5pass/22expect на Linux Bun1.4.4. Сырой/частичный ответ теперь записывается в wx0600 private file до JSON parsing; network/body/cap uncertainty сохраняется, POST автоматически не повторяется. ACCOUNT-HTTP-CASES.md и evidence/openobserve/account-http-*.
## SLI/SLO и подготовка Grafana
SLI API — доля бизнес-запросов без5xx, health исключён; учебная цель99%. Readiness SLI — доля accepted files, впервые ready≤10с; цель99% в5-минутной созревшей когорте. Immutable accepted_at/first_ready_at не переписываются Replay/Advance. Failed/stuck входят в denominator, legacy без acceptance исключены. Time-to-ready histogram измеряет завершённые попытки; он не заменяет denominator.
Панели показывают RPS/errors/p50/p95/p99, job outcomes/duration, backlog/age, DB used/pending/capacity, worker saturation, memory/CPU и readiness cohort. Instant gauges старше10с исключаются, чтобы старый service.instance.id после рестарта не удерживал ложный backlog/SLO. Гистограммы при нулевой активности могут давать NaN; это не подменяется нулевой latency.
Подготовлены Grafana-managed high-error, queue-lag, database-observer и два multiwindow burn alerts10/30с с for15с. Bool-сумма двух окон даёт0/1/2, оба плохих окна означают2 и threshold>1,5. NoData/Error не считаются здоровым SLO. Правила загружены и исполняются настоящим Grafana. Native scheduler отверг1с group и min_interval1с; обе неуспешные попытки сохранены. Canonical group10с работает без experimental flags. Фактические переходы и измерения приведены ниже.
Metrics datasource — native Prometheus type с явным jsonData.httpMethod=GET, UID prometheus и URL query gateway `/api/default/prometheus`. Logs/traces — готовый сторонний Windforce17/openobserve-grafana-plugin1.0.1, exact commit e10214739827dda9318705d49db262b28d67bcfe, unsigned только id openobserve. Verified release binary загружен без изменений; actual Chromium отрисовал canonical trace и оба navigation links.42 unique span IDs/41 parents сверены по реальному browser response; raw ns→UI duration проверены с точностью отображения. Официальным OpenObserve plugin он не называется. Native supplementary Logs volume query вообще не передаёт size. Native missing/0 дают корректную агрегацию; узкий9-case RED/GREEN исправил только request policy, не plugin. Optional service-map10000 не входит в принимаемый путь; row-size параметр не объявляется hard byte/work limit. Самодельный shim/plugin не создаётся.
## Запуск и остановка
Проверенная среда: Docker29.9.2, Compose5.6.1, Bun1.4.4, Linux amd64. Новые images сначала требуют разрешения. IMAGE-MANIFEST.json содержит public pinned refs/digests/sizes; текущие OpenObserve/Collector импортированы из проверенного OCI cache в локальные aliases. Повторный `scripts/up.sh` проверен на имеющихся volumes. Каждый полученный account token сохраняется отдельным checkpoint до создания следующего аккаунта. Intent записывается до POST и связывается с raw capture; при неоднозначном исходе POST автоматически не повторяется. Сохранённые tokens обязаны дать search200 с известным event, ingest200 и admin403 до публикации derived файлов/успешного старта;401/503 и другие статусы закрываются. Начальная PostgreSQL readiness использует TCP; Grafana env_file optional при первичном Compose parse, затем provisioner публикует dedicated query env до её старта. up.sh теперь включает Grafana, exact cached image/plugin checks и encrypted-first owner archival. Bounded warm/cold wait600с введён после измеренного fresh deadline120с:641 успешная миграция/ошибок0, HTTP ещё не слушал. Новый first boot с пустыми volumes прошёл целиком; предыдущие две неуспешные попытки сохранены отдельно.
После подготовки проверенных images рабочие команды из корня lab6:
```sh
./scripts/up.sh
bun scripts/check-backend-unchanged.ts
make check
make test BASE_URL=http://127.0.0.1:28206
bun student-tests/openobserve-gateway.ts
bun student-tests/openobserve-provisioned-queries.ts
bun student-tests/both-replicas.ts
./scripts/stop.sh
```
Все Compose helpers закрепляют absolute manifest и project, stop сохраняет volumes. Приложение —127.0.0.1:28206, OpenObserve operator endpoint28406, query gateway28906, Collector OTLP28706/internal metrics28806; Grafana28306. `.local` содержит generated synthetic credentials/cache/private receipts и исключена из git, Docker context и архива. Исходные задания/contracts/public tests сохраняются.
Для нового Docker29 host можно получить exact public amd64 refs из IMAGE-MANIFEST.json и после разрешения загрузить их обычным docker image pull, затем присвоить два проверенных local aliases из OPENOBSERVE-IDENTITIES.json. Никакие Azur данные/credentials/node metadata для развёртывания не нужны. Cache-discovery helpers относятся только к provenance текущего повторного использования и не входят в обязательный startup path.
## Настоящий Grafana и инциденты
Одобренная загрузка ограничена Grafana13.3.1 и готовым OpenObserve plugin1.0.1. Фактически получено559001001байт, меньше600000000; exact SHA/imageID находятся в evidence/grafana/download-ready.json и cache-ready.json. Дополнительные модели, браузер и образы не скачивались. Native health сообщает commit da3aa3ff574e7dbbf808c66b6daa744defed0441; публичный tag13.3.1 указывает5db6ba887630c7fa12e5bfd063f2fa854025468c. Tag readers служат reference, а не доказательством исходников compiled binary. Native startup/auth/failure outcomes остаются измеренными фактами.
Authenticated Chromium проверил18/18 dashboard panels с реальными finite active samples, а не толькоHTTP200; evidence/grafana/dashboard-panels.json и dashboard-panel18.png. Инертные NaN остаются отдельно. Trace33513303fdb8cdefbd4ff24c8394f3d3 визуально содержит API/publish/process/DB/S3,42 spans/41 parents/2 services. API188.030423мс отображается188.03мс, publisher68.126923→68.13мс, worker1153.117322мс→1.15с. См. trace-ui-first-readback.json, trace-first.png и linked-worker-tree.png.
Штатная ссылка в HTTP log details открывает тот же trace. Обратная находится в Span details → View logs for trace; worker span задаёт своё окно1794784791672000–1794784792825000мкс и находит одну matching worker.job строку с тем же trace/span. Исходный более широкий logs view содержит3 записи и HTTP request_id3cfa3390-ed20-4116-aa2f-9edefd0b38ab. Это два разных проверенных окна; требование3 строк для узкого span-window было ошибкой test oracle и его FAIL сохранён. evidence/grafana/both-native-links.json и PNG подтверждают реальные клики, без ручной подстановки обратной ссылки.
Обычный operator login работает. Доставленный в image frontend UserStorage.fetchResource возвращает null именно при missing-state404, а PageHistorySrv.load трактует это как empty и запускает обычное сохранение. На проверенном owner key были два cold GET404/console error; normal dashboard visit после throttle30с сделал POST201, reload того же ключа GET200 и unexpected errors0. Никаких seeding fixtures, vendor flags или blanket404 whitelist не применялось; literal raw console0 не заявляется. Точные route/status/count и bundle/map/source SHA находятся в history-normal-ui.json и history-source-binding.json.
Для SLO-трека использовались исходные5 правил без изменения порогов: group10с, bounded traffic2запроса/с и максимум4 одновременных,34 настоящих accepted uploads. PostgreSQL оставался живым/Ping200; ACCESS EXCLUSIVE lock на messages вызвал реальные application503 с UUID и postgres.query failure, поэтому LB503 не подменял app5xx. Короткий lock4.099с вызвал Pending и восстановился без Alerting. Длительный lock41.594с дал HighError Alerting через20.534с, FastBurn через28.585с; после release они стали Normal через19.126/17.114с. Наблюдения относятся к этому сценарию, не к performance benchmark.
Worker stop оставил обе API ready200. QueueLag стал Alerting через35.878с после остановки. Процесс worker запущен04:15:16.526UTC, ReadyBurn стал Alerting04:15:19.275, пока принятая очередь ещё не обработана; process start не означает бизнес-восстановление.34/34 файла достигли ready за47.89558–55.210172с; последний first_ready04:15:55.958UTC. ReadyBurn Normal04:15:39.294 отражает moving10/30с cohort, а не завершение всех файлов; QueueLag Normal04:16:07.400, финальные queue_depth/age0. Observer краткоPending, но неAlerting; API error rules при worker fault оставалисьNormal. Это observed false-positive check, не статистическая гарантия.
Исходная whole incident command закончиласьexit1 из-за phase-name oracle, который искал ReadyBurn только до запуска процесса. RESULT иVALIDATIONS сохранены как FAIL: incidents-original-command-fail.json и incidents-original-validations.json. Отдельный bounded readback тех же captured jobs/snapshots далexit0: incidents-causal-readback.json; он не повторял faults и не менял thresholds. Исправленная oracle требует complete same-ID accepted cohort и конечное причинное окно; empty queue samples больше не считаютсяdrained. RED3fail/1pass → GREEN4pass/9expect. Worker/PG восстановлены, own fault holders0, синтетическая сессия отозвана; encrypted private captures сохранены, plaintext secrets в evidence отсутствуют.
## Гипотеза и пределы SLO-трека
Учебные defaults99% API availability и99% first-ready≤10с заданы как engineering targets, а не результаты пользовательского исследования или обещанный production SLA.99% даёт бюджет1 неуспешный business request/file из100. Контролируемый обычный upload trace завершался примерно1.39с, поэтому10с оставляют запас для асинхронной доставки/планирования; это проверенный пример, не распределение всей нагрузки. Health исключён из API denominator,4xx не считаются dependency failure. Failed/stuck accepted files входят в readiness denominator.
Гипотеза: сочетание двух окон10/30с с for15с подавляет короткий spike, но обнаруживает длительную деградацию; queue-age alert отдельно обнаруживает worker outage при здоровом API. Burn threshold14.4 означает превышение допустимой1% доли нарушений в14.4раза, то есть более14.4% в обоих окнах. Короткие окна сжаты для ограниченного учебного эксперимента; это не рекомендация production paging policy. Методика и исходные случаи сохранены в student-tests/GRAFANA-CASES.md, исполняемый workload — grafana-incidents.ts.
Одна последовательность healthy→short DB lock→recovery→long DB lock→recovery→worker stop/uploads→recovery подтвердила ожидаемые переходы. Короткий spike действительно ухудшил availability, хотя alert не дошёл до firing: это сознательный обмен скорости обнаружения на меньший шум, а не отсутствие ошибки сервиса. Краткий observer Pending не превратился в ложный firing. Выборка по одному эпизоду не оценивает статистическую частоту ложных срабатываний.
Readiness burn использует moving matured acceptance cohorts, а не очередь незавершённых задач. Поэтому его Normal04:15:39 не доказывает завершение backlog: последняя job ready04:15:55, а QueueLag Normal04:16:07. Runbook требует одновременно проверять actual job facts и свежую queue age. Конечные budgets SDK/Collector не гарантируют бесконечное хранение при outage; send_failed counters не равны уникально потерянным событиям.
## Retry одной задачи
Для отдельного synthetic upload447122be-664f-4fef-8abf-4452676494a0 worker был временно приостановлен. После durable202 проверены точные111817байт staging fixture и сохранён W3C traceparent. Только этот объект временно отсутствовал; S3 и остальные файлы не останавливались/не менялись. Первая попытка записала processing_failed/queued, attempt1; точные байты восстановлены, автоматическая attempt2 опубликовала ready и исходный content остался byte-identical. Session отозвана, объект и worker восстановлены; исходный backend95 не менялся.
Grafana query того же trace3b58d0b8144926719059857bf888bada вернул58 spans с двумя publishers/consumers и правильными parent links, ERRORs3.get и worker.job outcomes retry/ready. Оба jobs_processed outcome counters положительны. Counter измеряет processing attempts, а не число уникальных файлов; retry означает неуспешную попытку с повтором, failed — отдельный terminal outcome, дополнительный permanent-failure job не запускался. Chromium отрисовал оба consumer attempts и красные S3 error children; retry-two-attempts.png/retry-storage-child.png показывают representative first errored attempt и начало следующего publisher, а не все58 строк одновременно. См. retry-owned-job.json и retry-telemetry.json.
## Чистый запуск и итоговые проверки
Fresh project mtuci-vp6-fresh-adf8b95f получил новые credentials и все новые named volumes, используя verified immutable image/plugin cache. В disposable copy согласованно изменены только Compose project/network, pinned compose wrapper и literal ownership guard startup provisioner. Production canonical selectors не ослаблены до inherited environment. Canonical volumes сохранены и остановлены перед повторным использованием своих loopback ports.
Whole command завершиласьexit0: full up.sh → original make check → original public make test → actual fresh all3 signals/parent/ns units → Grafana datasource health/dashboard18/exact5 rules → backend95 unchanged → stop. См. fresh-volume-green.json. Первая attempt отклонила несогласованное имя disposable project доSAcreate; вторая остановилась по120с deadline при продолжающихся успешных SQLite migrations. Обе FAIL сохранены и не переименованы; третья с600с bounded wait началась с новых пустых volumes. Это cold HDD/background correctness run, не benchmark.
Независимый обзор усилил metadata reader: HTTP200 с пустым/wrong UID/duplicate panels/rules теперь отклоняется. RED1pass/2fail → GREEN3pass/11expect; actual strict reader сверил UIDmessenger,18 configured panel IDs/titles и5 exact rule UIDs. Его PASS — metadata/content acceptance, а визуальное подтверждение18 панелей хранится отдельно.
Базовый native checkpoint имел31 tests/88 assertions и58 TS files. После подтверждённого source finding OO6-LIFECYCLE-1 проверены41 tests/136 assertions и все60 maintained TS helpers на Bun1.4.4/TypeScript7.0.2 без сетевых зависимостей; текущие receipts перечислены в evidence/grafana/final-source-checks.json и browser-lifecycle-delta.json. Неизменяемый backend95 сохраняет прежние native Go1.27.2 checks; финальный SHA guard подтверждён отдельно. Original README/COURSE/CONTRIBUTING/Makefile/contracts/tests/examples/CI не изменены. Авторские tests не подменяют независимый review.
## Исправление browser lifecycle по независимому review
В проверяемом helper был подтверждён code path, где newContext/newPage либо ошибка записи RESULT могли миновать close. Это source finding, а не наблюдавшаяся утечка текущего Chromium. Отдельная required-port factory теперь защищает весь setup/run немедленно после успешного acquire, закрывает ресурс до result reporting и сохраняет primary error отдельно от secondary close/report failures. Настоящий entrypoint не вызывается с optional fakes; зависимости тестов обязательны и явно синтетические.
Genuine native RED извлечённого старого порядка дал9fail/1pass, GREEN10pass/48 assertions; полный native suite41pass/136 assertions и typecheck60files PASS. Неизменность business scenario core подтверждена побайтно. Старые native UI/fault/retry receipts сохраняются как результаты прежнего checkpoint; browser, workload и faults после lifecycle fix не повторялись по согласованному ограничению. Текущий source closure подтверждён независимым reviewer; окончательная проверка нового ZIP выполняется отдельно. См. evidence/grafana/browser-lifecycle-delta.json.
## Явный предел browser acceptance
Новый three-scenario Chromium helper действительно выполнил оба canonical log↔trace clicks и retry trace render, console errors/warnings0. Обе aggregate команды оставлены FAIL, потому что browser Network сообщил один canceled static fetch `/public/build/img/icons/unicons/sync.svg`. Bounded single logs-init leaf измерил ERR_ABORTED/canceled:true, runtime initiator fetchFromPersistentCache; независимый GET того же asset дал200/image/svg+xml/343байта. Shipped react-inlinesvg provider использует CacheAPI.add(new Request(...)) с AbortSignal и имеет cleanup abort, но точный cleanup trigger динамически не доказан. Это nonblocking static asset cancellation limitation, не clean-network/wholeUI PASS; vendor не менялся, общий whitelist ERR_ABORTED не вводился. См. browser-three-functional-icon-aborted.json и icon-cancellation-diagnostic.json.
## Готовность пакета
Обязательные функциональные runtime сценарии подготовлены с указанным nonblocking browser пределом. Финальный ZIP формируется после проверки полного source freeze независимым reviewer; CRC, список entries и каждый source SHA проверяются отдельной native unzip командой. Все owned services/browser/listeners остановлены, volumes/private receipts/cache сохранены; подтверждение final-stop.json. Прежние113/8/16-file freezes, неуспешные команды и архивы сохраняются как история. Работа не отправлялась преподавателю; оценка и принятие не выдумываются.
+55
View File
@@ -0,0 +1,55 @@
# Диагностика учебного мессенджера
SLI API — доля бизнес-запросов без5xx, цель99% за минуту; health исключён. SLI обработки — доля принятых файлов, впервые достигших ready за10с, цель99% в5-минутной когорте с уже истёкшим10-секундным сроком. SQL использует immutable accepted_at/first_ready_at; replay/advance не переписывают первый исход. Legacy строки без acceptance не измеряются. Это учебные цели, не обещание production SLA.
Текущий runtime — API/worker → отдельный Collector → OpenObserve1.1.2. Настоящий Grafana13.3.1 проверен с18 dashboard panels, trace42spans, обоими log↔trace UI links и переходами Alerting→Normal при bounded отказах PostgreSQL/worker. Fresh-volume startup с original smoke прошёл; финальные source/artifact review receipts перечислены в REPORT.md. Точные текущие результаты и ограничения находятся в REPORT.md.
## Запуск и безопасная остановка
Команды выполняются из корня lab6 на omarchy. Docker data и builds находятся под `/mnt/data`; используйте background.slice с низким приоритетом по правилам workspace. `./scripts/up.sh` проверяет pinned cached OpenObserve/Collector/Grafana и verified plugin bytes, сохраняет существующий bootstrap, проверяет обе dedicated SA и запускает core и Grafana, сохраняя encrypted owner archive до первого использования. `./scripts/stop.sh` останавливает только закреплённый project `mtuci-vp6-medvedev`, включая его профили, и сохраняет volumes. Не используйте чужой COMPOSE_FILE или unqualified docker compose.
Loopback endpoints: приложение28206, operator OpenObserve28406, query gateway28906, Collector OTLP28706 и internal metrics28806. Grafana28306 запускается up.sh через исторически названный профиль grafana-pending после проверки pinned image/plugin. Обычная форма owner login временно включена; anonymous остаётсяViewer. Raw endpoint28406 нужен operator/bootstrap, он не является адресом datasource Grafana. Grafana состоит только в отдельной grafana-query-v2 с HAProxy; raw storage/Collector в этой сети отсутствуют. Host публикация28306 и UI проверены настоящим authenticated Chromium.
## Поиск причины инцидента
1. В Messenger dashboard проверьте RPS,5xx ratio и p50/p95/p99. Counter называется http_requests; histogram — http_duration_bucket. Фактические labels: method/route/status_class/service_name.401 при неверном пароле не означает отказ БД. Отсутствующий5xx series даёт0 только в числителе при существующем контроле нагрузки; потеря метрик не объявляется здоровой нагрузкой.
2. В Grafana Explore выберите OpenObserve Logs and Traces. Логи используют stream messenger и event/service_name/request_id/trace_id/span_id. Найдите конкретный HTTP trace_id из лога и откройте trace_id query того же stream. Канонические start_time/end_time — наносекунды, parent — reference_parent_span_id. Optional duration-filter UI готового plugin имеет известную несовместимость единиц и в принимаемый путь не входит. Для SQL mode native duration>5000 означает более5мс.
3. В цепочке upload найдите attachment.publish → attachment.process, postgres.acquire/query и s3.put/get/stat. Job переносит W3C traceparent/tracestate, поэтому worker продолжает trace API. request_id есть в HTTP-логе, но не является metric label. Тела сообщений, пароли и tokens не экспортируются.
4. При MessengerQueueLag проверьте queue_depth/queue_oldest_age_seconds, worker_busy/capacity и новые consumer spans. Queue age берётся из created_at текущей attachment_jobs версии. Она не начинается во время HTTP upload и не наследует возраст initial acceptance после advance.
5. При database_observer_up=0 либо отсутствии свежих observer/cohort samples не трактуйте прежние gauges как здоровый SLO. Dashboard/rules фильтруют samples старше10с. HTTP/process metrics продолжают экспорт; postgres.acquire, db_pool_pending_acquires и db_pool_acquire_duration показывают ожидание пула. Alert MessengerDatabaseObserverUnavailable отделяет потерю наблюдения от нулевой очереди.
6. Для ready SLO проверьте matured/timely cohort и первый COMMIT, а не только histogram завершённых попыток. time_to_ready измеряется до post-commit staging cleanup; failed/stuck остаются в denominator. OpenObserve добавляет разный start_time label каждому instrument, поэтому arithmetic и observer joins используют on(service_name,service_instance_id). Для общего shared DB берётся max копий cohort, а не их сумма.
7. После восстановления worker/БД проверьте уменьшение backlog и свежие gauges, повторите тот же запрос пользователя и сохраните UTC timestamps. Фактические переходы и UTC timestamps находятся в REPORT.md и evidence/grafana/incidents-causal-readback.json. Исходная whole command exit1 сохранена отдельно из-за phase-name oracle; запуск процесса не означает готовность backlog. Старые23 native syntax checks и17 positive controls не подменяют реальные Grafana receipts.
## Отказ telemetry и восстановление
OpenObserve имеет отдельный named volume и не является зависимостью успешного сообщения/обработки изображения. Настоящий bounded outage примерно114,541с прошёл с публичным API smoke; Collector queue после восстановления вернулась177781 →0байт, OOM=false. Это не доказательство отсутствия потерь telemetry или работы при бесконечном отказе.
SDK: очереди128, batch32, export timeout1с, retry выключен. Collector: memory_limiter128МиБ, hard container limit256МиБ, batch64/128,8МиБ queue на каждый из3 сигналов, retry максимум5с, timeout2с. Native queue capacity суммарно25165824байт. Exporter failed counters отражают попытки/components, а не точное количество уникально потерянных событий. Не сравнивайте рабочую память cgroup с process RSS как одну и ту же метрику.
Для проверки сохранности использовалась одна прежняя trace/log/metric generation без нового producer:42 spans,3 correlated logs и point-in-time HTTP counter127. Fingerprint до/после рестарта совпал. `student-tests/openobserve-persisted.ts` читает исторический fixture текущего стенда; на новом пустом volume этот fixture отсутствует и не должен выдавать PASS.
## Учётки и частичный bootstrap
Native Community service_account умеет и query, и ingest; это не native read-only роль. Два отдельных SA используются Collector и Grafana, gateway ограничивает query methods/paths/org/window. Owner/root не передаётся Grafana. Authorization-файл Collector содержит точные Basic bytes без CR/LF; после изменения credential нужен restart Collector и actual authenticated export.
`up.sh` вызывает production provisioner с отдельной required-port factory. Он сохраняет intent до POST, raw/partial response до JSON parse, затем checkpoint каждого token до следующего create. Только после live role=service_account, ingest200, search200 с известным собственным event и admin-rotation403 для обеих SA публикуются canonical/derived credentials. Проверка сначала создаёт свой identity_probe event и не требует messenger stream на первом запуске.
Private records находятся в `.local/openobserve-account-ingest.json`, query.json и соответствующих `.intent.json`; intent ссылается на claimed operation/create-*.raw.private. Повторный запуск использует checkpoint либо валидный captured native response и перепроверяет token. Если ответ не восстановим или token получил401/503/неверный admin status, запуск завершается ошибкой до core startup. Не удаляйте intent и не повторяйте account POST: сохраните private каталог и разберите неоднозначный исход. Автоматическая rotation/takeover не выполняется; несовпадающие generated credentials также закрываются без перезаписи.
## Границы безопасности и объёма
HAProxy query gateway проверяет identity, методы и exact paths. Search requested size0–1000 либо отсутствиеsize и окно до3600с не гарантируют число возвращённых SQL строк: native SQL LIMIT имеет приоритет. Provisioned SQL должен иметь явный bounded LIMIT; для Grafana задан bytecap1048576 и native engine budget5с/256МиБ. Go SDK отклонил2МиБ, browser proxy до guard вернул2097625байт. Existing HAProxy теперь требует identity и распознанный normalized одиночный Content-Length≤1048576 без CE/TE; тот же browser query получает502/35байт. Это не универсальный streaming counter.14 native response classes и ordinary positives сохранены отдельно. GET range консервативно ограничен floor-window1–3599с, ambiguous duplicate params закрываются. Body inspection64КиБ; измеренные incomplete/delayed bodies завершились408, oversized413, JSON duplicates422.
При запросе trace→logs откройте operation row, затем Span details → View logs for trace. Ссылка выбирает span time window; worker window может содержать одну worker.job строку, тогда как более широкий trace window содержит3 лога. Chain icon рядом operation открывает References, это другой control.
Один host и один PostgreSQL не защищают от потери host. Любые fault checks запускаются только на owned stand с синтетическими пользователями; не останавливайте shared runners, Azur или соседнюю lab7. После проверки остановите свой stand, сохранив volumes и evidence.
## Воспроизводимость dependencies и проверок
На проверенном omarchy images и unchanged plugin уже подготовлены. Выполните `bun scripts/preflight-openobserve-images.ts --grafana`, затем `./scripts/up.sh`; это cache-only старт, а не команда неограниченного скачивания. Image source refs/amd64 digests и sizes находятся в IMAGE-MANIFEST.json, фактические aliases/IDs и plugin SHA — в evidence/grafana/cache-ready.json и OPENOBSERVE-IDENTITIES.json. Grafana cold wait ограничен600с: на HDD fresh migrations превысили прежние120с.
На новом Docker29 host отдельно получите public linux/amd64 leaf images из IMAGE-MANIFEST.json и присвойте три local aliases, которые проверяет preflight. Не переносите private generated credentials. Готовый ZIP plugin берётся из Windforce17/openobserve-grafana-plugin release1.0.1; ожидаемые52781711байт/SHAe0c2cbcc7cf6d05bcfaa606db39e52249eb0dd234376e45a3f896353831c2fd5. Распакуйте public каталог openobserve в .local/grafana-plugins; plugin.json/module.js/nativebinary повторно проверяются до старта. Установка зависимостей на новом host требует его собственного допустимого бюджета; фактическая current600МБ approval не означает новый повторный pull. Новая установка на другом host не объявляется проверенной этим отчётом; пустые volumes на текущем проверены целиком.
Для source validation: `make check`, `bun test student-tests`, `bun scripts/check-backend-unchanged.ts`; public runtime smoke — `make test BASE_URL=http://127.0.0.1:28206`. Explicit fault/browser helpers перечислены в HELPERS.md; historical readback требует сохранённой fixture generation и не должен выдаватьPASS на пустом volume. Real browser helpers используют установленный Chromium/Playwright на omarchy, без скачивания browser.
Не повторяйте DB/worker/fresh/retry workload ради icon diagnostic: обязательные business scenarios уже имеют receipts. Strict browser helper сохраняет один canceled static icon как отдельный Network outcome; его aggregate exit1 не скрывает functional3/3 и console0 и не является доказательством отказа приложения.
+379
View File
@@ -0,0 +1,379 @@
{
"createdAt": "2026-11-16T05:55:51.381Z",
"assignment": "VP Lab6",
"student": "Медведев Андрей Михайлович, БПИ2303",
"target": "5, track2 SLO/incident detection; no awarded grade or instructor acceptance claimed",
"sourceOfCurrentResults": "REPORT.md",
"knownBrowserLimit": "Three functional UI scenarios completed; strict aggregate browser command retains one nonblocking canceled static sync.svg fetch, exact trigger unconfirmed; no clean-network PASS claimed",
"fileHashes": {
".dockerignore": "503384a6bf9e425743b9bb64488e7994f9c47e626a6c2c1d7b26714f052bd4d0",
".env.example": "820589ce3b26c382bbedd275815c672ac763c50e680484e82b747fe8eaef8a05",
".github/pull_request_template.md": "c0c01d34c0e1f1265a46cebc0b0feac1dbc4f4cf3deaafaccfc0e1d7ef808a10",
".github/workflows/contracts.yml": "a45d7932ef14130aff8020bce79f043cccaf9692d697088a754d309a03f7d3e8",
".gitignore": "b481dacab83380983e8c0ad05ac08f38428b5a3b0946fb347f2f30eb05e29ad4",
"BACKEND-READY.json": "0afe0f624bbe58a9acb8a711efd03d76edd19be6d11f8be943edb9cd015c7ff6",
"CONTRIBUTING.md": "2c01dd698cc9653dece49a94c9fd3f58d36e84764aaa643b46e26d7e599dc12e",
"COURSE.md": "c34d53cd082d194aa4b58f504c4e0348d25b6afec3d6a9c0963dd01c9c81e971",
"Dockerfile": "40095fcce1fc5c43eea0f7cbc0cce55ab6e630969abafdaddc64ce4f9a778fb4",
"Dockerfile.example": "fd7ac9491c0c3c663b083c41a762fb38bc2d212fa4a2ed3e6a0702af72108709",
"EVIDENCE.md": "3c9d1acf5f684e5b550b022e9ba071a363e1e4b5b08b1952b5056637fb80e974",
"HELPERS.md": "95f583fad7de0c68dc887f75b77f8a8fb4c71571a126e568687d17fa3a0ff211",
"IMAGE-MANIFEST.json": "b2d6c094136ca7f2b77ee900d30afeebe0f64992cf4212b5183396dbf671c5ce",
"MAINTAINERS.md": "f2e7e9d59e06f260ccf3acc9baf79c4a3b66e1bc370ff6e21cbefb338210630a",
"MIGRATION-OPENOBSERVE.md": "aa0d1ae47711cf22e248f5ef215ab8b9288ac28163d9e34e7e45e1e27ff56902",
"Makefile": "c05bd07058ae0949dd4905afdae7674bc7e3e63863a6cdf8b72661140d445826",
"OPENOBSERVE-CONTRACT.json": "8a5db4bf7e6cf70a7b11273b4ace02fe4e3ae687e25b2f483be64dc0550ed056",
"OPENOBSERVE-IDENTITIES.json": "7d8aa3a16b0cebb473b79554b0ce6c0ed67315392b1739cc433a71fcd3305612",
"README.md": "583fa7625d728ead8a6ee2cd2de9f117ed3f4a1d9696c0bb9fb602fe549a31db",
"REPORT.md": "1563b0781dec8039843bffb456fde029485fdc6433cf97b521c784c35fb1e719",
"RUNBOOK.md": "285f7ff45a9b83f7f26f6ef5c5140faf871a2f2d98fc3ea331ff5716a1e457d2",
"SUBMISSION.md": "f4de1597761d341c477d4beb1dfe2c9853cc740545c2feaba9ec6a529ef62ce6",
"TELEMETRY-SCHEMA.json": "c44ea15d3fc70b6bca35deca27c7296868bc3c1376e13efd043164be76af3c75",
"backend/.golangci.yaml": "215578dee14bcef1b66759a68543083e0a624611efcdb785197307b50cb7725d",
"backend/cmd/app/main.go": "00e7cd80cd07e9696a2eb761890b87fde26a09ed96efe01e181e62c98799509a",
"backend/cmd/worker/main.go": "405bdcfead49e30c988629db8478e4aee3e0ee6ffba94d7517c8676d5aff9dc4",
"backend/codegen.yaml": "e4d37ce5c5a9f969f696833cd59a6e1b86f99fb2b0de69101a1aa5f4a539aebe",
"backend/db/migrate.go": "33044d905354f24fff194fefd35ee9327d63d6e5062ded8496e518b6ce38dbea",
"backend/db/migrations/00001_core.sql": "b6bd8c70d51aed6c966e07eebffd262af2e694c8b3479c70bbf9a9eacf5a7aad",
"backend/db/migrations/00002_idempotency.sql": "97c98785547db8391c1fa0a685e4d4bcc810b8f79055c0e2fc6ef473cd38b37b",
"backend/db/migrations/00003_attachments.sql": "6b4f707dde32f0c3f531741c108bcf0f22be05c9e2ede327f27a1d9ef0cfcc7f",
"backend/db/migrations/00005_replay_audit.sql": "234a04177bca08621fa38b135cb8086bcdd55ae9084008b25814751e1dd742a4",
"backend/db/migrations/00006_slo_facts.sql": "2cdb8cdb1618a90eea76e59ac99f3d89a3665b2d6ade17502abe4261d5ffcf3b",
"backend/db/sqlc/queries/attachments.sql": "8e64f0e6169fab7664b0f4a97f2be353553b93c0c928d10678cb930e44de37da",
"backend/db/sqlc/queries/core.sql": "1aed3d0d2fc04650351d86d15837cf3239636844727e799aabf5ddd534429f33",
"backend/db/sqlc/queries/idempotency.sql": "5bdec86e85705f7805fb603e2c792bb87c11cf3156952fa48e7bcee35fd683ca",
"backend/db/sqlc/queries/observability.sql": "65c5dd26008542a80b7673d25631d1e7faeb0ae4caba661aee52d25fca26e4f3",
"backend/db/sqlc/queries/pipeline.sql": "9551971f39bced93c8690022495f15a87f7e14fa865d050a200a1b5d52c1acc1",
"backend/db/sqlc/queries/slo.sql": "6f9b0092fca3ee0dda312d421bdb9ea587e3f580983372af47dc54f3d79a1873",
"backend/db/sqlc/sqlc.yaml": "803b161e063692398c93152b11a08f9e42e1bf3c72a1e64553b82c494545fd10",
"backend/generator-overlay.yaml": "e666da2959b31befed713a8366237de53070a565f93850d6daae7a582a038819",
"backend/go.mod": "7f2b2a2764494f7be5f395d78bbf42f04196d3696363f89a0f38308e3a2f1cad",
"backend/go.sum": "8d58fbc798eb94634668719ac9ea68535b390621026a9097d0a3bbd0d122fbd2",
"backend/integration/object_metadata_test.go": "598e973575d57c6ee563f0b15d0111d37b27de6774c6f486c998dbc70594f15d",
"backend/internal/adapter/handler/api.gen.go": "4034501f5c661c87af3b81d0d1ecb9996050828569bea7e85b3604a2a8c8bdbe",
"backend/internal/adapter/handler/api.go": "9d9f109e13c9322287b29ed18e7913e0e8d5a9ef2dc4be23f10c7686de3f70fe",
"backend/internal/adapter/handler/attachments.go": "608827653c4d3e4252dff863d30007e93b1bf82579bbe71d8b647bf22128b1b2",
"backend/internal/adapter/handler/auth.go": "3889f58ac4c58b798192776ea05455d52735a2c024ba664ab9af5eb405187557",
"backend/internal/adapter/handler/errors.go": "39a5b916112fd9cc2be9979b6e78d6305e7c3c8ab5fa462f1b8e9f18c88c1a4a",
"backend/internal/adapter/handler/gate.go": "b662058ebc59010f30d21da12404837aa2329fe5e5590761efdd9140c1d24e9c",
"backend/internal/adapter/handler/gate_test.go": "f7938bdeb5617ed3611303c54818000de9861dde5915fa2c6fa0f74e1fa33268",
"backend/internal/adapter/handler/handler.go": "1a99601365ed52aa36f5ef2b6be4dc760e28e1ad5cc38eb6790d69c5559751ce",
"backend/internal/adapter/handler/json.go": "d6458b8e8e4426816ec8c855ff6c8d88a151161bc1560e4b98df0ea4b0dbf6be",
"backend/internal/adapter/handler/message_json_test.go": "8dd976c4cad81ee531c76165787b2c8b6052057d934ae4b09b60d8a261cf30e5",
"backend/internal/adapter/handler/middleware.go": "41ce83f0b7eaf25aecdbeb99d5292066f25e2043e08c0f681db2cfc80c01c528",
"backend/internal/adapter/handler/stats.go": "5cd375e5a0db8cc3596293d323e635b86fef22d8ec680f310ab5a460f8176f64",
"backend/internal/adapter/handler/utf8_test.go": "eac6a14b5f3b0ea9aa5329337afc6db27e17f5c17999874da9ce3592dc7a4205",
"backend/internal/adapter/imagefile/image.go": "8a9b399b22573992fd7fd4c87096f953b46c1bc7cad4e09c023923919d755375",
"backend/internal/adapter/imagefile/image_test.go": "a2dc0d3797a5c2ac7ad46a7453a34ea13aba94c3385d2692032705610026dc71",
"backend/internal/adapter/imagefile/stress_test.go": "3d7b0671f57ce5e01fa7e175a86c16a51d56d8f11ec6d3bfb09b2c8bfe8c04d8",
"backend/internal/adapter/imagefile/variants.go": "50762d1e2af635c9a9344d2721ee145bb9d9d43408f680fc280cdd493aeef886",
"backend/internal/adapter/imagefile/variants_integration_test.go": "d25e3371432128e58073ff61e1b87fa741bc32ee7d9bc6e8cb40ae3f3a5b4434",
"backend/internal/adapter/imagefile/variants_test.go": "79c24a3d5c0cb339a32fdea28eb56689703f35ad3c668c464ea7a4f5e82322e5",
"backend/internal/adapter/password/argon.go": "94643bf4b61ca390f8dd56a241e02e1325a994fc40887ffa0e2b8f8e5aebb4c7",
"backend/internal/adapter/queue/publish_trace_test.go": "057552f74a6af595439f7e68c7b0c8bbdb888093767f46dd559e8c656adec049",
"backend/internal/adapter/queue/streams.go": "c9f0864c264c23bb6927bb20de06c6a659c709b3228255059a45edaca953c360",
"backend/internal/adapter/repository/postgres/attachments.go": "88f8b8744d95407421eafc111ecc83bcb4b145f1683618d69cac19a8d1c39500",
"backend/internal/adapter/repository/postgres/observability.go": "b6001827364a226026e7a651da8a57918ec426e34cd64d3b6ef709814f0a5667",
"backend/internal/adapter/repository/postgres/pipeline.go": "9a367d7a50a9418e96851c253f61ace4a54753044e7fa672c73a3b4526a22245",
"backend/internal/adapter/repository/postgres/pipeline_integration_test.go": "58274475a02f1098a0f79a65a9e8d7608d05dfa26213f90c706a9239fabb586c",
"backend/internal/adapter/repository/postgres/queries/attachments.sql.go": "407ed2864afad2b039a33c571228e6ee6a9acdc515d957bb55dc600308b5eacc",
"backend/internal/adapter/repository/postgres/queries/core.sql.go": "9a0e082263cbea39535de9c7c72c41a39a551efc9ee57fdd7b08856831d074b3",
"backend/internal/adapter/repository/postgres/queries/db.go": "4863732830879d4162be215ff245375b953359de23bcd821ed7abc5c32a36c0f",
"backend/internal/adapter/repository/postgres/queries/idempotency.sql.go": "cf6fbc316e93339e0f938c7ca82007e29c3e32f1a21ee1433a2ce8784697f002",
"backend/internal/adapter/repository/postgres/queries/models.go": "dab26f6c8a5542ce223c234faac1e8ec8c7066507a84989ca3f620583956d0cd",
"backend/internal/adapter/repository/postgres/queries/observability.sql.go": "c2971da00171c126b129ca2010a1cd1a6223b11ed1b1b20632af19fe8a66f313",
"backend/internal/adapter/repository/postgres/queries/pipeline.sql.go": "43009cbd2754d478727cce5d4b263dcdcafe7ad8000de011f22795531e1fb3ec",
"backend/internal/adapter/repository/postgres/queries/slo.sql.go": "bcaa6b4515507e555caff58a0448861ffaeac2f21771a4bde587cf296f6c685e",
"backend/internal/adapter/repository/postgres/repository.go": "e380d7c0c52a2447653db9e4d13db432f4c1330e3205a0396510d957441bb978",
"backend/internal/adapter/repository/postgres/slo_integration_test.go": "e396c722fe8ea7ade19e2de5c9a070352efbde8e8712ae6e910f628a11c5c2b0",
"backend/internal/adapter/storage/bootstrap.go": "44c8f422826b29f85a918029bb71a62cd3018190f8efe889cbeee63d19d7039e",
"backend/internal/adapter/storage/readiness_test.go": "a6c79cba7af74ce67e69678085e5bb267a47f9dd7be57daf003ba03951274a5c",
"backend/internal/adapter/storage/s3.go": "99a717604c71da5c181bef74612a603abe3cfc913ccb3da84ed581a57a11ccd8",
"backend/internal/adapter/telemetry/dependencies.go": "09d5bcf97bfff77d42dbdca9637e015a2eb73e03ea3ff801d710e1340c85ea44",
"backend/internal/adapter/telemetry/dependencies_test.go": "d7738b7fe098b2d72967243dce8ab7a84135e570e4697371755b73245d908981",
"backend/internal/adapter/telemetry/gauge_failure_test.go": "4d8d3aaff9e471a36c0f65db0a23d799101c72f804b6d7270bc9a9570181b225",
"backend/internal/adapter/telemetry/gauges.go": "80782d0bdb1bbaa9132c6c3a471887e4a6d48839b448a5c3c558edf217a01642",
"backend/internal/adapter/telemetry/pool_integration_test.go": "b8e8d24ee46549c0898726f770f206701f147ea7f9cbb4271b24dcaa63f5a0dc",
"backend/internal/adapter/telemetry/telemetry.go": "db9ec2b20d504e7c50424414618e00fe1b81ce68b997c70ae65040260d31b890",
"backend/internal/adapter/telemetry/telemetry_test.go": "c727d6901ac3b8a5cc3e704645f8a7715a44d88f8bdf9bd990ac176c017fbba2",
"backend/internal/application/usecase/attachments.go": "0c835f88c72b90753b5458960f7b345e6e51b964070ce37eaae129a4c296b0ca",
"backend/internal/application/usecase/auth.go": "93802954db2c8eda3782acc71bca817947fd556986d3f6552c78c4b346d988ca",
"backend/internal/application/usecase/messenger.go": "c58aba0d3f5d0ab92fab5d2771b90a3af5ad06e6afe0c7cc46f481a429e89f48",
"backend/internal/application/usecase/pipeline.go": "05b4a6e87c2bfd86c06f5b8cc3e7b960a26c0a69726a366a3137158b1e9e2e45",
"backend/internal/application/usecase/pipeline_retry_test.go": "96a569aa26e23909c109dda14c8828901abe2914bbad2ec3506a10441a264dce",
"backend/internal/application/usecase/reconcile.go": "239edee16f767dd16b116791900994540a8f99ae3403fd0308f5e2c0797a892d",
"backend/internal/application/usecase/reconcile_pipeline_test.go": "75e1fb5fcc5ffb2188a2368688e5505a4bf54453c2800b5e46c8e9e23de89b7c",
"backend/internal/application/usecase/reconcile_test.go": "fdbcfad86851fc97a3cf635b6d884c41b43084eedc42b3c4847790c5eaad4a6f",
"backend/internal/config/config.go": "01fe90ef662e1bfe164b858818f68dc8e73d1ae3b1c3a07f83a8a7a00bb9d883",
"backend/internal/config/config_test.go": "aa9f1c670f525e4e895beccb807108e926cc9950bbf509ed71ecc96d6f64891d",
"backend/internal/domain/model/attachment.go": "e532f2c8e8c123d0fda6101bff4b21168527a0a446e53f566f5e5919b972133b",
"backend/internal/domain/model/chat.go": "3f7454f48ef442c277fd2741ce5aeeccf03d590bd2d31b9b89fa39f0ba05de7f",
"backend/internal/domain/model/errors.go": "d043547d742e94d9ef9f0799dced7c43dc2adc44d7d90906e891e5afee8de04d",
"backend/internal/domain/model/job.go": "564c219e526b99e06d2c882d7184fc6ee0b58d29c326be5969a38cc6b92b8c0a",
"backend/internal/domain/model/message.go": "0e0126663d2de04c9d14c8c7427f42a9fb733e3b170ceee7ed86c2138f51b23c",
"backend/internal/domain/model/model_test.go": "77bb8767ac1352bd3551e724eaaeaad4fe93eae248cc8c3008ee8b1c08adddca",
"backend/internal/domain/model/page.go": "cd6a07579bff01aa0d8a11b34fd000a804142b321cef302d386bdb95e9c7f953",
"backend/internal/domain/model/postgres_text_test.go": "8c245eef99a97261daf0aa16fec9edd7bcd3253ec4144c50b4f8e56e45e2186e",
"backend/internal/domain/model/text.go": "c9410cacd9d1b2fd7f3f2cb7e4112e637ecc494f6e4b6da649f936ce0c9a0457",
"backend/internal/domain/model/trace.go": "9ff395bce67142f6808fa7c652e50db2a68cf4e34c128fb04af408566a14220a",
"backend/internal/domain/model/user.go": "c20c8b8d8a9c0327a3a24afd74f3051d78caed1d2f0851baf340ef61ac502a92",
"backend/internal/domain/port/auth.go": "465c1a1f19075b6f8301eef8dbd7fd9a8d20255716d72c97e946db04c2850dda",
"backend/internal/domain/port/objects.go": "4d22b5ef5ff6fa13404884c160fe52bd7cef4099f84b29268137c27034a0f3ec",
"backend/internal/domain/port/pipeline.go": "5fe7c6909cbaf6a21a4be5d9ed0c2b1551681894d0054463cdba13ea09c21aac",
"backend/internal/domain/repository/repository.go": "0008747c69b32df9e4b7f36fd1ac157dee537d5e1e415bd48bd55b0b05852362",
"backend/pkg/logger/logger.go": "d839f31015eee692065c31d2538a4cb4c841b0d89e690f0ab8baf80168510556",
"backend/pkg/server/server.go": "2916acc42708dd493068452ac4eed5208cd166f910c517f16196d001aac5d269",
"backend/pkg/server/server_test.go": "7eccb3acd7f9e9ac0b724b17fa50f1f1ff94be043554fed0a5e0fad1e2cb9ee3",
"compose.example.yaml": "9bfd9eacae286dd9bffe29d6fe349534552560f966c3c3e6e6540e92d7357afb",
"compose.yaml": "a5147c4b94fbdaf1a25d62090bda256096e9d5945cf7ea2feda64089d4fc6b6c",
"config/grafana/dashboards/messenger.json": "36868638687c81b5c8dd73a6c627a4705c9235046b9cf34b3cb4792a7fa80bfb",
"config/grafana/provisioning/alerting/rules.yaml": "b4ae7ff647f294965277a81f6dd050dd98a733a222583ab860c29ce897dc1234",
"config/grafana/provisioning/dashboards/dashboards.yaml": "8e93508155f3b38dbd9594224ca002a586de83d621daf80e6e661e1811a88171",
"config/grafana/provisioning/datasources/datasources.yaml": "022644aed410205b19b0f09a8921c3ca83b233eeffea62723848deda32f01764",
"config/haproxy.cfg": "5fbd83685515fa68b463fd887b66f23cd2ffab9283c9ece6d015cc5634c7f5bd",
"config/openobserve-query.cfg": "b66161a1f7f6311be605d5a7482a5746d666d8072084c98e11921f0884c877be",
"config/otel-collector.yaml": "cd0fcfdc4055d39757a1654cd4de35b58eb64d3369ddb88c17f32f66678607ae",
"contracts/README.md": "72df27a559f88ffa3fd8c3c79d3c32f6db2c5986a0602677c51994c434a3fc4f",
"contracts/attachment-states.md": "cf0519e137410e4b34ab632aadb8ad8a381d4b73033e27b354c58eeacbdf86d6",
"contracts/examples.json": "381cc18c03789e935b391d007fb5843964be9cfd9f7530cb6b1996ed4a1c4c37",
"contracts/infrastructure.json": "45264a3db653bb79de8d8fcb8d70733924f3cbfcba0fa1253cecbc6828bce217",
"contracts/jobs.schema.json": "9ccf9b11466c0c3a26a941520ac5ff543241fc4935e3a6eebb01ef303779b10d",
"contracts/observability.md": "882c120c0e153a06c9cc824a802e317c58eb6590bc3d3a576f2e0a96f8597b62",
"contracts/openapi.json": "2c4074e8fac6bed6337fbbc0aaed0ae2321a76ef30ea68afad80a1f39699535a",
"docs/archive/openobserve-migration.md": "a59c2b008b5b4d490598cd6054c2310e7cd9fa9e449060396223078b40da5e77",
"docs/archive/pre-openobserve/README.md": "bd8ab63fa801404f0901f28112cb87a16fc57171b872a368ee481b9cb1118d87",
"docs/archive/pre-openobserve/alerts.yaml": "56b1df0e0ad30ae472fa7251299201cfac90aa18067272ffa8107ba7542aa4fa",
"docs/archive/pre-openobserve/loki.yaml": "cc980499348f0daac6f04811fe4bfccf8d1271b6bd32c5ccdcd8a265595cce39",
"docs/archive/pre-openobserve/prometheus.yaml": "5dbeb12445e996637ef95079afba8ae0a357edf943b2686f57074ae3185bb1c3",
"docs/archive/pre-openobserve/tempo.yaml": "25dec1d2e1f2f754da53530e611a221e75b08b2f3ebc102e2c892cae40b23eb7",
"docs/reference/OPENOBSERVE-BOUNDARY.md": "b44f91f0d9a41e04f1fcabb4424af98cad23a7f3d66ba304628517fd1095ce7f",
"docs/reference/README.md": "62f2bb6e4d28f52339de140101ac948f81323eb1c0392beef706f7411d932c69",
"evidence/README.md": "78e4bfd738d034597d15d8846accd8a8a5dc738a23a3d15f243f3992bd530cc3",
"evidence/both-replicas-collector-unavailable.txt": "3b1a79dff46aebf6a5a905c9204920c4790673818caa1f999c1f9758b4329ad4",
"evidence/both-replicas.json": "beba4aa327d18cdf9e59223ea5f039ee93afc7259d785b3cfb316f7692f908d9",
"evidence/core-resilience-collector-unavailable.txt": "e6194116ec933b8ebd41e96f1784c78499d35c1a3dd935273edea6f96fab8429",
"evidence/core-runtime-collector-unavailable.txt": "445bec2845a37adfd31ddddd32d6fc03bb3fed31879e7f54d925dda72c2b8a71",
"evidence/final-backend-check.txt": "28203f07080356926ab2955c6b751e20c63e8680c18a2249141ac74e65c5cdb4",
"evidence/final-core-runtime.txt": "3a2fe786410509e7994cf2aad4d48975a7c3c6b7a7d7c88e538b8c40634f4841",
"evidence/final-correlation-check.txt": "724e561788a436137f7bae0eab9e424790865936e3e4e206113eb3f1104fee10",
"evidence/grafana/BROWSER-LIFECYCLE-CASES.md": "3d16dd8df1a3deffe7ffd2d386c46d0cb051d45eff50ca7b5110ac8799693856",
"evidence/grafana/GRAFANA-START-CASES.md": "617c9795f61c79f4596d7159d595d3ed6b90cf09058567a34e39c8dbcde040db",
"evidence/grafana/INCIDENT-READBACK-CASES.md": "4cac56de696c8729a37bf8185ca24da551cfc63b1a9c2a65ce25a232b4d92229",
"evidence/grafana/RETRY-CASES.md": "b050f05b3da6e8a9b01c2f47352062e0c4f4c9cea0f026b0e9b2605775e91d7d",
"evidence/grafana/backend95-final.json": "13cf07e6eedb281dc404bb8d5b0a686f741955010e75c413529265361a0efcf6",
"evidence/grafana/both-native-links.json": "af989261f467a884b0557b71d35d86cb60a856331e67bc9772f6a80d48bf4152",
"evidence/grafana/both-native-links.png": "98a0cfcb31184c43a4fde9f09dd96d555b422dffa7d14806a8bb5484eed95e14",
"evidence/grafana/browser-cap-green.json": "f50a34c53d413868dd1f839a15388e7a8f7a55349ccec06cd4839899f782bc8d",
"evidence/grafana/browser-cap-red-sdk-cap-green.json": "2a9ae2e48cdd07b491de0c9dcf0533178283a307c85b9eda8c8b0a7d2b5ba7a7",
"evidence/grafana/browser-lifecycle-contract-check.txt": "d88519d8567f326ded8baf6de87e2a843ea8d94bd9647dcc1171f2c11cc9797f",
"evidence/grafana/browser-lifecycle-delta.json": "c13b6520d165a5d6a0c62c23b5a30825e82ad1278d394c4c796b2459b80064cc",
"evidence/grafana/browser-lifecycle-full-unit-native.txt": "3b4d243f3c37eccb4dc19e440b10c655cc35d0261954e1e691ac681e98484014",
"evidence/grafana/browser-lifecycle-green-native.txt": "1aae7403c76cea2f57faabd5aef8129e6a5ba8f1d38ee59cc5a71e12d7dd9baa",
"evidence/grafana/browser-lifecycle-green.txt": "ae49201e1d3b01c30bddaac6ae7bafd25d4907995689beeb4dcbc0a822db5da8",
"evidence/grafana/browser-lifecycle-red-native-finalcase.txt": "988b00cce40bb3b9d2daf3120471dead19d76cf79e2a8ebe2c2c99dfa51a95e9",
"evidence/grafana/browser-lifecycle-red-native.txt": "954b1720ab9adfe28d3e0a7a7d2696bfdf44ef40c59088e182fb881b2da41b9d",
"evidence/grafana/browser-lifecycle-red.txt": "95d8357344c8354b75d35ae26b19c2c7a9d0befd1ed9cc4c73a24d1ffd281e65",
"evidence/grafana/browser-lifecycle-typecheck-native.json": "a80154902fec39c8316841ffc963720db4f4c6c5d624eae11ddf8e02ce4bb56d",
"evidence/grafana/browser-partial-filter-negative.txt": "70dbc7df10cc2fef0d47eb53090ace6e62e712e5778f1eeee5a3450e18cf9d36",
"evidence/grafana/browser-three-functional-icon-aborted.json": "7fc2666b9b7568413a229e9ebff341e8e92687c1db818eb65ab12fa1c3e0a026",
"evidence/grafana/browser-three-functional-icon-unknown.json": "49fba0654677afaa2e931953eaca3aa71b4a631988f72a44c5acfd2bcc3097ad",
"evidence/grafana/cache-ready.json": "2b5a27e8357192157efbcd4f3dd91de6b4c06734ef4726058e4dd00b72c0d3e2",
"evidence/grafana/compose-bind-audit.json": "11fc05f93e0a592989a9f816197233ce28fa24413f67bbb960a6ee7df91103cc",
"evidence/grafana/current-up-green.json": "1dde382688fde124091407953403bdb82478a5b4ccc5284baa0385e2cab42825",
"evidence/grafana/dashboard-panel18.png": "bd3e3ae825834703b0954fcfa27492e9c01ad00c67b9682adbce42bf18bcf27c",
"evidence/grafana/dashboard-panels.json": "6a3c5e556595796e9dd369a6303c841afe9380a53a6706471c0b73b3590599f6",
"evidence/grafana/download-ready.json": "f0fe94027e06b129e20736191447747fbea80e6c2d78d97275bfe796d3acea91",
"evidence/grafana/final-source-checks.json": "1662e9779f9f8a84393b66e842049482a4f26bb662bf8f15064360fbf7672e9f",
"evidence/grafana/final-stop.json": "bb06cbb9ee1cd000c592a23504ce2d0f6ce06ae79d5deaf25d95d904c532324b",
"evidence/grafana/fresh-volume-deadline-red.json": "5ef744fa524b56e55473cce94c6021989df352cc694db324a499ffb624225bd9",
"evidence/grafana/fresh-volume-green.json": "e3f1de4693bdd6de103245ad5971e3ba2ed158c6e8333b6d9162ab13ddde2361",
"evidence/grafana/fresh-volume-identity-guard-fail.json": "fe0168d712d0ea7af2a98eadc01146faeb69a9e6e2b07c32c4a4b95213ad4e21",
"evidence/grafana/fresh-volume-operation.ts.txt": "b25a6d709298e4aee55337716187facb2cc7d5b22439f8f66fff90909dc86f27",
"evidence/grafana/grafana-credential-green.txt": "b9841a5d351721d69c5a8b50f5073c4a36d67d5f2afa4b0d3ddd14eda73d878c",
"evidence/grafana/grafana-credential-red.txt": "2cf2ac28c716a46796a7ec7f88203babfb43382f827743d5c67b38b93e36777c",
"evidence/grafana/grafana-provisioning-green.txt": "b14495dc3fa8c1e7727c79fe4fc92f00489eac914d10aa13aeb93a5cb2756f7c",
"evidence/grafana/grafana-provisioning-red.txt": "d9f62a53722dbc8a2252cdaa8c1031fd80c7f21f00995f866698aed1721bec4b",
"evidence/grafana/helpers-unit-current.txt": "cda61e6b50ba01006e192ef0fcf203a043e662804b536614ef9bb2d2ab16c3a7",
"evidence/grafana/histogram-native-green.json": "a30593005f1aab6c3391b39eae845d36c1867485485bc302556714672e7bbf8a",
"evidence/grafana/histogram-native-red.json": "011554cfbc7bb5ed2807ed173c8e4a3dfe582da4bc6572ead660df83c07365be",
"evidence/grafana/history-normal-ui.json": "653981b13525c996611e9b738c2087a8ddcf1facd801c8a9fcc25a1ce414086e",
"evidence/grafana/history-page-service-source.txt": "a8970b0388671b359a1721a6d6eb9913a5df4d30251fd40cb37621cfd0f5a2a2",
"evidence/grafana/history-source-binding.json": "822c9c536412cde4b5369a40c835ac5d09ce58d1d6cac7ce4b47735c58c4711b",
"evidence/grafana/history-user-storage-source.txt": "aee99a03048c1d7e639ad16e80d874508033a18f32c18955914dfc96624d1e28",
"evidence/grafana/icon-cancellation-diagnostic.json": "8d7c0a87260446bb33c6cd0c666bb15ecec035c693ba3b22ccfccd369c9b179c",
"evidence/grafana/icon-cancellation-observer.ts.txt": "135401581228d581ca363c8500411b6d03d68ada4c520146516ec281259f9522",
"evidence/grafana/incident-database-trace.json": "60fea5bc0031f17d90de475abcfcd52302657e7584cc44add0498cc8d7302041",
"evidence/grafana/incident-owned-attachment-facts.json": "bb0f8c3d242998a7f93b02eb12c0337c380df76524b9ebd33cdf175fae946c16",
"evidence/grafana/incident-readback-green.txt": "4b90f65a48664d0198671f52595ed2ec5aa052f2994d344564da09baff6519a0",
"evidence/grafana/incident-readback-red.txt": "5511d750cc3c6a26713cb5e8e0b1f7540b6de0b16ffe24f63b0f32f86fe54032",
"evidence/grafana/incidents-causal-readback.json": "5c25919d1b512aee235f84864943747c919d192398bcbf0fd7875a50c4113523",
"evidence/grafana/incidents-original-command-fail.json": "38d3084dd8225f376152df798b2714527854c9bb3e0c26062d56cc7c0888a0ae",
"evidence/grafana/incidents-original-validations.json": "5b7489355539a954172e99ca04bef460bbf10d4478b7acbae58a95883bb1608d",
"evidence/grafana/linked-worker-tree.png": "3eee13cb3ffb8916d926137d254c8b8159c69ebd899c373780da9d2e52c97c46",
"evidence/grafana/native-preflight-green.json": "3091949cbfa5f069a4300e8e7eb9aecb9f249b2c928926e4ec92357fb69faea1",
"evidence/grafana/offline-typecheck-final.json": "a2d898eb0810a6142b1aae3846e6a2fe227b313980124064191a315168dcccb1",
"evidence/grafana/offline-typecheck-source.ts.txt": "d3b6dbb8ebbca12f335fe30fb046a328805fe1174d95415cb16c95151e33e406",
"evidence/grafana/owned-fault-green.txt": "74548f55f46a6ca4fabfc6606eb06b43a72be3ab0348b80d98a5fe58b8183e17",
"evidence/grafana/owned-fault-red.txt": "a128ddbe95a530810b08517d01e874509e0be52d561930bea1b4a7001652fb80",
"evidence/grafana/provisioning-strict-native-green.json": "82ccbe34bb1c18d14e5f3b0c99a423aa9da16497c2a4965747cc1b38b911c919",
"evidence/grafana/query-boundary-positive-controls.json": "9df677633531ffafaaed62ead2bb69719e8709ab1cd2fcfde6e485ea86b0063d",
"evidence/grafana/response-boundary-green.json": "d961bdc510d00174df16a8aecf242afab85e1ef0292b446793d59dcb273dd111",
"evidence/grafana/response-boundary-leading-zero-old-oracle.json": "19459b5cf889fddc44243a9dc47859286f2775c99b6b46c8606bdb8966f6d11b",
"evidence/grafana/retry-owned-job.json": "ddd09853a4e0d7c4660986bd05e4454a23d511aba02fdea935895ba90482ee9f",
"evidence/grafana/retry-storage-child.png": "cec3b52e60b232493b9d42ae4691096f69ccbf61cf79b76f2ec9150fe1dfae9f",
"evidence/grafana/retry-telemetry.json": "e636d4bb3e8e63a013285b12fd3f87a37e233d4896184b7a7afa7ac0795cbff2",
"evidence/grafana/retry-two-attempts.png": "cec3b52e60b232493b9d42ae4691096f69ccbf61cf79b76f2ec9150fe1dfae9f",
"evidence/grafana/reviewed-source-guards.json": "b7ba00202ca3bbdbc1c1511e7c43601d9af67d5f04491112b608469615cdf2f7",
"evidence/grafana/reviews/browser-response-limit-advisory.md": "039b89da66153de5281433dce1c81f2f19813bd17a5ffa81f88b424439525afe",
"evidence/grafana/reviews/histogram-request.md": "2b82843724670360a97478f2ccc49e1eeb2c7b17299df5c3efe472789c8cbfdf",
"evidence/grafana/reviews/response-boundary-native.md": "7bfa21ff33b6c0405370cafea88798f22469974c408f3373fb4b1391396fb4dc",
"evidence/grafana/reviews/startup-lab6.md": "49adfc3c242e41b6b3bd666e5c684831aaf2cb9aed03e3f21980010602d26f89",
"evidence/grafana/scheduler-guard-green.txt": "483401c3036c4c2edac87ff34840643e7008f5f92cbe41beb9639379d50445e7",
"evidence/grafana/scheduler-guard-red.txt": "b4f125e8bbef1a659cbf261555af2b61254f7fa9dc0369e186eb6a7e73501b01",
"evidence/grafana/startup-type-delta.json": "82446bf6f4395055c421939f25b7eac0ce0976ede75853b11888ed142a60e459",
"evidence/grafana/stop-verification-source.ts.txt": "04b3ff4a525eecb8957e8b1816bf731dda5c86f7effa405757fd19c73a30ae1b",
"evidence/grafana/trace-first-dom.txt": "d3313b551114b7b646c56fe600a631e61b120211bdaa2945f7a28d0ede2241c0",
"evidence/grafana/trace-first.png": "5fb7e9741a566f1723a94a338c1e8670f7362d4286983e0146e19c8a58838247",
"evidence/grafana/trace-ui-first-readback.json": "20c1999c16b16386d045f2df168179dc6e4fac6c7ba2e59349b2c0442f8e626e",
"evidence/green-native-pool-pending.txt": "32f01e1e18a654209f9e9a2dc73512e0ac6cbad6b53117bf59b68535a77fe652",
"evidence/green-slo-cohort.txt": "ac7f0fa5e2d32ef2b276b2012cb5faa6c29e8a86678672be5005bf5db2831738",
"evidence/openobserve/account-http-green.txt": "44e54b6fda02f35da97f1cb43eaf93f87565b54ec502261c076afbab57f9569d",
"evidence/openobserve/account-http-legacy-red.txt": "1a799040d071b21dcc31e0fda622a529a015bd9326598b670d8764944f3df2bc",
"evidence/openobserve/account-http-legacy-source.ts.txt": "48cee4d7ff937b9bb8dcac1ada4e70f5297cd0912de12da5888da494b7e4818c",
"evidence/openobserve/backend-current-unchanged.json": "819dc5e831f017260420a86e4bee7a1c1288ba40ed0a635aebc429dd93b5ed18",
"evidence/openobserve/backend-unchanged.json": "0fe6245a25442cbffb0eb579014cc8def344821dc1073d2edccb4b03fee25448",
"evidence/openobserve/bootstrap-current-start-green.json": "e46c441f8bce3113f1b7abce01487461b767e4b253da8fd86314481714cb220e",
"evidence/openobserve/bootstrap-existing-green.json": "32b2d1d9ca4ddbd271547110ca99bc428269c936c6fe3cc1bc2a179a37062479",
"evidence/openobserve/bootstrap-green.txt": "8f2f5408ff9146209ff4a96a7816826391dd34cd1593e61225eb3605200e5988",
"evidence/openobserve/bootstrap-http-green.txt": "97f196059f53cb66745426360aa2d39b4ed3bfbe6b461868412e8f4d88fdb0bd",
"evidence/openobserve/bootstrap-legacy-red.txt": "c2f7dd8931a573fefc5327efb9a6ee72499e23820afbf02af1067150ad29f589",
"evidence/openobserve/bootstrap-legacy-source.ts.txt": "f55750b78d86c9bfbdef67ab9dee3cdd47006e2f26200e9ceff9bf88387a2f61",
"evidence/openobserve/bootstrap-native-green.txt": "6d7af944dda0d9c18334e184db08142f69e8fc7c76ca6f7ef27fecb9338fdd6d",
"evidence/openobserve/both-replicas.json": "08c5f313c3bbe9e13989c4dcc2662dcc92626965e549fd0ec015825adab55d76",
"evidence/openobserve/cohort-label-diagnosis.json": "0eeaac01ee37b1f350a9c27af900b718a447a0a7538219abcd2faf57aabe3859",
"evidence/openobserve/collector-before.json": "705e3a7886e87951372cd0c5bfacc0db6d974169fb044ef6dce873d51c596bf1",
"evidence/openobserve/collector-down.json": "5a667f612bfbe50212652b6aa12f99e3c37aefc721098da4b8c0a1d7cb8047c9",
"evidence/openobserve/collector-recovered.json": "37083f4206d9c2316d029f61f12c4c465b908e7767765ea800dee2200d394da1",
"evidence/openobserve/gateway-bounds-red.json": "d31db3119944f2473e91cd2370e4e2ad469c241680743b8a54e4400aea69ab1b",
"evidence/openobserve/gateway-green.json": "c4af6f6d4a0abde6229a2035983479524cdeb2ad7413cc3c74e90d4c05a41f83",
"evidence/openobserve/gateway-native-red.json": "59852a0de730679e7aede97d8c4625661f49a9d83e365ac81e9099ae343dacff",
"evidence/openobserve/historical-queries-initial.json": "a3ccb99e4adaa22da1a6a65cefb125bf2266a24042680216168426ac150f8e3e",
"evidence/openobserve/instant-time-grammar.json": "60951c38c6522dac4a55415e29a6de9039848b9dd407439637ed9ca89c832d01",
"evidence/openobserve/native-identities.json": "2f581efbd5c2058f06b0bea80a7524bab888a0a327456bfa820d6aa886f9114f",
"evidence/openobserve/native-signals-sa.json": "282e0c60967b20b0d7f5d3bdb2888200a955b902ddd1dfa3774798c785fc2fcf",
"evidence/openobserve/native-signals.json": "2f07a4237094f55cd567eef1a6c47b2f9238e0eb3ac727f5158fa8cfb72327d1",
"evidence/openobserve/network-final-green.json": "3605d63bf03482787b671b8142e1c5b9e20248e60059f18be117c3eef2e1532e",
"evidence/openobserve/network-green.json": "bd86757ec678fb55556305a322a1afbcbd8dfd884d22c8c5a1687626222688ed",
"evidence/openobserve/network-published-green.json": "7f61d91f7e1331870d49bf4ecb7a6148c8377ed398c2e06a226740e539af6f38",
"evidence/openobserve/network-red.json": "62ff3f820620ff8bf36db5566757207a6474d8e5c75875bbbb9967df6639aa1d",
"evidence/openobserve/parser-delayed-closed-initial-oracle.json": "eee979efe818623e3ddb1a806bca35626a699edb42df89023364d8f67ad1be04",
"evidence/openobserve/parser-green.json": "94a7569d2ef3e979bfe9d7d922bc4baea94c73093ea5ac0883c9bda7e9c11bab",
"evidence/openobserve/parser-initial.json": "a064da283a9bbc12cdc1bd0fb5ff09c9ee3c0f93f2e49fb8b13e65a7d255ca59",
"evidence/openobserve/persistence-after.json": "d0e7f6ef0882aae9c094f0e95b339d08ab9b0d6bd08832e0016684d875ab0f1e",
"evidence/openobserve/persistence-before.json": "3f2c0bb56fb145f35317cdc61aa6e336ab60b7e505b5cd20bf2bac71fe9e8429",
"evidence/openobserve/positive-queries-green.json": "86c4c665b556b51379d3d790884e2c141cdd978d60d97178f48b29f9af20892f",
"evidence/openobserve/positive-queries-red.json": "26dd20a064a011387940110b7ad8654828ddff44c46dc5bbf2be175251e439f8",
"evidence/openobserve/provisioned-queries.json": "823105a2f42539e5dfa677a7edd3b1d1e19de6ba0055554429faa71df8e5c7fb",
"evidence/openobserve/runtime-checks.json": "b814e1d4266a25d37eb076ad86303a43d92af7833d5143b3d89c97c8c7470f80",
"evidence/openobserve/schema-discovery.json": "bbaf7785bbb371cc599ad508806fdc0afd1ddde340af15276acee97e6cfddb7f",
"evidence/openobserve/ui-publication-green.json": "b756e5dd7a37f613db508c089f34aefd20ad1e316cafb5595a5ce438b1f1dffd",
"evidence/otlp-sdk-test.txt": "8053ec5d0ed4aafbbc74d004bdcdd431341c01b45d1290d3516567c090602cee",
"evidence/red-db-gauge-isolation.txt": "64ae35be982ab91f15281fc58a49f716a8c5ad6a759d1cfaba61cf7c42313b0e",
"evidence/red-drain-and-stream-span.txt": "7c39a391384df1d8036cfdad577634376fac7a5e31ec73e77083423d75d0fa6f",
"evidence/red-force-close.txt": "2f67d0da4a5498b1ac2be5a3c04a502899df5be7aa92b283b1c3ef35c6a207c5",
"evidence/red-nonfinite-sampler.txt": "08a055736ca187600d54897cba4448c6e7d3a2493c09b90d7a849a77e1379a64",
"evidence/red-pool-pending.txt": "dc5b29fb0d2decffd9bd130ba79a03e51f26113de79c406bca22a9383c32be91",
"evidence/red-preaccept-queue-age.txt": "766a0eae7b721db142f6c92cab28ef0bbf9d2517df9174577b01584ba75eeda1",
"evidence/red-producer-error-status.txt": "a8358166eb2b1a59912350f8bbf0ba9c099529dd2e4d20d616b7c40295b45ec9",
"evidence/red-ready-before-cleanup.txt": "0aa6ae3b7babfba49a9742d07192ffa9b4ce1dc5496fe63277e89ebc68d207ff",
"evidence/red-required-correlation-fields.txt": "65dc7c834687b35be96ce6cd8e719840296b5e2ca651fa90649c99ac3fd53a73",
"evidence/red-slo-cohort.txt": "d6bb5254e7709b2aab118efb5bd4f685bfe0e47056733413224d0ab6dd22e8e5",
"evidence/slo-backend-check.txt": "114320e7d89c0d899b9c434d1b50669ddd94c4cee3b915578ef0ef8df66e9b1a",
"examples/requests.http": "8c086fd540092b3e31a945a416632bcda6c4bea39253f86b076f791b362cf5e4",
"infra/README.md": "0e804fa51debba4c38d3cc84abf4704897665e0335b272060363f03aadd6354f",
"infra/otel/README.md": "0d6c93589f7b96b7045c1b82a261362697d25f14c9e1a4946b8ea8a977c1e842",
"lab.json": "807dbb02b180ddb13dc898c117e09495e15a462e83f4b55133ba174f089594ce",
"materials/loki-upstream.yaml": "b0512247b74062a2df69acdd26f1292fb7750d41330b3ee34d36d7511456ef90",
"materials/tempo-upstream.yaml": "85fc97739e982e4b79c901404152bcb5dee5334ef7921b2e5d5b3984bfbcb4e1",
"requirements-dev.txt": "4cefdb67007caf026d1613be7c519c01b4c8343db7c4f4b618677e01cf113525",
"scripts/bootstrap-grafana.ts": "f65c5819ff590eff79f4da52e4ca2512ebd2989b93d2902c1e83d33bf887e847",
"scripts/bootstrap-openobserve.ts": "65a99ee06582ce8d6972e5aaffc2501baeee40375566fb501878acea1760e490",
"scripts/bootstrap.ts": "b91575c133fde7fdb332b0712ce13673391823c672694fc413ee5a65255acf15",
"scripts/check-backend-unchanged.ts": "b8ab59114be4cc0d8693c4854b6c9963e4a202668c723ad4873dbee5bc8ba5fe",
"scripts/check_contract.py": "7c4402c2a6ab80d3b4676e1866d6ecc87b6daea14c080bcb9be9a7e8b959d671",
"scripts/complete-openobserve-cache.ts": "41d7bfff6f688381a14e4d7bc781433293330072d9b306ee2db4bb550ffe9ada",
"scripts/compose.sh": "5d873d3576699ea554d91ba61effc003af6de1e351dbb5691de7cb6708a86f57",
"scripts/copy-openobserve-cache.ts": "2a2ac853e350ec2091a68d62d8763f9ce1a3202ab7e5f026c6e3b4cce508fe27",
"scripts/discover-openobserve-cache.ts": "6e679d5bfe45817658722fcd1412850b48a9be397ae76cc547fda17c563117c0",
"scripts/image-manifests.ts": "e817db20aa036a479a3582384177d87563b1fb19e9ccd954674e84706dc4c7ae",
"scripts/inspect-openobserve-start.ts": "4128e1c4f38b303a5a2ea46d73008691c100c3c197cd0446e32a3bed95861177",
"scripts/lib/browser-lifecycle.ts": "feaccfdeedea5a9b9feb2012d84b4004374035e80700e9129f0ea0af27996749",
"scripts/lib/grafana-credential.ts": "ae5bffbc875a644da52096a4c5c12519d143df41eb1179ac56841232497e22a4",
"scripts/lib/grafana-provisioning.ts": "ac9df83b5f820c97ad99ade9d0531665de81a5d983c657651e9be388375c8b99",
"scripts/lib/grafana-scheduler.ts": "a1366acfee0a5f27b5aa893a8019131d04b8715eb17ef11c264442a3029fa9d7",
"scripts/lib/incident-readback.ts": "9c43089bf80e06cb208c6c9369286ee181269535f706888519cfa303b126bf04",
"scripts/lib/openobserve-account-http.ts": "e8afbdf05f697a60c4cfba7d1d05d983374bd0470c4b6824ec54804ea068d7f2",
"scripts/lib/openobserve-bootstrap.ts": "9107c48807a1c56f006fc20a67d2091e968d5c78e00e8a6c2cf70f7038ba9bcd",
"scripts/lib/owned-fault.ts": "236a404f9e18464721ecbee2be03130f18973e019f8711df9e516cfc0022b9bb",
"scripts/package.ts": "f7ab3ccd6160875838af97fb492c175f1f1d84c43c2471d4308960f97f0c0c80",
"scripts/preflight-openobserve-images.ts": "227eccbce60abffde448beab749e7eaabf73ad8337dbd3317a089701ee8c1637",
"scripts/prepare-cache-import.ts": "1e03c05227956194f2ba176df869b73e0d0b92d84896240be9e5efc02f111b40",
"scripts/provision-openobserve-accounts.ts": "abf009f68d5bbae4167920b2b0f2d9e018b0228db5c9d53e78febd9686dffa5f",
"scripts/render-openobserve-credentials.ts": "f120d6ede83f8584c27f5e9a29999bd1c63c142ff3c4b7aa11316125e5243552",
"scripts/restart.example.sh": "0328246ba75defbced992104e986fde973f63e3830d83251eced5953263e944b",
"scripts/restart.sh": "dfbdc591fcfe6b7262207fea240b15341ea6c7dcc4be40388e7e33dabfc9918f",
"scripts/stat-openobserve-cache.ts": "831c9b51b04141ebce3b6fa11ca63aac63e83f8bd273e2b63496c8da1bf14fc3",
"scripts/stop-one.example.sh": "1bcf34394520b89d95b7605ec56675be799eea920e83cfbba306d5dc0a6d9811",
"scripts/stop-one.sh": "aad731f184de9a2461b242e9dc4f1bc8c4388007839868069e3242c4d3b38c66",
"scripts/stop.sh": "2639f1ba8d2eddb454d395292cf2635d0d4c2ad9471784e70f66dce3ba61d498",
"scripts/up.sh": "e434a47ded97ab02eb8ec8939ab1f3d55e43700c5b3b7a7395743a19194460ce",
"scripts/validate_openapi.py": "4e981acbfc10dac58817bf922d5896e996798551fedc1b2385130e1cddb13d54",
"scripts/wait-grafana.ts": "0f4be1395bcdda9b6c48600470d815653d70756a7b338666e74a9333a1994d4e",
"scripts/wait-openobserve.ts": "d9d3791e9be47f8793bfef317f570450ecd48bbc53b4905003bb1c9cf25ec6f3",
"student-tests/ACCOUNT-HTTP-CASES.md": "b11742311905720521a203bb1d5b959a26648bc079bc9940f69107c162057b1c",
"student-tests/GRAFANA-CASES.md": "be619640aca5daf27d990f598156c40f01a0a04180d0e30bfe3f757b55ff536f",
"student-tests/both-replicas.ts": "580df3b1959234203a4316138725133887f1bf56323fa000519b63195d56986e",
"student-tests/browser-lifecycle.test.ts": "8b79795c4d68927a4dd0deac0d7007fcad71347f507715c564e125bfb96b23b4",
"student-tests/check-openobserve-config.ts": "fc085fd2d96d25e35b1b36426bb614c93fa1c10f62de0a60606b7e476ca3fa5b",
"student-tests/grafana-browser.ts": "c36c72e62c75a57b93111f449cdc64c4dcec7b7e9c63bf3a7d903390ae9d0aa1",
"student-tests/grafana-credential.test.ts": "4436d989985d8b37485c561ad29d2d03bff55996fb6dc4dade97c50eb5195512",
"student-tests/grafana-incident-readback.ts": "83ba78c13dd3acde5657b1c1e4f4ce4270298d057c930182753d007f9d6886f1",
"student-tests/grafana-incident-trace-readback.ts": "28b7d528b2f785fd5036213a12f0c8480f40592043851502ea9c0abc9f074102",
"student-tests/grafana-incidents.ts": "29a976806b7eac44a40d339e21f41fd0eb552679d5a20c82372f6b11071cc066",
"student-tests/grafana-native-histogram.ts": "c3bf3cef4d7403b19ca9d404ce1c6243cef678350256d615ae9ea15bc1274a94",
"student-tests/grafana-preflight.ts": "197750305e59d1f78af70cfdeeccfd7b3c8d8dce619e5a1ceee4ed96cf4105f2",
"student-tests/grafana-provisioning.test.ts": "08a3eaf0651e8f36450a6b9a69cb8f75ed5f48d873ec13a13431a55e32a87c9f",
"student-tests/grafana-response-boundary.ts": "f36fa129c053d8bc7da679d4af0d93af9b9b6df85e332f6745d7e27ab8ebc31b",
"student-tests/grafana-retry-readback.ts": "caa98b1881576f7b3191c947635bba92300e6ac3af05e9d4dd312d3cb512e304",
"student-tests/grafana-retry.ts": "418eac4c3409545f5fa1775d4c6e959dd0eac8dd3e7cef3073696524a4933481",
"student-tests/grafana-scheduler.test.ts": "5adb2a1fcfdfeb4f059813950e73249f7d99c0d9eb7dc9560ebf07ea4b0fda45",
"student-tests/incident-readback.test.ts": "69b2ac50bf1bfe615b7e8c9d537a637019b34168bba462fa34b4c05f4302e3c4",
"student-tests/openobserve-account-http.test.ts": "69a0b4494442eb96be7e783882da2f24afaf2c6fcf9bb849d30858e89cd049e9",
"student-tests/openobserve-bootstrap.test.ts": "9b093ee637db4eb168c4a5f6bb51b82a02f907c20c4550b0341783654d427667",
"student-tests/openobserve-cohort-query.ts": "281fea123719ddf163dc0faa41bdbe7292ac5ebeb25be8a828ab2bd23a8ab3a4",
"student-tests/openobserve-collector-snapshot.ts": "c9c17f959aae456986469861204750d40c9a51ad898b7b486b6fa87c6446daf9",
"student-tests/openobserve-gateway-boundaries.ts": "527d97a0a7d8678b52ca9625b135ec279ebc0748d07b0f03c5bf204d0555b5cb",
"student-tests/openobserve-gateway.ts": "bf8ba05aeb1f35bb3879a93cdeae0b2f20cfef5b5d3e72311ce7fdc59b5c626c",
"student-tests/openobserve-historical-queries.ts": "5dc1c85b61098516185b2651c03b5b4610090e12d9c4842b73136c4c372a026b",
"student-tests/openobserve-instant-time.ts": "699ad0610c416a9d2257e7033a084673da12712098fb6c6ec9c3f2a4488edf3f",
"student-tests/openobserve-network-run.ts": "70f70215a7b595328e67c8ecde6a91c189afb278e7fbba1181b6c083ebc48dcd",
"student-tests/openobserve-network.ts": "df55122b0fe5c98fdf26e387e630a5e5336040f4acbf73d69fe7037a49c644f5",
"student-tests/openobserve-persisted.ts": "360ba925485002675b5fdf14b0a7567e75a7060b3680efcf2ff3c3cb9f8c9d3a",
"student-tests/openobserve-positive-queries.ts": "5f9e24672abf60876364e0e7983bed464e77d478764ea2a9ba79a1fc6da85a3e",
"student-tests/openobserve-provisioned-queries.ts": "18ba76d0faa7212caee9fae59eafad896b03debb818fb3e5cf676010725143bc",
"student-tests/openobserve-readback.ts": "91721b73449339f91f1241ac697ad173e3253f7fea12b91dbd1bfb767b094022",
"student-tests/openobserve-signals.ts": "98aa701803949681ad7711e6b56290455e4ea33723394ce7927d30da9cc1cda4",
"student-tests/openobserve-ui-listener.ts": "89f27a970f4cb27b86043af95497f7b65ac3c92462e7815e8d6187eaf7fa3bea",
"student-tests/openobserve-ui-port.ts": "24b8df6fc9b40e576eb74b7f41c6382c78e6e756d349c2e96cd21e9facf5f990",
"student-tests/owned-fault.test.ts": "3a31c8c5228e74c3fe786e8753f3079431534882287444e756e8acaf22530fe6",
"student-tests/wait.ts": "9dff91ee5a9876d5c4758d66a65b3b42c29a6d3071c7e510babae561a84cf35b",
"tests/README.md": "e0bfa7e196797f6a5bf6e44d8ad2ffeb82651927ff5d8484eb1c2dda1ec7dec5",
"tests/client.py": "8d50976fc7c7bbb81e88a152f613c61ab25eaf53dba33c6a9dfab19f868d546d",
"tests/fixtures/README.md": "c001783760b4f87a7a8441a7ba55f5d080ff48e1397d3581d72a88d7691f96c2",
"tests/fixtures/sample.png": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75",
"tests/image_probe.py": "a9a4f89181c8e0fda6ce42e1a1af32fb42d8e3ae519e314207127c9b9b448b13",
"tests/load.py": "6d3511ea719c7a7e841083e73d435c86dcbfd9f4e5c2a07d62eb9614b01d9933",
"tests/resilience.py": "1f96663a174b3978d24eff66f2741f5f8c9b788d3242261d37a972720c52788b",
"tests/smoke.py": "52ac38eb343ff7d43e7543588e1da58ca06c9fce74b2b0d0bf0e4982bc23c1db"
}
}
+13
View File
@@ -0,0 +1,13 @@
# ЛР6 — БПИ2303 — Медведев Андрей Михайлович — цель5
Выбран трек2 «SLO и обнаружение инцидентов». Перенесена обязательная основа lab5; backend95 сохранён побайтно. По решению владельца Prometheus/Loki/Tempo заменены OpenObserve1.1.2, отдельные Collector и Grafana13.3.1 сохранены. Две API, PostgreSQL, S3, Valkey/broker и worker остаются в одном воспроизводимом Compose stack.
Проверены настоящий UI18 панелей, canonical trace42spans/41parents, оба log↔trace clicks, retry chain58spans, local alert firing/resolved при DB/worker faults, bounded telemetry outage, история после рестарта и обе API replicas. Fresh-volume fullstartup и original smoke прошли. Развёрнутый разбор гипотезы, методики, измерений и ограничений — [REPORT.md](REPORT.md); запуск/диагностика — [RUNBOOK.md](RUNBOOK.md); entrypoints — [HELPERS.md](HELPERS.md).
Из корня lab6 на подготовленном omarchy: `./scripts/up.sh`. Runtime dependencies и exact pinned cache/plugin hashes перечислены в RUNBOOK.md/IMAGE-MANIFEST.json. Source checks: `make check`, `bun test student-tests`, `bun scripts/check-backend-unchanged.ts`; original smoke: `make test BASE_URL=http://127.0.0.1:28206`. После проверки: `./scripts/stop.sh` с сохранением volumes.
Известный неблокирующий предел: Chromium отменяет один static sync.svg fetch; точный cleanup trigger не доказан. Aggregate browser commands сохранены FAIL при выполненных3/3 функциональных сценариях и console0. HTTP200 metadata не подменяет content/visual проверки. Старые неуспешные команды и отдельные исправленные readbacks не переписаны как PASS.
Оригинальные README/contracts/public tests/CI не ослаблены. Реальные credentials, keys, generated runtime files и caches не входят в source ZIP. Источники и SHA каждого файла входят в submission manifest; итоговая проверка ZIP использует native unzip CRC и per-file SHA. Результаты и исходники прошли независимые targeted reviews; полный final checkpoint передан отдельно.
При работе использовались AI agents для реализации и независимого обзора; измерения получены выполнением собственных synthetic correctness/fault checks. Student commits, instructor approval и выставленная оценка не выдумываются. PR/сообщения/работа преподавателю не отправлялись.
+363
View File
@@ -0,0 +1,363 @@
{
"source": "backend/internal/adapter/telemetry",
"validation": "Frozen SDK95 source unchanged; real native OpenObserve1.1.2/Collector0.163.2 all3 ingestion, native queries/correlation/persistence/outage verified; Grafana rendering and firing/resolved pending",
"sdk": {
"traceMetric": "1.47.0",
"logExporter": "0.23.0",
"endpoint": "OTEL_EXPORTER_OTLP_ENDPOINT",
"paths": [
"/v1/traces",
"/v1/metrics",
"/v1/logs"
],
"protocol": "HTTP/protobuf"
},
"resources": {
"service.name": [
"messenger-api",
"messenger-worker"
],
"service.version": "lab6-v1",
"service.instance.id": "random UUID per process",
"deployment.environment.name": "lab"
},
"trace": {
"propagator": "W3C traceparent/tracestate",
"spans": [
"METHOD route-template",
"attachment.publish",
"attachment.process",
"postgres.query",
"s3.put",
"s3.get",
"s3.stat",
"postgres.acquire"
],
"requestAttributes": [
"method",
"route",
"status_class"
],
"excluded": [
"SQL statement/arguments",
"object keys",
"username",
"password",
"Authorization",
"token",
"DSN",
"file payload"
]
},
"logs": {
"bodyAndEventName": [
"http.request",
"worker.delivery",
"worker.job",
"telemetry.database_observer"
],
"attributes": [
"event",
"outcome",
"request_id (HTTP only, response-generated UUID)"
],
"correlation": [
"trace_id",
"span_id"
],
"httpOutcomes": [
"2xx",
"3xx",
"4xx",
"5xx"
],
"deliveryOutcomes": [
"processed",
"dependency_error"
],
"jobOutcomes": [
"ready",
"retry",
"failed",
"stale"
]
},
"metrics": [
{
"name": "http_requests",
"kind": "counter",
"attributes": [
"method",
"route",
"status_class"
],
"healthExcluded": true
},
{
"name": "http_duration",
"kind": "histogram",
"unit": "s",
"attributes": [
"method",
"route",
"status_class"
],
"buckets": [
0.005,
0.01,
0.025,
0.05,
0.1,
0.25,
0.5,
1,
2,
5
]
},
{
"name": "jobs_processed",
"kind": "counter",
"attributes": [
"job_type",
"outcome"
]
},
{
"name": "job_duration",
"kind": "histogram",
"unit": "s",
"attributes": [
"job_type",
"outcome"
]
},
{
"name": "time_to_ready",
"kind": "histogram",
"unit": "s",
"attributes": [
"job_type"
]
},
{
"name": "queue_depth",
"kind": "gauge",
"attributes": [
"queue"
],
"queue": "attachment-jobs",
"source": "logical current-version attachment_jobs backlog, including durable unpublished intents"
},
{
"name": "queue_oldest_age_seconds",
"kind": "gauge",
"attributes": [
"queue"
],
"queue": "attachment-jobs",
"source": "logical current-version attachment_jobs backlog, including durable unpublished intents"
},
{
"name": "db_pool_used",
"kind": "gauge"
},
{
"name": "db_pool_capacity",
"kind": "gauge"
},
{
"name": "worker_busy",
"kind": "gauge",
"scope": "worker"
},
{
"name": "worker_capacity",
"kind": "gauge",
"scope": "worker"
},
{
"name": "process_resident_memory_bytes",
"kind": "gauge"
},
{
"name": "go_heap_bytes",
"kind": "gauge"
},
{
"name": "process_cpu",
"kind": "counter",
"unit": "s"
},
{
"name": "db_pool_pending_acquires",
"kind": "gauge",
"source": "native pgxpool AcquireTracer start/end; includes waiting and connection construction"
},
{
"name": "db_pool_acquire_duration",
"kind": "histogram",
"unit": "s",
"attributes": [
"outcome"
]
},
{
"name": "database_observer_up",
"kind": "gauge",
"semantics": "0 means DB-derived gauges unavailable; independent HTTP/process telemetry continues"
},
{
"name": "database_observer_failures",
"kind": "counter"
},
{
"name": "ready_slo_matured_10s",
"kind": "gauge",
"semantics": "immutable initial accepted_at/first_ready_at; cohorts age at least10s"
},
{
"name": "ready_slo_timely_10s",
"kind": "gauge",
"semantics": "immutable initial accepted_at/first_ready_at; cohorts age at least10s"
},
{
"name": "ready_slo_matured_30s",
"kind": "gauge",
"semantics": "immutable initial accepted_at/first_ready_at; cohorts age at least10s"
},
{
"name": "ready_slo_timely_30s",
"kind": "gauge",
"semantics": "immutable initial accepted_at/first_ready_at; cohorts age at least10s"
},
{
"name": "ready_slo_matured_300s",
"kind": "gauge",
"semantics": "immutable initial accepted_at/first_ready_at; cohorts age at least10s"
},
{
"name": "ready_slo_timely_300s",
"kind": "gauge",
"semantics": "immutable initial accepted_at/first_ready_at; cohorts age at least10s"
}
],
"bounds": {
"traceQueue": 128,
"traceBatch": 32,
"logQueue": 128,
"logBatch": 32,
"exportTimeoutSeconds": 1,
"sdkRetry": false,
"metricIntervalSeconds": 2,
"gaugeReadTimeoutMilliseconds": 500,
"defaultSamplingRatio": 1,
"httpDrainDefaultSeconds": 3,
"httpDrainMaximumSeconds": 4,
"note": "Lab7 retains its separately reviewed8/9s drain with15s supervisor; ForceClose shared on expiry",
"collectorQueueBytesPerSignal": 8388608,
"collectorRetryMaximumElapsedSeconds": 5,
"collectorMemoryLimiterMiB": 128,
"collectorContainerMiB": 256
},
"prometheusExpected": "Native OpenObserve names verified: no messenger namespace, no automatic total/seconds suffix; see OPENOBSERVE-CONTRACT.json and evidence/openobserve/schema-discovery.json",
"slo": {
"acceptedAt": "first QueueAttachment within durable handoff transaction, distinct from upload start",
"firstReadyAt": "first successful CompleteAttachment, COALESCE preserved through replay/advance",
"windowsSeconds": [
10,
30,
300
],
"maturationSeconds": 10,
"unknownLegacyRows": "excluded; no invented acceptance history",
"nativeResourceJoin": [
"service_name",
"service_instance_id"
],
"gaugeFreshnessSeconds": 10
},
"storageNative": {
"contract": "OPENOBSERVE-CONTRACT.json",
"logs": {
"stream": "messenger",
"timestamp": "_timestamp",
"body": "body",
"event": "event",
"requestId": "request_id",
"traceId": "trace_id",
"spanId": "span_id",
"service": "service_name",
"severity": "severity",
"outcome": "outcome"
},
"traces": {
"stream": "messenger",
"traceId": "trace_id",
"spanId": "span_id",
"parentSpanId": "reference_parent_span_id",
"parentTraceId": "reference_parent_trace_id",
"operation": "operation_name",
"service": "service_name",
"status": "span_status",
"start": "start_time",
"end": "end_time",
"duration": "duration",
"timestamp": "_timestamp",
"units": {
"start_time": "nanoseconds",
"end_time": "nanoseconds",
"duration": "microseconds",
"_timestamp": "microseconds"
}
},
"metrics": {
"nativePrometheusEndpoint": "/api/default/prometheus",
"counterNames": [
"http_requests",
"jobs_processed",
"database_observer_failures",
"process_cpu"
],
"histogramBases": [
"http_duration",
"job_duration",
"time_to_ready",
"db_pool_acquire_duration"
],
"histogramSuffixes": [
"_bucket",
"_count",
"_sum",
"_min",
"_max"
],
"gauges": [
"queue_depth",
"queue_oldest_age_seconds",
"db_pool_used",
"db_pool_capacity",
"db_pool_pending_acquires",
"worker_busy",
"worker_capacity",
"go_heap_bytes",
"process_resident_memory_bytes",
"database_observer_up",
"ready_slo_matured_10s",
"ready_slo_matured_30s",
"ready_slo_matured_300s",
"ready_slo_timely_10s",
"ready_slo_timely_30s",
"ready_slo_timely_300s"
],
"resourceLabels": [
"service_name",
"service_instance_id",
"service_version",
"deployment_environment_name"
],
"noLegacyExporterNamespace": true,
"noAutomaticTotalCounterSuffix": true
}
}
}
+36
View File
@@ -0,0 +1,36 @@
version: "2"
run:
allow-parallel-runners: true
linters:
default: standard
enable: [depguard]
settings:
errcheck:
# Close тела ответа и файла, открытого на чтение: ошибке некуда деться.
exclude-functions:
- (io.ReadCloser).Close
- (*os.File).Close
depguard:
rules:
# Гексагон: ядро и use-case'ы зависят от портов, а не от адаптеров и транспорта.
inner-pure:
files: ["**/internal/domain/**", "**/internal/application/**"]
deny:
- pkg: backend/internal/adapter
desc: домен и приложение зависят от портов, не от адаптеров
- pkg: backend/internal/config
desc: конфиг читает только cmd
- pkg: net/http
desc: HTTP только в adapter/handler
- pkg: github.com/go-chi
desc: роутер только в adapter/handler
- pkg: github.com/rs/zerolog
desc: логирование на границе, в адаптерах
domain-core:
files: ["**/internal/domain/**"]
deny:
- pkg: backend/internal/application
desc: домен не знает о use-case'ах
exclusions:
paths:
- ".*\\.gen\\.go$"
+225
View File
@@ -0,0 +1,225 @@
package main
import (
"context"
"encoding/json"
"errors"
"flag"
"fmt"
"net/http"
"os"
"os/signal"
"syscall"
"time"
"backend/db"
"backend/internal/adapter/handler"
"backend/internal/adapter/password"
"backend/internal/adapter/repository/postgres"
"backend/internal/adapter/storage"
"backend/internal/adapter/telemetry"
"backend/internal/application/usecase"
"backend/internal/config"
"backend/pkg/logger"
"backend/pkg/server"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/redis/go-redis/v9"
)
func main() {
healthcheck := flag.Bool("healthcheck", false, "проверить /health/ready и выйти (для Docker HEALTHCHECK)")
migrate := flag.Bool("migrate", false, "apply migrations and exit")
bootstrapS3 := flag.Bool("bootstrap-s3", false, "initialize private bucket and application IAM credentials")
reconcile := flag.Bool("reconcile", false, "reconcile at most 100 metadata rows and 100 objects; dry-run by default")
apply := flag.Bool("apply", false, "apply safe reconciliation changes")
grace := flag.Duration("grace", 10*time.Minute, "minimum safe object age, at least 60 seconds")
after := flag.String("after", "", "metadata UUID cursor")
objectAfter := flag.String("object-after", "", "object key cursor")
flag.Parse()
cfg, err := config.Load()
if err != nil {
fmt.Fprintln(os.Stderr, "config:", err)
os.Exit(1)
}
if *healthcheck {
os.Exit(probe(cfg.Port))
}
log := logger.New(cfg.LogLevel)
alias := uuid.NewString()
if err = os.Setenv("INSTANCE_ALIAS", alias); err != nil {
log.Fatal().Msg("instance alias initialization failed")
}
if err = os.WriteFile("/tmp/messenger-instance-id", []byte(alias), 0600); err != nil {
log.Fatal().Msg("instance alias file unavailable")
}
observability, err := telemetry.Setup(context.Background(), "messenger-api")
if err != nil {
log.Fatal().Msg("telemetry initialization failed")
}
defer func() {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
observability.Shutdown(ctx)
}()
dsn, err := config.ReadSecret(cfg.DatabaseURLFile)
if err != nil {
log.Fatal().Msg("database secret unavailable")
}
if *migrate {
if err = db.Migrate(context.Background(), dsn); err != nil {
log.Fatal().Err(err).Msg("migration failed")
}
return
}
s3path := os.Getenv("S3_CONFIG_FILE")
if s3path == "" {
s3path = "/run/secrets/s3_credentials"
}
s3config, err := storage.ReadConfig(s3path)
if err != nil {
log.Fatal().Msg("S3 application credentials unavailable")
}
objects, err := storage.FromConfig(s3config)
if err != nil {
log.Fatal().Msg("invalid S3 endpoint")
}
if *bootstrapS3 {
root, e := storage.ReadConfig("/run/secrets/s3_root")
if e != nil {
log.Fatal().Msg("S3 bootstrap credentials unavailable")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
if e = storage.Bootstrap(ctx, root, s3config); e != nil {
log.Fatal().Err(e).Msg("S3 bootstrap failed")
}
return
}
pc, err := pgxpool.ParseConfig(dsn)
if err != nil {
log.Fatal().Msg("invalid database configuration")
}
pc.MaxConns = cfg.MaxConns
pc.ConnConfig.Tracer = telemetry.DBTracer{}
pc.MinConns = 0
pc.ConnConfig.ConnectTimeout = time.Second
pc.ConnConfig.RuntimeParams["statement_timeout"] = "2000"
pc.ConnConfig.RuntimeParams["lock_timeout"] = "1000"
pool, err := pgxpool.NewWithConfig(context.Background(), pc)
if err != nil {
log.Fatal().Msg("database initialization failed")
}
defer pool.Close()
var cache *redis.Client
if url, e := config.ReadSecret(cfg.CacheURLFile); e == nil {
opts, e := redis.ParseURL(url)
if e != nil {
log.Fatal().Msg("invalid cache configuration")
}
opts.MaxRetries = 0
opts.DialTimeout = 80 * time.Millisecond
opts.ReadTimeout = 80 * time.Millisecond
opts.WriteTimeout = 80 * time.Millisecond
cache = redis.NewClient(opts)
defer func() { _ = cache.Close() }()
}
store := postgres.New(pool, cache)
if err = observability.Gauges(func(ctx context.Context) (map[string]float64, error) {
m, e := store.PipelineMetrics(ctx)
values := telemetry.PipelineValues(m)
values["db_pool_used"] = float64(pool.Stat().AcquiredConns())
values["db_pool_capacity"] = float64(pool.Stat().MaxConns())
return values, e
}); err != nil {
log.Fatal().Msg("telemetry gauge initialization failed")
}
if *reconcile {
cursor := uuid.Nil
if *after != "" {
cursor, err = uuid.Parse(*after)
if err != nil {
log.Fatal().Msg("invalid metadata cursor")
}
}
if len(*objectAfter) > 64 {
log.Fatal().Msg("object cursor too long")
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
report, e := usecase.NewReconciler(store, objects, time.Now).Run(ctx, cursor, *objectAfter, *grace, *apply)
if err = json.NewEncoder(os.Stdout).Encode(report); err != nil {
log.Fatal().Msg("reconciliation report write failed")
}
if e != nil {
log.Fatal().Err(e).Msg("reconciliation incomplete")
}
return
}
app := usecase.NewMessenger(store)
auth := usecase.NewAuth(store, password.New(), app, time.Duration(cfg.SessionTTLSeconds)*time.Second, time.Now)
attachments := usecase.NewAttachments(store, app, telemetry.Objects{Objects: objects})
probeDependencies := func(ctx context.Context) error {
if e := pool.Ping(ctx); e != nil {
return e
}
return objects.Ready(ctx)
}
h := handler.New(app, auth, attachments, probeDependencies, handler.Limits{
MaxBodyBytes: cfg.MaxBodyBytes,
MaxInflight: cfg.MaxInflight,
MaxQueue: cfg.MaxQueue,
QueueTimeout: cfg.QueueTimeout,
}, func() any {
return map[string]any{"pool_total": pool.Stat().TotalConns(), "pool_acquired": pool.Stat().AcquiredConns(), "pool_max": pool.Stat().MaxConns()}
}, &log)
srv := server.New(cfg.Addr(), h.Router())
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
defer stop()
serveErr := make(chan error, 1)
go func() { serveErr <- srv.ListenAndServe() }()
log.Info().Str("addr", cfg.Addr()).Str("env", cfg.Env).
Int("max_inflight", cfg.MaxInflight).
Int("max_queue", cfg.MaxQueue).Msg("server started")
select {
case err := <-serveErr:
log.Fatal().Err(err).Msg("server failed")
case <-ctx.Done():
}
started := time.Now()
log.Info().Dur("timeout", cfg.ShutdownTimeout).Msg("shutdown signal received, draining")
h.Drain()
shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.ShutdownTimeout)
defer cancel()
if err := srv.Shutdown(shutdownCtx); err != nil {
log.Error().Msg("shutdown timed out, forcing active requests closed")
if err := srv.Close(); err != nil {
log.Error().Msg("forced server close failed")
}
}
if err := <-serveErr; err != nil && !errors.Is(err, http.ErrServerClosed) {
log.Error().Err(err).Msg("server error")
}
log.Info().Dur("took", time.Since(started)).Msg("shutdown complete")
}
// probe — healthcheck без curl в образе distroless: сам бинарник ходит в
// /health/ready и возвращает код выхода для Docker.
func probe(port int) int {
client := http.Client{Timeout: 2 * time.Second}
resp, err := client.Get(fmt.Sprintf("http://127.0.0.1:%d/health/ready", port))
if err != nil {
return 1
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return 1
}
return 0
}
+151
View File
@@ -0,0 +1,151 @@
package main
import (
"backend/internal/adapter/imagefile"
"backend/internal/adapter/queue"
"backend/internal/adapter/repository/postgres"
"backend/internal/adapter/storage"
"backend/internal/adapter/telemetry"
"backend/internal/application/usecase"
"backend/internal/config"
"context"
"flag"
"fmt"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/redis/go-redis/v9"
"os"
"os/signal"
"strconv"
"syscall"
"time"
)
func main() {
replay := flag.String("replay", "", "replay failed attachment UUID")
advance := flag.String("advance", "", "new version for ready/failed attachment UUID")
fault := flag.String("fault", "", "test-only crash point: after_original/after_objects/after_commit")
flag.Parse()
cfg, e := config.Load()
must(e)
observability, e := telemetry.Setup(context.Background(), "messenger-worker")
must(e)
defer func() {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
observability.Shutdown(ctx)
}()
dsn, e := config.ReadSecret(cfg.DatabaseURLFile)
must(e)
pc, e := pgxpool.ParseConfig(dsn)
must(e)
pc.MaxConns = 2
pc.ConnConfig.Tracer = telemetry.DBTracer{}
pc.ConnConfig.ConnectTimeout = time.Second
pc.ConnConfig.RuntimeParams["statement_timeout"] = "2000"
pc.ConnConfig.RuntimeParams["lock_timeout"] = "1000"
pool, e := pgxpool.NewWithConfig(context.Background(), pc)
must(e)
defer pool.Close()
repo := postgres.New(pool, nil)
must(observability.Gauges(func(ctx context.Context) (map[string]float64, error) {
m, e := repo.PipelineMetrics(ctx)
values := telemetry.PipelineValues(m)
values["db_pool_used"] = float64(pool.Stat().AcquiredConns())
values["db_pool_capacity"] = float64(pool.Stat().MaxConns())
return values, e
}))
if *replay != "" || *advance != "" {
s := *replay
if s == "" {
s = *advance
}
id, e := uuid.Parse(s)
must(e)
var ok bool
if *replay != "" {
ok, e = repo.ReplayJob(context.Background(), id)
} else {
ok, e = repo.AdvancePipeline(context.Background(), id)
}
must(e)
fmt.Printf("{\"changed\":%v}\n", ok)
return
}
s3path := os.Getenv("S3_CONFIG_FILE")
if s3path == "" {
s3path = "/run/secrets/s3_credentials"
}
s3cfg, e := storage.ReadConfig(s3path)
must(e)
objects, e := storage.FromConfig(s3cfg)
must(e)
url, e := config.ReadSecret(cfg.CacheURLFile)
must(e)
opts, e := redis.ParseURL(url)
must(e)
opts.MaxRetries = 0
opts.DialTimeout = time.Second
opts.ReadTimeout = 2 * time.Second
opts.WriteTimeout = time.Second
client := redis.NewClient(opts)
defer func() { _ = client.Close() }()
attempts := 5
if s := os.Getenv("WORKER_MAX_ATTEMPTS"); s != "" {
attempts, e = strconv.Atoi(s)
must(e)
}
if attempts < 1 || attempts > 10 {
must(fmt.Errorf("attempts out of bounds"))
}
pipeline := usecase.NewPipeline(repo, telemetry.Objects{Objects: objects}, imagefile.NewTransformer(), int32(attempts))
pipeline.OnResult = observability.JobResult
if *fault != "" {
switch *fault {
case "after_original", "after_objects", "after_commit":
default:
must(fmt.Errorf("invalid fault point"))
}
pipeline.Fault = func(point string) error {
if point == *fault {
os.Exit(88)
}
return nil
}
}
broker := queue.New(client, repo, uuid.NewString())
ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
defer stop()
logError := func(error) { fmt.Fprintln(os.Stderr, "{\"event\":\"pipeline_dependency_error\"}") }
cleanupDone := make(chan struct{})
go func() {
defer close(cleanupDone)
ticker := time.NewTicker(time.Second)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
cleanup, cancel := context.WithTimeout(ctx, 5*time.Second)
e := pipeline.Cleanup(cleanup)
cancel()
if e != nil {
logError(e)
}
}
}
}()
done := make(chan struct{})
go func() { defer close(done); broker.RunPublisher(ctx, logError) }()
broker.RunWorker(ctx, observability.Worker(pipeline.Process), logError)
stop()
<-done
<-cleanupDone
}
func must(e error) {
if e != nil {
fmt.Fprintln(os.Stderr, "worker initialization/operation failed")
os.Exit(1)
}
}
+15
View File
@@ -0,0 +1,15 @@
package: handler
generate:
models: true
chi-server: true
strict-server: true
output: api.gen.go
output-options:
overlay:
path: ../../../generator-overlay.yaml
# date-time -> handler.Timestamp: ровно 6 цифр дробной части (см. timestamp.go)
type-mapping:
string:
formats:
date-time:
type: Timestamp
+25
View File
@@ -0,0 +1,25 @@
package db
import (
"context"
"database/sql"
"embed"
_ "github.com/jackc/pgx/v5/stdlib"
"github.com/pressly/goose/v3"
)
//go:embed migrations/*.sql
var migrations embed.FS
func Migrate(ctx context.Context, dsn string) error {
conn, err := sql.Open("pgx", dsn)
if err != nil {
return err
}
defer func() { _ = conn.Close() }()
goose.SetBaseFS(migrations)
if err = goose.SetDialect("postgres"); err != nil {
return err
}
return goose.UpContext(ctx, conn, "migrations")
}
+45
View File
@@ -0,0 +1,45 @@
-- +goose Up
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
chat_count integer NOT NULL DEFAULT 0 CHECK (chat_count BETWEEN 0 AND 1000),
message_count integer NOT NULL DEFAULT 0 CHECK (message_count BETWEEN 0 AND 1000000),
attachment_count integer NOT NULL DEFAULT 0 CHECK (attachment_count BETWEEN 0 AND 10000)
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id),
user_id uuid NOT NULL REFERENCES users(id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX member_chats ON chat_members(user_id,chat_id);
CREATE INDEX chats_order ON chats(created_at,id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL REFERENCES users(id),
text text NOT NULL CHECK (char_length(text) BETWEEN 0 AND 4000),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id,sender_id) REFERENCES chat_members(chat_id,user_id)
);
CREATE INDEX messages_history ON messages(chat_id,created_at,id);
CREATE TABLE sessions (
token_hash text PRIMARY KEY CHECK (length(token_hash)=64),
user_id uuid NOT NULL REFERENCES users(id),
expires_at timestamptz NOT NULL
);
CREATE INDEX sessions_user ON sessions(user_id,expires_at);
-- +goose Down
DROP TABLE sessions;
DROP TABLE messages;
DROP TABLE chat_members;
DROP TABLE chats;
DROP TABLE users;
@@ -0,0 +1,12 @@
-- +goose Up
CREATE TABLE idempotency (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
payload_hash text NOT NULL CHECK (length(payload_hash)=64),
message_id uuid NOT NULL REFERENCES messages(id) DEFERRABLE INITIALLY DEFERRED,
expires_at timestamptz NOT NULL,
PRIMARY KEY(user_id,chat_id,key)
);
-- +goose Down
DROP TABLE idempotency;
@@ -0,0 +1,49 @@
-- +goose Up
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL REFERENCES users(id),
filename text NOT NULL CHECK(char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK(content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK(sha256 ~ '^[a-f0-9]{64}$'),
status text NOT NULL CHECK(status IN ('uploading','queued','processing','ready','failed','missing','abandoned')),
pipeline_version integer NOT NULL DEFAULT 1 CHECK(pipeline_version>0),
error_code text NOT NULL DEFAULT '',
variants jsonb NOT NULL DEFAULT '[]' CHECK(jsonb_typeof(variants)='array' AND jsonb_array_length(variants)<=3),
staging_expired boolean NOT NULL DEFAULT false,
staging_cleaned boolean NOT NULL DEFAULT false,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY(chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
);
CREATE INDEX attachments_owner ON attachments(owner_id,id);
CREATE INDEX attachments_staging_cleanup ON attachments(updated_at,id) WHERE NOT staging_cleaned;
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages(id),
attachment_id uuid NOT NULL REFERENCES attachments(id),
ordinal integer NOT NULL CHECK(ordinal BETWEEN 0 AND 9),
PRIMARY KEY(message_id,ordinal),
UNIQUE(message_id,attachment_id)
);
CREATE INDEX attachment_references ON message_attachments(attachment_id);
CREATE TABLE attachment_jobs (
id uuid PRIMARY KEY,
attachment_id uuid NOT NULL REFERENCES attachments(id),
pipeline_version integer NOT NULL CHECK(pipeline_version>0),
attempt integer NOT NULL DEFAULT 0 CHECK(attempt>=0),
traceparent text NOT NULL DEFAULT '' CHECK(char_length(traceparent)<=55),
tracestate text NOT NULL DEFAULT '' CHECK(char_length(tracestate)<=512),
lease_id uuid NOT NULL DEFAULT '00000000-0000-0000-0000-000000000000',
lease_until timestamptz NOT NULL DEFAULT 'epoch',
next_attempt timestamptz NOT NULL DEFAULT 'epoch',
published_at timestamptz NOT NULL DEFAULT 'epoch',
completed boolean NOT NULL DEFAULT false,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
UNIQUE(attachment_id,pipeline_version)
);
CREATE INDEX pending_attachment_jobs ON attachment_jobs(next_attempt,published_at) WHERE NOT completed;
-- +goose Down
DROP TABLE attachment_jobs;
DROP TABLE message_attachments;
DROP TABLE attachments;
@@ -0,0 +1,4 @@
-- +goose Up
ALTER TABLE attachment_jobs ADD COLUMN replay_count bigint NOT NULL DEFAULT 0 CHECK(replay_count>=0), ADD COLUMN replayed_at timestamptz NOT NULL DEFAULT 'epoch';
-- +goose Down
ALTER TABLE attachment_jobs DROP COLUMN replay_count, DROP COLUMN replayed_at;
@@ -0,0 +1,8 @@
-- +goose Up
-- Historical upload/create time is not evidence of acceptance/readiness; leave
-- pre-migration rows unknown instead of inventing SLO history.
ALTER TABLE attachments ADD COLUMN accepted_at timestamptz, ADD COLUMN first_ready_at timestamptz;
CREATE INDEX attachments_slo_accepted ON attachments(accepted_at) WHERE accepted_at IS NOT NULL;
-- +goose Down
DROP INDEX attachments_slo_accepted;
ALTER TABLE attachments DROP COLUMN accepted_at, DROP COLUMN first_ready_at;
+15
View File
@@ -0,0 +1,15 @@
-- name: ChargeAttachment :one
UPDATE users SET attachment_count=attachment_count+1 WHERE id=$1 AND attachment_count<10000 RETURNING id;
-- name: InsertAttachment :one
INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status)
VALUES($1,$2,$3,$4,$5,$6,$7,'uploading') RETURNING *;
-- name: GetAttachment :one
SELECT * FROM attachments WHERE id=$1 FOR SHARE;
-- name: SetAttachmentStatus :execrows
UPDATE attachments SET status=$2,updated_at=clock_timestamp() WHERE id=$1 AND status=$3 AND pipeline_version=0;
-- name: InsertMessageAttachment :exec
INSERT INTO message_attachments(message_id,attachment_id,ordinal) VALUES($1,$2,$3);
-- name: MessageAttachments :many
SELECT attachment_id FROM message_attachments WHERE message_id=$1 ORDER BY ordinal LIMIT 10;
-- name: ReconcileAttachments :many
SELECT * FROM attachments WHERE id>$1 ORDER BY id LIMIT 101;
+42
View File
@@ -0,0 +1,42 @@
-- name: CreateUser :one
INSERT INTO users(id,username,display_name,password_hash) VALUES($1,$2,$3,$4)
RETURNING id,username,display_name,created_at;
-- name: GetUser :one
SELECT id,username,display_name,created_at FROM users WHERE id=$1;
-- name: Credentials :one
SELECT id,password_hash FROM users WHERE username=$1;
-- name: ChargeChat :one
UPDATE users SET chat_count=chat_count+1 WHERE id=$1 AND chat_count<1000 RETURNING id;
-- name: ChargeMessage :one
UPDATE users SET message_count=message_count+1 WHERE id=$1 AND message_count<1000000 RETURNING id;
-- name: InsertChat :one
INSERT INTO chats(id,title) VALUES($1,$2) RETURNING id,title,created_at;
-- name: InsertMembers :exec
INSERT INTO chat_members(chat_id,user_id) SELECT $1,unnest($2::uuid[]);
-- name: GetChat :one
SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id)::uuid[] AS members
FROM chats c WHERE c.id=$1;
-- name: ListChats :many
SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id)::uuid[] AS members
FROM chats c JOIN chat_members m ON m.chat_id=c.id
WHERE m.user_id=sqlc.arg(member_id) AND (c.created_at,c.id)>(sqlc.arg(after_time)::timestamptz,sqlc.arg(after_id)::uuid)
ORDER BY c.created_at,c.id LIMIT sqlc.arg(page_limit);
-- name: InsertMessage :one
INSERT INTO messages(id,chat_id,sender_id,text) VALUES($1,$2,$3,$4) RETURNING id,chat_id,sender_id,text,created_at;
-- name: ListMessages :many
SELECT m.id,m.chat_id,m.sender_id,m.text,m.created_at,
ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY ordinal LIMIT 10)::uuid[] AS attachment_ids
FROM messages m WHERE m.chat_id=$1 AND (m.created_at,m.id)>(sqlc.arg(after_time)::timestamptz,sqlc.arg(after_id)::uuid)
ORDER BY m.created_at,m.id LIMIT sqlc.arg(page_limit);
-- name: LockSessionOwner :one
SELECT id FROM users WHERE id=$1 FOR UPDATE;
-- name: DeleteExpiredSessions :exec
DELETE FROM sessions WHERE user_id=$1 AND expires_at<=clock_timestamp();
-- name: InsertSession :one
INSERT INTO sessions(token_hash,user_id,expires_at)
SELECT $1,$2,$3 WHERE (SELECT count(*) FROM sessions WHERE user_id=$2)<100
RETURNING user_id;
-- name: AuthenticateSession :one
SELECT user_id FROM sessions WHERE token_hash=$1 AND expires_at>clock_timestamp();
-- name: RevokeSession :exec
DELETE FROM sessions WHERE token_hash=$1;
+12
View File
@@ -0,0 +1,12 @@
-- name: ReserveKey :one
INSERT INTO idempotency(user_id,chat_id,key,payload_hash,message_id,expires_at)
VALUES($1,$2,$3,$4,$5,clock_timestamp()+interval '24 hours')
ON CONFLICT(user_id,chat_id,key) DO UPDATE SET payload_hash=excluded.payload_hash,
message_id=excluded.message_id,expires_at=excluded.expires_at
WHERE idempotency.expires_at<=clock_timestamp()
RETURNING message_id;
-- name: ReplayMessage :one
SELECT m.id,m.chat_id,m.sender_id,m.text,m.created_at,i.payload_hash,
ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY ordinal LIMIT 10)::uuid[] AS attachment_ids
FROM idempotency i JOIN messages m ON m.id=i.message_id
WHERE i.user_id=$1 AND i.chat_id=$2 AND i.key=$3;
@@ -0,0 +1,5 @@
-- name: PipelineMetrics :one
SELECT count(*)::bigint AS depth,
COALESCE(EXTRACT(EPOCH FROM clock_timestamp()-min(j.created_at)),0)::float8 AS oldest
FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id AND j.pipeline_version=a.pipeline_version
WHERE a.status IN ('queued','processing') AND NOT j.completed;
+50
View File
@@ -0,0 +1,50 @@
-- name: InsertJob :exec
INSERT INTO attachment_jobs(id,attachment_id,pipeline_version,traceparent,tracestate)
VALUES(sqlc.arg(job_id),sqlc.arg(attachment_id),sqlc.arg(pipeline_version),sqlc.arg(traceparent),sqlc.arg(tracestate));
-- name: QueueAttachment :one
UPDATE attachments SET accepted_at=COALESCE(accepted_at,clock_timestamp()),status='queued',updated_at=clock_timestamp() WHERE id=$1 AND status='uploading' AND NOT staging_expired RETURNING *;
-- name: PendingJobs :many
SELECT j.* FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id AND a.pipeline_version=j.pipeline_version
WHERE NOT j.completed AND a.status IN ('queued','processing') AND j.next_attempt<=clock_timestamp()
AND j.lease_until<clock_timestamp() AND j.published_at<clock_timestamp()-interval '2 seconds'
ORDER BY j.created_at,j.id LIMIT 100;
-- name: Published :exec
UPDATE attachment_jobs SET published_at=clock_timestamp() WHERE id=$1;
-- name: AcquireJob :one
UPDATE attachment_jobs j SET attempt=j.attempt+1,lease_id=sqlc.arg(lease_id),lease_until=clock_timestamp()+sqlc.arg(lease_seconds)::int*interval '1 second'
WHERE j.id=sqlc.arg(job_id) AND j.pipeline_version=sqlc.arg(pipeline_version) AND NOT j.completed
AND j.next_attempt<=clock_timestamp() AND j.lease_until<clock_timestamp()
AND EXISTS(SELECT 1 FROM attachments a WHERE a.id=j.attachment_id AND a.pipeline_version=j.pipeline_version AND a.status IN ('queued','processing'))
RETURNING j.*;
-- name: ProcessingAttachment :one
UPDATE attachments SET status='processing',updated_at=clock_timestamp()
WHERE id=sqlc.arg(attachment_id) AND pipeline_version=sqlc.arg(pipeline_version) AND status IN ('queued','processing') RETURNING *;
-- name: CompleteAttachment :execrows
UPDATE attachments a SET first_ready_at=COALESCE(a.first_ready_at,clock_timestamp()),status='ready',variants=sqlc.arg(variants)::jsonb,error_code='',updated_at=clock_timestamp()
FROM attachment_jobs j WHERE j.id=sqlc.arg(job_id) AND j.attachment_id=a.id AND j.pipeline_version=a.pipeline_version
AND j.lease_id=sqlc.arg(lease_id) AND j.lease_until>clock_timestamp() AND NOT j.completed AND a.status='processing';
-- name: CompleteJob :exec
UPDATE attachment_jobs SET completed=true,lease_until='epoch' WHERE id=sqlc.arg(job_id) AND lease_id=sqlc.arg(lease_id);
-- name: RetryAttachment :execrows
UPDATE attachments a SET status=sqlc.arg(next_status),error_code=sqlc.arg(error_code),updated_at=clock_timestamp()
FROM attachment_jobs j WHERE j.id=sqlc.arg(job_id) AND j.attachment_id=a.id AND j.pipeline_version=a.pipeline_version
AND j.lease_id=sqlc.arg(lease_id) AND j.lease_until>clock_timestamp() AND NOT j.completed AND a.status='processing';
-- name: RetryJob :exec
UPDATE attachment_jobs SET lease_until='epoch',next_attempt=clock_timestamp()+sqlc.arg(delay_ms)::int*interval '1 millisecond',completed=sqlc.arg(completed)
WHERE id=sqlc.arg(job_id) AND lease_id=sqlc.arg(lease_id);
-- name: ReplayAttachment :one
UPDATE attachments SET status='queued',error_code='',updated_at=clock_timestamp() WHERE id=$1 AND status='failed' AND NOT staging_expired RETURNING *;
-- name: ReplayJobs :exec
UPDATE attachment_jobs SET replay_count=replay_count+1,replayed_at=clock_timestamp(),completed=false,attempt=0,lease_until='epoch',next_attempt='epoch',published_at='epoch' WHERE attachment_id=$1 AND pipeline_version=$2;
-- name: AdvancePipeline :one
UPDATE attachments SET status='queued',error_code='',variants='[]',pipeline_version=pipeline_version+1,staging_expired=false,updated_at=clock_timestamp() WHERE id=$1 AND (status='ready' OR (status='failed' AND NOT staging_expired)) RETURNING *;
-- name: StagingCleanup :many
WITH candidates AS (
SELECT id FROM attachments WHERE NOT staging_cleaned AND (status='ready' OR (status IN ('failed','uploading') AND updated_at<clock_timestamp()-interval '24 hours'))
ORDER BY updated_at,id LIMIT 20
)
UPDATE attachments a SET staging_expired=(a.status<>'ready') FROM candidates c WHERE a.id=c.id AND NOT a.staging_cleaned
AND (a.status='ready' OR (a.status IN ('failed','uploading') AND a.updated_at<clock_timestamp()-interval '24 hours')) RETURNING a.*;
-- name: StagingCleaned :exec
UPDATE attachments SET staging_cleaned=true WHERE id=$1 AND (staging_expired OR status='ready');
+11
View File
@@ -0,0 +1,11 @@
-- name: ReadySLI :one
WITH clock AS (SELECT clock_timestamp() AS now), cohort AS (
SELECT a.accepted_at,a.first_ready_at,c.now FROM attachments a CROSS JOIN clock c
WHERE a.accepted_at>c.now-interval '310 seconds' AND a.accepted_at<=c.now-interval '10 seconds'
)
SELECT count(*) FILTER(WHERE accepted_at>now-interval '20 seconds')::bigint AS matured10,
count(*) FILTER(WHERE accepted_at>now-interval '20 seconds' AND first_ready_at>=accepted_at AND first_ready_at<=accepted_at+interval '10 seconds')::bigint AS timely10,
count(*) FILTER(WHERE accepted_at>now-interval '40 seconds')::bigint AS matured30,
count(*) FILTER(WHERE accepted_at>now-interval '40 seconds' AND first_ready_at>=accepted_at AND first_ready_at<=accepted_at+interval '10 seconds')::bigint AS timely30,
count(*)::bigint AS matured300,
count(*) FILTER(WHERE first_ready_at>=accepted_at AND first_ready_at<=accepted_at+interval '10 seconds')::bigint AS timely300 FROM cohort;
+16
View File
@@ -0,0 +1,16 @@
version: '2'
sql:
- schema: ../migrations
queries: queries
engine: postgresql
gen:
go:
package: queries
out: ../../internal/adapter/repository/postgres/queries
sql_package: pgx/v5
emit_empty_slices: true
overrides:
- db_type: uuid
go_type: github.com/google/uuid.UUID
- db_type: timestamptz
go_type: time.Time
+15
View File
@@ -0,0 +1,15 @@
overlay: 1.0.0
info:
title: Go DTO generation view; normative contract remains unchanged
version: 1.0.0
actions:
# oapi-codegen 2.8 emits duplicate Body fields for two binary MIME types.
# The adapter reads the original bounded request stream and validates both types.
- target: $.paths['/api/v1/chats/{chat_id}/attachments'].post.requestBody.content['image/jpeg']
remove: true
# anyOf expresses runtime requirements, not different Go representations.
# The domain enforces text-or-attachments; the adapter keeps strict JSON decoding.
- target: $.components.schemas.CreateMessage.anyOf
remove: true
- target: $.components.schemas.Message.anyOf
remove: true
+346
View File
@@ -0,0 +1,346 @@
module backend
go 1.27.2
tool (
github.com/golangci/golangci-lint/v2/cmd/golangci-lint
github.com/oapi-codegen/oapi-codegen/v2/cmd/oapi-codegen
github.com/pressly/goose/v3/cmd/goose
github.com/sqlc-dev/sqlc/cmd/sqlc
)
require (
github.com/go-chi/chi/v5 v5.5.0
github.com/google/uuid v1.6.0
github.com/oapi-codegen/runtime v1.7.0
github.com/rs/zerolog v1.35.1
github.com/spf13/viper v1.21.0
)
require (
4d63.com/gocheckcompilerdirectives v1.4.0 // indirect
4d63.com/gochecknoglobals v0.2.2 // indirect
cel.dev/expr v0.25.3 // indirect
charm.land/lipgloss/v2 v2.0.6 // indirect
codeberg.org/chavacava/garif v0.2.1 // indirect
codeberg.org/polyfloyd/go-errorlint v1.9.0 // indirect
dev.gaijin.team/go/exhaustruct/v4 v4.0.0 // indirect
dev.gaijin.team/go/exhaustruct/v5 v5.2.0 // indirect
dev.gaijin.team/go/golib v0.8.1 // indirect
filippo.io/edwards25519 v1.2.0 // indirect
github.com/4meepo/tagalign v1.4.4 // indirect
github.com/Abirdcfly/dupword v0.1.8 // indirect
github.com/AdminBenni/iota-mixing v1.0.0 // indirect
github.com/AlwxSin/noinlineerr v1.0.6 // indirect
github.com/Antonboom/errname v1.1.2 // indirect
github.com/Antonboom/nilnil v1.1.2 // indirect
github.com/Antonboom/testifylint v1.6.4 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.24.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.15.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.13.0 // indirect
github.com/AzureAD/microsoft-authentication-library-for-go v1.10.1 // indirect
github.com/BurntSushi/toml v1.6.0 // indirect
github.com/ClickHouse/ch-go v0.75.0 // indirect
github.com/ClickHouse/clickhouse-go-linter v1.2.1 // indirect
github.com/ClickHouse/clickhouse-go/v2 v2.51.0 // indirect
github.com/Djarvur/go-err113 v0.1.1 // indirect
github.com/Masterminds/semver/v3 v3.5.0 // indirect
github.com/MirrexOne/unqueryvet v1.5.4 // indirect
github.com/OpenPeeDeeP/depguard/v2 v2.2.1 // indirect
github.com/alecthomas/chroma/v2 v2.27.0 // indirect
github.com/alecthomas/go-check-sumtype v0.5.1-0.20260828200218-ae6904d28606 // indirect
github.com/alexkohler/nakedret/v2 v2.0.6 // indirect
github.com/alexkohler/prealloc v1.1.0 // indirect
github.com/alfatraining/structtag v1.0.0 // indirect
github.com/alingse/asasalint v0.0.11 // indirect
github.com/alingse/nilnesserr v0.2.0 // indirect
github.com/andybalholm/brotli v1.3.0 // indirect
github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect
github.com/ashanbrown/forbidigo/v2 v2.3.1 // indirect
github.com/ashanbrown/makezero/v2 v2.2.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bkielbasa/cyclop v1.2.3 // indirect
github.com/blizzy78/varnamelen v0.8.0 // indirect
github.com/bombsimon/wsl/v4 v4.7.0 // indirect
github.com/bombsimon/wsl/v5 v5.9.0 // indirect
github.com/breml/bidichk v0.3.3 // indirect
github.com/breml/errchkjson v0.4.1 // indirect
github.com/butuzov/ireturn v0.4.1 // indirect
github.com/butuzov/mirror v1.3.3 // indirect
github.com/catenacyber/perfsprint v0.10.1 // indirect
github.com/ccojocar/zxcvbn-go v1.0.4 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/charithe/durationcheck v0.0.11 // indirect
github.com/charmbracelet/colorprofile v0.4.3 // indirect
github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886 // indirect
github.com/charmbracelet/x/ansi v0.11.8 // indirect
github.com/charmbracelet/x/term v0.2.2 // indirect
github.com/charmbracelet/x/termios v0.1.1 // indirect
github.com/charmbracelet/x/windows v0.2.2 // indirect
github.com/ckaznocha/intrange v0.3.1 // indirect
github.com/clipperhouse/displaywidth v0.11.0 // indirect
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
github.com/coder/websocket v1.8.15 // indirect
github.com/coreos/go-semver v0.3.1 // indirect
github.com/cubicdaiya/gonp v1.0.4 // indirect
github.com/curioswitch/go-reassign v0.3.0 // indirect
github.com/daixiang0/gci v0.13.7 // indirect
github.com/dave/dst v0.27.3 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/denis-tingaikin/go-header v0.5.0 // indirect
github.com/dlclark/regexp2/v2 v2.2.1 // indirect
github.com/dprotaso/go-yit v0.0.0-20220510233725-9ba8df137936 // indirect
github.com/dustin/go-humanize v1.1.0 // indirect
github.com/elastic/go-sysinfo v1.15.5 // indirect
github.com/elastic/go-windows v1.0.2 // indirect
github.com/ettle/strcase v0.2.0 // indirect
github.com/fatih/color v1.19.0 // indirect
github.com/fatih/structtag v1.2.0 // indirect
github.com/firefart/nonamedreturns v1.0.8 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/fzipp/gocyclo v0.6.0 // indirect
github.com/getkin/kin-openapi v0.142.0 // indirect
github.com/ghostiam/protogetter v1.0.1 // indirect
github.com/go-critic/go-critic v0.15.0 // indirect
github.com/go-faster/city v1.0.1 // indirect
github.com/go-faster/errors v0.8.0 // indirect
github.com/go-logr/logr v1.4.4 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-openapi/jsonpointer v1.0.0 // indirect
github.com/go-openapi/swag/jsonname v0.26.0 // indirect
github.com/go-sql-driver/mysql v1.10.1 // indirect
github.com/go-toolsmith/astcast v1.1.0 // indirect
github.com/go-toolsmith/astcopy v1.1.0 // indirect
github.com/go-toolsmith/astequal v1.2.0 // indirect
github.com/go-toolsmith/astfmt v1.1.0 // indirect
github.com/go-toolsmith/astp v1.1.0 // indirect
github.com/go-toolsmith/strparse v1.1.0 // indirect
github.com/go-toolsmith/typep v1.1.0 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/go-xmlfmt/xmlfmt v1.1.3 // indirect
github.com/gobwas/glob v0.2.3 // indirect
github.com/godoc-lint/godoc-lint v0.11.4 // indirect
github.com/gofrs/flock v0.13.1 // indirect
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
github.com/golang-jwt/jwt/v5 v5.3.1 // indirect
github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/golangci/asciicheck v0.5.0 // indirect
github.com/golangci/canonicalheader v0.0.0-20260827115959-a25c71c521f6 // indirect
github.com/golangci/dupl v0.0.0-20260401084720-c99c5cf5c202 // indirect
github.com/golangci/go-printf-func-name v0.1.1 // indirect
github.com/golangci/gofmt v0.0.0-20260820135601-e84e05053792 // indirect
github.com/golangci/golangci-lint/v2 v2.14.0 // indirect
github.com/golangci/golines v0.15.0 // indirect
github.com/golangci/misspell v0.8.0 // indirect
github.com/golangci/plugin-module-register v0.1.2 // indirect
github.com/golangci/revgrep v0.8.0 // indirect
github.com/golangci/rowserrcheck v0.0.0-20260419091836-c5f79b8a11ba // indirect
github.com/golangci/swaggoswag v0.0.0-20250504205917-77f2aca3143e // indirect
github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e // indirect
github.com/google/cel-go v0.28.0 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/gordonklaus/ineffassign v0.2.0 // indirect
github.com/gostaticanalysis/analysisutil v0.7.1 // indirect
github.com/gostaticanalysis/comment v1.5.0 // indirect
github.com/gostaticanalysis/forcetypeassert v0.2.0 // indirect
github.com/gostaticanalysis/nilerr v0.1.2 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect
github.com/hashicorp/go-immutable-radix/v2 v2.1.0 // indirect
github.com/hashicorp/go-version v1.9.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/hexops/gotextdiff v1.0.3 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
github.com/jackc/pgx/v5 v5.13.0 // indirect
github.com/jackc/puddle/v2 v2.2.3 // indirect
github.com/jgautheron/goconst v1.11.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jjti/go-spancheck v0.6.5 // indirect
github.com/joho/godotenv v1.5.1 // indirect
github.com/jonboulle/clockwork v0.5.0 // indirect
github.com/julz/importas v0.2.0 // indirect
github.com/karamaru-alpha/copyloopvar v1.2.2 // indirect
github.com/kisielk/errcheck v1.20.0 // indirect
github.com/kkHAIKE/contextcheck v1.1.6 // indirect
github.com/klauspost/compress v1.20.4 // indirect
github.com/klauspost/cpuid/v2 v2.4.0 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/kulti/thelper v0.7.1 // indirect
github.com/kunwardeep/paralleltest v1.0.15 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/ldez/exptostd v0.4.5 // indirect
github.com/ldez/gomoddirectives v0.10.0 // indirect
github.com/ldez/grignotin v0.10.1 // indirect
github.com/ldez/structtags v0.6.1 // indirect
github.com/ldez/tagliatelle v0.8.0 // indirect
github.com/ldez/usetesting v0.5.0 // indirect
github.com/leonklingele/grouper v1.1.2 // indirect
github.com/lucasb-eyer/go-colorful v1.4.1 // indirect
github.com/macabu/inamedparam v0.2.0 // indirect
github.com/manuelarte/embeddedstructfieldcheck v0.4.0 // indirect
github.com/manuelarte/funcorder v0.6.0 // indirect
github.com/maratori/testableexamples v1.0.1 // indirect
github.com/maratori/testpackage v1.1.2 // indirect
github.com/matoous/godox v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.15 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect
github.com/mattn/go-runewidth v0.0.24 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mfridman/interpolate v0.0.2 // indirect
github.com/mfridman/xflag v0.1.0 // indirect
github.com/mgechev/revive v1.17.0 // indirect
github.com/microsoft/go-mssqldb v1.11.2 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/minio-go/v7 v7.4.1 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/moricho/tparallel v0.3.2 // indirect
github.com/muesli/cancelreader v0.2.2 // indirect
github.com/nakabonne/nestif v0.3.1 // indirect
github.com/ncruces/go-sqlite3 v0.32.0 // indirect
github.com/ncruces/go-strftime v1.1.0 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/nishanths/exhaustive v0.13.0 // indirect
github.com/nishanths/predeclared v0.2.2 // indirect
github.com/nunnatsa/ginkgolinter v0.24.0 // indirect
github.com/oapi-codegen/oapi-codegen/v2 v2.8.0 // indirect
github.com/oasdiff/yaml v0.1.1 // indirect
github.com/oasdiff/yaml3 v0.0.14 // indirect
github.com/paulmach/orb v0.14.0 // indirect
github.com/pelletier/go-toml/v2 v2.4.3 // indirect
github.com/pganalyze/pg_query_go/v6 v6.2.2 // indirect
github.com/philhofer/fwd v1.2.0 // indirect
github.com/pierrec/lz4/v4 v4.1.33 // indirect
github.com/pingcap/errors v0.11.5-0.20250523034308-74f78ae071ee // indirect
github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 // indirect
github.com/pingcap/log v1.1.0 // indirect
github.com/pingcap/tidb/pkg/parser v0.0.0-20260418072757-ce92298d1124 // indirect
github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
github.com/pressly/goose/v3 v3.30.0 // indirect
github.com/prometheus/client_golang v1.12.1 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.32.1 // indirect
github.com/prometheus/procfs v0.23.0 // indirect
github.com/quasilyte/go-ruleguard v0.4.5 // indirect
github.com/quasilyte/go-ruleguard/dsl v0.3.23 // indirect
github.com/quasilyte/gogrep v0.5.0 // indirect
github.com/quasilyte/regex/syntax v0.0.0-20210819130434-b3f0c404a727 // indirect
github.com/quasilyte/stdinfo v0.0.0-20220114132959-f7386bf02567 // indirect
github.com/raeperd/recvcheck v0.3.1 // indirect
github.com/redis/go-redis/v9 v9.23.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/riza-io/grpc-go v0.2.0 // indirect
github.com/rogpeppe/go-internal v1.16.0 // indirect
github.com/rs/xid v1.6.0 // indirect
github.com/ryancurrah/gomodguard v1.4.1 // indirect
github.com/ryancurrah/gomodguard/v2 v2.1.3 // indirect
github.com/ryanrolds/sqlclosecheck v0.6.0 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/sanposhiho/wastedassign/v2 v2.1.0 // indirect
github.com/santhosh-tekuri/jsonschema/v6 v6.0.3 // indirect
github.com/sashamelentyev/interfacebloat v1.1.0 // indirect
github.com/sashamelentyev/usestdlibvars v1.29.0 // indirect
github.com/securego/gosec/v2 v2.29.0 // indirect
github.com/segmentio/asm v1.2.1 // indirect
github.com/sethvargo/go-retry v0.5.0 // indirect
github.com/shopspring/decimal v1.6.0 // indirect
github.com/sirupsen/logrus v1.10.2 // indirect
github.com/sivchari/containedctx v1.0.3 // indirect
github.com/sonatard/noctx v0.5.1 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/sourcegraph/go-diff v0.9.0 // indirect
github.com/speakeasy-api/jsonpath v0.6.3 // indirect
github.com/speakeasy-api/openapi v1.24.0 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/spf13/cobra v1.10.2 // indirect
github.com/spf13/pflag v1.0.10 // indirect
github.com/sqlc-dev/doubleclick v1.0.0 // indirect
github.com/sqlc-dev/sqlc v1.31.1 // indirect
github.com/ssgreg/nlreturn/v2 v2.2.1 // indirect
github.com/stbenjam/no-sprintf-host-port v0.3.1 // indirect
github.com/stretchr/objx v0.5.3 // indirect
github.com/stretchr/testify v1.12.1 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tetafro/godot v1.5.6 // indirect
github.com/tetratelabs/wazero v1.11.0 // indirect
github.com/timakin/bodyclose v0.0.0-20260723120731-857993a2939c // indirect
github.com/timonwong/loggercheck v0.12.0 // indirect
github.com/tinylib/msgp v1.6.4 // indirect
github.com/tomarrell/wrapcheck/v2 v2.12.0 // indirect
github.com/tommy-muehle/go-mnd/v2 v2.5.1 // indirect
github.com/tursodatabase/libsql-client-go v0.0.0-20260528064733-9d5d30a29a60 // indirect
github.com/ultraware/funlen v0.2.0 // indirect
github.com/ultraware/whitespace v0.2.0 // indirect
github.com/uudashr/gocognit v1.2.1 // indirect
github.com/uudashr/iface v1.5.1 // indirect
github.com/vertica/vertica-sql-go v1.3.9 // indirect
github.com/vmware-labs/yaml-jsonpath v0.3.2 // indirect
github.com/wasilibs/go-pgquery v0.0.0-20250409022910-10ac41983c07 // indirect
github.com/wasilibs/wazero-helpers v0.0.0-20240620070341-3dff1577cd52 // indirect
github.com/xen0n/gosmopolitan v1.3.0 // indirect
github.com/xo/terminfo v1.0.0 // indirect
github.com/yagipy/maintidx v1.0.0 // indirect
github.com/yandex-cloud/go-genproto v0.137.0 // indirect
github.com/ydb-platform/ydb-go-genproto v0.0.0-20260810123728-f0c151ab31b9 // indirect
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.5.3 // indirect
github.com/ydb-platform/ydb-go-sdk/v3 v3.156.1 // indirect
github.com/ydb-platform/ydb-go-yc v0.12.5 // indirect
github.com/ydb-platform/ydb-go-yc-metadata v0.6.2 // indirect
github.com/yeya24/promlinter v0.3.0 // indirect
github.com/ykadowak/zerologlint v0.1.5 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
github.com/ziutek/mymysql v1.5.4 // indirect
gitlab.com/bosi/decorder v0.4.2 // indirect
go-simpler.org/musttag v0.14.0 // indirect
go-simpler.org/sloglint v0.12.0 // indirect
go.augendre.info/arangolint v0.4.0 // indirect
go.augendre.info/fatcontext v0.10.1 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel v1.47.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.23.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.47.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.47.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.47.0 // indirect
go.opentelemetry.io/otel/log v1.47.0 // indirect
go.opentelemetry.io/otel/metric v1.47.0 // indirect
go.opentelemetry.io/otel/sdk v1.47.0 // indirect
go.opentelemetry.io/otel/sdk/log v1.47.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.47.0 // indirect
go.opentelemetry.io/otel/trace v1.47.0 // indirect
go.opentelemetry.io/proto/otlp v1.11.0 // indirect
go.uber.org/atomic v1.12.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.28.0 // indirect
go.yaml.in/yaml/v3 v3.0.6 // indirect
golang.org/x/crypto v0.59.0 // indirect
golang.org/x/exp v0.0.0-20261109185158-a52e3aef5920 // indirect
golang.org/x/exp/typeparams v0.0.0-20260811152304-ee035b5b010f // indirect
golang.org/x/mod v0.42.0 // indirect
golang.org/x/net v0.62.0 // indirect
golang.org/x/sync v0.25.0 // indirect
golang.org/x/sys v0.50.0 // indirect
golang.org/x/text v0.44.0 // indirect
golang.org/x/tools v0.52.1-0.20261106145633-d407941a339a // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20261110000943-09e21b237f2c // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20261110000943-09e21b237f2c // indirect
google.golang.org/grpc v1.85.0 // indirect
google.golang.org/protobuf v1.36.12 // indirect
gopkg.in/ini.v1 v1.67.3 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
honnef.co/go/tools v0.8.1 // indirect
howett.net/plist v1.0.1 // indirect
modernc.org/libc v1.80.2 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.12.1 // indirect
modernc.org/sqlite v1.62.0 // indirect
mvdan.cc/gofumpt v0.12.0 // indirect
mvdan.cc/unparam v0.0.0-20260823230713-2fa3d841b0c8 // indirect
)
+1335
View File
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,67 @@
package integration
import (
"backend/internal/adapter/storage"
"backend/internal/domain/port"
"bytes"
"context"
"github.com/google/uuid"
"os"
"testing"
"time"
)
func TestS3InventoryAndStatReturnComparableVersionMetadata(t *testing.T) {
if os.Getenv("RUN_ISOLATED_RECONCILIATION") != "1" {
t.Skip("explicit isolated stand required")
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
cfg, e := storage.ReadConfig("/run/secrets/s3_credentials")
if e != nil {
t.Fatal("configuration unavailable")
}
s, e := storage.FromConfig(cfg)
if e != nil {
t.Fatal(e)
}
key := "originals/" + uuid.NewString()
defer func() {
if err := s.Remove(context.Background(), key); err != nil {
t.Error("object fixture cleanup failed")
}
}()
if e = s.Put(ctx, key, bytes.NewReader([]byte("synthetic-version-probe")), 23, "application/octet-stream"); e != nil {
t.Fatal(e)
}
stat, e := s.Stat(ctx, key)
if e != nil {
t.Fatal(e)
}
var listed port.Object
after := ""
found := false
for page := 0; page < 10; page++ {
rows, more, e := s.List(ctx, after)
if e != nil {
t.Fatal(e)
}
for _, row := range rows {
if row.Key == key {
listed = row
found = true
break
}
}
if found || !more {
break
}
after = rows[len(rows)-1].Key
}
if !found {
t.Fatal("own object absent from bounded inventory")
}
if listed.ETag != stat.ETag || listed.Size != stat.Size || !listed.Modified.Equal(stat.Modified) {
t.Fatalf("inconsistent version metadata: etag_equal=%t size_equal=%t list_time=%s stat_time=%s", listed.ETag == stat.ETag, listed.Size == stat.Size, listed.Modified.Format(time.RFC3339Nano), stat.Modified.Format(time.RFC3339Nano))
}
}
File diff suppressed because it is too large Load Diff
+111
View File
@@ -0,0 +1,111 @@
package handler
import (
"context"
"github.com/google/uuid"
"net/http"
"backend/internal/domain/model"
)
// Реализация StrictServerInterface: вызвать use-case и перевести результат в
// API-модели. Ошибки уходят в writeError, он один решает про HTTP-статус.
func (h *API) Liveness(context.Context, LivenessRequestObject) (LivenessResponseObject, error) {
return Liveness200JSONResponse{Body: Health{Status: Ok}}, nil
}
// Readiness rejects draining instances and unavailable required dependencies.
func (h *API) Readiness(ctx context.Context, _ ReadinessRequestObject) (ReadinessResponseObject, error) {
if !h.ready.Load() {
return nil, errDraining
}
if err := h.probe(ctx); err != nil {
return nil, model.ErrUnavailable
}
return Readiness200JSONResponse{Body: Health{Status: Ok}}, nil
}
func (h *API) RegisterUser(ctx context.Context, req RegisterUserRequestObject) (RegisterUserResponseObject, error) {
if req.Body.Password == nil {
return nil, &model.ValidationError{Message: "password is required"}
}
user, err := h.auth.Register(ctx, req.Body.Username, req.Body.DisplayName, *req.Body.Password)
if err != nil {
return nil, err
}
return RegisterUser201JSONResponse{Body: toUser(user)}, nil
}
func (h *API) CurrentUser(ctx context.Context, _ CurrentUserRequestObject) (CurrentUserResponseObject, error) {
return CurrentUser200JSONResponse{Body: toUser(currentUser(ctx))}, nil
}
func (h *API) CreateChat(ctx context.Context, req CreateChatRequestObject) (CreateChatResponseObject, error) {
chat, err := h.app.CreateChat(ctx, currentUser(ctx).ID, req.Body.Title, req.Body.MemberIds)
if err != nil {
return nil, err
}
return CreateChat201JSONResponse{Body: toChat(chat)}, nil
}
func (h *API) ListChats(ctx context.Context, req ListChatsRequestObject) (ListChatsResponseObject, error) {
page, err := h.app.ListChats(ctx, currentUser(ctx).ID, req.Params.Limit, req.Params.Cursor)
if err != nil {
return nil, err
}
return ListChats200JSONResponse{Body: ChatPage{Items: mapAll(page.Items, toChat), NextCursor: page.NextCursor}}, nil
}
func (h *API) GetChat(ctx context.Context, req GetChatRequestObject) (GetChatResponseObject, error) {
chat, err := h.app.GetChat(ctx, currentUser(ctx).ID, req.ChatId)
if err != nil {
return nil, err
}
return GetChat200JSONResponse{Body: toChat(chat)}, nil
}
func (h *API) CreateMessage(ctx context.Context, req CreateMessageRequestObject) (CreateMessageResponseObject, error) {
r := ctx.Value(sessionRequestKey{}).(*http.Request)
ids := []uuid.UUID{}
if req.Body.AttachmentIds != nil {
ids = *req.Body.AttachmentIds
}
msg, err := h.app.SendMessage(ctx, currentUser(ctx).ID, req.ChatId, req.Body.Text, r.Header.Get("Idempotency-Key"), ids)
if err != nil {
return nil, err
}
return CreateMessage201JSONResponse{Body: toMessage(msg)}, nil
}
func (h *API) ListMessages(ctx context.Context, req ListMessagesRequestObject) (ListMessagesResponseObject, error) {
page, err := h.app.ListMessages(ctx, currentUser(ctx).ID, req.ChatId, req.Params.Limit, req.Params.Cursor)
if err != nil {
return nil, err
}
return ListMessages200JSONResponse{Body: MessagePage{Items: mapAll(page.Items, toMessage), NextCursor: page.NextCursor}}, nil
}
func toUser(u model.User) User {
return User{Id: u.ID, Username: u.Username, DisplayName: u.DisplayName, CreatedAt: Timestamp(u.CreatedAt)}
}
func toChat(c model.Chat) Chat {
return Chat{Id: c.ID, Title: c.Title, MemberIds: c.MemberIDs, CreatedAt: Timestamp(c.CreatedAt)}
}
func toMessage(m model.Message) Message {
ids := m.AttachmentIDs
if ids == nil {
ids = []uuid.UUID{}
}
return Message{Id: m.ID, ChatId: m.ChatID, SenderId: m.SenderID, Text: m.Text, CreatedAt: Timestamp(m.CreatedAt), AttachmentIds: ids}
}
func mapAll[T, R any](items []T, f func(T) R) []R {
out := make([]R, len(items))
for i, item := range items {
out[i] = f(item)
}
return out
}
@@ -0,0 +1,96 @@
package handler
import (
"backend/internal/adapter/imagefile"
"backend/internal/domain/model"
"context"
"errors"
"io"
"net/http"
"strconv"
)
func toAttachment(a model.Attachment) Attachment {
variants := make([]Variant, 0, len(a.Variants))
for _, v := range a.Variants {
variants = append(variants, Variant{Name: VariantName(v.Name), ContentType: VariantContentType(v.ContentType), SizeBytes: int(v.SizeBytes), Sha256: v.SHA256, Width: v.Width, Height: v.Height})
}
var code *string
if a.ErrorCode != "" {
code = &a.ErrorCode
}
return Attachment{Id: a.ID, ChatId: a.ChatID, OwnerId: a.OwnerID, Filename: a.Filename, ContentType: AttachmentContentType(a.ContentType), SizeBytes: int(a.SizeBytes), Sha256: a.SHA256, Status: AttachmentStatus(a.Status), Variants: variants, ErrorCode: code, CreatedAt: Timestamp(a.CreatedAt), UpdatedAt: Timestamp(a.UpdatedAt)}
}
func (h *API) UploadAttachment(ctx context.Context, req UploadAttachmentRequestObject) (UploadAttachmentResponseObject, error) {
caller := currentUser(ctx).ID
if _, err := h.app.GetChat(ctx, caller, req.ChatId); err != nil {
return nil, err
}
select {
case h.uploadGate <- struct{}{}:
defer func() { <-h.uploadGate }()
case <-ctx.Done():
return nil, errOverloaded
}
r := ctx.Value(sessionRequestKey{}).(*http.Request)
if r.ContentLength > model.MaxImageBytes {
return nil, errPayloadTooLarge
}
f, err := imagefile.Inspect(r.Body, req.Params.Filename, r.Header.Get("Content-Type"))
if errors.Is(err, imagefile.ErrTooLarge) {
return nil, errPayloadTooLarge
}
if errors.Is(err, imagefile.ErrUnsupported) {
return nil, errUnsupportedImage
}
if err != nil {
return nil, err
}
defer f.Dispose()
a, err := h.attachments.Upload(ctx, caller, req.ChatId, req.Params.Filename, f.ContentType, f.SHA256, f.Size, f.File)
if err != nil {
return nil, err
}
return UploadAttachment202JSONResponse{Body: toAttachment(a)}, nil
}
func (h *API) GetAttachment(ctx context.Context, req GetAttachmentRequestObject) (GetAttachmentResponseObject, error) {
a, e := h.attachments.Get(ctx, currentUser(ctx).ID, req.AttachmentId)
if e != nil {
return nil, e
}
return GetAttachment200JSONResponse{Body: toAttachment(a)}, nil
}
// Closing in the response visitor keeps the S3 stream alive until serialization completes.
type originalResponse struct {
body io.ReadCloser
size int64
mime string
}
func (r originalResponse) VisitDownloadAttachmentResponse(w http.ResponseWriter) error {
defer r.body.Close()
w.Header().Set("Content-Type", r.mime)
w.Header().Set("Content-Length", strconv.FormatInt(r.size, 10))
w.Header().Set("X-Content-Type-Options", "nosniff")
w.WriteHeader(200)
_, err := io.CopyN(w, r.body, r.size)
if err != nil {
panic(http.ErrAbortHandler)
}
return nil
}
func (h *API) DownloadAttachment(ctx context.Context, req DownloadAttachmentRequestObject) (DownloadAttachmentResponseObject, error) {
if _, e := h.attachments.Get(ctx, currentUser(ctx).ID, req.AttachmentId); e != nil {
return nil, e
}
variant := "original"
if req.Params.Variant != nil {
variant = string(*req.Params.Variant)
}
a, r, e := h.attachments.Content(ctx, currentUser(ctx).ID, req.AttachmentId, variant)
if e != nil {
return nil, e
}
return originalResponse{r, a.SizeBytes, a.ContentType}, nil
}
+47
View File
@@ -0,0 +1,47 @@
package handler
import (
"backend/internal/domain/model"
"context"
"net/http"
"strings"
"time"
"unicode/utf8"
)
type sessionRequestKey struct{}
func (h *API) CreateSession(ctx context.Context, _ CreateSessionRequestObject) (CreateSessionResponseObject, error) {
r := ctx.Value(sessionRequestKey{}).(*http.Request)
username, password, ok := r.BasicAuth()
if !ok || len(username) > 32 || len(password) > 512 || utf8.RuneCountInString(password) > 128 {
return nil, model.ErrUnauthorized
}
s, err := h.auth.Login(ctx, username, password)
if err != nil {
return nil, err
}
return CreateSession201JSONResponse{Body: Session{Token: s.Token, TokenType: Bearer, ExpiresAt: Timestamp(s.ExpiresAt), User: toUser(s.User)}}, nil
}
func (h *API) RevokeSession(ctx context.Context, _ RevokeSessionRequestObject) (RevokeSessionResponseObject, error) {
r := ctx.Value(sessionRequestKey{}).(*http.Request)
parts := strings.Fields(r.Header.Get("Authorization"))
if err := h.auth.Logout(ctx, parts[1]); err != nil {
return nil, err
}
return RevokeSession204Response{}, nil
}
func (h *API) requestContext(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
duration := 5 * time.Second
if strings.HasSuffix(r.URL.Path, "/attachments") || strings.HasSuffix(r.URL.Path, "/content") {
duration = 20 * time.Second
}
ctx, cancel := context.WithTimeout(r.Context(), duration)
defer cancel()
if r.URL.Path == "/api/v1/auth/sessions" {
w.Header().Set("WWW-Authenticate", `Basic realm="messenger", charset="UTF-8"`)
}
next.ServeHTTP(w, r.WithContext(context.WithValue(ctx, sessionRequestKey{}, r)))
})
}
@@ -0,0 +1,85 @@
package handler
import (
"encoding/json"
"errors"
"net/http"
"backend/internal/domain/model"
)
var (
errRouteNotFound = errors.New("route not found")
errMethodNotAllowed = errors.New("method not allowed")
errPayloadTooLarge = errors.New("payload too large")
errOverloaded = errors.New("overloaded")
errDraining = errors.New("draining")
errUnsupportedImage = errors.New("unsupported image")
)
type apiError struct {
status int
code ErrorErrorCode
message string
}
// classify — единственное место, где ошибка превращается в HTTP-статус.
func classify(err error) apiError {
var invalid *model.ValidationError
switch {
case errors.As(err, &invalid):
return apiError{http.StatusBadRequest, InvalidRequest, invalid.Message}
case errors.Is(err, model.ErrUnauthorized):
return apiError{http.StatusUnauthorized, Unauthorized, "нужна действующая сессия"}
case errors.Is(err, model.ErrNotFound), errors.Is(err, errRouteNotFound):
return apiError{http.StatusNotFound, NotFound, "объект не найден"}
case errors.Is(err, errMethodNotAllowed):
return apiError{http.StatusMethodNotAllowed, InvalidRequest, "метод не поддерживается"}
case errors.Is(err, model.ErrUsernameTaken):
return apiError{http.StatusConflict, Conflict, "username уже занят"}
case errors.Is(err, model.ErrConflict):
return apiError{http.StatusConflict, Conflict, "конфликт запроса"}
case errors.Is(err, model.ErrUnavailable):
return apiError{http.StatusServiceUnavailable, Unavailable, "зависимость недоступна"}
case errors.Is(err, errPayloadTooLarge):
return apiError{http.StatusRequestEntityTooLarge, PayloadTooLarge, "тело запроса больше допустимого"}
case errors.Is(err, errUnsupportedImage):
return apiError{http.StatusUnsupportedMediaType, UnsupportedMediaType, "нужен корректный PNG или JPEG с верным Content-Type"}
case errors.Is(err, model.ErrStorageFull):
return apiError{http.StatusServiceUnavailable, Unavailable, "хранилище заполнено, запись невозможна; чтение работает"}
case errors.Is(err, errOverloaded):
return apiError{http.StatusServiceUnavailable, Unavailable, "сервер перегружен, повторите позже"}
case errors.Is(err, errDraining):
return apiError{http.StatusServiceUnavailable, Unavailable, "сервер останавливается"}
default:
return apiError{http.StatusInternalServerError, InternalError, "внутренняя ошибка"}
}
}
func (h *API) writeError(w http.ResponseWriter, r *http.Request, err error) {
e := classify(err)
if e.status == http.StatusInternalServerError {
h.log.Error().Err(err).Str("request_id", requestIDFrom(r.Context())).Msg("unexpected error")
}
if errors.Is(err, errOverloaded) {
w.Header().Set("Retry-After", "1")
}
var body Error
body.Error.Code = e.code
body.Error.Message = e.message
body.Error.RequestId = requestIDFrom(r.Context())
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(e.status)
_ = json.NewEncoder(w).Encode(body)
}
// bodyError — JSON синтаксически верен (это проверил jsonBody), но не
// подходит к схеме: лишнее поле или неверный тип.
func (h *API) bodyError(w http.ResponseWriter, r *http.Request, _ error) {
h.writeError(w, r, &model.ValidationError{Message: "тело запроса не соответствует схеме: лишнее поле или неверный тип"})
}
// paramError — path или query параметр не разобрался (UUID, целое число).
func (h *API) paramError(w http.ResponseWriter, r *http.Request, _ error) {
h.writeError(w, r, &model.ValidationError{Message: "некорректный параметр запроса"})
}
+78
View File
@@ -0,0 +1,78 @@
package handler
import (
"net/http"
"sync/atomic"
"time"
)
// gate ограничивает одновременную обработку запросов API: не больше inflight
// в работе и не больше queue в ожидании слота. Лишние сразу получают 503 с
// Retry-After, ждавшие дольше timeout — тоже 503. Так перегрузка превращается
// в быстрый предсказуемый отказ, а не в рост горутин, памяти и задержки.
type gate struct {
slots chan struct{} // nil — без лимита
maxQueue int64
timeout time.Duration
waiting atomic.Int64
rejected atomic.Int64
}
func newGate(inflight, queue int, timeout time.Duration) *gate {
g := &gate{maxQueue: int64(queue), timeout: timeout}
if inflight > 0 {
g.slots = make(chan struct{}, inflight)
}
return g
}
func (g *gate) middleware(reject func(http.ResponseWriter, *http.Request, error)) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if g.slots == nil {
next.ServeHTTP(w, r)
return
}
if !g.acquire(r) {
g.rejected.Add(1)
reject(w, r, errOverloaded)
return
}
defer func() { <-g.slots }()
next.ServeHTTP(w, r)
})
}
}
func (g *gate) acquire(r *http.Request) bool {
select {
case g.slots <- struct{}{}:
return true
default:
}
if g.waiting.Add(1) > g.maxQueue {
g.waiting.Add(-1)
return false
}
defer g.waiting.Add(-1)
timer := time.NewTimer(g.timeout)
defer timer.Stop()
select {
case g.slots <- struct{}{}:
return true
case <-timer.C:
return false
case <-r.Context().Done():
return false
}
}
type gateStats struct {
Inflight int `json:"inflight"`
Waiting int64 `json:"waiting"`
Rejected int64 `json:"rejected"`
}
func (g *gate) stats() gateStats {
return gateStats{Inflight: len(g.slots), Waiting: g.waiting.Load(), Rejected: g.rejected.Load()}
}
@@ -0,0 +1,102 @@
package handler
import (
"errors"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
)
// gatedServer пропускает запросы через gate в обработчик, который держит слот,
// пока тест не закроет release.
func gatedServer(t *testing.T, g *gate) (*httptest.Server, chan struct{}, chan struct{}) {
t.Helper()
entered, release := make(chan struct{}, 10), make(chan struct{})
reject := func(w http.ResponseWriter, _ *http.Request, err error) {
if !errors.Is(err, errOverloaded) {
t.Errorf("unexpected error %v", err)
}
w.Header().Set("Retry-After", "1")
w.WriteHeader(http.StatusServiceUnavailable)
}
h := g.middleware(reject)(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
entered <- struct{}{}
<-release
w.WriteHeader(http.StatusOK)
}))
srv := httptest.NewServer(h)
t.Cleanup(srv.Close)
return srv, entered, release
}
func get(t *testing.T, url string) int {
t.Helper()
resp, err := http.Get(url)
if err != nil {
t.Error(err)
return 0
}
resp.Body.Close()
return resp.StatusCode
}
func waitFor(t *testing.T, cond func() bool) {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for !cond() {
if time.Now().After(deadline) {
t.Fatal("condition not reached")
}
time.Sleep(time.Millisecond)
}
}
func TestGateRejectsBeyondQueueAndRecovers(t *testing.T) {
t.Parallel()
g := newGate(1, 1, time.Minute)
srv, entered, release := gatedServer(t, g)
statuses := make(chan int, 2)
var wg sync.WaitGroup
wg.Go(func() { statuses <- get(t, srv.URL) }) // занимает слот
<-entered
wg.Go(func() { statuses <- get(t, srv.URL) }) // ждёт в очереди
waitFor(t, func() bool { return g.waiting.Load() == 1 })
if code := get(t, srv.URL); code != http.StatusServiceUnavailable {
t.Fatalf("third request: %d, want 503", code)
}
close(release)
wg.Wait()
close(statuses)
for code := range statuses {
if code != http.StatusOK {
t.Fatalf("admitted request got %d", code)
}
}
if code := get(t, srv.URL); code != http.StatusOK {
t.Fatalf("after load: %d, want recovery to 200", code)
}
if g.stats().Rejected != 1 {
t.Fatalf("rejected %d", g.stats().Rejected)
}
}
func TestGateTimesOutQueuedRequest(t *testing.T) {
t.Parallel()
g := newGate(1, 10, 20*time.Millisecond)
srv, entered, release := gatedServer(t, g)
defer close(release)
go get(t, srv.URL)
<-entered
start := time.Now()
if code := get(t, srv.URL); code != http.StatusServiceUnavailable {
t.Fatalf("queued request: %d, want 503 after timeout", code)
}
if time.Since(start) > time.Second {
t.Fatal("queue timeout not applied")
}
}
+103
View File
@@ -0,0 +1,103 @@
package handler
import (
"context"
"net/http"
"sync/atomic"
"time"
"backend/internal/adapter/telemetry"
"backend/internal/application/usecase"
"github.com/go-chi/chi/v5"
"github.com/rs/zerolog"
)
// Limits — защитные лимиты HTTP-слоя (трек 5 «управляемая деградация»).
// Нулевое значение отключает лимит.
type Limits struct {
MaxBodyBytes int64 // больше — 413
MaxInflight int // одновременно обрабатываемые запросы API
MaxQueue int // ждущие свободного слота; больше — сразу 503
QueueTimeout time.Duration // дольше ждал слот — 503
}
// API — driving HTTP-адаптер: реализует StrictServerInterface из api.gen.go.
//
//go:generate go tool oapi-codegen -config ../../../codegen.yaml ../../../../contracts/openapi.json
type API struct {
app *usecase.Messenger
auth *usecase.Auth
attachments *usecase.Attachments
uploadGate chan struct{}
probe func(context.Context) error
limits Limits
gate *gate
storeStats func() any
ready atomic.Bool
log *zerolog.Logger
}
// New собирает HTTP-адаптер. storeStats отдаёт состояние хранилища для
// /debug/stats.
func New(app *usecase.Messenger, auth *usecase.Auth, attachments *usecase.Attachments, probe func(context.Context) error, limits Limits, storeStats func() any, log *zerolog.Logger) *API {
h := &API{
app: app,
auth: auth,
attachments: attachments,
uploadGate: make(chan struct{}, 1),
probe: probe,
limits: limits,
gate: newGate(limits.MaxInflight, limits.MaxQueue, limits.QueueTimeout),
storeStats: storeStats,
log: log,
}
h.ready.Store(true)
return h
}
// Drain переводит /health/ready в 503 перед остановкой сервера.
func (h *API) Drain() { h.ready.Store(false) }
func (h *API) Router() http.Handler {
r := chi.NewRouter()
r.Use(requestID, telemetry.HTTP, h.recoverer, h.accessLog, h.requestContext)
r.NotFound(func(w http.ResponseWriter, r *http.Request) {
h.writeError(w, r, errRouteNotFound)
})
r.MethodNotAllowed(func(w http.ResponseWriter, r *http.Request) {
h.writeError(w, r, errMethodNotAllowed)
})
strict := NewStrictHandlerWithOptions(h, nil, StrictHTTPServerOptions{
RequestErrorHandlerFunc: h.bodyError,
ResponseErrorHandlerFunc: h.writeError,
})
api := ServerInterfaceWrapper{Handler: strict, ErrorHandlerFunc: h.paramError}
// Health и статистика — мимо очереди: под перегрузкой они должны отвечать.
r.Get("/health/live", api.Liveness)
r.Get("/health/ready", api.Readiness)
r.Get("/debug/stats", h.stats)
r.Route("/api/v1", func(r chi.Router) {
r.Use(h.gate.middleware(h.writeError))
r.With(h.jsonBody).Post("/users", api.RegisterUser)
r.Post("/auth/sessions", api.CreateSession)
// Сначала аутентификация, потом тело и доступ к ресурсу.
r.Group(func(r chi.Router) {
r.Use(h.authenticate, h.validQuery, h.jsonBody)
r.Delete("/auth/sessions/current", api.RevokeSession)
r.Get("/users/me", api.CurrentUser)
r.Post("/chats", api.CreateChat)
r.Get("/chats", api.ListChats)
r.Get("/chats/{chat_id}", api.GetChat)
r.Post("/chats/{chat_id}/messages", api.CreateMessage)
r.Get("/chats/{chat_id}/messages", api.ListMessages)
r.Post("/chats/{chat_id}/attachments", api.UploadAttachment)
r.Get("/attachments/{attachment_id}", api.GetAttachment)
r.Get("/attachments/{attachment_id}/content", api.DownloadAttachment)
})
})
return r
}
+65
View File
@@ -0,0 +1,65 @@
package handler
import (
"bytes"
"encoding/json"
"fmt"
"time"
)
// Timestamp подставляется кодогенератором вместо time.Time (codegen.yaml,
// type-mapping): RFC 3339 UTC с ровно шестью цифрами дробной части.
// time.Time срезает хвостовые нули, а fromisoformat в Python 3.9–3.10
// (на нём публичные тесты) понимает дробную часть только из 3 или 6 цифр.
type Timestamp time.Time
const timestampLayout = "2006-01-02T15:04:05.000000Z"
func (t Timestamp) MarshalJSON() ([]byte, error) {
return []byte(`"` + time.Time(t).UTC().Format(timestampLayout) + `"`), nil
}
func (t *Timestamp) UnmarshalJSON(data []byte) error {
var parsed time.Time
if err := json.Unmarshal(data, &parsed); err != nil {
return err
}
*t = Timestamp(parsed)
return nil
}
// Тела запросов с лишними полями — 400 по контракту, а сгенерированный
// декодер их молча игнорирует. Строгий разбор подключается методами на
// сгенерированных типах.
func (c *CreateUser) UnmarshalJSON(data []byte) error {
type plain CreateUser
return decodeStrict(data, (*plain)(c))
}
func (c *CreateChat) UnmarshalJSON(data []byte) error {
type plain CreateChat
return decodeStrict(data, (*plain)(c))
}
func (c *CreateMessage) UnmarshalJSON(data []byte) error {
var fields map[string]json.RawMessage
if err := json.Unmarshal(data, &fields); err != nil {
return err
}
raw, ok := fields["text"]
if !ok || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) {
return fmt.Errorf("text must be a present string")
}
if ids, ok := fields["attachment_ids"]; ok && bytes.Equal(bytes.TrimSpace(ids), []byte("null")) {
return fmt.Errorf("attachment_ids must be an array")
}
type plain CreateMessage
return decodeStrict(data, (*plain)(c))
}
func decodeStrict(data []byte, dst any) error {
dec := json.NewDecoder(bytes.NewReader(data))
dec.DisallowUnknownFields()
return dec.Decode(dst)
}
@@ -0,0 +1,17 @@
package handler
import (
"encoding/json"
"testing"
)
func TestMessageJSONRequiresStringTextAndArrayIDs(t *testing.T) {
for _, body := range []string{`{"attachment_ids":[]}`, `{"text":null,"attachment_ids":[]}`, `{"text":"ok","attachment_ids":null}`} {
t.Run(body, func(t *testing.T) {
var message CreateMessage
if err := json.Unmarshal([]byte(body), &message); err == nil {
t.Fatal("accepted a missing/null required field")
}
})
}
}
@@ -0,0 +1,164 @@
package handler
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/url"
"os"
"runtime/debug"
"strings"
"time"
"unicode/utf8"
"backend/internal/domain/model"
"github.com/go-chi/chi/v5"
"github.com/go-chi/chi/v5/middleware"
"github.com/google/uuid"
"go.opentelemetry.io/otel/trace"
)
type ctxKey int
const (
requestIDKey ctxKey = iota
userKey
)
// requestID ставит X-Request-Id в каждый ответ, включая ошибки и health.
func requestID(next http.Handler) http.Handler {
instance := os.Getenv("INSTANCE_ALIAS")
if instance == "" {
instance = uuid.NewString()
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
id := uuid.NewString()
w.Header().Set("X-Request-Id", id)
w.Header().Set("X-Instance-Id", instance)
next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), requestIDKey, id)))
})
}
func requestIDFrom(ctx context.Context) string {
id, _ := ctx.Value(requestIDKey).(string)
return id
}
// recoverer превращает панику в 500 без деталей; стек — только в лог.
func (h *API) recoverer(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
defer func() {
rec := recover()
if rec == nil {
return
}
if rec == http.ErrAbortHandler {
panic(rec)
}
h.log.Error().Interface("panic", rec).Bytes("stack", debug.Stack()).
Str("request_id", requestIDFrom(r.Context())).Msg("panic")
h.writeError(w, r, errors.New("panic"))
}()
next.ServeHTTP(w, r)
})
}
// accessLog пишет метод, шаблон маршрута, статус и длительность. Тело, query
// и заголовки не логируются: там текст сообщений и идентификаторы.
func (h *API) accessLog(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
ww := middleware.NewWrapResponseWriter(w, r.ProtoMajor)
next.ServeHTTP(ww, r)
route := chi.RouteContext(r.Context()).RoutePattern()
if route == "" {
route = "unmatched"
}
event := h.log.Info()
if ww.Status() >= http.StatusInternalServerError {
event = h.log.Warn()
}
event.Str("request_id", requestIDFrom(r.Context())).
Str("trace_id", trace.SpanContextFromContext(r.Context()).TraceID().String()).
Str("span_id", trace.SpanContextFromContext(r.Context()).SpanID().String()).
Str("method", r.Method).
Str("route", route).
Int("status", ww.Status()).
Dur("duration", time.Since(start)).
Msg("request")
})
}
// authenticate verifies persisted Bearer sessions for protected business routes.
// The session-creation endpoint separately validates Basic credentials.
func (h *API) authenticate(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
parts := strings.Fields(r.Header.Get("Authorization"))
if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") {
w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`)
h.writeError(w, r, model.ErrUnauthorized)
return
}
user, err := h.auth.Authenticate(r.Context(), parts[1])
if err != nil {
if errors.Is(err, model.ErrUnauthorized) {
w.Header().Set("WWW-Authenticate", `Bearer realm="messenger"`)
}
h.writeError(w, r, err)
return
}
next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), userKey, user)))
})
}
// validQuery отсекает query с битым %-экранированием: url.Values молча
// выбросил бы такой параметр, и cursor=%%% превратился бы в «с начала».
func (h *API) validQuery(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if _, err := url.ParseQuery(r.URL.RawQuery); err != nil {
h.writeError(w, r, &model.ValidationError{Message: "некорректная строка запроса"})
return
}
next.ServeHTTP(w, r)
})
}
func currentUser(ctx context.Context) model.User {
user, _ := ctx.Value(userKey).(model.User)
return user
}
// jsonBody читает тело POST с лимитом размера (иначе 413) и отсекает
// синтаксически неверный JSON (400) до кодогенерированного декодера: тот
// читает только первое JSON-значение и пропустил бы мусор после него.
func (h *API) jsonBody(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || strings.HasSuffix(r.URL.Path, "/attachments") {
next.ServeHTTP(w, r)
return
}
reader := io.Reader(r.Body)
if h.limits.MaxBodyBytes > 0 {
reader = http.MaxBytesReader(w, r.Body, h.limits.MaxBodyBytes)
}
body, err := io.ReadAll(reader)
var tooLarge *http.MaxBytesError
switch {
case errors.As(err, &tooLarge):
h.writeError(w, r, errPayloadTooLarge)
return
case err != nil:
h.writeError(w, r, &model.ValidationError{Message: "тело запроса не прочитано"})
return
case !utf8.Valid(body) || !json.Valid(body):
h.writeError(w, r, &model.ValidationError{Message: "некорректный JSON"})
return
}
r.Body = io.NopCloser(bytes.NewReader(body))
next.ServeHTTP(w, r)
})
}
+47
View File
@@ -0,0 +1,47 @@
package handler
import (
"bufio"
"encoding/json"
"net/http"
"os"
"runtime"
"runtime/debug"
"strconv"
"strings"
)
// stats — расширение за пределами контракта (contracts/extensions.md): память
// процесса, учтённый объём хранилища и очередь. Нужен для графиков трека 5.
func (h *API) stats(w http.ResponseWriter, _ *http.Request) {
var mem runtime.MemStats
runtime.ReadMemStats(&mem)
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"rss_bytes": rssBytes(),
"heap_alloc": mem.HeapAlloc,
"heap_sys": mem.HeapSys,
"go_memory_limit": debug.SetMemoryLimit(-1),
"num_gc": mem.NumGC,
"goroutines": runtime.NumGoroutine(),
"gate": h.gate.stats(),
"store": h.storeStats(),
})
}
// rssBytes читает VmRSS из /proc (только Linux, то есть в контейнере); иначе 0.
func rssBytes() int64 {
f, err := os.Open("/proc/self/status")
if err != nil {
return 0
}
defer f.Close()
sc := bufio.NewScanner(f)
for sc.Scan() {
if kb, ok := strings.CutPrefix(sc.Text(), "VmRSS:"); ok {
n, _ := strconv.ParseInt(strings.TrimSpace(strings.TrimSuffix(kb, "kB")), 10, 64)
return n * 1024
}
}
return 0
}
@@ -0,0 +1,21 @@
package handler
import (
"bytes"
"net/http"
"net/http/httptest"
"testing"
)
func TestRejectsInvalidUTF8BeforeJSONDecoding(t *testing.T) {
body := append([]byte(`{"display_name":"`), 0xff)
body = append(body, []byte(`"}`)...)
request := httptest.NewRequest(http.MethodPost, "/api/v1/users", bytes.NewReader(body))
response := httptest.NewRecorder()
h := &API{limits: Limits{MaxBodyBytes: 65536}}
called := false
requestID(h.jsonBody(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { called = true; w.WriteHeader(204) }))).ServeHTTP(response, request)
if response.Code != 400 || called {
t.Fatalf("invalid UTF-8 reached decoder: status=%d called=%t", response.Code, called)
}
}
@@ -0,0 +1,72 @@
package imagefile
import (
"backend/internal/domain/model"
"crypto/sha256"
"encoding/hex"
"errors"
"image"
_ "image/jpeg"
_ "image/png"
"io"
"os"
)
var ErrTooLarge = errors.New("image too large")
var ErrUnsupported = errors.New("invalid image or MIME")
type File struct {
*os.File
Size int64
SHA256, ContentType string
}
func (f *File) Dispose() { _ = f.Close(); _ = os.Remove(f.Name()) }
func Inspect(body io.Reader, filename, mime string) (out *File, err error) {
if err = model.ValidateFilename(filename); err != nil {
return nil, err
}
if mime != "image/png" && mime != "image/jpeg" {
return nil, ErrUnsupported
}
f, e := os.CreateTemp("", "messenger-upload-*")
if e != nil {
return nil, e
}
defer func() {
if err != nil {
_ = f.Close()
_ = os.Remove(f.Name())
}
}()
hash := sha256.New()
n, e := io.Copy(io.MultiWriter(f, hash), io.LimitReader(body, model.MaxImageBytes+1))
if e != nil {
return nil, &model.ValidationError{Message: "incomplete image upload"}
}
if n > model.MaxImageBytes {
return nil, ErrTooLarge
}
if _, e = f.Seek(0, io.SeekStart); e != nil {
return nil, e
}
cfg, format, e := image.DecodeConfig(f)
if e != nil || "image/"+format != mime {
return nil, ErrUnsupported
}
pixels := int64(cfg.Width) * int64(cfg.Height)
if cfg.Width < 1 || cfg.Height < 1 || pixels > model.MaxImagePixels {
return nil, ErrTooLarge
}
if _, e = f.Seek(0, io.SeekStart); e != nil {
return nil, e
}
decoded, actual, e := image.Decode(f)
if e != nil || actual != format || decoded.Bounds().Dx() != cfg.Width || decoded.Bounds().Dy() != cfg.Height {
return nil, ErrUnsupported
}
if _, e = f.Seek(0, io.SeekStart); e != nil {
return nil, e
}
return &File{f, n, hex.EncodeToString(hash.Sum(nil)), mime}, nil
}
@@ -0,0 +1,81 @@
package imagefile
import (
"backend/internal/domain/model"
"bytes"
"encoding/binary"
"errors"
"hash/crc32"
"image"
"image/png"
"io"
"testing"
)
func pngBytes(t *testing.T) []byte {
t.Helper()
var b bytes.Buffer
if e := png.Encode(&b, image.NewNRGBA(image.Rect(0, 0, 3, 2))); e != nil {
t.Fatal(e)
}
return b.Bytes()
}
func TestOriginalBytesArePreserved(t *testing.T) {
raw := pngBytes(t)
f, e := Inspect(bytes.NewReader(raw), "пример.png", "image/png")
if e != nil {
t.Fatal(e)
}
defer f.Dispose()
got, e := io.ReadAll(f)
if e != nil {
t.Fatal(e)
}
if !bytes.Equal(got, raw) {
t.Fatal("original changed")
}
}
func TestRejectsInvalidImageAndFilename(t *testing.T) {
for _, tc := range []struct {
name, mime string
body []byte
want error
}{
{"a.png", "image/jpeg", pngBytes(t), ErrUnsupported},
{"a.png", "image/png", []byte("not an image"), ErrUnsupported},
{"a.png", "application/octet-stream", pngBytes(t), ErrUnsupported},
{"a.png", "image/png", append(pngBytes(t)[:20], 0), ErrUnsupported},
} {
t.Run(tc.mime+string(tc.body[:1]), func(t *testing.T) {
_, e := Inspect(bytes.NewReader(tc.body), tc.name, tc.mime)
if !errors.Is(e, tc.want) {
t.Fatalf("got %v, want %v", e, tc.want)
}
})
}
for _, name := range []string{"../x.png", "a\\b", "a\x00b", "a\nb", ""} {
if _, e := Inspect(bytes.NewReader(pngBytes(t)), name, "image/png"); e == nil {
t.Fatalf("accepted unsafe filename %q", name)
}
}
}
func TestRejectsOversizedStreamBeforeDecoding(t *testing.T) {
_, e := Inspect(io.LimitReader(zeroReader{}, model.MaxImageBytes+1), "a.png", "image/png")
if !errors.Is(e, ErrTooLarge) {
t.Fatalf("got %v", e)
}
}
type zeroReader struct{}
func (zeroReader) Read(p []byte) (int, error) { clear(p); return len(p), nil }
func TestRejectsPixelBombBeforeDecoding(t *testing.T) {
raw := pngBytes(t)
binary.BigEndian.PutUint32(raw[16:20], 50000)
binary.BigEndian.PutUint32(raw[20:24], 50000)
binary.BigEndian.PutUint32(raw[29:33], crc32.ChecksumIEEE(raw[12:29]))
_, e := Inspect(bytes.NewReader(raw), "huge.png", "image/png")
if !errors.Is(e, ErrTooLarge) {
t.Fatalf("pixel limit did not apply before decode: %v", e)
}
}
@@ -0,0 +1,135 @@
package imagefile
import (
"backend/internal/adapter/password"
"bytes"
"compress/zlib"
"encoding/binary"
"hash/crc32"
"io"
"os"
"runtime"
"runtime/debug"
"strconv"
"strings"
"sync"
"testing"
)
// A normal PNG encoder allocates several full-width filter rows itself.
// Construct a tiny, highly compressible adversarial fixture as a stream instead.
func extremePNG(t *testing.T, interlaced bool) []byte {
t.Helper()
const width = 40_000_000
const height = 1
var idat bytes.Buffer
z := zlib.NewWriter(&idat)
passes := [][4]int{{0, 0, 1, 1}}
if interlaced {
passes = [][4]int{{0, 0, 8, 8}, {4, 0, 8, 8}, {0, 4, 4, 8}, {2, 0, 4, 4}, {0, 2, 2, 4}, {1, 0, 2, 2}, {0, 1, 1, 2}}
}
for _, p := range passes {
w := (width - p[0] + p[2] - 1) / p[2]
h := (height - p[1] + p[3] - 1) / p[3]
if width <= p[0] || height <= p[1] || w <= 0 || h <= 0 {
continue
}
for row := 0; row < h; row++ {
if _, e := z.Write([]byte{0}); e != nil {
t.Fatal(e)
}
if _, e := io.CopyN(z, zeroReader{}, int64(w)*8); e != nil {
t.Fatal(e)
}
}
}
if e := z.Close(); e != nil {
t.Fatal(e)
}
var out bytes.Buffer
out.Write([]byte{137, 80, 78, 71, 13, 10, 26, 10})
chunk := func(kind string, data []byte) {
var word [4]byte
binary.BigEndian.PutUint32(word[:], uint32(len(data)))
out.Write(word[:])
out.WriteString(kind)
out.Write(data)
crc := crc32.NewIEEE()
_, _ = crc.Write([]byte(kind))
_, _ = crc.Write(data)
binary.BigEndian.PutUint32(word[:], crc.Sum32())
out.Write(word[:])
}
header := make([]byte, 13)
binary.BigEndian.PutUint32(header[:4], width)
binary.BigEndian.PutUint32(header[4:8], height)
header[8] = 16
header[9] = 6
if interlaced {
header[12] = 1
}
chunk("IHDR", header)
chunk("IDAT", idat.Bytes())
chunk("IEND", nil)
return out.Bytes()
}
func TestWorstCaseImagesWithinBudget(t *testing.T) {
if os.Getenv("RUN_IMAGE_STRESS") != "1" {
t.Skip("explicit isolated memory experiment required")
}
old := debug.SetMemoryLimit(1536 * 1024 * 1024)
defer debug.SetMemoryLimit(old)
for _, interlaced := range []bool{false, true} {
raw := extremePNG(t, interlaced)
runtime.GC()
stop := make(chan struct{})
var workers sync.WaitGroup
for i := 0; i < 3; i++ {
workers.Add(1)
go func() {
defer workers.Done()
for {
select {
case <-stop:
return
default:
if _, e := password.New().Hash("synthetic-concurrent-KDF"); e != nil {
t.Error(e)
return
}
}
}
}()
}
for repeat := 0; repeat < 2; repeat++ {
f, e := Inspect(bytes.NewReader(raw), "wide.png", "image/png")
if e != nil {
close(stop)
workers.Wait()
t.Fatal(e)
}
f.Dispose()
}
close(stop)
workers.Wait()
runtime.GC()
}
raw, e := os.ReadFile("/proc/self/status")
if e != nil {
t.Fatal(e)
}
peak := int64(0)
for _, line := range strings.Split(string(raw), "\n") {
if strings.HasPrefix(line, "VmHWM:") {
fields := strings.Fields(line)
peak, e = strconv.ParseInt(fields[1], 10, 64)
if e != nil {
t.Fatal(e)
}
}
}
if peak <= 0 || peak >= 2*1024*1024 {
t.Fatalf("RSS high-water mark out of budget: %d KiB", peak)
}
t.Logf("PASS native RGBA64 40000000x1, normal+Adam7, two sequential validations each, 3 concurrent Argon2 workers; peak_rss_kib=%d hard_limit_kib=2097152", peak)
}
@@ -0,0 +1,129 @@
package imagefile
import (
"backend/internal/domain/model"
"backend/internal/domain/port"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"fmt"
"image"
"io"
"os"
"os/exec"
)
func FitDimensions(w, h, max int) (int, int) {
if w <= max && h <= max {
return w, h
}
if w >= h {
return max, maxInt(1, h*max/w)
}
return maxInt(1, w*max/h), max
}
func maxInt(a, b int) int {
if a > b {
return a
}
return b
}
type Transformer struct{}
func NewTransformer() port.Transformer { return Transformer{} }
// ImageMagick is a constrained subprocess; no input names or shell fragments are accepted.
func (Transformer) Transform(ctx context.Context, r io.Reader) ([]port.ProcessedVariant, error) {
input, err := os.CreateTemp("", "pipeline-original-*")
if err != nil {
return nil, err
}
defer func() { _ = input.Close(); _ = os.Remove(input.Name()) }()
n, err := io.Copy(input, io.LimitReader(r, model.MaxImageBytes+1))
if err != nil {
return nil, err
}
if n > model.MaxImageBytes {
return nil, model.ErrInvalidImage
}
if _, err = input.Seek(0, 0); err != nil {
return nil, err
}
config, _, err := image.DecodeConfig(input)
if err != nil || int64(config.Width)*int64(config.Height) > model.MaxImagePixels {
return nil, model.ErrInvalidImage
}
probe := exec.CommandContext(ctx, "convert", "-limit", "memory", "256MiB", "-limit", "map", "256MiB", "-limit", "disk", "0", input.Name()+"[0]", "-auto-orient", "-format", "%w %h", "info:")
raw, err := probe.Output()
if err != nil {
return nil, fmt.Errorf("image subprocess unavailable: %w", err)
}
var originalW, originalH int
if _, err = fmt.Sscanf(string(raw), "%d %d", &originalW, &originalH); err != nil || int64(originalW)*int64(originalH) > model.MaxImagePixels {
return nil, model.ErrInvalidImage
}
var variants []port.ProcessedVariant
for _, v := range []struct {
name, mime, geometry, format string
crop bool
}{{"thumbnail", "image/jpeg", "256x256>", "jpeg", false}, {"crop", "image/jpeg", "128x128^", "jpeg", true}, {"optimized", "image/webp", "1280x1280>", "webp", false}} {
geometry := v.geometry
if !v.crop {
max := 256
if v.name == "optimized" {
max = 1280
}
w, h := FitDimensions(originalW, originalH, max)
geometry = fmt.Sprintf("%dx%d!", w, h)
}
args := []string{"-limit", "memory", "256MiB", "-limit", "map", "256MiB", "-limit", "disk", "0", "-limit", "thread", "1", input.Name() + "[0]", "-auto-orient", "-background", "white", "-alpha", "remove", "-alpha", "off", "-strip", "-resize", geometry}
if v.crop {
args = append(args, "-gravity", "center", "-extent", "128x128")
}
args = append(args, "-quality", "82", v.format+":-")
cmd := exec.CommandContext(ctx, "convert", args...)
var out limitedBuffer
cmd.Stdout = &out
if err = cmd.Run(); err != nil {
return nil, fmt.Errorf("image subprocess failed: %w", err)
}
b := out.Bytes()
if len(b) == 0 {
return nil, fmt.Errorf("empty variant")
}
w, h := 0, 0
if v.format == "webp" {
// Read only geometry through the native decoder; never dump diagnostics/payload.
probe := exec.CommandContext(ctx, "identify", "-format", "%w %h", "webp:-")
probe.Stdin = bytes.NewReader(b)
raw, e := probe.Output()
if e != nil {
return nil, fmt.Errorf("invalid generated WebP")
}
if _, e = fmt.Sscanf(string(raw), "%d %d", &w, &h); e != nil {
return nil, e
}
} else {
// JPEG dimensions are read through the same strict decoder used for uploads.
cfg, _, e := image.DecodeConfig(bytes.NewReader(b))
if e != nil {
return nil, e
}
w, h = cfg.Width, cfg.Height
}
sum := sha256.Sum256(b)
variants = append(variants, port.ProcessedVariant{Metadata: model.Variant{Name: v.name, ContentType: v.mime, SHA256: hex.EncodeToString(sum[:]), SizeBytes: int64(len(b)), Width: w, Height: h}, Bytes: b})
}
return variants, nil
}
type limitedBuffer struct{ bytes.Buffer }
func (b *limitedBuffer) Write(p []byte) (int, error) {
if b.Len()+len(p) > 16*1024*1024 {
return 0, fmt.Errorf("variant too large")
}
return b.Buffer.Write(p)
}
@@ -0,0 +1,93 @@
package imagefile
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"image"
"image/color"
"image/jpeg"
"image/png"
"os"
"testing"
"time"
)
func TestActualVariantsHonorDimensionsAlphaAndEXIF(t *testing.T) {
if os.Getenv("RUN_VARIANT_TESTS") != "1" {
t.Skip("requires actual ImageMagick runtime")
}
for _, c := range []struct {
name string
w, h int
orientation bool
}{{"landscape", 320, 200, false}, {"tiny", 1, 1, false}, {"portrait", 23, 67, false}, {"exif", 40, 20, true}} {
t.Run(c.name, func(t *testing.T) {
img := image.NewNRGBA(image.Rect(0, 0, c.w, c.h))
img.Set(0, 0, color.NRGBA{R: 255, A: 0})
var encoded bytes.Buffer
if c.orientation {
if e := jpeg.Encode(&encoded, img, &jpeg.Options{Quality: 82}); e != nil {
t.Fatal(e)
}
} else {
if e := png.Encode(&encoded, img); e != nil {
t.Fatal(e)
}
}
input := encoded.Bytes()
if c.orientation {
payload := []byte{'E', 'x', 'i', 'f', 0, 0, 'I', 'I', 42, 0, 8, 0, 0, 0, 1, 0, 18, 1, 3, 0, 1, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0}
marker := append([]byte{255, 225, 0, byte(len(payload) + 2)}, payload...)
input = append(append(append([]byte{}, input[:2]...), marker...), input[2:]...)
}
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
variants, e := NewTransformer().Transform(ctx, bytes.NewReader(input))
if e != nil {
t.Fatal(e)
}
if len(variants) != 3 {
t.Fatalf("want three variants, got %d", len(variants))
}
w, h := c.w, c.h
if c.orientation {
w, h = h, w
}
for _, v := range variants {
s := sha256.Sum256(v.Bytes)
if v.Metadata.SHA256 != hex.EncodeToString(s[:]) || v.Metadata.SizeBytes != int64(len(v.Bytes)) {
t.Fatal("content digest/size mismatch")
}
wantW, wantH := w, h
if v.Metadata.Name == "thumbnail" {
wantW, wantH = FitDimensions(w, h, 256)
}
if v.Metadata.Name == "crop" {
wantW, wantH = 128, 128
}
if v.Metadata.Width != wantW || v.Metadata.Height != wantH {
t.Fatalf("%s: got %dx%d want %dx%d", v.Metadata.Name, v.Metadata.Width, v.Metadata.Height, wantW, wantH)
}
if v.Metadata.ContentType == "image/jpeg" {
decoded, e := jpeg.Decode(bytes.NewReader(v.Bytes))
if e != nil {
t.Fatal(e)
}
if decoded.Bounds().Dx() != wantW || decoded.Bounds().Dy() != wantH {
t.Fatal("actual JPEG geometry mismatch")
}
if !c.orientation {
r, g, b, _ := decoded.At(0, 0).RGBA()
if r < 64000 || g < 64000 || b < 64000 {
t.Fatal("alpha was not composited on white")
}
}
} else if len(v.Bytes) < 16 || string(v.Bytes[:4]) != "RIFF" || string(v.Bytes[8:12]) != "WEBP" {
t.Fatal("optimized file is not WebP")
}
}
})
}
}
@@ -0,0 +1,12 @@
package imagefile
import "testing"
func TestFitDimensionsPreservesAspectAndDoesNotUpscale(t *testing.T) {
for _, c := range []struct{ w, h, max, wantW, wantH int }{{320, 200, 256, 256, 160}, {1, 1, 256, 1, 1}, {1, 400, 256, 1, 256}, {400, 1, 256, 256, 1}, {200, 320, 256, 160, 256}, {1280, 1280, 1280, 1280, 1280}} {
w, h := FitDimensions(c.w, c.h, c.max)
if w != c.wantW || h != c.wantH {
t.Errorf("%+v: got %dx%d", c, w, h)
}
}
}
@@ -0,0 +1,36 @@
package password
import (
"backend/internal/domain/port"
"crypto/rand"
"crypto/subtle"
"encoding/base64"
"fmt"
"golang.org/x/crypto/argon2"
"strings"
)
type Argon struct{}
func New() port.Passwords { return Argon{} }
func (Argon) Hash(password string) (string, error) {
salt := make([]byte, 16)
if _, err := rand.Read(salt); err != nil {
return "", err
}
hash := argon2.IDKey([]byte(password), salt, 2, 32*1024, 1, 32)
return fmt.Sprintf("argon2id$v19$m32768t2p1$%s$%s", base64.RawStdEncoding.EncodeToString(salt), base64.RawStdEncoding.EncodeToString(hash)), nil
}
func (Argon) Verify(password, encoded string) bool {
parts := strings.Split(encoded, "$")
if len(parts) != 5 || parts[0] != "argon2id" || parts[1] != "v19" || parts[2] != "m32768t2p1" {
return false
}
salt, e1 := base64.RawStdEncoding.DecodeString(parts[3])
want, e2 := base64.RawStdEncoding.DecodeString(parts[4])
if e1 != nil || e2 != nil || len(salt) != 16 || len(want) != 32 {
return false
}
got := argon2.IDKey([]byte(password), salt, 2, 32*1024, 1, 32)
return subtle.ConstantTimeCompare(got, want) == 1
}
@@ -0,0 +1,47 @@
package queue
import (
"backend/internal/domain/model"
"backend/internal/domain/port"
"context"
"errors"
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/codes"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
"net"
"testing"
"time"
)
type reviewPendingStore struct{ port.PipelineStore }
func (reviewPendingStore) PendingJobs(context.Context) ([]model.Job, error) {
return []model.Job{{ID: uuid.New(), AttachmentID: uuid.New(), PipelineVersion: 1, CreatedAt: time.Now()}}, nil
}
func TestReviewPublishTransportFailureMarksProducerSpan(t *testing.T) {
exporter := tracetest.NewInMemoryExporter()
provider := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exporter))
previous := otel.GetTracerProvider()
otel.SetTracerProvider(provider)
defer func() {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
_ = provider.Shutdown(ctx)
otel.SetTracerProvider(previous)
}()
client := redis.NewClient(&redis.Options{MaxRetries: -1, Dialer: func(context.Context, string, string) (net.Conn, error) {
return nil, errors.New("synthetic transport failure")
}})
defer func() { _ = client.Close() }()
broker := New(client, reviewPendingStore{}, "fake-review")
if err := broker.Publish(context.Background()); err == nil {
t.Fatal("negative control did not fail")
}
spans := exporter.GetSpans()
if len(spans) != 1 || spans[0].Name != "attachment.publish" || spans[0].Status.Code != codes.Error {
t.Fatalf("failed publish not marked as failed producer span; count=%d", len(spans))
}
}
+150
View File
@@ -0,0 +1,150 @@
package queue
import (
"backend/internal/domain/model"
"backend/internal/domain/port"
"bytes"
"context"
"encoding/json"
"errors"
"github.com/google/uuid"
"github.com/redis/go-redis/v9"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
"time"
)
const stream = "attachment-jobs"
type Broker struct {
client *redis.Client
store port.PipelineStore
consumer string
}
func New(client *redis.Client, store port.PipelineStore, consumer string) *Broker {
return &Broker{client, store, consumer}
}
func (b *Broker) Publish(ctx context.Context) error {
jobs, e := b.store.PendingJobs(ctx)
if e != nil {
return e
}
for _, j := range jobs {
parent := otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier{"traceparent": j.Traceparent, "tracestate": j.Tracestate})
send, span := otel.Tracer("messenger").Start(parent, "attachment.publish", trace.WithSpanKind(trace.SpanKindProducer))
carrier := propagation.MapCarrier{}
otel.GetTextMapPropagator().Inject(send, carrier)
j.Traceparent = carrier.Get("traceparent")
j.Tracestate = carrier.Get("tracestate")
j.Attempt++
payload := struct {
Schema int `json:"schema_version"`
Type string `json:"type"`
model.Job
}{1, "attachment.process.v1", j}
raw, e := json.Marshal(payload)
if e != nil {
span.SetStatus(codes.Error, "job_encoding_error")
span.End()
return e
}
if e = b.client.XAdd(send, &redis.XAddArgs{Stream: stream, MaxLen: 10000, Approx: true, Values: map[string]any{"job": string(raw)}}).Err(); e != nil {
span.SetStatus(codes.Error, "broker_error")
span.End()
return e
}
if e = b.store.Published(send, j.ID); e != nil {
span.SetStatus(codes.Error, "outbox_persistence_error")
span.End()
return e
}
span.End()
}
return nil
}
func (b *Broker) RunPublisher(ctx context.Context, onError func(error)) {
tick := time.NewTicker(500 * time.Millisecond)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
if e := b.Publish(ctx); e != nil {
onError(e)
}
}
}
}
func (b *Broker) ack(ctx context.Context, id string) {
if b.client.XAck(ctx, stream, "workers", id).Err() == nil {
_ = b.client.XDel(ctx, stream, id).Err()
}
}
func (b *Broker) consume(ctx context.Context, m redis.XMessage, process func(context.Context, model.Job) error, onError func(error)) {
raw, ok := m.Values["job"].(string)
if !ok || len(raw) > 2048 {
b.ack(ctx, m.ID)
return
}
var envelope struct {
Schema int `json:"schema_version"`
Type string `json:"type"`
model.Job
}
decoder := json.NewDecoder(bytes.NewBufferString(raw))
decoder.DisallowUnknownFields()
if decoder.Decode(&envelope) != nil || envelope.Schema != 1 || envelope.Type != "attachment.process.v1" || envelope.ID == uuid.Nil || envelope.AttachmentID == uuid.Nil || envelope.PipelineVersion < 1 || envelope.Attempt < 1 {
b.ack(ctx, m.ID)
return
}
work, cancel := context.WithTimeout(ctx, 20*time.Second)
e := process(work, envelope.Job)
cancel()
if e != nil {
onError(e)
} else {
b.ack(ctx, m.ID)
}
}
func (b *Broker) RunWorker(ctx context.Context, process func(context.Context, model.Job) error, onError func(error)) {
for ctx.Err() == nil {
e := b.client.XGroupCreateMkStream(ctx, stream, "workers", "0").Err()
if e != nil && e.Error() != "BUSYGROUP Consumer Group name already exists" {
onError(e)
pause(ctx)
continue
}
// Reclaim abandoned PEL entries; the DB lease and version fence are authoritative.
claimed, _, e := b.client.XAutoClaim(ctx, &redis.XAutoClaimArgs{Stream: stream, Group: "workers", Consumer: b.consumer, MinIdle: 30 * time.Second, Start: "0-0", Count: 1}).Result()
if e == nil && len(claimed) > 0 {
for _, m := range claimed {
b.consume(ctx, m, process, onError)
}
continue
}
rows, e := b.client.XReadGroup(ctx, &redis.XReadGroupArgs{Group: "workers", Consumer: b.consumer, Streams: []string{stream, ">"}, Count: 1, Block: time.Second}).Result()
if errors.Is(e, redis.Nil) {
continue
}
if e != nil {
onError(e)
pause(ctx)
continue
}
for _, row := range rows {
for _, m := range row.Messages {
b.consume(ctx, m, process, onError)
}
}
}
}
func pause(ctx context.Context) {
select {
case <-ctx.Done():
case <-time.After(200 * time.Millisecond):
}
}
@@ -0,0 +1,57 @@
package postgres
import (
"backend/internal/adapter/repository/postgres/queries"
"backend/internal/domain/model"
"context"
"encoding/json"
"github.com/google/uuid"
)
func attachment(r queries.Attachment) model.Attachment {
var variants []model.Variant
_ = json.Unmarshal(r.Variants, &variants)
return model.Attachment{ID: r.ID, ChatID: r.ChatID, OwnerID: r.OwnerID, Filename: r.Filename, ContentType: r.ContentType, SizeBytes: r.SizeBytes, SHA256: r.Sha256, Status: r.Status, CreatedAt: r.CreatedAt, UpdatedAt: r.UpdatedAt, PipelineVersion: r.PipelineVersion, ErrorCode: r.ErrorCode, Variants: variants, StagingExpired: r.StagingExpired, StagingCleaned: r.StagingCleaned}
}
func (s *Store) ReserveAttachment(ctx context.Context, a model.Attachment) (model.Attachment, error) {
tx, e := s.pool.Begin(ctx)
if e != nil {
return model.Attachment{}, mapErr(e)
}
defer func() { _ = tx.Rollback(ctx) }()
q := s.q.WithTx(tx)
if _, e = q.ChargeAttachment(ctx, a.OwnerID); e != nil {
return model.Attachment{}, model.ErrStorageFull
}
r, e := q.InsertAttachment(ctx, queries.InsertAttachmentParams{ID: a.ID, ChatID: a.ChatID, OwnerID: a.OwnerID, Filename: a.Filename, ContentType: a.ContentType, SizeBytes: a.SizeBytes, Sha256: a.SHA256})
if e != nil {
return model.Attachment{}, mapErr(e)
}
if e = tx.Commit(ctx); e != nil {
return model.Attachment{}, mapErr(e)
}
return attachment(r), nil
}
func (s *Store) GetAttachment(ctx context.Context, id uuid.UUID) (model.Attachment, error) {
r, e := s.q.GetAttachment(ctx, id)
return attachment(r), mapErr(e)
}
func (s *Store) SetAttachmentStatus(ctx context.Context, id uuid.UUID, from, to string) (bool, error) {
n, e := s.q.SetAttachmentStatus(ctx, queries.SetAttachmentStatusParams{ID: id, Status: to, Status_2: from})
return n == 1, mapErr(e)
}
func (s *Store) ReconcileAttachments(ctx context.Context, after uuid.UUID) ([]model.Attachment, bool, error) {
rows, e := s.q.ReconcileAttachments(ctx, after)
if e != nil {
return nil, false, mapErr(e)
}
more := len(rows) > 100
if more {
rows = rows[:100]
}
out := make([]model.Attachment, len(rows))
for i, r := range rows {
out[i] = attachment(r)
}
return out, more, nil
}
@@ -0,0 +1,15 @@
package postgres
import (
"backend/internal/domain/model"
"context"
)
func (s *Store) PipelineMetrics(ctx context.Context) (model.PipelineMetrics, error) {
r, e := s.q.PipelineMetrics(ctx)
if e != nil {
return model.PipelineMetrics{}, mapErr(e)
}
cohort, e := s.q.ReadySLI(ctx)
return model.PipelineMetrics{Depth: r.Depth, Oldest: r.Oldest, ReadyMatured: [3]int64{cohort.Matured10, cohort.Matured30, cohort.Matured300}, ReadyTimely: [3]int64{cohort.Timely10, cohort.Timely30, cohort.Timely300}}, mapErr(e)
}
@@ -0,0 +1,171 @@
package postgres
import (
"backend/internal/adapter/repository/postgres/queries"
"backend/internal/domain/model"
"context"
"encoding/json"
"errors"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"time"
)
func job(r queries.AttachmentJob) model.Job {
return model.Job{ID: r.ID, AttachmentID: r.AttachmentID, PipelineVersion: r.PipelineVersion, Attempt: r.Attempt, CreatedAt: r.CreatedAt, Traceparent: r.Traceparent, Tracestate: r.Tracestate}
}
func (s *Store) QueueAttachment(ctx context.Context, a model.Attachment, tp, ts string) (model.Attachment, error) {
tx, e := s.pool.Begin(ctx)
if e != nil {
return model.Attachment{}, mapErr(e)
}
defer func() { _ = tx.Rollback(ctx) }()
q := s.q.WithTx(tx)
r, e := q.QueueAttachment(ctx, a.ID)
if e != nil {
return model.Attachment{}, mapErr(e)
}
if e = q.InsertJob(ctx, queries.InsertJobParams{JobID: uuid.New(), AttachmentID: r.ID, PipelineVersion: r.PipelineVersion, Traceparent: tp, Tracestate: ts}); e != nil {
return model.Attachment{}, mapErr(e)
}
if e = tx.Commit(ctx); e != nil {
return model.Attachment{}, mapErr(e)
}
return attachment(r), nil
}
func (s *Store) PendingJobs(ctx context.Context) ([]model.Job, error) {
rows, e := s.q.PendingJobs(ctx)
if e != nil {
return nil, mapErr(e)
}
out := make([]model.Job, len(rows))
for i, r := range rows {
out[i] = job(r)
}
return out, nil
}
func (s *Store) Published(ctx context.Context, id uuid.UUID) error {
return mapErr(s.q.Published(ctx, id))
}
func (s *Store) AcquireJob(ctx context.Context, j model.Job, lease uuid.UUID, d time.Duration) (model.Job, model.Attachment, bool, error) {
tx, e := s.pool.Begin(ctx)
if e != nil {
return j, model.Attachment{}, false, mapErr(e)
}
defer func() { _ = tx.Rollback(ctx) }()
q := s.q.WithTx(tx)
r, e := q.AcquireJob(ctx, queries.AcquireJobParams{LeaseID: lease, LeaseSeconds: int32(d / time.Second), JobID: j.ID, PipelineVersion: j.PipelineVersion})
if errors.Is(e, pgx.ErrNoRows) {
return j, model.Attachment{}, false, nil
}
if e != nil {
return j, model.Attachment{}, false, mapErr(e)
}
a, e := q.ProcessingAttachment(ctx, queries.ProcessingAttachmentParams{AttachmentID: r.AttachmentID, PipelineVersion: r.PipelineVersion})
if e != nil {
return j, model.Attachment{}, false, mapErr(e)
}
if e = tx.Commit(ctx); e != nil {
return j, model.Attachment{}, false, mapErr(e)
}
return job(r), attachment(a), true, nil
}
func (s *Store) CompleteJob(ctx context.Context, j model.Job, lease uuid.UUID, v []model.Variant) (bool, error) {
raw, e := json.Marshal(v)
if e != nil {
return false, e
}
tx, e := s.pool.Begin(ctx)
if e != nil {
return false, mapErr(e)
}
defer func() { _ = tx.Rollback(ctx) }()
q := s.q.WithTx(tx)
n, e := q.CompleteAttachment(ctx, queries.CompleteAttachmentParams{Variants: raw, JobID: j.ID, LeaseID: lease})
if e != nil {
return false, mapErr(e)
}
if n != 1 {
return false, nil
}
if e = q.CompleteJob(ctx, queries.CompleteJobParams{JobID: j.ID, LeaseID: lease}); e != nil {
return false, mapErr(e)
}
return true, mapErr(tx.Commit(ctx))
}
func (s *Store) RetryJob(ctx context.Context, j model.Job, lease uuid.UUID, code string, delay time.Duration, failed bool) error {
status := "queued"
if failed {
status = "failed"
}
tx, e := s.pool.Begin(ctx)
if e != nil {
return mapErr(e)
}
defer func() { _ = tx.Rollback(ctx) }()
q := s.q.WithTx(tx)
n, e := q.RetryAttachment(ctx, queries.RetryAttachmentParams{NextStatus: status, ErrorCode: code, JobID: j.ID, LeaseID: lease})
if e != nil {
return mapErr(e)
}
if n == 0 {
return nil
}
if e = q.RetryJob(ctx, queries.RetryJobParams{DelayMs: int32(delay / time.Millisecond), Completed: failed, JobID: j.ID, LeaseID: lease}); e != nil {
return mapErr(e)
}
return mapErr(tx.Commit(ctx))
}
func (s *Store) ReplayJob(ctx context.Context, id uuid.UUID) (bool, error) {
tx, e := s.pool.Begin(ctx)
if e != nil {
return false, mapErr(e)
}
defer func() { _ = tx.Rollback(ctx) }()
q := s.q.WithTx(tx)
a, e := q.ReplayAttachment(ctx, id)
if errors.Is(e, pgx.ErrNoRows) {
return false, nil
}
if e != nil {
return false, mapErr(e)
}
if e = q.ReplayJobs(ctx, queries.ReplayJobsParams{AttachmentID: id, PipelineVersion: a.PipelineVersion}); e != nil {
return false, mapErr(e)
}
return true, mapErr(tx.Commit(ctx))
}
func (s *Store) AdvancePipeline(ctx context.Context, id uuid.UUID) (bool, error) {
tx, e := s.pool.Begin(ctx)
if e != nil {
return false, mapErr(e)
}
defer func() { _ = tx.Rollback(ctx) }()
q := s.q.WithTx(tx)
a, e := q.AdvancePipeline(ctx, id)
if errors.Is(e, pgx.ErrNoRows) {
return false, nil
}
if e != nil {
return false, mapErr(e)
}
if e = q.InsertJob(ctx, queries.InsertJobParams{JobID: uuid.New(), AttachmentID: id, PipelineVersion: a.PipelineVersion}); e != nil {
return false, mapErr(e)
}
return true, mapErr(tx.Commit(ctx))
}
func (s *Store) StagingCleanup(ctx context.Context) ([]model.Attachment, error) {
rows, e := s.q.StagingCleanup(ctx)
if e != nil {
return nil, mapErr(e)
}
out := make([]model.Attachment, len(rows))
for i, r := range rows {
out[i] = attachment(r)
}
return out, nil
}
func (s *Store) StagingCleaned(ctx context.Context, id uuid.UUID) error {
return mapErr(s.q.StagingCleaned(ctx, id))
}
@@ -0,0 +1,331 @@
package postgres
import (
"backend/internal/adapter/storage"
"backend/internal/application/usecase"
"backend/internal/config"
"backend/internal/domain/model"
"backend/internal/domain/port"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"io"
"os"
"reflect"
"sync"
"syscall"
"testing"
"time"
)
func TestReplayAfterCompletedPreviousVersionCleanup(t *testing.T) {
if os.Getenv("RUN_PIPELINE_DB_TESTS") != "1" {
t.Skip("explicit isolated database required")
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
raw, e := config.ReadSecret("/run/secrets/database_url")
if e != nil {
t.Fatal("database file unavailable")
}
pool, e := pgxpool.New(ctx, raw)
if e != nil {
t.Fatal("database connection unavailable")
}
defer pool.Close()
s := New(pool, nil)
user, e := s.CreateUser(ctx, model.UserDraft{Username: "v2_" + uuid.NewString()[:8], DisplayName: "Synthetic version test", PasswordHash: "disabled-synthetic"})
if e != nil {
t.Fatal(e)
}
var chatID, attachmentID uuid.UUID
defer func() {
cleanup, c := context.WithTimeout(context.Background(), 5*time.Second)
defer c()
for _, q := range []struct {
sql string
id uuid.UUID
}{{"DELETE FROM attachment_jobs WHERE attachment_id=$1", attachmentID}, {"DELETE FROM attachments WHERE id=$1", attachmentID}, {"DELETE FROM chat_members WHERE chat_id=$1", chatID}, {"DELETE FROM chats WHERE id=$1", chatID}, {"DELETE FROM users WHERE id=$1", user.ID}} {
if _, e := pool.Exec(cleanup, q.sql, q.id); e != nil {
t.Error("version fixture cleanup failed")
}
}
}()
chat, e := s.CreateChat(ctx, model.ChatDraft{Title: "Owned version fixture", Creator: user.ID, MemberIDs: []uuid.UUID{user.ID}})
if e != nil {
t.Fatal(e)
}
chatID = chat.ID
attachmentID = uuid.New()
a, e := s.ReserveAttachment(ctx, model.Attachment{ID: attachmentID, ChatID: chatID, OwnerID: user.ID, Filename: "test.png", ContentType: "image/png", SizeBytes: 3, SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"})
if e != nil {
t.Fatal(e)
}
if _, e = s.QueueAttachment(ctx, a, "", ""); e != nil {
t.Fatal(e)
}
jobs, e := s.PendingJobs(ctx)
if e != nil {
t.Fatal(e)
}
var j model.Job
for _, candidate := range jobs {
if candidate.AttachmentID == attachmentID {
j = candidate
}
}
lease := uuid.New()
j, _, ok, e := s.AcquireJob(ctx, j, lease, 30*time.Second)
if e != nil || !ok {
t.Fatal("version1 acquisition failed")
}
if ok, e = s.CompleteJob(ctx, j, lease, []model.Variant{}); e != nil || !ok {
t.Fatal("version1 completion failed")
}
if _, e = s.StagingCleanup(ctx); e != nil {
t.Fatal(e)
}
if e = s.StagingCleaned(ctx, attachmentID); e != nil {
t.Fatal(e)
}
if ok, e = s.AdvancePipeline(ctx, attachmentID); e != nil || !ok {
t.Fatal("version2 advance failed")
}
jobs, e = s.PendingJobs(ctx)
if e != nil {
t.Fatal(e)
}
for _, candidate := range jobs {
if candidate.AttachmentID == attachmentID && candidate.PipelineVersion == 2 {
j = candidate
}
}
lease = uuid.New()
j, _, ok, e = s.AcquireJob(ctx, j, lease, 30*time.Second)
if e != nil || !ok {
t.Fatal("version2 acquisition failed")
}
if e = s.RetryJob(ctx, j, lease, "retry_exhausted", 0, true); e != nil {
t.Fatal(e)
}
ok, e = s.ReplayJob(ctx, attachmentID)
if e != nil {
t.Fatal(e)
}
if !ok {
t.Fatal("ready version1 staging cleanup permanently blocked replay of failed version2 despite retained original")
}
var replays int
var replayed time.Time
if e = pool.QueryRow(ctx, "SELECT replay_count,replayed_at FROM attachment_jobs WHERE attachment_id=$1 AND pipeline_version=2", attachmentID).Scan(&replays, &replayed); e != nil || replays != 1 || replayed.IsZero() {
t.Fatal("explicit replay has no durable audit count/timestamp")
}
}
// This deterministic transformer makes v1/v2 effects distinguishable; the Store,
// pipeline ownership/leases, transactions and S3 operations remain real.
type versionTransformer struct {
body string
err error
}
func (v versionTransformer) Transform(context.Context, io.Reader) ([]port.ProcessedVariant, error) {
if v.err != nil {
return nil, v.err
}
out := []port.ProcessedVariant{}
for _, name := range []string{"thumbnail", "crop", "optimized"} {
body := []byte(v.body + name)
sum := sha256.Sum256(body)
out = append(out, port.ProcessedVariant{Metadata: model.Variant{Name: name, ContentType: "image/jpeg", SHA256: hex.EncodeToString(sum[:]), SizeBytes: int64(len(body)), Width: 1, Height: 1}, Bytes: body})
}
return out, nil
}
func TestVersionedPipelineDuplicateReorderRetryAndCleanup(t *testing.T) {
if os.Getenv("RUN_PIPELINE_DB_TESTS") != "1" {
t.Skip("explicit isolated database and S3 required")
}
ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second)
defer cancel()
raw, e := config.ReadSecret("/run/secrets/database_url")
if e != nil {
t.Fatal("database file unavailable")
}
pool, e := pgxpool.New(ctx, raw)
if e != nil {
t.Fatal("database unavailable")
}
defer pool.Close()
s := New(pool, nil)
cfg, e := storage.ReadConfig("/run/secrets/s3_credentials")
if e != nil {
t.Fatal("S3 config unavailable")
}
objects, e := storage.FromConfig(cfg)
if e != nil {
t.Fatal(e)
}
user, e := s.CreateUser(ctx, model.UserDraft{Username: "matrix_" + uuid.NewString()[:8], DisplayName: "Synthetic fault matrix", PasswordHash: "disabled"})
if e != nil {
t.Fatal(e)
}
chat, e := s.CreateChat(ctx, model.ChatDraft{Title: "Owned pipeline matrix", Creator: user.ID, MemberIDs: []uuid.UUID{user.ID}})
if e != nil {
t.Fatal(e)
}
id := uuid.New()
body := []byte("immutable-original")
sum := sha256.Sum256(body)
defer func() {
c, stop := context.WithTimeout(context.Background(), 5*time.Second)
defer stop()
for _, key := range []string{"staging/" + id.String(), usecase.ObjectKey(id)} {
if e := objects.Remove(c, key); e != nil {
t.Error("object fixture cleanup failed")
}
}
for version := int32(1); version <= 3; version++ {
for _, name := range []string{"thumbnail", "crop", "optimized"} {
if e := objects.Remove(c, usecase.VariantKey(id, version, name)); e != nil {
t.Error("variant fixture cleanup failed")
}
}
}
for _, q := range []struct {
sql string
id uuid.UUID
}{{"DELETE FROM attachment_jobs WHERE attachment_id=$1", id}, {"DELETE FROM attachments WHERE id=$1", id}, {"DELETE FROM chat_members WHERE chat_id=$1", chat.ID}, {"DELETE FROM chats WHERE id=$1", chat.ID}, {"DELETE FROM users WHERE id=$1", user.ID}} {
if _, e := pool.Exec(c, q.sql, q.id); e != nil {
t.Error("database fixture cleanup failed")
}
}
}()
a, e := s.ReserveAttachment(ctx, model.Attachment{ID: id, ChatID: chat.ID, OwnerID: user.ID, Filename: "matrix.png", ContentType: "image/png", SizeBytes: int64(len(body)), SHA256: hex.EncodeToString(sum[:])})
if e != nil {
t.Fatal(e)
}
if e = objects.Put(ctx, "staging/"+id.String(), bytes.NewReader(body), int64(len(body)), "image/png"); e != nil {
t.Fatal(e)
}
if _, e = s.QueueAttachment(ctx, a, "", ""); e != nil {
t.Fatal(e)
}
next := func(version int32) model.Job {
t.Helper()
jobs, e := s.PendingJobs(ctx)
if e != nil {
t.Fatal(e)
}
for _, j := range jobs {
if j.AttachmentID == id && j.PipelineVersion == version {
return j
}
}
t.Fatal("owned current job not found")
return model.Job{}
}
j1 := next(1)
p1 := usecase.NewPipeline(s, objects, versionTransformer{body: "version1-"}, 5)
p1.Fault = func(point string) error {
if point == "after_commit" {
return errors.New("synthetic crash after commit")
}
return nil
}
if e = p1.Process(ctx, j1); e == nil {
t.Fatal("fault did not interrupt after commit")
}
a, e = s.GetAttachment(ctx, id)
if e != nil || a.Status != "ready" || a.StagingCleaned {
t.Fatal("commit fault did not preserve ready/pending cleanup")
}
if e = p1.Process(ctx, j1); e != nil {
t.Fatal(e)
}
if _, e = objects.Get(ctx, "staging/"+id.String()); !errors.Is(e, model.ErrNotFound) {
t.Fatal("completed redelivery did not remove staging")
}
if ok, e := s.AdvancePipeline(ctx, id); e != nil || !ok {
t.Fatal("advance v2 failed")
}
j2 := next(2)
// Old delivery arriving before v2 and a fabricated future delivery are no-ops.
if e = p1.Process(ctx, j1); e != nil {
t.Fatal(e)
}
future := j2
future.PipelineVersion = 3
if e = p1.Process(ctx, future); e != nil {
t.Fatal(e)
}
p2 := usecase.NewPipeline(s, objects, versionTransformer{body: "version2-"}, 5)
var group sync.WaitGroup
errs := make(chan error, 2)
for i := 0; i < 2; i++ {
group.Add(1)
go func() { defer group.Done(); errs <- p2.Process(ctx, j2) }()
}
group.Wait()
close(errs)
for e := range errs {
if e != nil {
t.Fatal(e)
}
}
a, e = s.GetAttachment(ctx, id)
if e != nil || a.PipelineVersion != 2 || a.Status != "ready" || len(a.Variants) != 3 {
t.Fatal("duplicate current job lost result")
}
snapshot := a.Variants
for _, j := range []model.Job{j1, j2, j1, future, j2} {
if e = p1.Process(ctx, j); e != nil {
t.Fatal(e)
}
}
a, e = s.GetAttachment(ctx, id)
if e != nil || !reflect.DeepEqual(snapshot, a.Variants) {
t.Fatal("stale or duplicate job overwrote current metadata")
}
for _, v := range a.Variants {
r, e := objects.Get(ctx, usecase.VariantKey(id, 2, v.Name))
if e != nil {
t.Fatal(e)
}
got, e := io.ReadAll(r)
r.Close()
if e != nil || string(got) != "version2-"+v.Name {
t.Fatal("old transformation overwrote v2 objects")
}
}
if ok, e := s.AdvancePipeline(ctx, id); e != nil || !ok {
t.Fatal("advance v3 failed")
}
j3 := next(3)
broken := usecase.NewPipeline(s, objects, versionTransformer{err: syscall.ENOSPC}, 5)
for i := 0; i < 5; i++ {
if _, e = pool.Exec(ctx, "UPDATE attachment_jobs SET next_attempt='epoch' WHERE id=$1", j3.ID); e != nil {
t.Fatal(e)
}
if e = broken.Process(ctx, j3); e != nil {
t.Fatal(e)
}
}
a, e = s.GetAttachment(ctx, id)
if e != nil || a.Status != "failed" || a.ErrorCode != "retry_exhausted" {
t.Fatal("transient infrastructure failure did not exhaust bounded retry")
}
if ok, e := s.ReplayJob(ctx, id); e != nil || !ok {
t.Fatal("explicit replay blocked despite retained immutable source")
}
if e = p2.Process(ctx, j3); e != nil {
t.Fatal(e)
}
a, e = s.GetAttachment(ctx, id)
if e != nil || a.Status != "ready" || a.PipelineVersion != 3 {
t.Fatal("replayed job did not recover")
}
}
@@ -0,0 +1,199 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: attachments.sql
package queries
import (
"context"
"github.com/google/uuid"
)
const chargeAttachment = `-- name: ChargeAttachment :one
UPDATE users SET attachment_count=attachment_count+1 WHERE id=$1 AND attachment_count<10000 RETURNING id
`
func (q *Queries) ChargeAttachment(ctx context.Context, id uuid.UUID) (uuid.UUID, error) {
row := q.db.QueryRow(ctx, chargeAttachment, id)
var id_2 uuid.UUID
err := row.Scan(&id_2)
return id_2, err
}
const getAttachment = `-- name: GetAttachment :one
SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, pipeline_version, error_code, variants, staging_expired, staging_cleaned, created_at, updated_at, accepted_at, first_ready_at FROM attachments WHERE id=$1 FOR SHARE
`
func (q *Queries) GetAttachment(ctx context.Context, id uuid.UUID) (Attachment, error) {
row := q.db.QueryRow(ctx, getAttachment, id)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.PipelineVersion,
&i.ErrorCode,
&i.Variants,
&i.StagingExpired,
&i.StagingCleaned,
&i.CreatedAt,
&i.UpdatedAt,
&i.AcceptedAt,
&i.FirstReadyAt,
)
return i, err
}
const insertAttachment = `-- name: InsertAttachment :one
INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,status)
VALUES($1,$2,$3,$4,$5,$6,$7,'uploading') RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, pipeline_version, error_code, variants, staging_expired, staging_cleaned, created_at, updated_at, accepted_at, first_ready_at
`
type InsertAttachmentParams struct {
ID uuid.UUID
ChatID uuid.UUID
OwnerID uuid.UUID
Filename string
ContentType string
SizeBytes int64
Sha256 string
}
func (q *Queries) InsertAttachment(ctx context.Context, arg InsertAttachmentParams) (Attachment, error) {
row := q.db.QueryRow(ctx, insertAttachment,
arg.ID,
arg.ChatID,
arg.OwnerID,
arg.Filename,
arg.ContentType,
arg.SizeBytes,
arg.Sha256,
)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.PipelineVersion,
&i.ErrorCode,
&i.Variants,
&i.StagingExpired,
&i.StagingCleaned,
&i.CreatedAt,
&i.UpdatedAt,
&i.AcceptedAt,
&i.FirstReadyAt,
)
return i, err
}
const insertMessageAttachment = `-- name: InsertMessageAttachment :exec
INSERT INTO message_attachments(message_id,attachment_id,ordinal) VALUES($1,$2,$3)
`
type InsertMessageAttachmentParams struct {
MessageID uuid.UUID
AttachmentID uuid.UUID
Ordinal int32
}
func (q *Queries) InsertMessageAttachment(ctx context.Context, arg InsertMessageAttachmentParams) error {
_, err := q.db.Exec(ctx, insertMessageAttachment, arg.MessageID, arg.AttachmentID, arg.Ordinal)
return err
}
const messageAttachments = `-- name: MessageAttachments :many
SELECT attachment_id FROM message_attachments WHERE message_id=$1 ORDER BY ordinal LIMIT 10
`
func (q *Queries) MessageAttachments(ctx context.Context, messageID uuid.UUID) ([]uuid.UUID, error) {
rows, err := q.db.Query(ctx, messageAttachments, messageID)
if err != nil {
return nil, err
}
defer rows.Close()
items := []uuid.UUID{}
for rows.Next() {
var attachment_id uuid.UUID
if err := rows.Scan(&attachment_id); err != nil {
return nil, err
}
items = append(items, attachment_id)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const reconcileAttachments = `-- name: ReconcileAttachments :many
SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, pipeline_version, error_code, variants, staging_expired, staging_cleaned, created_at, updated_at, accepted_at, first_ready_at FROM attachments WHERE id>$1 ORDER BY id LIMIT 101
`
func (q *Queries) ReconcileAttachments(ctx context.Context, id uuid.UUID) ([]Attachment, error) {
rows, err := q.db.Query(ctx, reconcileAttachments, id)
if err != nil {
return nil, err
}
defer rows.Close()
items := []Attachment{}
for rows.Next() {
var i Attachment
if err := rows.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.PipelineVersion,
&i.ErrorCode,
&i.Variants,
&i.StagingExpired,
&i.StagingCleaned,
&i.CreatedAt,
&i.UpdatedAt,
&i.AcceptedAt,
&i.FirstReadyAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const setAttachmentStatus = `-- name: SetAttachmentStatus :execrows
UPDATE attachments SET status=$2,updated_at=clock_timestamp() WHERE id=$1 AND status=$3 AND pipeline_version=0
`
type SetAttachmentStatusParams struct {
ID uuid.UUID
Status string
Status_2 string
}
func (q *Queries) SetAttachmentStatus(ctx context.Context, arg SetAttachmentStatusParams) (int64, error) {
result, err := q.db.Exec(ctx, setAttachmentStatus, arg.ID, arg.Status, arg.Status_2)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
@@ -0,0 +1,358 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: core.sql
package queries
import (
"context"
"time"
"github.com/google/uuid"
)
const authenticateSession = `-- name: AuthenticateSession :one
SELECT user_id FROM sessions WHERE token_hash=$1 AND expires_at>clock_timestamp()
`
func (q *Queries) AuthenticateSession(ctx context.Context, tokenHash string) (uuid.UUID, error) {
row := q.db.QueryRow(ctx, authenticateSession, tokenHash)
var user_id uuid.UUID
err := row.Scan(&user_id)
return user_id, err
}
const chargeChat = `-- name: ChargeChat :one
UPDATE users SET chat_count=chat_count+1 WHERE id=$1 AND chat_count<1000 RETURNING id
`
func (q *Queries) ChargeChat(ctx context.Context, id uuid.UUID) (uuid.UUID, error) {
row := q.db.QueryRow(ctx, chargeChat, id)
var id_2 uuid.UUID
err := row.Scan(&id_2)
return id_2, err
}
const chargeMessage = `-- name: ChargeMessage :one
UPDATE users SET message_count=message_count+1 WHERE id=$1 AND message_count<1000000 RETURNING id
`
func (q *Queries) ChargeMessage(ctx context.Context, id uuid.UUID) (uuid.UUID, error) {
row := q.db.QueryRow(ctx, chargeMessage, id)
var id_2 uuid.UUID
err := row.Scan(&id_2)
return id_2, err
}
const createUser = `-- name: CreateUser :one
INSERT INTO users(id,username,display_name,password_hash) VALUES($1,$2,$3,$4)
RETURNING id,username,display_name,created_at
`
type CreateUserParams struct {
ID uuid.UUID
Username string
DisplayName string
PasswordHash string
}
type CreateUserRow struct {
ID uuid.UUID
Username string
DisplayName string
CreatedAt time.Time
}
func (q *Queries) CreateUser(ctx context.Context, arg CreateUserParams) (CreateUserRow, error) {
row := q.db.QueryRow(ctx, createUser,
arg.ID,
arg.Username,
arg.DisplayName,
arg.PasswordHash,
)
var i CreateUserRow
err := row.Scan(
&i.ID,
&i.Username,
&i.DisplayName,
&i.CreatedAt,
)
return i, err
}
const credentials = `-- name: Credentials :one
SELECT id,password_hash FROM users WHERE username=$1
`
type CredentialsRow struct {
ID uuid.UUID
PasswordHash string
}
func (q *Queries) Credentials(ctx context.Context, username string) (CredentialsRow, error) {
row := q.db.QueryRow(ctx, credentials, username)
var i CredentialsRow
err := row.Scan(&i.ID, &i.PasswordHash)
return i, err
}
const deleteExpiredSessions = `-- name: DeleteExpiredSessions :exec
DELETE FROM sessions WHERE user_id=$1 AND expires_at<=clock_timestamp()
`
func (q *Queries) DeleteExpiredSessions(ctx context.Context, userID uuid.UUID) error {
_, err := q.db.Exec(ctx, deleteExpiredSessions, userID)
return err
}
const getChat = `-- name: GetChat :one
SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id)::uuid[] AS members
FROM chats c WHERE c.id=$1
`
type GetChatRow struct {
ID uuid.UUID
Title string
CreatedAt time.Time
Members []uuid.UUID
}
func (q *Queries) GetChat(ctx context.Context, id uuid.UUID) (GetChatRow, error) {
row := q.db.QueryRow(ctx, getChat, id)
var i GetChatRow
err := row.Scan(
&i.ID,
&i.Title,
&i.CreatedAt,
&i.Members,
)
return i, err
}
const getUser = `-- name: GetUser :one
SELECT id,username,display_name,created_at FROM users WHERE id=$1
`
type GetUserRow struct {
ID uuid.UUID
Username string
DisplayName string
CreatedAt time.Time
}
func (q *Queries) GetUser(ctx context.Context, id uuid.UUID) (GetUserRow, error) {
row := q.db.QueryRow(ctx, getUser, id)
var i GetUserRow
err := row.Scan(
&i.ID,
&i.Username,
&i.DisplayName,
&i.CreatedAt,
)
return i, err
}
const insertChat = `-- name: InsertChat :one
INSERT INTO chats(id,title) VALUES($1,$2) RETURNING id,title,created_at
`
type InsertChatParams struct {
ID uuid.UUID
Title string
}
func (q *Queries) InsertChat(ctx context.Context, arg InsertChatParams) (Chat, error) {
row := q.db.QueryRow(ctx, insertChat, arg.ID, arg.Title)
var i Chat
err := row.Scan(&i.ID, &i.Title, &i.CreatedAt)
return i, err
}
const insertMembers = `-- name: InsertMembers :exec
INSERT INTO chat_members(chat_id,user_id) SELECT $1,unnest($2::uuid[])
`
type InsertMembersParams struct {
ChatID uuid.UUID
Column2 []uuid.UUID
}
func (q *Queries) InsertMembers(ctx context.Context, arg InsertMembersParams) error {
_, err := q.db.Exec(ctx, insertMembers, arg.ChatID, arg.Column2)
return err
}
const insertMessage = `-- name: InsertMessage :one
INSERT INTO messages(id,chat_id,sender_id,text) VALUES($1,$2,$3,$4) RETURNING id,chat_id,sender_id,text,created_at
`
type InsertMessageParams struct {
ID uuid.UUID
ChatID uuid.UUID
SenderID uuid.UUID
Text string
}
func (q *Queries) InsertMessage(ctx context.Context, arg InsertMessageParams) (Message, error) {
row := q.db.QueryRow(ctx, insertMessage,
arg.ID,
arg.ChatID,
arg.SenderID,
arg.Text,
)
var i Message
err := row.Scan(
&i.ID,
&i.ChatID,
&i.SenderID,
&i.Text,
&i.CreatedAt,
)
return i, err
}
const insertSession = `-- name: InsertSession :one
INSERT INTO sessions(token_hash,user_id,expires_at)
SELECT $1,$2,$3 WHERE (SELECT count(*) FROM sessions WHERE user_id=$2)<100
RETURNING user_id
`
type InsertSessionParams struct {
TokenHash string
UserID uuid.UUID
ExpiresAt time.Time
}
func (q *Queries) InsertSession(ctx context.Context, arg InsertSessionParams) (uuid.UUID, error) {
row := q.db.QueryRow(ctx, insertSession, arg.TokenHash, arg.UserID, arg.ExpiresAt)
var user_id uuid.UUID
err := row.Scan(&user_id)
return user_id, err
}
const listChats = `-- name: ListChats :many
SELECT c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id)::uuid[] AS members
FROM chats c JOIN chat_members m ON m.chat_id=c.id
WHERE m.user_id=$1 AND (c.created_at,c.id)>($2::timestamptz,$3::uuid)
ORDER BY c.created_at,c.id LIMIT $4
`
type ListChatsParams struct {
MemberID uuid.UUID
AfterTime time.Time
AfterID uuid.UUID
PageLimit int32
}
type ListChatsRow struct {
ID uuid.UUID
Title string
CreatedAt time.Time
Members []uuid.UUID
}
func (q *Queries) ListChats(ctx context.Context, arg ListChatsParams) ([]ListChatsRow, error) {
rows, err := q.db.Query(ctx, listChats,
arg.MemberID,
arg.AfterTime,
arg.AfterID,
arg.PageLimit,
)
if err != nil {
return nil, err
}
defer rows.Close()
items := []ListChatsRow{}
for rows.Next() {
var i ListChatsRow
if err := rows.Scan(
&i.ID,
&i.Title,
&i.CreatedAt,
&i.Members,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listMessages = `-- name: ListMessages :many
SELECT m.id,m.chat_id,m.sender_id,m.text,m.created_at,
ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY ordinal LIMIT 10)::uuid[] AS attachment_ids
FROM messages m WHERE m.chat_id=$1 AND (m.created_at,m.id)>($2::timestamptz,$3::uuid)
ORDER BY m.created_at,m.id LIMIT $4
`
type ListMessagesParams struct {
ChatID uuid.UUID
AfterTime time.Time
AfterID uuid.UUID
PageLimit int32
}
type ListMessagesRow struct {
ID uuid.UUID
ChatID uuid.UUID
SenderID uuid.UUID
Text string
CreatedAt time.Time
AttachmentIds []uuid.UUID
}
func (q *Queries) ListMessages(ctx context.Context, arg ListMessagesParams) ([]ListMessagesRow, error) {
rows, err := q.db.Query(ctx, listMessages,
arg.ChatID,
arg.AfterTime,
arg.AfterID,
arg.PageLimit,
)
if err != nil {
return nil, err
}
defer rows.Close()
items := []ListMessagesRow{}
for rows.Next() {
var i ListMessagesRow
if err := rows.Scan(
&i.ID,
&i.ChatID,
&i.SenderID,
&i.Text,
&i.CreatedAt,
&i.AttachmentIds,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const lockSessionOwner = `-- name: LockSessionOwner :one
SELECT id FROM users WHERE id=$1 FOR UPDATE
`
func (q *Queries) LockSessionOwner(ctx context.Context, id uuid.UUID) (uuid.UUID, error) {
row := q.db.QueryRow(ctx, lockSessionOwner, id)
var id_2 uuid.UUID
err := row.Scan(&id_2)
return id_2, err
}
const revokeSession = `-- name: RevokeSession :exec
DELETE FROM sessions WHERE token_hash=$1
`
func (q *Queries) RevokeSession(ctx context.Context, tokenHash string) error {
_, err := q.db.Exec(ctx, revokeSession, tokenHash)
return err
}
@@ -0,0 +1,32 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
package queries
import (
"context"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
)
type DBTX interface {
Exec(context.Context, string, ...interface{}) (pgconn.CommandTag, error)
Query(context.Context, string, ...interface{}) (pgx.Rows, error)
QueryRow(context.Context, string, ...interface{}) pgx.Row
}
func New(db DBTX) *Queries {
return &Queries{db: db}
}
type Queries struct {
db DBTX
}
func (q *Queries) WithTx(tx pgx.Tx) *Queries {
return &Queries{
db: tx,
}
}
@@ -0,0 +1,81 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: idempotency.sql
package queries
import (
"context"
"time"
"github.com/google/uuid"
)
const replayMessage = `-- name: ReplayMessage :one
SELECT m.id,m.chat_id,m.sender_id,m.text,m.created_at,i.payload_hash,
ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY ordinal LIMIT 10)::uuid[] AS attachment_ids
FROM idempotency i JOIN messages m ON m.id=i.message_id
WHERE i.user_id=$1 AND i.chat_id=$2 AND i.key=$3
`
type ReplayMessageParams struct {
UserID uuid.UUID
ChatID uuid.UUID
Key string
}
type ReplayMessageRow struct {
ID uuid.UUID
ChatID uuid.UUID
SenderID uuid.UUID
Text string
CreatedAt time.Time
PayloadHash string
AttachmentIds []uuid.UUID
}
func (q *Queries) ReplayMessage(ctx context.Context, arg ReplayMessageParams) (ReplayMessageRow, error) {
row := q.db.QueryRow(ctx, replayMessage, arg.UserID, arg.ChatID, arg.Key)
var i ReplayMessageRow
err := row.Scan(
&i.ID,
&i.ChatID,
&i.SenderID,
&i.Text,
&i.CreatedAt,
&i.PayloadHash,
&i.AttachmentIds,
)
return i, err
}
const reserveKey = `-- name: ReserveKey :one
INSERT INTO idempotency(user_id,chat_id,key,payload_hash,message_id,expires_at)
VALUES($1,$2,$3,$4,$5,clock_timestamp()+interval '24 hours')
ON CONFLICT(user_id,chat_id,key) DO UPDATE SET payload_hash=excluded.payload_hash,
message_id=excluded.message_id,expires_at=excluded.expires_at
WHERE idempotency.expires_at<=clock_timestamp()
RETURNING message_id
`
type ReserveKeyParams struct {
UserID uuid.UUID
ChatID uuid.UUID
Key string
PayloadHash string
MessageID uuid.UUID
}
func (q *Queries) ReserveKey(ctx context.Context, arg ReserveKeyParams) (uuid.UUID, error) {
row := q.db.QueryRow(ctx, reserveKey,
arg.UserID,
arg.ChatID,
arg.Key,
arg.PayloadHash,
arg.MessageID,
)
var message_id uuid.UUID
err := row.Scan(&message_id)
return message_id, err
}
@@ -0,0 +1,100 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
package queries
import (
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype"
)
type Attachment struct {
ID uuid.UUID
ChatID uuid.UUID
OwnerID uuid.UUID
Filename string
ContentType string
SizeBytes int64
Sha256 string
Status string
PipelineVersion int32
ErrorCode string
Variants []byte
StagingExpired bool
StagingCleaned bool
CreatedAt time.Time
UpdatedAt time.Time
AcceptedAt pgtype.Timestamptz
FirstReadyAt pgtype.Timestamptz
}
type AttachmentJob struct {
ID uuid.UUID
AttachmentID uuid.UUID
PipelineVersion int32
Attempt int32
Traceparent string
Tracestate string
LeaseID uuid.UUID
LeaseUntil time.Time
NextAttempt time.Time
PublishedAt time.Time
Completed bool
CreatedAt time.Time
ReplayCount int64
ReplayedAt time.Time
}
type Chat struct {
ID uuid.UUID
Title string
CreatedAt time.Time
}
type ChatMember struct {
ChatID uuid.UUID
UserID uuid.UUID
}
type Idempotency struct {
UserID uuid.UUID
ChatID uuid.UUID
Key string
PayloadHash string
MessageID uuid.UUID
ExpiresAt time.Time
}
type Message struct {
ID uuid.UUID
ChatID uuid.UUID
SenderID uuid.UUID
Text string
CreatedAt time.Time
}
type MessageAttachment struct {
MessageID uuid.UUID
AttachmentID uuid.UUID
Ordinal int32
}
type Session struct {
TokenHash string
UserID uuid.UUID
ExpiresAt time.Time
}
type User struct {
ID uuid.UUID
Username string
DisplayName string
PasswordHash string
CreatedAt time.Time
ChatCount int32
MessageCount int32
AttachmentCount int32
}
@@ -0,0 +1,29 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: observability.sql
package queries
import (
"context"
)
const pipelineMetrics = `-- name: PipelineMetrics :one
SELECT count(*)::bigint AS depth,
COALESCE(EXTRACT(EPOCH FROM clock_timestamp()-min(j.created_at)),0)::float8 AS oldest
FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id AND j.pipeline_version=a.pipeline_version
WHERE a.status IN ('queued','processing') AND NOT j.completed
`
type PipelineMetricsRow struct {
Depth int64
Oldest float64
}
func (q *Queries) PipelineMetrics(ctx context.Context) (PipelineMetricsRow, error) {
row := q.db.QueryRow(ctx, pipelineMetrics)
var i PipelineMetricsRow
err := row.Scan(&i.Depth, &i.Oldest)
return i, err
}
@@ -0,0 +1,403 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: pipeline.sql
package queries
import (
"context"
"github.com/google/uuid"
)
const acquireJob = `-- name: AcquireJob :one
UPDATE attachment_jobs j SET attempt=j.attempt+1,lease_id=$1,lease_until=clock_timestamp()+$2::int*interval '1 second'
WHERE j.id=$3 AND j.pipeline_version=$4 AND NOT j.completed
AND j.next_attempt<=clock_timestamp() AND j.lease_until<clock_timestamp()
AND EXISTS(SELECT 1 FROM attachments a WHERE a.id=j.attachment_id AND a.pipeline_version=j.pipeline_version AND a.status IN ('queued','processing'))
RETURNING j.id, j.attachment_id, j.pipeline_version, j.attempt, j.traceparent, j.tracestate, j.lease_id, j.lease_until, j.next_attempt, j.published_at, j.completed, j.created_at, j.replay_count, j.replayed_at
`
type AcquireJobParams struct {
LeaseID uuid.UUID
LeaseSeconds int32
JobID uuid.UUID
PipelineVersion int32
}
func (q *Queries) AcquireJob(ctx context.Context, arg AcquireJobParams) (AttachmentJob, error) {
row := q.db.QueryRow(ctx, acquireJob,
arg.LeaseID,
arg.LeaseSeconds,
arg.JobID,
arg.PipelineVersion,
)
var i AttachmentJob
err := row.Scan(
&i.ID,
&i.AttachmentID,
&i.PipelineVersion,
&i.Attempt,
&i.Traceparent,
&i.Tracestate,
&i.LeaseID,
&i.LeaseUntil,
&i.NextAttempt,
&i.PublishedAt,
&i.Completed,
&i.CreatedAt,
&i.ReplayCount,
&i.ReplayedAt,
)
return i, err
}
const advancePipeline = `-- name: AdvancePipeline :one
UPDATE attachments SET status='queued',error_code='',variants='[]',pipeline_version=pipeline_version+1,staging_expired=false,updated_at=clock_timestamp() WHERE id=$1 AND (status='ready' OR (status='failed' AND NOT staging_expired)) RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, pipeline_version, error_code, variants, staging_expired, staging_cleaned, created_at, updated_at, accepted_at, first_ready_at
`
func (q *Queries) AdvancePipeline(ctx context.Context, id uuid.UUID) (Attachment, error) {
row := q.db.QueryRow(ctx, advancePipeline, id)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.PipelineVersion,
&i.ErrorCode,
&i.Variants,
&i.StagingExpired,
&i.StagingCleaned,
&i.CreatedAt,
&i.UpdatedAt,
&i.AcceptedAt,
&i.FirstReadyAt,
)
return i, err
}
const completeAttachment = `-- name: CompleteAttachment :execrows
UPDATE attachments a SET first_ready_at=COALESCE(a.first_ready_at,clock_timestamp()),status='ready',variants=$1::jsonb,error_code='',updated_at=clock_timestamp()
FROM attachment_jobs j WHERE j.id=$2 AND j.attachment_id=a.id AND j.pipeline_version=a.pipeline_version
AND j.lease_id=$3 AND j.lease_until>clock_timestamp() AND NOT j.completed AND a.status='processing'
`
type CompleteAttachmentParams struct {
Variants []byte
JobID uuid.UUID
LeaseID uuid.UUID
}
func (q *Queries) CompleteAttachment(ctx context.Context, arg CompleteAttachmentParams) (int64, error) {
result, err := q.db.Exec(ctx, completeAttachment, arg.Variants, arg.JobID, arg.LeaseID)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const completeJob = `-- name: CompleteJob :exec
UPDATE attachment_jobs SET completed=true,lease_until='epoch' WHERE id=$1 AND lease_id=$2
`
type CompleteJobParams struct {
JobID uuid.UUID
LeaseID uuid.UUID
}
func (q *Queries) CompleteJob(ctx context.Context, arg CompleteJobParams) error {
_, err := q.db.Exec(ctx, completeJob, arg.JobID, arg.LeaseID)
return err
}
const insertJob = `-- name: InsertJob :exec
INSERT INTO attachment_jobs(id,attachment_id,pipeline_version,traceparent,tracestate)
VALUES($1,$2,$3,$4,$5)
`
type InsertJobParams struct {
JobID uuid.UUID
AttachmentID uuid.UUID
PipelineVersion int32
Traceparent string
Tracestate string
}
func (q *Queries) InsertJob(ctx context.Context, arg InsertJobParams) error {
_, err := q.db.Exec(ctx, insertJob,
arg.JobID,
arg.AttachmentID,
arg.PipelineVersion,
arg.Traceparent,
arg.Tracestate,
)
return err
}
const pendingJobs = `-- name: PendingJobs :many
SELECT j.id, j.attachment_id, j.pipeline_version, j.attempt, j.traceparent, j.tracestate, j.lease_id, j.lease_until, j.next_attempt, j.published_at, j.completed, j.created_at, j.replay_count, j.replayed_at FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id AND a.pipeline_version=j.pipeline_version
WHERE NOT j.completed AND a.status IN ('queued','processing') AND j.next_attempt<=clock_timestamp()
AND j.lease_until<clock_timestamp() AND j.published_at<clock_timestamp()-interval '2 seconds'
ORDER BY j.created_at,j.id LIMIT 100
`
func (q *Queries) PendingJobs(ctx context.Context) ([]AttachmentJob, error) {
rows, err := q.db.Query(ctx, pendingJobs)
if err != nil {
return nil, err
}
defer rows.Close()
items := []AttachmentJob{}
for rows.Next() {
var i AttachmentJob
if err := rows.Scan(
&i.ID,
&i.AttachmentID,
&i.PipelineVersion,
&i.Attempt,
&i.Traceparent,
&i.Tracestate,
&i.LeaseID,
&i.LeaseUntil,
&i.NextAttempt,
&i.PublishedAt,
&i.Completed,
&i.CreatedAt,
&i.ReplayCount,
&i.ReplayedAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const processingAttachment = `-- name: ProcessingAttachment :one
UPDATE attachments SET status='processing',updated_at=clock_timestamp()
WHERE id=$1 AND pipeline_version=$2 AND status IN ('queued','processing') RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, pipeline_version, error_code, variants, staging_expired, staging_cleaned, created_at, updated_at, accepted_at, first_ready_at
`
type ProcessingAttachmentParams struct {
AttachmentID uuid.UUID
PipelineVersion int32
}
func (q *Queries) ProcessingAttachment(ctx context.Context, arg ProcessingAttachmentParams) (Attachment, error) {
row := q.db.QueryRow(ctx, processingAttachment, arg.AttachmentID, arg.PipelineVersion)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.PipelineVersion,
&i.ErrorCode,
&i.Variants,
&i.StagingExpired,
&i.StagingCleaned,
&i.CreatedAt,
&i.UpdatedAt,
&i.AcceptedAt,
&i.FirstReadyAt,
)
return i, err
}
const published = `-- name: Published :exec
UPDATE attachment_jobs SET published_at=clock_timestamp() WHERE id=$1
`
func (q *Queries) Published(ctx context.Context, id uuid.UUID) error {
_, err := q.db.Exec(ctx, published, id)
return err
}
const queueAttachment = `-- name: QueueAttachment :one
UPDATE attachments SET accepted_at=COALESCE(accepted_at,clock_timestamp()),status='queued',updated_at=clock_timestamp() WHERE id=$1 AND status='uploading' AND NOT staging_expired RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, pipeline_version, error_code, variants, staging_expired, staging_cleaned, created_at, updated_at, accepted_at, first_ready_at
`
func (q *Queries) QueueAttachment(ctx context.Context, id uuid.UUID) (Attachment, error) {
row := q.db.QueryRow(ctx, queueAttachment, id)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.PipelineVersion,
&i.ErrorCode,
&i.Variants,
&i.StagingExpired,
&i.StagingCleaned,
&i.CreatedAt,
&i.UpdatedAt,
&i.AcceptedAt,
&i.FirstReadyAt,
)
return i, err
}
const replayAttachment = `-- name: ReplayAttachment :one
UPDATE attachments SET status='queued',error_code='',updated_at=clock_timestamp() WHERE id=$1 AND status='failed' AND NOT staging_expired RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, pipeline_version, error_code, variants, staging_expired, staging_cleaned, created_at, updated_at, accepted_at, first_ready_at
`
func (q *Queries) ReplayAttachment(ctx context.Context, id uuid.UUID) (Attachment, error) {
row := q.db.QueryRow(ctx, replayAttachment, id)
var i Attachment
err := row.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.PipelineVersion,
&i.ErrorCode,
&i.Variants,
&i.StagingExpired,
&i.StagingCleaned,
&i.CreatedAt,
&i.UpdatedAt,
&i.AcceptedAt,
&i.FirstReadyAt,
)
return i, err
}
const replayJobs = `-- name: ReplayJobs :exec
UPDATE attachment_jobs SET replay_count=replay_count+1,replayed_at=clock_timestamp(),completed=false,attempt=0,lease_until='epoch',next_attempt='epoch',published_at='epoch' WHERE attachment_id=$1 AND pipeline_version=$2
`
type ReplayJobsParams struct {
AttachmentID uuid.UUID
PipelineVersion int32
}
func (q *Queries) ReplayJobs(ctx context.Context, arg ReplayJobsParams) error {
_, err := q.db.Exec(ctx, replayJobs, arg.AttachmentID, arg.PipelineVersion)
return err
}
const retryAttachment = `-- name: RetryAttachment :execrows
UPDATE attachments a SET status=$1,error_code=$2,updated_at=clock_timestamp()
FROM attachment_jobs j WHERE j.id=$3 AND j.attachment_id=a.id AND j.pipeline_version=a.pipeline_version
AND j.lease_id=$4 AND j.lease_until>clock_timestamp() AND NOT j.completed AND a.status='processing'
`
type RetryAttachmentParams struct {
NextStatus string
ErrorCode string
JobID uuid.UUID
LeaseID uuid.UUID
}
func (q *Queries) RetryAttachment(ctx context.Context, arg RetryAttachmentParams) (int64, error) {
result, err := q.db.Exec(ctx, retryAttachment,
arg.NextStatus,
arg.ErrorCode,
arg.JobID,
arg.LeaseID,
)
if err != nil {
return 0, err
}
return result.RowsAffected(), nil
}
const retryJob = `-- name: RetryJob :exec
UPDATE attachment_jobs SET lease_until='epoch',next_attempt=clock_timestamp()+$1::int*interval '1 millisecond',completed=$2
WHERE id=$3 AND lease_id=$4
`
type RetryJobParams struct {
DelayMs int32
Completed bool
JobID uuid.UUID
LeaseID uuid.UUID
}
func (q *Queries) RetryJob(ctx context.Context, arg RetryJobParams) error {
_, err := q.db.Exec(ctx, retryJob,
arg.DelayMs,
arg.Completed,
arg.JobID,
arg.LeaseID,
)
return err
}
const stagingCleaned = `-- name: StagingCleaned :exec
UPDATE attachments SET staging_cleaned=true WHERE id=$1 AND (staging_expired OR status='ready')
`
func (q *Queries) StagingCleaned(ctx context.Context, id uuid.UUID) error {
_, err := q.db.Exec(ctx, stagingCleaned, id)
return err
}
const stagingCleanup = `-- name: StagingCleanup :many
WITH candidates AS (
SELECT id FROM attachments WHERE NOT staging_cleaned AND (status='ready' OR (status IN ('failed','uploading') AND updated_at<clock_timestamp()-interval '24 hours'))
ORDER BY updated_at,id LIMIT 20
)
UPDATE attachments a SET staging_expired=(a.status<>'ready') FROM candidates c WHERE a.id=c.id AND NOT a.staging_cleaned
AND (a.status='ready' OR (a.status IN ('failed','uploading') AND a.updated_at<clock_timestamp()-interval '24 hours')) RETURNING a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.status, a.pipeline_version, a.error_code, a.variants, a.staging_expired, a.staging_cleaned, a.created_at, a.updated_at, a.accepted_at, a.first_ready_at
`
func (q *Queries) StagingCleanup(ctx context.Context) ([]Attachment, error) {
rows, err := q.db.Query(ctx, stagingCleanup)
if err != nil {
return nil, err
}
defer rows.Close()
items := []Attachment{}
for rows.Next() {
var i Attachment
if err := rows.Scan(
&i.ID,
&i.ChatID,
&i.OwnerID,
&i.Filename,
&i.ContentType,
&i.SizeBytes,
&i.Sha256,
&i.Status,
&i.PipelineVersion,
&i.ErrorCode,
&i.Variants,
&i.StagingExpired,
&i.StagingCleaned,
&i.CreatedAt,
&i.UpdatedAt,
&i.AcceptedAt,
&i.FirstReadyAt,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
@@ -0,0 +1,46 @@
// Code generated by sqlc. DO NOT EDIT.
// versions:
// sqlc v1.31.1
// source: slo.sql
package queries
import (
"context"
)
const readySLI = `-- name: ReadySLI :one
WITH clock AS (SELECT clock_timestamp() AS now), cohort AS (
SELECT a.accepted_at,a.first_ready_at,c.now FROM attachments a CROSS JOIN clock c
WHERE a.accepted_at>c.now-interval '310 seconds' AND a.accepted_at<=c.now-interval '10 seconds'
)
SELECT count(*) FILTER(WHERE accepted_at>now-interval '20 seconds')::bigint AS matured10,
count(*) FILTER(WHERE accepted_at>now-interval '20 seconds' AND first_ready_at>=accepted_at AND first_ready_at<=accepted_at+interval '10 seconds')::bigint AS timely10,
count(*) FILTER(WHERE accepted_at>now-interval '40 seconds')::bigint AS matured30,
count(*) FILTER(WHERE accepted_at>now-interval '40 seconds' AND first_ready_at>=accepted_at AND first_ready_at<=accepted_at+interval '10 seconds')::bigint AS timely30,
count(*)::bigint AS matured300,
count(*) FILTER(WHERE first_ready_at>=accepted_at AND first_ready_at<=accepted_at+interval '10 seconds')::bigint AS timely300 FROM cohort
`
type ReadySLIRow struct {
Matured10 int64
Timely10 int64
Matured30 int64
Timely30 int64
Matured300 int64
Timely300 int64
}
func (q *Queries) ReadySLI(ctx context.Context) (ReadySLIRow, error) {
row := q.db.QueryRow(ctx, readySLI)
var i ReadySLIRow
err := row.Scan(
&i.Matured10,
&i.Timely10,
&i.Matured30,
&i.Timely30,
&i.Matured300,
&i.Timely300,
)
return i, err
}
@@ -0,0 +1,243 @@
package postgres
import (
"context"
"encoding/json"
"errors"
"time"
"backend/internal/adapter/repository/postgres/queries"
"backend/internal/domain/model"
"backend/internal/domain/repository"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/redis/go-redis/v9"
)
//go:generate go tool sqlc generate -f ../../../../db/sqlc/sqlc.yaml
type Store struct {
pool *pgxpool.Pool
q *queries.Queries
cache *redis.Client
}
func New(pool *pgxpool.Pool, cache *redis.Client) repository.Persistent {
return &Store{pool, queries.New(pool), cache}
}
func mapErr(err error) error {
if err == nil {
return nil
}
if errors.Is(err, pgx.ErrNoRows) {
return model.ErrNotFound
}
var pg *pgconn.PgError
if errors.As(err, &pg) {
switch pg.Code {
case "23505":
return model.ErrUsernameTaken
case "23503":
return model.ErrUnknownMember
case "23514":
return &model.ValidationError{Message: "constraint violation"}
}
}
return model.ErrUnavailable
}
func (s *Store) CreateUser(ctx context.Context, d model.UserDraft) (model.User, error) {
row, err := s.q.CreateUser(ctx, queries.CreateUserParams{ID: uuid.New(), Username: d.Username, DisplayName: d.DisplayName, PasswordHash: d.PasswordHash})
return model.User{ID: row.ID, Username: row.Username, DisplayName: row.DisplayName, CreatedAt: row.CreatedAt}, mapErr(err)
}
func (s *Store) GetUser(ctx context.Context, id uuid.UUID) (model.User, error) {
key := "profile:" + id.String()
if s.cache != nil {
c, cancel := context.WithTimeout(ctx, 80*time.Millisecond)
raw, err := s.cache.Get(c, key).Bytes()
cancel()
if err == nil {
var u model.User
if json.Unmarshal(raw, &u) == nil && u.ID == id {
return u, nil
}
}
}
row, err := s.q.GetUser(ctx, id)
if err != nil {
return model.User{}, mapErr(err)
}
user := model.User{ID: row.ID, Username: row.Username, DisplayName: row.DisplayName, CreatedAt: row.CreatedAt}
if s.cache != nil {
raw, e := json.Marshal(user)
if e == nil {
c, cancel := context.WithTimeout(ctx, 80*time.Millisecond)
_ = s.cache.Set(c, key, raw, time.Minute).Err()
cancel()
}
}
return user, nil
}
func (s *Store) CreateChat(ctx context.Context, d model.ChatDraft) (model.Chat, error) {
tx, err := s.pool.Begin(ctx)
if err != nil {
return model.Chat{}, mapErr(err)
}
defer func() { _ = tx.Rollback(ctx) }()
q := s.q.WithTx(tx)
if _, err = q.ChargeChat(ctx, d.Creator); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return model.Chat{}, model.ErrStorageFull
}
return model.Chat{}, mapErr(err)
}
row, err := q.InsertChat(ctx, queries.InsertChatParams{ID: uuid.New(), Title: d.Title})
if err != nil {
return model.Chat{}, mapErr(err)
}
if err = q.InsertMembers(ctx, queries.InsertMembersParams{ChatID: row.ID, Column2: d.MemberIDs}); err != nil {
return model.Chat{}, mapErr(err)
}
if err = tx.Commit(ctx); err != nil {
return model.Chat{}, mapErr(err)
}
return model.Chat{ID: row.ID, Title: row.Title, CreatedAt: row.CreatedAt, MemberIDs: d.MemberIDs}, nil
}
func (s *Store) GetChat(ctx context.Context, id uuid.UUID) (model.Chat, error) {
r, e := s.q.GetChat(ctx, id)
return model.Chat{ID: r.ID, Title: r.Title, CreatedAt: r.CreatedAt, MemberIDs: r.Members}, mapErr(e)
}
func position(q model.PageQuery) (time.Time, uuid.UUID) {
if q.After == nil {
return time.Unix(0, 0).UTC(), uuid.Nil
}
return q.After.CreatedAt, q.After.ID
}
func (s *Store) ListChats(ctx context.Context, id uuid.UUID, p model.PageQuery) ([]model.Chat, bool, error) {
at, aid := position(p)
rows, err := s.q.ListChats(ctx, queries.ListChatsParams{MemberID: id, AfterTime: at, AfterID: aid, PageLimit: int32(p.Limit + 1)})
if err != nil {
return nil, false, mapErr(err)
}
more := len(rows) > p.Limit
if more {
rows = rows[:p.Limit]
}
out := make([]model.Chat, len(rows))
for i, r := range rows {
out[i] = model.Chat{ID: r.ID, Title: r.Title, CreatedAt: r.CreatedAt, MemberIDs: r.Members}
}
return out, more, nil
}
func (s *Store) CreateMessage(ctx context.Context, d model.MessageDraft) (model.Message, error) {
tx, err := s.pool.Begin(ctx)
if err != nil {
return model.Message{}, mapErr(err)
}
defer func() { _ = tx.Rollback(ctx) }()
q := s.q.WithTx(tx)
messageID := uuid.New()
reserved, e := q.ReserveKey(ctx, queries.ReserveKeyParams{UserID: d.SenderID, ChatID: d.ChatID, Key: d.Key, PayloadHash: d.PayloadHash, MessageID: messageID})
if errors.Is(e, pgx.ErrNoRows) {
r, e := q.ReplayMessage(ctx, queries.ReplayMessageParams{UserID: d.SenderID, ChatID: d.ChatID, Key: d.Key})
if e != nil {
return model.Message{}, mapErr(e)
}
if r.PayloadHash != d.PayloadHash {
return model.Message{}, model.ErrConflict
}
return model.Message{ID: r.ID, ChatID: r.ChatID, SenderID: r.SenderID, Text: r.Text, CreatedAt: r.CreatedAt, AttachmentIDs: r.AttachmentIds}, nil
}
if e != nil {
return model.Message{}, mapErr(e)
}
messageID = reserved
for _, id := range d.AttachmentIDs {
a, e := q.GetAttachment(ctx, id)
if errors.Is(e, pgx.ErrNoRows) {
return model.Message{}, model.ErrNotFound
}
if e != nil {
return model.Message{}, mapErr(e)
}
if a.ChatID != d.ChatID {
return model.Message{}, model.ErrNotFound
}
if a.Status != "ready" && a.Status != "queued" && a.Status != "processing" {
return model.Message{}, model.ErrConflict
}
}
if _, err = q.ChargeMessage(ctx, d.SenderID); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return model.Message{}, model.ErrStorageFull
}
return model.Message{}, mapErr(err)
}
r, err := q.InsertMessage(ctx, queries.InsertMessageParams{ID: messageID, ChatID: d.ChatID, SenderID: d.SenderID, Text: d.Text})
if err != nil {
return model.Message{}, mapErr(err)
}
for i, id := range d.AttachmentIDs {
if e := q.InsertMessageAttachment(ctx, queries.InsertMessageAttachmentParams{MessageID: messageID, AttachmentID: id, Ordinal: int32(i)}); e != nil {
return model.Message{}, mapErr(e)
}
}
if err = tx.Commit(ctx); err != nil {
return model.Message{}, mapErr(err)
}
return model.Message{ID: r.ID, ChatID: r.ChatID, SenderID: r.SenderID, Text: r.Text, CreatedAt: r.CreatedAt, AttachmentIDs: d.AttachmentIDs}, nil
}
func (s *Store) ListMessages(ctx context.Context, id uuid.UUID, p model.PageQuery) ([]model.Message, bool, error) {
at, aid := position(p)
rows, err := s.q.ListMessages(ctx, queries.ListMessagesParams{ChatID: id, AfterTime: at, AfterID: aid, PageLimit: int32(p.Limit + 1)})
if err != nil {
return nil, false, mapErr(err)
}
more := len(rows) > p.Limit
if more {
rows = rows[:p.Limit]
}
out := make([]model.Message, len(rows))
for i, r := range rows {
out[i] = model.Message{ID: r.ID, ChatID: r.ChatID, SenderID: r.SenderID, Text: r.Text, CreatedAt: r.CreatedAt, AttachmentIDs: r.AttachmentIds}
}
return out, more, nil
}
func (s *Store) Credentials(ctx context.Context, name string) (uuid.UUID, string, error) {
r, e := s.q.Credentials(ctx, name)
if errors.Is(e, pgx.ErrNoRows) {
return uuid.Nil, "", model.ErrUnauthorized
}
return r.ID, r.PasswordHash, mapErr(e)
}
func (s *Store) CreateSession(ctx context.Context, hash string, id uuid.UUID, expiry time.Time) error {
tx, e := s.pool.Begin(ctx)
if e != nil {
return mapErr(e)
}
defer func() { _ = tx.Rollback(ctx) }()
q := s.q.WithTx(tx)
if _, e = q.LockSessionOwner(ctx, id); e != nil {
return mapErr(e)
}
if e = q.DeleteExpiredSessions(ctx, id); e != nil {
return mapErr(e)
}
if _, e = q.InsertSession(ctx, queries.InsertSessionParams{TokenHash: hash, UserID: id, ExpiresAt: expiry}); e != nil {
if errors.Is(e, pgx.ErrNoRows) {
return model.ErrStorageFull
}
return mapErr(e)
}
return mapErr(tx.Commit(ctx))
}
func (s *Store) AuthenticateSession(ctx context.Context, hash string) (uuid.UUID, error) {
r, e := s.q.AuthenticateSession(ctx, hash)
if errors.Is(e, pgx.ErrNoRows) {
return uuid.Nil, model.ErrUnauthorized
}
return r, mapErr(e)
}
func (s *Store) RevokeSession(ctx context.Context, hash string) error {
return mapErr(s.q.RevokeSession(ctx, hash))
}
@@ -0,0 +1,145 @@
package postgres
import (
"backend/db"
"backend/internal/config"
"backend/internal/domain/model"
"context"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgxpool"
"net/url"
"os"
"strings"
"testing"
"time"
)
func TestReadySLIIncludesFailedStuckReplayAndReprocessedAcceptances(t *testing.T) {
if os.Getenv("RUN_SLO_DB_TESTS") != "1" {
t.Skip("isolated real PostgreSQL required")
}
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
dsn, e := config.ReadSecret("/run/secrets/database_url")
if e != nil {
t.Fatal("database file unavailable")
}
operator, e := pgxpool.New(ctx, dsn)
if e != nil {
t.Fatal("operator unavailable")
}
defer operator.Close()
schema := "slo_" + strings.ReplaceAll(uuid.NewString(), "-", "")
if _, e = operator.Exec(ctx, "CREATE SCHEMA "+schema); e != nil {
t.Fatal("isolated schema failed")
}
defer func() {
c, stop := context.WithTimeout(context.Background(), 5*time.Second)
defer stop()
if _, e := operator.Exec(c, "DROP SCHEMA "+schema+" CASCADE"); e != nil {
t.Error("isolated schema cleanup failed")
}
}()
u, e := url.Parse(dsn)
if e != nil {
t.Fatal("DSN unavailable")
}
query := u.Query()
query.Set("search_path", schema)
u.RawQuery = query.Encode()
if e = db.Migrate(ctx, u.String()); e != nil {
t.Fatal("isolated real migrations failed")
}
pool, e := pgxpool.New(ctx, u.String())
if e != nil {
t.Fatal("isolated pool unavailable")
}
defer pool.Close()
s := New(pool, nil)
user, e := s.CreateUser(ctx, model.UserDraft{Username: "slo_" + uuid.NewString()[:8], DisplayName: "Synthetic SLO", PasswordHash: "disabled"})
if e != nil {
t.Fatal(e)
}
chat, e := s.CreateChat(ctx, model.ChatDraft{Title: "Owned cohort fixture", Creator: user.ID, MemberIDs: []uuid.UUID{user.ID}})
if e != nil {
t.Fatal(e)
}
reserve := func() model.Attachment {
t.Helper()
a, e := s.ReserveAttachment(ctx, model.Attachment{ID: uuid.New(), ChatID: chat.ID, OwnerID: user.ID, Filename: "cohort.png", ContentType: "image/png", SHA256: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", SizeBytes: 1})
if e != nil {
t.Fatal(e)
}
return a
}
var replayID uuid.UUID
for _, f := range []struct {
age, latency int
status string
advance bool
}{{15, 5, "ready", false}, {16, 11, "ready", false}, {17, -1, "failed", false}, {18, -1, "processing", false}, {35, 6, "ready", false}, {100, 6, "ready", false}, {5, 1, "ready", false}, {500, 2, "ready", false}, {15, 2, "ready", true}} {
a := reserve()
if _, e = s.QueueAttachment(ctx, a, "", ""); e != nil {
t.Fatal(e)
}
if _, e = pool.Exec(ctx, "UPDATE attachments SET accepted_at=clock_timestamp()-$2*interval '1 second',first_ready_at=CASE WHEN $3::int>=0 THEN clock_timestamp()-($2-$3)*interval '1 second' ELSE NULL END,status=$4 WHERE id=$1", a.ID, f.age, f.latency, f.status); e != nil {
t.Fatal(e)
}
if f.advance {
if ok, e := s.AdvancePipeline(ctx, a.ID); e != nil || !ok {
t.Fatal("actual version advance failed")
}
}
if f.status == "failed" {
replayID = a.ID
}
}
// A late explicit replay may become ready, but cannot erase the missed original deadline.
if ok, e := s.ReplayJob(ctx, replayID); e != nil || !ok {
t.Fatal("actual replay failed")
}
jobs, e := s.PendingJobs(ctx)
if e != nil {
t.Fatal(e)
}
for _, j := range jobs {
if j.AttachmentID == replayID {
lease := uuid.New()
owned, _, ok, e := s.AcquireJob(ctx, j, lease, 30*time.Second)
if e != nil || !ok {
t.Fatal("replay acquisition failed")
}
if ok, e = s.CompleteJob(ctx, owned, lease, []model.Variant{}); e != nil || !ok {
t.Fatal("replayed late completion failed")
}
}
}
// Upload can start long before durable acceptance. QueueAttachment must establish
// its own timestamp, while legacy rows with unknown acceptance remain unmeasured.
oldUpload := reserve()
if _, e = pool.Exec(ctx, "UPDATE attachments SET created_at=clock_timestamp()-interval '1 hour' WHERE id=$1", oldUpload.ID); e != nil {
t.Fatal(e)
}
if _, e = s.QueueAttachment(ctx, oldUpload, "", ""); e != nil {
t.Fatal(e)
}
var age float64
if e = pool.QueryRow(ctx, "SELECT EXTRACT(EPOCH FROM clock_timestamp()-accepted_at)::float8 FROM attachments WHERE id=$1", oldUpload.ID).Scan(&age); e != nil || age < 0 || age > 2 {
t.Fatal("acceptance clock incorrectly reused upload start")
}
legacy := reserve()
if _, e = pool.Exec(ctx, "UPDATE attachments SET created_at=clock_timestamp()-interval '15 seconds',status='ready' WHERE id=$1", legacy.ID); e != nil {
t.Fatal(e)
}
r, e := s.(*Store).q.ReadySLI(ctx)
if e != nil {
t.Fatal(e)
}
if r.Matured10 != 5 || r.Timely10 != 2 || r.Matured30 != 6 || r.Timely30 != 3 || r.Matured300 != 7 || r.Timely300 != 4 {
t.Fatal("SLO denominator or immutable first-ready outcome changed after replay/advance or counted immature/unknown upload")
}
metrics, e := s.PipelineMetrics(ctx)
if e != nil || metrics.Oldest < 0 || metrics.Oldest > 5 {
t.Fatal("queue age included pre-accept upload time instead of durable logical job")
}
}
@@ -0,0 +1,86 @@
package storage
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/signer"
"io"
"net/http"
"net/url"
"os"
"time"
)
type Config struct{ Endpoint, Access, Secret, Bucket string }
func ReadConfig(path string) (Config, error) {
var c Config
b, e := os.ReadFile(path)
if e != nil {
return c, fmt.Errorf("S3 credentials unavailable")
}
if e = json.Unmarshal(b, &c); e != nil {
return c, fmt.Errorf("invalid S3 credentials format")
}
if c.Endpoint == "" || c.Access == "" || c.Secret == "" || c.Bucket == "" {
return c, fmt.Errorf("incomplete S3 configuration")
}
return c, nil
}
func FromConfig(c Config) (*S3, error) { return New(c.Endpoint, c.Access, c.Secret, c.Bucket) }
// RustFS provides S3 SDKs, but its documented administrative API is separate.
// Use the SDK's SigV4 signer; do not implement signing or persist credentials in command arguments.
func admin(ctx context.Context, c Config, path string, body []byte) error {
r, e := http.NewRequestWithContext(ctx, http.MethodPut, "http://"+c.Endpoint+"/rustfs/admin/v3/"+path, bytes.NewReader(body))
if e != nil {
return e
}
sum := sha256.Sum256(body)
r.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum[:]))
r.Header.Set("Content-Type", "application/json")
signed := signer.SignV4(*r, c.Access, c.Secret, "", "us-east-1")
response, e := (&http.Client{Timeout: 5 * time.Second}).Do(signed)
if e != nil {
return fmt.Errorf("S3 admin transport failed")
}
defer response.Body.Close()
_, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 64*1024))
if response.StatusCode != 200 && response.StatusCode != 204 {
return fmt.Errorf("S3 admin operation rejected: HTTP %d", response.StatusCode)
}
return nil
}
func Bootstrap(ctx context.Context, root, app Config) error {
s, e := FromConfig(root)
if e != nil {
return e
}
exists, e := s.Client.BucketExists(ctx, root.Bucket)
if e != nil {
return fmt.Errorf("bucket availability check failed")
}
if !exists {
if e = s.Client.MakeBucket(ctx, root.Bucket, minio.MakeBucketOptions{Region: "us-east-1"}); e != nil {
return fmt.Errorf("bucket creation failed")
}
}
policy := map[string]any{"Version": "2012-10-17", "Statement": []any{
map[string]any{"Effect": "Allow", "Action": []string{"s3:GetBucketLocation", "s3:ListBucket"}, "Resource": []string{"arn:aws:s3:::" + app.Bucket}},
map[string]any{"Effect": "Allow", "Action": []string{"s3:PutObject", "s3:GetObject", "s3:DeleteObject"}, "Resource": []string{"arn:aws:s3:::" + app.Bucket + "/originals/*", "arn:aws:s3:::" + app.Bucket + "/staging/*", "arn:aws:s3:::" + app.Bucket + "/variants/*"}},
}}
raw, _ := json.Marshal(policy)
if e = admin(ctx, root, "add-canned-policy?name=messenger", raw); e != nil {
return e
}
raw, _ = json.Marshal(map[string]string{"secretKey": app.Secret, "status": "enabled"})
if e = admin(ctx, root, "add-user?accessKey="+url.QueryEscape(app.Access), raw); e != nil {
return e
}
return admin(ctx, root, "set-user-or-group-policy?policyName=messenger&userOrGroup="+url.QueryEscape(app.Access)+"&isGroup=false", nil)
}
@@ -0,0 +1,21 @@
package storage
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
)
func TestMissingBucketFailsReadiness(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(404) }))
defer server.Close()
store, err := New(strings.TrimPrefix(server.URL, "http://"), "synthetic-access", "synthetic-secret", "missing")
if err != nil {
t.Fatal(err)
}
if err = store.Ready(context.Background()); err == nil {
t.Fatal("missing bucket reported ready")
}
}
+81
View File
@@ -0,0 +1,81 @@
package storage
import (
"backend/internal/domain/model"
"backend/internal/domain/port"
"context"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"io"
"net"
"net/http"
"time"
)
type S3 struct {
Client *minio.Client
Bucket string
}
func New(endpoint, access, secret, bucket string) (*S3, error) {
client, err := minio.New(endpoint, &minio.Options{Creds: credentials.NewStaticV4(access, secret, ""), Secure: false, Region: "us-east-1", Transport: &http.Transport{DialContext: (&net.Dialer{Timeout: time.Second}).DialContext, ResponseHeaderTimeout: 3 * time.Second, MaxIdleConnsPerHost: 8}})
if err != nil {
return nil, err
}
return &S3{client, bucket}, nil
}
func mapError(err error) error {
if err == nil {
return nil
}
r := minio.ToErrorResponse(err)
if r.Code == "NoSuchKey" || r.Code == "NoSuchObject" || r.Code == "NotFound" {
return model.ErrNotFound
}
return model.ErrUnavailable
}
func (s *S3) Put(ctx context.Context, key string, r io.Reader, size int64, mime string) error {
_, err := s.Client.PutObject(ctx, s.Bucket, key, r, size, minio.PutObjectOptions{ContentType: mime, DisableMultipart: true})
return mapError(err)
}
func (s *S3) Get(ctx context.Context, key string) (io.ReadCloser, error) {
obj, err := s.Client.GetObject(ctx, s.Bucket, key, minio.GetObjectOptions{})
if err != nil {
return nil, mapError(err)
}
if _, err = obj.Stat(); err != nil {
_ = obj.Close()
return nil, mapError(err)
}
return obj, nil
}
func (s *S3) Stat(ctx context.Context, key string) (port.Object, error) {
r, e := s.Client.StatObject(ctx, s.Bucket, key, minio.StatObjectOptions{})
return port.Object{Key: key, ETag: r.ETag, Size: r.Size, Modified: r.LastModified.UTC().Truncate(time.Second)}, mapError(e)
}
func (s *S3) Remove(ctx context.Context, key string) error {
return mapError(s.Client.RemoveObject(ctx, s.Bucket, key, minio.RemoveObjectOptions{}))
}
func (s *S3) List(ctx context.Context, after string) ([]port.Object, bool, error) {
ctx, cancel := context.WithCancel(ctx)
defer cancel()
out := []port.Object{}
for r := range s.Client.ListObjects(ctx, s.Bucket, minio.ListObjectsOptions{Prefix: "originals/", StartAfter: after, Recursive: true, MaxKeys: 100}) {
if r.Err != nil {
return nil, false, mapError(r.Err)
}
// HTTP Last-Modified has second precision; RustFS inventory also carries milliseconds.
out = append(out, port.Object{Key: r.Key, ETag: r.ETag, Size: r.Size, Modified: r.LastModified.UTC().Truncate(time.Second)})
if len(out) > 100 {
return out[:100], true, nil
}
}
return out, false, nil
}
func (s *S3) Ready(ctx context.Context) error {
exists, err := s.Client.BucketExists(ctx, s.Bucket)
if err != nil || !exists {
return model.ErrUnavailable
}
return nil
}
@@ -0,0 +1,126 @@
package telemetry
import (
"backend/internal/domain/port"
"context"
"errors"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/trace"
"io"
"sync"
"sync/atomic"
"time"
)
type DBTracer struct{}
func (DBTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, _ pgx.TraceQueryStartData) context.Context {
ctx, _ = otel.Tracer("messenger").Start(ctx, "postgres.query", trace.WithSpanKind(trace.SpanKindClient))
return ctx
}
func (DBTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, d pgx.TraceQueryEndData) {
s := trace.SpanFromContext(ctx)
if d.Err != nil {
s.SetStatus(codes.Error, "database_error")
}
s.End()
}
type Objects struct{ port.Objects }
func (s Objects) Put(ctx context.Context, k string, r io.Reader, n int64, m string) error {
ctx, span := otel.Tracer("messenger").Start(ctx, "s3.put", trace.WithSpanKind(trace.SpanKindClient))
defer span.End()
e := s.Objects.Put(ctx, k, r, n, m)
if e != nil {
span.SetStatus(codes.Error, "storage_error")
}
return e
}
func (s Objects) Get(ctx context.Context, k string) (io.ReadCloser, error) {
ctx, span := otel.Tracer("messenger").Start(ctx, "s3.get", trace.WithSpanKind(trace.SpanKindClient))
r, e := s.Objects.Get(ctx, k)
if e != nil {
span.SetStatus(codes.Error, "storage_error")
span.End()
return nil, e
}
return &tracedReader{reader: r, span: span}, nil
}
// The caller owns Close, as for every io.ReadCloser. Keeping the span open until
// Close includes streaming body reads and late Close errors in the same boundary.
type tracedReader struct {
reader io.ReadCloser
span trace.Span
once sync.Once
closed atomic.Bool
closeError error
}
func (r *tracedReader) Read(p []byte) (int, error) {
if r.closed.Load() {
return 0, io.ErrClosedPipe
}
n, e := r.reader.Read(p)
if e != nil && !errors.Is(e, io.EOF) {
r.span.SetStatus(codes.Error, "storage_error")
}
return n, e
}
func (r *tracedReader) Close() error {
r.once.Do(func() {
r.closed.Store(true)
r.closeError = r.reader.Close()
if r.closeError != nil {
r.span.SetStatus(codes.Error, "storage_error")
}
r.span.End()
})
return r.closeError
}
func (s Objects) Stat(ctx context.Context, k string) (port.Object, error) {
ctx, span := otel.Tracer("messenger").Start(ctx, "s3.stat", trace.WithSpanKind(trace.SpanKindClient))
defer span.End()
r, e := s.Objects.Stat(ctx, k)
if e != nil {
span.SetStatus(codes.Error, "storage_error")
}
return r, e
}
type acquireKey struct{}
type acquireState struct {
telemetry *Telemetry
started time.Time
span trace.Span
}
func (DBTracer) TraceAcquireStart(ctx context.Context, _ *pgxpool.Pool, _ pgxpool.TraceAcquireStartData) context.Context {
t := current.Load()
if t == nil {
return ctx
}
t.poolPending.Add(1)
ctx, span := otel.Tracer("messenger").Start(ctx, "postgres.acquire", trace.WithSpanKind(trace.SpanKindClient))
return context.WithValue(ctx, acquireKey{}, acquireState{t, time.Now(), span})
}
func (DBTracer) TraceAcquireEnd(ctx context.Context, _ *pgxpool.Pool, data pgxpool.TraceAcquireEndData) {
s, ok := ctx.Value(acquireKey{}).(acquireState)
if !ok {
return
}
s.telemetry.poolPending.Add(-1)
outcome := "success"
if data.Err != nil {
outcome = "error"
s.span.SetStatus(codes.Error, "database_acquire_error")
}
s.telemetry.dbAcquire.Record(ctx, time.Since(s.started).Seconds(), metric.WithAttributes(attribute.String("outcome", outcome)))
s.span.End()
}
@@ -0,0 +1,85 @@
package telemetry
import (
"backend/internal/domain/port"
"context"
"errors"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/codes"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
"io"
"strings"
"testing"
)
type readerObjects struct {
port.Objects
r io.ReadCloser
}
func (o readerObjects) Get(context.Context, string) (io.ReadCloser, error) { return o.r, nil }
type controlledReadCloser struct {
io.Reader
readError, closeError error
closes int
}
func (r *controlledReadCloser) Read(p []byte) (int, error) {
if r.readError != nil {
return 0, r.readError
}
return r.Reader.Read(p)
}
func (r *controlledReadCloser) Close() error { r.closes++; return r.closeError }
func TestS3SpanCoversBodyReadAndClosesExactlyOnce(t *testing.T) {
for _, failure := range []string{"none", "read", "close"} {
t.Run(failure, func(t *testing.T) {
recorder := tracetest.NewSpanRecorder()
provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))
previous := otel.GetTracerProvider()
otel.SetTracerProvider(provider)
defer otel.SetTracerProvider(previous)
defer func() {
if e := provider.Shutdown(context.Background()); e != nil {
t.Error("provider cleanup failed")
}
}()
reader := &controlledReadCloser{Reader: strings.NewReader("synthetic body")}
if failure == "read" {
reader.readError = io.ErrUnexpectedEOF
}
if failure == "close" {
reader.closeError = io.ErrClosedPipe
}
objects := Objects{Objects: readerObjects{r: reader}}
r, e := objects.Get(context.Background(), "synthetic-key")
if e != nil {
t.Fatal(e)
}
if len(recorder.Ended()) != 0 {
t.Error("S3 span ended before body transfer")
}
_, readErr := io.ReadAll(r)
closeErr := r.Close()
_ = r.Close()
spans := recorder.Ended()
if len(spans) != 1 {
t.Fatal("stream span not ended exactly once")
}
if reader.closes != 1 {
t.Error("underlying reader closed repeatedly")
}
if failure != "none" && spans[0].Status().Code != codes.Error {
t.Error("body/close error missing from S3 span")
}
if failure == "read" && !errors.Is(readErr, io.ErrUnexpectedEOF) {
t.Error("read error lost")
}
if failure == "close" && !errors.Is(closeErr, io.ErrClosedPipe) {
t.Error("close error lost")
}
})
}
}
@@ -0,0 +1,77 @@
package telemetry
import (
"context"
"errors"
metricpb "go.opentelemetry.io/proto/otlp/collector/metrics/v1"
"google.golang.org/protobuf/proto"
"io"
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"time"
)
func TestReviewFailedDBGaugeDoesNotSuppressUnrelatedHTTPMetrics(t *testing.T) {
var metricExports atomic.Int64
var invalidDB, observerDown, independentHTTP atomic.Bool
sink := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/metrics" {
metricExports.Add(1)
raw, _ := io.ReadAll(r.Body)
data := &metricpb.ExportMetricsServiceRequest{}
if proto.Unmarshal(raw, data) == nil {
for _, rm := range data.ResourceMetrics {
for _, sm := range rm.ScopeMetrics {
for _, m := range sm.Metrics {
switch m.Name {
case "queue_depth", "ready_slo_matured_10s":
invalidDB.Store(true)
case "http_requests":
independentHTTP.Store(true)
case "database_observer_up":
for _, p := range m.GetGauge().GetDataPoints() {
if p.GetAsDouble() == 0 {
observerDown.Store(true)
}
}
}
}
}
}
}
}
w.Header().Set("Content-Type", "application/x-protobuf")
w.WriteHeader(200)
}))
defer sink.Close()
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", sink.URL)
t.Setenv("OTEL_SDK_DISABLED", "false")
t.Setenv("OTEL_TRACES_SAMPLER_ARG", "1")
sdk, err := Setup(context.Background(), "messenger-api")
if err != nil {
t.Fatal(err)
}
defer func() {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
sdk.Shutdown(ctx)
current.Store(nil)
}()
if err = sdk.Gauges(func(context.Context) (map[string]float64, error) {
return map[string]float64{"queue_depth": 0, "ready_slo_matured_10s": 0}, errors.New("synthetic database unavailable")
}); err != nil {
t.Fatal(err)
}
sdk.requests.Add(context.Background(), 1)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
flushErr := sdk.meters.ForceFlush(ctx)
if invalidDB.Load() || !observerDown.Load() || !independentHTTP.Load() {
t.Error("failed gauge leaked healthy zeros or omitted independent metrics/observer-down signal")
}
if metricExports.Load() == 0 {
t.Fatalf("independent HTTP metrics suppressed by failed DB gauge; flush error present=%t", flushErr != nil)
}
}
@@ -0,0 +1,95 @@
package telemetry
import (
"backend/internal/domain/model"
"context"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
"os"
"runtime"
"strconv"
"strings"
"syscall"
"time"
)
func (t *Telemetry) Gauges(source func(context.Context) (map[string]float64, error)) error {
if t == nil {
return nil
}
meter := t.meters.Meter("messenger")
names := []string{"queue_depth", "queue_oldest_age_seconds", "db_pool_used", "db_pool_capacity", "database_observer_up", "db_pool_pending_acquires", "worker_busy", "worker_capacity", "process_resident_memory_bytes", "go_heap_bytes", "ready_slo_matured_10s", "ready_slo_timely_10s", "ready_slo_matured_30s", "ready_slo_timely_30s", "ready_slo_matured_300s", "ready_slo_timely_300s"}
gauges := make([]metric.Float64ObservableGauge, len(names))
obs := make([]metric.Observable, len(names))
for i, name := range names {
g, e := meter.Float64ObservableGauge(name)
if e != nil {
return e
}
gauges[i] = g
obs[i] = g
}
cpu, e := meter.Float64ObservableCounter("process_cpu", metric.WithUnit("s"))
if e != nil {
return e
}
failures, e := meter.Float64ObservableCounter("database_observer_failures")
if e != nil {
return e
}
obs = append(obs, cpu, failures)
_, e = meter.RegisterCallback(func(ctx context.Context, o metric.Observer) error {
ctx, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
defer cancel()
values, err := source(ctx)
if err != nil {
values = nil
t.observationFailures.Add(1)
t.Event(ctx, "telemetry.database_observer", "unavailable")
}
if values == nil {
values = map[string]float64{}
}
o.ObserveFloat64(failures, float64(t.observationFailures.Load()))
values["database_observer_up"] = 1
if err != nil {
values["database_observer_up"] = 0
}
values["db_pool_pending_acquires"] = float64(t.poolPending.Load())
if t.worker {
values["worker_capacity"] = 1
values["worker_busy"] = float64(t.busy.Load())
}
var self, children syscall.Rusage
if syscall.Getrusage(syscall.RUSAGE_SELF, &self) == nil && syscall.Getrusage(syscall.RUSAGE_CHILDREN, &children) == nil {
o.ObserveFloat64(cpu, float64(self.Utime.Sec+self.Stime.Sec+children.Utime.Sec+children.Stime.Sec)+float64(self.Utime.Usec+self.Stime.Usec+children.Utime.Usec+children.Stime.Usec)/1e6)
}
var mem runtime.MemStats
runtime.ReadMemStats(&mem)
values["go_heap_bytes"] = float64(mem.HeapAlloc)
if raw, e := os.ReadFile("/proc/self/statm"); e == nil {
parts := strings.Fields(string(raw))
if len(parts) > 1 {
n, e := strconv.ParseUint(parts[1], 10, 64)
if e == nil {
values["process_resident_memory_bytes"] = float64(n * uint64(os.Getpagesize()))
}
}
}
for i, name := range names {
if v, ok := values[name]; ok {
if name == "queue_depth" || name == "queue_oldest_age_seconds" {
o.ObserveFloat64(gauges[i], v, metric.WithAttributes(attribute.String("queue", "attachment-jobs")))
} else {
o.ObserveFloat64(gauges[i], v)
}
}
}
return nil
}, obs...)
return e
}
func PipelineValues(m model.PipelineMetrics) map[string]float64 {
return map[string]float64{"queue_depth": float64(m.Depth), "queue_oldest_age_seconds": m.Oldest, "ready_slo_matured_10s": float64(m.ReadyMatured[0]), "ready_slo_timely_10s": float64(m.ReadyTimely[0]), "ready_slo_matured_30s": float64(m.ReadyMatured[1]), "ready_slo_timely_30s": float64(m.ReadyTimely[1]), "ready_slo_matured_300s": float64(m.ReadyMatured[2]), "ready_slo_timely_300s": float64(m.ReadyTimely[2])}
}
@@ -0,0 +1,112 @@
package telemetry
import (
"backend/internal/config"
"context"
"github.com/jackc/pgx/v5/pgxpool"
metricpb "go.opentelemetry.io/proto/otlp/collector/metrics/v1"
"google.golang.org/protobuf/proto"
"io"
"net/http"
"net/http/httptest"
"os"
"sync/atomic"
"testing"
"time"
)
func TestNativePoolWaitExportsPendingAcquire(t *testing.T) {
if os.Getenv("RUN_TELEMETRY_DB_TESTS") != "1" {
t.Skip("isolated real PostgreSQL required")
}
var observed atomic.Int64
observed.Store(-1)
sink := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/metrics" {
raw, _ := io.ReadAll(r.Body)
data := &metricpb.ExportMetricsServiceRequest{}
if proto.Unmarshal(raw, data) == nil {
for _, rm := range data.ResourceMetrics {
for _, sm := range rm.ScopeMetrics {
for _, m := range sm.Metrics {
if m.Name == "db_pool_pending_acquires" {
for _, p := range m.GetGauge().GetDataPoints() {
observed.Store(int64(p.GetAsDouble()))
}
}
}
}
}
}
}
w.Header().Set("Content-Type", "application/x-protobuf")
w.WriteHeader(200)
}))
defer sink.Close()
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", sink.URL)
t.Setenv("OTEL_SDK_DISABLED", "false")
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
sdk, e := Setup(ctx, "messenger-api")
if e != nil {
t.Fatal(e)
}
defer func() { sdk.Shutdown(context.Background()); current.Store(nil) }()
dsn, e := config.ReadSecret("/run/secrets/database_url")
if e != nil {
t.Fatal("database file unavailable")
}
pc, e := pgxpool.ParseConfig(dsn)
if e != nil {
t.Fatal("database config unavailable")
}
pc.MaxConns = 1
pc.ConnConfig.Tracer = DBTracer{}
pool, e := pgxpool.NewWithConfig(ctx, pc)
if e != nil {
t.Fatal("pool unavailable")
}
defer pool.Close()
held, e := pool.Acquire(ctx)
if e != nil {
t.Fatal("first acquisition failed")
}
defer held.Release()
done := make(chan error, 1)
go func() {
c, e := pool.Acquire(ctx)
if e == nil {
c.Release()
}
done <- e
}()
waiting := false
for i := 0; i < 1000; i++ {
if sdk.poolPending.Load() == 1 {
waiting = true
break
}
time.Sleep(time.Millisecond)
}
if !waiting {
t.Fatal("native blocked acquisition not tracked")
}
if e = sdk.Gauges(func(context.Context) (map[string]float64, error) {
return map[string]float64{"db_pool_used": float64(pool.Stat().AcquiredConns()), "db_pool_capacity": 1}, nil
}); e != nil {
t.Fatal(e)
}
if e = sdk.meters.ForceFlush(ctx); e != nil {
t.Fatal(e)
}
if observed.Load() != 1 {
t.Fatal("actual OTLP pending-acquires gauge did not show native pool wait")
}
held.Release()
if e = <-done; e != nil {
t.Fatal("queued acquisition failed after release")
}
if sdk.poolPending.Load() != 0 {
t.Fatal("completed native acquisition still pending")
}
}
@@ -0,0 +1,216 @@
package telemetry
import (
"backend/internal/domain/model"
"context"
"fmt"
"github.com/go-chi/chi/v5"
chimiddleware "github.com/go-chi/chi/v5/middleware"
"github.com/google/uuid"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
logapi "go.opentelemetry.io/otel/log"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/propagation"
sdklog "go.opentelemetry.io/otel/sdk/log"
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/trace"
"math"
"net/http"
"os"
"strconv"
"sync/atomic"
"time"
)
type Telemetry struct {
traces *sdktrace.TracerProvider
meters *sdkmetric.MeterProvider
logs *sdklog.LoggerProvider
logger logapi.Logger
requests metric.Int64Counter
durations metric.Float64Histogram
jobs metric.Int64Counter
jobDuration metric.Float64Histogram
readyDuration metric.Float64Histogram
busy atomic.Int64
poolPending atomic.Int64
observationFailures atomic.Int64
dbAcquire metric.Float64Histogram
worker bool
}
var current atomic.Pointer[Telemetry]
func Setup(ctx context.Context, service string) (*Telemetry, error) {
endpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
if endpoint == "" || os.Getenv("OTEL_SDK_DISABLED") == "true" {
return nil, nil
}
ratio := 1.0
if s := os.Getenv("OTEL_TRACES_SAMPLER_ARG"); s != "" {
parsed, err := strconv.ParseFloat(s, 64)
ratio = parsed
if err != nil || math.IsNaN(ratio) || math.IsInf(ratio, 0) || ratio < 0 || ratio > 1 {
return nil, fmt.Errorf("invalid sampling ratio")
}
}
resource := resource.NewWithAttributes("", attribute.String("service.name", service), attribute.String("service.version", "lab6-v1"), attribute.String("service.instance.id", uuid.NewString()), attribute.String("deployment.environment.name", "lab"))
tx, e := otlptracehttp.New(ctx, otlptracehttp.WithTimeout(time.Second), otlptracehttp.WithRetry(otlptracehttp.RetryConfig{Enabled: false}))
if e != nil {
return nil, e
}
mx, e := otlpmetrichttp.New(ctx, otlpmetrichttp.WithTimeout(time.Second), otlpmetrichttp.WithRetry(otlpmetrichttp.RetryConfig{Enabled: false}))
if e != nil {
return nil, e
}
lx, e := otlploghttp.New(ctx, otlploghttp.WithTimeout(time.Second), otlploghttp.WithRetry(otlploghttp.RetryConfig{Enabled: false}))
if e != nil {
return nil, e
}
t := &Telemetry{
worker: service == "messenger-worker",
traces: sdktrace.NewTracerProvider(sdktrace.WithResource(resource), sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(ratio))), sdktrace.WithBatcher(tx, sdktrace.WithMaxQueueSize(128), sdktrace.WithMaxExportBatchSize(32), sdktrace.WithBatchTimeout(time.Second), sdktrace.WithExportTimeout(time.Second))),
meters: sdkmetric.NewMeterProvider(sdkmetric.WithResource(resource), sdkmetric.WithReader(sdkmetric.NewPeriodicReader(mx, sdkmetric.WithInterval(2*time.Second), sdkmetric.WithTimeout(time.Second)))),
logs: sdklog.NewLoggerProvider(sdklog.WithResource(resource), sdklog.WithProcessor(sdklog.NewBatchProcessor(lx, sdklog.WithMaxQueueSize(128), sdklog.WithExportMaxBatchSize(32), sdklog.WithExportInterval(time.Second), sdklog.WithExportTimeout(time.Second)))),
}
otel.SetTracerProvider(t.traces)
otel.SetMeterProvider(t.meters)
otel.SetTextMapPropagator(propagation.TraceContext{})
t.logger = t.logs.Logger("messenger")
meter := t.meters.Meter("messenger")
t.requests, e = meter.Int64Counter("http_requests", metric.WithDescription("Business HTTP requests"))
if e != nil {
return nil, e
}
t.durations, e = meter.Float64Histogram("http_duration", metric.WithUnit("s"), metric.WithExplicitBucketBoundaries(.005, .01, .025, .05, .1, .25, .5, 1, 2, 5))
if e != nil {
return nil, e
}
t.jobs, e = meter.Int64Counter("jobs_processed", metric.WithDescription("Owned processing attempts by result"))
if e != nil {
return nil, e
}
t.jobDuration, e = meter.Float64Histogram("job_duration", metric.WithUnit("s"), metric.WithExplicitBucketBoundaries(.01, .05, .1, .25, .5, 1, 2, 5, 10, 20))
if e != nil {
return nil, e
}
t.dbAcquire, e = meter.Float64Histogram("db_pool_acquire_duration", metric.WithUnit("s"), metric.WithExplicitBucketBoundaries(.001, .005, .01, .025, .05, .1, .25, .5, 1, 2))
if e != nil {
return nil, e
}
t.readyDuration, e = meter.Float64Histogram("time_to_ready", metric.WithUnit("s"), metric.WithExplicitBucketBoundaries(.1, .5, 1, 2, 5, 10, 30, 60, 120))
if e != nil {
return nil, e
}
current.Store(t)
return t, nil
}
func (t *Telemetry) Shutdown(ctx context.Context) {
if t == nil {
return
}
_ = t.traces.Shutdown(ctx)
_ = t.meters.Shutdown(ctx)
_ = t.logs.Shutdown(ctx)
}
func (t *Telemetry) Event(ctx context.Context, event, outcome string, extra ...attribute.KeyValue) {
if t == nil {
return
}
var record logapi.Record
record.SetTimestamp(time.Now())
record.SetSeverity(logapi.SeverityInfo)
record.SetSeverityText("INFO")
record.SetBody(attribute.StringValue(event))
record.SetEventName(event)
record.AddAttributes(attribute.String("event", event), attribute.String("outcome", outcome))
record.AddAttributes(extra...)
t.logger.Emit(ctx, record)
}
func HTTP(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
t := current.Load()
if t == nil {
next.ServeHTTP(w, r)
return
}
ctx := otel.GetTextMapPropagator().Extract(r.Context(), propagation.HeaderCarrier(r.Header))
ctx, span := otel.Tracer("messenger").Start(ctx, "HTTP", trace.WithSpanKind(trace.SpanKindServer))
defer span.End()
carrier := propagation.MapCarrier{}
otel.GetTextMapPropagator().Inject(ctx, carrier)
ctx = model.WithTrace(ctx, carrier.Get("traceparent"), carrier.Get("tracestate"))
ww := chimiddleware.NewWrapResponseWriter(w, r.ProtoMajor)
start := time.Now()
next.ServeHTTP(ww, r.WithContext(ctx))
route := chi.RouteContext(r.Context()).RoutePattern()
if route == "" {
route = "unmatched"
}
code := ww.Status()
if code == 0 {
code = 200
}
outcome := fmt.Sprintf("%dxx", code/100)
method := r.Method
switch method {
case "GET", "POST", "DELETE", "PUT", "PATCH", "OPTIONS", "HEAD":
default:
method = "OTHER"
}
attrs := []attribute.KeyValue{attribute.String("method", method), attribute.String("route", route), attribute.String("status_class", outcome)}
span.SetName(r.Method + " " + route)
span.SetAttributes(attrs...)
if code >= 500 {
span.SetStatus(codes.Error, "server_error")
}
if route != "/health/ready" && route != "/health/live" {
t.requests.Add(ctx, 1, metric.WithAttributes(attrs...))
t.durations.Record(ctx, time.Since(start).Seconds(), metric.WithAttributes(attrs...))
}
if id := ww.Header().Get("X-Request-Id"); id != "" {
t.Event(ctx, "http.request", outcome, attribute.String("request_id", id))
} else {
t.Event(ctx, "http.request", outcome)
}
})
}
func (t *Telemetry) Worker(next func(context.Context, model.Job) error) func(context.Context, model.Job) error {
return func(ctx context.Context, j model.Job) error {
if t == nil {
return next(ctx, j)
}
t.busy.Add(1)
defer t.busy.Add(-1)
ctx = otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier{"traceparent": j.Traceparent, "tracestate": j.Tracestate})
ctx, span := otel.Tracer("messenger").Start(ctx, "attachment.process", trace.WithSpanKind(trace.SpanKindConsumer))
defer span.End()
e := next(ctx, j)
outcome := "processed"
if e != nil {
outcome = "dependency_error"
span.SetStatus(codes.Error, outcome)
}
t.Event(ctx, "worker.delivery", outcome)
return e
}
}
func (t *Telemetry) JobResult(ctx context.Context, j model.Job, outcome string, d time.Duration) {
if t == nil {
return
}
attrs := metric.WithAttributes(attribute.String("job_type", "attachment.process.v1"), attribute.String("outcome", outcome))
t.jobs.Add(ctx, 1, attrs)
t.jobDuration.Record(ctx, d.Seconds(), attrs)
if outcome == "ready" {
t.readyDuration.Record(ctx, time.Since(j.CreatedAt).Seconds(), metric.WithAttributes(attribute.String("job_type", "attachment.process.v1")))
}
t.Event(ctx, "worker.job", outcome)
}
@@ -0,0 +1,220 @@
package telemetry
import (
"backend/internal/domain/model"
"bytes"
"context"
"github.com/go-chi/chi/v5"
"github.com/jackc/pgx/v5/pgxpool"
"go.opentelemetry.io/otel/trace"
logpb "go.opentelemetry.io/proto/otlp/collector/logs/v1"
metricpb "go.opentelemetry.io/proto/otlp/collector/metrics/v1"
tracepb "go.opentelemetry.io/proto/otlp/collector/trace/v1"
"google.golang.org/protobuf/proto"
"io"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
)
func TestRealOTLPExportsCorrelatedSignalsWithoutRequestSecrets(t *testing.T) {
var mu sync.Mutex
traces := &tracepb.ExportTraceServiceRequest{}
logs := &logpb.ExportLogsServiceRequest{}
metrics := &metricpb.ExportMetricsServiceRequest{}
var exportErr error
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
raw, e := io.ReadAll(io.LimitReader(r.Body, 2<<20))
mu.Lock()
defer mu.Unlock()
if e != nil {
exportErr = e
}
if bytes.Contains(raw, []byte("synthetic-password-secret")) || bytes.Contains(raw, []byte("synthetic-token-secret")) {
exportErr = io.ErrUnexpectedEOF
}
switch r.URL.Path {
case "/v1/traces":
v := &tracepb.ExportTraceServiceRequest{}
if e := proto.Unmarshal(raw, v); e != nil {
exportErr = e
}
traces.ResourceSpans = append(traces.ResourceSpans, v.ResourceSpans...)
case "/v1/logs":
v := &logpb.ExportLogsServiceRequest{}
if e := proto.Unmarshal(raw, v); e != nil {
exportErr = e
}
logs.ResourceLogs = append(logs.ResourceLogs, v.ResourceLogs...)
case "/v1/metrics":
v := &metricpb.ExportMetricsServiceRequest{}
if e := proto.Unmarshal(raw, v); e != nil {
exportErr = e
}
metrics.ResourceMetrics = append(metrics.ResourceMetrics, v.ResourceMetrics...)
default:
exportErr = io.ErrUnexpectedEOF
}
w.Header().Set("Content-Type", "application/x-protobuf")
w.WriteHeader(200)
}))
defer server.Close()
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", server.URL)
t.Setenv("OTEL_SDK_DISABLED", "false")
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
sdk, e := Setup(ctx, "messenger-api")
if e != nil {
t.Fatal(e)
}
defer sdk.Shutdown(context.Background())
defer current.Store(nil)
var accepted model.Job
router := chi.NewRouter()
router.Use(HTTP)
router.Post("/api/v1/work", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("X-Request-Id", "synthetic-request-id")
carrier := model.TraceFrom(r.Context())
accepted = model.Job{Traceparent: carrier.Traceparent, Tracestate: carrier.Tracestate, CreatedAt: time.Now()}
w.WriteHeader(202)
})
request := httptest.NewRequest("POST", "/api/v1/work", bytes.NewBufferString(`{"password":"synthetic-password-secret"}`))
request.Header.Set("Authorization", "Bearer synthetic-token-secret")
router.ServeHTTP(httptest.NewRecorder(), request)
var workerTrace trace.TraceID
if e = sdk.Worker(func(c context.Context, j model.Job) error {
workerTrace = trace.SpanContextFromContext(c).TraceID()
acquired := DBTracer{}.TraceAcquireStart(c, nil, pgxpool.TraceAcquireStartData{})
if sdk.poolPending.Load() != 1 {
t.Error("pending acquire not tracked")
}
DBTracer{}.TraceAcquireEnd(acquired, nil, pgxpool.TraceAcquireEndData{})
if sdk.poolPending.Load() != 0 {
t.Error("completed acquire still pending")
}
sdk.JobResult(c, j, "ready", time.Millisecond)
return nil
})(ctx, accepted); e != nil {
t.Fatal(e)
}
if e = sdk.Gauges(func(context.Context) (map[string]float64, error) {
return map[string]float64{"queue_depth": 2, "db_pool_capacity": 8}, nil
}); e != nil {
t.Fatal(e)
}
if e = sdk.traces.ForceFlush(ctx); e != nil {
t.Fatal(e)
}
if e = sdk.logs.ForceFlush(ctx); e != nil {
t.Fatal(e)
}
if e = sdk.meters.ForceFlush(ctx); e != nil {
t.Fatal(e)
}
mu.Lock()
defer mu.Unlock()
if exportErr != nil {
t.Fatal("OTLP decoding or secret exclusion failed")
}
spanCount, correlatedLogs := 0, 0
requestCorrelated, queueNamed := false, false
for _, rs := range traces.ResourceSpans {
for _, ss := range rs.ScopeSpans {
for _, s := range ss.Spans {
if bytes.Equal(s.TraceId, workerTrace[:]) {
spanCount++
}
}
}
}
if spanCount != 3 {
t.Fatal("API and worker spans lost common trace")
}
for _, rl := range logs.ResourceLogs {
for _, sl := range rl.ScopeLogs {
for _, l := range sl.LogRecords {
for _, a := range l.Attributes {
if a.Key == "request_id" && a.Value.GetStringValue() == "synthetic-request-id" {
requestCorrelated = true
}
}
if bytes.Equal(l.TraceId, workerTrace[:]) && len(l.SpanId) == 8 {
correlatedLogs++
}
}
}
}
if correlatedLogs < 3 {
t.Fatal("structured logs lost trace/span correlation")
}
seen := map[string]bool{}
for _, rm := range metrics.ResourceMetrics {
for _, sm := range rm.ScopeMetrics {
for _, m := range sm.Metrics {
seen[m.Name] = true
for _, p := range m.GetSum().GetDataPoints() {
for _, a := range p.Attributes {
if a.Key == "request_id" {
t.Error("request id leaked into counter labels")
}
}
}
for _, p := range m.GetHistogram().GetDataPoints() {
for _, a := range p.Attributes {
if a.Key == "request_id" {
t.Error("request id leaked into histogram labels")
}
}
}
if m.Name == "queue_depth" {
for _, p := range m.GetGauge().GetDataPoints() {
for _, a := range p.Attributes {
if a.Key == "queue" && a.Value.GetStringValue() == "attachment-jobs" {
queueNamed = true
}
if a.Key == "request_id" {
t.Error("request identifier leaked into metric labels")
}
}
}
}
}
}
}
if !requestCorrelated || !queueNamed {
t.Error("OTLP logs lost request-id correlation or queue metric omitted queue name")
}
for _, name := range []string{"http_requests", "http_duration", "jobs_processed", "job_duration", "time_to_ready", "queue_depth", "db_pool_capacity", "process_resident_memory_bytes", "db_pool_pending_acquires", "db_pool_acquire_duration"} {
if !seen[name] {
t.Fatal("actual metric was not exported: " + name)
}
}
}
func TestInvalidSamplingRatioFailsBeforeProviderSetup(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/x-protobuf")
w.WriteHeader(200)
}))
defer server.Close()
t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", server.URL)
t.Setenv("OTEL_SDK_DISABLED", "false")
defer current.Store(nil)
for _, ratio := range []string{"NaN", "+Inf", "-Inf", "1.1", "-0.1", "not-a-number"} {
t.Run(ratio, func(t *testing.T) {
t.Setenv("OTEL_TRACES_SAMPLER_ARG", ratio)
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
sdk, e := Setup(ctx, "messenger-api")
if e == nil {
sdk.Shutdown(ctx)
t.Fatal("invalid/nonfinite sampling ratio accepted")
}
if sdk != nil {
t.Fatal("failed configuration published a provider")
}
})
}
}
@@ -0,0 +1,80 @@
package usecase
import (
"backend/internal/domain/model"
"backend/internal/domain/port"
"backend/internal/domain/repository"
"context"
"github.com/google/uuid"
"io"
)
type Attachments struct {
repo repository.AsyncAttachments
messenger *Messenger
objects port.Objects
}
func NewAttachments(r repository.AsyncAttachments, m *Messenger, o port.Objects) *Attachments {
return &Attachments{r, m, o}
}
func ObjectKey(id uuid.UUID) string { return "originals/" + id.String() }
func (a *Attachments) Upload(ctx context.Context, caller, chat uuid.UUID, name, mime, sha string, size int64, body io.Reader) (model.Attachment, error) {
if _, e := a.messenger.GetChat(ctx, caller, chat); e != nil {
return model.Attachment{}, e
}
row, e := a.repo.ReserveAttachment(ctx, model.Attachment{ID: uuid.New(), ChatID: chat, OwnerID: caller, Filename: name, ContentType: mime, SHA256: sha, SizeBytes: size})
if e != nil {
return model.Attachment{}, e
}
if e = a.objects.Put(ctx, "staging/"+row.ID.String(), body, size, mime); e != nil {
return model.Attachment{}, model.ErrUnavailable
}
carrier := model.TraceFrom(ctx)
return a.repo.QueueAttachment(ctx, row, carrier.Traceparent, carrier.Tracestate)
}
func (a *Attachments) Get(ctx context.Context, caller, id uuid.UUID) (model.Attachment, error) {
row, e := a.repo.GetAttachment(ctx, id)
if e != nil {
return model.Attachment{}, e
}
if _, e = a.messenger.GetChat(ctx, caller, row.ChatID); e != nil {
return model.Attachment{}, e
}
return row, nil
}
func (a *Attachments) Content(ctx context.Context, caller, id uuid.UUID, variant string) (model.Attachment, io.ReadCloser, error) {
row, e := a.Get(ctx, caller, id)
if e != nil {
return row, nil, e
}
if row.Status != "ready" {
return row, nil, model.ErrConflict
}
key := ObjectKey(id)
if variant != "original" {
found := false
for _, v := range row.Variants {
if v.Name == variant {
found = true
key = VariantKey(id, row.PipelineVersion, variant)
row.SizeBytes = v.SizeBytes
row.ContentType = v.ContentType
row.SHA256 = v.SHA256
break
}
}
if !found {
return row, nil, model.ErrNotFound
}
}
info, e := a.objects.Stat(ctx, key)
if e != nil || info.Size != row.SizeBytes {
return row, nil, model.ErrUnavailable
}
r, e := a.objects.Get(ctx, key)
if e != nil {
return row, nil, model.ErrUnavailable
}
return row, r, nil
}
@@ -0,0 +1,86 @@
package usecase
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"time"
"unicode/utf8"
"backend/internal/domain/model"
"backend/internal/domain/port"
)
type Auth struct {
sessions port.Sessions
passwords port.Passwords
messenger *Messenger
ttl time.Duration
now func() time.Time
}
type Session struct {
Token string
ExpiresAt time.Time
User model.User
}
func NewAuth(s port.Sessions, p port.Passwords, m *Messenger, ttl time.Duration, now func() time.Time) *Auth {
return &Auth{s, p, m, ttl, now}
}
func (a *Auth) Register(ctx context.Context, username, name, password string) (model.User, error) {
if n := utf8.RuneCountInString(password); n < 12 || n > 128 || !utf8.ValidString(password) {
return model.User{}, &model.ValidationError{Message: "password: 12–128 Unicode characters"}
}
draft, err := model.NewUserDraft(username, name)
if err != nil {
return model.User{}, err
}
hash, err := a.passwords.Hash(password)
if err != nil {
return model.User{}, err
}
draft.PasswordHash = hash
return a.messenger.repo.CreateUser(ctx, draft)
}
func (a *Auth) Login(ctx context.Context, username, password string) (Session, error) {
id, hash, err := a.sessions.Credentials(ctx, username)
if err != nil {
return Session{}, err
}
if !a.passwords.Verify(password, hash) {
return Session{}, model.ErrUnauthorized
}
user, err := a.messenger.repo.GetUser(ctx, id)
if err != nil {
return Session{}, err
}
raw := make([]byte, 32)
if _, err = rand.Read(raw); err != nil {
return Session{}, err
}
token := base64.RawURLEncoding.EncodeToString(raw)
expires := a.now().UTC().Add(a.ttl)
if err = a.sessions.CreateSession(ctx, TokenHash(token), id, expires); err != nil {
return Session{}, err
}
return Session{token, expires, user}, nil
}
func TokenHash(token string) string {
sum := sha256.Sum256([]byte(token))
return hex.EncodeToString(sum[:])
}
func (a *Auth) Authenticate(ctx context.Context, token string) (model.User, error) {
if len(token) != 43 {
return model.User{}, model.ErrUnauthorized
}
id, err := a.sessions.AuthenticateSession(ctx, TokenHash(token))
if err != nil {
return model.User{}, err
}
return a.messenger.repo.GetUser(ctx, id)
}
func (a *Auth) Logout(ctx context.Context, token string) error {
return a.sessions.RevokeSession(ctx, TokenHash(token))
}
@@ -0,0 +1,102 @@
package usecase
import (
"context"
"backend/internal/domain/model"
"backend/internal/domain/repository"
"github.com/google/uuid"
)
const chatsCursorScope = "chats"
// Page — страница списка; NextCursor == nil, если следующей записи нет.
type Page[T any] struct {
Items []T
NextCursor *string
}
type Messenger struct {
repo repository.Repository
}
func NewMessenger(repo repository.Repository) *Messenger {
return &Messenger{repo: repo}
}
func (m *Messenger) CreateChat(ctx context.Context, caller uuid.UUID, title string, memberIDs []uuid.UUID) (model.Chat, error) {
draft, err := model.NewChatDraft(caller, title, memberIDs)
if err != nil {
return model.Chat{}, err
}
return m.repo.CreateChat(ctx, draft)
}
func (m *Messenger) ListChats(ctx context.Context, caller uuid.UUID, limit *int, cursor *string) (Page[model.Chat], error) {
q, err := model.NewPageQuery(limit, cursor, chatsCursorScope)
if err != nil {
return Page[model.Chat]{}, err
}
chats, more, err := m.repo.ListChats(ctx, caller, q)
if err != nil {
return Page[model.Chat]{}, err
}
return newPage(chats, more, chatsCursorScope, model.Chat.Position), nil
}
// GetChat отдаёт чат только участнику; постороннему — ErrNotFound, чтобы не
// раскрывать существование чужого чата.
func (m *Messenger) GetChat(ctx context.Context, caller, chatID uuid.UUID) (model.Chat, error) {
chat, err := m.repo.GetChat(ctx, chatID)
if err != nil {
return model.Chat{}, err
}
if !chat.HasMember(caller) {
return model.Chat{}, model.ErrNotFound
}
return chat, nil
}
// SendMessage сначала проверяет доступ, затем тело: посторонний получает 404
// даже на невалидное сообщение. Проверка членства и запись не атомарны, но
// участники чата неизменяемы, а чаты не удаляются, поэтому гонки здесь нет.
func (m *Messenger) SendMessage(ctx context.Context, caller, chatID uuid.UUID, text, key string, attachments []uuid.UUID) (model.Message, error) {
if _, err := m.GetChat(ctx, caller, chatID); err != nil {
return model.Message{}, err
}
draft, err := model.NewMessageDraft(chatID, caller, text, attachments)
if err != nil {
return model.Message{}, err
}
draft, err = model.WithKey(draft, key)
if err != nil {
return model.Message{}, err
}
return m.repo.CreateMessage(ctx, draft)
}
func (m *Messenger) ListMessages(ctx context.Context, caller, chatID uuid.UUID, limit *int, cursor *string) (Page[model.Message], error) {
if _, err := m.GetChat(ctx, caller, chatID); err != nil {
return Page[model.Message]{}, err
}
scope := "messages:" + chatID.String()
q, err := model.NewPageQuery(limit, cursor, scope)
if err != nil {
return Page[model.Message]{}, err
}
messages, more, err := m.repo.ListMessages(ctx, chatID, q)
if err != nil {
return Page[model.Message]{}, err
}
return newPage(messages, more, scope, model.Message.Position), nil
}
func newPage[T any](items []T, more bool, scope string, position func(T) model.Position) Page[T] {
page := Page[T]{Items: items}
if more && len(items) > 0 {
next := model.EncodeCursor(scope, position(items[len(items)-1]))
page.NextCursor = &next
}
return page
}
@@ -0,0 +1,159 @@
package usecase
import (
"backend/internal/domain/model"
"backend/internal/domain/port"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"github.com/google/uuid"
"io"
"math/rand/v2"
"time"
)
type Pipeline struct {
store port.PipelineStore
objects port.Objects
transform port.Transformer
MaxAttempts int32
Fault func(string) error
OnResult func(context.Context, model.Job, string, time.Duration)
}
func NewPipeline(s port.PipelineStore, o port.Objects, t port.Transformer, max int32) *Pipeline {
return &Pipeline{store: s, objects: o, transform: t, MaxAttempts: max, Fault: func(string) error { return nil }}
}
func VariantKey(id uuid.UUID, version int32, name string) string {
return fmt.Sprintf("variants/%s/v%d/%s", id, version, name)
}
func (p *Pipeline) Process(ctx context.Context, j model.Job) error {
lease := uuid.New()
owned, a, ok, e := p.store.AcquireJob(ctx, j, lease, 30*time.Second)
if e != nil {
return e
}
if !ok {
return p.Cleanup(ctx)
}
j = owned
started := time.Now()
outcome := "retry"
if j.Attempt >= p.MaxAttempts {
outcome = "failed"
}
reported := false
defer func() {
if !reported && p.OnResult != nil {
p.OnResult(ctx, j, outcome, time.Since(started))
}
}()
if j.Attempt > p.MaxAttempts {
return p.store.RetryJob(ctx, j, lease, "retry_exhausted", 0, true)
}
// Reprocessing uses the immutable original after staging was safely removed.
key := "staging/" + a.ID.String()
r, e := p.objects.Get(ctx, key)
if errors.Is(e, model.ErrNotFound) {
r, e = p.objects.Get(ctx, ObjectKey(a.ID))
}
if e != nil {
return p.retry(ctx, j, lease, e)
}
original, e := io.ReadAll(io.LimitReader(r, model.MaxImageBytes+1))
_ = r.Close()
if e != nil || int64(len(original)) != a.SizeBytes {
return p.retry(ctx, j, lease, model.ErrUnavailable)
}
sum := sha256.Sum256(original)
if hex.EncodeToString(sum[:]) != a.SHA256 {
outcome = "failed"
return p.retry(ctx, j, lease, model.ErrInvalidImage)
}
if e = p.objects.Put(ctx, ObjectKey(a.ID), bytes.NewReader(original), a.SizeBytes, a.ContentType); e != nil {
return p.retry(ctx, j, lease, e)
}
if e = p.Fault("after_original"); e != nil {
return e
}
variants, e := p.transform.Transform(ctx, bytes.NewReader(original))
if e != nil {
if errors.Is(e, model.ErrInvalidImage) {
outcome = "failed"
}
return p.retry(ctx, j, lease, e)
}
result := make([]model.Variant, 0, len(variants))
for _, v := range variants {
if e = p.objects.Put(ctx, VariantKey(a.ID, j.PipelineVersion, v.Metadata.Name), bytes.NewReader(v.Bytes), int64(len(v.Bytes)), v.Metadata.ContentType); e != nil {
return p.retry(ctx, j, lease, e)
}
result = append(result, v.Metadata)
}
if e = p.Fault("after_objects"); e != nil {
return e
}
ok, e = p.store.CompleteJob(ctx, j, lease, result)
if e != nil {
return e
}
if !ok {
outcome = "stale"
return nil
}
outcome = "ready"
if p.OnResult != nil {
p.OnResult(ctx, j, outcome, time.Since(started))
}
reported = true
if e = p.Fault("after_commit"); e != nil {
return e
}
return p.Cleanup(ctx)
}
func (p *Pipeline) retry(ctx context.Context, j model.Job, lease uuid.UUID, cause error) error {
failed := j.Attempt >= p.MaxAttempts
code := "processing_failed"
if failed {
code = "retry_exhausted"
}
if errors.Is(cause, model.ErrInvalidImage) {
failed = true
code = "invalid_image"
}
delay := time.Duration((1<<min(j.Attempt, 5))*100+rand.IntN(100)) * time.Millisecond
if e := p.store.RetryJob(ctx, j, lease, code, delay, failed); e != nil {
return e
}
// This delivery has durably recorded its retry/failed effect. Ack it; the
// transactional outbox schedules the next attempt instead of retaining a PEL entry.
return nil
}
func (p *Pipeline) Cleanup(ctx context.Context) error {
rows, e := p.store.StagingCleanup(ctx)
if e != nil {
return e
}
for _, a := range rows {
if a.Status == "ready" {
info, e := p.objects.Stat(ctx, ObjectKey(a.ID))
if e != nil {
return e
}
if info.Size != a.SizeBytes {
return model.ErrUnavailable
}
}
if e = p.objects.Remove(ctx, "staging/"+a.ID.String()); e != nil {
return e
}
if e = p.store.StagingCleaned(ctx, a.ID); e != nil {
return e
}
}
return nil
}
@@ -0,0 +1,190 @@
package usecase
import (
"backend/internal/domain/model"
"backend/internal/domain/port"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"github.com/google/uuid"
"io"
"syscall"
"testing"
"time"
)
type pipelineStoreFixture struct {
failed bool
code string
complete bool
a model.Attachment
cleaned int
}
func (f *pipelineStoreFixture) PendingJobs(context.Context) ([]model.Job, error) { return nil, nil }
func (f *pipelineStoreFixture) Published(context.Context, uuid.UUID) error { return nil }
func (f *pipelineStoreFixture) AcquireJob(_ context.Context, j model.Job, _ uuid.UUID, _ time.Duration) (model.Job, model.Attachment, bool, error) {
j.Attempt = 1
return j, f.a, !f.complete, nil
}
func (f *pipelineStoreFixture) CompleteJob(context.Context, model.Job, uuid.UUID, []model.Variant) (bool, error) {
f.complete = true
f.a.Status = "ready"
return true, nil
}
func (f *pipelineStoreFixture) RetryJob(_ context.Context, _ model.Job, _ uuid.UUID, code string, _ time.Duration, failed bool) error {
f.failed = failed
f.code = code
return nil
}
func (f *pipelineStoreFixture) ReplayJob(context.Context, uuid.UUID) (bool, error) { return false, nil }
func (f *pipelineStoreFixture) AdvancePipeline(context.Context, uuid.UUID) (bool, error) {
return false, nil
}
func (f *pipelineStoreFixture) StagingCleanup(context.Context) ([]model.Attachment, error) {
if f.complete && f.cleaned == 0 {
return []model.Attachment{f.a}, nil
}
return nil, nil
}
func (f *pipelineStoreFixture) StagingCleaned(context.Context, uuid.UUID) error {
f.cleaned++
return nil
}
type pipelineObjectsFixture struct {
content []byte
removeErr error
removed int
}
func (f *pipelineObjectsFixture) Get(context.Context, string) (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(f.content)), nil
}
func (f *pipelineObjectsFixture) Put(context.Context, string, io.Reader, int64, string) error {
return nil
}
func (f *pipelineObjectsFixture) Stat(context.Context, string) (port.Object, error) {
return port.Object{Size: int64(len(f.content))}, nil
}
func (f *pipelineObjectsFixture) Remove(context.Context, string) error {
if f.removeErr != nil {
return f.removeErr
}
f.removed++
return nil
}
func (f *pipelineObjectsFixture) List(context.Context, string) ([]port.Object, bool, error) {
return nil, false, nil
}
func (f *pipelineObjectsFixture) Ready(context.Context) error { return nil }
type transformFixture struct{ err error }
func (f transformFixture) Transform(context.Context, io.Reader) ([]port.ProcessedVariant, error) {
return nil, f.err
}
func TestInfrastructureTransformFailureIsRetried(t *testing.T) {
raw := []byte("valid durable bytes")
sum := sha256.Sum256(raw)
f := &pipelineStoreFixture{a: model.Attachment{ID: uuid.New(), SizeBytes: int64(len(raw)), SHA256: hex.EncodeToString(sum[:])}}
for _, failure := range []error{syscall.ENOSPC, errors.New("convert executable unavailable")} {
p := NewPipeline(f, &pipelineObjectsFixture{content: raw}, transformFixture{failure}, 5)
if e := p.Process(context.Background(), model.Job{ID: uuid.New(), PipelineVersion: 1}); e != nil {
t.Fatal(e)
}
if f.failed || f.code != "processing_failed" {
t.Fatalf("temporary transform infrastructure failure became permanent: failed=%t code=%s", f.failed, f.code)
}
}
}
func TestCompletedDeliveryRecoversStagingCleanupAfterCrash(t *testing.T) {
raw := []byte("immutable original")
sum := sha256.Sum256(raw)
store := &pipelineStoreFixture{a: model.Attachment{ID: uuid.New(), SizeBytes: int64(len(raw)), SHA256: hex.EncodeToString(sum[:])}}
objects := &pipelineObjectsFixture{content: raw}
p := NewPipeline(store, objects, transformFixture{}, 5)
p.Fault = func(point string) error {
if point == "after_commit" {
return errors.New("worker crash")
}
return nil
}
j := model.Job{ID: uuid.New(), PipelineVersion: 1}
if e := p.Process(context.Background(), j); e == nil {
t.Fatal("crash was not injected")
}
if objects.removed != 0 {
t.Fatal("staging removed before simulated crash")
}
p = NewPipeline(store, objects, transformFixture{}, 5)
if e := p.Process(context.Background(), j); e != nil {
t.Fatal(e)
}
if objects.removed != 1 || store.cleaned != 1 || store.a.Status != "ready" {
t.Fatalf("completed redelivery did not recover cleanup: removed=%d durable_cleaned=%d status=%s", objects.removed, store.cleaned, store.a.Status)
}
if e := p.Process(context.Background(), j); e != nil {
t.Fatal(e)
}
if objects.removed != 1 {
t.Fatal("completed cleanup was repeated")
}
}
type blockedCleanupObjects struct {
*pipelineObjectsFixture
entered, release chan struct{}
}
func (f *blockedCleanupObjects) Remove(ctx context.Context, key string) error {
close(f.entered)
select {
case <-f.release:
return f.pipelineObjectsFixture.Remove(ctx, key)
case <-ctx.Done():
return ctx.Err()
}
}
func TestReadyTelemetryPublishedBeforeSlowStagingCleanup(t *testing.T) {
raw := []byte("immutable original")
sum := sha256.Sum256(raw)
store := &pipelineStoreFixture{a: model.Attachment{ID: uuid.New(), SizeBytes: int64(len(raw)), SHA256: hex.EncodeToString(sum[:])}}
objects := &blockedCleanupObjects{pipelineObjectsFixture: &pipelineObjectsFixture{content: raw}, entered: make(chan struct{}), release: make(chan struct{})}
p := NewPipeline(store, objects, transformFixture{}, 5)
events := make(chan string, 2)
p.OnResult = func(_ context.Context, _ model.Job, outcome string, _ time.Duration) { events <- outcome }
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
done := make(chan error, 1)
go func() { done <- p.Process(ctx, model.Job{ID: uuid.New(), PipelineVersion: 1, CreatedAt: time.Now()}) }()
select {
case <-objects.entered:
case <-ctx.Done():
t.Fatal("cleanup was not reached")
}
reported := 0
select {
case outcome := <-events:
reported++
if outcome != "ready" {
t.Error("wrong published outcome")
}
default:
t.Error("ready telemetry was delayed until staging cleanup")
}
close(objects.release)
if e := <-done; e != nil {
t.Fatal(e)
}
select {
case <-events:
reported++
default:
}
if reported != 1 {
t.Error("ready attempt emitted an incorrect event count")
}
}
@@ -0,0 +1,166 @@
package usecase
import (
"backend/internal/domain/model"
"backend/internal/domain/port"
"backend/internal/domain/repository"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"github.com/google/uuid"
"io"
"strings"
"time"
)
type ReconcileReport struct {
Metadata, Objects, Orphans, Missing, Recovered, Fresh, Deleted, Changed int
PipelineSkipped int
NextMetadata, NextObject string
}
type Reconciler struct {
repo repository.AttachmentRepository
objects port.Objects
now func() time.Time
}
func NewReconciler(repo repository.AttachmentRepository, objects port.Objects, now func() time.Time) *Reconciler {
return &Reconciler{repo, objects, now}
}
func (r *Reconciler) Run(ctx context.Context, after uuid.UUID, objectAfter string, grace time.Duration, apply bool) (report ReconcileReport, runErr error) {
report = ReconcileReport{}
defer func() {
if runErr != nil {
report.NextMetadata = ""
report.NextObject = ""
}
}()
if grace < time.Minute {
return report, &model.ValidationError{Message: "reconciliation grace must be at least 60 seconds"}
}
now := r.now().UTC()
old := func(t time.Time) bool { age := now.Sub(t); return !t.IsZero() && age >= grace && age >= 0 }
rows, more, e := r.repo.ReconcileAttachments(ctx, after)
if e != nil {
return report, e
}
for _, a := range rows {
report.Metadata++
if more {
report.NextMetadata = a.ID.String()
}
if !old(a.CreatedAt) {
report.Fresh++
continue
}
// Only the fenced pipeline can publish versioned results. The legacy
// reconciler knows nothing about required variants, retries or outbox jobs.
if a.PipelineVersion > 0 {
report.PipelineSkipped++
continue
}
info, e := r.objects.Stat(ctx, ObjectKey(a.ID))
target := a.Status
if errors.Is(e, model.ErrNotFound) {
switch a.Status {
case "ready":
report.Missing++
target = "missing"
case "uploading":
report.Missing++
target = "abandoned"
}
} else if e != nil {
return report, e
} else if a.Status != "ready" {
valid := false
if info.Size == a.SizeBytes {
body, err := r.objects.Get(ctx, ObjectKey(a.ID))
if err != nil {
return report, err
}
hash := sha256.New()
n, err := io.Copy(hash, io.LimitReader(body, a.SizeBytes+1))
_ = body.Close()
if err != nil {
return report, model.ErrUnavailable
}
valid = n == a.SizeBytes && hex.EncodeToString(hash.Sum(nil)) == a.SHA256
}
if valid {
report.Recovered++
target = "ready"
} else {
report.Missing++
target = "missing"
}
} else if info.Size != a.SizeBytes {
report.Missing++
target = "missing"
}
if apply && target != a.Status {
changed, e := r.repo.SetAttachmentStatus(ctx, a.ID, a.Status, target)
if e != nil {
return report, e
}
if changed {
report.Changed++
}
}
}
objects, more, e := r.objects.List(ctx, objectAfter)
if e != nil {
return report, e
}
for _, o := range objects {
report.Objects++
if more {
report.NextObject = o.Key
}
if !old(o.Modified) {
report.Fresh++
continue
}
raw, ok := strings.CutPrefix(o.Key, "originals/")
if !ok {
continue
}
id, e := uuid.Parse(raw)
if e != nil || ObjectKey(id) != o.Key {
continue
}
if _, e = r.repo.GetAttachment(ctx, id); e == nil {
continue
} else if !errors.Is(e, model.ErrNotFound) {
return report, e
}
report.Orphans++
if !apply {
continue
}
fresh, e := r.objects.Stat(ctx, o.Key)
if errors.Is(e, model.ErrNotFound) {
continue
}
if e != nil {
return report, e
}
if fresh.ETag != o.ETag || fresh.Size != o.Size || !fresh.Modified.Equal(o.Modified) || !old(fresh.Modified) {
report.Fresh++
continue
}
if _, e = r.repo.GetAttachment(ctx, id); e == nil {
continue
} else if !errors.Is(e, model.ErrNotFound) {
return report, e
}
// Object keys are random UUIDs, immutable and never reused by the upload API.
// An absent metadata row cannot later adopt an old inventory key.
if e = r.objects.Remove(ctx, o.Key); e != nil {
return report, e
}
report.Deleted++
}
return report, nil
}
@@ -0,0 +1,31 @@
package usecase
import (
"backend/internal/domain/model"
"backend/internal/domain/port"
"context"
"crypto/sha256"
"encoding/hex"
"github.com/google/uuid"
"testing"
"time"
)
func TestReconcilerCannotPublishVersionedPipelineResult(t *testing.T) {
for _, status := range []string{"uploading", "queued", "processing", "failed", "ready"} {
t.Run(status, func(t *testing.T) {
id := uuid.New()
raw := []byte("only original survived worker crash")
sum := sha256.Sum256(raw)
repo := &fixtureRepo{rows: map[uuid.UUID]model.Attachment{id: {ID: id, Status: status, PipelineVersion: 1, SizeBytes: int64(len(raw)), SHA256: hex.EncodeToString(sum[:]), CreatedAt: fixedNow.Add(-2 * time.Minute)}}}
objects := &fixtureObjects{items: map[string]port.Object{ObjectKey(id): {Key: ObjectKey(id), Size: int64(len(raw)), Modified: fixedNow.Add(-2 * time.Minute)}}, content: raw}
report, e := NewReconciler(repo, objects, func() time.Time { return fixedNow }).Run(context.Background(), uuid.Nil, "", time.Minute, true)
if e != nil {
t.Fatal(e)
}
if repo.rows[id].Status != status || report.Recovered != 0 || report.Changed != 0 {
t.Fatalf("pipeline state was changed by legacy reconciler: status %s -> %s, report %+v", status, repo.rows[id].Status, report)
}
})
}
}
@@ -0,0 +1,184 @@
package usecase
import (
"backend/internal/domain/model"
"backend/internal/domain/port"
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"github.com/google/uuid"
"io"
"testing"
"time"
)
type fixtureRepo struct {
rows map[uuid.UUID]model.Attachment
failure error
more bool
}
func (f *fixtureRepo) ReserveAttachment(context.Context, model.Attachment) (model.Attachment, error) {
panic("not used")
}
func (f *fixtureRepo) GetAttachment(_ context.Context, id uuid.UUID) (model.Attachment, error) {
if f.failure != nil {
return model.Attachment{}, f.failure
}
a, ok := f.rows[id]
if !ok {
return a, model.ErrNotFound
}
return a, nil
}
func (f *fixtureRepo) SetAttachmentStatus(_ context.Context, id uuid.UUID, from, to string) (bool, error) {
a := f.rows[id]
if a.Status != from {
return false, nil
}
a.Status = to
f.rows[id] = a
return true, nil
}
func (f *fixtureRepo) ReconcileAttachments(context.Context, uuid.UUID) ([]model.Attachment, bool, error) {
out := []model.Attachment{}
for _, a := range f.rows {
out = append(out, a)
}
return out, f.more, f.failure
}
type fixtureObjects struct {
items map[string]port.Object
content []byte
failure error
removed int
changed bool
}
func (f *fixtureObjects) Put(context.Context, string, io.Reader, int64, string) error {
panic("not used")
}
func (f *fixtureObjects) Get(context.Context, string) (io.ReadCloser, error) {
return io.NopCloser(bytes.NewReader(f.content)), f.failure
}
func (f *fixtureObjects) Stat(_ context.Context, key string) (port.Object, error) {
if f.failure != nil {
return port.Object{}, f.failure
}
o, ok := f.items[key]
if !ok {
return o, model.ErrNotFound
}
if f.changed {
o.ETag = "changed"
}
return o, nil
}
func (f *fixtureObjects) Remove(_ context.Context, key string) error {
delete(f.items, key)
f.removed++
return nil
}
func (f *fixtureObjects) List(context.Context, string) ([]port.Object, bool, error) {
out := []port.Object{}
for _, o := range f.items {
out = append(out, o)
}
return out, false, f.failure
}
var fixedNow = time.Date(2026, 11, 12, 12, 0, 0, 0, time.UTC)
func TestOrphanCleanupHonorsGraceAndDryRun(t *testing.T) {
for _, tc := range []struct {
name string
age time.Duration
apply, used, changed bool
wantDeleted int
}{
{name: "old dry run", age: 2 * time.Minute},
{name: "old apply", age: 2 * time.Minute, apply: true, wantDeleted: 1},
{name: "fresh protected", age: 10 * time.Second, apply: true},
{name: "future timestamp protected", age: -time.Minute, apply: true},
{name: "referenced protected", age: 2 * time.Minute, apply: true, used: true},
{name: "changed inventory protected", age: 2 * time.Minute, apply: true, changed: true},
} {
t.Run(tc.name, func(t *testing.T) {
id := uuid.MustParse("00000000-0000-0000-0000-000000000001")
key := ObjectKey(id)
repo := &fixtureRepo{rows: map[uuid.UUID]model.Attachment{}}
if tc.used {
repo.rows[id] = model.Attachment{ID: id, Status: "ready", CreatedAt: fixedNow}
}
objects := &fixtureObjects{items: map[string]port.Object{key: {Key: key, ETag: "original", Modified: fixedNow.Add(-tc.age)}}, changed: tc.changed}
report, e := NewReconciler(repo, objects, func() time.Time { return fixedNow }).Run(context.Background(), uuid.Nil, "", time.Minute, tc.apply)
if e != nil {
t.Fatal(e)
}
if objects.removed != tc.wantDeleted {
t.Fatalf("deleted %d, want %d; report=%+v", objects.removed, tc.wantDeleted, report)
}
if tc.wantDeleted > 0 {
_, e = NewReconciler(repo, objects, func() time.Time { return fixedNow }).Run(context.Background(), uuid.Nil, "", time.Minute, true)
if e != nil {
t.Fatal(e)
}
if objects.removed != 1 {
t.Fatal("second run deleted something twice")
}
}
})
}
}
func TestMissingAndPendingMetadataRepair(t *testing.T) {
for _, status := range []string{"ready", "uploading", "missing"} {
t.Run(status, func(t *testing.T) {
id := uuid.New()
raw := []byte("original bytes")
sum := sha256.Sum256(raw)
repo := &fixtureRepo{rows: map[uuid.UUID]model.Attachment{id: {ID: id, Status: status, CreatedAt: fixedNow.Add(-2 * time.Minute), UpdatedAt: fixedNow.Add(-2 * time.Minute), SizeBytes: int64(len(raw)), SHA256: hex.EncodeToString(sum[:])}}}
objects := &fixtureObjects{items: map[string]port.Object{}, content: raw}
want := "missing"
if status != "ready" {
objects.items[ObjectKey(id)] = port.Object{Key: ObjectKey(id), Size: int64(len(raw)), Modified: fixedNow.Add(-2 * time.Minute)}
want = "ready"
}
_, e := NewReconciler(repo, objects, func() time.Time { return fixedNow }).Run(context.Background(), uuid.Nil, "", time.Minute, true)
if e != nil {
t.Fatal(e)
}
if got := repo.rows[id].Status; got != want {
t.Fatalf("status %s, want %s", got, want)
}
if objects.removed != 0 {
t.Fatal("metadata repair removed original")
}
})
}
}
func TestDependencyFailureNeverDeletes(t *testing.T) {
repo := &fixtureRepo{rows: map[uuid.UUID]model.Attachment{}, failure: model.ErrUnavailable}
objects := &fixtureObjects{items: map[string]port.Object{}}
_, e := NewReconciler(repo, objects, func() time.Time { return fixedNow }).Run(context.Background(), uuid.Nil, "", time.Minute, true)
if !errors.Is(e, model.ErrUnavailable) {
t.Fatalf("expected dependency error, got %v", e)
}
if objects.removed != 0 {
t.Fatal("deleted during failed inventory")
}
}
func TestFailureDoesNotReturnCursorPastFailedRecord(t *testing.T) {
id := uuid.New()
repo := &fixtureRepo{rows: map[uuid.UUID]model.Attachment{id: {ID: id, Status: "ready", CreatedAt: fixedNow.Add(-2 * time.Minute)}}, more: true}
objects := &fixtureObjects{failure: model.ErrUnavailable}
report, err := NewReconciler(repo, objects, func() time.Time { return fixedNow }).Run(context.Background(), uuid.Nil, "", time.Minute, true)
if !errors.Is(err, model.ErrUnavailable) {
t.Fatalf("expected failure, got %v", err)
}
if report.NextMetadata != "" || report.NextObject != "" {
t.Fatal("failed inventory must restart rather than skip the failed item")
}
}
+86
View File
@@ -0,0 +1,86 @@
package config
import (
"fmt"
"os"
"strings"
"time"
"github.com/spf13/viper"
)
// Config приходит только из окружения (12-factor): в контейнер не копируются
// ни .env, ни файлы конфигурации. Значения по умолчанию — ниже.
type Config struct {
Host string `mapstructure:"APP_HOST"`
Port int `mapstructure:"APP_PORT"`
Env string `mapstructure:"APP_ENV"`
LogLevel string `mapstructure:"LOG_LEVEL"`
ShutdownTimeout time.Duration `mapstructure:"SHUTDOWN_TIMEOUT"`
MaxBodyBytes int64 `mapstructure:"MAX_BODY_BYTES"`
MaxInflight int `mapstructure:"MAX_INFLIGHT"`
MaxQueue int `mapstructure:"MAX_QUEUE"`
QueueTimeout time.Duration `mapstructure:"QUEUE_TIMEOUT"`
DatabaseURLFile string `mapstructure:"DATABASE_URL_FILE"`
CacheURLFile string `mapstructure:"CACHE_URL_FILE"`
SessionTTLSeconds int `mapstructure:"SESSION_TTL_SECONDS"`
MaxConns int32 `mapstructure:"PG_MAX_CONNS"`
}
var defaults = map[string]any{
"APP_HOST": "0.0.0.0",
"APP_PORT": 8080,
"APP_ENV": "lab",
"LOG_LEVEL": "info",
"SHUTDOWN_TIMEOUT": "3s",
"MAX_BODY_BYTES": 64 << 10,
"MAX_INFLIGHT": 4,
"MAX_QUEUE": 32,
"QUEUE_TIMEOUT": "1s",
"DATABASE_URL_FILE": "/run/secrets/database_url",
"CACHE_URL_FILE": "/run/secrets/cache_url",
"SESSION_TTL_SECONDS": 3600,
"PG_MAX_CONNS": 8,
}
func Load() (Config, error) {
v := viper.New()
for key, value := range defaults {
v.SetDefault(key, value)
}
v.AutomaticEnv()
var cfg Config
if err := v.Unmarshal(&cfg); err != nil {
return Config{}, fmt.Errorf("parse config: %w", err)
}
if cfg.ShutdownTimeout <= 0 || cfg.ShutdownTimeout > 4*time.Second {
return Config{}, fmt.Errorf("SHUTDOWN_TIMEOUT must be in (0,4s]: reserve exporter teardown within8s stop")
}
if cfg.SessionTTLSeconds < 1 || cfg.SessionTTLSeconds > 86400 {
return Config{}, fmt.Errorf("SESSION_TTL_SECONDS must be 1..86400")
}
if cfg.MaxConns < 1 || cfg.MaxConns > 64 {
return Config{}, fmt.Errorf("PG_MAX_CONNS must be 1..64")
}
if cfg.MaxInflight < 1 || cfg.MaxInflight > 4 || cfg.MaxQueue < 0 || cfg.MaxQueue > 128 || cfg.QueueTimeout <= 0 || cfg.QueueTimeout > 2*time.Second {
return Config{}, fmt.Errorf("HTTP gate exceeds the tested resource budget")
}
if cfg.MaxBodyBytes < 1 || cfg.MaxBodyBytes > 65536 {
return Config{}, fmt.Errorf("MAX_BODY_BYTES must be 1..65536")
}
return cfg, nil
}
func (c Config) Addr() string { return fmt.Sprintf("%s:%d", c.Host, c.Port) }
func ReadSecret(path string) (string, error) {
b, e := os.ReadFile(path)
if e != nil {
return "", fmt.Errorf("required secret file unavailable")
}
v := strings.TrimSpace(string(b))
if v == "" {
return "", fmt.Errorf("required secret is empty")
}
return v, nil
}
+39
View File
@@ -0,0 +1,39 @@
package config
import (
"testing"
"time"
)
func TestRejectsConfigurationOutsideResourceBudget(t *testing.T) {
for _, tc := range []struct{ key, value string }{{"MAX_INFLIGHT", "0"}, {"MAX_INFLIGHT", "5"}, {"MAX_QUEUE", "-1"}, {"MAX_QUEUE", "129"}, {"QUEUE_TIMEOUT", "0s"}, {"QUEUE_TIMEOUT", "3s"}, {"MAX_BODY_BYTES", "0"}, {"MAX_BODY_BYTES", "65537"}, {"PG_MAX_CONNS", "0"}, {"PG_MAX_CONNS", "65"}, {"SESSION_TTL_SECONDS", "0"}, {"SESSION_TTL_SECONDS", "86401"}} {
t.Run(tc.key+tc.value, func(t *testing.T) {
t.Setenv(tc.key, tc.value)
if _, e := Load(); e == nil {
t.Fatal("accepted configuration beyond tested bounds")
}
})
}
}
func TestPoolSizeIsConfigurableWithinBounds(t *testing.T) {
t.Setenv("PG_MAX_CONNS", "1")
cfg, e := Load()
if e != nil {
t.Fatal(e)
}
if cfg.MaxConns != 1 {
t.Fatal("configured pool size ignored")
}
}
func TestTelemetryDrainLeavesRoomForExporterWithinStopDeadline(t *testing.T) {
t.Setenv("SHUTDOWN_TIMEOUT", "5s")
if _, e := Load(); e == nil {
t.Fatal("HTTP drain leaves no safe reserve for exporter/pool teardown before8s stop")
}
t.Setenv("SHUTDOWN_TIMEOUT", "3s")
cfg, e := Load()
if e != nil || cfg.ShutdownTimeout != 3*time.Second {
t.Fatal("bounded drain rejected")
}
}
@@ -0,0 +1,35 @@
package model
import (
"github.com/google/uuid"
"strings"
"time"
"unicode"
"unicode/utf8"
)
const MaxImageBytes int64 = 10 * 1024 * 1024
const MaxImagePixels int64 = 40_000_000
type Attachment struct {
ID, ChatID, OwnerID uuid.UUID
Filename, ContentType, SHA256, Status string
SizeBytes int64
CreatedAt, UpdatedAt time.Time
PipelineVersion int32
ErrorCode string
Variants []Variant
StagingExpired, StagingCleaned bool
}
func ValidateFilename(name string) error {
if !utf8.ValidString(name) || utf8.RuneCountInString(name) < 1 || utf8.RuneCountInString(name) > 255 || strings.TrimSpace(name) == "" {
return invalidf("filename: 1–255 characters required")
}
for _, r := range name {
if r == '/' || r == '\\' || unicode.IsControl(r) {
return invalidf("filename: invalid character")
}
}
return nil
}
+52
View File
@@ -0,0 +1,52 @@
package model
import (
"slices"
"time"
"github.com/google/uuid"
)
const maxChatMembers = 100
// Chat неизменяем после создания: участники задаются один раз, удаления в API нет.
type Chat struct {
ID uuid.UUID
Title string
MemberIDs []uuid.UUID
CreatedAt time.Time
}
func (c Chat) HasMember(userID uuid.UUID) bool {
return slices.Contains(c.MemberIDs, userID)
}
// ChatDraft — проверенные данные нового чата. Существование участников
// проверяет хранилище атомарно с созданием.
type ChatDraft struct {
Creator uuid.UUID
Title string
MemberIDs []uuid.UUID
}
func NewChatDraft(creator uuid.UUID, title string, memberIDs []uuid.UUID) (ChatDraft, error) {
if err := requireText("title", title, 1, 100); err != nil {
return ChatDraft{}, err
}
if len(memberIDs) == 0 || len(memberIDs) > maxChatMembers {
return ChatDraft{}, invalidf("member_ids: от 1 до %d участников", maxChatMembers)
}
seen := make(map[uuid.UUID]struct{}, len(memberIDs))
for _, id := range memberIDs {
if _, dup := seen[id]; dup {
return ChatDraft{}, invalidf("member_ids: повторяющийся участник")
}
seen[id] = struct{}{}
}
if _, ok := seen[creator]; !ok {
return ChatDraft{}, invalidf("member_ids: должен содержать создателя чата")
}
return ChatDraft{Creator: creator, Title: title, MemberIDs: slices.Clone(memberIDs)}, nil
}
func (c Chat) Position() Position { return Position{CreatedAt: c.CreatedAt, ID: c.ID} }
+33
View File
@@ -0,0 +1,33 @@
package model
import (
"errors"
"fmt"
)
var (
ErrUnavailable = errors.New("dependency unavailable")
ErrInvalidImage = errors.New("invalid staged image")
ErrConflict = errors.New("conflict")
ErrNotFound = errors.New("not found")
ErrUnauthorized = errors.New("unauthorized")
ErrUsernameTaken = errors.New("username taken")
// ErrStorageFull — исчерпан бюджет памяти хранилища. Подтверждённые данные
// не вытесняются, поэтому новые записи отклоняются, а чтение продолжает работать.
ErrStorageFull = errors.New("storage full")
// ErrUnknownMember возвращает хранилище, если среди участников нового чата
// есть несуществующий пользователь; для клиента это ошибка ввода (400).
ErrUnknownMember error = &ValidationError{Message: "member_ids: неизвестный пользователь"}
)
// ValidationError — нарушение правил входных данных; Message безопасно
// показывать клиенту.
type ValidationError struct {
Message string
}
func (e *ValidationError) Error() string { return e.Message }
func invalidf(format string, args ...any) error {
return &ValidationError{Message: fmt.Sprintf(format, args...)}
}
+31
View File
@@ -0,0 +1,31 @@
package model
import (
"github.com/google/uuid"
"time"
)
type Variant struct {
Name string `json:"name"`
ContentType string `json:"content_type"`
SHA256 string `json:"sha256"`
SizeBytes int64 `json:"size_bytes"`
Width int `json:"width"`
Height int `json:"height"`
}
type Job struct {
ID uuid.UUID `json:"job_id"`
AttachmentID uuid.UUID `json:"attachment_id"`
PipelineVersion int32 `json:"pipeline_version"`
Attempt int32 `json:"attempt"`
CreatedAt time.Time `json:"created_at"`
Traceparent string `json:"traceparent,omitempty"`
Tracestate string `json:"tracestate,omitempty"`
}
type PipelineMetrics struct {
Depth int64
Oldest float64
ReadyMatured [3]int64
ReadyTimely [3]int64
}
+82
View File
@@ -0,0 +1,82 @@
package model
import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"regexp"
"slices"
"strings"
"time"
"unicode/utf8"
"github.com/google/uuid"
)
const MaxMessageRunes = 4000
type Message struct {
ID uuid.UUID
ChatID uuid.UUID
SenderID uuid.UUID
Text string
CreatedAt time.Time
AttachmentIDs []uuid.UUID
}
// MessageDraft — проверенное сообщение; отправитель берётся из идентификации,
// а не из тела запроса.
type MessageDraft struct {
ChatID uuid.UUID
SenderID uuid.UUID
Text string
Key string
PayloadHash string
AttachmentIDs []uuid.UUID
}
var keyPattern = regexp.MustCompile(`^[A-Za-z0-9._:-]{1,128}$`)
func WithKey(d MessageDraft, key string) (MessageDraft, error) {
if !keyPattern.MatchString(key) {
return MessageDraft{}, invalidf("Idempotency-Key: 1–128 ASCII characters required")
}
canonical, _ := json.Marshal(struct {
Text string
Attachments []uuid.UUID
}{d.Text, d.AttachmentIDs})
sum := sha256.Sum256(canonical)
d.Key = key
d.PayloadHash = hex.EncodeToString(sum[:])
return d, nil
}
func NewMessageDraft(chatID, senderID uuid.UUID, text string, attachments ...[]uuid.UUID) (MessageDraft, error) {
ids := []uuid.UUID{}
if len(attachments) > 0 {
ids = slices.Clone(attachments[0])
if ids == nil {
ids = []uuid.UUID{}
}
}
if len(ids) > 10 {
return MessageDraft{}, invalidf("attachment_ids: at most 10")
}
seen := map[uuid.UUID]bool{}
for _, id := range ids {
if seen[id] {
return MessageDraft{}, invalidf("attachment_ids: duplicate")
}
seen[id] = true
}
if len(ids) == 0 || strings.TrimSpace(text) != "" {
if err := requireText("text", text, 1, MaxMessageRunes); err != nil {
return MessageDraft{}, err
}
} else if !utf8.ValidString(text) || utf8.RuneCountInString(text) > MaxMessageRunes || strings.ContainsRune(text, 0) {
return MessageDraft{}, invalidf("text: invalid optional text")
}
return MessageDraft{ChatID: chatID, SenderID: senderID, Text: text, AttachmentIDs: ids}, nil
}
func (m Message) Position() Position { return Position{CreatedAt: m.CreatedAt, ID: m.ID} }
+126
View File
@@ -0,0 +1,126 @@
package model
import (
"errors"
"strings"
"testing"
"time"
"github.com/google/uuid"
)
func isValidationError(err error) bool {
var v *ValidationError
return errors.As(err, &v)
}
func TestNewUserDraft(t *testing.T) {
t.Parallel()
cases := []struct {
name, username, display string
ok bool
}{
{"minimal", "abc", "A", true},
{"max username 32", strings.Repeat("a", 32), "A", true},
{"username too short", "ab", "A", false},
{"username too long", strings.Repeat("a", 33), "A", false},
{"uppercase rejected", "Alice", "A", false},
{"cyrillic username rejected", "алиса", "A", false},
{"display 100 emoji ok", "abc", strings.Repeat("😀", 100), true},
{"display 101 code points rejected", "abc", strings.Repeat("ж", 101), false},
{"display whitespace only", "abc", " \t\n", false},
{"display empty", "abc", "", false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, err := NewUserDraft(tc.username, tc.display)
if tc.ok && err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !tc.ok && !isValidationError(err) {
t.Fatalf("want ValidationError, got %v", err)
}
})
}
}
func TestNewChatDraft(t *testing.T) {
t.Parallel()
me, other := uuid.New(), uuid.New()
tooMany := []uuid.UUID{me}
for range 100 {
tooMany = append(tooMany, uuid.New())
}
cases := []struct {
name string
members []uuid.UUID
ok bool
}{
{"creator only", []uuid.UUID{me}, true},
{"two members", []uuid.UUID{me, other}, true},
{"empty", nil, false},
{"creator missing", []uuid.UUID{other}, false},
{"duplicate member", []uuid.UUID{me, other, other}, false},
{"over 100 members", tooMany, false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, err := NewChatDraft(me, "Чат", tc.members)
if tc.ok != (err == nil) {
t.Fatalf("ok=%v, err=%v", tc.ok, err)
}
})
}
}
func TestNewMessageDraftLimitsInCodePoints(t *testing.T) {
t.Parallel()
if _, err := NewMessageDraft(uuid.New(), uuid.New(), strings.Repeat("👋", MaxMessageRunes)); err != nil {
t.Fatalf("4000 four-byte code points must pass: %v", err)
}
if _, err := NewMessageDraft(uuid.New(), uuid.New(), strings.Repeat("a", MaxMessageRunes+1)); !isValidationError(err) {
t.Fatalf("4001 code points must be rejected, got %v", err)
}
}
func TestCursorRoundTripAndScope(t *testing.T) {
t.Parallel()
pos := Position{CreatedAt: time.Date(2026, 9, 1, 12, 0, 0, 123456000, time.UTC), ID: uuid.New()}
cursor := EncodeCursor("messages:a", pos)
q, err := NewPageQuery(nil, &cursor, "messages:a")
if err != nil || q.After == nil || *q.After != pos || q.Limit != DefaultPageLimit {
t.Fatalf("round trip failed: %+v, %v", q, err)
}
if _, err := NewPageQuery(nil, &cursor, "messages:b"); !isValidationError(err) {
t.Fatalf("cursor of another resource must be rejected, got %v", err)
}
for _, bad := range []string{"", "!!!", EncodeCursor("x", pos)[:5]} {
if _, err := NewPageQuery(nil, &bad, "messages:a"); !isValidationError(err) {
t.Fatalf("garbage cursor %q must be rejected, got %v", bad, err)
}
}
}
func TestPageLimitBounds(t *testing.T) {
t.Parallel()
for limit, ok := range map[int]bool{0: false, 1: true, 100: true, 101: false, -1: false} {
_, err := NewPageQuery(&limit, nil, "chats")
if ok != (err == nil) {
t.Errorf("limit=%d: ok=%v, err=%v", limit, ok, err)
}
}
}
func TestPositionOrderTiesBrokenByID(t *testing.T) {
t.Parallel()
at := time.Now()
low := Position{CreatedAt: at, ID: uuid.MustParse("00000000-0000-4000-8000-000000000001")}
high := Position{CreatedAt: at, ID: uuid.MustParse("ffffffff-0000-4000-8000-000000000000")}
later := Position{CreatedAt: at.Add(time.Microsecond), ID: low.ID}
if !high.After(low) || low.After(high) || !later.After(high) || low.After(low) {
t.Fatal("order must be (created_at, id) ascending")
}
}
+84
View File
@@ -0,0 +1,84 @@
package model
import (
"bytes"
"encoding/base64"
"strconv"
"strings"
"time"
"github.com/google/uuid"
)
const (
DefaultPageLimit = 50
MaxPageLimit = 100
)
// Position — ключ keyset-пагинации: история упорядочена по (created_at, id).
type Position struct {
CreatedAt time.Time
ID uuid.UUID
}
// After сообщает, стоит ли p строго после other в порядке (created_at, id).
func (p Position) After(other Position) bool {
if !p.CreatedAt.Equal(other.CreatedAt) {
return p.CreatedAt.After(other.CreatedAt)
}
return bytes.Compare(p.ID[:], other.ID[:]) > 0
}
// PageQuery — проверенный запрос страницы: limit 1–100 и позиция после
// последнего элемента предыдущей страницы (nil — с начала).
type PageQuery struct {
Limit int
After *Position
}
// NewPageQuery проверяет limit и курсор. scope привязывает курсор к ресурсу:
// курсор истории одного чата не подходит к другому чату или к списку чатов.
func NewPageQuery(limit *int, cursor *string, scope string) (PageQuery, error) {
q := PageQuery{Limit: DefaultPageLimit}
if limit != nil {
if *limit < 1 || *limit > MaxPageLimit {
return PageQuery{}, invalidf("limit: от 1 до %d", MaxPageLimit)
}
q.Limit = *limit
}
if cursor != nil {
pos, err := decodeCursor(*cursor, scope)
if err != nil {
return PageQuery{}, err
}
q.After = &pos
}
return q, nil
}
// EncodeCursor делает непрозрачную для клиента строку из позиции и scope.
func EncodeCursor(scope string, p Position) string {
raw := scope + "|" + strconv.FormatInt(p.CreatedAt.UnixMicro(), 10) + "|" + p.ID.String()
return base64.RawURLEncoding.EncodeToString([]byte(raw))
}
func decodeCursor(cursor, scope string) (Position, error) {
errBad := invalidf("cursor: некорректное значение")
raw, err := base64.RawURLEncoding.DecodeString(cursor)
if err != nil {
return Position{}, errBad
}
parts := strings.Split(string(raw), "|")
if len(parts) != 3 || parts[0] != scope {
return Position{}, errBad
}
micros, err := strconv.ParseInt(parts[1], 10, 64)
if err != nil {
return Position{}, errBad
}
id, err := uuid.Parse(parts[2])
if err != nil {
return Position{}, errBad
}
return Position{CreatedAt: time.UnixMicro(micros).UTC(), ID: id}, nil
}
@@ -0,0 +1,13 @@
package model
import "testing"
func TestRejectsNULInPersistedText(t *testing.T) {
for _, field := range []string{"display_name", "title", "text"} {
t.Run(field, func(t *testing.T) {
if err := requireText(field, "a\x00b", 1, 4000); err == nil {
t.Fatal("PostgreSQL text must reject NUL as a validation error")
}
})
}
}

Some files were not shown because too many files have changed in this diff Show More