test(storage): record verified faults and isolate inventory cost
This commit is contained in:
@@ -6,8 +6,9 @@ keyset pagination, общий Idempotency-Key, HAProxy и две независ
|
||||
Добавлены приватный S3, вложения и повторяемый reconciler.
|
||||
|
||||
Публикация и окончательные результаты будут внесены после проверок.
|
||||
Сейчас завершены реализация и статические проверки; runtime-доказательства
|
||||
ещё собираются. Этот промежуточный статус не является заявлением PASS.
|
||||
Публичный smoke и 108 собственных HTTP-проверок прошли на полном стенде
|
||||
(362.40 s). Проверки отказов/исследование и clean clone ещё выполняются;
|
||||
окончательная готовность будет указана после их завершения.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
@@ -18,7 +19,8 @@ MinIO с точным module version скачивается восемью по
|
||||
с ограниченным timeout/числом повторов; затем стандартный `go mod download`
|
||||
проверяет h1 через Go checksum database перед сборкой. Проверки TLS и
|
||||
checksum не отключены. Go parallelism
|
||||
ограничен двумя потоками, зависимости и компилятор используют BuildKit cache.
|
||||
задан `GOMAXPROCS=2` и `GOFLAGS=-p=2`; зависимости и компилятор
|
||||
используют BuildKit cache.
|
||||
|
||||
```sh
|
||||
uv sync --locked
|
||||
@@ -103,8 +105,15 @@ endpoint `s3:9000`, region us-east-1, bucket messenger, byte/pixel limits и
|
||||
Readiness проверяет PG и S3. При потере S3 HAProxy может исключить все API и
|
||||
вернуть собственный 503; прямой внутренний API возвращает JSON unavailable.
|
||||
|
||||
Таблица наблюдений и ссылки на evidence будут заполнены после выполнения
|
||||
storage, cluster, HTTP и clean-clone проверок.
|
||||
| Проверка | Команда | Наблюдение | Доказательство |
|
||||
| --- | --- | --- | --- |
|
||||
| SQL migrations | `docker compose run --rm --no-deps migrate`, повтор | 3 migrations, checksum match, repeat idempotent | [migrations-first.json](evidence/migrations-first.json) |
|
||||
| Публичный API | `make test` | PASS: upload201, Message, original SHA-256 и ACL | [http-first.json](evidence/http-first.json) |
|
||||
| Прежние возможности и негативные файлы | `uv run pytest student-tests -q` | 108 passed, 362.40 s; в том числе chunked >10 MiB, corrupt/MIME/pixels/filename/foreign chat | [http-first.json](evidence/http-first.json) |
|
||||
| Закреплённый S3 | `go mod download`, `minio --version`, inspect | module h1 verified; binary 2025-10-15/go1.24.8/arm64; 2 API, реальные CPU/RAM/ports | [minio-source-verification.json](evidence/minio-source-verification.json) |
|
||||
|
||||
Строки storage/cluster/research/clean clone будут добавлены после выполнения
|
||||
этих проверок. Source HTTP = `f5a7a74`, OCI revision API совпадает с ним.
|
||||
|
||||
## Методика исследования
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"date": "2026-11-15T18:04:52.576254+00:00",
|
||||
"source_commit": "f5a7a74f679d936d851fa0e58f6cfb59dcd62c50",
|
||||
"result": "PASS",
|
||||
"smoke": {
|
||||
"command": "make test PYTHON=.venv/bin/python BASE_URL=http://127.0.0.1:8084",
|
||||
"seconds": 8.702,
|
||||
"exit_code": 0,
|
||||
"output": ".venv/bin/python tests/smoke.py --base-url \"http://127.0.0.1:8084\" \nPASS: lab4 public API scenario; infrastructure and grading evidence remain separate.\n"
|
||||
},
|
||||
"http": {
|
||||
"command": ".venv/bin/pytest student-tests -q",
|
||||
"seconds": 362.959,
|
||||
"exit_code": 0,
|
||||
"output": "........................................................................ [ 66%]\n.................................... [100%]\n108 passed in 362.40s (0:06:02)\n"
|
||||
}
|
||||
}
|
||||
@@ -8,5 +8,85 @@
|
||||
"module_checksum": "h1:6TdolSCLSs2nwm8i0PpWDqf9iX2Ty9WQK8wmr7dCnUM=",
|
||||
"go_mod_checksum": "h1:yCWDkwWO9IWpGsT4mreDDN/B/QVmK2zC666uInRAcqE=",
|
||||
"validation": "standard go mod download with default Go checksum database after ranged HTTPS download",
|
||||
"compiler_build_status": "in progress; source validation only"
|
||||
"compiler_build_status": "PASS",
|
||||
"runtime_version": "minio version RELEASE.2025-10-15T17-29-55Z (commit-id=9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a)\nRuntime: go1.24.8 linux/arm64\nLicense: GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html\nCopyright: 2015-0000 MinIO, Inc.",
|
||||
"runtime_snapshot": [
|
||||
{
|
||||
"role": "api",
|
||||
"container": "c9c495521bf2",
|
||||
"pid": 39030,
|
||||
"started_at": "2026-11-15T18:03:15.404279669Z",
|
||||
"ports": {},
|
||||
"cpu_nano": 500000000,
|
||||
"memory_limit": 805306368,
|
||||
"instance": "e91b8ff7122d4842abb3aa0ff38e0352"
|
||||
},
|
||||
{
|
||||
"role": "api",
|
||||
"container": "2af84f3c87dd",
|
||||
"pid": 39163,
|
||||
"started_at": "2026-11-15T18:03:15.735686586Z",
|
||||
"ports": {},
|
||||
"cpu_nano": 500000000,
|
||||
"memory_limit": 805306368,
|
||||
"instance": "17cd27ba20064c5eb9580379e963f419"
|
||||
},
|
||||
{
|
||||
"role": "cache",
|
||||
"container": "e33ddf35f293",
|
||||
"pid": 5356,
|
||||
"started_at": "2026-11-15T17:39:31.281698344Z",
|
||||
"ports": {},
|
||||
"cpu_nano": 250000000,
|
||||
"memory_limit": 134217728
|
||||
},
|
||||
{
|
||||
"role": "lb",
|
||||
"container": "6d7d02d323ed",
|
||||
"pid": 39796,
|
||||
"started_at": "2026-11-15T18:03:26.954206841Z",
|
||||
"ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8084"
|
||||
}
|
||||
]
|
||||
},
|
||||
"cpu_nano": 500000000,
|
||||
"memory_limit": 134217728
|
||||
},
|
||||
{
|
||||
"role": "postgres",
|
||||
"container": "a5ffac5a4228",
|
||||
"pid": 5418,
|
||||
"started_at": "2026-11-15T17:39:31.614022427Z",
|
||||
"ports": {},
|
||||
"cpu_nano": 1000000000,
|
||||
"memory_limit": 536870912
|
||||
},
|
||||
{
|
||||
"role": "reconciler",
|
||||
"container": "c94b86137075",
|
||||
"pid": 39092,
|
||||
"started_at": "2026-11-15T18:03:15.578851294Z",
|
||||
"ports": {},
|
||||
"cpu_nano": 250000000,
|
||||
"memory_limit": 268435456
|
||||
},
|
||||
{
|
||||
"role": "s3",
|
||||
"container": "acc08930aff2",
|
||||
"pid": 38412,
|
||||
"started_at": "2026-11-15T18:03:03.039434803Z",
|
||||
"ports": {},
|
||||
"cpu_nano": 1000000000,
|
||||
"memory_limit": 1073741824
|
||||
}
|
||||
],
|
||||
"runtime_image_revision": {
|
||||
"api": "f5a7a74f679d936d851fa0e58f6cfb59dcd62c50",
|
||||
"s3": "9e49d5e7a648f00e26f2246f4dc28e6b07f8c84a",
|
||||
"bootstrap": "7394ce0dd2a80935aded936b09fa12cbb3cb8096"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,502 @@
|
||||
{
|
||||
"source_commit": "f5a7a74f679d936d851fa0e58f6cfb59dcd62c50",
|
||||
"date": "2026-11-15T18:13:30.360880+00:00",
|
||||
"complete": true,
|
||||
"runtime": [
|
||||
{
|
||||
"role": "api",
|
||||
"container": "c9c495521bf2",
|
||||
"pid": 39030,
|
||||
"started_at": "2026-11-15T18:03:15.404279669Z",
|
||||
"ports": {},
|
||||
"cpu_nano": 500000000,
|
||||
"memory_limit": 805306368,
|
||||
"instance": "e91b8ff7122d4842abb3aa0ff38e0352"
|
||||
},
|
||||
{
|
||||
"role": "api",
|
||||
"container": "2af84f3c87dd",
|
||||
"pid": 39163,
|
||||
"started_at": "2026-11-15T18:03:15.735686586Z",
|
||||
"ports": {},
|
||||
"cpu_nano": 500000000,
|
||||
"memory_limit": 805306368,
|
||||
"instance": "17cd27ba20064c5eb9580379e963f419"
|
||||
},
|
||||
{
|
||||
"role": "cache",
|
||||
"container": "e33ddf35f293",
|
||||
"pid": 5356,
|
||||
"started_at": "2026-11-15T17:39:31.281698344Z",
|
||||
"ports": {},
|
||||
"cpu_nano": 250000000,
|
||||
"memory_limit": 134217728
|
||||
},
|
||||
{
|
||||
"role": "lb",
|
||||
"container": "6d7d02d323ed",
|
||||
"pid": 39796,
|
||||
"started_at": "2026-11-15T18:03:26.954206841Z",
|
||||
"ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8084"
|
||||
}
|
||||
]
|
||||
},
|
||||
"cpu_nano": 500000000,
|
||||
"memory_limit": 134217728
|
||||
},
|
||||
{
|
||||
"role": "postgres",
|
||||
"container": "a5ffac5a4228",
|
||||
"pid": 5418,
|
||||
"started_at": "2026-11-15T17:39:31.614022427Z",
|
||||
"ports": {},
|
||||
"cpu_nano": 1000000000,
|
||||
"memory_limit": 536870912
|
||||
},
|
||||
{
|
||||
"role": "reconciler",
|
||||
"container": "c94b86137075",
|
||||
"pid": 39092,
|
||||
"started_at": "2026-11-15T18:03:15.578851294Z",
|
||||
"ports": {},
|
||||
"cpu_nano": 250000000,
|
||||
"memory_limit": 268435456
|
||||
},
|
||||
{
|
||||
"role": "s3",
|
||||
"container": "acc08930aff2",
|
||||
"pid": 38412,
|
||||
"started_at": "2026-11-15T18:03:03.039434803Z",
|
||||
"ports": {},
|
||||
"cpu_nano": 1000000000,
|
||||
"memory_limit": 1073741824
|
||||
}
|
||||
],
|
||||
"upload_replica_stopped_downloaded_elsewhere": {
|
||||
"stopped_container": "2af84f3c87dd",
|
||||
"uploader": "17cd27ba20064c5eb9580379e963f419",
|
||||
"downloader": "e91b8ff7122d4842abb3aa0ff38e0352",
|
||||
"sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75"
|
||||
},
|
||||
"private_bucket_and_iam": {
|
||||
"denied_head_is_not_missing": 503,
|
||||
"anonymous_get": 403,
|
||||
"list_other_bucket": "AccessDenied",
|
||||
"create_bucket": "AccessDenied",
|
||||
"write_outside_prefix": "AccessDenied",
|
||||
"console_internal_network": "unreachable"
|
||||
},
|
||||
"bootstrap_repeat_preserved_original": true,
|
||||
"aborted_upload_has_no_intent_or_bytes": true,
|
||||
"pg_intent_failure_has_no_bytes": true,
|
||||
"s3_unavailable": {
|
||||
"status": 503,
|
||||
"error": "unavailable"
|
||||
},
|
||||
"concurrent_uploads": {
|
||||
"concurrency": 10,
|
||||
"requests": 10,
|
||||
"successes": 10,
|
||||
"errors": 0,
|
||||
"bytes_per_image": 6017095,
|
||||
"pixels": 2890000,
|
||||
"duration_seconds": 5.149,
|
||||
"successful_rps": 1.942,
|
||||
"latency_scope": "upload only; duration includes verified downloads and memory sampler",
|
||||
"p50_ms": 2162.965,
|
||||
"p95_ms": 4465.512,
|
||||
"p99_ms": 4465.512,
|
||||
"cpu_before": {
|
||||
"c9c495521bf2e0eb21d94eebc8ac06ea3735ee282401f8bbf4918203b05e69ea": "usage_usec 148242470\nuser_usec 139413925\nsystem_usec 8828544\nnice_usec 0\nnr_periods 5268\nnr_throttled 2720\nthrottled_usec 566509749\nnr_bursts 0\nburst_usec 0",
|
||||
"2af84f3c87dd51105775135adaa2864b852c68c3b3d929f566d714ee4bbf13a2": "usage_usec 7938759\nuser_usec 7311622\nsystem_usec 627137\nnice_usec 0\nnr_periods 338\nnr_throttled 144\nthrottled_usec 17342812\nnr_bursts 0\nburst_usec 0"
|
||||
},
|
||||
"cpu_after": {
|
||||
"c9c495521bf2e0eb21d94eebc8ac06ea3735ee282401f8bbf4918203b05e69ea": "usage_usec 150694911\nuser_usec 141253264\nsystem_usec 9441647\nnice_usec 0\nnr_periods 5320\nnr_throttled 2767\nthrottled_usec 573793811\nnr_bursts 0\nburst_usec 0",
|
||||
"2af84f3c87dd51105775135adaa2864b852c68c3b3d929f566d714ee4bbf13a2": "usage_usec 9567439\nuser_usec 8568377\nsystem_usec 999061\nnice_usec 0\nnr_periods 386\nnr_throttled 171\nthrottled_usec 20452342\nnr_bursts 0\nburst_usec 0"
|
||||
},
|
||||
"memory_samples": [
|
||||
{
|
||||
"at": 0.0,
|
||||
"container": "c9c495521bf2",
|
||||
"memory.current": 128004096,
|
||||
"memory.peak": 215093248
|
||||
},
|
||||
{
|
||||
"at": 0.931,
|
||||
"container": "2af84f3c87dd",
|
||||
"memory.current": 128983040,
|
||||
"memory.peak": 152731648
|
||||
},
|
||||
{
|
||||
"at": 2.318,
|
||||
"container": "c9c495521bf2",
|
||||
"memory.current": 163500032,
|
||||
"memory.peak": 215093248
|
||||
},
|
||||
{
|
||||
"at": 3.846,
|
||||
"container": "2af84f3c87dd",
|
||||
"memory.current": 126009344,
|
||||
"memory.peak": 152731648
|
||||
},
|
||||
{
|
||||
"at": 4.515,
|
||||
"container": "c9c495521bf2",
|
||||
"memory.current": 153980928,
|
||||
"memory.peak": 215093248
|
||||
},
|
||||
{
|
||||
"at": 4.862,
|
||||
"container": "2af84f3c87dd",
|
||||
"memory.current": 125583360,
|
||||
"memory.peak": 152731648
|
||||
}
|
||||
],
|
||||
"results": [
|
||||
{
|
||||
"id": "7df54468-dffb-48e1-b435-bb64e73b087c",
|
||||
"latency_ms": 774.2156249878462,
|
||||
"instance": "ba0474b668fd43b294ece08957bf0648"
|
||||
},
|
||||
{
|
||||
"id": "e1ba3763-89f2-497f-a152-6008f4737e08",
|
||||
"latency_ms": 1630.1402079989202,
|
||||
"instance": "e91b8ff7122d4842abb3aa0ff38e0352"
|
||||
},
|
||||
{
|
||||
"id": "69c854d8-ca53-46aa-9dbd-646b68bc42be",
|
||||
"latency_ms": 3091.3296249927953,
|
||||
"instance": "e91b8ff7122d4842abb3aa0ff38e0352"
|
||||
},
|
||||
{
|
||||
"id": "03328959-bf30-4e57-a233-4084e599bbfc",
|
||||
"latency_ms": 2098.3562500041444,
|
||||
"instance": "e91b8ff7122d4842abb3aa0ff38e0352"
|
||||
},
|
||||
{
|
||||
"id": "483eff54-3137-4764-bc89-3feccf7d00e6",
|
||||
"latency_ms": 4303.714374982519,
|
||||
"instance": "e91b8ff7122d4842abb3aa0ff38e0352"
|
||||
},
|
||||
{
|
||||
"id": "f4eb43f0-65bf-462c-bfd2-9c8d88ff1d34",
|
||||
"latency_ms": 1105.6364579999354,
|
||||
"instance": "e91b8ff7122d4842abb3aa0ff38e0352"
|
||||
},
|
||||
{
|
||||
"id": "90a80cf9-d955-4bcc-9f16-01587a3f2f66",
|
||||
"latency_ms": 3825.5119579844177,
|
||||
"instance": "e91b8ff7122d4842abb3aa0ff38e0352"
|
||||
},
|
||||
{
|
||||
"id": "3e2a2c9c-4204-4883-b18a-5b1deeca70c3",
|
||||
"latency_ms": 2162.9652910050936,
|
||||
"instance": "ba0474b668fd43b294ece08957bf0648"
|
||||
},
|
||||
{
|
||||
"id": "ea4b8e91-80fe-4741-8d0b-c77d2da6656f",
|
||||
"latency_ms": 4465.512333001243,
|
||||
"instance": "e91b8ff7122d4842abb3aa0ff38e0352"
|
||||
},
|
||||
{
|
||||
"id": "031a9c2b-4c93-4dd4-935b-8104080a0a86",
|
||||
"latency_ms": 1169.7680839861277,
|
||||
"instance": "ba0474b668fd43b294ece08957bf0648"
|
||||
}
|
||||
],
|
||||
"window_max_memory_current": 163500032,
|
||||
"lifetime_memory_peak": 215093248
|
||||
},
|
||||
"reconciliation": {
|
||||
"dry_run": {
|
||||
"apply": false,
|
||||
"grace_seconds": 300,
|
||||
"objects_scanned": 20,
|
||||
"rows_scanned": 20,
|
||||
"head_requests": 8,
|
||||
"bytes_read": 111817,
|
||||
"fresh_protected": 13,
|
||||
"ready_protected": 5,
|
||||
"linked_protected": 0,
|
||||
"actions": {
|
||||
"delete_orphan": 1,
|
||||
"recover_pending": 1,
|
||||
"fail_pending_missing": 1,
|
||||
"delete_failed_object": 1,
|
||||
"mark_missing": 1
|
||||
},
|
||||
"applied": {},
|
||||
"details": [
|
||||
{
|
||||
"action": "delete_orphan",
|
||||
"identity": "objects/verification-f49e0d70d0f54f62bce62b4a3dccdbdc/orphan"
|
||||
},
|
||||
{
|
||||
"action": "recover_pending",
|
||||
"identity": "06e24ef9-2a84-43d1-bc08-facf33155e9f"
|
||||
},
|
||||
{
|
||||
"action": "fail_pending_missing",
|
||||
"identity": "44b0d154-ed76-4f79-a356-7feb85a3b14c"
|
||||
},
|
||||
{
|
||||
"action": "delete_failed_object",
|
||||
"identity": "54e914d0-5a4c-4dc9-982c-ef887467700c"
|
||||
},
|
||||
{
|
||||
"action": "mark_missing",
|
||||
"identity": "fa66fcd8-ca7e-41e4-8f25-25ff119c38e9"
|
||||
}
|
||||
],
|
||||
"duration_seconds": 0.0604,
|
||||
"complete": true
|
||||
},
|
||||
"apply": {
|
||||
"apply": true,
|
||||
"grace_seconds": 300,
|
||||
"objects_scanned": 20,
|
||||
"rows_scanned": 20,
|
||||
"head_requests": 10,
|
||||
"bytes_read": 111817,
|
||||
"fresh_protected": 13,
|
||||
"ready_protected": 5,
|
||||
"linked_protected": 0,
|
||||
"actions": {
|
||||
"delete_orphan": 1,
|
||||
"recover_pending": 1,
|
||||
"fail_pending_missing": 1,
|
||||
"delete_failed_object": 1,
|
||||
"mark_missing": 1
|
||||
},
|
||||
"applied": {
|
||||
"delete_orphan": 1,
|
||||
"recover_pending": 1,
|
||||
"fail_pending_missing": 1,
|
||||
"delete_failed_object": 1,
|
||||
"mark_missing": 1
|
||||
},
|
||||
"details": [
|
||||
{
|
||||
"action": "delete_orphan",
|
||||
"identity": "objects/verification-f49e0d70d0f54f62bce62b4a3dccdbdc/orphan"
|
||||
},
|
||||
{
|
||||
"action": "recover_pending",
|
||||
"identity": "06e24ef9-2a84-43d1-bc08-facf33155e9f"
|
||||
},
|
||||
{
|
||||
"action": "fail_pending_missing",
|
||||
"identity": "44b0d154-ed76-4f79-a356-7feb85a3b14c"
|
||||
},
|
||||
{
|
||||
"action": "delete_failed_object",
|
||||
"identity": "54e914d0-5a4c-4dc9-982c-ef887467700c"
|
||||
},
|
||||
{
|
||||
"action": "mark_missing",
|
||||
"identity": "fa66fcd8-ca7e-41e4-8f25-25ff119c38e9"
|
||||
}
|
||||
],
|
||||
"duration_seconds": 0.106004,
|
||||
"complete": true
|
||||
},
|
||||
"repeat": {
|
||||
"apply": true,
|
||||
"grace_seconds": 300,
|
||||
"objects_scanned": 18,
|
||||
"rows_scanned": 20,
|
||||
"head_requests": 6,
|
||||
"bytes_read": 0,
|
||||
"fresh_protected": 13,
|
||||
"ready_protected": 6,
|
||||
"linked_protected": 1,
|
||||
"actions": {},
|
||||
"applied": {},
|
||||
"details": [],
|
||||
"duration_seconds": 0.102749,
|
||||
"complete": true
|
||||
},
|
||||
"s3_grace_wait_seconds": 331.301,
|
||||
"fresh_and_ready_unlinked_protected": true,
|
||||
"old_failed_unlinked_deleted_and_fresh_failed_protected": true,
|
||||
"missing_used_id_and_link_preserved": true,
|
||||
"dry_run_unchanged": true
|
||||
},
|
||||
"cost_research": {
|
||||
"cells": [
|
||||
{
|
||||
"extra_objects": 0,
|
||||
"repetition": 1,
|
||||
"apply": false,
|
||||
"grace_seconds": 300,
|
||||
"objects_scanned": 18,
|
||||
"rows_scanned": 20,
|
||||
"head_requests": 6,
|
||||
"bytes_read": 0,
|
||||
"fresh_protected": 13,
|
||||
"ready_protected": 6,
|
||||
"linked_protected": 1,
|
||||
"actions": {},
|
||||
"applied": {},
|
||||
"details": [],
|
||||
"duration_seconds": 0.183485,
|
||||
"complete": true
|
||||
},
|
||||
{
|
||||
"extra_objects": 0,
|
||||
"repetition": 2,
|
||||
"apply": false,
|
||||
"grace_seconds": 300,
|
||||
"objects_scanned": 18,
|
||||
"rows_scanned": 20,
|
||||
"head_requests": 6,
|
||||
"bytes_read": 0,
|
||||
"fresh_protected": 13,
|
||||
"ready_protected": 6,
|
||||
"linked_protected": 1,
|
||||
"actions": {},
|
||||
"applied": {},
|
||||
"details": [],
|
||||
"duration_seconds": 0.043382,
|
||||
"complete": true
|
||||
},
|
||||
{
|
||||
"extra_objects": 0,
|
||||
"repetition": 3,
|
||||
"apply": false,
|
||||
"grace_seconds": 300,
|
||||
"objects_scanned": 18,
|
||||
"rows_scanned": 20,
|
||||
"head_requests": 6,
|
||||
"bytes_read": 0,
|
||||
"fresh_protected": 13,
|
||||
"ready_protected": 6,
|
||||
"linked_protected": 1,
|
||||
"actions": {},
|
||||
"applied": {},
|
||||
"details": [],
|
||||
"duration_seconds": 0.046774,
|
||||
"complete": true
|
||||
},
|
||||
{
|
||||
"extra_objects": 100,
|
||||
"repetition": 1,
|
||||
"apply": false,
|
||||
"grace_seconds": 300,
|
||||
"objects_scanned": 118,
|
||||
"rows_scanned": 20,
|
||||
"head_requests": 6,
|
||||
"bytes_read": 0,
|
||||
"fresh_protected": 112,
|
||||
"ready_protected": 6,
|
||||
"linked_protected": 1,
|
||||
"actions": {},
|
||||
"applied": {},
|
||||
"details": [],
|
||||
"duration_seconds": 0.078726,
|
||||
"complete": true
|
||||
},
|
||||
{
|
||||
"extra_objects": 100,
|
||||
"repetition": 2,
|
||||
"apply": false,
|
||||
"grace_seconds": 300,
|
||||
"objects_scanned": 118,
|
||||
"rows_scanned": 20,
|
||||
"head_requests": 6,
|
||||
"bytes_read": 0,
|
||||
"fresh_protected": 112,
|
||||
"ready_protected": 6,
|
||||
"linked_protected": 1,
|
||||
"actions": {},
|
||||
"applied": {},
|
||||
"details": [],
|
||||
"duration_seconds": 0.088476,
|
||||
"complete": true
|
||||
},
|
||||
{
|
||||
"extra_objects": 100,
|
||||
"repetition": 3,
|
||||
"apply": false,
|
||||
"grace_seconds": 300,
|
||||
"objects_scanned": 118,
|
||||
"rows_scanned": 20,
|
||||
"head_requests": 6,
|
||||
"bytes_read": 0,
|
||||
"fresh_protected": 112,
|
||||
"ready_protected": 6,
|
||||
"linked_protected": 1,
|
||||
"actions": {},
|
||||
"applied": {},
|
||||
"details": [],
|
||||
"duration_seconds": 0.206964,
|
||||
"complete": true
|
||||
},
|
||||
{
|
||||
"extra_objects": 1100,
|
||||
"repetition": 1,
|
||||
"apply": false,
|
||||
"grace_seconds": 300,
|
||||
"objects_scanned": 1118,
|
||||
"rows_scanned": 20,
|
||||
"head_requests": 6,
|
||||
"bytes_read": 0,
|
||||
"fresh_protected": 1112,
|
||||
"ready_protected": 6,
|
||||
"linked_protected": 1,
|
||||
"actions": {},
|
||||
"applied": {},
|
||||
"details": [],
|
||||
"duration_seconds": 0.197227,
|
||||
"complete": true
|
||||
},
|
||||
{
|
||||
"extra_objects": 1100,
|
||||
"repetition": 2,
|
||||
"apply": false,
|
||||
"grace_seconds": 300,
|
||||
"objects_scanned": 1118,
|
||||
"rows_scanned": 20,
|
||||
"head_requests": 10,
|
||||
"bytes_read": 0,
|
||||
"fresh_protected": 1108,
|
||||
"ready_protected": 10,
|
||||
"linked_protected": 1,
|
||||
"actions": {},
|
||||
"applied": {},
|
||||
"details": [],
|
||||
"duration_seconds": 0.145977,
|
||||
"complete": true
|
||||
},
|
||||
{
|
||||
"extra_objects": 1100,
|
||||
"repetition": 3,
|
||||
"apply": false,
|
||||
"grace_seconds": 300,
|
||||
"objects_scanned": 1118,
|
||||
"rows_scanned": 20,
|
||||
"head_requests": 16,
|
||||
"bytes_read": 0,
|
||||
"fresh_protected": 1102,
|
||||
"ready_protected": 16,
|
||||
"linked_protected": 1,
|
||||
"actions": {},
|
||||
"applied": {},
|
||||
"details": [],
|
||||
"duration_seconds": 0.199997,
|
||||
"complete": true
|
||||
}
|
||||
],
|
||||
"repeats": 3,
|
||||
"method": "same running stand; 0/100/1100 fresh one-byte objects; dry-run; serial ascending dataset; no warmup",
|
||||
"median_seconds": {
|
||||
"0": 0.046774,
|
||||
"100": 0.088476,
|
||||
"1100": 0.197227
|
||||
}
|
||||
}
|
||||
}
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 83 KiB |
@@ -0,0 +1,159 @@
|
||||
"""Controlled inventory experiment on the teaching stand, with own fixtures."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import statistics
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
|
||||
from scripts.stand import BASE, actor, auth, command, compose, ready, sql
|
||||
from scripts.verify_storage import PNG, PREFIX, inside
|
||||
|
||||
|
||||
def probe():
|
||||
# Run the actual code with the reconciler service's .25 CPU /256 MiB.
|
||||
code = (
|
||||
PREFIX
|
||||
+ """
|
||||
import pathlib,resource
|
||||
started=time.process_time()
|
||||
result=reconcile(d,s,apply=False,grace=3600)
|
||||
result['cpu_seconds']=time.process_time()-started
|
||||
result['rss_peak_bytes']=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss*1024
|
||||
result['cgroup_memory_peak_bytes']=int(pathlib.Path('/sys/fs/cgroup/memory.peak').read_text())
|
||||
print(json.dumps(result)); s.close(); d.close()
|
||||
"""
|
||||
)
|
||||
return json.loads(
|
||||
command(
|
||||
[
|
||||
"docker",
|
||||
"compose",
|
||||
"run",
|
||||
"--rm",
|
||||
"--no-deps",
|
||||
"-T",
|
||||
"reconciler",
|
||||
"python",
|
||||
"-",
|
||||
],
|
||||
input=code,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--output", default="evidence/cost-repeat.json")
|
||||
args = parser.parse_args()
|
||||
output = Path(args.output)
|
||||
assert not output.exists()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
prefix = "objects/cost-" + uuid4().hex + "/"
|
||||
result = {
|
||||
"date": datetime.now(UTC).isoformat(),
|
||||
"source_commit": command(["git", "rev-parse", "HEAD"]),
|
||||
"complete": False,
|
||||
"cells": [],
|
||||
"warmups": [],
|
||||
"method": "0/100/1100 extra fresh one-byte objects; one warmup then 3 repeats per dataset; real reconciler service .25 CPU/256MiB; grace3600; 8 PNG fixtures with deliberately aged PG metadata (2days); other runtime objects remain real",
|
||||
"fixture_rows": 8,
|
||||
"grace_seconds": 3600,
|
||||
}
|
||||
chat = None
|
||||
ready()
|
||||
compose("stop", "reconciler")
|
||||
try:
|
||||
with httpx.Client(
|
||||
base_url=BASE, trust_env=False, timeout=20
|
||||
) as client:
|
||||
user, _, session = actor(client)
|
||||
r = client.post(
|
||||
"/api/v1/chats",
|
||||
headers=auth(session),
|
||||
json={
|
||||
"title": "Controlled inventory",
|
||||
"member_ids": [user["id"]],
|
||||
},
|
||||
)
|
||||
assert r.status_code == 201
|
||||
chat = r.json()["id"]
|
||||
seed = f"""
|
||||
from uuid import uuid4
|
||||
data=bytes.fromhex({PNG.hex()!r})
|
||||
for i in range(8):
|
||||
identity=uuid4(); key={prefix!r}+'ready-'+str(i)
|
||||
s.put(key=key,stream=io.BytesIO(data),size=len(data),content_type='image/png',sha256=hashlib.sha256(data).hexdigest())
|
||||
with d.connection() as c:
|
||||
c.execute("INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,state,uploader_instance,expires_at,created_at) VALUES(%s,%s,%s,'fixture.png','image/png',%s,%s,%s,'ready','cost-fixture',clock_timestamp()-interval '1 day',clock_timestamp()-interval '2 days')",(identity,{chat!r},{user["id"]!r},len(data),hashlib.sha256(data).hexdigest(),key))
|
||||
print('true')
|
||||
"""
|
||||
inside(seed)
|
||||
for count in [0, 100, 1100]:
|
||||
if count:
|
||||
start = 0 if count == 100 else 100
|
||||
inside(
|
||||
f"\nfor i in range({start},{count}):\n s.put(key={prefix!r}+'extra-'+str(i),stream=io.BytesIO(b'x'),size=1,content_type='application/octet-stream',sha256=hashlib.sha256(b'x').hexdigest())\nprint('true')"
|
||||
)
|
||||
result["warmups"].append({"extra_objects": count, **probe()})
|
||||
for repeat in range(3):
|
||||
cell = {
|
||||
"extra_objects": count,
|
||||
"repetition": repeat + 1,
|
||||
**probe(),
|
||||
}
|
||||
result["cells"].append(cell)
|
||||
assert cell["complete"] and not cell["applied"]
|
||||
assert cell["ready_protected"] >= 8
|
||||
controls = {
|
||||
(v["rows_scanned"], v["head_requests"], v["bytes_read"])
|
||||
for v in result["cells"]
|
||||
}
|
||||
assert len(controls) == 1, (
|
||||
"Age boundary changed work; choose a fresh stand"
|
||||
)
|
||||
for count in [0, 100, 1100]:
|
||||
assert (
|
||||
len(
|
||||
{
|
||||
v["objects_scanned"]
|
||||
for v in result["cells"]
|
||||
if v["extra_objects"] == count
|
||||
}
|
||||
)
|
||||
== 1
|
||||
)
|
||||
result["stable_rows_head_bytes"] = list(next(iter(controls)))
|
||||
result["median_seconds"] = {
|
||||
str(n): statistics.median(
|
||||
v["duration_seconds"]
|
||||
for v in result["cells"]
|
||||
if v["extra_objects"] == n
|
||||
)
|
||||
for n in [0, 100, 1100]
|
||||
}
|
||||
result["complete"] = True
|
||||
finally:
|
||||
if chat:
|
||||
sql(
|
||||
"DELETE FROM attachments WHERE chat_id='"
|
||||
+ chat
|
||||
+ "' AND uploader_instance='cost-fixture'"
|
||||
)
|
||||
inside(
|
||||
f"\nfor o in s.objects():\n if o.object_name.startswith({prefix!r}): s.delete(key=o.object_name)\nprint('true')"
|
||||
)
|
||||
compose("up", "-d", "--no-build", "--no-recreate", "reconciler")
|
||||
output.write_text(
|
||||
json.dumps(result, ensure_ascii=False, indent=2) + "\n"
|
||||
)
|
||||
print(
|
||||
"PASS: controlled inventory cost, 9 measurements and constant metadata/HEAD work"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -386,7 +386,14 @@ except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':j
|
||||
inside(f"s.delete(key={key(used['id'])!r}); print('true')")
|
||||
age(used["id"])
|
||||
age(saved["id"])
|
||||
# Fresh cgroups make memory.peak specific to this experiment.
|
||||
compose("restart", "api")
|
||||
cluster.wait_count(2)
|
||||
ready()
|
||||
result["concurrent_uploads"] = concurrency(client, session, chat)
|
||||
result["concurrent_uploads"][
|
||||
"api_restarted_before_memory_measurement"
|
||||
] = True
|
||||
while time.monotonic() - grace_started < 302:
|
||||
print(
|
||||
"Waiting for genuine S3 LastModified grace: "
|
||||
@@ -464,7 +471,7 @@ except urllib.error.HTTPError as e: print(json.dumps({{'status':e.code,'error':j
|
||||
f"print(json.dumps(s.head(key={key(failed_old['id'])!r}) is None and s.head(key={key(failed_fresh['id'])!r}) is not None))"
|
||||
)
|
||||
repeat = run_reconcile(True)
|
||||
assert not repeat["applied"].get("delete_orphan")
|
||||
assert not repeat["applied"]
|
||||
assert repeat["linked_protected"] >= 1
|
||||
result["reconciliation"] = {
|
||||
"dry_run": dry,
|
||||
|
||||
Reference in New Issue
Block a user