test: capture API failover under sustained load
This commit is contained in:
@@ -0,0 +1,207 @@
|
||||
{
|
||||
"kind": "failover",
|
||||
"label": "baseline",
|
||||
"started": "2026-10-15T10:55:52Z",
|
||||
"profile": {
|
||||
"name": "failover",
|
||||
"limits": {
|
||||
"api_replicas": 2,
|
||||
"api_cpus": 1.0,
|
||||
"api_memory": "512m",
|
||||
"db_cpus": 2.0,
|
||||
"db_memory": "1g",
|
||||
"db_max_connections": 100,
|
||||
"cache_cpus": 0.5,
|
||||
"cache_memory": "128m",
|
||||
"lb_cpus": 1.0,
|
||||
"lb_memory": "256m",
|
||||
"gen_cpus": 2.0,
|
||||
"gen_memory": "1g",
|
||||
"sampler_cpus": 0.25,
|
||||
"sampler_memory": "256m"
|
||||
},
|
||||
"users": 4,
|
||||
"read_messages": 1000,
|
||||
"write_rate": 100.0,
|
||||
"read_rate": 100.0,
|
||||
"gen_procs": 2,
|
||||
"session_ttl": 21600,
|
||||
"seed": 20261014,
|
||||
"modes": [
|
||||
"stop",
|
||||
"kill",
|
||||
"pause"
|
||||
],
|
||||
"targets": 2,
|
||||
"before_s": 10.0,
|
||||
"fail_s": 15.0,
|
||||
"after_s": 10.0,
|
||||
"restore_s": 10.0,
|
||||
"rollout_before_s": 0.0,
|
||||
"rollout_budget_s": 0.0,
|
||||
"rollout_after_s": 0.0,
|
||||
"bounds": [
|
||||
[
|
||||
"stop",
|
||||
2.5
|
||||
],
|
||||
[
|
||||
"kill",
|
||||
2.5
|
||||
],
|
||||
[
|
||||
"pause",
|
||||
6.5
|
||||
]
|
||||
]
|
||||
},
|
||||
"host": {
|
||||
"platform": "macOS-26.6.2-arm64-arm-64bit-Mach-O",
|
||||
"machine": "arm64",
|
||||
"cpu_model": "Apple M4",
|
||||
"cpu_count": 10,
|
||||
"ram_gb": 16.0,
|
||||
"swap_used_mib": 0.0,
|
||||
"power": "Battery Power",
|
||||
"memory_pressure": null,
|
||||
"memory_free_percent": 79,
|
||||
"swapouts": 0,
|
||||
"loadavg_1m": 1.34
|
||||
},
|
||||
"docker": {
|
||||
"ServerVersion": "28.0.4",
|
||||
"OperatingSystem": "Docker Desktop",
|
||||
"KernelVersion": "6.10.14-linuxkit",
|
||||
"Architecture": "aarch64",
|
||||
"NCPU": 10,
|
||||
"MemTotal": 8218034176,
|
||||
"CgroupVersion": "2",
|
||||
"CgroupDriver": "cgroupfs",
|
||||
"compose_version": "2.34.0-desktop.1",
|
||||
"other_running_containers": []
|
||||
},
|
||||
"budget": {
|
||||
"limits_cpus": 7.75,
|
||||
"limits_memory_gib": 3.62,
|
||||
"vm_cpus": 10,
|
||||
"vm_memory_gib": 7.65
|
||||
},
|
||||
"host_warnings": [
|
||||
"host on battery power: CPU may be throttled"
|
||||
],
|
||||
"image": {
|
||||
"tag": "mtusi-messenger-vladimirceva:bench-baseline",
|
||||
"id": "sha256:ac17d94cda500d4a65418c184deec9428e706ff40a68a7789b068c973c7d4e05",
|
||||
"created": "2026-10-15T10:39:12.700675009Z",
|
||||
"src_sha256": "0349e9a2701784673391c0ba4bc2eedba7013098d6ff7f871261aab36f78354a",
|
||||
"versions": {
|
||||
"python": "3.13.15",
|
||||
"fastapi": "0.142.1",
|
||||
"starlette": "1.7.0",
|
||||
"uvicorn": "0.54.0",
|
||||
"pydantic": "2.13.5",
|
||||
"sqlalchemy": "2.1.1",
|
||||
"asyncpg": "0.31.0",
|
||||
"alembic": "1.20.0",
|
||||
"redis": "8.1.0",
|
||||
"argon2-cffi": "25.1.0"
|
||||
},
|
||||
"app_defaults": {
|
||||
"pool_size": 10,
|
||||
"pool_overflow": 5,
|
||||
"pool_timeout": 5.0,
|
||||
"operation_timeout": 6.0,
|
||||
"statement_timeout_ms": 5000,
|
||||
"kdf_concurrency": 4,
|
||||
"keep_alive": 60,
|
||||
"shutdown_timeout": 5
|
||||
}
|
||||
},
|
||||
"code": {
|
||||
"src_sha256": "0349e9a2701784673391c0ba4bc2eedba7013098d6ff7f871261aab36f78354a",
|
||||
"git_commit": "006ec24d208670f6405bc1bd02f2aafe207e2b30",
|
||||
"git_dirty": false
|
||||
},
|
||||
"setup": {
|
||||
"users": 4,
|
||||
"read_chat": "691d7f26-cb3d-4fbf-8037-95c2b0ff2f98",
|
||||
"read_messages": 1000,
|
||||
"cursors": 20,
|
||||
"fill_s": 0.94,
|
||||
"fill_rps": 1069.0
|
||||
},
|
||||
"stand": {
|
||||
"postgres": {
|
||||
"image": "postgres:18.6-trixie@sha256:5a5a84b19854a9ffaa54082c166ff4ec27473a361e496e5ea167f298f2da9722",
|
||||
"version": "18.6 (Debian 18.6-1.pgdg13+2)",
|
||||
"settings": {
|
||||
"checkpoint_timeout": "5min",
|
||||
"fsync": "on",
|
||||
"jit": "on",
|
||||
"max_connections": "100",
|
||||
"max_wal_size": "1GB",
|
||||
"password_encryption": "scram-sha-256",
|
||||
"shared_buffers": "128MB",
|
||||
"synchronous_commit": "on",
|
||||
"wal_level": "replica"
|
||||
}
|
||||
},
|
||||
"valkey": {
|
||||
"image": "valkey/valkey:9.1.2-trixie@sha256:418652cfb58ef879d4978c33553735d7147016032d5aefaa14c828e611eb9dfd",
|
||||
"version": "9.1.2"
|
||||
},
|
||||
"haproxy": {
|
||||
"image": "haproxy:3.4.6-trixie@sha256:e273915691a8eab3f105bf3e39fd4c29a4ecf334e33421903b5589d25e14db26"
|
||||
},
|
||||
"limits": {
|
||||
"api": {
|
||||
"nano_cpus": 1000000000,
|
||||
"memory": 536870912,
|
||||
"memory_swap": 536870912,
|
||||
"published_ports": {}
|
||||
},
|
||||
"postgres": {
|
||||
"nano_cpus": 2000000000,
|
||||
"memory": 1073741824,
|
||||
"memory_swap": 1073741824,
|
||||
"published_ports": {}
|
||||
},
|
||||
"valkey": {
|
||||
"nano_cpus": 500000000,
|
||||
"memory": 134217728,
|
||||
"memory_swap": 134217728,
|
||||
"published_ports": {}
|
||||
},
|
||||
"haproxy": {
|
||||
"nano_cpus": 1000000000,
|
||||
"memory": 268435456,
|
||||
"memory_swap": 268435456,
|
||||
"published_ports": {}
|
||||
}
|
||||
},
|
||||
"api_replicas": 2,
|
||||
"app_env": {
|
||||
"APP_HOST": "0.0.0.0",
|
||||
"APP_PORT": "8080",
|
||||
"CACHE_HOST": "valkey",
|
||||
"CACHE_PORT": "6379",
|
||||
"DB_HOST": "postgres",
|
||||
"DB_NAME": "bench",
|
||||
"DB_PORT": "5432",
|
||||
"DB_USER": "bench",
|
||||
"IDEMPOTENCY_TTL_SECONDS": "86400",
|
||||
"LOG_LEVEL": "INFO",
|
||||
"SESSION_TTL_SECONDS": "21600",
|
||||
"SHUTDOWN_TIMEOUT": "5"
|
||||
}
|
||||
},
|
||||
"sampler_ticks": 281,
|
||||
"finished": "2026-10-15T11:00:43Z",
|
||||
"host_state_end": {
|
||||
"slept_seconds": 0.0,
|
||||
"swapouts_start": 0,
|
||||
"swapouts_end": 0,
|
||||
"power_end": "Battery Power",
|
||||
"swapouts_delta": 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
phase,mode,replica,target,slot,action_at_ms,restore_at_ms,detection_s,bound_s,over_bound,lb_return_s,errors_before,errors_fail,errors_after,error_kinds,first_error_s,last_error_s,recovery_s,ok_rps_before,ok_rps_fail,ok_rps_after,p95_before_ms,p95_fail_ms,p95_after_ms,max_fail_ms,target_after_down,writes,retried,recovered,rows,distinct_texts,verified,lb_errors
|
||||
p1-stop-r1,stop,r1,cce5762ff431,api-2,1792061771104.8,1792061786580.0,1.79,2.5,False,2.03,0,1,0,IncompleteReadError:1,0.09,0.09,1.0,200.0,199.9,199.9,6.5,13.3,6.9,512.6,0,4500,0,0,4500,4500,True,
|
||||
p2-kill-r1,kill,r1,cce5762ff431,api-2,1792061817739.5,1792061832912.1,1.21,2.5,False,1.8,1,0,0,IncompleteReadError:1,-0.0,-0.0,0.0,199.8,200.0,199.9,6.4,9.0,6.7,536.1,0,4500,1,1,4500,4500,True,
|
||||
p3-pause-r1,pause,r1,cce5762ff431,api-2,1792061864401.4,1792061879442.8,5.89,6.5,False,1.04,1,584,0,504:585,-0.01,5.83,6.0,199.7,161.2,199.9,6.9,7.8,6.7,9998.4,0,4500,12,12,4500,4500,True,504:585
|
||||
p4-stop-r2,stop,r2,dad5d68a3f49,api-1,1792061911082.5,1792061926559.0,1.27,2.5,False,0.81,0,0,0,,,,0.0,200.2,199.9,200.0,6.6,8.6,6.6,528.0,0,4500,0,0,4500,4500,True,
|
||||
p5-kill-r2,kill,r2,dad5d68a3f49,api-1,1792061957859.1,1792061973051.8,0.26,2.5,False,1.92,0,0,0,,,,0.0,199.8,200.1,199.9,6.5,7.5,6.6,507.9,0,4500,0,0,4500,4500,True,
|
||||
p6-pause-r2,pause,r2,dad5d68a3f49,api-1,1792062004497.9,1792062019540.0,6.06,6.5,False,0.98,1,553,0,504:554,-0.01,5.52,6.0,199.9,163.2,199.9,6.8,7.6,6.9,10000.6,0,4500,548,548,4500,4500,True,504:554
|
||||
|
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,19 @@
|
||||
phase,window,api_cpu_cores,api_cpu_max,api_throttled_max,api_memory_max_mib,db_cpu_cores,db_memory_mib,lb_cpu_cores,cache_cpu_cores,gen_cpu_cores,vm_busy_share,api_cpu_us_per_req,db_cpu_us_per_req
|
||||
p1-stop-r1,before,0.47,0.243,0.0,70.1,0.085,143.3,0.015,0.006,0.028,0.0814,2350.0,425.0
|
||||
p1-stop-r1,fail,0.343,0.343,0.0,71.3,0.071,143.3,0.013,0.005,0.024,0.0563,1715.9,355.2
|
||||
p1-stop-r1,after,0.484,0.243,0.0,71.3,0.091,137.2,0.016,0.006,0.026,0.0726,2421.2,455.2
|
||||
p2-kill-r1,before,0.51,0.259,0.0,71.3,0.089,138.5,0.017,0.006,0.03,0.074,2552.6,445.4
|
||||
p2-kill-r1,fail,0.351,0.351,0.0,72.1,0.069,138.3,0.014,0.006,0.024,0.0597,1755.0,345.0
|
||||
p2-kill-r1,after,0.489,0.252,0.0,71.8,0.097,147.8,0.015,0.006,0.029,0.0737,2446.2,485.2
|
||||
p3-pause-r1,before,0.509,0.267,0.0,72.3,0.099,149.4,0.016,0.007,0.03,0.0784,2548.8,495.7
|
||||
p3-pause-r1,fail,0.326,0.306,0.0,86.7,0.065,162.2,0.017,0.006,0.035,0.0563,2022.3,403.2
|
||||
p3-pause-r1,after,0.522,0.275,0.0,94.1,0.104,162.5,0.017,0.007,0.029,0.0812,2611.3,520.3
|
||||
p4-stop-r2,before,0.496,0.255,0.0,94.1,0.115,163.5,0.016,0.006,0.029,0.0801,2477.5,574.4
|
||||
p4-stop-r2,fail,0.35,0.35,0.0,95.1,0.072,163.4,0.014,0.006,0.023,0.0578,1750.9,360.2
|
||||
p4-stop-r2,after,0.521,0.275,0.005,94.8,0.094,168.3,0.016,0.006,0.027,0.0768,2605.0,470.0
|
||||
p5-kill-r2,before,0.529,0.266,0.0,94.6,0.096,169.1,0.017,0.006,0.03,0.0758,2647.6,480.5
|
||||
p5-kill-r2,fail,0.344,0.344,0.0,95.1,0.085,169.1,0.013,0.006,0.023,0.0622,1719.1,424.8
|
||||
p5-kill-r2,after,0.488,0.251,0.0,94.9,0.092,175.9,0.016,0.006,0.027,0.0746,2441.2,460.2
|
||||
p6-pause-r2,before,0.502,0.257,0.0,94.6,0.094,177.0,0.016,0.006,0.031,0.0756,2511.3,470.2
|
||||
p6-pause-r2,fail,0.328,0.311,0.0,94.9,0.065,177.7,0.018,0.006,0.039,0.0555,2009.8,398.3
|
||||
p6-pause-r2,after,0.505,0.266,0.0051,94.7,0.116,193.4,0.017,0.006,0.029,0.0808,2526.3,580.3
|
||||
|
@@ -0,0 +1,45 @@
|
||||
# Отказ реплики под нагрузкой: baseline
|
||||
|
||||
Код: src_sha256 `0349e9a270178467`, commit 006ec24d2086. Хост: Apple M4, 16.0 ГиБ; ВМ Docker 10 CPU, 7.7 ГиБ, Docker 28.0.4, compose 2.34.0-desktop.1. Начало 2026-10-15T10:55:52Z, конец 2026-10-15T11:00:43Z.
|
||||
|
||||
Нагрузка: открытая модель, 100 записей с `Idempotency-Key` и 100 чтений страницы истории в секунду через HAProxy стенда; две реплики с лимитами рабочего стенда. Фаза: 10 с до действия, действие, 15 с отказа, возврат той же реплики, ещё 10 с.
|
||||
|
||||
Отметки — часы ВМ Docker: действие и возврат — события Docker, исключение и возврат в LB — журнал HAProxy, запросы — генератор (момент по расписанию). Обнаружение — от действия до `DOWN` слота; восстановление — от действия до начала секунды, после которой до возврата нет ошибок и успешных ≥ 90 % поданной нагрузки.
|
||||
|
||||
| Фаза | Режим | Реплика | Обнаружение, с (граница) | Возврат в LB, с | Ошибки до / отказ / после | Виды ошибок | Восстановление, с | Успешных/с до / отказ / после | p95, мс до / отказ / после | max в отказе, мс | Повтор: не прошло → прошло | Сверка |
|
||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
||||
| p1-stop-r1 | stop | r1 (cce5762ff431) | 1.79 (2.5) | 2.03 | 0 / 1 / 0 | IncompleteReadError:1 | 1 | 200.0 / 199.9 / 199.9 | 6.5 / 13.3 / 6.9 | 512.6 | 0 → 0 | ok: 4500 ключей = 4500 строк |
|
||||
| p2-kill-r1 | kill | r1 (cce5762ff431) | 1.21 (2.5) | 1.8 | 1 / 0 / 0 | IncompleteReadError:1 | 0 | 199.8 / 200.0 / 199.9 | 6.4 / 9 / 6.7 | 536.1 | 1 → 1 | ok: 4500 ключей = 4500 строк |
|
||||
| p3-pause-r1 | pause | r1 (cce5762ff431) | 5.89 (6.5) | 1.04 | 1 / 584 / 0 | 504:585 | 6 | 199.7 / 161.2 / 199.9 | 6.9 / 7.8 / 6.7 | 9998.4 | 12 → 12 | ok: 4500 ключей = 4500 строк |
|
||||
| p4-stop-r2 | stop | r2 (dad5d68a3f49) | 1.27 (2.5) | 0.81 | 0 / 0 / 0 | — | 0 | 200.2 / 199.9 / 200.0 | 6.6 / 8.6 / 6.6 | 528 | 0 → 0 | ok: 4500 ключей = 4500 строк |
|
||||
| p5-kill-r2 | kill | r2 (dad5d68a3f49) | 0.26 (2.5) | 1.92 | 0 / 0 / 0 | — | 0 | 199.8 / 200.1 / 199.9 | 6.5 / 7.5 / 6.6 | 507.9 | 0 → 0 | ok: 4500 ключей = 4500 строк |
|
||||
| p6-pause-r2 | pause | r2 (dad5d68a3f49) | 6.06 (6.5) | 0.98 | 1 / 553 / 0 | 504:554 | 6 | 199.9 / 163.2 / 199.9 | 6.8 / 7.6 / 6.9 | 10000.6 | 548 → 548 | ok: 4500 ключей = 4500 строк |
|
||||
|
||||
## CPU и память по окнам фазы
|
||||
|
||||
Сэмплер (cgroup каждого контейнера раз в секунду). Окна: до действия (от первого запроса + 2 с), отказ (действие → возврат), после (возврат в LB → конец нагрузки). CPU реплик — сумма; на успешный запрос — только по окнам с нагрузкой.
|
||||
|
||||
| Фаза | Окно | API, ядер (макс. реплика) | PostgreSQL | HAProxy | Valkey | Генератор | ВМ занята | API, мкс/запрос | PostgreSQL, мкс/запрос |
|
||||
|---|---|---|---|---|---|---|---|---|---|
|
||||
| p1-stop-r1 | before | 0.47 (0.243) | 0.085 | 0.015 | 0.006 | 0.028 | 0.0814 | 2350 | 425 |
|
||||
| p1-stop-r1 | fail | 0.343 (0.343) | 0.071 | 0.013 | 0.005 | 0.024 | 0.0563 | 1715.9 | 355.2 |
|
||||
| p1-stop-r1 | after | 0.484 (0.243) | 0.091 | 0.016 | 0.006 | 0.026 | 0.0726 | 2421.2 | 455.2 |
|
||||
| p2-kill-r1 | before | 0.51 (0.259) | 0.089 | 0.017 | 0.006 | 0.03 | 0.074 | 2552.6 | 445.4 |
|
||||
| p2-kill-r1 | fail | 0.351 (0.351) | 0.069 | 0.014 | 0.006 | 0.024 | 0.0597 | 1755 | 345 |
|
||||
| p2-kill-r1 | after | 0.489 (0.252) | 0.097 | 0.015 | 0.006 | 0.029 | 0.0737 | 2446.2 | 485.2 |
|
||||
| p3-pause-r1 | before | 0.509 (0.267) | 0.099 | 0.016 | 0.007 | 0.03 | 0.0784 | 2548.8 | 495.7 |
|
||||
| p3-pause-r1 | fail | 0.326 (0.306) | 0.065 | 0.017 | 0.006 | 0.035 | 0.0563 | 2022.3 | 403.2 |
|
||||
| p3-pause-r1 | after | 0.522 (0.275) | 0.104 | 0.017 | 0.007 | 0.029 | 0.0812 | 2611.3 | 520.3 |
|
||||
| p4-stop-r2 | before | 0.496 (0.255) | 0.115 | 0.016 | 0.006 | 0.029 | 0.0801 | 2477.5 | 574.4 |
|
||||
| p4-stop-r2 | fail | 0.35 (0.35) | 0.072 | 0.014 | 0.006 | 0.023 | 0.0578 | 1750.9 | 360.2 |
|
||||
| p4-stop-r2 | after | 0.521 (0.275) | 0.094 | 0.016 | 0.006 | 0.027 | 0.0768 | 2605 | 470 |
|
||||
| p5-kill-r2 | before | 0.529 (0.266) | 0.096 | 0.017 | 0.006 | 0.03 | 0.0758 | 2647.6 | 480.5 |
|
||||
| p5-kill-r2 | fail | 0.344 (0.344) | 0.085 | 0.013 | 0.006 | 0.023 | 0.0622 | 1719.1 | 424.8 |
|
||||
| p5-kill-r2 | after | 0.488 (0.251) | 0.092 | 0.016 | 0.006 | 0.027 | 0.0746 | 2441.2 | 460.2 |
|
||||
| p6-pause-r2 | before | 0.502 (0.257) | 0.094 | 0.016 | 0.006 | 0.031 | 0.0756 | 2511.3 | 470.2 |
|
||||
| p6-pause-r2 | fail | 0.328 (0.311) | 0.065 | 0.018 | 0.006 | 0.039 | 0.0555 | 2009.8 | 398.3 |
|
||||
| p6-pause-r2 | after | 0.505 (0.266) | 0.116 | 0.017 | 0.006 | 0.029 | 0.0808 | 2526.3 | 580.3 |
|
||||
|
||||
## Предупреждения о хосте
|
||||
|
||||
- host on battery power: CPU may be throttled
|
||||
@@ -0,0 +1,277 @@
|
||||
phase,second,sent,ok_write,ok_read,ok_target,ok_other,ok_old,ok_new,errors,error_kinds,p95_ms,max_ms
|
||||
p1-stop-r1,-10,194,97,97,97,97,,,0,,5.6,11.6
|
||||
p1-stop-r1,-9,200,100,100,100,100,,,0,,6.8,10.8
|
||||
p1-stop-r1,-8,200,100,100,100,100,,,0,,6.6,35.7
|
||||
p1-stop-r1,-7,200,100,100,100,100,,,0,,6.7,9.3
|
||||
p1-stop-r1,-6,200,100,100,100,100,,,0,,6.4,7.5
|
||||
p1-stop-r1,-5,200,100,100,100,100,,,0,,6.0,7.3
|
||||
p1-stop-r1,-4,200,100,100,100,100,,,0,,6.1,7.3
|
||||
p1-stop-r1,-3,200,100,100,100,100,,,0,,6.6,10.3
|
||||
p1-stop-r1,-2,200,100,100,100,100,,,0,,6.6,9.0
|
||||
p1-stop-r1,-1,200,100,100,100,100,,,0,,6.6,7.7
|
||||
p1-stop-r1,0,200,100,99,8,191,,,1,IncompleteReadError:1,506.1,512.6
|
||||
p1-stop-r1,1,200,100,100,0,200,,,0,,506.6,507.1
|
||||
p1-stop-r1,2,200,100,100,0,200,,,0,,7.4,8.5
|
||||
p1-stop-r1,3,200,100,100,0,200,,,0,,7.4,10.1
|
||||
p1-stop-r1,4,200,100,100,0,200,,,0,,8.5,10.8
|
||||
p1-stop-r1,5,200,100,100,0,200,,,0,,8.6,10.6
|
||||
p1-stop-r1,6,200,100,100,0,200,,,0,,7.3,9.1
|
||||
p1-stop-r1,7,200,100,100,0,200,,,0,,7.1,9.0
|
||||
p1-stop-r1,8,200,100,100,0,200,,,0,,8.1,16.2
|
||||
p1-stop-r1,9,200,100,100,0,200,,,0,,7.4,11.2
|
||||
p1-stop-r1,10,200,100,100,0,200,,,0,,7.1,8.6
|
||||
p1-stop-r1,11,200,100,100,0,200,,,0,,8.1,16.0
|
||||
p1-stop-r1,12,200,100,100,0,200,,,0,,8.4,9.8
|
||||
p1-stop-r1,13,200,100,100,0,200,,,0,,7.6,12.0
|
||||
p1-stop-r1,14,200,100,100,0,200,,,0,,7.4,8.3
|
||||
p1-stop-r1,15,200,100,100,0,200,,,0,,7.7,10.5
|
||||
p1-stop-r1,16,200,100,100,0,200,,,0,,7.7,9.1
|
||||
p1-stop-r1,17,200,100,100,50,150,,,0,,8.0,31.9
|
||||
p1-stop-r1,18,200,100,100,100,100,,,0,,6.2,9.5
|
||||
p1-stop-r1,19,200,100,100,100,100,,,0,,6.7,11.9
|
||||
p1-stop-r1,20,200,100,100,100,100,,,0,,6.7,9.6
|
||||
p1-stop-r1,21,200,100,100,100,100,,,0,,6.8,10.2
|
||||
p1-stop-r1,22,200,100,100,100,100,,,0,,6.8,8.7
|
||||
p1-stop-r1,23,200,100,100,100,100,,,0,,7.1,10.2
|
||||
p1-stop-r1,24,200,100,100,100,100,,,0,,7.2,8.2
|
||||
p1-stop-r1,25,200,100,100,100,100,,,0,,6.7,39.2
|
||||
p1-stop-r1,26,200,100,100,100,100,,,0,,6.5,7.5
|
||||
p1-stop-r1,27,200,100,100,100,100,,,0,,6.7,8.4
|
||||
p1-stop-r1,28,200,100,100,100,100,,,0,,7.3,10.4
|
||||
p1-stop-r1,29,200,100,100,100,100,,,0,,8.7,15.9
|
||||
p1-stop-r1,30,200,100,100,100,100,,,0,,6.5,7.8
|
||||
p1-stop-r1,31,200,100,100,100,100,,,0,,6.1,9.6
|
||||
p1-stop-r1,32,200,100,100,100,100,,,0,,6.1,7.0
|
||||
p1-stop-r1,33,200,100,100,100,100,,,0,,6.6,8.7
|
||||
p1-stop-r1,34,200,100,100,100,100,,,0,,6.8,13.4
|
||||
p1-stop-r1,35,6,3,3,3,3,,,0,,5.4,5.4
|
||||
p2-kill-r1,-10,188,94,94,94,94,,,0,,7.2,30.6
|
||||
p2-kill-r1,-9,200,100,100,100,100,,,0,,6.4,7.5
|
||||
p2-kill-r1,-8,200,100,100,100,100,,,0,,6.4,7.3
|
||||
p2-kill-r1,-7,200,100,100,100,100,,,0,,6.5,9.2
|
||||
p2-kill-r1,-6,200,100,100,100,100,,,0,,6.8,13.3
|
||||
p2-kill-r1,-5,200,100,100,100,100,,,0,,6.2,6.7
|
||||
p2-kill-r1,-4,200,100,100,100,100,,,0,,6.4,7.6
|
||||
p2-kill-r1,-3,200,100,100,100,100,,,0,,6.6,9.6
|
||||
p2-kill-r1,-2,200,100,100,100,100,,,0,,6.2,6.7
|
||||
p2-kill-r1,-1,200,99,100,99,100,,,1,IncompleteReadError:1,6.2,7.4
|
||||
p2-kill-r1,0,200,100,100,0,200,,,0,,507.0,536.1
|
||||
p2-kill-r1,1,200,100,100,0,200,,,0,,505.6,507.4
|
||||
p2-kill-r1,2,200,100,100,0,200,,,0,,7.8,10.0
|
||||
p2-kill-r1,3,200,100,100,0,200,,,0,,6.8,9.0
|
||||
p2-kill-r1,4,200,100,100,0,200,,,0,,7.7,13.3
|
||||
p2-kill-r1,5,200,100,100,0,200,,,0,,6.9,9.0
|
||||
p2-kill-r1,6,200,100,100,0,200,,,0,,7.0,9.6
|
||||
p2-kill-r1,7,200,100,100,0,200,,,0,,7.4,9.2
|
||||
p2-kill-r1,8,200,100,100,0,200,,,0,,7.4,9.4
|
||||
p2-kill-r1,9,200,100,100,0,200,,,0,,7.3,9.7
|
||||
p2-kill-r1,10,200,100,100,0,200,,,0,,7.6,9.7
|
||||
p2-kill-r1,11,200,100,100,0,200,,,0,,7.5,9.1
|
||||
p2-kill-r1,12,200,100,100,0,200,,,0,,8.3,9.4
|
||||
p2-kill-r1,13,200,100,100,0,200,,,0,,8.4,9.5
|
||||
p2-kill-r1,14,200,100,100,0,200,,,0,,8.1,11.4
|
||||
p2-kill-r1,15,200,100,100,0,200,,,0,,7.9,16.9
|
||||
p2-kill-r1,16,200,100,100,3,197,,,0,,6.5,67.9
|
||||
p2-kill-r1,17,200,100,100,100,100,,,0,,6.3,52.7
|
||||
p2-kill-r1,18,200,100,100,100,100,,,0,,6.4,7.1
|
||||
p2-kill-r1,19,200,100,100,100,100,,,0,,8.5,10.0
|
||||
p2-kill-r1,20,200,100,100,100,100,,,0,,7.1,10.0
|
||||
p2-kill-r1,21,200,100,100,100,100,,,0,,6.6,8.9
|
||||
p2-kill-r1,22,200,100,100,100,100,,,0,,6.5,8.3
|
||||
p2-kill-r1,23,200,100,100,100,100,,,0,,6.1,11.9
|
||||
p2-kill-r1,24,200,100,100,100,100,,,0,,6.6,34.6
|
||||
p2-kill-r1,25,200,100,100,100,100,,,0,,6.7,8.9
|
||||
p2-kill-r1,26,200,100,100,100,100,,,0,,6.3,8.2
|
||||
p2-kill-r1,27,200,100,100,100,100,,,0,,7.1,16.5
|
||||
p2-kill-r1,28,200,100,100,100,100,,,0,,6.3,6.9
|
||||
p2-kill-r1,29,200,100,100,100,100,,,0,,6.3,6.9
|
||||
p2-kill-r1,30,200,100,100,100,100,,,0,,6.8,8.2
|
||||
p2-kill-r1,31,200,100,100,100,100,,,0,,6.8,9.0
|
||||
p2-kill-r1,32,200,100,100,100,100,,,0,,6.9,8.6
|
||||
p2-kill-r1,33,200,100,100,100,100,,,0,,6.1,6.8
|
||||
p2-kill-r1,34,200,100,100,100,100,,,0,,6.9,14.5
|
||||
p2-kill-r1,35,12,6,6,6,6,,,0,,6.5,6.5
|
||||
p3-pause-r1,-10,196,98,98,98,98,,,0,,7.4,17.6
|
||||
p3-pause-r1,-9,200,100,100,100,100,,,0,,6.9,9.4
|
||||
p3-pause-r1,-8,200,100,100,100,100,,,0,,6.5,8.0
|
||||
p3-pause-r1,-7,200,100,100,100,100,,,0,,6.7,7.9
|
||||
p3-pause-r1,-6,200,100,100,100,100,,,0,,6.7,8.3
|
||||
p3-pause-r1,-5,200,100,100,100,100,,,0,,6.7,8.8
|
||||
p3-pause-r1,-4,200,100,100,100,100,,,0,,6.7,8.2
|
||||
p3-pause-r1,-3,200,100,100,100,100,,,0,,7.5,13.7
|
||||
p3-pause-r1,-2,200,100,100,100,100,,,0,,7.4,21.2
|
||||
p3-pause-r1,-1,200,100,99,99,100,,,1,504:1,6.6,7.1
|
||||
p3-pause-r1,0,200,98,2,0,100,,,100,504:100,8.5,16.3
|
||||
p3-pause-r1,1,200,99,1,0,100,,,100,504:100,7.2,24.5
|
||||
p3-pause-r1,2,200,96,4,0,100,,,100,504:100,8.0,20.2
|
||||
p3-pause-r1,3,200,99,1,0,100,,,100,504:100,8.6,42.7
|
||||
p3-pause-r1,4,200,97,3,0,100,,,100,504:100,7.6,18.0
|
||||
p3-pause-r1,5,200,99,17,5,111,,,84,504:84,30.5,9998.4
|
||||
p3-pause-r1,6,200,100,100,0,200,,,0,,7.2,9.2
|
||||
p3-pause-r1,7,200,100,100,0,200,,,0,,8.7,13.1
|
||||
p3-pause-r1,8,200,100,100,0,200,,,0,,7.6,9.9
|
||||
p3-pause-r1,9,200,100,100,0,200,,,0,,8.7,10.5
|
||||
p3-pause-r1,10,200,100,100,0,200,,,0,,7.2,8.2
|
||||
p3-pause-r1,11,200,100,100,0,200,,,0,,7.4,8.0
|
||||
p3-pause-r1,12,200,100,100,0,200,,,0,,6.8,8.1
|
||||
p3-pause-r1,13,200,100,100,0,200,,,0,,7.5,9.1
|
||||
p3-pause-r1,14,200,100,100,0,200,,,0,,7.5,8.5
|
||||
p3-pause-r1,15,200,100,100,0,200,,,0,,6.2,9.8
|
||||
p3-pause-r1,16,200,100,100,91,109,,,0,,6.6,7.3
|
||||
p3-pause-r1,17,200,100,100,100,100,,,0,,6.4,14.6
|
||||
p3-pause-r1,18,200,100,100,100,100,,,0,,6.4,7.2
|
||||
p3-pause-r1,19,200,100,100,100,100,,,0,,6.6,9.7
|
||||
p3-pause-r1,20,200,100,100,100,100,,,0,,6.9,17.5
|
||||
p3-pause-r1,21,200,100,100,100,100,,,0,,6.3,7.0
|
||||
p3-pause-r1,22,200,100,100,100,100,,,0,,6.9,9.5
|
||||
p3-pause-r1,23,200,100,100,100,100,,,0,,6.5,7.6
|
||||
p3-pause-r1,24,200,100,100,100,100,,,0,,6.3,7.1
|
||||
p3-pause-r1,25,200,100,100,100,100,,,0,,7.0,8.5
|
||||
p3-pause-r1,26,200,100,100,100,100,,,0,,6.6,10.3
|
||||
p3-pause-r1,27,200,100,100,100,100,,,0,,7.1,8.3
|
||||
p3-pause-r1,28,200,100,100,100,100,,,0,,6.8,16.1
|
||||
p3-pause-r1,29,200,100,100,100,100,,,0,,6.5,10.3
|
||||
p3-pause-r1,30,200,100,100,100,100,,,0,,6.9,15.4
|
||||
p3-pause-r1,31,200,100,100,100,100,,,0,,6.7,7.5
|
||||
p3-pause-r1,32,200,100,100,100,100,,,0,,6.5,7.6
|
||||
p3-pause-r1,33,200,100,100,100,100,,,0,,7.2,8.9
|
||||
p3-pause-r1,34,200,100,100,100,100,,,0,,6.7,8.1
|
||||
p3-pause-r1,35,4,2,2,2,2,,,0,,4.9,4.9
|
||||
p4-stop-r2,-10,192,96,96,96,96,,,0,,7.8,32.4
|
||||
p4-stop-r2,-9,200,100,100,100,100,,,0,,7.1,11.2
|
||||
p4-stop-r2,-8,200,100,100,100,100,,,0,,6.1,7.8
|
||||
p4-stop-r2,-7,200,100,100,100,100,,,0,,6.5,8.6
|
||||
p4-stop-r2,-6,200,100,100,100,100,,,0,,6.1,10.3
|
||||
p4-stop-r2,-5,200,100,100,100,100,,,0,,6.2,7.3
|
||||
p4-stop-r2,-4,200,100,100,100,100,,,0,,7.3,8.7
|
||||
p4-stop-r2,-3,200,100,100,100,100,,,0,,6.8,7.7
|
||||
p4-stop-r2,-2,200,100,100,100,100,,,0,,6.0,7.0
|
||||
p4-stop-r2,-1,200,100,100,100,100,,,0,,6.7,7.4
|
||||
p4-stop-r2,0,200,100,100,7,193,,,0,,506.4,511.7
|
||||
p4-stop-r2,1,200,100,100,0,200,,,0,,505.7,528.0
|
||||
p4-stop-r2,2,200,100,100,0,200,,,0,,7.0,9.0
|
||||
p4-stop-r2,3,200,100,100,0,200,,,0,,7.5,10.8
|
||||
p4-stop-r2,4,200,100,100,0,200,,,0,,7.8,10.3
|
||||
p4-stop-r2,5,200,100,100,0,200,,,0,,8.5,9.8
|
||||
p4-stop-r2,6,200,100,100,0,200,,,0,,7.0,9.0
|
||||
p4-stop-r2,7,200,100,100,0,200,,,0,,6.7,8.2
|
||||
p4-stop-r2,8,200,100,100,0,200,,,0,,6.9,15.2
|
||||
p4-stop-r2,9,200,100,100,0,200,,,0,,7.8,10.4
|
||||
p4-stop-r2,10,200,100,100,0,200,,,0,,8.5,9.9
|
||||
p4-stop-r2,11,200,100,100,0,200,,,0,,8.6,10.7
|
||||
p4-stop-r2,12,200,100,100,0,200,,,0,,7.2,8.3
|
||||
p4-stop-r2,13,200,100,100,0,200,,,0,,6.8,8.6
|
||||
p4-stop-r2,14,200,100,100,0,200,,,0,,6.6,8.9
|
||||
p4-stop-r2,15,200,100,100,0,200,,,0,,7.4,18.4
|
||||
p4-stop-r2,16,200,100,100,72,128,,,0,,6.7,31.1
|
||||
p4-stop-r2,17,200,100,100,100,100,,,0,,6.3,7.2
|
||||
p4-stop-r2,18,200,100,100,100,100,,,0,,6.1,7.3
|
||||
p4-stop-r2,19,200,100,100,100,100,,,0,,6.3,7.3
|
||||
p4-stop-r2,20,200,100,100,100,100,,,0,,6.3,8.9
|
||||
p4-stop-r2,21,200,100,100,100,100,,,0,,7.3,15.6
|
||||
p4-stop-r2,22,200,100,100,100,100,,,0,,6.5,8.3
|
||||
p4-stop-r2,23,200,100,100,100,100,,,0,,6.4,11.1
|
||||
p4-stop-r2,24,200,100,100,100,100,,,0,,6.5,30.3
|
||||
p4-stop-r2,25,200,100,100,100,100,,,0,,7.4,9.5
|
||||
p4-stop-r2,26,200,100,100,100,100,,,0,,7.8,9.5
|
||||
p4-stop-r2,27,200,100,100,100,100,,,0,,6.3,7.0
|
||||
p4-stop-r2,28,200,100,100,100,100,,,0,,5.9,17.8
|
||||
p4-stop-r2,29,200,100,100,100,100,,,0,,5.9,6.8
|
||||
p4-stop-r2,30,200,100,100,100,100,,,0,,6.2,6.7
|
||||
p4-stop-r2,31,200,100,100,100,100,,,0,,7.6,12.3
|
||||
p4-stop-r2,32,200,100,100,100,100,,,0,,6.3,12.0
|
||||
p4-stop-r2,33,200,100,100,100,100,,,0,,7.4,11.7
|
||||
p4-stop-r2,34,200,100,100,100,100,,,0,,6.2,6.6
|
||||
p4-stop-r2,35,8,4,4,4,4,,,0,,6.5,6.5
|
||||
p5-kill-r2,-10,192,96,96,96,96,,,0,,7.1,26.8
|
||||
p5-kill-r2,-9,200,100,100,100,100,,,0,,6.3,7.3
|
||||
p5-kill-r2,-8,200,100,100,100,100,,,0,,6.5,8.9
|
||||
p5-kill-r2,-7,200,100,100,100,100,,,0,,6.3,6.9
|
||||
p5-kill-r2,-6,200,100,100,100,100,,,0,,7.5,13.3
|
||||
p5-kill-r2,-5,200,100,100,100,100,,,0,,6.2,6.8
|
||||
p5-kill-r2,-4,200,100,100,100,100,,,0,,6.4,9.3
|
||||
p5-kill-r2,-3,200,100,100,100,100,,,0,,8.0,20.5
|
||||
p5-kill-r2,-2,200,100,100,100,100,,,0,,6.3,7.4
|
||||
p5-kill-r2,-1,200,100,100,100,100,,,0,,6.5,7.9
|
||||
p5-kill-r2,0,200,100,100,0,200,,,0,,506.2,507.9
|
||||
p5-kill-r2,1,200,100,100,0,200,,,0,,6.2,8.6
|
||||
p5-kill-r2,2,200,100,100,0,200,,,0,,7.3,8.0
|
||||
p5-kill-r2,3,200,100,100,0,200,,,0,,6.5,7.1
|
||||
p5-kill-r2,4,200,100,100,0,200,,,0,,7.9,10.4
|
||||
p5-kill-r2,5,200,100,100,0,200,,,0,,6.6,9.3
|
||||
p5-kill-r2,6,200,100,100,0,200,,,0,,6.8,8.7
|
||||
p5-kill-r2,7,200,100,100,0,200,,,0,,7.1,8.4
|
||||
p5-kill-r2,8,200,100,100,0,200,,,0,,7.5,15.0
|
||||
p5-kill-r2,9,200,100,100,0,200,,,0,,6.5,7.6
|
||||
p5-kill-r2,10,200,100,100,0,200,,,0,,7.8,9.9
|
||||
p5-kill-r2,11,200,100,100,0,200,,,0,,7.0,8.7
|
||||
p5-kill-r2,12,200,100,100,0,200,,,0,,7.7,9.7
|
||||
p5-kill-r2,13,200,100,100,0,200,,,0,,7.5,9.8
|
||||
p5-kill-r2,14,200,100,100,0,200,,,0,,8.2,13.2
|
||||
p5-kill-r2,15,200,100,100,0,200,,,0,,11.0,27.6
|
||||
p5-kill-r2,16,200,100,100,0,200,,,0,,6.6,10.1
|
||||
p5-kill-r2,17,200,100,100,88,112,,,0,,6.7,48.6
|
||||
p5-kill-r2,18,200,100,100,100,100,,,0,,6.3,7.0
|
||||
p5-kill-r2,19,200,100,100,100,100,,,0,,6.5,8.9
|
||||
p5-kill-r2,20,200,100,100,100,100,,,0,,6.7,7.3
|
||||
p5-kill-r2,21,200,100,100,100,100,,,0,,6.2,6.6
|
||||
p5-kill-r2,22,200,100,100,100,100,,,0,,6.6,7.3
|
||||
p5-kill-r2,23,200,100,100,100,100,,,0,,6.5,8.6
|
||||
p5-kill-r2,24,200,100,100,100,100,,,0,,6.8,34.1
|
||||
p5-kill-r2,25,200,100,100,100,100,,,0,,6.3,7.5
|
||||
p5-kill-r2,26,200,100,100,100,100,,,0,,6.8,8.8
|
||||
p5-kill-r2,27,200,100,100,100,100,,,0,,6.3,13.8
|
||||
p5-kill-r2,28,200,100,100,100,100,,,0,,7.3,9.0
|
||||
p5-kill-r2,29,200,100,100,100,100,,,0,,6.4,8.6
|
||||
p5-kill-r2,30,200,100,100,100,100,,,0,,6.3,9.3
|
||||
p5-kill-r2,31,200,100,100,100,100,,,0,,6.4,7.6
|
||||
p5-kill-r2,32,200,100,100,100,100,,,0,,5.9,6.8
|
||||
p5-kill-r2,33,200,100,100,100,100,,,0,,6.4,7.4
|
||||
p5-kill-r2,34,200,100,100,100,100,,,0,,7.4,10.8
|
||||
p5-kill-r2,35,8,4,4,4,4,,,0,,7.8,7.8
|
||||
p6-pause-r2,-10,190,95,95,95,95,,,0,,7.1,27.1
|
||||
p6-pause-r2,-9,200,100,100,100,100,,,0,,6.4,8.2
|
||||
p6-pause-r2,-8,200,100,100,100,100,,,0,,7.0,8.3
|
||||
p6-pause-r2,-7,200,100,100,100,100,,,0,,6.8,9.1
|
||||
p6-pause-r2,-6,200,100,100,100,100,,,0,,6.6,7.3
|
||||
p6-pause-r2,-5,200,100,100,100,100,,,0,,6.6,8.3
|
||||
p6-pause-r2,-4,200,100,100,100,100,,,0,,6.5,8.7
|
||||
p6-pause-r2,-3,200,100,100,100,100,,,0,,7.1,11.4
|
||||
p6-pause-r2,-2,200,100,100,100,100,,,0,,6.7,8.2
|
||||
p6-pause-r2,-1,200,99,100,99,100,,,1,504:1,7.1,7.8
|
||||
p6-pause-r2,0,200,2,98,0,100,,,100,504:100,7.6,22.7
|
||||
p6-pause-r2,1,200,0,100,0,100,,,100,504:100,6.6,7.6
|
||||
p6-pause-r2,2,200,1,99,0,100,,,100,504:100,7.9,9.0
|
||||
p6-pause-r2,3,200,0,100,0,100,,,100,504:100,6.8,7.8
|
||||
p6-pause-r2,4,200,3,97,0,100,,,100,504:100,6.9,23.8
|
||||
p6-pause-r2,5,200,47,100,47,100,,,53,504:53,9947.6,10000.6
|
||||
p6-pause-r2,6,200,100,100,6,194,,,0,,7.2,9530.3
|
||||
p6-pause-r2,7,200,100,100,0,200,,,0,,7.5,14.1
|
||||
p6-pause-r2,8,200,100,100,0,200,,,0,,7.5,13.9
|
||||
p6-pause-r2,9,200,100,100,0,200,,,0,,7.9,8.7
|
||||
p6-pause-r2,10,200,100,100,0,200,,,0,,6.7,11.8
|
||||
p6-pause-r2,11,200,100,100,0,200,,,0,,7.2,8.5
|
||||
p6-pause-r2,12,200,100,100,0,200,,,0,,6.8,7.7
|
||||
p6-pause-r2,13,200,100,100,0,200,,,0,,6.9,8.2
|
||||
p6-pause-r2,14,200,100,100,0,200,,,0,,7.2,8.2
|
||||
p6-pause-r2,15,200,100,100,0,200,,,0,,7.2,9.5
|
||||
p6-pause-r2,16,200,100,100,97,103,,,0,,6.8,8.2
|
||||
p6-pause-r2,17,200,100,100,100,100,,,0,,7.0,13.0
|
||||
p6-pause-r2,18,200,100,100,100,100,,,0,,5.9,6.6
|
||||
p6-pause-r2,19,200,100,100,100,100,,,0,,6.3,7.3
|
||||
p6-pause-r2,20,200,100,100,100,100,,,0,,7.6,15.7
|
||||
p6-pause-r2,21,200,100,100,100,100,,,0,,6.8,8.7
|
||||
p6-pause-r2,22,200,100,100,100,100,,,0,,6.9,7.5
|
||||
p6-pause-r2,23,200,100,100,100,100,,,0,,7.0,8.5
|
||||
p6-pause-r2,24,200,100,100,100,100,,,0,,6.4,8.5
|
||||
p6-pause-r2,25,200,100,100,100,100,,,0,,6.2,6.8
|
||||
p6-pause-r2,26,200,100,100,100,100,,,0,,6.6,7.4
|
||||
p6-pause-r2,27,200,100,100,100,100,,,0,,6.9,10.6
|
||||
p6-pause-r2,28,200,100,100,100,100,,,0,,7.6,15.4
|
||||
p6-pause-r2,29,200,100,100,100,100,,,0,,7.3,14.1
|
||||
p6-pause-r2,30,200,100,100,100,100,,,0,,6.8,14.6
|
||||
p6-pause-r2,31,200,100,100,100,100,,,0,,6.6,8.2
|
||||
p6-pause-r2,32,200,100,100,100,100,,,0,,6.5,7.0
|
||||
p6-pause-r2,33,200,100,100,100,100,,,0,,7.0,9.0
|
||||
p6-pause-r2,34,200,100,100,100,100,,,0,,7.1,9.4
|
||||
p6-pause-r2,35,10,5,5,5,5,,,0,,7.2,7.2
|
||||
|
Reference in New Issue
Block a user