# Lab 4: memory under 16 simultaneous uploads, reconciler faults + cost, load.py
# 2026-10-25T12:03:36Z, commit f9f5ab9 bench: сценарий прерванного apply на 2000 сирот, понятные подписи итогов

$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe bench/upload_memory.py --output evidence/memory
generating files ...
{"scenario": "png-9.4MiB x16, slots=4 (default)", "settings": {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}, "clients": 16, "s3_sampled_max_mib": 548.3, "s3_anon_after_mib": 386.7, "s3_limit_mib": 1024.0, "incidents": {}, "file_mib_each": 9.42, "types": ["image/png"], "status_codes": {"201": 16}, "error_reasons": {}, "wall_seconds": 4.37, "latency_s": {"p50": 3.62, "max": 4.34}, "replicas": {"f2c108a7e1b2": {"idle_mib": 75.3, "peak_mib": 156.3, "sampled_max_mib": 156.1, "python_vmhwm_mib": 152.6, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 81.0}, "2e1bacf102e6": {"idle_mib": 74.5, "peak_mib": 163.2, "sampled_max_mib": 161.7, "python_vmhwm_mib": 152.9, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 88.7}}}
{"scenario": "png-9.4MiB x32, slots=4 (default)", "settings": {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}, "clients": 32, "s3_sampled_max_mib": 800.5, "s3_anon_after_mib": 399.1, "s3_limit_mib": 1024.0, "incidents": {}, "file_mib_each": 9.42, "types": ["image/png"], "status_codes": {"201": 25, "503": 7}, "error_reasons": {"Слишком много одновременных загрузок, повторите позже": 7}, "wall_seconds": 6.96, "latency_s": {"p50": 4.99, "max": 6.91}, "replicas": {"5e7cbb39228d": {"idle_mib": 74.9, "peak_mib": 155.1, "sampled_max_mib": 153.6, "python_vmhwm_mib": 143.8, "uploads_served": 16, "limit_mib": 512.0, "peak_minus_idle_mib": 80.2}, "0036ead083c5": {"idle_mib": 74.6, "peak_mib": 187.5, "sampled_max_mib": 185.9, "python_vmhwm_mib": 181.2, "uploads_served": 16, "limit_mib": 512.0, "peak_minus_idle_mib": 112.9}}}
{"scenario": "png-9.4MiB x32, slots=16 (bound raised)", "settings": {"UPLOAD_CONCURRENCY": "16", "IMAGE_VERIFY_CONCURRENCY": "1"}, "clients": 32, "s3_sampled_max_mib": 1023.9, "s3_anon_after_mib": 411.8, "s3_limit_mib": 1024.0, "incidents": {}, "file_mib_each": 9.42, "types": ["image/png"], "status_codes": {"201": 24, "503": 8}, "error_reasons": {"Хранилище файлов временно недоступно": 8}, "wall_seconds": 10.2, "latency_s": {"p50": 7.46, "max": 10.04}, "replicas": {"04b50b0bdc7f": {"idle_mib": 75.2, "peak_mib": 284.6, "sampled_max_mib": 284.3, "python_vmhwm_mib": 285.9, "uploads_served": 16, "limit_mib": 512.0, "peak_minus_idle_mib": 209.4}, "f860d892caaf": {"idle_mib": 75.1, "peak_mib": 268.2, "sampled_max_mib": 265.8, "python_vmhwm_mib": 266.7, "uploads_served": 16, "limit_mib": 512.0, "peak_minus_idle_mib": 193.1}}}
{"scenario": "jpeg-40Mpx-progressive-4:4:4 x16, verify=1 (default)", "settings": {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}, "clients": 16, "s3_sampled_max_mib": 990.1, "s3_anon_after_mib": 386.0, "s3_limit_mib": 1024.0, "incidents": {}, "file_mib_each": 0.6, "types": ["image/jpeg"], "status_codes": {"201": 16}, "error_reasons": {}, "wall_seconds": 5.77, "latency_s": {"p50": 3.58, "max": 5.74}, "replicas": {"2c78d3ce98f5": {"idle_mib": 75.0, "peak_mib": 317.1, "sampled_max_mib": 315.5, "python_vmhwm_mib": 107.5, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 242.1}, "2220bc508f5c": {"idle_mib": 74.7, "peak_mib": 316.5, "sampled_max_mib": 315.5, "python_vmhwm_mib": 107.2, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 241.8}}}
{"scenario": "jpeg-40Mpx-progressive-4:4:4 x16, verify=4", "settings": {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "4"}, "clients": 16, "s3_sampled_max_mib": 995.3, "s3_anon_after_mib": 386.6, "s3_limit_mib": 1024.0, "incidents": {"mtusi-lab4-api-1": "restarts=0 oom_kill=6", "mtusi-lab4-api-2": "restarts=0 oom_kill=7"}, "file_mib_each": 0.6, "types": ["image/jpeg"], "status_codes": {"201": 2, "503": 14}, "error_reasons": {"Не хватило ресурсов для проверки изображения, повторите позже": 13, "Слишком много одновременных загрузок, повторите позже": 1}, "wall_seconds": 8.15, "latency_s": {"p50": 8.12, "max": 8.12}, "replicas": {"984d6a81b4b8": {"idle_mib": 86.2, "peak_mib": 512.0, "sampled_max_mib": 511.9, "python_vmhwm_mib": 110.7, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 425.8}, "42f112fb4559": {"idle_mib": 74.5, "peak_mib": 512.0, "sampled_max_mib": 507.9, "python_vmhwm_mib": 92.3, "uploads_served": 8, "limit_mib": 512.0, "peak_minus_idle_mib": 437.5}}}
{"date": "2026-10-25 15:08:28", "host": "Windows-10-10.0.19045-SP0", "cpus": 8, "commit": "f9f5ab9", "api_limits": "1000000000 536870912", "jpeg_file_mib": 0.6, "png_file_mib": 9.42}
[exit 0, 292.6 s]

$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe bench/reconcile_faults.py --output evidence/reconcile
{
 "dry_run_changed_nothing": true,
 "per_key": {
  "dry-run": {
   "healthy used A": null,
   "healthy unused B": null,
   "used, object deleted C": {
    "kind": "missing_object",
    "result": "planned",
    "used": true
   },
   "unused, object truncated D": {
    "kind": "size_mismatch",
    "result": "planned",
    "used": false
   },
   "quarantined but healthy E": {
    "kind": "restored_object",
    "result": "planned",
    "used": true
   },
   "same-size corruption F": {
    "kind": "checksum_mismatch",
    "result": "planned",
    "used": false
   },
   "old orphan object": {
    "kind": "orphan_object",
    "result": "planned",
    "used": false
   },
   "stale intent + object": {
    "kind": "stale_intent",
    "result": "planned",
    "used": false
   },
   "stale intent, no object": {
    "kind": "stale_intent",
    "result": "planned",
    "used": false
   },
   "fresh orphan (in-flight)": {
    "kind": "fresh_orphan_object",
    "result": "not_actionable",
    "used": false
   },
   "fresh intent (in-flight)": {
    "kind": "fresh_intent",
    "result": "not_actionable",
    "used": false
   }
  },
  "apply": {
   "healthy used A": null,
   "healthy unused B": null,
   "used, object deleted C": {
    "kind": "missing_object",
    "result": "applied",
    "used": true
   },
   "unused, object truncated D": {
    "kind": "size_mismatch",
    "result": "applied",
    "used": false
   },
   "quarantined but healthy E": {
    "kind": "restored_object",
    "result": "applied",
    "used": true
   },
   "same-size corruption F": {
    "kind": "checksum_mismatch",
    "result": "applied",
    "used": false
   },
   "old orphan object": {
    "kind": "orphan_object",
    "result": "applied",
    "used": false
   },
   "stale intent + object": {
    "kind": "stale_intent",
    "result": "applied",
    "used": false
   },
   "stale intent, no object": {
    "kind": "stale_intent",
    "result": "applied",
    "used": false
   },
   "fresh orphan (in-flight)": {
    "kind": "fresh_orphan_object",
    "result": "not_actionable",
    "used": false
   },
   "fresh intent (in-flight)": {
    "kind": "fresh_intent",
    "result": "not_actionable",
    "used": false
   }
  },
  "apply again": {
   "healthy used A": null,
   "healthy unused B": null,
   "used, object deleted C": null,
   "unused, object truncated D": null,
   "quarantined but healthy E": null,
   "same-size corruption F": null,
   "old orphan object": null,
   "stale intent + object": null,
   "stale intent, no object": null,
   "fresh orphan (in-flight)": {
    "kind": "fresh_orphan_object",
    "result": "not_actionable",
    "used": false
   },
   "fresh intent (in-flight)": {
    "kind": "fresh_intent",
    "result": "not_actionable",
    "used": false
   }
  },
  "after restore of C": {
   "healthy used A": null,
   "healthy unused B": null,
   "used, object deleted C": {
    "kind": "restored_object",
    "result": "applied",
    "used": true
   },
   "unused, object truncated D": null,
   "quarantined but healthy E": null,
   "same-size corruption F": null,
   "old orphan object": null,
   "stale intent + object": null,
   "stale intent, no object": null,
   "fresh orphan (in-flight)": {
    "kind": "orphan_object",
    "result": "applied",
    "used": false
   },
   "fresh intent (in-flight)": {
    "kind": "stale_intent",
    "result": "applied",
    "used": false
   }
  }
 },
 "objects_present_at_end (fresh items were older than --grace by the later runs)": {
  "healthy used A": true,
  "healthy unused B": true,
  "used, object deleted C": true,
  "unused, object truncated D": true,
  "quarantined but healthy E": true,
  "same-size corruption F": true,
  "old orphan object": false,
  "stale intent + object": false,
  "stale intent, no object": false,
  "fresh orphan (in-flight)": false,
  "fresh intent (in-flight)": false
 },
 "rows_after": {
  "C": {
   "status": "ready",
   "lost_reason": null
  },
  "D": {
   "status": "ready",
   "lost_reason": "size_mismatch"
  },
  "E": {
   "status": "ready",
   "lost_reason": null
  },
  "F": {
   "status": "ready",
   "lost_reason": "checksum_mismatch"
  },
  "A": {
   "status": "ready",
   "lost_reason": null
  },
  "B": {
   "status": "ready",
   "lost_reason": null
  }
 },
 "content_status_after": {
  "A": 200,
  "B": 200,
  "C": 200,
  "D": 404,
  "E": 200,
  "F": 404
 },
 "durations_s": [
  3.94,
  3.983,
  3.0,
  3.548
 ]
}
{
 "orphans_injected": 2000,
 "interrupted_run": {
  "exit_code": 1,
  "error": "store_unavailable: Unavailable",
  "applied": 440
 },
 "orphans_left_after_interrupt": 1560,
 "rerun": {
  "exit_code": 0,
  "applied": 1562,
  "error": null
 },
 "third": {
  "exit_code": 0,
  "planned": 0
 },
 "orphans_left_at_end": 0
}
[exit 0, 182.1 s]

$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\.venv\Scripts\python.exe bench/reconcile_cost.py --output evidence/reconcile
removed 0 synthetic attachments
Traceback (most recent call last):
  File "C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\bench\reconcile_cost.py", line 155, in <module>
    sys.exit(main())
             ~~~~^^
  File "C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\bench\reconcile_cost.py", line 127, in main
    add(target - present, owner, chat)
    ~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\bench\reconcile_cost.py", line 89, in add
    psql("INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, "
    ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
         "object_key, status, created_at, updated_at) "
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
         f"SELECT v.id::uuid, '{chat}', '{owner}', 'bench-reconcile', 'image/png', {len(OBJECT)}, "
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
         f"decode('{SHA}', 'hex'), 1, 1, v.k, 'ready', clock_timestamp() - interval '2 hours', "
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
         f"clock_timestamp() - interval '2 hours' FROM (VALUES {values}) AS v(id, k)")
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\bench\reconcile_cost.py", line 62, in psql
    return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-v",
           ~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
                   "ON_ERROR_STOP=1", "-c", sql).stdout.strip()
                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab4-clean\bench\reconcile_cost.py", line 57, in compose
    return subprocess.run(["docker", "compose", *args], cwd=ROOT, input=stdin, capture_output=True, text=True,
           ~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
                          encoding="utf-8", timeout=timeout, check=check)
                          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\Lib\subprocess.py", line 555, in run
    with Popen(*popenargs, **kwargs) as process:
         ~~~~~^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\Lib\subprocess.py", line 1039, in __init__
    self._execute_child(args, executable, preexec_fn, close_fds,
    ~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
                        pass_fds, cwd, env,
                        ^^^^^^^^^^^^^^^^^^^
    ...<5 lines>...
                        gid, gids, uid, umask,
                        ^^^^^^^^^^^^^^^^^^^^^^
                        start_new_session, process_group)
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\Lib\subprocess.py", line 1553, in _execute_child
    hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
                       ~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^
                             # no special security
                             ^^^^^^^^^^^^^^^^^^^^^
    ...<4 lines>...
                             cwd,
                             ^^^^
                             startupinfo)
                             ^^^^^^^^^^^^
FileNotFoundError: [WinError 206] Имя файла или его расширение имеет слишком большую длину
[exit 1, 1.3 s]

$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2
2 API replica(s) UP in HAProxy after 0.6 s: ['379d13a0775a', '862067d68c1c']
[exit 0, 1.4 s]

$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe tests/load.py --duration 30 --concurrency 4 --output evidence/load.json
{
  "lab": 4,
  "workload": "closed-loop POST messages; no automatic retries",
  "requested_duration_seconds": 30.0,
  "elapsed_seconds": 30.091,
  "concurrency": 4,
  "successful_requests": 5416,
  "failed_requests": 0,
  "error_ratio": 0.0,
  "successful_rps": 179.989,
  "successful_latency_ms": {
    "p50": 17.684,
    "p95": 49.053,
    "p99": 63.81
  },
  "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
[exit 0, 30.7 s]

$ docker compose ps -a --format table {{.Service}}	{{.Status}}
SERVICE    STATUS
api        Up 3 minutes (healthy)
api        Up 3 minutes (healthy)
lb         Up 24 minutes (healthy)
migrate    Exited (0) 10 minutes ago
postgres   Up 11 minutes (healthy)
s3         Up 6 minutes (healthy)
s3-init    Exited (0) 10 minutes ago
valkey     Up 12 minutes (healthy)
[exit 0, 0.4 s]
