date: 2026-11-05T21:59:08.897951+00:00 commit: 3fb741e6a84d2b187a0266488b018f0b684dcce5
env: COMPOSE_PROJECT_NAME=mtusi2026lab3-track5-20261105 LB_HTTP_PORT=18083 LB_STATS_PORT=23083 API_REPLICAS=2 NO_PROXY=localhost,127.0.0.1,::1 RATE=100

$ make check validate lint
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
.venv/bin/python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
.venv/bin/ruff check src student-tests alembic scripts/probe.py scripts/limits.py scripts/scaling.py tools
All checks passed!
.venv/bin/ruff format --check src student-tests alembic scripts/probe.py scripts/limits.py scripts/scaling.py tools
109 files already formatted
.venv/bin/ty check src student-tests alembic
All checks passed!
exit: 0

$ uv lock --check
Resolved 64 packages in 8ms
exit: 0

$ make student-test
.venv/bin/pytest
============================= test session starts ==============================
platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0
rootdir: <repo>
configfile: pyproject.toml
testpaths: student-tests
plugins: asyncio-1.4.0, anyio-4.15.1
asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function
collected 302 items / 12 deselected / 290 selected

student-tests/api/test_access.py .........                               [  3%]
student-tests/api/test_bounded_contexts.py ..................            [  9%]
student-tests/api/test_cache.py ......                                   [ 11%]
student-tests/api/test_concurrency.py ....                               [ 12%]
student-tests/api/test_consistency.py ........                           [ 15%]
student-tests/api/test_drain.py ..                                       [ 16%]
student-tests/api/test_events.py ......                                  [ 18%]
student-tests/api/test_idempotency.py ...............                    [ 23%]
student-tests/api/test_instance.py ....                                  [ 24%]
student-tests/api/test_migrations.py ......................              [ 32%]
student-tests/api/test_negative.py ..................................... [ 45%]
                                                                         [ 45%]
student-tests/api/test_operability.py ..............                     [ 50%]
student-tests/api/test_pagination.py ..................                  [ 56%]
student-tests/api/test_pool.py ............                              [ 60%]
student-tests/api/test_sessions.py ..................                    [ 66%]
student-tests/api/test_timeout.py ......                                 [ 68%]
student-tests/api/test_wire_format.py ..                                 [ 69%]
student-tests/unit/test_authentication.py .....................          [ 76%]
student-tests/unit/test_cursor.py ...........                            [ 80%]
student-tests/unit/test_domain.py .....................................  [ 93%]
student-tests/unit/test_probe.py ..............                          [ 97%]
student-tests/unit/test_scaling.py .....                                 [ 99%]
student-tests/unit/test_timeouts.py .                                    [100%]

===================== 290 passed, 12 deselected in 29.58s ======================
exit: 0

$ docker compose exec -T api python -c 'import sqlalchemy; from messenger.messaging.infrastructure.postgres import POST_MESSAGE; print("SQLAlchemy", sqlalchemy.__version__, "compiled cache key", POST_MESSAGE._generate_cache_key() is not None); assert POST_MESSAGE._generate_cache_key() is not None'
SQLAlchemy 2.0.45 compiled cache key True
exit: 0

$ make test
python3 tests/smoke.py --base-url "http://127.0.0.1:18083"
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
exit: 0

$ docker compose ps -aq
a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236
d4dfe6128c9d5ffd0f471d8dd3495146663fdfc126f23b795919ef18ee7d6624
cf56513187ef071b21df06da39f07f7042f5f84fe393b3760ec4bbc5b6433dac
87a881f5164136678a8904b634e6ee8cb55873c1cce8421bd11820671bf2afb5
e47f7b0129172b1fa6721915c4fbd78bed1944a92a868f64d2076fa89e85e609
16b45d1c308b4718beb5e389ded43c6ef802d873939bf981fceaa1de83309148
exit: 0

$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18083"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
exit: 0

$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18083"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 2.624}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
exit: 0

$ docker compose ps -a
NAME                                       IMAGE                                                                                                 COMMAND                  SERVICE    CREATED         STATUS                      PORTS
mtusi2026lab3-track5-20261105-api-11       mtusi-messenger:mtusi2026lab3-track5-20261105                                                         "python -m gunicorn …"   api        6 minutes ago   Exited (0) 12 seconds ago
mtusi2026lab3-track5-20261105-api-12       mtusi-messenger:mtusi2026lab3-track5-20261105                                                         "python -m gunicorn …"   api        6 minutes ago   Up 19 seconds (healthy)
mtusi2026lab3-track5-20261105-lb-1         haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b         "docker-entrypoint.s…"   lb         6 minutes ago   Up 20 seconds (healthy)     127.0.0.1:18083->8080/tcp, 127.0.0.1:23083->8404/tcp
mtusi2026lab3-track5-20261105-migrate-1    mtusi-messenger:mtusi2026lab3-track5-20261105                                                         "alembic upgrade head"   migrate    6 minutes ago   Exited (0) 19 seconds ago
mtusi2026lab3-track5-20261105-postgres-1   postgres:17.11-alpine@sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b         "docker-entrypoint.s…"   postgres   6 minutes ago   Up 21 seconds (healthy)
mtusi2026lab3-track5-20261105-valkey-1     valkey/valkey:8.1.11-alpine@sha256:2602ace2be6569bc2bbb06f873776516ab9089821525be34f4305c67eb9db02a   "tini -- docker-entr…"   valkey     6 minutes ago   Up 21 seconds (healthy)
exit: 0

$ make restore
scripts/restore.sh
 Container mtusi2026lab3-track5-20261105-api-12  Running
 Container mtusi2026lab3-track5-20261105-api-11  Starting
 Container mtusi2026lab3-track5-20261105-api-11  Started
 Container mtusi2026lab3-track5-20261105-api-11  Waiting
 Container mtusi2026lab3-track5-20261105-api-12  Waiting
 Container mtusi2026lab3-track5-20261105-api-11  Healthy
 Container mtusi2026lab3-track5-20261105-api-12  Healthy
exit: 0

$ python3 scripts/limits.py --out evidence/track1-sqlalchemy/revalidation/limits.txt
date: 2026-11-05T22:00:08Z  commit: 3fb741e
$ docker compose --profile bench config --format json  (declared)
service   cpus/mem    ports                            secrets                               volumes     networks                ro
--------  ----------  -------------------------------  ------------------------------------  ----------  ----------------------  -----
api       1/512MiB    -                                cache_url,cursor_secret,database_url  -           backend(internal)       true
k6        2/512MiB    -                                -                                     -           edge                    false
lb        1/128MiB    127.0.0.1:18083,127.0.0.1:23083  -                                     -           backend(internal),edge  true
migrate   0.5/256MiB  -                                database_url                          -           backend(internal)       false
postgres  2/1024MiB   -                                postgres_password                     pgdata      backend(internal)       false
valkey    0.5/128MiB  -                                valkey.conf                           valkeydata  backend(internal)       false

$ docker inspect  (running containers: hostname = X-Instance-Id, applied limits)
service   hostname      cpus  memory   published
--------  ------------  ----  -------  ---------------------------------------------------
api       a855a190991a  1     512MiB
api       d4dfe6128c9d  1     512MiB
lb        cf56513187ef  1     128MiB   127.0.0.1:18083->8080/tcp,127.0.0.1:23083->8404/tcp
postgres  e47f7b012917  2     1024MiB
valkey    16b45d1c308b  0.5   128MiB

PASS: limits, ports, secrets, volumes and networks match the policy
exit: 0

$ python3 scripts/probe.py check --rounds 20 --out evidence/track1-sqlalchemy/revalidation/lb-check.json
{
  "instances_one_token": {
    "a855a190991a": 20,
    "d4dfe6128c9d": 20
  },
  "same_key": {
    "rounds": 20,
    "parallel": 8,
    "statuses": {
      "201": 160
    },
    "instances": {
      "d4dfe6128c9d": 80,
      "a855a190991a": 80
    },
    "identical_bodies_rounds": 20,
    "single_copy_rounds": 20,
    "conflict_rounds": 20
  },
  "pass": true
}
exit: 0

$ python3 scripts/probe.py retry --out evidence/track1-sqlalchemy/revalidation/retry.json
{
  "variants": [
    {
      "variant": "timeout",
      "lost_attempt": {
        "status": 0,
        "client_timeout_s": 0.5,
        "chat_lock_held_s": 1.5,
        "replica": "a855a190991a",
        "replica_killed": false,
        "replica_answered": 201
      },
      "retry": {
        "after_s": 1.546,
        "status": 201,
        "attempts": 1,
        "replica": "d4dfe6128c9d"
      },
      "copies_in_history": 1,
      "same_message": true,
      "written_by": "lost attempt",
      "pass": true
    },
    {
      "variant": "kill",
      "lost_attempt": {
        "status": 0,
        "client_timeout_s": 0.5,
        "chat_lock_held_s": 1.5,
        "replica": "d4dfe6128c9d",
        "replica_killed": true,
        "replica_answered": null
      },
      "retry": {
        "after_s": 1.536,
        "status": 201,
        "attempts": 1,
        "replica": "a855a190991a"
      },
      "copies_in_history": 1,
      "same_message": true,
      "written_by": "lost attempt",
      "pass": true
    }
  ],
  "pass": true
}
exit: 0

$ sh scripts/detection.sh evidence/track1-sqlalchemy/revalidation/detection/run1
 Container mtusi2026lab3-track5-20261105-api-11  Running
 Container mtusi2026lab3-track5-20261105-api-12  Running
 Container mtusi2026lab3-track5-20261105-api-11  Waiting
 Container mtusi2026lab3-track5-20261105-api-12  Waiting
 Container mtusi2026lab3-track5-20261105-api-11  Healthy
 Container mtusi2026lab3-track5-20261105-api-12  Healthy
a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236
 Container mtusi2026lab3-track5-20261105-api-12  Running
 Container mtusi2026lab3-track5-20261105-api-11  Starting
 Container mtusi2026lab3-track5-20261105-api-11  Started
 Container mtusi2026lab3-track5-20261105-api-11  Waiting
 Container mtusi2026lab3-track5-20261105-api-12  Waiting
 Container mtusi2026lab3-track5-20261105-api-11  Healthy
 Container mtusi2026lab3-track5-20261105-api-12  Healthy
{
  "action": [
    "docker",
    "stop",
    "-t",
    "7",
    "a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236"
  ],
  "action_exit": 0,
  "action_seconds": 2.632,
  "lb_up_before": 2,
  "lb_samples_missing": 0,
  "lb_up_counted_as_detected": 1,
  "detection_seconds": 0.899,
  "declared_max_seconds": 1.5,
  "measurement_slack_seconds": 0.25,
  "lb_answer_after_detection": {
    "path": "/health/live",
    "status": 200,
    "x_instance_id_present": true,
    "error_code": null
  },
  "restore": "scripts/restore.sh",
  "restore_seconds": 3.602,
  "lb_up_after_restore": 2,
  "checks": {
    "action_ok": true,
    "lb_observed": true,
    "detected_in_time": true,
    "lb_answer_ok": true,
    "excluded_before_action_end": true,
    "restored": true
  },
  "pass": true
}
a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236
 Container mtusi2026lab3-track5-20261105-api-12  Running
 Container mtusi2026lab3-track5-20261105-api-11  Starting
 Container mtusi2026lab3-track5-20261105-api-11  Started
 Container mtusi2026lab3-track5-20261105-api-11  Waiting
 Container mtusi2026lab3-track5-20261105-api-12  Waiting
 Container mtusi2026lab3-track5-20261105-api-11  Healthy
 Container mtusi2026lab3-track5-20261105-api-12  Healthy
{
  "action": [
    "docker",
    "kill",
    "a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236"
  ],
  "action_exit": 0,
  "action_seconds": 0.115,
  "lb_up_before": 2,
  "lb_samples_missing": 0,
  "lb_up_counted_as_detected": 1,
  "detection_seconds": 0.273,
  "declared_max_seconds": 1.5,
  "measurement_slack_seconds": 0.25,
  "lb_answer_after_detection": {
    "path": "/health/live",
    "status": 200,
    "x_instance_id_present": true,
    "error_code": null
  },
  "restore": "scripts/restore.sh",
  "restore_seconds": 3.056,
  "lb_up_after_restore": 2,
  "checks": {
    "action_ok": true,
    "lb_observed": true,
    "detected_in_time": true,
    "lb_answer_ok": true,
    "restored": true
  },
  "pass": true
}
a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236
a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236
{
  "action": [
    "docker",
    "pause",
    "a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236"
  ],
  "action_exit": 0,
  "action_seconds": 0.058,
  "lb_up_before": 2,
  "lb_samples_missing": 0,
  "lb_up_counted_as_detected": 1,
  "detection_seconds": 3.073,
  "declared_max_seconds": 3.5,
  "measurement_slack_seconds": 0.25,
  "lb_answer_after_detection": {
    "path": "/health/live",
    "status": 200,
    "x_instance_id_present": true,
    "error_code": null
  },
  "restore": "docker unpause a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236 && scripts/lb-wait.sh",
  "restore_seconds": 2.399,
  "lb_up_after_restore": 2,
  "checks": {
    "action_ok": true,
    "lb_observed": true,
    "detected_in_time": true,
    "lb_answer_ok": true,
    "restored": true
  },
  "pass": true
}
e47f7b0129172b1fa6721915c4fbd78bed1944a92a868f64d2076fa89e85e609
e47f7b0129172b1fa6721915c4fbd78bed1944a92a868f64d2076fa89e85e609
{
  "action": [
    "docker",
    "stop",
    "e47f7b0129172b1fa6721915c4fbd78bed1944a92a868f64d2076fa89e85e609"
  ],
  "action_exit": 0,
  "action_seconds": 0.163,
  "lb_up_before": 2,
  "lb_samples_missing": 0,
  "lb_up_counted_as_detected": 0,
  "detection_seconds": 1.201,
  "declared_max_seconds": 1.5,
  "measurement_slack_seconds": 0.25,
  "lb_answer_after_detection": {
    "path": "/health/live",
    "status": 503,
    "x_instance_id_present": false,
    "error_code": "unavailable"
  },
  "restore": "docker start e47f7b0129172b1fa6721915c4fbd78bed1944a92a868f64d2076fa89e85e609 && scripts/lb-wait.sh",
  "restore_seconds": 2.428,
  "lb_up_after_restore": 2,
  "checks": {
    "action_ok": true,
    "lb_observed": true,
    "detected_in_time": true,
    "lb_answer_ok": true,
    "restored": true
  },
  "pass": true
}
exit: 0

$ sh scripts/detection.sh evidence/track1-sqlalchemy/revalidation/detection/run2
 Container mtusi2026lab3-track5-20261105-api-11  Running
 Container mtusi2026lab3-track5-20261105-api-12  Running
 Container mtusi2026lab3-track5-20261105-api-11  Waiting
 Container mtusi2026lab3-track5-20261105-api-12  Waiting
 Container mtusi2026lab3-track5-20261105-api-11  Healthy
 Container mtusi2026lab3-track5-20261105-api-12  Healthy
a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236
 Container mtusi2026lab3-track5-20261105-api-12  Running
 Container mtusi2026lab3-track5-20261105-api-11  Starting
 Container mtusi2026lab3-track5-20261105-api-11  Started
 Container mtusi2026lab3-track5-20261105-api-11  Waiting
 Container mtusi2026lab3-track5-20261105-api-12  Waiting
 Container mtusi2026lab3-track5-20261105-api-11  Healthy
 Container mtusi2026lab3-track5-20261105-api-12  Healthy
{
  "action": [
    "docker",
    "stop",
    "-t",
    "7",
    "a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236"
  ],
  "action_exit": 0,
  "action_seconds": 2.575,
  "lb_up_before": 2,
  "lb_samples_missing": 0,
  "lb_up_counted_as_detected": 1,
  "detection_seconds": 1.326,
  "declared_max_seconds": 1.5,
  "measurement_slack_seconds": 0.25,
  "lb_answer_after_detection": {
    "path": "/health/live",
    "status": 200,
    "x_instance_id_present": true,
    "error_code": null
  },
  "restore": "scripts/restore.sh",
  "restore_seconds": 2.5,
  "lb_up_after_restore": 2,
  "checks": {
    "action_ok": true,
    "lb_observed": true,
    "detected_in_time": true,
    "lb_answer_ok": true,
    "excluded_before_action_end": true,
    "restored": true
  },
  "pass": true
}
a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236
 Container mtusi2026lab3-track5-20261105-api-12  Running
 Container mtusi2026lab3-track5-20261105-api-11  Starting
 Container mtusi2026lab3-track5-20261105-api-11  Started
 Container mtusi2026lab3-track5-20261105-api-11  Waiting
 Container mtusi2026lab3-track5-20261105-api-12  Waiting
 Container mtusi2026lab3-track5-20261105-api-11  Healthy
 Container mtusi2026lab3-track5-20261105-api-12  Healthy
{
  "action": [
    "docker",
    "kill",
    "a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236"
  ],
  "action_exit": 0,
  "action_seconds": 0.163,
  "lb_up_before": 2,
  "lb_samples_missing": 0,
  "lb_up_counted_as_detected": 1,
  "detection_seconds": 0.335,
  "declared_max_seconds": 1.5,
  "measurement_slack_seconds": 0.25,
  "lb_answer_after_detection": {
    "path": "/health/live",
    "status": 200,
    "x_instance_id_present": true,
    "error_code": null
  },
  "restore": "scripts/restore.sh",
  "restore_seconds": 1.94,
  "lb_up_after_restore": 2,
  "checks": {
    "action_ok": true,
    "lb_observed": true,
    "detected_in_time": true,
    "lb_answer_ok": true,
    "restored": true
  },
  "pass": true
}
a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236
a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236
{
  "action": [
    "docker",
    "pause",
    "a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236"
  ],
  "action_exit": 0,
  "action_seconds": 0.058,
  "lb_up_before": 2,
  "lb_samples_missing": 0,
  "lb_up_counted_as_detected": 1,
  "detection_seconds": 2.783,
  "declared_max_seconds": 3.5,
  "measurement_slack_seconds": 0.25,
  "lb_answer_after_detection": {
    "path": "/health/live",
    "status": 200,
    "x_instance_id_present": true,
    "error_code": null
  },
  "restore": "docker unpause a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236 && scripts/lb-wait.sh",
  "restore_seconds": 2.418,
  "lb_up_after_restore": 2,
  "checks": {
    "action_ok": true,
    "lb_observed": true,
    "detected_in_time": true,
    "lb_answer_ok": true,
    "restored": true
  },
  "pass": true
}
e47f7b0129172b1fa6721915c4fbd78bed1944a92a868f64d2076fa89e85e609
e47f7b0129172b1fa6721915c4fbd78bed1944a92a868f64d2076fa89e85e609
{
  "action": [
    "docker",
    "stop",
    "e47f7b0129172b1fa6721915c4fbd78bed1944a92a868f64d2076fa89e85e609"
  ],
  "action_exit": 0,
  "action_seconds": 0.158,
  "lb_up_before": 2,
  "lb_samples_missing": 0,
  "lb_up_counted_as_detected": 0,
  "detection_seconds": 1.203,
  "declared_max_seconds": 1.5,
  "measurement_slack_seconds": 0.25,
  "lb_answer_after_detection": {
    "path": "/health/live",
    "status": 503,
    "x_instance_id_present": false,
    "error_code": "unavailable"
  },
  "restore": "docker start e47f7b0129172b1fa6721915c4fbd78bed1944a92a868f64d2076fa89e85e609 && scripts/lb-wait.sh",
  "restore_seconds": 2.448,
  "lb_up_after_restore": 2,
  "checks": {
    "action_ok": true,
    "lb_observed": true,
    "detected_in_time": true,
    "lb_answer_ok": true,
    "restored": true
  },
  "pass": true
}
exit: 0

$ sh scripts/detection.sh evidence/track1-sqlalchemy/revalidation/detection/run3
 Container mtusi2026lab3-track5-20261105-api-11  Running
 Container mtusi2026lab3-track5-20261105-api-12  Running
 Container mtusi2026lab3-track5-20261105-api-11  Waiting
 Container mtusi2026lab3-track5-20261105-api-12  Waiting
 Container mtusi2026lab3-track5-20261105-api-11  Healthy
 Container mtusi2026lab3-track5-20261105-api-12  Healthy
a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236
 Container mtusi2026lab3-track5-20261105-api-12  Running
 Container mtusi2026lab3-track5-20261105-api-11  Starting
 Container mtusi2026lab3-track5-20261105-api-11  Started
 Container mtusi2026lab3-track5-20261105-api-11  Waiting
 Container mtusi2026lab3-track5-20261105-api-12  Waiting
 Container mtusi2026lab3-track5-20261105-api-11  Healthy
 Container mtusi2026lab3-track5-20261105-api-12  Healthy
{
  "action": [
    "docker",
    "stop",
    "-t",
    "7",
    "a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236"
  ],
  "action_exit": 0,
  "action_seconds": 2.509,
  "lb_up_before": 2,
  "lb_samples_missing": 0,
  "lb_up_counted_as_detected": 1,
  "detection_seconds": 1.283,
  "declared_max_seconds": 1.5,
  "measurement_slack_seconds": 0.25,
  "lb_answer_after_detection": {
    "path": "/health/live",
    "status": 200,
    "x_instance_id_present": true,
    "error_code": null
  },
  "restore": "scripts/restore.sh",
  "restore_seconds": 3.531,
  "lb_up_after_restore": 2,
  "checks": {
    "action_ok": true,
    "lb_observed": true,
    "detected_in_time": true,
    "lb_answer_ok": true,
    "excluded_before_action_end": true,
    "restored": true
  },
  "pass": true
}
a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236
 Container mtusi2026lab3-track5-20261105-api-12  Running
 Container mtusi2026lab3-track5-20261105-api-11  Starting
 Container mtusi2026lab3-track5-20261105-api-11  Started
 Container mtusi2026lab3-track5-20261105-api-11  Waiting
 Container mtusi2026lab3-track5-20261105-api-12  Waiting
 Container mtusi2026lab3-track5-20261105-api-11  Healthy
 Container mtusi2026lab3-track5-20261105-api-12  Healthy
{
  "action": [
    "docker",
    "kill",
    "a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236"
  ],
  "action_exit": 0,
  "action_seconds": 0.163,
  "lb_up_before": 2,
  "lb_samples_missing": 0,
  "lb_up_counted_as_detected": 1,
  "detection_seconds": 0.795,
  "declared_max_seconds": 1.5,
  "measurement_slack_seconds": 0.25,
  "lb_answer_after_detection": {
    "path": "/health/live",
    "status": 200,
    "x_instance_id_present": true,
    "error_code": null
  },
  "restore": "scripts/restore.sh",
  "restore_seconds": 2.474,
  "lb_up_after_restore": 2,
  "checks": {
    "action_ok": true,
    "lb_observed": true,
    "detected_in_time": true,
    "lb_answer_ok": true,
    "restored": true
  },
  "pass": true
}
a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236
a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236
{
  "action": [
    "docker",
    "pause",
    "a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236"
  ],
  "action_exit": 0,
  "action_seconds": 0.054,
  "lb_up_before": 2,
  "lb_samples_missing": 0,
  "lb_up_counted_as_detected": 1,
  "detection_seconds": 3.281,
  "declared_max_seconds": 3.5,
  "measurement_slack_seconds": 0.25,
  "lb_answer_after_detection": {
    "path": "/health/live",
    "status": 200,
    "x_instance_id_present": true,
    "error_code": null
  },
  "restore": "docker unpause a855a190991aa22e198214a7fbc8dc11cf73f0867be70f961c160188e62ea236 && scripts/lb-wait.sh",
  "restore_seconds": 2.878,
  "lb_up_after_restore": 2,
  "checks": {
    "action_ok": true,
    "lb_observed": true,
    "detected_in_time": true,
    "lb_answer_ok": true,
    "restored": true
  },
  "pass": true
}
e47f7b0129172b1fa6721915c4fbd78bed1944a92a868f64d2076fa89e85e609
e47f7b0129172b1fa6721915c4fbd78bed1944a92a868f64d2076fa89e85e609
{
  "action": [
    "docker",
    "stop",
    "e47f7b0129172b1fa6721915c4fbd78bed1944a92a868f64d2076fa89e85e609"
  ],
  "action_exit": 0,
  "action_seconds": 0.168,
  "lb_up_before": 2,
  "lb_samples_missing": 0,
  "lb_up_counted_as_detected": 0,
  "detection_seconds": 1.255,
  "declared_max_seconds": 1.5,
  "measurement_slack_seconds": 0.25,
  "lb_answer_after_detection": {
    "path": "/health/live",
    "status": 503,
    "x_instance_id_present": false,
    "error_code": "unavailable"
  },
  "restore": "docker start e47f7b0129172b1fa6721915c4fbd78bed1944a92a868f64d2076fa89e85e609 && scripts/lb-wait.sh",
  "restore_seconds": 2.424,
  "lb_up_after_restore": 2,
  "checks": {
    "action_ok": true,
    "lb_observed": true,
    "detected_in_time": true,
    "lb_answer_ok": true,
    "restored": true
  },
  "pass": true
}
exit: 0

$ sh scripts/failover-each.sh evidence/track1-sqlalchemy/revalidation/failover-each
 Container mtusi2026lab3-track5-20261105-api-11  Running
 Container mtusi2026lab3-track5-20261105-api-12  Running
 Container mtusi2026lab3-track5-20261105-api-11  Waiting
 Container mtusi2026lab3-track5-20261105-api-12  Waiting
 Container mtusi2026lab3-track5-20261105-api-11  Healthy
 Container mtusi2026lab3-track5-20261105-api-12  Healthy
{"label": "a855a190991a", "action": ["sh", "-c", "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' api && { scripts/stand-usage.sh & sleep 10; wait; } && scripts/restore.sh"], "action_exit": 0, "action_seconds": 14.586, "action_finished_before_load_end": true, "lb_up_before": 2, "lb_up_min": 1, "lb_samples_missing": 0, "detection_seconds": 1.018, "rejoin_seconds": 14.48, "probe": {"attempts": 661, "failed_attempts": 0, "first_failure_s": null, "first_success_s": 0.058, "recovery_s": 0.0, "instances_before": {"d4dfe6128c9d": 112, "a855a190991a": 127}, "instances_during_outage": {"d4dfe6128c9d": 124}, "instances_after": {"a855a190991a": 134, "d4dfe6128c9d": 153}}, "history": {"confirmed": 661, "stored": 661, "exact_match": true, "unique": true}, "lb_counters": {"wretr": 0, "wredis": 0, "econ": 0, "eresp": 0, "hrsp_5xx": 0}, "k6": {"requests": 6000, "rps": 100.0, "ok_rps": 100.0, "failed_ratio": 0, "dropped_iterations": 0, "p50_ms": 2.51, "p95_ms": 4.33, "p99_ms": 5.15, "exit": 0, "rate": 100}, "checks": {"action_ok": true, "load_errors_ok": true, "load_delivered": true, "lb_observed": true, "write_recovery_ok": true, "history_preserved": true, "detected": true, "target_excluded": true, "rejoined": true}, "pass": true}
 Container mtusi2026lab3-track5-20261105-api-11  Running
 Container mtusi2026lab3-track5-20261105-api-12  Running
 Container mtusi2026lab3-track5-20261105-api-11  Waiting
 Container mtusi2026lab3-track5-20261105-api-12  Waiting
 Container mtusi2026lab3-track5-20261105-api-11  Healthy
 Container mtusi2026lab3-track5-20261105-api-12  Healthy
{"label": "d4dfe6128c9d", "action": ["sh", "-c", "scripts/stop-one.sh && docker compose ps -a --format '{{.ID}} {{.Service}} {{.Status}}' api && { scripts/stand-usage.sh & sleep 10; wait; } && scripts/restore.sh"], "action_exit": 0, "action_seconds": 16.248, "action_finished_before_load_end": true, "lb_up_before": 2, "lb_up_min": 1, "lb_samples_missing": 0, "detection_seconds": 1.145, "rejoin_seconds": 16.127, "probe": {"attempts": 661, "failed_attempts": 0, "first_failure_s": null, "first_success_s": 0.096, "recovery_s": 0.0, "instances_before": {"a855a190991a": 120, "d4dfe6128c9d": 120}, "instances_during_outage": {"a855a190991a": 138}, "instances_after": {"a855a190991a": 132, "d4dfe6128c9d": 139}}, "history": {"confirmed": 661, "stored": 661, "exact_match": true, "unique": true}, "lb_counters": {"wretr": 0, "wredis": 0, "econ": 0, "eresp": 0, "hrsp_5xx": 0}, "k6": {"requests": 6001, "rps": 100.02, "ok_rps": 100.02, "failed_ratio": 0, "dropped_iterations": 0, "p50_ms": 2.5, "p95_ms": 4.39, "p99_ms": 5.23, "exit": 0, "rate": 100}, "checks": {"action_ok": true, "load_errors_ok": true, "load_delivered": true, "lb_observed": true, "write_recovery_ok": true, "history_preserved": true, "detected": true, "target_excluded": true, "rejoined": true}, "pass": true}
 Container mtusi2026lab3-track5-20261105-api-11  Running
 Container mtusi2026lab3-track5-20261105-api-12  Running
 Container mtusi2026lab3-track5-20261105-api-11  Waiting
 Container mtusi2026lab3-track5-20261105-api-12  Waiting
 Container mtusi2026lab3-track5-20261105-api-11  Healthy
 Container mtusi2026lab3-track5-20261105-api-12  Healthy
exit: 0

$ docker compose up -d --no-deps --force-recreate --wait api
env override: {'API_IMAGE': 'mtusi-messenger:5c8f605cd3bd40245109348463d3e7111379b2d3'}
 Container mtusi2026lab3-track5-20261105-api-11  Recreate
 Container mtusi2026lab3-track5-20261105-api-12  Recreate
 Container mtusi2026lab3-track5-20261105-api-11  Recreated
 Container mtusi2026lab3-track5-20261105-api-12  Recreated
 Container mtusi2026lab3-track5-20261105-api-11  Starting
 Container mtusi2026lab3-track5-20261105-api-11  Started
 Container mtusi2026lab3-track5-20261105-api-12  Starting
 Container mtusi2026lab3-track5-20261105-api-12  Started
 Container mtusi2026lab3-track5-20261105-api-11  Waiting
 Container mtusi2026lab3-track5-20261105-api-12  Waiting
 Container mtusi2026lab3-track5-20261105-api-11  Healthy
 Container mtusi2026lab3-track5-20261105-api-12  Healthy
exit: 0

$ scripts/lb-wait.sh
exit: 0

$ python3 scripts/probe.py load --expect replace --duration 90 --out evidence/track1-sqlalchemy/revalidation/rolling -- scripts/rolling-update.sh
{"label": "action", "action": ["scripts/rolling-update.sh"], "action_exit": 0, "action_seconds": 11.16, "action_finished_before_load_end": true, "lb_up_before": 2, "lb_up_min": 2, "lb_samples_missing": 0, "detection_seconds": null, "rejoin_seconds": null, "probe": {"attempts": 938, "failed_attempts": 0, "first_failure_s": null, "first_success_s": 0.037, "recovery_s": 0.0, "instances_before": {"150b3322c7a1": 101, "e2760a6b2294": 136}, "instances_during_outage": {}, "instances_after": {"99fbbfb05387": 288, "a75dfd985b18": 309}}, "history": {"confirmed": 938, "stored": 938, "exact_match": true, "unique": true}, "lb_counters": {"wretr": 0, "wredis": 0, "econ": 0, "eresp": 0, "hrsp_5xx": 0}, "k6": {"requests": 9001, "rps": 100.01, "ok_rps": 100.01, "failed_ratio": 0, "dropped_iterations": 0, "p50_ms": 2.58, "p95_ms": 4.18, "p99_ms": 5.16, "exit": 0, "rate": 100}, "checks": {"action_ok": true, "load_errors_ok": true, "load_delivered": true, "lb_observed": true, "write_recovery_ok": true, "history_preserved": true, "capacity_kept": true, "all_replaced": true}, "pass": true}
exit: 0

$ make test-faults
BASE_URL="http://127.0.0.1:18083" .venv/bin/pytest -m stack -v -s student-tests/stack
============================= test session starts ==============================
platform darwin -- Python 3.14.5, pytest-9.1.1, pluggy-1.6.0 -- <repo>/.venv/bin/python
cachedir: .pytest_cache
rootdir: <repo>
configfile: pyproject.toml
plugins: asyncio-1.4.0, anyio-4.15.1
asyncio: mode=Mode.AUTO, debug=False, asyncio_default_fixture_loop_scope=function, asyncio_default_test_loop_scope=function
collecting ... collected 12 items

student-tests/stack/test_faults.py::test_registration_race_yields_exactly_one_user гонка 50 регистраций: 201×1, 409×49
PASSED
student-tests/stack/test_faults.py::test_chat_creation_rolls_back_on_unknown_member чат с несуществующим участником: 400 invalid_request
PASSED
student-tests/stack/test_faults.py::test_wrong_password_and_unknown_user_get_same_basic_challenge неверный пароль: 401 за 0.012 с
неизвестный пользователь: 401 за 0.013 с
PASSED
student-tests/stack/test_faults.py::test_logout_then_token_is_dead logout: 204, затем 401, повтор 401
PASSED
student-tests/stack/test_faults.py::test_valkey_serves_repeated_checks 5 проверок токена: keyspace_hits 26658 → 26663
PASSED
student-tests/stack/test_faults.py::test_valkey_stopped_is_bypassed_but_logout_waits valkey остановлен, действующий токен: 200 за 0.006 с
valkey остановлен, неизвестный токен: 401 за 0.004 с
valkey остановлен: ready 200, вход 201, выход 503
valkey снова запущен: повторный выход 204
PASSED
student-tests/stack/test_faults.py::test_valkey_restart_keeps_cache_and_revocations рестарт valkey: ключей 74 → 74
рестарт valkey: живой 200, отозванный 401
PASSED
student-tests/stack/test_faults.py::test_valkey_flushall_loses_nothing FLUSHALL: живой 200, отозванный 401
PASSED
student-tests/stack/test_faults.py::test_postgres_stopped_gives_bounded_503_and_recovers postgres остановлен, ready: 503 за 0.004 с
postgres остановлен, me: 503 за 0.003 с
postgres остановлен, stranger: 503 за 0.003 с
postgres остановлен, login: 503 за 0.002 с
postgres остановлен, register: 503 за 0.014 с
live внутри реплики 200; LB исключил все реплики ещё через 0.71 с
live через LB без реплик: 503 за 0.000 с
postgres запущен: готовность через 2.1 с, прежний токен 200
postgres запущен: 40 регистраций 201 через 2 реплики без рестарта API
PASSED
student-tests/stack/test_faults.py::test_api_restart_keeps_session_without_extending_it рестарт api: готовность через 1.8 с, токен 200
expires_at при входе 2026-11-05T23:05:45.791644Z, в БД после рестарта 2026-11-05T23:05:45.791644Z
PASSED
student-tests/stack/test_faults.py::test_full_restart_keeps_session_until_logout restart.sh: готовность через 0.0 с, прежний токен 200, выход 204, затем 401
PASSED
student-tests/stack/test_faults.py::test_session_expires_after_restart_at_original_time срок сессии сокращён до 20 с, рестарт api: готовность через 1.5 с, до срока 200, после срока 401
PASSED

============================= 12 passed in 48.52s ==============================
exit: 0

$ docker compose exec -T api python -c 'import sqlalchemy; from messenger.messaging.infrastructure.postgres import POST_MESSAGE; print("SQLAlchemy", sqlalchemy.__version__, "compiled cache key", POST_MESSAGE._generate_cache_key() is not None); assert POST_MESSAGE._generate_cache_key() is not None'
SQLAlchemy 2.0.45 compiled cache key True
exit: 0

$ make test
python3 tests/smoke.py --base-url "http://127.0.0.1:18083"
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
exit: 0

$ docker compose ps -a
NAME                                       IMAGE                                                                                                 COMMAND                  SERVICE    CREATED          STATUS                      PORTS
mtusi2026lab3-track5-20261105-api-13       mtusi-messenger:mtusi2026lab3-track5-20261105                                                         "python -m gunicorn …"   api        2 minutes ago    Up 19 seconds (healthy)
mtusi2026lab3-track5-20261105-api-14       mtusi-messenger:mtusi2026lab3-track5-20261105                                                         "python -m gunicorn …"   api        2 minutes ago    Up 19 seconds (healthy)
mtusi2026lab3-track5-20261105-lb-1         haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b         "docker-entrypoint.s…"   lb         12 minutes ago   Up 27 seconds (healthy)     127.0.0.1:18083->8080/tcp, 127.0.0.1:23083->8404/tcp
mtusi2026lab3-track5-20261105-migrate-1    mtusi-messenger:mtusi2026lab3-track5-20261105                                                         "alembic upgrade head"   migrate    12 minutes ago   Exited (0) 25 seconds ago
mtusi2026lab3-track5-20261105-postgres-1   postgres:17.11-alpine@sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b         "docker-entrypoint.s…"   postgres   12 minutes ago   Up 27 seconds (healthy)
mtusi2026lab3-track5-20261105-valkey-1     valkey/valkey:8.1.11-alpine@sha256:2602ace2be6569bc2bbb06f873776516ab9089821525be34f4305c67eb9db02a   "tini -- docker-entr…"   valkey     12 minutes ago   Up 27 seconds (healthy)
exit: 0
