date: 2026-11-06T08:20:27.740580+00:00 commit: b063c8780eabf63cdb9da3802f82440750308d6b
$ sh -c 'make test && python3 scripts/probe.py check --rounds 20 --out evidence/final-validation/lb-check.json && python3 scripts/probe.py retry --out evidence/final-validation/retry.json && make restore && make test-failover ACTION_SCRIPT=scripts/stop-one.sh && docker compose ps -a && make restore && curl -s -o /dev/null http://127.0.0.1:18083/health/live && make test-failover ACTION_SCRIPT=scripts/stop-one.sh && docker compose ps -a && make restore && make test-persistence ACTION_SCRIPT=scripts/restart.sh && python3 scripts/limits.py --out evidence/final-validation/limits.txt'
python3 tests/smoke.py --base-url "http://127.0.0.1:18083"
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
{
  "instances_one_token": {
    "ad1ce3e2864d": 20,
    "9aefe0725018": 20
  },
  "same_key": {
    "rounds": 20,
    "parallel": 8,
    "statuses": {
      "201": 160
    },
    "instances": {
      "9aefe0725018": 80,
      "ad1ce3e2864d": 80
    },
    "identical_bodies_rounds": 20,
    "single_copy_rounds": 20,
    "conflict_rounds": 20
  },
  "pass": true
}
{
  "variants": [
    {
      "variant": "timeout",
      "lost_attempt": {
        "status": 0,
        "client_timeout_s": 0.5,
        "chat_lock_held_s": 1.5,
        "replica": "ad1ce3e2864d",
        "replica_killed": false,
        "replica_answered": 201
      },
      "retry": {
        "after_s": 1.539,
        "status": 201,
        "attempts": 1,
        "replica": "9aefe0725018"
      },
      "copies_in_history": 1,
      "same_message": true,
      "written_by": "lost attempt",
      "pass": true
    },
    {
      "variant": "kill",
      "lost_attempt": {
        "status": 0,
        "client_timeout_s": 0.5,
        "chat_lock_held_s": 1.5,
        "replica": "9aefe0725018",
        "replica_killed": true,
        "replica_answered": null
      },
      "retry": {
        "after_s": 1.55,
        "status": 201,
        "attempts": 1,
        "replica": "ad1ce3e2864d"
      },
      "copies_in_history": 1,
      "same_message": true,
      "written_by": "lost attempt",
      "pass": true
    }
  ],
  "pass": true
}
scripts/restore.sh
 Container hlpipe-lab3-validation-20261106-api-1  Running
 Container hlpipe-lab3-validation-20261106-api-2  Running
 Container hlpipe-lab3-validation-20261106-api-2  Waiting
 Container hlpipe-lab3-validation-20261106-api-1  Waiting
 Container hlpipe-lab3-validation-20261106-api-2  Healthy
 Container hlpipe-lab3-validation-20261106-api-1  Healthy
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18083"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 132, "failed_attempts": 0, "first_recovery_seconds": 2.962}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
NAME                                         IMAGE                                                                                                 COMMAND                  SERVICE    CREATED              STATUS                          PORTS
hlpipe-lab3-validation-20261106-api-1        mtusi-messenger:hlpipe-lab3-validation-20261106                                                       "python -m gunicorn …"   api        About a minute ago   Exited (0) 12 seconds ago
hlpipe-lab3-validation-20261106-api-2        mtusi-messenger:hlpipe-lab3-validation-20261106                                                       "python -m gunicorn …"   api        About a minute ago   Up About a minute (healthy)
hlpipe-lab3-validation-20261106-lb-1         haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b         "docker-entrypoint.s…"   lb         About a minute ago   Up About a minute (healthy)     127.0.0.1:18083->8080/tcp, 127.0.0.1:23083->8404/tcp
hlpipe-lab3-validation-20261106-migrate-1    mtusi-messenger:hlpipe-lab3-validation-20261106                                                       "alembic upgrade head"   migrate    About a minute ago   Exited (0) About a minute ago
hlpipe-lab3-validation-20261106-postgres-1   postgres:17.11-alpine@sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b         "docker-entrypoint.s…"   postgres   About a minute ago   Up About a minute (healthy)
hlpipe-lab3-validation-20261106-valkey-1     valkey/valkey:8.1.11-alpine@sha256:2602ace2be6569bc2bbb06f873776516ab9089821525be34f4305c67eb9db02a   "tini -- docker-entr…"   valkey     About a minute ago   Up About a minute (healthy)
scripts/restore.sh
 Container hlpipe-lab3-validation-20261106-api-2  Running
 Container hlpipe-lab3-validation-20261106-api-1  Starting
 Container hlpipe-lab3-validation-20261106-api-1  Started
 Container hlpipe-lab3-validation-20261106-api-2  Waiting
 Container hlpipe-lab3-validation-20261106-api-1  Waiting
 Container hlpipe-lab3-validation-20261106-api-2  Healthy
 Container hlpipe-lab3-validation-20261106-api-1  Healthy
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:18083"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 132, "failed_attempts": 0, "first_recovery_seconds": 2.734}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
NAME                                         IMAGE                                                                                                 COMMAND                  SERVICE    CREATED              STATUS                          PORTS
hlpipe-lab3-validation-20261106-api-1        mtusi-messenger:hlpipe-lab3-validation-20261106                                                       "python -m gunicorn …"   api        About a minute ago   Up 19 seconds (healthy)
hlpipe-lab3-validation-20261106-api-2        mtusi-messenger:hlpipe-lab3-validation-20261106                                                       "python -m gunicorn …"   api        About a minute ago   Exited (0) 12 seconds ago
hlpipe-lab3-validation-20261106-lb-1         haproxy:3.2.26-alpine@sha256:5893840576071e6217cd52ebdcc14f454c6f7a631056fa6f4e0f40d3cac4215b         "docker-entrypoint.s…"   lb         About a minute ago   Up About a minute (healthy)     127.0.0.1:18083->8080/tcp, 127.0.0.1:23083->8404/tcp
hlpipe-lab3-validation-20261106-migrate-1    mtusi-messenger:hlpipe-lab3-validation-20261106                                                       "alembic upgrade head"   migrate    About a minute ago   Exited (0) About a minute ago
hlpipe-lab3-validation-20261106-postgres-1   postgres:17.11-alpine@sha256:266d3c257594e195b309ae9d1805e54c788f1d7b375f56bf6c367aac6c917e7b         "docker-entrypoint.s…"   postgres   About a minute ago   Up About a minute (healthy)
hlpipe-lab3-validation-20261106-valkey-1     valkey/valkey:8.1.11-alpine@sha256:2602ace2be6569bc2bbb06f873776516ab9089821525be34f4305c67eb9db02a   "tini -- docker-entr…"   valkey     About a minute ago   Up About a minute (healthy)
scripts/restore.sh
 Container hlpipe-lab3-validation-20261106-api-1  Running
 Container hlpipe-lab3-validation-20261106-api-2  Starting
 Container hlpipe-lab3-validation-20261106-api-2  Started
 Container hlpipe-lab3-validation-20261106-api-2  Waiting
 Container hlpipe-lab3-validation-20261106-api-1  Waiting
 Container hlpipe-lab3-validation-20261106-api-2  Healthy
 Container hlpipe-lab3-validation-20261106-api-1  Healthy
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:18083"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
date: 2026-11-06T08:21:22Z  commit: b063c87
$ docker compose --profile bench config --format json  (declared)
service   cpus/mem    ports                            secrets                               volumes     networks                ro
--------  ----------  -------------------------------  ------------------------------------  ----------  ----------------------  -----
api       1/512MiB    -                                cache_url,cursor_secret,database_url  -           backend(internal)       true
k6        2/512MiB    -                                -                                     -           edge                    false
lb        1/128MiB    127.0.0.1:18083,127.0.0.1:23083  -                                     -           backend(internal),edge  true
migrate   0.5/256MiB  -                                database_url                          -           backend(internal)       false
postgres  2/1024MiB   -                                postgres_password                     pgdata      backend(internal)       false
valkey    0.5/128MiB  -                                valkey.conf                           valkeydata  backend(internal)       false

$ docker inspect  (running containers: hostname = X-Instance-Id, applied limits)
service   hostname      cpus  memory   published
--------  ------------  ----  -------  ---------------------------------------------------
api       9aefe0725018  1     512MiB
api       ad1ce3e2864d  1     512MiB
lb        f7b001f9f369  1     128MiB   127.0.0.1:18083->8080/tcp,127.0.0.1:23083->8404/tcp
postgres  97a45d6b25a7  2     1024MiB
valkey    47b097b4ad8c  0.5   128MiB

PASS: limits, ports, secrets, volumes and networks match the policy

exit: 0
