lab5: очереди и асинхронная обработка изображений

Durable staging handoff, transactional outbox, at-least-once worker with lease fencing, idempotent versioned pipeline, three variants, operator tooling, contract tests and evidence. Includes the review fixes: reprocess source selection and failure settlement, replay guards, stale-job retirement, PNG structural/raster validation, migration rollback safety and audit events.
This commit is contained in:
Vladislav Kunetskiy
2026-10-31 14:34:24 +03:00
parent 6c5ba026f1
commit 01b885e846
99 changed files with 10164 additions and 1452 deletions
+165
View File
@@ -0,0 +1,165 @@
#!/usr/bin/env python3
"""Track 2 evidence: versioned pipeline, duplicate delivery, stale-version guard.
Runs N repetitions against the live stand and writes raw CSV/Markdown evidence.
This is a correctness harness, not a load test.
python3 scripts/lab5_track2.py --reps 5
"""
import argparse
import csv
import hashlib
import json
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
sys.path.insert(0, str(ROOT / 'student-tests'))
from client import Client, Failure, check # noqa: E402
from tests_helpers import compose, jobctl, psql # noqa: E402
FIXTURE = ROOT / 'tests/fixtures/sample.png'
EVIDENCE = ROOT / 'evidence/lab5/track2'
def wait_ready(client, actor, attachment, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
current = client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=actor, schema='Attachment')['json']
check(current['status'] != 'failed', 'attachment failed: ' + str(current.get('error_code')))
if current['status'] == 'ready':
return current
time.sleep(0.2)
raise Failure('attachment did not become ready')
def wait_version(attachment_id, version, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
row = psql("select pipeline_version || '|' || status from attachments where id='%s'" % attachment_id)
if row == str(version) + '|ready':
return
time.sleep(0.2)
raise Failure('attachment did not reach pipeline version %d' % version)
def variants_by_name(attachment):
return {v['name']: v for v in attachment['variants']}
def scenario_duplicate_before_effect(client, actor, chat):
compose('stop', 'worker')
try:
attached, _ = client.attachment(actor, chat)
time.sleep(1)
job = psql("select job_id from attachment_jobs where attachment_id='%s'" % attached['id'])
check(bool(job), 'job id missing')
jobctl('duplicate', '-job', job)
jobctl('duplicate', '-job', job)
finally:
compose('up', '-d', '--no-deps', 'worker', check=False)
ready = wait_ready(client, actor, attached)
names = sorted(variants_by_name(ready))
check(names == ['crop', 'optimized', 'thumbnail'], 'duplicate delivery changed the variant set')
check(psql("select count(*) from attachment_variants where attachment_id='%s'" % attached['id']) == '3',
'duplicate delivery created extra variant rows')
return 'duplicate_before_effect', True, 'attempts=' + psql(
"select attempt from attachment_jobs where attachment_id='%s'" % attached['id'])
def scenario_stale_version(client, actor, chat):
attached, content = client.attachment(actor, chat)
v1 = wait_ready(client, actor, attached)
v1_variants = variants_by_name(v1)
original_sha = hashlib.sha256(content).hexdigest()
check(v1['sha256'] == original_sha, 'original checksum changed before reprocess')
v1_job = psql("select job_id from attachment_jobs where attachment_id='%s' and pipeline_version=1" % attached['id'])
check(bool(v1_job), 'v1 job missing')
reprocess = jobctl('reprocess', '-attachment', attached['id'], '-pipeline', '2')
check(reprocess.returncode == 0, 'reprocess failed: ' + reprocess.stderr)
wait_version(attached['id'], 2)
v2 = wait_ready(client, actor, attached)
v2_variants = variants_by_name(v2)
check(sorted(v2_variants) == ['crop', 'optimized', 'thumbnail'], 'v2 variant set incomplete')
check(v2_variants['thumbnail']['sha256'] != v1_variants['thumbnail']['sha256'],
'v2 bytes must differ from v1 (pipeline parameters changed)')
check(v2['sha256'] == original_sha, 'original changed during reprocess')
before = json.dumps(v2_variants, sort_keys=True)
# Deliver the old v1 message after v2 is published. It must be acknowledged
# without touching the serving version or its artifacts.
jobctl('duplicate', '-job', v1_job)
time.sleep(2)
after_attachment = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=actor, schema='Attachment')['json']
after_variants = variants_by_name(after_attachment)
check(json.dumps(after_variants, sort_keys=True) == before, 'stale v1 delivery overwrote v2 variants')
check(psql("select pipeline_version from attachments where id='%s'" % attached['id']) == '2',
'serving pipeline version regressed')
check(psql("select status from attachment_jobs where job_id='%s'" % v1_job) == 'ready',
'v1 job state changed unexpectedly')
return 'stale_version_after_v2', True, 'v1_job=' + v1_job
def main():
parser = argparse.ArgumentParser(description='Lab5 track 2 correctness harness')
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--reps', type=int, default=3)
args = parser.parse_args()
EVIDENCE.mkdir(parents=True, exist_ok=True)
rows = []
client = Client(args.base_url, None)
for rep in range(1, args.reps + 1):
alice, _, _, chat = client.bootstrap()
for scenario in (scenario_duplicate_before_effect, scenario_stale_version):
name, passed, detail = scenario(client, alice, chat)
rows.append({'scenario': name, 'rep': rep, 'pass': int(passed), 'detail': detail})
print('D%s rep=%d: %s' % (name, rep, 'PASS' if passed else 'FAIL'))
with (EVIDENCE / 'runs.csv').open('w', newline='') as handle:
writer = csv.DictWriter(handle, fieldnames=['scenario', 'rep', 'pass', 'detail'])
writer.writeheader()
writer.writerows(rows)
summary = [
'# Track 2 — versioned pipeline and duplicate/stale delivery',
'',
'Hypothesis: with the effect key `(attachment_id, pipeline_version, variant)`,',
'deterministic object keys, a serving version that changes only when all v2',
'variants are committed, and stale-version acknowledgements, an at-least-once',
'pipeline loses no accepted work and never overwrites a newer result with an',
'older duplicate.',
'',
'Method: N=%d repetitions. Each repetition creates fresh users/chat and runs:' % args.reps,
'- duplicate pipeline publication while the worker is stopped, followed by recovery;',
'- v1 ready -> explicit reprocess to v2 -> old v1 message delivered after v2 ready.',
'',
'Invariants checked: unique variant rows per effect key; ready remains ready;',
'serving pipeline version never regresses; original bytes/checksum unchanged;',
'stale job state is not rewritten.',
'',
'Limitations: this harness covers duplicate and stale/reorder delivery. Crash',
'windows (before/after side effect, before/after ack, outbox gap) and lease',
'takeover are covered by student-tests/lab5_faults.py; both are single-node',
'Compose evidence, not a production availability claim. Delivery is',
'at-least-once; exactly-once is not claimed.',
]
(EVIDENCE / 'summary.md').write_text('\n'.join(summary) + '\n')
passed = sum(row['pass'] for row in rows)
print('PASS: track2 %d/%d scenarios' % (passed, len(rows)))
if __name__ == '__main__':
main()
+13 -7
View File
@@ -1,5 +1,5 @@
#!/bin/sh
# Проверка критерия «LB исключает неготовую реплику» (lab3).
# Проверка критерия «LB исключает неготовую реплику» (lab5).
#
# Рядом с рабочими репликами поднимается третья с тем же сетевым alias, но её
# доступ к БД идёт через временный TCP-прокси. Пока прокси жив, реплика готова и
@@ -23,10 +23,10 @@ if [ -f .env ]; then
set +a
fi
OUT=${OUT:-evidence/lb-readiness.txt}
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab4}
OUT=${OUT:-evidence/lab5/lb-readiness-detail.txt}
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab5}
NET=${PROJECT}_default
IMAGE=${IMAGE:-mtusi-messenger:lab4}
IMAGE=${IMAGE:-mtusi-messenger:lab5}
STATS=${STATS:-http://127.0.0.1:8404/stats;csv}
BASE_URL=${BASE_URL:-http://localhost:8080}
CANDIDATE=lb-readiness-candidate
@@ -77,7 +77,7 @@ wait_readiness() {
}
up_slots() {
curl -s --noproxy '*' "$STATS" | awk -F, '$1 ~ /^api/ && $18 == "UP" && $2 != "BACKEND" { printf "%s ", $2 }'
curl -s --noproxy '*' "$STATS" | awk -F, '$1 ~ /^api/ && $18 ~ /^UP/ && $2 != "BACKEND" { printf "%s ", $2 }'
}
wait_new_up_slot() {
@@ -120,7 +120,7 @@ wait_slot_status() {
}
{
echo "# LB excludes a not-ready but reachable replica (lab3)"
echo "# LB excludes a not-ready but reachable replica (lab5)"
echo "# generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
echo "# lb: $(docker inspect -f '{{.Config.Image}}' "$(docker ps -q --filter label=com.docker.compose.service=lb)" 2>/dev/null)"
echo
@@ -139,6 +139,11 @@ wait_slot_status() {
-e POSTGRES_USER="${POSTGRES_USER:-messenger}" \
-e POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-messenger}" -e POSTGRES_SSL_MODE=disable \
-e REDIS_ENABLED=true -e REDIS_HOST=redis -e CACHE_ENABLED=true \
-e S3_ENDPOINT=http://seaweedfs:8333 -e S3_REGION="${S3_REGION:-us-east-1}" \
-e S3_BUCKET="${S3_BUCKET:-messenger-attachments}" \
-e S3_ACCESS_KEY_ID="${S3_ACCESS_KEY_ID:-lab4app}" \
-e S3_SECRET_ACCESS_KEY="${S3_SECRET_ACCESS_KEY:-lab4app-secret}" \
-e S3_USE_PATH_STYLE=true \
"$IMAGE" >/dev/null
if ! wait_readiness 200; then
@@ -170,7 +175,8 @@ wait_slot_status() {
if wait_slot_status "$slot" DOWN; then
echo " $slot went DOWN ${SWAIT_SECONDS}s after its database was cut (check: inter 2s fall 2)"
else
echo " WARNING: $slot is still not DOWN" >&2
echo " FAIL: $slot is still not DOWN" >&2
exit 1
fi
echo " stats after failure: $(stats)"
+10 -10
View File
@@ -1,6 +1,6 @@
#!/bin/sh
set -eu
# Restart hook: bounce API replicas and every state store of the lab4 stand.
# Restart hook: bounce API replicas, worker/relay and every state store of the lab5 stand.
#
# scripts/restart.sh
#
@@ -29,16 +29,16 @@ compose() {
$COMPOSE_BIN --env-file .env -f compose.yaml "$@"
}
# 1. State stores: restart the running ones, then make sure they are healthy.
# Volumes are attached as before, nothing is dropped or re-seeded.
compose up -d --wait postgres redis seaweedfs
compose restart postgres redis seaweedfs
compose up -d --wait postgres redis seaweedfs
# 1. State stores and broker: restart the running ones, then make sure they are
# healthy. Volumes are attached as before, nothing is dropped or re-seeded.
compose up -d --wait postgres redis seaweedfs broker
compose restart postgres redis seaweedfs broker
compose up -d --wait postgres redis seaweedfs broker
# 2. API replicas: restart in place (same containers, same addresses) and bring
# back any replica that was stopped by the fault-injection hook.
compose restart api
compose up -d --wait --no-deps api
# 2. API replicas, worker and relay: restart in place and bring back any replica
# stopped by a fault-injection hook. The worker reconnects to the broker.
compose restart api worker relay
compose up -d --wait --no-deps api worker relay
# 3. Load balancer keeps its upstream; re-resolve in case addresses changed.
compose restart lb
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT_DIR"
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
$COMPOSE_BIN --env-file .env -f compose.yaml up -d --wait --no-deps broker
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT_DIR"
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
$COMPOSE_BIN --env-file .env -f compose.yaml up -d --wait --no-deps worker
+6
View File
@@ -0,0 +1,6 @@
#!/bin/sh
set -eu
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT_DIR"
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
$COMPOSE_BIN --env-file .env -f compose.yaml stop broker
+2 -2
View File
@@ -1,6 +1,6 @@
#!/bin/sh
set -eu
# Fault-injection hook: stop exactly ONE api replica of the lab4 stand.
# Fault-injection hook: stop exactly ONE api replica of the lab5 stand.
#
# TARGET_INSTANCE_ID=<hostname|container name|short id> scripts/stop-one.sh
#
@@ -13,7 +13,7 @@ set -eu
# No credentials are printed.
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab4}
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab5}
candidates=$(docker ps -q \
--filter "label=com.docker.compose.project=${PROJECT}" \
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
# Fault-injection hook: stop the lab5 worker and leave it stopped.
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$ROOT_DIR"
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
$COMPOSE_BIN --env-file .env -f compose.yaml stop worker