lab5: очереди и асинхронная обработка изображений
Durable staging handoff, transactional outbox, at-least-once worker with lease fencing, idempotent versioned pipeline, three variants, operator tooling, contract tests and evidence. Includes the review fixes: reprocess source selection and failure settlement, replay guards, stale-job retirement, PNG structural/raster validation, migration rollback safety and audit events.
This commit is contained in:
Executable
+165
@@ -0,0 +1,165 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 2 evidence: versioned pipeline, duplicate delivery, stale-version guard.
|
||||
|
||||
Runs N repetitions against the live stand and writes raw CSV/Markdown evidence.
|
||||
This is a correctness harness, not a load test.
|
||||
|
||||
python3 scripts/lab5_track2.py --reps 5
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
sys.path.insert(0, str(ROOT / 'student-tests'))
|
||||
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
from tests_helpers import compose, jobctl, psql # noqa: E402
|
||||
|
||||
FIXTURE = ROOT / 'tests/fixtures/sample.png'
|
||||
EVIDENCE = ROOT / 'evidence/lab5/track2'
|
||||
|
||||
|
||||
def wait_ready(client, actor, attachment, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
current = client.call('GET', '/api/v1/attachments/' + attachment['id'], actor=actor, schema='Attachment')['json']
|
||||
check(current['status'] != 'failed', 'attachment failed: ' + str(current.get('error_code')))
|
||||
if current['status'] == 'ready':
|
||||
return current
|
||||
time.sleep(0.2)
|
||||
raise Failure('attachment did not become ready')
|
||||
|
||||
|
||||
def wait_version(attachment_id, version, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
row = psql("select pipeline_version || '|' || status from attachments where id='%s'" % attachment_id)
|
||||
if row == str(version) + '|ready':
|
||||
return
|
||||
time.sleep(0.2)
|
||||
raise Failure('attachment did not reach pipeline version %d' % version)
|
||||
|
||||
|
||||
def variants_by_name(attachment):
|
||||
return {v['name']: v for v in attachment['variants']}
|
||||
|
||||
|
||||
def scenario_duplicate_before_effect(client, actor, chat):
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
attached, _ = client.attachment(actor, chat)
|
||||
time.sleep(1)
|
||||
job = psql("select job_id from attachment_jobs where attachment_id='%s'" % attached['id'])
|
||||
check(bool(job), 'job id missing')
|
||||
jobctl('duplicate', '-job', job)
|
||||
jobctl('duplicate', '-job', job)
|
||||
finally:
|
||||
compose('up', '-d', '--no-deps', 'worker', check=False)
|
||||
|
||||
ready = wait_ready(client, actor, attached)
|
||||
names = sorted(variants_by_name(ready))
|
||||
check(names == ['crop', 'optimized', 'thumbnail'], 'duplicate delivery changed the variant set')
|
||||
check(psql("select count(*) from attachment_variants where attachment_id='%s'" % attached['id']) == '3',
|
||||
'duplicate delivery created extra variant rows')
|
||||
|
||||
return 'duplicate_before_effect', True, 'attempts=' + psql(
|
||||
"select attempt from attachment_jobs where attachment_id='%s'" % attached['id'])
|
||||
|
||||
|
||||
def scenario_stale_version(client, actor, chat):
|
||||
attached, content = client.attachment(actor, chat)
|
||||
v1 = wait_ready(client, actor, attached)
|
||||
v1_variants = variants_by_name(v1)
|
||||
original_sha = hashlib.sha256(content).hexdigest()
|
||||
check(v1['sha256'] == original_sha, 'original checksum changed before reprocess')
|
||||
|
||||
v1_job = psql("select job_id from attachment_jobs where attachment_id='%s' and pipeline_version=1" % attached['id'])
|
||||
check(bool(v1_job), 'v1 job missing')
|
||||
|
||||
reprocess = jobctl('reprocess', '-attachment', attached['id'], '-pipeline', '2')
|
||||
check(reprocess.returncode == 0, 'reprocess failed: ' + reprocess.stderr)
|
||||
wait_version(attached['id'], 2)
|
||||
v2 = wait_ready(client, actor, attached)
|
||||
v2_variants = variants_by_name(v2)
|
||||
check(sorted(v2_variants) == ['crop', 'optimized', 'thumbnail'], 'v2 variant set incomplete')
|
||||
check(v2_variants['thumbnail']['sha256'] != v1_variants['thumbnail']['sha256'],
|
||||
'v2 bytes must differ from v1 (pipeline parameters changed)')
|
||||
check(v2['sha256'] == original_sha, 'original changed during reprocess')
|
||||
before = json.dumps(v2_variants, sort_keys=True)
|
||||
|
||||
# Deliver the old v1 message after v2 is published. It must be acknowledged
|
||||
# without touching the serving version or its artifacts.
|
||||
jobctl('duplicate', '-job', v1_job)
|
||||
time.sleep(2)
|
||||
|
||||
after_attachment = client.call('GET', '/api/v1/attachments/' + attached['id'], actor=actor, schema='Attachment')['json']
|
||||
after_variants = variants_by_name(after_attachment)
|
||||
check(json.dumps(after_variants, sort_keys=True) == before, 'stale v1 delivery overwrote v2 variants')
|
||||
check(psql("select pipeline_version from attachments where id='%s'" % attached['id']) == '2',
|
||||
'serving pipeline version regressed')
|
||||
check(psql("select status from attachment_jobs where job_id='%s'" % v1_job) == 'ready',
|
||||
'v1 job state changed unexpectedly')
|
||||
|
||||
return 'stale_version_after_v2', True, 'v1_job=' + v1_job
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Lab5 track 2 correctness harness')
|
||||
parser.add_argument('--base-url', default='http://localhost:8080')
|
||||
parser.add_argument('--reps', type=int, default=3)
|
||||
args = parser.parse_args()
|
||||
|
||||
EVIDENCE.mkdir(parents=True, exist_ok=True)
|
||||
rows = []
|
||||
|
||||
client = Client(args.base_url, None)
|
||||
|
||||
for rep in range(1, args.reps + 1):
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
for scenario in (scenario_duplicate_before_effect, scenario_stale_version):
|
||||
name, passed, detail = scenario(client, alice, chat)
|
||||
rows.append({'scenario': name, 'rep': rep, 'pass': int(passed), 'detail': detail})
|
||||
print('D%s rep=%d: %s' % (name, rep, 'PASS' if passed else 'FAIL'))
|
||||
|
||||
with (EVIDENCE / 'runs.csv').open('w', newline='') as handle:
|
||||
writer = csv.DictWriter(handle, fieldnames=['scenario', 'rep', 'pass', 'detail'])
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
summary = [
|
||||
'# Track 2 — versioned pipeline and duplicate/stale delivery',
|
||||
'',
|
||||
'Hypothesis: with the effect key `(attachment_id, pipeline_version, variant)`,',
|
||||
'deterministic object keys, a serving version that changes only when all v2',
|
||||
'variants are committed, and stale-version acknowledgements, an at-least-once',
|
||||
'pipeline loses no accepted work and never overwrites a newer result with an',
|
||||
'older duplicate.',
|
||||
'',
|
||||
'Method: N=%d repetitions. Each repetition creates fresh users/chat and runs:' % args.reps,
|
||||
'- duplicate pipeline publication while the worker is stopped, followed by recovery;',
|
||||
'- v1 ready -> explicit reprocess to v2 -> old v1 message delivered after v2 ready.',
|
||||
'',
|
||||
'Invariants checked: unique variant rows per effect key; ready remains ready;',
|
||||
'serving pipeline version never regresses; original bytes/checksum unchanged;',
|
||||
'stale job state is not rewritten.',
|
||||
'',
|
||||
'Limitations: this harness covers duplicate and stale/reorder delivery. Crash',
|
||||
'windows (before/after side effect, before/after ack, outbox gap) and lease',
|
||||
'takeover are covered by student-tests/lab5_faults.py; both are single-node',
|
||||
'Compose evidence, not a production availability claim. Delivery is',
|
||||
'at-least-once; exactly-once is not claimed.',
|
||||
]
|
||||
(EVIDENCE / 'summary.md').write_text('\n'.join(summary) + '\n')
|
||||
|
||||
passed = sum(row['pass'] for row in rows)
|
||||
print('PASS: track2 %d/%d scenarios' % (passed, len(rows)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,5 +1,5 @@
|
||||
#!/bin/sh
|
||||
# Проверка критерия «LB исключает неготовую реплику» (lab3).
|
||||
# Проверка критерия «LB исключает неготовую реплику» (lab5).
|
||||
#
|
||||
# Рядом с рабочими репликами поднимается третья с тем же сетевым alias, но её
|
||||
# доступ к БД идёт через временный TCP-прокси. Пока прокси жив, реплика готова и
|
||||
@@ -23,10 +23,10 @@ if [ -f .env ]; then
|
||||
set +a
|
||||
fi
|
||||
|
||||
OUT=${OUT:-evidence/lb-readiness.txt}
|
||||
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab4}
|
||||
OUT=${OUT:-evidence/lab5/lb-readiness-detail.txt}
|
||||
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab5}
|
||||
NET=${PROJECT}_default
|
||||
IMAGE=${IMAGE:-mtusi-messenger:lab4}
|
||||
IMAGE=${IMAGE:-mtusi-messenger:lab5}
|
||||
STATS=${STATS:-http://127.0.0.1:8404/stats;csv}
|
||||
BASE_URL=${BASE_URL:-http://localhost:8080}
|
||||
CANDIDATE=lb-readiness-candidate
|
||||
@@ -77,7 +77,7 @@ wait_readiness() {
|
||||
}
|
||||
|
||||
up_slots() {
|
||||
curl -s --noproxy '*' "$STATS" | awk -F, '$1 ~ /^api/ && $18 == "UP" && $2 != "BACKEND" { printf "%s ", $2 }'
|
||||
curl -s --noproxy '*' "$STATS" | awk -F, '$1 ~ /^api/ && $18 ~ /^UP/ && $2 != "BACKEND" { printf "%s ", $2 }'
|
||||
}
|
||||
|
||||
wait_new_up_slot() {
|
||||
@@ -120,7 +120,7 @@ wait_slot_status() {
|
||||
}
|
||||
|
||||
{
|
||||
echo "# LB excludes a not-ready but reachable replica (lab3)"
|
||||
echo "# LB excludes a not-ready but reachable replica (lab5)"
|
||||
echo "# generated: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "# lb: $(docker inspect -f '{{.Config.Image}}' "$(docker ps -q --filter label=com.docker.compose.service=lb)" 2>/dev/null)"
|
||||
echo
|
||||
@@ -139,6 +139,11 @@ wait_slot_status() {
|
||||
-e POSTGRES_USER="${POSTGRES_USER:-messenger}" \
|
||||
-e POSTGRES_PASSWORD="${POSTGRES_PASSWORD:-messenger}" -e POSTGRES_SSL_MODE=disable \
|
||||
-e REDIS_ENABLED=true -e REDIS_HOST=redis -e CACHE_ENABLED=true \
|
||||
-e S3_ENDPOINT=http://seaweedfs:8333 -e S3_REGION="${S3_REGION:-us-east-1}" \
|
||||
-e S3_BUCKET="${S3_BUCKET:-messenger-attachments}" \
|
||||
-e S3_ACCESS_KEY_ID="${S3_ACCESS_KEY_ID:-lab4app}" \
|
||||
-e S3_SECRET_ACCESS_KEY="${S3_SECRET_ACCESS_KEY:-lab4app-secret}" \
|
||||
-e S3_USE_PATH_STYLE=true \
|
||||
"$IMAGE" >/dev/null
|
||||
|
||||
if ! wait_readiness 200; then
|
||||
@@ -170,7 +175,8 @@ wait_slot_status() {
|
||||
if wait_slot_status "$slot" DOWN; then
|
||||
echo " $slot went DOWN ${SWAIT_SECONDS}s after its database was cut (check: inter 2s fall 2)"
|
||||
else
|
||||
echo " WARNING: $slot is still not DOWN" >&2
|
||||
echo " FAIL: $slot is still not DOWN" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo " stats after failure: $(stats)"
|
||||
|
||||
+10
-10
@@ -1,6 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Restart hook: bounce API replicas and every state store of the lab4 stand.
|
||||
# Restart hook: bounce API replicas, worker/relay and every state store of the lab5 stand.
|
||||
#
|
||||
# scripts/restart.sh
|
||||
#
|
||||
@@ -29,16 +29,16 @@ compose() {
|
||||
$COMPOSE_BIN --env-file .env -f compose.yaml "$@"
|
||||
}
|
||||
|
||||
# 1. State stores: restart the running ones, then make sure they are healthy.
|
||||
# Volumes are attached as before, nothing is dropped or re-seeded.
|
||||
compose up -d --wait postgres redis seaweedfs
|
||||
compose restart postgres redis seaweedfs
|
||||
compose up -d --wait postgres redis seaweedfs
|
||||
# 1. State stores and broker: restart the running ones, then make sure they are
|
||||
# healthy. Volumes are attached as before, nothing is dropped or re-seeded.
|
||||
compose up -d --wait postgres redis seaweedfs broker
|
||||
compose restart postgres redis seaweedfs broker
|
||||
compose up -d --wait postgres redis seaweedfs broker
|
||||
|
||||
# 2. API replicas: restart in place (same containers, same addresses) and bring
|
||||
# back any replica that was stopped by the fault-injection hook.
|
||||
compose restart api
|
||||
compose up -d --wait --no-deps api
|
||||
# 2. API replicas, worker and relay: restart in place and bring back any replica
|
||||
# stopped by a fault-injection hook. The worker reconnects to the broker.
|
||||
compose restart api worker relay
|
||||
compose up -d --wait --no-deps api worker relay
|
||||
|
||||
# 3. Load balancer keeps its upstream; re-resolve in case addresses changed.
|
||||
compose restart lb
|
||||
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT_DIR"
|
||||
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
|
||||
$COMPOSE_BIN --env-file .env -f compose.yaml up -d --wait --no-deps broker
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT_DIR"
|
||||
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
|
||||
$COMPOSE_BIN --env-file .env -f compose.yaml up -d --wait --no-deps worker
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT_DIR"
|
||||
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
|
||||
$COMPOSE_BIN --env-file .env -f compose.yaml stop broker
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Fault-injection hook: stop exactly ONE api replica of the lab4 stand.
|
||||
# Fault-injection hook: stop exactly ONE api replica of the lab5 stand.
|
||||
#
|
||||
# TARGET_INSTANCE_ID=<hostname|container name|short id> scripts/stop-one.sh
|
||||
#
|
||||
@@ -13,7 +13,7 @@ set -eu
|
||||
# No credentials are printed.
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
|
||||
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab4}
|
||||
PROJECT=${COMPOSE_PROJECT_NAME:-mtusi-messenger-lab5}
|
||||
|
||||
candidates=$(docker ps -q \
|
||||
--filter "label=com.docker.compose.project=${PROJECT}" \
|
||||
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Fault-injection hook: stop the lab5 worker and leave it stopped.
|
||||
ROOT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT_DIR"
|
||||
COMPOSE_BIN=${COMPOSE_BIN:-docker compose}
|
||||
$COMPOSE_BIN --env-file .env -f compose.yaml stop worker
|
||||
Reference in New Issue
Block a user