tests: полный вывод сценариев, ключи mc ls, разбивка времени reconciler

This commit is contained in:
2026-11-04 20:42:57 +03:00
parent 3b403cc352
commit 3ffd41cf3c
6 changed files with 51 additions and 17 deletions
+3 -1
View File
@@ -172,7 +172,9 @@ services:
reconciler:
image: mtusi2026lab4-api:local
entrypoint: ["/app/reconcile", "--apply", "--interval", "${RECONCILE_INTERVAL:-10m}", "--grace", "${RECONCILE_GRACE:-1h}"]
environment: *api-env
environment:
<<: *api-env
CACHE_URL_FILE: "" # Redis reconciler не нужен
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
# HTTP не обслуживает, поэтому вместо HEALTHCHECK образа API — проверка, что
# процесс reconcile жив (нужна и для docker compose up --wait).
+25 -7
View File
@@ -68,6 +68,9 @@ type Report struct {
Skipped map[string]int `json:"skipped"`
Calls map[string]int `json:"calls"`
HeapAllocMB float64 `json:"heap_alloc_mb"`
// WaitMS — сколько merge-цикл простоял в ожидании очередного элемента каждого
// потока: показывает, кто (S3-листинг или запрос к БД) задаёт время обхода.
WaitMS map[string]int64 `json:"wait_ms"`
Errors []string `json:"errors"`
Items []Item `json:"items"`
}
@@ -105,7 +108,7 @@ func Run(ctx context.Context, repo Repo, store storage.Store, opt Options) (*Rep
rep := &Report{
DryRun: !opt.Apply, GraceSeconds: opt.Grace.Seconds(), Cutoff: cutoff,
Found: map[string]int{}, Fixed: map[string]int{}, Skipped: map[string]int{}, Calls: map[string]int{},
Errors: []string{}, Items: []Item{},
Errors: []string{}, Items: []Item{}, WaitMS: map[string]int64{},
}
rc := &reconciler{repo: repo, store: store, opt: opt, rep: rep, items: map[string]int{}}
@@ -139,26 +142,41 @@ func Run(ctx context.Context, repo Repo, store storage.Store, opt Options) (*Rep
})
}()
o, okO := <-objs
a, okA := <-rows
var waitS3, waitDB time.Duration
nextObj := func() (storage.Object, bool) {
t := time.Now()
o, ok := <-objs
waitS3 += time.Since(t)
return o, ok
}
nextRow := func() (*domain.Attachment, bool) {
t := time.Now()
a, ok := <-rows
waitDB += time.Since(t)
return a, ok
}
o, okO := nextObj()
a, okA := nextRow()
for okO || okA {
switch {
case okA && (!okO || a.ObjectKey < o.Key):
rep.RowsScanned++
rc.rowWithoutObject(ctx, a, cutoff)
a, okA = <-rows
a, okA = nextRow()
case okO && (!okA || o.Key < a.ObjectKey):
rep.ObjectsScanned++
rc.objectWithoutRow(ctx, o, cutoff)
o, okO = <-objs
o, okO = nextObj()
default:
rep.RowsScanned++
rep.ObjectsScanned++
rc.both(ctx, a, o, cutoff)
a, okA = <-rows
o, okO = <-objs
a, okA = nextRow()
o, okO = nextObj()
}
}
rep.WaitMS["s3_list"] = waitS3.Milliseconds()
rep.WaitMS["db_stream"] = waitDB.Milliseconds()
if err := <-objErr; err != nil {
return rep, fmt.Errorf("list objects: %w", err)
}
+8 -3
View File
@@ -86,17 +86,22 @@ files() {
tail -3 evidence/attachments_negative.txt
for s in failover s3-outage interrupt security db-gap memory; do
step "вложения: $s"
$PY student-tests/attachments_ops.py "$s" --base-url "$BASE" --output "evidence/files_$s.json" | tail -1
$PY student-tests/attachments_ops.py "$s" --base-url "$BASE" --output "evidence/files_$s.json" \
> "evidence/files_$s.log" 2>&1 || echo "FAILED: $s (см. evidence/files_$s.log)"
tail -2 "evidence/files_$s.log"
done
docker compose logs --no-color api 2>&1 | grep -E 'compensat|s3 get failed|object missing' | tail -20 > evidence/compensation_log.txt || true
}
track2() {
step "reconciler: внедрение расхождений с обеих сторон"
$PY student-tests/reconcile_faults.py --base-url "$BASE" --output evidence/track2/faults.json | tail -1
$PY student-tests/reconcile_faults.py --base-url "$BASE" --output evidence/track2/faults.json \
> evidence/track2/faults.log 2>&1 || echo "FAILED: reconcile_faults (см. evidence/track2/faults.log)"
tail -2 evidence/track2/faults.log
step "reconciler: стоимость обхода"
$PY student-tests/reconcile_cost.py --base-url "$BASE" --sizes "${COST_SIZES:-1000,10000,30000}" \
--output evidence/track2/cost.json | tail -1
--output evidence/track2/cost.json > evidence/track2/cost.log 2>&1 || echo "FAILED: reconcile_cost"
tail -2 evidence/track2/cost.log
}
case "$WHAT" in
+11 -5
View File
@@ -144,9 +144,14 @@ def scenario_db_gap(c):
check(objs == '0', 'compensation must delete the object, objects=' + objs)
# Reconciler (короткий grace для теста) убирает брошенное намерение.
psql("UPDATE attachments SET updated_at = now() - interval '30 minutes' WHERE chat_id = '%s'" % chat['id'])
out = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/reconcile', 'reconciler',
'--apply', '--grace', '10m').stdout.strip().splitlines()[-1]
rep = json.loads(out)
for _ in range(10):
r = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/reconcile', 'reconciler',
'--apply', '--grace', '10m', check=False)
if r.returncode != 3: # 3 — advisory lock у фонового reconciler
break
time.sleep(2)
check(r.returncode in (0, 4), 'reconcile failed: ' + r.stderr[-300:])
rep = json.loads(r.stdout.strip().splitlines()[-1])
after = psql("SELECT count(*) FROM attachments WHERE chat_id = '%s'" % chat['id'])
check(after == '0', 'reconciler did not remove stale intent')
return {'client_status': st, 'seconds': round(time.time() - t0, 1), 'after_gap': {'rows': rows.split()[-1], 'objects': int(objs)},
@@ -172,7 +177,7 @@ def scenario_security(c):
check(own == 'ok', 'api key cannot read own bucket')
check(mb.returncode != 0, 'api key must not create buckets')
check(admin.returncode != 0, 'api key must not use admin API')
check(buckets.returncode != 0 or BUCKET not in buckets.stdout or 'evil' not in buckets.stdout, 'unexpected bucket listing')
check(buckets.returncode != 0 or 'evil' not in buckets.stdout, 'api key sees a bucket it created?!')
# Bootstrap идемпотентен: повторный запуск успешен, объект на месте.
before = mc('mc ls --recursive s3/%s | wc -l' % BUCKET).stdout.strip()
r1 = compose('run', '--rm', 's3-init')
@@ -183,7 +188,8 @@ def scenario_security(c):
check('0.0.0.0' not in ports and '127.0.0.1' not in ports, 's3 port is published: ' + ports)
return {'anonymous_get': '403', 'anonymous_list': '403', 'api_key_own_bucket': 'allowed',
'api_key_make_bucket': 'denied', 'api_key_admin': 'denied',
'api_key_list_all_buckets': 'denied' if buckets.returncode != 0 else 'allowed',
'api_key_visible_buckets': sorted(l.split()[-1].rstrip('/') for l in buckets.stdout.splitlines() if l.strip())
if buckets.returncode == 0 else 'denied',
'bootstrap_runs': [r1.returncode, r2.returncode], 'objects_before_after': [int(before), int(after)],
's3_published_ports': ports or 'none'}
+1
View File
@@ -63,6 +63,7 @@ def main():
'duration_ms_runs': ms, 'duration_ms_median': statistics.median(ms),
'us_per_pair': round(1000 * statistics.median(ms) / max(1, r['rows_scanned']), 1),
'calls': r['calls'], 'heap_alloc_mb': r['heap_alloc_mb'], 'found': r['found'],
'wait_ms_runs': [x.get('wait_ms') for x in reps],
'seed_seconds': round(seed_s, 1)})
print(json.dumps(runs[-1], ensure_ascii=False))
# 1% расхождений на самом большом наборе: удалённые объекты и сироты.
+3 -1
View File
@@ -44,7 +44,9 @@ def snapshot(chat_id):
rows = psql("SELECT id || ':' || status || ':' || COALESCE(size_bytes, 0) FROM attachments "
"WHERE chat_id = '%s' ORDER BY id" % chat_id)
objs = mc('mc ls --recursive --json s3/%s/a/%s/' % (BUCKET, chat_id)).stdout
keys = sorted((json.loads(l)['key'], json.loads(l)['size']) for l in objs.splitlines() if l.strip())
# mc ls --json отдаёт key относительно листинга — дополняем до полного ключа.
keys = sorted(('a/%s/%s' % (chat_id, json.loads(l)['key']), json.loads(l)['size'])
for l in objs.splitlines() if l.strip() and 'key' in json.loads(l))
return rows, keys