forked from mtusi2026/mtusi2026lab4
tests: полный вывод сценариев, ключи mc ls, разбивка времени reconciler
This commit is contained in:
+3
-1
@@ -172,7 +172,9 @@ services:
|
||||
reconciler:
|
||||
image: mtusi2026lab4-api:local
|
||||
entrypoint: ["/app/reconcile", "--apply", "--interval", "${RECONCILE_INTERVAL:-10m}", "--grace", "${RECONCILE_GRACE:-1h}"]
|
||||
environment: *api-env
|
||||
environment:
|
||||
<<: *api-env
|
||||
CACHE_URL_FILE: "" # Redis reconciler не нужен
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||
# HTTP не обслуживает, поэтому вместо HEALTHCHECK образа API — проверка, что
|
||||
# процесс reconcile жив (нужна и для docker compose up --wait).
|
||||
|
||||
@@ -68,6 +68,9 @@ type Report struct {
|
||||
Skipped map[string]int `json:"skipped"`
|
||||
Calls map[string]int `json:"calls"`
|
||||
HeapAllocMB float64 `json:"heap_alloc_mb"`
|
||||
// WaitMS — сколько merge-цикл простоял в ожидании очередного элемента каждого
|
||||
// потока: показывает, кто (S3-листинг или запрос к БД) задаёт время обхода.
|
||||
WaitMS map[string]int64 `json:"wait_ms"`
|
||||
Errors []string `json:"errors"`
|
||||
Items []Item `json:"items"`
|
||||
}
|
||||
@@ -105,7 +108,7 @@ func Run(ctx context.Context, repo Repo, store storage.Store, opt Options) (*Rep
|
||||
rep := &Report{
|
||||
DryRun: !opt.Apply, GraceSeconds: opt.Grace.Seconds(), Cutoff: cutoff,
|
||||
Found: map[string]int{}, Fixed: map[string]int{}, Skipped: map[string]int{}, Calls: map[string]int{},
|
||||
Errors: []string{}, Items: []Item{},
|
||||
Errors: []string{}, Items: []Item{}, WaitMS: map[string]int64{},
|
||||
}
|
||||
rc := &reconciler{repo: repo, store: store, opt: opt, rep: rep, items: map[string]int{}}
|
||||
|
||||
@@ -139,26 +142,41 @@ func Run(ctx context.Context, repo Repo, store storage.Store, opt Options) (*Rep
|
||||
})
|
||||
}()
|
||||
|
||||
o, okO := <-objs
|
||||
a, okA := <-rows
|
||||
var waitS3, waitDB time.Duration
|
||||
nextObj := func() (storage.Object, bool) {
|
||||
t := time.Now()
|
||||
o, ok := <-objs
|
||||
waitS3 += time.Since(t)
|
||||
return o, ok
|
||||
}
|
||||
nextRow := func() (*domain.Attachment, bool) {
|
||||
t := time.Now()
|
||||
a, ok := <-rows
|
||||
waitDB += time.Since(t)
|
||||
return a, ok
|
||||
}
|
||||
o, okO := nextObj()
|
||||
a, okA := nextRow()
|
||||
for okO || okA {
|
||||
switch {
|
||||
case okA && (!okO || a.ObjectKey < o.Key):
|
||||
rep.RowsScanned++
|
||||
rc.rowWithoutObject(ctx, a, cutoff)
|
||||
a, okA = <-rows
|
||||
a, okA = nextRow()
|
||||
case okO && (!okA || o.Key < a.ObjectKey):
|
||||
rep.ObjectsScanned++
|
||||
rc.objectWithoutRow(ctx, o, cutoff)
|
||||
o, okO = <-objs
|
||||
o, okO = nextObj()
|
||||
default:
|
||||
rep.RowsScanned++
|
||||
rep.ObjectsScanned++
|
||||
rc.both(ctx, a, o, cutoff)
|
||||
a, okA = <-rows
|
||||
o, okO = <-objs
|
||||
a, okA = nextRow()
|
||||
o, okO = nextObj()
|
||||
}
|
||||
}
|
||||
rep.WaitMS["s3_list"] = waitS3.Milliseconds()
|
||||
rep.WaitMS["db_stream"] = waitDB.Milliseconds()
|
||||
if err := <-objErr; err != nil {
|
||||
return rep, fmt.Errorf("list objects: %w", err)
|
||||
}
|
||||
|
||||
@@ -86,17 +86,22 @@ files() {
|
||||
tail -3 evidence/attachments_negative.txt
|
||||
for s in failover s3-outage interrupt security db-gap memory; do
|
||||
step "вложения: $s"
|
||||
$PY student-tests/attachments_ops.py "$s" --base-url "$BASE" --output "evidence/files_$s.json" | tail -1
|
||||
$PY student-tests/attachments_ops.py "$s" --base-url "$BASE" --output "evidence/files_$s.json" \
|
||||
> "evidence/files_$s.log" 2>&1 || echo "FAILED: $s (см. evidence/files_$s.log)"
|
||||
tail -2 "evidence/files_$s.log"
|
||||
done
|
||||
docker compose logs --no-color api 2>&1 | grep -E 'compensat|s3 get failed|object missing' | tail -20 > evidence/compensation_log.txt || true
|
||||
}
|
||||
|
||||
track2() {
|
||||
step "reconciler: внедрение расхождений с обеих сторон"
|
||||
$PY student-tests/reconcile_faults.py --base-url "$BASE" --output evidence/track2/faults.json | tail -1
|
||||
$PY student-tests/reconcile_faults.py --base-url "$BASE" --output evidence/track2/faults.json \
|
||||
> evidence/track2/faults.log 2>&1 || echo "FAILED: reconcile_faults (см. evidence/track2/faults.log)"
|
||||
tail -2 evidence/track2/faults.log
|
||||
step "reconciler: стоимость обхода"
|
||||
$PY student-tests/reconcile_cost.py --base-url "$BASE" --sizes "${COST_SIZES:-1000,10000,30000}" \
|
||||
--output evidence/track2/cost.json | tail -1
|
||||
--output evidence/track2/cost.json > evidence/track2/cost.log 2>&1 || echo "FAILED: reconcile_cost"
|
||||
tail -2 evidence/track2/cost.log
|
||||
}
|
||||
|
||||
case "$WHAT" in
|
||||
|
||||
@@ -144,9 +144,14 @@ def scenario_db_gap(c):
|
||||
check(objs == '0', 'compensation must delete the object, objects=' + objs)
|
||||
# Reconciler (короткий grace для теста) убирает брошенное намерение.
|
||||
psql("UPDATE attachments SET updated_at = now() - interval '30 minutes' WHERE chat_id = '%s'" % chat['id'])
|
||||
out = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/reconcile', 'reconciler',
|
||||
'--apply', '--grace', '10m').stdout.strip().splitlines()[-1]
|
||||
rep = json.loads(out)
|
||||
for _ in range(10):
|
||||
r = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/reconcile', 'reconciler',
|
||||
'--apply', '--grace', '10m', check=False)
|
||||
if r.returncode != 3: # 3 — advisory lock у фонового reconciler
|
||||
break
|
||||
time.sleep(2)
|
||||
check(r.returncode in (0, 4), 'reconcile failed: ' + r.stderr[-300:])
|
||||
rep = json.loads(r.stdout.strip().splitlines()[-1])
|
||||
after = psql("SELECT count(*) FROM attachments WHERE chat_id = '%s'" % chat['id'])
|
||||
check(after == '0', 'reconciler did not remove stale intent')
|
||||
return {'client_status': st, 'seconds': round(time.time() - t0, 1), 'after_gap': {'rows': rows.split()[-1], 'objects': int(objs)},
|
||||
@@ -172,7 +177,7 @@ def scenario_security(c):
|
||||
check(own == 'ok', 'api key cannot read own bucket')
|
||||
check(mb.returncode != 0, 'api key must not create buckets')
|
||||
check(admin.returncode != 0, 'api key must not use admin API')
|
||||
check(buckets.returncode != 0 or BUCKET not in buckets.stdout or 'evil' not in buckets.stdout, 'unexpected bucket listing')
|
||||
check(buckets.returncode != 0 or 'evil' not in buckets.stdout, 'api key sees a bucket it created?!')
|
||||
# Bootstrap идемпотентен: повторный запуск успешен, объект на месте.
|
||||
before = mc('mc ls --recursive s3/%s | wc -l' % BUCKET).stdout.strip()
|
||||
r1 = compose('run', '--rm', 's3-init')
|
||||
@@ -183,7 +188,8 @@ def scenario_security(c):
|
||||
check('0.0.0.0' not in ports and '127.0.0.1' not in ports, 's3 port is published: ' + ports)
|
||||
return {'anonymous_get': '403', 'anonymous_list': '403', 'api_key_own_bucket': 'allowed',
|
||||
'api_key_make_bucket': 'denied', 'api_key_admin': 'denied',
|
||||
'api_key_list_all_buckets': 'denied' if buckets.returncode != 0 else 'allowed',
|
||||
'api_key_visible_buckets': sorted(l.split()[-1].rstrip('/') for l in buckets.stdout.splitlines() if l.strip())
|
||||
if buckets.returncode == 0 else 'denied',
|
||||
'bootstrap_runs': [r1.returncode, r2.returncode], 'objects_before_after': [int(before), int(after)],
|
||||
's3_published_ports': ports or 'none'}
|
||||
|
||||
|
||||
@@ -63,6 +63,7 @@ def main():
|
||||
'duration_ms_runs': ms, 'duration_ms_median': statistics.median(ms),
|
||||
'us_per_pair': round(1000 * statistics.median(ms) / max(1, r['rows_scanned']), 1),
|
||||
'calls': r['calls'], 'heap_alloc_mb': r['heap_alloc_mb'], 'found': r['found'],
|
||||
'wait_ms_runs': [x.get('wait_ms') for x in reps],
|
||||
'seed_seconds': round(seed_s, 1)})
|
||||
print(json.dumps(runs[-1], ensure_ascii=False))
|
||||
# 1% расхождений на самом большом наборе: удалённые объекты и сироты.
|
||||
|
||||
@@ -44,7 +44,9 @@ def snapshot(chat_id):
|
||||
rows = psql("SELECT id || ':' || status || ':' || COALESCE(size_bytes, 0) FROM attachments "
|
||||
"WHERE chat_id = '%s' ORDER BY id" % chat_id)
|
||||
objs = mc('mc ls --recursive --json s3/%s/a/%s/' % (BUCKET, chat_id)).stdout
|
||||
keys = sorted((json.loads(l)['key'], json.loads(l)['size']) for l in objs.splitlines() if l.strip())
|
||||
# mc ls --json отдаёт key относительно листинга — дополняем до полного ключа.
|
||||
keys = sorted(('a/%s/%s' % (chat_id, json.loads(l)['key']), json.loads(l)['size'])
|
||||
for l in objs.splitlines() if l.strip() and 'key' in json.loads(l))
|
||||
return rows, keys
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user