From 3ffd41cf3c5aea139688fabb7ad11e64c23bbc42 Mon Sep 17 00:00:00 2001 From: CharaWein Date: Wed, 4 Nov 2026 20:42:57 +0300 Subject: [PATCH] =?UTF-8?q?tests:=20=D0=BF=D0=BE=D0=BB=D0=BD=D1=8B=D0=B9?= =?UTF-8?q?=20=D0=B2=D1=8B=D0=B2=D0=BE=D0=B4=20=D1=81=D1=86=D0=B5=D0=BD?= =?UTF-8?q?=D0=B0=D1=80=D0=B8=D0=B5=D0=B2,=20=D0=BA=D0=BB=D1=8E=D1=87?= =?UTF-8?q?=D0=B8=20mc=20ls,=20=D1=80=D0=B0=D0=B7=D0=B1=D0=B8=D0=B2=D0=BA?= =?UTF-8?q?=D0=B0=20=D0=B2=D1=80=D0=B5=D0=BC=D0=B5=D0=BD=D0=B8=20reconcile?= =?UTF-8?q?r?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- compose.yaml | 4 +++- internal/reconcile/reconcile.go | 32 ++++++++++++++++++++++++------- scripts/collect-evidence.sh | 11 ++++++++--- student-tests/attachments_ops.py | 16 +++++++++++----- student-tests/reconcile_cost.py | 1 + student-tests/reconcile_faults.py | 4 +++- 6 files changed, 51 insertions(+), 17 deletions(-) diff --git a/compose.yaml b/compose.yaml index c4c44b30..dc2fa534 100644 --- a/compose.yaml +++ b/compose.yaml @@ -172,7 +172,9 @@ services: reconciler: image: mtusi2026lab4-api:local entrypoint: ["/app/reconcile", "--apply", "--interval", "${RECONCILE_INTERVAL:-10m}", "--grace", "${RECONCILE_GRACE:-1h}"] - environment: *api-env + environment: + <<: *api-env + CACHE_URL_FILE: "" # Redis reconciler не нужен secrets: [database_url, s3_access_key_id, s3_secret_access_key] # HTTP не обслуживает, поэтому вместо HEALTHCHECK образа API — проверка, что # процесс reconcile жив (нужна и для docker compose up --wait). diff --git a/internal/reconcile/reconcile.go b/internal/reconcile/reconcile.go index 50067075..368ee54a 100644 --- a/internal/reconcile/reconcile.go +++ b/internal/reconcile/reconcile.go @@ -68,6 +68,9 @@ type Report struct { Skipped map[string]int `json:"skipped"` Calls map[string]int `json:"calls"` HeapAllocMB float64 `json:"heap_alloc_mb"` + // WaitMS — сколько merge-цикл простоял в ожидании очередного элемента каждого + // потока: показывает, кто (S3-листинг или запрос к БД) задаёт время обхода. + WaitMS map[string]int64 `json:"wait_ms"` Errors []string `json:"errors"` Items []Item `json:"items"` } @@ -105,7 +108,7 @@ func Run(ctx context.Context, repo Repo, store storage.Store, opt Options) (*Rep rep := &Report{ DryRun: !opt.Apply, GraceSeconds: opt.Grace.Seconds(), Cutoff: cutoff, Found: map[string]int{}, Fixed: map[string]int{}, Skipped: map[string]int{}, Calls: map[string]int{}, - Errors: []string{}, Items: []Item{}, + Errors: []string{}, Items: []Item{}, WaitMS: map[string]int64{}, } rc := &reconciler{repo: repo, store: store, opt: opt, rep: rep, items: map[string]int{}} @@ -139,26 +142,41 @@ func Run(ctx context.Context, repo Repo, store storage.Store, opt Options) (*Rep }) }() - o, okO := <-objs - a, okA := <-rows + var waitS3, waitDB time.Duration + nextObj := func() (storage.Object, bool) { + t := time.Now() + o, ok := <-objs + waitS3 += time.Since(t) + return o, ok + } + nextRow := func() (*domain.Attachment, bool) { + t := time.Now() + a, ok := <-rows + waitDB += time.Since(t) + return a, ok + } + o, okO := nextObj() + a, okA := nextRow() for okO || okA { switch { case okA && (!okO || a.ObjectKey < o.Key): rep.RowsScanned++ rc.rowWithoutObject(ctx, a, cutoff) - a, okA = <-rows + a, okA = nextRow() case okO && (!okA || o.Key < a.ObjectKey): rep.ObjectsScanned++ rc.objectWithoutRow(ctx, o, cutoff) - o, okO = <-objs + o, okO = nextObj() default: rep.RowsScanned++ rep.ObjectsScanned++ rc.both(ctx, a, o, cutoff) - a, okA = <-rows - o, okO = <-objs + a, okA = nextRow() + o, okO = nextObj() } } + rep.WaitMS["s3_list"] = waitS3.Milliseconds() + rep.WaitMS["db_stream"] = waitDB.Milliseconds() if err := <-objErr; err != nil { return rep, fmt.Errorf("list objects: %w", err) } diff --git a/scripts/collect-evidence.sh b/scripts/collect-evidence.sh index a18cff60..9183cc98 100755 --- a/scripts/collect-evidence.sh +++ b/scripts/collect-evidence.sh @@ -86,17 +86,22 @@ files() { tail -3 evidence/attachments_negative.txt for s in failover s3-outage interrupt security db-gap memory; do step "вложения: $s" - $PY student-tests/attachments_ops.py "$s" --base-url "$BASE" --output "evidence/files_$s.json" | tail -1 + $PY student-tests/attachments_ops.py "$s" --base-url "$BASE" --output "evidence/files_$s.json" \ + > "evidence/files_$s.log" 2>&1 || echo "FAILED: $s (см. evidence/files_$s.log)" + tail -2 "evidence/files_$s.log" done docker compose logs --no-color api 2>&1 | grep -E 'compensat|s3 get failed|object missing' | tail -20 > evidence/compensation_log.txt || true } track2() { step "reconciler: внедрение расхождений с обеих сторон" - $PY student-tests/reconcile_faults.py --base-url "$BASE" --output evidence/track2/faults.json | tail -1 + $PY student-tests/reconcile_faults.py --base-url "$BASE" --output evidence/track2/faults.json \ + > evidence/track2/faults.log 2>&1 || echo "FAILED: reconcile_faults (см. evidence/track2/faults.log)" + tail -2 evidence/track2/faults.log step "reconciler: стоимость обхода" $PY student-tests/reconcile_cost.py --base-url "$BASE" --sizes "${COST_SIZES:-1000,10000,30000}" \ - --output evidence/track2/cost.json | tail -1 + --output evidence/track2/cost.json > evidence/track2/cost.log 2>&1 || echo "FAILED: reconcile_cost" + tail -2 evidence/track2/cost.log } case "$WHAT" in diff --git a/student-tests/attachments_ops.py b/student-tests/attachments_ops.py index cab39ddc..b1b3e09a 100755 --- a/student-tests/attachments_ops.py +++ b/student-tests/attachments_ops.py @@ -144,9 +144,14 @@ def scenario_db_gap(c): check(objs == '0', 'compensation must delete the object, objects=' + objs) # Reconciler (короткий grace для теста) убирает брошенное намерение. psql("UPDATE attachments SET updated_at = now() - interval '30 minutes' WHERE chat_id = '%s'" % chat['id']) - out = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/reconcile', 'reconciler', - '--apply', '--grace', '10m').stdout.strip().splitlines()[-1] - rep = json.loads(out) + for _ in range(10): + r = compose('run', '--rm', '--no-deps', '-T', '--entrypoint', '/app/reconcile', 'reconciler', + '--apply', '--grace', '10m', check=False) + if r.returncode != 3: # 3 — advisory lock у фонового reconciler + break + time.sleep(2) + check(r.returncode in (0, 4), 'reconcile failed: ' + r.stderr[-300:]) + rep = json.loads(r.stdout.strip().splitlines()[-1]) after = psql("SELECT count(*) FROM attachments WHERE chat_id = '%s'" % chat['id']) check(after == '0', 'reconciler did not remove stale intent') return {'client_status': st, 'seconds': round(time.time() - t0, 1), 'after_gap': {'rows': rows.split()[-1], 'objects': int(objs)}, @@ -172,7 +177,7 @@ def scenario_security(c): check(own == 'ok', 'api key cannot read own bucket') check(mb.returncode != 0, 'api key must not create buckets') check(admin.returncode != 0, 'api key must not use admin API') - check(buckets.returncode != 0 or BUCKET not in buckets.stdout or 'evil' not in buckets.stdout, 'unexpected bucket listing') + check(buckets.returncode != 0 or 'evil' not in buckets.stdout, 'api key sees a bucket it created?!') # Bootstrap идемпотентен: повторный запуск успешен, объект на месте. before = mc('mc ls --recursive s3/%s | wc -l' % BUCKET).stdout.strip() r1 = compose('run', '--rm', 's3-init') @@ -183,7 +188,8 @@ def scenario_security(c): check('0.0.0.0' not in ports and '127.0.0.1' not in ports, 's3 port is published: ' + ports) return {'anonymous_get': '403', 'anonymous_list': '403', 'api_key_own_bucket': 'allowed', 'api_key_make_bucket': 'denied', 'api_key_admin': 'denied', - 'api_key_list_all_buckets': 'denied' if buckets.returncode != 0 else 'allowed', + 'api_key_visible_buckets': sorted(l.split()[-1].rstrip('/') for l in buckets.stdout.splitlines() if l.strip()) + if buckets.returncode == 0 else 'denied', 'bootstrap_runs': [r1.returncode, r2.returncode], 'objects_before_after': [int(before), int(after)], 's3_published_ports': ports or 'none'} diff --git a/student-tests/reconcile_cost.py b/student-tests/reconcile_cost.py index 3016c107..ad96ed5a 100755 --- a/student-tests/reconcile_cost.py +++ b/student-tests/reconcile_cost.py @@ -63,6 +63,7 @@ def main(): 'duration_ms_runs': ms, 'duration_ms_median': statistics.median(ms), 'us_per_pair': round(1000 * statistics.median(ms) / max(1, r['rows_scanned']), 1), 'calls': r['calls'], 'heap_alloc_mb': r['heap_alloc_mb'], 'found': r['found'], + 'wait_ms_runs': [x.get('wait_ms') for x in reps], 'seed_seconds': round(seed_s, 1)}) print(json.dumps(runs[-1], ensure_ascii=False)) # 1% расхождений на самом большом наборе: удалённые объекты и сироты. diff --git a/student-tests/reconcile_faults.py b/student-tests/reconcile_faults.py index 4c687775..8f8d936b 100755 --- a/student-tests/reconcile_faults.py +++ b/student-tests/reconcile_faults.py @@ -44,7 +44,9 @@ def snapshot(chat_id): rows = psql("SELECT id || ':' || status || ':' || COALESCE(size_bytes, 0) FROM attachments " "WHERE chat_id = '%s' ORDER BY id" % chat_id) objs = mc('mc ls --recursive --json s3/%s/a/%s/' % (BUCKET, chat_id)).stdout - keys = sorted((json.loads(l)['key'], json.loads(l)['size']) for l in objs.splitlines() if l.strip()) + # mc ls --json отдаёт key относительно листинга — дополняем до полного ключа. + keys = sorted(('a/%s/%s' % (chat_id, json.loads(l)['key']), json.loads(l)['size']) + for l in objs.splitlines() if l.strip() and 'key' in json.loads(l)) return rows, keys