forked from mtusi2026/mtusi2026lab5
трек 2: read-only аудит инвариантов I1-I7 и автоматическая матрица отказов
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> audit: БД только READ ONLY-транзакциями, S3 ключом auditor (Read+List): потерянная работа, один эффект на версию, опубликованные объекты и SHA-256, монотонная версия, источник у активной работы, посторонние объекты, попытки. pipeline_faults: дубль, reorder, crash relay после confirm, crash worker до/после побочного эффекта и после COMMIT, временные ошибки, два worker, устаревшая версия (v2 медленная, v3 быстрая; дубль v1), reorder версий в очереди, зависший worker (docker pause).
This commit is contained in:
@@ -0,0 +1,190 @@
|
||||
"""Track 2: read-only checker of the pipeline invariants: `python -m messenger.audit`
|
||||
(compose service `auditor`, profile tools; `make audit`).
|
||||
|
||||
Read-only by construction: every database transaction is `READ ONLY` (PostgreSQL refuses any
|
||||
write), and the S3 key pair of the auditor has Read + List only. Output: one JSON report; exit 1
|
||||
if any invariant is violated.
|
||||
|
||||
Invariants (docs/PIPELINE.md):
|
||||
I1 no lost work - a queued/processing attachment has an active job; an active job has an
|
||||
outbox row (published or due) - nothing waits for a message that nobody sends.
|
||||
I2 one effect - per (attachment, pipeline_version) either no variant rows or exactly the
|
||||
three variants, all written by ONE successful attempt of a job of that version.
|
||||
I3 published data - every published object (original, variants) exists with the stored size
|
||||
(--verify-checksums: and the stored SHA-256).
|
||||
I4 monotonic - published_version = the highest version with a succeeded job; a ready
|
||||
attachment is never queued/processing again (published_version stays set).
|
||||
I5 source kept - an active job whose original is not published still has its staging copy
|
||||
(state present AND the object exists in the staging bucket).
|
||||
I6 no stray data - every object in the buckets is referenced: published, staging of a live row,
|
||||
or planned by an attempt whose objects were not cleaned yet. An object of an
|
||||
already cleaned attempt is reported as `leaked_object` (written by a stalled
|
||||
attempt after its cleanup: garbage, never a loss).
|
||||
I7 attempts - a succeeded job has exactly one succeeded attempt; at most one running attempt
|
||||
per job, and it is the lease holder.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
|
||||
from .config import ConfigError, Settings
|
||||
from .db import Database
|
||||
from .errors import Unavailable
|
||||
from .pipeline import VARIANT_NAMES
|
||||
from .storage import ObjectMissing, SENTINEL_KEY, Storage
|
||||
|
||||
|
||||
async def rows(db: Database, sql: str, params=()) -> list[tuple]:
|
||||
async def work(conn):
|
||||
await conn.execute("SET TRANSACTION READ ONLY")
|
||||
cur = await conn.execute(sql, params)
|
||||
return await cur.fetchall()
|
||||
return await db.run(work, deadline=db.deadline)
|
||||
|
||||
|
||||
async def list_all(storage: Storage, prefix: str) -> dict[str, int]:
|
||||
found, after = {}, None
|
||||
while True:
|
||||
page, more = await storage.list_page(prefix, after, 1000)
|
||||
for item in page:
|
||||
found[item.key] = item.size
|
||||
if not more or not page:
|
||||
return found
|
||||
after = page[-1].key
|
||||
|
||||
|
||||
async def audit(settings: Settings, verify: bool) -> dict:
|
||||
db = Database(settings, name="messenger-audit")
|
||||
final = Storage(settings)
|
||||
staging = Storage(settings, bucket=settings.s3_staging_bucket)
|
||||
await db.open()
|
||||
await final.open()
|
||||
await staging.open()
|
||||
violations: list[dict] = []
|
||||
notes: list[dict] = []
|
||||
|
||||
def bad(kind: str, **detail) -> None:
|
||||
violations.append({"invariant": kind, **{k: str(v) for k, v in detail.items()}})
|
||||
|
||||
try:
|
||||
# I1: queued/processing attachment without an active job; active job without outbox row.
|
||||
for (aid,) in await rows(db, "SELECT a.id FROM attachments a WHERE a.status IN ('queued', 'processing') "
|
||||
"AND NOT EXISTS (SELECT 1 FROM jobs j WHERE j.attachment_id = a.id "
|
||||
"AND j.state IN ('queued', 'processing'))"):
|
||||
bad("I1_no_active_job", attachment_id=aid)
|
||||
for (jid,) in await rows(db, "SELECT j.id FROM jobs j WHERE j.state = 'queued' AND NOT EXISTS "
|
||||
"(SELECT 1 FROM outbox o WHERE o.job_id = j.id)"):
|
||||
bad("I1_queued_without_outbox", job_id=jid)
|
||||
|
||||
# I2: exactly three variants per published version, one successful attempt of that version.
|
||||
for aid, version, names, tokens in await rows(
|
||||
db, "SELECT attachment_id, pipeline_version, array_agg(name ORDER BY name), "
|
||||
"array_agg(DISTINCT attempt_token) FROM attachment_variants GROUP BY 1, 2"):
|
||||
if sorted(names) != sorted(VARIANT_NAMES):
|
||||
bad("I2_variant_set", attachment_id=aid, version=version, names=names)
|
||||
if len(tokens) != 1:
|
||||
bad("I2_mixed_attempts", attachment_id=aid, version=version, attempts=len(tokens))
|
||||
for aid, version, outcome, jversion in await rows(
|
||||
db, "SELECT v.attachment_id, v.pipeline_version, ja.outcome, j.pipeline_version "
|
||||
"FROM attachment_variants v JOIN job_attempts ja ON ja.token = v.attempt_token "
|
||||
"JOIN jobs j ON j.id = ja.job_id WHERE ja.outcome <> 'succeeded' OR j.pipeline_version "
|
||||
"<> v.pipeline_version OR j.attachment_id <> v.attachment_id"):
|
||||
bad("I2_variant_from_unsuccessful_attempt", attachment_id=aid, version=version, outcome=outcome)
|
||||
|
||||
# I4: published_version = highest succeeded version; ready <=> published.
|
||||
for aid, status, published, best in await rows(
|
||||
db, "SELECT a.id, a.status, a.published_version, (SELECT max(j.pipeline_version) FROM jobs j "
|
||||
"WHERE j.attachment_id = a.id AND j.state = 'succeeded') FROM attachments a "
|
||||
"WHERE a.published_version IS DISTINCT FROM (SELECT max(j.pipeline_version) FROM jobs j "
|
||||
"WHERE j.attachment_id = a.id AND j.state = 'succeeded') "
|
||||
"OR (a.published_version IS NOT NULL AND a.status <> 'ready')"):
|
||||
bad("I4_published_version", attachment_id=aid, status=status, published=published, best_succeeded=best)
|
||||
|
||||
# I7: attempts.
|
||||
for (jid,) in await rows(db, "SELECT j.id FROM jobs j WHERE j.state = 'succeeded' AND (SELECT count(*) "
|
||||
"FROM job_attempts a WHERE a.job_id = j.id AND a.outcome = 'succeeded') <> 1"):
|
||||
bad("I7_succeeded_attempts", job_id=jid)
|
||||
for jid, token in await rows(db, "SELECT a.job_id, a.token FROM job_attempts a JOIN jobs j ON j.id = a.job_id "
|
||||
"WHERE a.outcome = 'running' AND j.lease_token IS DISTINCT FROM a.token"):
|
||||
bad("I7_running_not_holder", job_id=jid, token=token)
|
||||
|
||||
# I3 + I5 + I6: storage.
|
||||
published = {k: (s, h) for k, s, h in await rows(
|
||||
db, "SELECT object_key, size_bytes, sha256 FROM attachment_variants UNION ALL "
|
||||
"SELECT object_key, size_bytes, sha256 FROM attachments WHERE object_key IS NOT NULL")}
|
||||
staging_live = {k: (s, st, status) for k, s, st, status in await rows(
|
||||
db, "SELECT staging_key, size_bytes, staging_state, status FROM attachments WHERE staging_key IS NOT NULL")}
|
||||
planned_open, planned_cleaned = set(), set()
|
||||
for keys, cleaned in await rows(db, "SELECT planned_keys, objects_cleaned_at IS NOT NULL FROM job_attempts"):
|
||||
(planned_cleaned if cleaned else planned_open).update(keys)
|
||||
final_objects = await list_all(final, "")
|
||||
staging_objects = await list_all(staging, "")
|
||||
for key, (size, sha) in published.items():
|
||||
if key not in final_objects:
|
||||
bad("I3_published_missing", key=key)
|
||||
elif final_objects[key] != size:
|
||||
bad("I3_published_size", key=key, stored=size, actual=final_objects[key])
|
||||
elif verify:
|
||||
data = await final.read_all(key, limit=settings.max_upload_bytes)
|
||||
if hashlib.sha256(data).digest() != bytes(sha):
|
||||
bad("I3_published_checksum", key=key)
|
||||
for key, (size, state, status) in staging_live.items():
|
||||
# 'pending' may lack its object (the PUT failed: the janitor removes the row); accepted
|
||||
# work (queued/processing) must have it.
|
||||
active = status in ("queued", "processing")
|
||||
if state == "present" and key not in staging_objects and active:
|
||||
bad("I5_staging_missing", key=key, status=status)
|
||||
for (aid,) in await rows(db, "SELECT a.id FROM attachments a WHERE a.status IN ('queued', 'processing') "
|
||||
"AND a.object_key IS NULL AND a.staging_state <> 'present'"):
|
||||
bad("I5_active_without_source", attachment_id=aid)
|
||||
for key in final_objects:
|
||||
if key == SENTINEL_KEY or key in published or key in planned_open:
|
||||
continue
|
||||
if key in planned_cleaned:
|
||||
notes.append({"kind": "leaked_object", "key": key})
|
||||
else:
|
||||
bad("I6_unknown_final_object", key=key)
|
||||
for key in staging_objects:
|
||||
if key == SENTINEL_KEY:
|
||||
continue
|
||||
row = staging_live.get(key)
|
||||
if row is None:
|
||||
bad("I6_unknown_staging_object", key=key)
|
||||
elif row[1] == "deleted":
|
||||
notes.append({"kind": "staging_marked_deleted_still_present", "key": key})
|
||||
counts = dict(await rows(db, "SELECT status, count(*) FROM attachments GROUP BY status"))
|
||||
jobs = dict(await rows(db, "SELECT state, count(*) FROM jobs GROUP BY state"))
|
||||
return {"ok": not violations, "violations": violations, "notes": notes,
|
||||
"attachments": counts, "jobs": jobs,
|
||||
"objects": {"final": len(final_objects), "staging": len(staging_objects),
|
||||
"published": len(published), "checksums_verified": verify}}
|
||||
finally:
|
||||
await staging.close()
|
||||
await final.close()
|
||||
await db.close()
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(prog="audit", description="Read-only check of the pipeline invariants")
|
||||
p.add_argument("--verify-checksums", action="store_true", help="read every published object and compare SHA-256")
|
||||
args = p.parse_args(argv)
|
||||
try:
|
||||
settings = Settings.from_env()
|
||||
except ConfigError as exc:
|
||||
print(f"configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
report = asyncio.run(audit(settings, args.verify_checksums))
|
||||
except (Unavailable, ObjectMissing) as exc:
|
||||
print(json.dumps({"ok": False, "error": "store_unavailable", "detail": type(exc).__name__}))
|
||||
return 1
|
||||
print(json.dumps(report, ensure_ascii=False, indent=1, default=str))
|
||||
return 0 if report["ok"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user