forked from mtusi2026/mtusi2026lab4
62 lines
3.6 KiB
Python
62 lines
3.6 KiB
Python
"""Controlled DB/S3 fault injection; use an isolated local Compose stack."""
|
|
from client import Client, check, run
|
|
from pathlib import Path
|
|
import subprocess, time, uuid, os, json
|
|
|
|
def command(args, data=None):
|
|
r=subprocess.run(args,input=data,text=True,capture_output=True)
|
|
check(r.returncode==0,'operator command failed: '+args[0])
|
|
return r.stdout.strip()
|
|
|
|
def sql(query):
|
|
return command(['docker','compose','exec','-T','db','sh','-c','psql -U "$(cat /run/secrets/postgres_user)" -d "$(cat /run/secrets/postgres_db)" -At'],query)
|
|
|
|
def mc(action):
|
|
return command(['docker','compose','exec','-T','minio','sh','-c',
|
|
'mc alias set proof http://localhost:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null && '+action])
|
|
|
|
def main():
|
|
c=Client(timeout=30); a,b,e,chat=c.bootstrap()
|
|
protected,protected_data=c.attachment(a,chat)
|
|
c.message(a,chat,'protected',attachments=[protected['id']])
|
|
protected_key=sql("select object_key from attachments where id='"+protected['id']+"';")
|
|
att,data=c.attachment(a,chat)
|
|
key=sql("select object_key from attachments where id='"+att['id']+"';")
|
|
orphan='proof-'+uuid.uuid4().hex
|
|
mc('mc cp /etc/hostname proof/messenger-attachments/'+orphan)
|
|
sql("update attachments set created_at=now()-interval '20 minutes' where id='"+att['id']+"';")
|
|
mc('mc rm proof/messenger-attachments/'+key)
|
|
fresh='fresh-'+uuid.uuid4().hex
|
|
# Objects must age past the configured one-minute test grace before scanning.
|
|
time.sleep(62)
|
|
mc('mc cp /etc/hostname proof/messenger-attachments/'+fresh)
|
|
# API replicas run scheduled reconciliation with dry-run=true, interval=3s, grace=1min.
|
|
time.sleep(6)
|
|
check(sql("select status from attachments where id='"+att['id']+"';")=='READY','dry-run mutated DB')
|
|
mc('mc stat proof/messenger-attachments/'+orphan)
|
|
logs=command(['docker','compose','logs','--no-log-prefix','api'])
|
|
proof='\n'.join(line for line in logs.splitlines() if 'Reconcile' in line)
|
|
Path('evidence/reconciler-dry-run-log.txt').write_text(proof)
|
|
check('orphan dryRun=true' in proof and 'missing dryRun=true' in proof,'dry-run did not report findings')
|
|
print('PASS dry-run: old orphan and missing metadata detected; no mutations',flush=True)
|
|
env=dict(os.environ,RECONCILER_INTERVAL_MS='3000',RECONCILER_GRACE_PERIOD_MINUTES='1',RECONCILER_DRY_RUN='false')
|
|
r=subprocess.run(['docker','compose','up','-d','--no-deps','api'],env=env,capture_output=True)
|
|
check(r.returncode==0,'recreate failed')
|
|
for _ in range(90):
|
|
if sql("select status from attachments where id='"+att['id']+"';")=='BROKEN': break
|
|
time.sleep(1)
|
|
check(sql("select status from attachments where id='"+att['id']+"';")=='BROKEN','missing not repaired')
|
|
check(orphan not in mc('mc ls proof/messenger-attachments/'),'orphan retained')
|
|
mc('mc stat proof/messenger-attachments/'+fresh)
|
|
print('PASS apply: orphan removed, missing marked BROKEN, fresh object retained',flush=True)
|
|
time.sleep(5)
|
|
check(sql("select status from attachments where id='"+att['id']+"';")=='BROKEN','repeat changed result')
|
|
mc('mc stat proof/messenger-attachments/'+protected_key)
|
|
check(sql("select status from attachments where id='"+protected['id']+"';")=='READY','used attachment damaged')
|
|
logs=command(['docker','compose','logs','--no-log-prefix','api'])
|
|
Path('evidence/reconciler-apply-log.txt').write_text('\n'.join(line for line in logs.splitlines() if 'Reconcile' in line))
|
|
print('PASS repeat is idempotent; referenced attachment and bytes retained',flush=True)
|
|
|
|
if __name__ == '__main__':
|
|
run(main)
|