Files

62 lines
3.6 KiB
Python

"""Controlled DB/S3 fault injection; use an isolated local Compose stack."""
from client import Client, check, run
from pathlib import Path
import subprocess, time, uuid, os, json
def command(args, data=None):
r=subprocess.run(args,input=data,text=True,capture_output=True)
check(r.returncode==0,'operator command failed: '+args[0])
return r.stdout.strip()
def sql(query):
return command(['docker','compose','exec','-T','db','sh','-c','psql -U "$(cat /run/secrets/postgres_user)" -d "$(cat /run/secrets/postgres_db)" -At'],query)
def mc(action):
return command(['docker','compose','exec','-T','minio','sh','-c',
'mc alias set proof http://localhost:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null && '+action])
def main():
c=Client(timeout=30); a,b,e,chat=c.bootstrap()
protected,protected_data=c.attachment(a,chat)
c.message(a,chat,'protected',attachments=[protected['id']])
protected_key=sql("select object_key from attachments where id='"+protected['id']+"';")
att,data=c.attachment(a,chat)
key=sql("select object_key from attachments where id='"+att['id']+"';")
orphan='proof-'+uuid.uuid4().hex
mc('mc cp /etc/hostname proof/messenger-attachments/'+orphan)
sql("update attachments set created_at=now()-interval '20 minutes' where id='"+att['id']+"';")
mc('mc rm proof/messenger-attachments/'+key)
fresh='fresh-'+uuid.uuid4().hex
# Objects must age past the configured one-minute test grace before scanning.
time.sleep(62)
mc('mc cp /etc/hostname proof/messenger-attachments/'+fresh)
# API replicas run scheduled reconciliation with dry-run=true, interval=3s, grace=1min.
time.sleep(6)
check(sql("select status from attachments where id='"+att['id']+"';")=='READY','dry-run mutated DB')
mc('mc stat proof/messenger-attachments/'+orphan)
logs=command(['docker','compose','logs','--no-log-prefix','api'])
proof='\n'.join(line for line in logs.splitlines() if 'Reconcile' in line)
Path('evidence/reconciler-dry-run-log.txt').write_text(proof)
check('orphan dryRun=true' in proof and 'missing dryRun=true' in proof,'dry-run did not report findings')
print('PASS dry-run: old orphan and missing metadata detected; no mutations',flush=True)
env=dict(os.environ,RECONCILER_INTERVAL_MS='3000',RECONCILER_GRACE_PERIOD_MINUTES='1',RECONCILER_DRY_RUN='false')
r=subprocess.run(['docker','compose','up','-d','--no-deps','api'],env=env,capture_output=True)
check(r.returncode==0,'recreate failed')
for _ in range(90):
if sql("select status from attachments where id='"+att['id']+"';")=='BROKEN': break
time.sleep(1)
check(sql("select status from attachments where id='"+att['id']+"';")=='BROKEN','missing not repaired')
check(orphan not in mc('mc ls proof/messenger-attachments/'),'orphan retained')
mc('mc stat proof/messenger-attachments/'+fresh)
print('PASS apply: orphan removed, missing marked BROKEN, fresh object retained',flush=True)
time.sleep(5)
check(sql("select status from attachments where id='"+att['id']+"';")=='BROKEN','repeat changed result')
mc('mc stat proof/messenger-attachments/'+protected_key)
check(sql("select status from attachments where id='"+protected['id']+"';")=='READY','used attachment damaged')
logs=command(['docker','compose','logs','--no-log-prefix','api'])
Path('evidence/reconciler-apply-log.txt').write_text('\n'.join(line for line in logs.splitlines() if 'Reconcile' in line))
print('PASS repeat is idempotent; referenced attachment and bytes retained',flush=True)
if __name__ == '__main__':
run(main)