Files
mtusi2026lab4/student-tests/consistency.py
T

106 lines
7.0 KiB
Python

#!/usr/bin/env python3
import sys,time,json,uuid,hashlib,subprocess,socket
from pathlib import Path
sys.path.insert(0,str(Path(__file__).resolve().parent))
from attachments import ROOT,compose,sql
sys.path.insert(0,str(ROOT/'tests'))
from client import Client,arguments,check,run
def s3(method,key,data=b'',signed=True,bucket='messenger'):
command='curl -sS --max-time 20 -w "\\n%{http_code}" '
if signed: command+='--aws-sigv4 aws:amz:us-east-1:s3 --user "$(cat /run/secrets/s3_access):$(cat /run/secrets/s3_secret)" '
command+='-H x-amz-content-sha256:'+hashlib.sha256(data).hexdigest()+' -X '+method+' '
if method=='PUT':command+='--data-binary @- '
command+='http://s3:8333/'+bucket+'/'+key
raw=compose('exec','-T','api','sh','-c',command,input=data)
body,status=raw.rsplit(b'\n',1);return int(status),body
def reconcile(apply=False):
args=['exec','-T','-e','RECONCILE_GRACE_SECONDS=120','api','dotnet','Messenger.Api.dll','--reconcile']
if apply:args.append('--apply')
out=compose(*args).decode()
return json.loads(next(line for line in out.splitlines() if line.startswith('{')))
def main():
args=arguments('S3/PostgreSQL failures and reconciliation')
c=Client(args.base_url,args.ca_file,timeout=100);a,b,e,chat=c.bootstrap();events=[]
image=(ROOT/'tests/fixtures/sample.png').read_bytes()
linked,_=c.attachment(a,chat);c.message(a,chat,'keep',attachments=[linked['id']])
abandoned,_=c.attachment(a,chat)
missing,_=c.attachment(a,chat);c.message(a,chat,'missing',attachments=[missing['id']])
key=lambda att:sql("SELECT object_key FROM attachments WHERE id='"+att['id']+"'")
linkedkey=key(linked);missingkey=key(missing)
check(s3('GET',linkedkey,signed=False)[0]==403,'bucket permits anonymous download')
check(s3('PUT','blocked',image,bucket='other-bucket')[0] in (403,404),'API credentials can write another bucket')
check(s3('DELETE',missingkey)[0]==204,'could not inject missing object')
events.append('Private bucket and scoped credentials; injected missing S3 object')
# Fail the DB write after successful S3 PUT. No production test switch is installed.
sql("CREATE FUNCTION test_attachment_failure() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename='injected-db-failure.png' THEN RAISE EXCEPTION 'test'; END IF; RETURN NEW; END $$; CREATE TRIGGER test_attachment_failure BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION test_attachment_failure();")
try:
c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=injected-db-failure.png',503,actor=a,raw=image,headers={'Content-Type':'image/png'})
finally:
sql('DROP TRIGGER test_attachment_failure ON attachments; DROP FUNCTION test_attachment_failure();')
meta_missing,_=c.attachment(a,chat);c.message(a,chat,'metadata link',attachments=[meta_missing['id']]);meta_key=key(meta_missing)
sql("DELETE FROM attachments WHERE id='"+meta_missing['id']+"'")
orphan='originals/'+uuid.uuid4().hex
check(s3('PUT',orphan,image)[0]==200,'orphan injection failed')
# Partial HTTP upload is never published.
from urllib.parse import urlsplit
target=urlsplit(args.base_url);sock=socket.create_connection((target.hostname,target.port or 80))
head=(f"POST /api/v1/chats/{chat['id']}/attachments?filename=partial.png HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer {a['token']}\r\nContent-Type: image/png\r\nContent-Length: 100000\r\n\r\n").encode()
sock.sendall(head+image[:20]);sock.close();time.sleep(1)
check(sql("SELECT count(*) FROM attachments WHERE filename='partial.png'")=='0','partial upload was published')
# S3 outage: metadata remains, synchronous bytes/upload path is unavailable.
compose('stop','s3')
try:
# HAProxy can itself produce 503 if both readiness checks have already failed.
import urllib.request,urllib.error
deadline=time.monotonic()+10
while True:
try:
urllib.request.urlopen(args.base_url+'/health/ready',timeout=5)
check(time.monotonic()<deadline,'ready during S3 outage');time.sleep(.3)
except urllib.error.HTTPError as ex:
check(ex.code==503,'unexpected readiness status');break
finally:
compose('start','s3')
deadline=time.monotonic()+100
while True:
try:c.call('GET','/health/ready');break
except Exception:
if time.monotonic()>deadline:raise
time.sleep(1)
events.append('Interrupted upload not published; DB failure after PUT leaves reconcilable orphan; S3 outage recovers')
fresh=reconcile();check(not any(x.get('key')==orphan or x.get('id') in [linked['id'],missing['id'],abandoned['id']] for x in fresh['actions']),'fresh attachment was selected')
print('Fresh records protected; waiting for the 120-second safety interval',flush=True)
created=time.monotonic()
# Wait for server-side LastModified as well as metadata age; timestamps are not forged.
remaining=max(0,135-(time.time()-__import__('datetime').datetime.fromisoformat(linked['created_at'].replace('Z','+00:00')).timestamp()))
while remaining>0:
time.sleep(min(20,remaining));remaining-=min(20,remaining)
dry=reconcile();check(s3('GET',orphan)[0]==200,'dry-run mutated S3')
kinds={x['kind'] for x in dry['actions']};check({'orphan','missing','unreferenced'}<=kinds,'not all inconsistencies detected')
apply=reconcile(True)
check(s3('GET',orphan)[0]==404,'orphan not deleted')
check(sql("SELECT count(*) FROM attachments WHERE id='"+abandoned['id']+"'")=='0','abandoned metadata left')
check(sql("SELECT status FROM attachments WHERE id='"+missing['id']+"'")=='missing','missing original not marked')
check(s3('GET',linkedkey)[1]==image,'used original removed or changed')
check(s3('GET',meta_key)[1]==image,'used original without metadata was removed')
new,_=c.attachment(a,chat);freshkey=key(new)
again=reconcile(True)
check(s3('GET',freshkey)[0]==200,'fresh original removed')
check(sql("SELECT count(*) FROM attachments WHERE id='"+linked['id']+"'")=='1','used metadata removed')
check(not any(x['kind'] in ('orphan','unreferenced') for x in again['actions']),'second run has remaining garbage')
scan_samples=[reconcile() for _ in range(3)]
report={'scan_samples':scan_samples,'checks':events+['dry-run is read-only','old orphan and unused data removed','referenced missing marked, link retained','fresh and referenced originals preserved, including missing metadata','repeat is idempotent'],
'dry_run':dry,'apply':apply,'repeat':again,'safety_wait_seconds':time.monotonic()-created,'assumption':'exclusive PostgreSQL advisory gate; all app writers use shared gate; 120s minimum grace; no automatic restore without backup'}
h=hashlib.sha256()
for source in sorted((ROOT/'src/Messenger.Api').rglob('*.cs')):
if 'bin' in source.parts or 'obj' in source.parts:continue
h.update(str(source.relative_to(ROOT/'src/Messenger.Api')).encode());h.update(source.read_bytes())
report['source_sha256']=h.hexdigest()
report['environment']=subprocess.check_output(['docker','info','--format','{{.OSType}} {{.Architecture}} {{.NCPU}} CPUs {{.MemTotal}} bytes RAM']).decode().strip()
(ROOT/'evidence/consistency.json').write_text(json.dumps(report,indent=2)+'\n')
print('PASS: storage failures, dry-run, safe repair, repeat and protected files')
if __name__=='__main__':run(main)