forked from mtusi2026/mtusi2026lab4
106 lines
7.0 KiB
Python
106 lines
7.0 KiB
Python
#!/usr/bin/env python3
|
|
import sys,time,json,uuid,hashlib,subprocess,socket
|
|
from pathlib import Path
|
|
sys.path.insert(0,str(Path(__file__).resolve().parent))
|
|
from attachments import ROOT,compose,sql
|
|
sys.path.insert(0,str(ROOT/'tests'))
|
|
from client import Client,arguments,check,run
|
|
|
|
def s3(method,key,data=b'',signed=True,bucket='messenger'):
|
|
command='curl -sS --max-time 20 -w "\\n%{http_code}" '
|
|
if signed: command+='--aws-sigv4 aws:amz:us-east-1:s3 --user "$(cat /run/secrets/s3_access):$(cat /run/secrets/s3_secret)" '
|
|
command+='-H x-amz-content-sha256:'+hashlib.sha256(data).hexdigest()+' -X '+method+' '
|
|
if method=='PUT':command+='--data-binary @- '
|
|
command+='http://s3:8333/'+bucket+'/'+key
|
|
raw=compose('exec','-T','api','sh','-c',command,input=data)
|
|
body,status=raw.rsplit(b'\n',1);return int(status),body
|
|
|
|
def reconcile(apply=False):
|
|
args=['exec','-T','-e','RECONCILE_GRACE_SECONDS=120','api','dotnet','Messenger.Api.dll','--reconcile']
|
|
if apply:args.append('--apply')
|
|
out=compose(*args).decode()
|
|
return json.loads(next(line for line in out.splitlines() if line.startswith('{')))
|
|
|
|
def main():
|
|
args=arguments('S3/PostgreSQL failures and reconciliation')
|
|
c=Client(args.base_url,args.ca_file,timeout=100);a,b,e,chat=c.bootstrap();events=[]
|
|
image=(ROOT/'tests/fixtures/sample.png').read_bytes()
|
|
linked,_=c.attachment(a,chat);c.message(a,chat,'keep',attachments=[linked['id']])
|
|
abandoned,_=c.attachment(a,chat)
|
|
missing,_=c.attachment(a,chat);c.message(a,chat,'missing',attachments=[missing['id']])
|
|
key=lambda att:sql("SELECT object_key FROM attachments WHERE id='"+att['id']+"'")
|
|
linkedkey=key(linked);missingkey=key(missing)
|
|
check(s3('GET',linkedkey,signed=False)[0]==403,'bucket permits anonymous download')
|
|
check(s3('PUT','blocked',image,bucket='other-bucket')[0] in (403,404),'API credentials can write another bucket')
|
|
check(s3('DELETE',missingkey)[0]==204,'could not inject missing object')
|
|
events.append('Private bucket and scoped credentials; injected missing S3 object')
|
|
# Fail the DB write after successful S3 PUT. No production test switch is installed.
|
|
sql("CREATE FUNCTION test_attachment_failure() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename='injected-db-failure.png' THEN RAISE EXCEPTION 'test'; END IF; RETURN NEW; END $$; CREATE TRIGGER test_attachment_failure BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION test_attachment_failure();")
|
|
try:
|
|
c.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=injected-db-failure.png',503,actor=a,raw=image,headers={'Content-Type':'image/png'})
|
|
finally:
|
|
sql('DROP TRIGGER test_attachment_failure ON attachments; DROP FUNCTION test_attachment_failure();')
|
|
meta_missing,_=c.attachment(a,chat);c.message(a,chat,'metadata link',attachments=[meta_missing['id']]);meta_key=key(meta_missing)
|
|
sql("DELETE FROM attachments WHERE id='"+meta_missing['id']+"'")
|
|
orphan='originals/'+uuid.uuid4().hex
|
|
check(s3('PUT',orphan,image)[0]==200,'orphan injection failed')
|
|
# Partial HTTP upload is never published.
|
|
from urllib.parse import urlsplit
|
|
target=urlsplit(args.base_url);sock=socket.create_connection((target.hostname,target.port or 80))
|
|
head=(f"POST /api/v1/chats/{chat['id']}/attachments?filename=partial.png HTTP/1.1\r\nHost: localhost\r\nAuthorization: Bearer {a['token']}\r\nContent-Type: image/png\r\nContent-Length: 100000\r\n\r\n").encode()
|
|
sock.sendall(head+image[:20]);sock.close();time.sleep(1)
|
|
check(sql("SELECT count(*) FROM attachments WHERE filename='partial.png'")=='0','partial upload was published')
|
|
# S3 outage: metadata remains, synchronous bytes/upload path is unavailable.
|
|
compose('stop','s3')
|
|
try:
|
|
# HAProxy can itself produce 503 if both readiness checks have already failed.
|
|
import urllib.request,urllib.error
|
|
deadline=time.monotonic()+10
|
|
while True:
|
|
try:
|
|
urllib.request.urlopen(args.base_url+'/health/ready',timeout=5)
|
|
check(time.monotonic()<deadline,'ready during S3 outage');time.sleep(.3)
|
|
except urllib.error.HTTPError as ex:
|
|
check(ex.code==503,'unexpected readiness status');break
|
|
finally:
|
|
compose('start','s3')
|
|
deadline=time.monotonic()+100
|
|
while True:
|
|
try:c.call('GET','/health/ready');break
|
|
except Exception:
|
|
if time.monotonic()>deadline:raise
|
|
time.sleep(1)
|
|
events.append('Interrupted upload not published; DB failure after PUT leaves reconcilable orphan; S3 outage recovers')
|
|
fresh=reconcile();check(not any(x.get('key')==orphan or x.get('id') in [linked['id'],missing['id'],abandoned['id']] for x in fresh['actions']),'fresh attachment was selected')
|
|
print('Fresh records protected; waiting for the 120-second safety interval',flush=True)
|
|
created=time.monotonic()
|
|
# Wait for server-side LastModified as well as metadata age; timestamps are not forged.
|
|
remaining=max(0,135-(time.time()-__import__('datetime').datetime.fromisoformat(linked['created_at'].replace('Z','+00:00')).timestamp()))
|
|
while remaining>0:
|
|
time.sleep(min(20,remaining));remaining-=min(20,remaining)
|
|
dry=reconcile();check(s3('GET',orphan)[0]==200,'dry-run mutated S3')
|
|
kinds={x['kind'] for x in dry['actions']};check({'orphan','missing','unreferenced'}<=kinds,'not all inconsistencies detected')
|
|
apply=reconcile(True)
|
|
check(s3('GET',orphan)[0]==404,'orphan not deleted')
|
|
check(sql("SELECT count(*) FROM attachments WHERE id='"+abandoned['id']+"'")=='0','abandoned metadata left')
|
|
check(sql("SELECT status FROM attachments WHERE id='"+missing['id']+"'")=='missing','missing original not marked')
|
|
check(s3('GET',linkedkey)[1]==image,'used original removed or changed')
|
|
check(s3('GET',meta_key)[1]==image,'used original without metadata was removed')
|
|
new,_=c.attachment(a,chat);freshkey=key(new)
|
|
again=reconcile(True)
|
|
check(s3('GET',freshkey)[0]==200,'fresh original removed')
|
|
check(sql("SELECT count(*) FROM attachments WHERE id='"+linked['id']+"'")=='1','used metadata removed')
|
|
check(not any(x['kind'] in ('orphan','unreferenced') for x in again['actions']),'second run has remaining garbage')
|
|
scan_samples=[reconcile() for _ in range(3)]
|
|
report={'scan_samples':scan_samples,'checks':events+['dry-run is read-only','old orphan and unused data removed','referenced missing marked, link retained','fresh and referenced originals preserved, including missing metadata','repeat is idempotent'],
|
|
'dry_run':dry,'apply':apply,'repeat':again,'safety_wait_seconds':time.monotonic()-created,'assumption':'exclusive PostgreSQL advisory gate; all app writers use shared gate; 120s minimum grace; no automatic restore without backup'}
|
|
h=hashlib.sha256()
|
|
for source in sorted((ROOT/'src/Messenger.Api').rglob('*.cs')):
|
|
if 'bin' in source.parts or 'obj' in source.parts:continue
|
|
h.update(str(source.relative_to(ROOT/'src/Messenger.Api')).encode());h.update(source.read_bytes())
|
|
report['source_sha256']=h.hexdigest()
|
|
report['environment']=subprocess.check_output(['docker','info','--format','{{.OSType}} {{.Architecture}} {{.NCPU}} CPUs {{.MemTotal}} bytes RAM']).decode().strip()
|
|
(ROOT/'evidence/consistency.json').write_text(json.dumps(report,indent=2)+'\n')
|
|
print('PASS: storage failures, dry-run, safe repair, repeat and protected files')
|
|
if __name__=='__main__':run(main)
|