Document scaling experiment, measured database boundary and verified failure recovery

This commit is contained in:
2026-11-06 18:16:18 +03:00
parent 029bf9b377
commit a0ca0af168
52 changed files with 28893 additions and 95 deletions
+11 -3
View File
@@ -1,4 +1,5 @@
"""Record non-secret container identity/lifecycle fields for restart evidence."""
import json
import subprocess
import sys
@@ -8,7 +9,14 @@ ids = subprocess.check_output(["docker", "compose", "ps", "-q"], text=True).spli
rows = []
for cid in ids:
raw = json.loads(subprocess.check_output(["docker", "inspect", cid], text=True))[0]
rows.append({"id": cid, "service": raw["Config"]["Labels"]["com.docker.compose.service"],
"image_id": raw["Image"], "started_at": raw["State"]["StartedAt"],
"cpu_quota": raw["HostConfig"]["NanoCpus"], "memory": raw["HostConfig"]["Memory"]})
rows.append(
{
"id": cid,
"service": raw["Config"]["Labels"]["com.docker.compose.service"],
"image_id": raw["Image"],
"started_at": raw["State"]["StartedAt"],
"cpu_quota": raw["HostConfig"]["NanoCpus"],
"memory": raw["HostConfig"]["Memory"],
}
)
Path(sys.argv[1]).write_text(json.dumps(rows, indent=2) + "\n")
+15 -4
View File
@@ -1,4 +1,5 @@
"""Map a process alias to its real container, then leave exactly it stopped."""
import json
import os
import subprocess
@@ -7,12 +8,22 @@ wanted = os.environ["TARGET_INSTANCE_ID"]
ids = subprocess.check_output(["docker", "compose", "ps", "-q", "api"], text=True).split()
for cid in ids:
result = subprocess.run(
["docker", "exec", cid, "python", "-c",
"import urllib.request; r=urllib.request.urlopen('http://127.0.0.1:8080/health/live'); print(r.headers['X-Instance-Id'])"],
text=True, capture_output=True, check=False,
[
"docker",
"exec",
cid,
"python",
"-c",
"import urllib.request; r=urllib.request.urlopen('http://127.0.0.1:8080/health/live'); print(r.headers['X-Instance-Id'])",
],
text=True,
capture_output=True,
check=False,
)
if result.returncode == 0 and result.stdout.strip() == wanted:
subprocess.run(["docker", "stop", "--time", "8", cid], check=True, stdout=subprocess.DEVNULL)
subprocess.run(
["docker", "stop", "--time", "8", cid], check=True, stdout=subprocess.DEVNULL
)
print(json.dumps({"stopped_container": cid, "instance_id": wanted}))
break
else:
+46
View File
@@ -0,0 +1,46 @@
"""Check the LB's own safe JSON error when every API is stopped, then recover."""
import json
import subprocess
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client
def main():
c = Client("http://127.0.0.1:8083")
actor = c.user("lbdown")
subprocess.run(["docker", "compose", "stop", "api"], cwd=ROOT, check=True)
try:
time.sleep(4)
request = urllib.request.Request(c.base + "/api/v1/users/me",
headers={"Authorization": "Bearer " + actor["token"]})
try:
response = urllib.request.urlopen(request, timeout=3)
except urllib.error.HTTPError as exc:
response = exc
with response:
body = json.loads(response.read())
result = {"status": response.code, "body": body,
"request_id": response.headers.get("X-Request-Id"),
"instance_id": response.headers.get("X-Instance-Id")}
assert result["status"] == 503
assert body["error"]["code"] == "unavailable"
assert result["request_id"] and body["error"]["request_id"] == result["request_id"]
assert result["instance_id"] is None
finally:
subprocess.run(["docker", "compose", "start", "api"], cwd=ROOT, check=True)
subprocess.run([sys.executable, "scripts/wait_replicas.py", "2"], cwd=ROOT, check=True)
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
result["recovered_existing_session"] = True
(ROOT / "evidence/lb-no-ready-api.json").write_text(json.dumps(result, indent=2) + "\n")
print(json.dumps(result, indent=2))
if __name__ == "__main__":
main()
+9 -3
View File
@@ -1,4 +1,5 @@
"""A live process with lost DB network must be excluded by the LB."""
import json
import subprocess
import sys
@@ -38,9 +39,14 @@ print(json.dumps({'status':r.code,'alias':r.headers['X-Instance-Id']}))"""
alias = response["headers"]["X-Instance-Id"]
assert alias != initial["alias"]
observations.append(alias)
result = {"target": target, "failed_instance": initial["alias"], "direct_readiness": 503,
"seconds_until_verification": time.monotonic() - started,
"lb_successful_requests": len(observations), "lb_instances": sorted(set(observations))}
result = {
"target": target,
"failed_instance": initial["alias"],
"direct_readiness": 503,
"seconds_until_verification": time.monotonic() - started,
"lb_successful_requests": len(observations),
"lb_instances": sorted(set(observations)),
}
finally:
cmd("docker", "network", "connect", "--alias", "api", backend, target)
subprocess.run([sys.executable, "scripts/wait_replicas.py", "2"], cwd=ROOT, check=True)
+40 -15
View File
@@ -1,4 +1,5 @@
"""Stop each real API process under traffic; retain confirmed writes and source aliases."""
import json
import os
import subprocess
@@ -19,10 +20,17 @@ def compose(*args):
def aliases():
result = {}
for cid in compose("ps", "-q", "api").split():
alias = subprocess.check_output([
"docker", "exec", cid, "python", "-c",
"import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
], text=True).strip()
alias = subprocess.check_output(
[
"docker",
"exec",
cid,
"python",
"-c",
"import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
],
text=True,
).strip()
result[cid] = alias
return result
@@ -43,8 +51,13 @@ def main():
while not stop.is_set():
try:
response = c.message(actor, chat, "failover-proof")
successful.append({"at": time.monotonic(), "id": response["json"]["id"],
"instance": response["headers"]["X-Instance-Id"]})
successful.append(
{
"at": time.monotonic(),
"id": response["json"]["id"],
"instance": response["headers"]["X-Instance-Id"],
}
)
except Failure:
errors.append(time.monotonic())
@@ -53,8 +66,12 @@ def main():
thread.start()
time.sleep(1)
stopped_at = time.monotonic()
subprocess.run(["scripts/stop-one.sh"], cwd=ROOT,
env={**os.environ, "TARGET_INSTANCE_ID": target_alias}, check=True)
subprocess.run(
["scripts/stop-one.sh"],
cwd=ROOT,
env={**os.environ, "TARGET_INSTANCE_ID": target_alias},
check=True,
)
time.sleep(4)
stop.set()
thread.join()
@@ -62,14 +79,22 @@ def main():
after = [x for x in successful if x["at"] > stopped_at]
assert after and all(x["instance"] != target_alias for x in after[-5:])
assert min(x["at"] for x in after) - stopped_at <= 10
state = json.loads(subprocess.check_output(["docker", "inspect", target_container], text=True))[0]["State"]
state = json.loads(
subprocess.check_output(["docker", "inspect", target_container], text=True)
)[0]["State"]
assert not state["Running"]
runs.append({"target_container": target_container, "target_alias": target_alias,
"successful_writes": len(successful), "failed_attempts": len(errors),
"successful_rps": len(successful) / elapsed,
"recovery_seconds": min(x["at"] for x in after) - stopped_at,
"observed_after": sorted({x["instance"] for x in after[-5:]}),
"target_running_after_hook": state["Running"]})
runs.append(
{
"target_container": target_container,
"target_alias": target_alias,
"successful_writes": len(successful),
"failed_attempts": len(errors),
"successful_rps": len(successful) / elapsed,
"recovery_seconds": min(x["at"] for x in after) - stopped_at,
"observed_after": sorted({x["instance"] for x in after[-5:]}),
"target_running_after_hook": state["Running"],
}
)
subprocess.run(["docker", "start", target_container], check=True, stdout=subprocess.DEVNULL)
subprocess.run([sys.executable, "scripts/wait_replicas.py", "2"], cwd=ROOT, check=True)
# Verify every acknowledged write, without assuming failed requests did not commit.
+37 -14
View File
@@ -1,4 +1,5 @@
"""Sequential real image replacement under message traffic."""
import json
import os
import subprocess
@@ -13,7 +14,9 @@ from client import Client, Failure
def snapshots():
ids = subprocess.check_output(["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True).split()
ids = subprocess.check_output(
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
).split()
rows = []
for cid in ids:
raw = json.loads(subprocess.check_output(["docker", "inspect", cid], text=True))[0]
@@ -22,28 +25,43 @@ def snapshots():
def main():
subprocess.run(["docker", "build", "-t", "mtusi-vasilev:lab3-rolling", "-"],
input="FROM mtusi-vasilev:lab3\nLABEL org.mtusi.rolling-proof=revision-2\n", text=True, check=True)
subprocess.run(
["docker", "build", "-t", "mtusi-vasilev:lab3-rolling", "-"],
input="FROM mtusi-vasilev:lab3\nLABEL org.mtusi.rolling-proof=revision-2\n",
text=True,
check=True,
)
before = snapshots()
c = Client("http://127.0.0.1:8083")
actor, _, _, chat = c.bootstrap()
stop = threading.Event()
successes, errors = [], []
def traffic():
while not stop.is_set():
try:
response = c.message(actor, chat, "rolling-proof")
successes.append({"id": response["json"]["id"], "at": time.monotonic(),
"instance": response["headers"]["X-Instance-Id"]})
successes.append(
{
"id": response["json"]["id"],
"at": time.monotonic(),
"instance": response["headers"]["X-Instance-Id"],
}
)
except Failure:
errors.append(time.monotonic())
started = time.monotonic()
thread = threading.Thread(target=traffic)
thread.start()
try:
time.sleep(1)
subprocess.run(["scripts/rolling-update.sh"], cwd=ROOT,
env={**os.environ, "API_IMAGE": "mtusi-vasilev:lab3-rolling"}, check=True)
subprocess.run(
["scripts/rolling-update.sh"],
cwd=ROOT,
env={**os.environ, "API_IMAGE": "mtusi-vasilev:lab3-rolling"},
check=True,
)
time.sleep(2)
finally:
stop.set()
@@ -64,13 +82,18 @@ def main():
break
assert {x["id"] for x in successes} <= {x["id"] for x in items}
times = [started, *[x["at"] for x in successes], started + elapsed]
result = {"before": before, "after": after, "successful_writes": len(successes),
"failed_attempts": len(errors), "elapsed_seconds": elapsed,
"successful_rps": len(successes) / elapsed,
"observed_instances": sorted({x["instance"] for x in successes}),
"largest_success_gap_seconds": max(b-a for a,b in zip(times,times[1:])),
"all_confirmed_present": True,
"note": "Target image differs by revision label; same API/schema. This verifies sequential replacement, not arbitrary schema compatibility."}
result = {
"before": before,
"after": after,
"successful_writes": len(successes),
"failed_attempts": len(errors),
"elapsed_seconds": elapsed,
"successful_rps": len(successes) / elapsed,
"observed_instances": sorted({x["instance"] for x in successes}),
"largest_success_gap_seconds": max(b - a for a, b in zip(times, times[1:])),
"all_confirmed_present": True,
"note": "Target image differs by revision label; same API/schema. This verifies sequential replacement, not arbitrary schema compatibility.",
}
(ROOT / "evidence/rolling-update.json").write_text(json.dumps(result, indent=2) + "\n")
print(json.dumps(result, indent=2))
+103
View File
@@ -0,0 +1,103 @@
import argparse
import concurrent.futures
import importlib.util
import json
import subprocess
import time
from pathlib import Path
parser=argparse.ArgumentParser()
parser.add_argument('--lab',type=int,default=3)
parser.add_argument('--repo',type=Path)
parser.add_argument('--base-url')
parser.add_argument('--output',type=Path)
args=parser.parse_args()
repo=(args.repo or Path(__file__).resolve().parents[1]).resolve()
spec=importlib.util.spec_from_file_location('pub',repo/'tests/client.py')
m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
c=m.Client(args.base_url or f'http://127.0.0.1:{8080+args.lab}',timeout=10)
rows=[]
def compose(*args):
return subprocess.check_output(['docker','compose',*args],cwd=repo,text=True).strip()
def sql(query):
return subprocess.check_output(['docker','exec',pg,'psql','-U','messenger','-d','messenger','-At','-c',query],text=True).strip()
def direct(cid,path,token=None):
code='''import json,sys,time,urllib.request,urllib.error
p=json.load(sys.stdin);start=time.monotonic()
r=urllib.request.Request('http://127.0.0.1:8080'+p['path'],headers={'Authorization':'Bearer '+p['token']} if p.get('token') else {})
try: response=urllib.request.urlopen(r,timeout=9)
except urllib.error.HTTPError as e: response=e
with response:
body=response.read();result={'status':response.code,'elapsed_s':time.monotonic()-start,'request_id':response.headers.get('X-Request-Id'),'instance_id':response.headers.get('X-Instance-Id'),'body':json.loads(body)}
print(json.dumps(result))'''
out=subprocess.check_output(['docker','exec','-i',cid,'python','-c',code],input=json.dumps({'path':path,'token':token}),text=True,timeout=15)
return json.loads(out)
def check(label,fn):
start=time.monotonic()
try: detail=fn(); row={'case':label,'status':'PASS','detail':detail}
except Exception as e: row={'case':label,'status':'FAIL','error':str(e)[:500]}
row['elapsed_s']=time.monotonic()-start; rows.append(row);print(row,flush=True)
pg=compose('ps','-q','postgres');apis=compose('ps','-q','api').split()
alice,bob,eve,chat=c.bootstrap()
path='/api/v1/chats/'+chat['id']+'/messages'
original=c.message(alice,chat,'fault replay',key='root-fault-replay')['json']
def acl_replay():
sql(f"SET session_replication_role=replica; DELETE FROM members WHERE chat_id='{chat['id']}' AND user_id='{alice['id']}';")
try:
for text in ('fault replay','different'):
c.call('POST',path,404,actor=alice,body={'text':text},headers={'Idempotency-Key':'root-fault-replay'})
finally: sql(f"INSERT INTO members VALUES ('{chat['id']}','{alice['id']}',0);")
return {'same_body':404,'changed_body':404}
def schema_ready():
sql('ALTER TABLE message_requests RENAME TO message_requests_root_probe;')
try:
values=[direct(cid,'/health/ready') for cid in apis]
assert all(x['status']==503 for x in values),values
return values
finally: sql('ALTER TABLE message_requests_root_probe RENAME TO message_requests;')
def paused_pg():
# Warm both pools before freezing the real server.
for cid in apis: assert direct(cid,'/api/v1/users/me',alice['token'])['status']==200
subprocess.run(['docker','pause',pg],check=True,capture_output=True)
try:
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as ex:
values=list(ex.map(lambda item:direct(apis[0],item,alice['token']),('/health/ready','/api/v1/users/me')))
assert all(x['status']==503 and x['elapsed_s']<8 for x in values),values
assert all(x['request_id']==x['body']['error']['request_id'] and x['instance_id'] for x in values)
finally: subprocess.run(['docker','unpause',pg],check=True,capture_output=True)
deadline=time.monotonic()+25
while time.monotonic()<deadline:
try:
recovered=direct(apis[0],'/api/v1/users/me',alice['token'])
if recovered['status']==200: break
except Exception: pass
time.sleep(.5)
else: raise AssertionError('API did not recover')
return {'during_pause':values,'recovered_status':recovered['status']}
def logout_lock():
deadline=time.monotonic()+20
while time.monotonic()<deadline:
try:
c.call('GET','/health/ready',schema='Health');break
except Exception: time.sleep(.3)
actor=c.user('lockout')
proc=subprocess.Popen(['docker','exec','-i',pg,'psql','-U','messenger','-d','messenger','-At'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
try:
proc.stdin.write("BEGIN; LOCK TABLE sessions IN SHARE MODE; SELECT 'ROOT_LOCKED';\n");proc.stdin.flush()
while True:
line=proc.stdout.readline()
if 'ROOT_LOCKED' in line: break
if not line: raise AssertionError('lock setup failed')
c.call('DELETE','/api/v1/auth/sessions/current',503,actor=actor)
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
finally:
proc.stdin.write('ROLLBACK;\n\\q\n');proc.stdin.flush();proc.communicate(timeout=8)
c.call('DELETE','/api/v1/auth/sessions/current',204,actor=actor)
c.call('GET','/api/v1/users/me',401,actor=actor)
return {'blocked_logout':503,'retry':204,'old_token':401}
check('ACL precedes both replay and conflict',acl_replay)
check('readiness detects missing mandatory table',schema_ready)
check('paused PostgreSQL bounded API error and recovery',paused_pg)
check('logout cannot falsely succeed under DB lock',logout_lock)
out=args.output or repo/'evidence/teacher-faults-repeat.json';out.write_text(json.dumps({'tests':rows},ensure_ascii=False,indent=2)+'\n')
raise SystemExit(any(x['status']=='FAIL' for x in rows))
+193
View File
@@ -0,0 +1,193 @@
"""Independent HTTP regressions derived from teacher review; no production mutation hooks."""
import argparse
import base64
import concurrent.futures
import hashlib
import importlib.util
import json
import struct
import subprocess
import time
import urllib.parse
import uuid
import zlib
from pathlib import Path
def png_chunk(kind, content):
return struct.pack('!I',len(content))+kind+content+struct.pack('!I',zlib.crc32(kind+content)&0xffffffff)
def tiny_png(filter_byte=0, bad_idat=False, width=1, height=1):
return (b'\x89PNG\r\n\x1a\n'+png_chunk(b'IHDR',struct.pack('!IIBBBBB',width,height,8,2,0,0,0))
+png_chunk(b'IDAT',b'not-a-zlib-stream' if bad_idat else zlib.compress(bytes([filter_byte,0,0,0])))
+png_chunk(b'IEND',b''))
def main():
p=argparse.ArgumentParser()
p.add_argument('--lab',type=int,choices=(3,4,5),required=True)
p.add_argument('--repo',type=Path,required=True)
p.add_argument('--base-url',required=True)
p.add_argument('--output',type=Path,required=True)
a=p.parse_args()
spec=importlib.util.spec_from_file_location('public_client',a.repo/'tests/client.py')
mod=importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
c=mod.Client(a.base_url,timeout=12)
results=[]
def case(label,fn):
started=time.monotonic()
try:
fn()
row={'case':label,'status':'PASS'}
except Exception as e:
row={'case':label,'status':'FAIL','error':str(e)[:700]}
row['elapsed_s']=round(time.monotonic()-started,4)
results.append(row)
print(row,flush=True)
alice,bob,eve,chat=c.bootstrap()
path='/api/v1/chats/'+chat['id']+'/messages'
def msg(body,key=None,actor=alice,status=201):
return c.call('POST',path,status,body=body,actor=actor,headers={'Idempotency-Key':uuid.uuid4().hex if key is None else key},schema='Message' if status==201 else None)
def equal(actual,expected):
assert actual==expected,'value did not match expected result'
for body in ({'text':None},{'text':123},{'text':True},{'text':[]},{'text':'x','TEXT':'y'},{'Text':'x'},{'text':'x','attachment_ids':None},{'text':'x','attachment_ids':{}},{}):
case('message rejects wrong types/extra '+str(body),lambda body=body:msg(body,status=400))
for key in ('','contains space','x'*129,'bad/key'):
case('invalid idempotency key '+repr(key[:20]),lambda key=key:msg({'text':'x'},key=key,status=400))
case('missing idempotency key',lambda:c.call('POST',path,400,body={'text':'x'},actor=alice))
for body in (b'{bad',b'\xff',b'{"text":"x"}{}'):
case('malformed UTF8/JSON '+repr(body),lambda body=body:c.call('POST',path,400,raw=body,actor=alice,headers={'Content-Type':'application/json','Idempotency-Key':uuid.uuid4().hex}))
for endpoint in ('/api/v1/chats',path):
for query in ('limit=','limit=0','limit=101','limit=abc','limit=1&limit=2','cursor=','cursor=W10','cursor=bnVsbA'):
case('pagination '+endpoint+'?'+query,lambda endpoint=endpoint,query=query:c.call('GET',endpoint+'?'+query,400,actor=alice))
c.message(alice,chat,'cursor1')
c.message(alice,chat,'cursor2')
cursor=c.call('GET',path+'?limit=1',actor=alice,schema='MessagePage')['json']['next_cursor']
decoded=json.loads(base64.urlsafe_b64decode(cursor+'='*(-len(cursor)%4)))
if isinstance(decoded,list) and len(decoded)==4:
for index,value in ((2,None),(2,1),(2,'2026-13-01T00:00:00Z'),(3,False),(3,[]),(3,{}),(3,'not-a-uuid')):
data=list(decoded); data[index]=value
corrupt=base64.urlsafe_b64encode(json.dumps(data).encode()).decode().rstrip('=')
case('typed cursor '+str((index,value)),lambda corrupt=corrupt:c.call('GET',path+'?cursor='+corrupt,400,actor=alice))
case('invalid UUID path',lambda:c.call('GET','/api/v1/chats/not-a-uuid',400,actor=alice))
for bad in ([],{},123,False):
case('member UUID '+str(bad),lambda bad=bad:c.call('POST','/api/v1/chats',400,actor=alice,body={'title':'x','member_ids':[alice['id'],bad]}))
case('unknown member400',lambda:c.call('POST','/api/v1/chats',400,actor=alice,body={'title':'x','member_ids':[alice['id'],str(uuid.uuid4())]}))
for body in (b'{bad',b'{}',b'null',b' '):
auth=base64.b64encode((alice['registration']['username']+':'+alice['registration']['password']).encode()).decode()
case('login rejects body '+repr(body),lambda body=body,auth=auth:c.call('POST','/api/v1/auth/sessions',400,raw=body,headers={'Authorization':'Basic '+auth,'Content-Type':'application/json'}))
case('nonASCII Bearer401',lambda:c.call('GET','/api/v1/users/me',401,headers={'Authorization':'Bearer é'}))
for suffix in ('\n',' '):
case('username trailing whitespace',lambda suffix=suffix:c.call('POST','/api/v1/users',400,body={'username':'review'+uuid.uuid4().hex[:8]+suffix,'display_name':'x','password':'Password123456'}))
case('blank password400',lambda:c.call('POST','/api/v1/users',400,body={'username':'r'+uuid.uuid4().hex[:15],'display_name':'x','password':' '*12}))
def unicode_login():
username='r'+uuid.uuid4().hex[:15]
password='я:'*64
user=c.call('POST','/api/v1/users',201,body={'username':username,'display_name':'Никита\0имя','password':password},schema='User')['json']
equal(user['display_name'],'Никита\0имя')
auth=base64.b64encode((username+':'+password).encode()).decode()
session=c.call('POST','/api/v1/auth/sessions',201,headers={'Authorization':'Basic '+auth},schema='Session')['json']
equal(session['user'],user)
case('UTF8 colon128 password and NUL display name',unicode_login)
def canonical_repeat():
key=uuid.uuid4().hex
original=msg({'text':'canonical'},key)['json']
equal(msg({'attachment_ids':[],'text':'canonical'},key)['json'],original)
msg({'text':'changed'},key,status=409)
case('omitted attachment_ids equals [] and conflict',canonical_repeat)
def concurrent_repeat():
key=uuid.uuid4().hex
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as ex:
rows=list(ex.map(lambda _:msg({'text':'concurrent'},key)['json'],range(32)))
equal(len({x['id'] for x in rows}),1)
equal(len({json.dumps(x,sort_keys=True) for x in rows}),1)
case('32 concurrent same key all201 same message',concurrent_repeat)
def independent_scope():
key=uuid.uuid4().hex
one=msg({'text':'scope'},key)['json']
two=msg({'text':'scope'},key,actor=bob)['json']
assert one['id']!=two['id'] and two['sender_id']==bob['id']
another=c.call('POST','/api/v1/chats',201,actor=alice,body={'title':'another','member_ids':[alice['id']]},schema='Chat')['json']
three=c.message(alice,another,'scope',key=key)['json']
assert three['id']!=one['id'] and three['chat_id']==another['id']
case('key scope user and chat',independent_scope)
case('outsider cannot send',lambda:msg({'text':'outsider'},actor=eve,status=404))
def nul_message():
key=uuid.uuid4().hex
original=msg({'text':'abc\0def'},key)['json']
equal(original['text'],'abc\0def')
equal(msg({'text':'abc\0def'},key)['json'],original)
allrows=c.call('GET',path+'?limit=100',actor=bob,schema='MessagePage')['json']['items']
equal(next(x for x in allrows if x['id']==original['id']),original)
case('NUL message exact post replay history',nul_message)
def logout():
actor=c.user('logout')
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
c.call('DELETE','/api/v1/auth/sessions/current',204,actor=actor)
for _ in range(4): c.call('GET','/api/v1/users/me',401,actor=actor)
case('logout visible through all replicas',logout)
if a.lab>=4:
fixture=(a.repo/'tests/fixtures/sample.png').read_bytes()
upload='/api/v1/chats/'+chat['id']+'/attachments'
def upl(raw=fixture,mime='image/png',name='sample.png',expected=None):
return c.call('POST',upload+'?filename='+urllib.parse.quote(name,safe=''),expected or (201 if a.lab==4 else 202),actor=alice,raw=raw,headers={'Content-Type':mime},schema='Attachment' if expected is None else None)
for label,raw,mime in (
('header only',fixture[:33],'image/png'),('bad IDAT',tiny_png(bad_idat=True),'image/png'),
('invalid filter',tiny_png(filter_byte=5),'image/png'),
('empty JPEG scan',bytes.fromhex('ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9'),'image/jpeg'),
('MIME mismatch',fixture,'image/jpeg')):
case('image rejects '+label,lambda raw=raw,mime=mime:upl(raw=raw,mime=mime,expected=415))
jpeg_code="from PIL import Image; from io import BytesIO; import random,sys; im=Image.frombytes('RGB',(800,600),random.Random(42).randbytes(800*600*3)); b=BytesIO(); im.save(b,format='JPEG',quality=95); sys.stdout.buffer.write(b.getvalue())"
jpeg=subprocess.check_output([str(a.repo.absolute()/'.venv/bin/python'),'-c',jpeg_code])
case('valid real JPEG accepted',lambda:upl(raw=jpeg,mime='image/jpeg',name='real.jpg'))
case('half JPEG entropy with EOI rejected',lambda:upl(raw=jpeg[:len(jpeg)//2]+b'\xff\xd9',mime='image/jpeg',name='half.jpg',expected=415))
sos=jpeg.index(b'\xff\xda')
bad_sos=bytearray(jpeg);bad_sos[sos+5]=250
case('JPEG SOS unknown component rejected',lambda:upl(raw=bytes(bad_sos),mime='image/jpeg',name='sos.jpg',expected=415))
palette=(b'\x89PNG\r\n\x1a\n'+png_chunk(b'IHDR',struct.pack('!IIBBBBB',1,1,8,3,0,0,0))+png_chunk(b'PLTE',bytes(771))+png_chunk(b'IDAT',zlib.compress(b'\0\0'))+png_chunk(b'IEND',b''))
case('PNG invalid palette257 rejected',lambda:upl(raw=palette,expected=415))
for name in ('../x.png','a\\b.png','a\0b.png','a\x7fb.png','a\x85b.png',' '):
case('filename '+repr(name),lambda name=name:upl(name=name,expected=400))
case('filename missing',lambda:c.call('POST',upload,400,actor=alice,raw=fixture,headers={'Content-Type':'image/png'}))
case('pixel bomb413',lambda:upl(raw=tiny_png(width=20000,height=10000),expected=413))
case('too many bytes413',lambda:upl(raw=fixture+b'x'*(10485761-len(fixture)),expected=413))
attachments=[]
for _ in range(2):
att=upl()['json']
if a.lab==5: att=c.wait_ready(alice,att)
attachments.append(att)
def attachment_checks():
ids=[x['id'] for x in reversed(attachments)]
key=uuid.uuid4().hex
sent=msg({'text':'','attachment_ids':ids},key)['json']
equal(sent['attachment_ids'],ids)
equal(msg({'text':'','attachment_ids':ids},key)['json'],sent)
msg({'text':'','attachment_ids':list(reversed(ids))},key,status=409)
rows=c.call('GET',path+'?limit=100',actor=bob,schema='MessagePage')['json']['items']
equal(next(x for x in rows if x['id']==sent['id']),sent)
for att in attachments:
route='/api/v1/attachments/'+att['id']
for suffix in ('','/content'): c.call('GET',route+suffix,404,actor=eve)
content=c.call('GET',route+'/content',actor=bob)['bytes']
equal(hashlib.sha256(content).hexdigest(),att['sha256'])
msg({'text':'unknown','attachment_ids':[str(uuid.uuid4())]},status=404)
case('attachment order ACL original checksum',attachment_checks)
if a.lab==4:
case('ready missing thumbnail404',lambda:c.call('GET','/api/v1/attachments/'+attachments[0]['id']+'/content?variant=thumbnail',404,actor=alice))
# Valid PNG padded by a legal ancillary chunk, exactly 10 MiB.
def exact_limit():
base=tiny_png()
content=base[:-12]+png_chunk(b'tEXt',b'padding\0'+b'x'*(10485760-len(base)-12-8))+base[-12:]
equal(len(content),10485760)
att=upl(raw=content)['json']
equal(att['size_bytes'],10485760)
case('exact10MiB allowed through LB',exact_limit)
report={'lab':a.lab,'base_url':a.base_url,'tests':results,'passed':sum(x['status']=='PASS' for x in results),'failed':sum(x['status']=='FAIL' for x in results)}
a.output.parent.mkdir(parents=True,exist_ok=True)
a.output.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
print('TOTAL',report['passed'],'PASS',report['failed'],'FAIL',flush=True)
return int(report['failed']>0)
if __name__=='__main__': raise SystemExit(main())