Document scaling experiment, measured database boundary and verified failure recovery
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
"""Record non-secret container identity/lifecycle fields for restart evidence."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -8,7 +9,14 @@ ids = subprocess.check_output(["docker", "compose", "ps", "-q"], text=True).spli
|
||||
rows = []
|
||||
for cid in ids:
|
||||
raw = json.loads(subprocess.check_output(["docker", "inspect", cid], text=True))[0]
|
||||
rows.append({"id": cid, "service": raw["Config"]["Labels"]["com.docker.compose.service"],
|
||||
"image_id": raw["Image"], "started_at": raw["State"]["StartedAt"],
|
||||
"cpu_quota": raw["HostConfig"]["NanoCpus"], "memory": raw["HostConfig"]["Memory"]})
|
||||
rows.append(
|
||||
{
|
||||
"id": cid,
|
||||
"service": raw["Config"]["Labels"]["com.docker.compose.service"],
|
||||
"image_id": raw["Image"],
|
||||
"started_at": raw["State"]["StartedAt"],
|
||||
"cpu_quota": raw["HostConfig"]["NanoCpus"],
|
||||
"memory": raw["HostConfig"]["Memory"],
|
||||
}
|
||||
)
|
||||
Path(sys.argv[1]).write_text(json.dumps(rows, indent=2) + "\n")
|
||||
|
||||
+15
-4
@@ -1,4 +1,5 @@
|
||||
"""Map a process alias to its real container, then leave exactly it stopped."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
@@ -7,12 +8,22 @@ wanted = os.environ["TARGET_INSTANCE_ID"]
|
||||
ids = subprocess.check_output(["docker", "compose", "ps", "-q", "api"], text=True).split()
|
||||
for cid in ids:
|
||||
result = subprocess.run(
|
||||
["docker", "exec", cid, "python", "-c",
|
||||
"import urllib.request; r=urllib.request.urlopen('http://127.0.0.1:8080/health/live'); print(r.headers['X-Instance-Id'])"],
|
||||
text=True, capture_output=True, check=False,
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cid,
|
||||
"python",
|
||||
"-c",
|
||||
"import urllib.request; r=urllib.request.urlopen('http://127.0.0.1:8080/health/live'); print(r.headers['X-Instance-Id'])",
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode == 0 and result.stdout.strip() == wanted:
|
||||
subprocess.run(["docker", "stop", "--time", "8", cid], check=True, stdout=subprocess.DEVNULL)
|
||||
subprocess.run(
|
||||
["docker", "stop", "--time", "8", cid], check=True, stdout=subprocess.DEVNULL
|
||||
)
|
||||
print(json.dumps({"stopped_container": cid, "instance_id": wanted}))
|
||||
break
|
||||
else:
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Check the LB's own safe JSON error when every API is stopped, then recover."""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client
|
||||
|
||||
|
||||
def main():
|
||||
c = Client("http://127.0.0.1:8083")
|
||||
actor = c.user("lbdown")
|
||||
subprocess.run(["docker", "compose", "stop", "api"], cwd=ROOT, check=True)
|
||||
try:
|
||||
time.sleep(4)
|
||||
request = urllib.request.Request(c.base + "/api/v1/users/me",
|
||||
headers={"Authorization": "Bearer " + actor["token"]})
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=3)
|
||||
except urllib.error.HTTPError as exc:
|
||||
response = exc
|
||||
with response:
|
||||
body = json.loads(response.read())
|
||||
result = {"status": response.code, "body": body,
|
||||
"request_id": response.headers.get("X-Request-Id"),
|
||||
"instance_id": response.headers.get("X-Instance-Id")}
|
||||
assert result["status"] == 503
|
||||
assert body["error"]["code"] == "unavailable"
|
||||
assert result["request_id"] and body["error"]["request_id"] == result["request_id"]
|
||||
assert result["instance_id"] is None
|
||||
finally:
|
||||
subprocess.run(["docker", "compose", "start", "api"], cwd=ROOT, check=True)
|
||||
subprocess.run([sys.executable, "scripts/wait_replicas.py", "2"], cwd=ROOT, check=True)
|
||||
c.call("GET", "/api/v1/users/me", actor=actor, schema="User")
|
||||
result["recovered_existing_session"] = True
|
||||
(ROOT / "evidence/lb-no-ready-api.json").write_text(json.dumps(result, indent=2) + "\n")
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,4 +1,5 @@
|
||||
"""A live process with lost DB network must be excluded by the LB."""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -38,9 +39,14 @@ print(json.dumps({'status':r.code,'alias':r.headers['X-Instance-Id']}))"""
|
||||
alias = response["headers"]["X-Instance-Id"]
|
||||
assert alias != initial["alias"]
|
||||
observations.append(alias)
|
||||
result = {"target": target, "failed_instance": initial["alias"], "direct_readiness": 503,
|
||||
"seconds_until_verification": time.monotonic() - started,
|
||||
"lb_successful_requests": len(observations), "lb_instances": sorted(set(observations))}
|
||||
result = {
|
||||
"target": target,
|
||||
"failed_instance": initial["alias"],
|
||||
"direct_readiness": 503,
|
||||
"seconds_until_verification": time.monotonic() - started,
|
||||
"lb_successful_requests": len(observations),
|
||||
"lb_instances": sorted(set(observations)),
|
||||
}
|
||||
finally:
|
||||
cmd("docker", "network", "connect", "--alias", "api", backend, target)
|
||||
subprocess.run([sys.executable, "scripts/wait_replicas.py", "2"], cwd=ROOT, check=True)
|
||||
|
||||
+40
-15
@@ -1,4 +1,5 @@
|
||||
"""Stop each real API process under traffic; retain confirmed writes and source aliases."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
@@ -19,10 +20,17 @@ def compose(*args):
|
||||
def aliases():
|
||||
result = {}
|
||||
for cid in compose("ps", "-q", "api").split():
|
||||
alias = subprocess.check_output([
|
||||
"docker", "exec", cid, "python", "-c",
|
||||
"import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
|
||||
], text=True).strip()
|
||||
alias = subprocess.check_output(
|
||||
[
|
||||
"docker",
|
||||
"exec",
|
||||
cid,
|
||||
"python",
|
||||
"-c",
|
||||
"import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])",
|
||||
],
|
||||
text=True,
|
||||
).strip()
|
||||
result[cid] = alias
|
||||
return result
|
||||
|
||||
@@ -43,8 +51,13 @@ def main():
|
||||
while not stop.is_set():
|
||||
try:
|
||||
response = c.message(actor, chat, "failover-proof")
|
||||
successful.append({"at": time.monotonic(), "id": response["json"]["id"],
|
||||
"instance": response["headers"]["X-Instance-Id"]})
|
||||
successful.append(
|
||||
{
|
||||
"at": time.monotonic(),
|
||||
"id": response["json"]["id"],
|
||||
"instance": response["headers"]["X-Instance-Id"],
|
||||
}
|
||||
)
|
||||
except Failure:
|
||||
errors.append(time.monotonic())
|
||||
|
||||
@@ -53,8 +66,12 @@ def main():
|
||||
thread.start()
|
||||
time.sleep(1)
|
||||
stopped_at = time.monotonic()
|
||||
subprocess.run(["scripts/stop-one.sh"], cwd=ROOT,
|
||||
env={**os.environ, "TARGET_INSTANCE_ID": target_alias}, check=True)
|
||||
subprocess.run(
|
||||
["scripts/stop-one.sh"],
|
||||
cwd=ROOT,
|
||||
env={**os.environ, "TARGET_INSTANCE_ID": target_alias},
|
||||
check=True,
|
||||
)
|
||||
time.sleep(4)
|
||||
stop.set()
|
||||
thread.join()
|
||||
@@ -62,14 +79,22 @@ def main():
|
||||
after = [x for x in successful if x["at"] > stopped_at]
|
||||
assert after and all(x["instance"] != target_alias for x in after[-5:])
|
||||
assert min(x["at"] for x in after) - stopped_at <= 10
|
||||
state = json.loads(subprocess.check_output(["docker", "inspect", target_container], text=True))[0]["State"]
|
||||
state = json.loads(
|
||||
subprocess.check_output(["docker", "inspect", target_container], text=True)
|
||||
)[0]["State"]
|
||||
assert not state["Running"]
|
||||
runs.append({"target_container": target_container, "target_alias": target_alias,
|
||||
"successful_writes": len(successful), "failed_attempts": len(errors),
|
||||
"successful_rps": len(successful) / elapsed,
|
||||
"recovery_seconds": min(x["at"] for x in after) - stopped_at,
|
||||
"observed_after": sorted({x["instance"] for x in after[-5:]}),
|
||||
"target_running_after_hook": state["Running"]})
|
||||
runs.append(
|
||||
{
|
||||
"target_container": target_container,
|
||||
"target_alias": target_alias,
|
||||
"successful_writes": len(successful),
|
||||
"failed_attempts": len(errors),
|
||||
"successful_rps": len(successful) / elapsed,
|
||||
"recovery_seconds": min(x["at"] for x in after) - stopped_at,
|
||||
"observed_after": sorted({x["instance"] for x in after[-5:]}),
|
||||
"target_running_after_hook": state["Running"],
|
||||
}
|
||||
)
|
||||
subprocess.run(["docker", "start", target_container], check=True, stdout=subprocess.DEVNULL)
|
||||
subprocess.run([sys.executable, "scripts/wait_replicas.py", "2"], cwd=ROOT, check=True)
|
||||
# Verify every acknowledged write, without assuming failed requests did not commit.
|
||||
|
||||
+37
-14
@@ -1,4 +1,5 @@
|
||||
"""Sequential real image replacement under message traffic."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
@@ -13,7 +14,9 @@ from client import Client, Failure
|
||||
|
||||
|
||||
def snapshots():
|
||||
ids = subprocess.check_output(["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True).split()
|
||||
ids = subprocess.check_output(
|
||||
["docker", "compose", "ps", "-q", "api"], cwd=ROOT, text=True
|
||||
).split()
|
||||
rows = []
|
||||
for cid in ids:
|
||||
raw = json.loads(subprocess.check_output(["docker", "inspect", cid], text=True))[0]
|
||||
@@ -22,28 +25,43 @@ def snapshots():
|
||||
|
||||
|
||||
def main():
|
||||
subprocess.run(["docker", "build", "-t", "mtusi-vasilev:lab3-rolling", "-"],
|
||||
input="FROM mtusi-vasilev:lab3\nLABEL org.mtusi.rolling-proof=revision-2\n", text=True, check=True)
|
||||
subprocess.run(
|
||||
["docker", "build", "-t", "mtusi-vasilev:lab3-rolling", "-"],
|
||||
input="FROM mtusi-vasilev:lab3\nLABEL org.mtusi.rolling-proof=revision-2\n",
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
before = snapshots()
|
||||
c = Client("http://127.0.0.1:8083")
|
||||
actor, _, _, chat = c.bootstrap()
|
||||
stop = threading.Event()
|
||||
successes, errors = [], []
|
||||
|
||||
def traffic():
|
||||
while not stop.is_set():
|
||||
try:
|
||||
response = c.message(actor, chat, "rolling-proof")
|
||||
successes.append({"id": response["json"]["id"], "at": time.monotonic(),
|
||||
"instance": response["headers"]["X-Instance-Id"]})
|
||||
successes.append(
|
||||
{
|
||||
"id": response["json"]["id"],
|
||||
"at": time.monotonic(),
|
||||
"instance": response["headers"]["X-Instance-Id"],
|
||||
}
|
||||
)
|
||||
except Failure:
|
||||
errors.append(time.monotonic())
|
||||
|
||||
started = time.monotonic()
|
||||
thread = threading.Thread(target=traffic)
|
||||
thread.start()
|
||||
try:
|
||||
time.sleep(1)
|
||||
subprocess.run(["scripts/rolling-update.sh"], cwd=ROOT,
|
||||
env={**os.environ, "API_IMAGE": "mtusi-vasilev:lab3-rolling"}, check=True)
|
||||
subprocess.run(
|
||||
["scripts/rolling-update.sh"],
|
||||
cwd=ROOT,
|
||||
env={**os.environ, "API_IMAGE": "mtusi-vasilev:lab3-rolling"},
|
||||
check=True,
|
||||
)
|
||||
time.sleep(2)
|
||||
finally:
|
||||
stop.set()
|
||||
@@ -64,13 +82,18 @@ def main():
|
||||
break
|
||||
assert {x["id"] for x in successes} <= {x["id"] for x in items}
|
||||
times = [started, *[x["at"] for x in successes], started + elapsed]
|
||||
result = {"before": before, "after": after, "successful_writes": len(successes),
|
||||
"failed_attempts": len(errors), "elapsed_seconds": elapsed,
|
||||
"successful_rps": len(successes) / elapsed,
|
||||
"observed_instances": sorted({x["instance"] for x in successes}),
|
||||
"largest_success_gap_seconds": max(b-a for a,b in zip(times,times[1:])),
|
||||
"all_confirmed_present": True,
|
||||
"note": "Target image differs by revision label; same API/schema. This verifies sequential replacement, not arbitrary schema compatibility."}
|
||||
result = {
|
||||
"before": before,
|
||||
"after": after,
|
||||
"successful_writes": len(successes),
|
||||
"failed_attempts": len(errors),
|
||||
"elapsed_seconds": elapsed,
|
||||
"successful_rps": len(successes) / elapsed,
|
||||
"observed_instances": sorted({x["instance"] for x in successes}),
|
||||
"largest_success_gap_seconds": max(b - a for a, b in zip(times, times[1:])),
|
||||
"all_confirmed_present": True,
|
||||
"note": "Target image differs by revision label; same API/schema. This verifies sequential replacement, not arbitrary schema compatibility.",
|
||||
}
|
||||
(ROOT / "evidence/rolling-update.json").write_text(json.dumps(result, indent=2) + "\n")
|
||||
print(json.dumps(result, indent=2))
|
||||
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import importlib.util
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--lab',type=int,default=3)
|
||||
parser.add_argument('--repo',type=Path)
|
||||
parser.add_argument('--base-url')
|
||||
parser.add_argument('--output',type=Path)
|
||||
args=parser.parse_args()
|
||||
repo=(args.repo or Path(__file__).resolve().parents[1]).resolve()
|
||||
spec=importlib.util.spec_from_file_location('pub',repo/'tests/client.py')
|
||||
m=importlib.util.module_from_spec(spec);spec.loader.exec_module(m)
|
||||
c=m.Client(args.base_url or f'http://127.0.0.1:{8080+args.lab}',timeout=10)
|
||||
rows=[]
|
||||
def compose(*args):
|
||||
return subprocess.check_output(['docker','compose',*args],cwd=repo,text=True).strip()
|
||||
def sql(query):
|
||||
return subprocess.check_output(['docker','exec',pg,'psql','-U','messenger','-d','messenger','-At','-c',query],text=True).strip()
|
||||
def direct(cid,path,token=None):
|
||||
code='''import json,sys,time,urllib.request,urllib.error
|
||||
p=json.load(sys.stdin);start=time.monotonic()
|
||||
r=urllib.request.Request('http://127.0.0.1:8080'+p['path'],headers={'Authorization':'Bearer '+p['token']} if p.get('token') else {})
|
||||
try: response=urllib.request.urlopen(r,timeout=9)
|
||||
except urllib.error.HTTPError as e: response=e
|
||||
with response:
|
||||
body=response.read();result={'status':response.code,'elapsed_s':time.monotonic()-start,'request_id':response.headers.get('X-Request-Id'),'instance_id':response.headers.get('X-Instance-Id'),'body':json.loads(body)}
|
||||
print(json.dumps(result))'''
|
||||
out=subprocess.check_output(['docker','exec','-i',cid,'python','-c',code],input=json.dumps({'path':path,'token':token}),text=True,timeout=15)
|
||||
return json.loads(out)
|
||||
def check(label,fn):
|
||||
start=time.monotonic()
|
||||
try: detail=fn(); row={'case':label,'status':'PASS','detail':detail}
|
||||
except Exception as e: row={'case':label,'status':'FAIL','error':str(e)[:500]}
|
||||
row['elapsed_s']=time.monotonic()-start; rows.append(row);print(row,flush=True)
|
||||
pg=compose('ps','-q','postgres');apis=compose('ps','-q','api').split()
|
||||
alice,bob,eve,chat=c.bootstrap()
|
||||
path='/api/v1/chats/'+chat['id']+'/messages'
|
||||
original=c.message(alice,chat,'fault replay',key='root-fault-replay')['json']
|
||||
def acl_replay():
|
||||
sql(f"SET session_replication_role=replica; DELETE FROM members WHERE chat_id='{chat['id']}' AND user_id='{alice['id']}';")
|
||||
try:
|
||||
for text in ('fault replay','different'):
|
||||
c.call('POST',path,404,actor=alice,body={'text':text},headers={'Idempotency-Key':'root-fault-replay'})
|
||||
finally: sql(f"INSERT INTO members VALUES ('{chat['id']}','{alice['id']}',0);")
|
||||
return {'same_body':404,'changed_body':404}
|
||||
def schema_ready():
|
||||
sql('ALTER TABLE message_requests RENAME TO message_requests_root_probe;')
|
||||
try:
|
||||
values=[direct(cid,'/health/ready') for cid in apis]
|
||||
assert all(x['status']==503 for x in values),values
|
||||
return values
|
||||
finally: sql('ALTER TABLE message_requests_root_probe RENAME TO message_requests;')
|
||||
def paused_pg():
|
||||
# Warm both pools before freezing the real server.
|
||||
for cid in apis: assert direct(cid,'/api/v1/users/me',alice['token'])['status']==200
|
||||
subprocess.run(['docker','pause',pg],check=True,capture_output=True)
|
||||
try:
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as ex:
|
||||
values=list(ex.map(lambda item:direct(apis[0],item,alice['token']),('/health/ready','/api/v1/users/me')))
|
||||
assert all(x['status']==503 and x['elapsed_s']<8 for x in values),values
|
||||
assert all(x['request_id']==x['body']['error']['request_id'] and x['instance_id'] for x in values)
|
||||
finally: subprocess.run(['docker','unpause',pg],check=True,capture_output=True)
|
||||
deadline=time.monotonic()+25
|
||||
while time.monotonic()<deadline:
|
||||
try:
|
||||
recovered=direct(apis[0],'/api/v1/users/me',alice['token'])
|
||||
if recovered['status']==200: break
|
||||
except Exception: pass
|
||||
time.sleep(.5)
|
||||
else: raise AssertionError('API did not recover')
|
||||
return {'during_pause':values,'recovered_status':recovered['status']}
|
||||
def logout_lock():
|
||||
deadline=time.monotonic()+20
|
||||
while time.monotonic()<deadline:
|
||||
try:
|
||||
c.call('GET','/health/ready',schema='Health');break
|
||||
except Exception: time.sleep(.3)
|
||||
actor=c.user('lockout')
|
||||
proc=subprocess.Popen(['docker','exec','-i',pg,'psql','-U','messenger','-d','messenger','-At'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,text=True)
|
||||
try:
|
||||
proc.stdin.write("BEGIN; LOCK TABLE sessions IN SHARE MODE; SELECT 'ROOT_LOCKED';\n");proc.stdin.flush()
|
||||
while True:
|
||||
line=proc.stdout.readline()
|
||||
if 'ROOT_LOCKED' in line: break
|
||||
if not line: raise AssertionError('lock setup failed')
|
||||
c.call('DELETE','/api/v1/auth/sessions/current',503,actor=actor)
|
||||
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
finally:
|
||||
proc.stdin.write('ROLLBACK;\n\\q\n');proc.stdin.flush();proc.communicate(timeout=8)
|
||||
c.call('DELETE','/api/v1/auth/sessions/current',204,actor=actor)
|
||||
c.call('GET','/api/v1/users/me',401,actor=actor)
|
||||
return {'blocked_logout':503,'retry':204,'old_token':401}
|
||||
check('ACL precedes both replay and conflict',acl_replay)
|
||||
check('readiness detects missing mandatory table',schema_ready)
|
||||
check('paused PostgreSQL bounded API error and recovery',paused_pg)
|
||||
check('logout cannot falsely succeed under DB lock',logout_lock)
|
||||
out=args.output or repo/'evidence/teacher-faults-repeat.json';out.write_text(json.dumps({'tests':rows},ensure_ascii=False,indent=2)+'\n')
|
||||
raise SystemExit(any(x['status']=='FAIL' for x in rows))
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Independent HTTP regressions derived from teacher review; no production mutation hooks."""
|
||||
import argparse
|
||||
import base64
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import struct
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.parse
|
||||
import uuid
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def png_chunk(kind, content):
|
||||
return struct.pack('!I',len(content))+kind+content+struct.pack('!I',zlib.crc32(kind+content)&0xffffffff)
|
||||
|
||||
|
||||
def tiny_png(filter_byte=0, bad_idat=False, width=1, height=1):
|
||||
return (b'\x89PNG\r\n\x1a\n'+png_chunk(b'IHDR',struct.pack('!IIBBBBB',width,height,8,2,0,0,0))
|
||||
+png_chunk(b'IDAT',b'not-a-zlib-stream' if bad_idat else zlib.compress(bytes([filter_byte,0,0,0])))
|
||||
+png_chunk(b'IEND',b''))
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser()
|
||||
p.add_argument('--lab',type=int,choices=(3,4,5),required=True)
|
||||
p.add_argument('--repo',type=Path,required=True)
|
||||
p.add_argument('--base-url',required=True)
|
||||
p.add_argument('--output',type=Path,required=True)
|
||||
a=p.parse_args()
|
||||
spec=importlib.util.spec_from_file_location('public_client',a.repo/'tests/client.py')
|
||||
mod=importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(mod)
|
||||
c=mod.Client(a.base_url,timeout=12)
|
||||
results=[]
|
||||
def case(label,fn):
|
||||
started=time.monotonic()
|
||||
try:
|
||||
fn()
|
||||
row={'case':label,'status':'PASS'}
|
||||
except Exception as e:
|
||||
row={'case':label,'status':'FAIL','error':str(e)[:700]}
|
||||
row['elapsed_s']=round(time.monotonic()-started,4)
|
||||
results.append(row)
|
||||
print(row,flush=True)
|
||||
alice,bob,eve,chat=c.bootstrap()
|
||||
path='/api/v1/chats/'+chat['id']+'/messages'
|
||||
def msg(body,key=None,actor=alice,status=201):
|
||||
return c.call('POST',path,status,body=body,actor=actor,headers={'Idempotency-Key':uuid.uuid4().hex if key is None else key},schema='Message' if status==201 else None)
|
||||
def equal(actual,expected):
|
||||
assert actual==expected,'value did not match expected result'
|
||||
for body in ({'text':None},{'text':123},{'text':True},{'text':[]},{'text':'x','TEXT':'y'},{'Text':'x'},{'text':'x','attachment_ids':None},{'text':'x','attachment_ids':{}},{}):
|
||||
case('message rejects wrong types/extra '+str(body),lambda body=body:msg(body,status=400))
|
||||
for key in ('','contains space','x'*129,'bad/key'):
|
||||
case('invalid idempotency key '+repr(key[:20]),lambda key=key:msg({'text':'x'},key=key,status=400))
|
||||
case('missing idempotency key',lambda:c.call('POST',path,400,body={'text':'x'},actor=alice))
|
||||
for body in (b'{bad',b'\xff',b'{"text":"x"}{}'):
|
||||
case('malformed UTF8/JSON '+repr(body),lambda body=body:c.call('POST',path,400,raw=body,actor=alice,headers={'Content-Type':'application/json','Idempotency-Key':uuid.uuid4().hex}))
|
||||
for endpoint in ('/api/v1/chats',path):
|
||||
for query in ('limit=','limit=0','limit=101','limit=abc','limit=1&limit=2','cursor=','cursor=W10','cursor=bnVsbA'):
|
||||
case('pagination '+endpoint+'?'+query,lambda endpoint=endpoint,query=query:c.call('GET',endpoint+'?'+query,400,actor=alice))
|
||||
c.message(alice,chat,'cursor1')
|
||||
c.message(alice,chat,'cursor2')
|
||||
cursor=c.call('GET',path+'?limit=1',actor=alice,schema='MessagePage')['json']['next_cursor']
|
||||
decoded=json.loads(base64.urlsafe_b64decode(cursor+'='*(-len(cursor)%4)))
|
||||
if isinstance(decoded,list) and len(decoded)==4:
|
||||
for index,value in ((2,None),(2,1),(2,'2026-13-01T00:00:00Z'),(3,False),(3,[]),(3,{}),(3,'not-a-uuid')):
|
||||
data=list(decoded); data[index]=value
|
||||
corrupt=base64.urlsafe_b64encode(json.dumps(data).encode()).decode().rstrip('=')
|
||||
case('typed cursor '+str((index,value)),lambda corrupt=corrupt:c.call('GET',path+'?cursor='+corrupt,400,actor=alice))
|
||||
case('invalid UUID path',lambda:c.call('GET','/api/v1/chats/not-a-uuid',400,actor=alice))
|
||||
for bad in ([],{},123,False):
|
||||
case('member UUID '+str(bad),lambda bad=bad:c.call('POST','/api/v1/chats',400,actor=alice,body={'title':'x','member_ids':[alice['id'],bad]}))
|
||||
case('unknown member400',lambda:c.call('POST','/api/v1/chats',400,actor=alice,body={'title':'x','member_ids':[alice['id'],str(uuid.uuid4())]}))
|
||||
for body in (b'{bad',b'{}',b'null',b' '):
|
||||
auth=base64.b64encode((alice['registration']['username']+':'+alice['registration']['password']).encode()).decode()
|
||||
case('login rejects body '+repr(body),lambda body=body,auth=auth:c.call('POST','/api/v1/auth/sessions',400,raw=body,headers={'Authorization':'Basic '+auth,'Content-Type':'application/json'}))
|
||||
case('nonASCII Bearer401',lambda:c.call('GET','/api/v1/users/me',401,headers={'Authorization':'Bearer é'}))
|
||||
for suffix in ('\n',' '):
|
||||
case('username trailing whitespace',lambda suffix=suffix:c.call('POST','/api/v1/users',400,body={'username':'review'+uuid.uuid4().hex[:8]+suffix,'display_name':'x','password':'Password123456'}))
|
||||
case('blank password400',lambda:c.call('POST','/api/v1/users',400,body={'username':'r'+uuid.uuid4().hex[:15],'display_name':'x','password':' '*12}))
|
||||
def unicode_login():
|
||||
username='r'+uuid.uuid4().hex[:15]
|
||||
password='я:'*64
|
||||
user=c.call('POST','/api/v1/users',201,body={'username':username,'display_name':'Никита\0имя','password':password},schema='User')['json']
|
||||
equal(user['display_name'],'Никита\0имя')
|
||||
auth=base64.b64encode((username+':'+password).encode()).decode()
|
||||
session=c.call('POST','/api/v1/auth/sessions',201,headers={'Authorization':'Basic '+auth},schema='Session')['json']
|
||||
equal(session['user'],user)
|
||||
case('UTF8 colon128 password and NUL display name',unicode_login)
|
||||
def canonical_repeat():
|
||||
key=uuid.uuid4().hex
|
||||
original=msg({'text':'canonical'},key)['json']
|
||||
equal(msg({'attachment_ids':[],'text':'canonical'},key)['json'],original)
|
||||
msg({'text':'changed'},key,status=409)
|
||||
case('omitted attachment_ids equals [] and conflict',canonical_repeat)
|
||||
def concurrent_repeat():
|
||||
key=uuid.uuid4().hex
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=16) as ex:
|
||||
rows=list(ex.map(lambda _:msg({'text':'concurrent'},key)['json'],range(32)))
|
||||
equal(len({x['id'] for x in rows}),1)
|
||||
equal(len({json.dumps(x,sort_keys=True) for x in rows}),1)
|
||||
case('32 concurrent same key all201 same message',concurrent_repeat)
|
||||
def independent_scope():
|
||||
key=uuid.uuid4().hex
|
||||
one=msg({'text':'scope'},key)['json']
|
||||
two=msg({'text':'scope'},key,actor=bob)['json']
|
||||
assert one['id']!=two['id'] and two['sender_id']==bob['id']
|
||||
another=c.call('POST','/api/v1/chats',201,actor=alice,body={'title':'another','member_ids':[alice['id']]},schema='Chat')['json']
|
||||
three=c.message(alice,another,'scope',key=key)['json']
|
||||
assert three['id']!=one['id'] and three['chat_id']==another['id']
|
||||
case('key scope user and chat',independent_scope)
|
||||
case('outsider cannot send',lambda:msg({'text':'outsider'},actor=eve,status=404))
|
||||
def nul_message():
|
||||
key=uuid.uuid4().hex
|
||||
original=msg({'text':'abc\0def'},key)['json']
|
||||
equal(original['text'],'abc\0def')
|
||||
equal(msg({'text':'abc\0def'},key)['json'],original)
|
||||
allrows=c.call('GET',path+'?limit=100',actor=bob,schema='MessagePage')['json']['items']
|
||||
equal(next(x for x in allrows if x['id']==original['id']),original)
|
||||
case('NUL message exact post replay history',nul_message)
|
||||
def logout():
|
||||
actor=c.user('logout')
|
||||
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
c.call('DELETE','/api/v1/auth/sessions/current',204,actor=actor)
|
||||
for _ in range(4): c.call('GET','/api/v1/users/me',401,actor=actor)
|
||||
case('logout visible through all replicas',logout)
|
||||
if a.lab>=4:
|
||||
fixture=(a.repo/'tests/fixtures/sample.png').read_bytes()
|
||||
upload='/api/v1/chats/'+chat['id']+'/attachments'
|
||||
def upl(raw=fixture,mime='image/png',name='sample.png',expected=None):
|
||||
return c.call('POST',upload+'?filename='+urllib.parse.quote(name,safe=''),expected or (201 if a.lab==4 else 202),actor=alice,raw=raw,headers={'Content-Type':mime},schema='Attachment' if expected is None else None)
|
||||
for label,raw,mime in (
|
||||
('header only',fixture[:33],'image/png'),('bad IDAT',tiny_png(bad_idat=True),'image/png'),
|
||||
('invalid filter',tiny_png(filter_byte=5),'image/png'),
|
||||
('empty JPEG scan',bytes.fromhex('ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9'),'image/jpeg'),
|
||||
('MIME mismatch',fixture,'image/jpeg')):
|
||||
case('image rejects '+label,lambda raw=raw,mime=mime:upl(raw=raw,mime=mime,expected=415))
|
||||
jpeg_code="from PIL import Image; from io import BytesIO; import random,sys; im=Image.frombytes('RGB',(800,600),random.Random(42).randbytes(800*600*3)); b=BytesIO(); im.save(b,format='JPEG',quality=95); sys.stdout.buffer.write(b.getvalue())"
|
||||
jpeg=subprocess.check_output([str(a.repo.absolute()/'.venv/bin/python'),'-c',jpeg_code])
|
||||
case('valid real JPEG accepted',lambda:upl(raw=jpeg,mime='image/jpeg',name='real.jpg'))
|
||||
case('half JPEG entropy with EOI rejected',lambda:upl(raw=jpeg[:len(jpeg)//2]+b'\xff\xd9',mime='image/jpeg',name='half.jpg',expected=415))
|
||||
sos=jpeg.index(b'\xff\xda')
|
||||
bad_sos=bytearray(jpeg);bad_sos[sos+5]=250
|
||||
case('JPEG SOS unknown component rejected',lambda:upl(raw=bytes(bad_sos),mime='image/jpeg',name='sos.jpg',expected=415))
|
||||
palette=(b'\x89PNG\r\n\x1a\n'+png_chunk(b'IHDR',struct.pack('!IIBBBBB',1,1,8,3,0,0,0))+png_chunk(b'PLTE',bytes(771))+png_chunk(b'IDAT',zlib.compress(b'\0\0'))+png_chunk(b'IEND',b''))
|
||||
case('PNG invalid palette257 rejected',lambda:upl(raw=palette,expected=415))
|
||||
for name in ('../x.png','a\\b.png','a\0b.png','a\x7fb.png','a\x85b.png',' '):
|
||||
case('filename '+repr(name),lambda name=name:upl(name=name,expected=400))
|
||||
case('filename missing',lambda:c.call('POST',upload,400,actor=alice,raw=fixture,headers={'Content-Type':'image/png'}))
|
||||
case('pixel bomb413',lambda:upl(raw=tiny_png(width=20000,height=10000),expected=413))
|
||||
case('too many bytes413',lambda:upl(raw=fixture+b'x'*(10485761-len(fixture)),expected=413))
|
||||
attachments=[]
|
||||
for _ in range(2):
|
||||
att=upl()['json']
|
||||
if a.lab==5: att=c.wait_ready(alice,att)
|
||||
attachments.append(att)
|
||||
def attachment_checks():
|
||||
ids=[x['id'] for x in reversed(attachments)]
|
||||
key=uuid.uuid4().hex
|
||||
sent=msg({'text':'','attachment_ids':ids},key)['json']
|
||||
equal(sent['attachment_ids'],ids)
|
||||
equal(msg({'text':'','attachment_ids':ids},key)['json'],sent)
|
||||
msg({'text':'','attachment_ids':list(reversed(ids))},key,status=409)
|
||||
rows=c.call('GET',path+'?limit=100',actor=bob,schema='MessagePage')['json']['items']
|
||||
equal(next(x for x in rows if x['id']==sent['id']),sent)
|
||||
for att in attachments:
|
||||
route='/api/v1/attachments/'+att['id']
|
||||
for suffix in ('','/content'): c.call('GET',route+suffix,404,actor=eve)
|
||||
content=c.call('GET',route+'/content',actor=bob)['bytes']
|
||||
equal(hashlib.sha256(content).hexdigest(),att['sha256'])
|
||||
msg({'text':'unknown','attachment_ids':[str(uuid.uuid4())]},status=404)
|
||||
case('attachment order ACL original checksum',attachment_checks)
|
||||
if a.lab==4:
|
||||
case('ready missing thumbnail404',lambda:c.call('GET','/api/v1/attachments/'+attachments[0]['id']+'/content?variant=thumbnail',404,actor=alice))
|
||||
# Valid PNG padded by a legal ancillary chunk, exactly 10 MiB.
|
||||
def exact_limit():
|
||||
base=tiny_png()
|
||||
content=base[:-12]+png_chunk(b'tEXt',b'padding\0'+b'x'*(10485760-len(base)-12-8))+base[-12:]
|
||||
equal(len(content),10485760)
|
||||
att=upl(raw=content)['json']
|
||||
equal(att['size_bytes'],10485760)
|
||||
case('exact10MiB allowed through LB',exact_limit)
|
||||
report={'lab':a.lab,'base_url':a.base_url,'tests':results,'passed':sum(x['status']=='PASS' for x in results),'failed':sum(x['status']=='FAIL' for x in results)}
|
||||
a.output.parent.mkdir(parents=True,exist_ok=True)
|
||||
a.output.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
||||
print('TOTAL',report['passed'],'PASS',report['failed'],'FAIL',flush=True)
|
||||
return int(report['failed']>0)
|
||||
|
||||
if __name__=='__main__': raise SystemExit(main())
|
||||
Reference in New Issue
Block a user