feat(lab4): private S3 attachments and safe storage reconciliation
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
docker compose restart postgres s3 api1 api2 lb
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart postgres s3 api1 api2 lb
|
||||
@@ -0,0 +1,174 @@
|
||||
"""Reproduce evidence on a LOCAL synthetic Compose stand, Python 3.9+ and Docker.
|
||||
|
||||
No secrets in summaries. Fails immediately on a broken check; restores stopped services.
|
||||
"""
|
||||
import concurrent.futures
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OUT = ROOT/'evidence'
|
||||
OUT.mkdir(exist_ok=True)
|
||||
os.chdir(ROOT)
|
||||
os.environ['PYTHONUTF8'] = '1'
|
||||
|
||||
|
||||
def command(args, file=None):
|
||||
result = subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.PIPE,
|
||||
text=True,encoding='utf-8',errors='replace')
|
||||
if file:
|
||||
(OUT/file).write_text(result.stdout,encoding='utf-8')
|
||||
if result.returncode:
|
||||
print(result.stdout+result.stderr)
|
||||
raise RuntimeError('Command failed: '+' '.join(args[:5]))
|
||||
return result.stdout
|
||||
|
||||
|
||||
def ready():
|
||||
for _ in range(90):
|
||||
try:
|
||||
with urllib.request.urlopen(os.environ['BASE_URL']+'/health/ready',timeout=3) as r:
|
||||
if r.status==200:
|
||||
return
|
||||
except (OSError,urllib.error.URLError):
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise RuntimeError('Readiness deadline')
|
||||
|
||||
|
||||
def snapshot(name):
|
||||
# No environment/inspect Config: Docker environments can contain secrets.
|
||||
data = command(['docker','compose','ps','--all','--format','json'])
|
||||
# Compose emits JSON Lines; evidence files must be valid standalone JSON.
|
||||
try:
|
||||
parsed = json.loads(data)
|
||||
rows = parsed if isinstance(parsed,list) else [parsed]
|
||||
except json.JSONDecodeError:
|
||||
rows = [json.loads(line) for line in data.splitlines() if line.strip()]
|
||||
(OUT/name).write_text(json.dumps(rows,indent=2),encoding='utf-8')
|
||||
starts = command(['docker','inspect','--format',
|
||||
'{{.Name}} {{.State.StartedAt}}',
|
||||
'chernenko-lab4-postgres-1','chernenko-lab4-s3-1',
|
||||
'chernenko-lab4-api1-1','chernenko-lab4-api2-1','chernenko-lab4-lb-1'])
|
||||
(OUT/(Path(name).stem+'-started.txt')).write_text(starts,encoding='utf-8')
|
||||
|
||||
|
||||
def execute_inside(script,filename):
|
||||
return command(['docker','compose','exec','-T','api1','python',script],filename)
|
||||
|
||||
|
||||
def main():
|
||||
command(['docker','compose','config','--quiet'],'compose-config.txt')
|
||||
(OUT/'compose-config.txt').write_text('PASS: docker compose config --quiet (exit 0)\n',encoding='utf-8')
|
||||
address = command(['docker','compose','port','lb','8080']).strip()
|
||||
os.environ['BASE_URL'] = 'http://'+address
|
||||
command(['docker','compose','exec','-T','api1','make','check','validate','PYTHON=python'],'check-validate.txt')
|
||||
ready()
|
||||
command(['docker','compose','exec','-T','api1','make','test','PYTHON=python','BASE_URL=http://lb:8080'],'smoke.txt')
|
||||
execute_inside('student-tests/acceptance.py','acceptance.json')
|
||||
snapshot('before-restart.json')
|
||||
hook = 'scripts/restart.cmd' if os.name=='nt' else 'scripts/restart.sh'
|
||||
command([sys.executable,'-X','utf8','tests/resilience.py','--mode','persistence',
|
||||
'--action-script',hook],'persistence.txt')
|
||||
snapshot('after-restart.json')
|
||||
# Both failure directions deterministically, including bytes and existing session.
|
||||
sys.path.insert(0,str(ROOT/'tests'))
|
||||
from client import Client, check
|
||||
c = Client(timeout=20)
|
||||
a,b,e,ch = c.bootstrap()
|
||||
att,raw = c.attachment(a,ch)
|
||||
c.message(a,ch,'survives',attachments=[att['id']])
|
||||
outages = []
|
||||
for target in ('api1','api2'):
|
||||
command(['docker','compose','stop',target])
|
||||
try:
|
||||
time.sleep(3)
|
||||
result = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=b)
|
||||
check(result['bytes']==raw,'failover changed bytes')
|
||||
check(result['headers']['X-Instance-Id']!=target,'stopped replica served response')
|
||||
c.message(a,ch,'after '+target)
|
||||
outages.append(dict(stopped=target,served_by=result['headers']['X-Instance-Id'],checksum_equal=True))
|
||||
finally:
|
||||
command(['docker','compose','start',target])
|
||||
time.sleep(4)
|
||||
ready()
|
||||
hook = 'scripts/stop-one.cmd' if os.name=='nt' else 'scripts/stop-one.sh'
|
||||
try:
|
||||
command([sys.executable,'-X','utf8','tests/resilience.py','--mode','failover',
|
||||
'--action-script',hook],'failover.txt')
|
||||
snapshot('after-failover.json')
|
||||
finally:
|
||||
command(['docker','compose','start','api1','api2'])
|
||||
time.sleep(4)
|
||||
ready()
|
||||
# Internal API avoids LB-generated 503 masking application behavior. Sessions
|
||||
# are passed only over stdin, never saved in evidence or process arguments.
|
||||
for dependency in ('s3','postgres'):
|
||||
try:
|
||||
command(['docker','compose','stop',dependency])
|
||||
result = subprocess.run(['docker','compose','exec','-T','api2','python',
|
||||
'student-tests/storage_outage.py',dependency],input=json.dumps(
|
||||
{'actor':{'id':a['id'],'token':a['token']},'chat':ch,'attachment':att}),text=True,encoding='utf-8',
|
||||
stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
|
||||
(OUT/(dependency+'-outage.json')).write_text(result.stdout,encoding='utf-8')
|
||||
if result.returncode:
|
||||
raise RuntimeError(dependency+' outage assertions failed: '+result.stdout)
|
||||
finally:
|
||||
command(['docker','compose','start',dependency])
|
||||
time.sleep(4)
|
||||
ready()
|
||||
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=b)['bytes']==raw,
|
||||
'storage restart changed original')
|
||||
(OUT/'replica-directions.json').write_text(json.dumps(outages,indent=2),encoding='utf-8')
|
||||
execute_inside('student-tests/research.py','research.json')
|
||||
samples = []
|
||||
done = threading.Event()
|
||||
def sample():
|
||||
while not done.is_set():
|
||||
raw = command(['docker','stats','--no-stream','--format','{{json .}}',
|
||||
'chernenko-lab4-api1-1','chernenko-lab4-api2-1'])
|
||||
samples.append({'elapsed':round(time.monotonic()-start,3),
|
||||
'replicas':[json.loads(x) for x in raw.splitlines() if x.strip()]})
|
||||
done.wait(0.2)
|
||||
start = time.monotonic()
|
||||
thread = threading.Thread(target=sample)
|
||||
thread.start()
|
||||
probes = {service:subprocess.Popen(['docker','compose','exec','-T',service,'python',
|
||||
'student-tests/memory_probe.py'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,text=True,encoding='utf-8') for service in ('api1','api2')}
|
||||
try:
|
||||
# Separate generator container: its image generation RAM is not charged
|
||||
# to either measured API replica.
|
||||
command(['docker','compose','run','--rm','--no-deps','migrate','python',
|
||||
'student-tests/upload_load.py'],'uploads.json')
|
||||
finally:
|
||||
done.set()
|
||||
thread.join()
|
||||
(OUT/'memory.json').write_text(json.dumps(samples,indent=2),encoding='utf-8')
|
||||
peaks = {}
|
||||
for service,process in probes.items():
|
||||
output,error = process.communicate(input='done',timeout=10)
|
||||
if process.returncode:
|
||||
raise RuntimeError('Memory probe failed: '+error)
|
||||
peaks[service] = json.loads(output)
|
||||
(OUT/'memory-peaks.json').write_text(json.dumps(peaks,indent=2),encoding='utf-8')
|
||||
command([sys.executable,'-X','utf8','tests/load.py','--duration','30','--concurrency','4',
|
||||
'--output','evidence/messages-load.json'],'messages-load.txt')
|
||||
command(['docker','info','--format','CPUs={{.NCPU}} memory_bytes={{.MemTotal}} OS={{.OperatingSystem}}'],
|
||||
'hardware.txt')
|
||||
command(['docker','compose','images'],'images.txt')
|
||||
command(['docker','compose','exec','-T','api1','python','-m','pip','freeze'],'python-packages.txt')
|
||||
command(['docker','compose','run','--rm','bootstrap'],'bootstrap.txt')
|
||||
snapshot('final-state.json')
|
||||
print('PASS: all lab4 scenarios. Sanitized evidence in evidence/')
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,30 @@
|
||||
"""Create LOCAL secrets once; never print their values or overwrite existing state."""
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
folder = root/'secrets'
|
||||
folder.mkdir(exist_ok=True)
|
||||
folder.chmod(0o700)
|
||||
password = folder/'postgres_password'
|
||||
if not password.exists():
|
||||
password.write_text(secrets.token_hex(24))
|
||||
values = {
|
||||
'database_url':'postgresql://messenger:'+password.read_text().strip()+'@postgres:5432/messenger',
|
||||
'minio_root_user':'root_'+secrets.token_hex(8),
|
||||
'minio_root_password':secrets.token_hex(24),
|
||||
's3_access_key_id':'app_'+secrets.token_hex(8),
|
||||
's3_secret_access_key':secrets.token_hex(24),
|
||||
}
|
||||
for name,value in values.items():
|
||||
path = folder/name
|
||||
if not path.exists():
|
||||
path.write_text(value)
|
||||
# Parent directory is owner-only; individual secret mounts are readable by
|
||||
# the unprivileged container user even on native Linux bind mounts.
|
||||
path.chmod(0o444)
|
||||
password.chmod(0o444)
|
||||
env = root/'.env'
|
||||
if not env.exists():
|
||||
env.write_text('SESSION_TTL_SECONDS=3600\n')
|
||||
print('Local secrets ready; no values printed')
|
||||
@@ -0,0 +1,5 @@
|
||||
@echo off
|
||||
cd /d "%~dp0.."
|
||||
if "%TARGET_INSTANCE_ID%"=="api1" (docker compose stop -t 1 api1 & exit /b)
|
||||
if "%TARGET_INSTANCE_ID%"=="api2" (docker compose stop -t 1 api2 & exit /b)
|
||||
exit /b 1
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
case "${TARGET_INSTANCE_ID:-}" in
|
||||
api1|api2) docker compose stop -t 1 "$TARGET_INSTANCE_ID" ;;
|
||||
*) echo 'Unknown instance alias' >&2; exit 1 ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user