feat(lab4): private S3 attachments and safe storage reconciliation

This commit is contained in:
Quzzqq
2026-11-06 16:51:18 +03:00
parent ec540a1a72
commit 435a6465ec
61 changed files with 3106 additions and 33 deletions
+3
View File
@@ -0,0 +1,3 @@
@echo off
cd /d "%~dp0.."
docker compose restart postgres s3 api1 api2 lb
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose restart postgres s3 api1 api2 lb
+174
View File
@@ -0,0 +1,174 @@
"""Reproduce evidence on a LOCAL synthetic Compose stand, Python 3.9+ and Docker.
No secrets in summaries. Fails immediately on a broken check; restores stopped services.
"""
import concurrent.futures
import json
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT/'evidence'
OUT.mkdir(exist_ok=True)
os.chdir(ROOT)
os.environ['PYTHONUTF8'] = '1'
def command(args, file=None):
result = subprocess.run(args,stdout=subprocess.PIPE,stderr=subprocess.PIPE,
text=True,encoding='utf-8',errors='replace')
if file:
(OUT/file).write_text(result.stdout,encoding='utf-8')
if result.returncode:
print(result.stdout+result.stderr)
raise RuntimeError('Command failed: '+' '.join(args[:5]))
return result.stdout
def ready():
for _ in range(90):
try:
with urllib.request.urlopen(os.environ['BASE_URL']+'/health/ready',timeout=3) as r:
if r.status==200:
return
except (OSError,urllib.error.URLError):
pass
time.sleep(1)
raise RuntimeError('Readiness deadline')
def snapshot(name):
# No environment/inspect Config: Docker environments can contain secrets.
data = command(['docker','compose','ps','--all','--format','json'])
# Compose emits JSON Lines; evidence files must be valid standalone JSON.
try:
parsed = json.loads(data)
rows = parsed if isinstance(parsed,list) else [parsed]
except json.JSONDecodeError:
rows = [json.loads(line) for line in data.splitlines() if line.strip()]
(OUT/name).write_text(json.dumps(rows,indent=2),encoding='utf-8')
starts = command(['docker','inspect','--format',
'{{.Name}} {{.State.StartedAt}}',
'chernenko-lab4-postgres-1','chernenko-lab4-s3-1',
'chernenko-lab4-api1-1','chernenko-lab4-api2-1','chernenko-lab4-lb-1'])
(OUT/(Path(name).stem+'-started.txt')).write_text(starts,encoding='utf-8')
def execute_inside(script,filename):
return command(['docker','compose','exec','-T','api1','python',script],filename)
def main():
command(['docker','compose','config','--quiet'],'compose-config.txt')
(OUT/'compose-config.txt').write_text('PASS: docker compose config --quiet (exit 0)\n',encoding='utf-8')
address = command(['docker','compose','port','lb','8080']).strip()
os.environ['BASE_URL'] = 'http://'+address
command(['docker','compose','exec','-T','api1','make','check','validate','PYTHON=python'],'check-validate.txt')
ready()
command(['docker','compose','exec','-T','api1','make','test','PYTHON=python','BASE_URL=http://lb:8080'],'smoke.txt')
execute_inside('student-tests/acceptance.py','acceptance.json')
snapshot('before-restart.json')
hook = 'scripts/restart.cmd' if os.name=='nt' else 'scripts/restart.sh'
command([sys.executable,'-X','utf8','tests/resilience.py','--mode','persistence',
'--action-script',hook],'persistence.txt')
snapshot('after-restart.json')
# Both failure directions deterministically, including bytes and existing session.
sys.path.insert(0,str(ROOT/'tests'))
from client import Client, check
c = Client(timeout=20)
a,b,e,ch = c.bootstrap()
att,raw = c.attachment(a,ch)
c.message(a,ch,'survives',attachments=[att['id']])
outages = []
for target in ('api1','api2'):
command(['docker','compose','stop',target])
try:
time.sleep(3)
result = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=b)
check(result['bytes']==raw,'failover changed bytes')
check(result['headers']['X-Instance-Id']!=target,'stopped replica served response')
c.message(a,ch,'after '+target)
outages.append(dict(stopped=target,served_by=result['headers']['X-Instance-Id'],checksum_equal=True))
finally:
command(['docker','compose','start',target])
time.sleep(4)
ready()
hook = 'scripts/stop-one.cmd' if os.name=='nt' else 'scripts/stop-one.sh'
try:
command([sys.executable,'-X','utf8','tests/resilience.py','--mode','failover',
'--action-script',hook],'failover.txt')
snapshot('after-failover.json')
finally:
command(['docker','compose','start','api1','api2'])
time.sleep(4)
ready()
# Internal API avoids LB-generated 503 masking application behavior. Sessions
# are passed only over stdin, never saved in evidence or process arguments.
for dependency in ('s3','postgres'):
try:
command(['docker','compose','stop',dependency])
result = subprocess.run(['docker','compose','exec','-T','api2','python',
'student-tests/storage_outage.py',dependency],input=json.dumps(
{'actor':{'id':a['id'],'token':a['token']},'chat':ch,'attachment':att}),text=True,encoding='utf-8',
stdout=subprocess.PIPE,stderr=subprocess.STDOUT)
(OUT/(dependency+'-outage.json')).write_text(result.stdout,encoding='utf-8')
if result.returncode:
raise RuntimeError(dependency+' outage assertions failed: '+result.stdout)
finally:
command(['docker','compose','start',dependency])
time.sleep(4)
ready()
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=b)['bytes']==raw,
'storage restart changed original')
(OUT/'replica-directions.json').write_text(json.dumps(outages,indent=2),encoding='utf-8')
execute_inside('student-tests/research.py','research.json')
samples = []
done = threading.Event()
def sample():
while not done.is_set():
raw = command(['docker','stats','--no-stream','--format','{{json .}}',
'chernenko-lab4-api1-1','chernenko-lab4-api2-1'])
samples.append({'elapsed':round(time.monotonic()-start,3),
'replicas':[json.loads(x) for x in raw.splitlines() if x.strip()]})
done.wait(0.2)
start = time.monotonic()
thread = threading.Thread(target=sample)
thread.start()
probes = {service:subprocess.Popen(['docker','compose','exec','-T',service,'python',
'student-tests/memory_probe.py'],stdin=subprocess.PIPE,stdout=subprocess.PIPE,
stderr=subprocess.PIPE,text=True,encoding='utf-8') for service in ('api1','api2')}
try:
# Separate generator container: its image generation RAM is not charged
# to either measured API replica.
command(['docker','compose','run','--rm','--no-deps','migrate','python',
'student-tests/upload_load.py'],'uploads.json')
finally:
done.set()
thread.join()
(OUT/'memory.json').write_text(json.dumps(samples,indent=2),encoding='utf-8')
peaks = {}
for service,process in probes.items():
output,error = process.communicate(input='done',timeout=10)
if process.returncode:
raise RuntimeError('Memory probe failed: '+error)
peaks[service] = json.loads(output)
(OUT/'memory-peaks.json').write_text(json.dumps(peaks,indent=2),encoding='utf-8')
command([sys.executable,'-X','utf8','tests/load.py','--duration','30','--concurrency','4',
'--output','evidence/messages-load.json'],'messages-load.txt')
command(['docker','info','--format','CPUs={{.NCPU}} memory_bytes={{.MemTotal}} OS={{.OperatingSystem}}'],
'hardware.txt')
command(['docker','compose','images'],'images.txt')
command(['docker','compose','exec','-T','api1','python','-m','pip','freeze'],'python-packages.txt')
command(['docker','compose','run','--rm','bootstrap'],'bootstrap.txt')
snapshot('final-state.json')
print('PASS: all lab4 scenarios. Sanitized evidence in evidence/')
if __name__=='__main__':
main()
+30
View File
@@ -0,0 +1,30 @@
"""Create LOCAL secrets once; never print their values or overwrite existing state."""
from pathlib import Path
import secrets
root = Path(__file__).resolve().parents[1]
folder = root/'secrets'
folder.mkdir(exist_ok=True)
folder.chmod(0o700)
password = folder/'postgres_password'
if not password.exists():
password.write_text(secrets.token_hex(24))
values = {
'database_url':'postgresql://messenger:'+password.read_text().strip()+'@postgres:5432/messenger',
'minio_root_user':'root_'+secrets.token_hex(8),
'minio_root_password':secrets.token_hex(24),
's3_access_key_id':'app_'+secrets.token_hex(8),
's3_secret_access_key':secrets.token_hex(24),
}
for name,value in values.items():
path = folder/name
if not path.exists():
path.write_text(value)
# Parent directory is owner-only; individual secret mounts are readable by
# the unprivileged container user even on native Linux bind mounts.
path.chmod(0o444)
password.chmod(0o444)
env = root/'.env'
if not env.exists():
env.write_text('SESSION_TTL_SECONDS=3600\n')
print('Local secrets ready; no values printed')
+5
View File
@@ -0,0 +1,5 @@
@echo off
cd /d "%~dp0.."
if "%TARGET_INSTANCE_ID%"=="api1" (docker compose stop -t 1 api1 & exit /b)
if "%TARGET_INSTANCE_ID%"=="api2" (docker compose stop -t 1 api2 & exit /b)
exit /b 1
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
case "${TARGET_INSTANCE_ID:-}" in
api1|api2) docker compose stop -t 1 "$TARGET_INSTANCE_ID" ;;
*) echo 'Unknown instance alias' >&2; exit 1 ;;
esac