готово

This commit is contained in:
2026-11-12 19:38:03 +03:00
parent ec540a1a72
commit f3ba37f736
142 changed files with 10257 additions and 0 deletions
+63
View File
@@ -0,0 +1,63 @@
"""Export all public working files, including new files; exclude local secrets/state."""
import hashlib
import json
import os
import tarfile
from pathlib import Path
import subprocess
import sys
ROOT=Path(__file__).resolve().parents[1]
subprocess.run([sys.executable,str(ROOT/'scripts/check-course-files.py')],check=True)
paths=subprocess.check_output(['git','ls-files','--cached','--others','--exclude-standard','-z'],cwd=ROOT).split(b'\0')
files={}
for raw in paths:
if not raw:continue
name=os.fsdecode(raw)
if name=='SOURCE_MANIFEST.json':continue
path=ROOT/name
if path.is_file():files[name]=hashlib.sha256(path.read_bytes()).hexdigest()
# Actual non-example credentials must not occur in exported source/evidence.
from dotenv import dotenv_values
example=dotenv_values(ROOT/'student.env.example')
private=[v.encode() for k,v in dotenv_values(ROOT/'.env').items()
if v and len(v)>=12 and v!=example.get(k)
and any(word in k for word in ('PASSWORD','SECRET','TOKEN','API_KEY','DATABASE_URL'))]
unsafe=[]
for name in files:
path=ROOT/name
if name.startswith(('.env','secrets/','backups/','data/')) and name!='.env.example':
unsafe.append(name)
elif any(secret in path.read_bytes() for secret in private):unsafe.append(name)
if unsafe:
raise SystemExit('Не экспортированы файлы с приватными данными: '+', '.join(unsafe))
proof=json.loads((ROOT/'evidence/lab4-clean-start.json').read_text())
changed=[name for name,digest in proof['code_manifest'].items() if files.get(name)!=digest]
if changed:raise SystemExit('После clean-start изменился код: '+', '.join(changed))
required=('Dockerfile','compose.yaml','src/core/request_deadline.py','src/maintenance/reconcile.py',
'student.mk','student.env.example','requirements-student.txt','STUDENT_README.md',
'STUDENT_REPORT.md','course-files.sha256.json','Dockerfile.dockerignore',
'src/core/storage_bootstrap.py','infra/s3/api-policy.json','infra/s3/reconciler-policy.json',
'migrations/versions/008_reconciliation.py',
'scripts/setup-env.py','scripts/check-course-files.py','scripts/test-reconciliation.sh','tests/reconciliation.py',
'tests/lab4_review.py','tests/fixtures/sample.png')
assert all(name in files for name in required),'Required source missing from export'
manifest={'format':1,'lab':4,'author':'Каменский Андрей, БПИ2301',
'clean_start_code_matches':True,'private_credentials_scan':'PASS','files':files}
(ROOT/'SOURCE_MANIFEST.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2)+'\n')
destination=ROOT/'dist';destination.mkdir(exist_ok=True)
archive=destination/'mtusi2026lab4-submission.tar.gz'
with tarfile.open(archive,'w:gz') as output:
for name in sorted([*files,'SOURCE_MANIFEST.json']):
output.add(ROOT/name,arcname='mtusi2026lab4/'+name,recursive=False)
with tarfile.open(archive,'r:gz') as source:
bundled={member.name.removeprefix('mtusi2026lab4/') for member in source.getmembers()}
assert set(files).issubset(bundled) and 'SOURCE_MANIFEST.json' in bundled
assert '.env' not in bundled and not any(name.startswith(('.git/','.venv/')) for name in bundled)
receipt={'result':'PASS','archive':str(archive.relative_to(ROOT)),
'sha256':hashlib.sha256(archive.read_bytes()).hexdigest(),
'source_files':len(files),'required_files_present':True,'clean_start_code_matches':True,
'private_credentials_scan':'PASS','local_env_excluded':True,'course_files_unchanged':True}
# Receipt is separate to avoid a circular archive hash in its own contents.
(destination/'submission-receipt.json').write_text(json.dumps(receipt,ensure_ascii=False,indent=2)+'\n')
print(json.dumps(receipt,ensure_ascii=False,indent=2))
+38
View File
@@ -0,0 +1,38 @@
"""Plot measured scan cost and per-container memory from final acceptance outputs."""
import json
import statistics
from pathlib import Path
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
root=Path(__file__).resolve().parents[1]
def json_rows(name):
result=[]
for line in (root/'evidence'/name).read_text().splitlines():
try:result.append(json.loads(line))
except ValueError:pass
return result
runs=next(r['runs'] for r in json_rows('lab4-final-reconciliation.txt') if r.get('experiment')=='scan_cost')
memory=next(r for r in json_rows('lab4-final-upload-memory.txt') if r.get('result')=='PASS')
fig,axes=plt.subplots(1,2,figsize=(11,4),layout='constrained')
sizes=[10,100,1005]
medians=[statistics.median(r['seconds'] for r in runs if r['objects']==n) for n in sizes]
lows=[m-min(r['seconds'] for r in runs if r['objects']==n) for n,m in zip(sizes,medians)]
highs=[max(r['seconds'] for r in runs if r['objects']==n)-m for n,m in zip(sizes,medians)]
axes[0].bar([str(n) for n in sizes],medians,color='#3766a3',yerr=[lows,highs],capsize=5)
axes[0].set(title='Metadata scan: median and range of 3 runs',xlabel='Objects',ylabel='Seconds')
axes[0].grid(axis='y',alpha=.2)
base=[r['baseline_bytes']/1024**2 for r in memory['memory']]
peak=[r['sampled_peak_bytes']/1024**2 for r in memory['memory']]
axes[1].bar([-.18,.82],base,width=.36,label='Baseline',color='#9bb8d9')
axes[1].bar([.18,1.18],peak,width=.36,label='Sampled peak',color='#3766a3')
axes[1].set(xticks=[0,1],xticklabels=['API 1','API 2'],ylabel='Cgroup memory, MiB',
title='10 concurrent PNG uploads; 20 ms sampling',ylim=(0,max(peak)*1.3))
axes[1].text(.02,.95,'Container limit: 512 MiB',transform=axes[1].transAxes,va='top')
axes[1].legend(loc='upper right');axes[1].grid(axis='y',alpha=.2)
output=root/'evidence/lab4-research';output.mkdir(exist_ok=True)
fig.savefig(output/'results.png',dpi=180)
fig.savefig(output/'results.svg')
(output/'results.json').write_text(json.dumps({'scan_runs':runs,'upload_experiment':memory},ensure_ascii=False,indent=2)+'\n')
print('Created evidence/lab4-research/results.png and results.json')
+31
View File
@@ -0,0 +1,31 @@
"""Управление только API-контейнерами текущего Compose-проекта."""
import json
import os
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def docker(*args):
return subprocess.check_output(["docker", *args], cwd=ROOT, text=True).strip()
def compose(*args):
return docker("compose", "--env-file", os.environ.get("ENV_FILE", ".env"), *args)
def containers():
return compose("ps", "--all", "-q", "api").split()
def describe(container):
# Никогда не выводим Config.Env: там credentials.
return json.loads(docker("inspect", "--format", "{{json .State}}", container))
def instance_id(container):
code = "import urllib.request; r=urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2); print(r.headers['X-Instance-Id'])"
return docker("exec", container, "python", "-c", code)
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# Полный restart работающих ролей; volumes не удаляются.
docker compose --env-file "${ENV_FILE:-.env}" restart postgres redis minio
docker compose --env-file "${ENV_FILE:-.env}" up -d --wait --force-recreate api
docker compose --env-file "${ENV_FILE:-.env}" restart reconciler lb
+26
View File
@@ -0,0 +1,26 @@
"""Create local synthetic credentials without overwriting an existing .env."""
import argparse
import os
import secrets
from pathlib import Path
root=Path(__file__).resolve().parents[1]
parser=argparse.ArgumentParser(description=__doc__)
parser.add_argument('--output',type=Path,default=root/'.env')
args=parser.parse_args()
values={
'POSTGRES_PASSWORD':secrets.token_hex(24),
'ADMIN_PASSWORD':secrets.token_hex(24),
'MINIO_ROOT_PASSWORD':secrets.token_hex(24),
'S3_SECRET_ACCESS_KEY':secrets.token_hex(24),
'S3_RECONCILE_SECRET_ACCESS_KEY':secrets.token_hex(24),
}
lines=(root/'student.env.example').read_text().splitlines()
content='\n'.join(key+'='+values[key] if '=' in line and (key:=line.split('=',1)[0]) in values else line for line in lines)+'\n'
try:
descriptor=os.open(args.output,os.O_WRONLY|os.O_CREAT|os.O_EXCL,0o600)
except FileExistsError:
parser.exit(1,'Файл уже существует; его настройки и пароли сохранены.\n')
with os.fdopen(descriptor,'w') as stream:
stream.write(content)
print('Создан файл локальных настроек с новыми паролями; значения не выводятся.')
+34
View File
@@ -0,0 +1,34 @@
"""Сопоставить X-Instance-Id реальному контейнеру и оставить его остановленным."""
import json
import os
import time
from replicas import containers, describe, docker, instance_id
target = os.environ["TARGET_INSTANCE_ID"]
running = [c for c in containers() if describe(c)["Running"]]
if len(running) < 2:
raise SystemExit("Need at least two running API containers")
for container in running:
if instance_id(container) == target:
before = describe(container)
started = time.monotonic()
docker("stop", "--time", "10", container)
after = describe(container)
assert not after["Running"]
print(
json.dumps(
{
"container": container[:12],
"instance_id": target,
"started_at": before["StartedAt"],
"finished_at": after["FinishedAt"],
"stop_seconds": round(time.monotonic() - started, 3),
"exit_code": after["ExitCode"],
}
)
)
break
else:
raise SystemExit("Requested instance not found in this Compose project")
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec "${PYTHON:-python3}" scripts/stop-one.py
+92
View File
@@ -0,0 +1,92 @@
"""Build the complete source snapshot and boot a separate stack with empty volumes.
The active project and its data are untouched. Credentials and source copies are
kept in a temporary directory; only our UUID-named test project is removed.
"""
import hashlib
import json
import os
import secrets
import shutil
import socket
import subprocess
import sys
import time
from pathlib import Path
from tempfile import TemporaryDirectory
ROOT=Path(__file__).resolve().parents[1]
project='lab4-clean-'+secrets.token_hex(4)
with socket.socket() as sock:
sock.bind(('127.0.0.1',0));port=sock.getsockname()[1]
inventory=subprocess.run(['git','ls-files','--cached','--others','--exclude-standard','-z'],cwd=ROOT,capture_output=True)
if inventory.returncode==0:
files=inventory.stdout.split(b'\0')
else:
manifest_path=ROOT/'SOURCE_MANIFEST.json'
if not manifest_path.is_file():
raise SystemExit('Нужен Git clone или архив с SOURCE_MANIFEST.json')
files=[os.fsencode(name) for name in json.loads(manifest_path.read_text())['files']]
proof={'project':project,'empty_volumes':True,'source':'complete working snapshot including new files',
'build_cache':'API rebuilt with --no-cache; pinned MinIO build may use BuildKit cache'}
with TemporaryDirectory(prefix='lab4-clean-') as temporary:
source=Path(temporary)/'source';source.mkdir()
manifest={}
for raw in files:
if not raw:continue
relative=Path(os.fsdecode(raw))
original=ROOT/relative
if not original.is_file():continue
target=source/relative;target.parent.mkdir(parents=True,exist_ok=True)
shutil.copy2(original,target)
if relative.parts[0] in ('src','migrations','infra') or str(relative) in ('Dockerfile','Dockerfile.dockerignore','requirements.txt','requirements-student.txt',
'student.env.example','student.mk','compose.yaml','alembic.ini'):
manifest[str(relative)]=hashlib.sha256(original.read_bytes()).hexdigest()
subprocess.run([sys.executable,str(source/'scripts/setup-env.py')],cwd=source,check=True,capture_output=True)
env_file=source/'.env'
text=env_file.read_text().replace('APP_PORT=8080','APP_PORT='+str(port))
env_file.write_text(text)
environment={**os.environ,'COMPOSE_PROJECT_NAME':project,'ENV_FILE':str(env_file),
'APP_PORT':str(port),'APP_IMAGE':'mtusi-messenger:'+project,
'MINIO_IMAGE':'mtusi-minio:'+project}
# A clean launch must not inherit deployment overrides from the current shell.
for name in ('POSTGRES_PASSWORD','POSTGRES_USER','POSTGRES_DB','MINIO_ROOT_USER',
'MINIO_ROOT_PASSWORD','S3_ACCESS_KEY_ID','S3_SECRET_ACCESS_KEY','S3_RECONCILE_ACCESS_KEY_ID',
'S3_RECONCILE_SECRET_ACCESS_KEY','UPLOAD_CONCURRENCY','API_REPLICAS','API_MEMORY'):
environment.pop(name,None)
command=['docker','compose','--env-file',str(env_file)]
def run(*args,timeout=300):
return subprocess.run(args,cwd=source,env=environment,text=True,check=True,
capture_output=True,timeout=timeout)
def run_build(*args):
started=time.perf_counter()
result=run(*command,'build',*args,timeout=900)
return {'seconds':round(time.perf_counter()-started,3),'exit_code':result.returncode}
try:
proof['api_build']=run_build('--no-cache','api')
proof['minio_build']=run_build('minio')
run(*command,'up','-d','--no-build','--wait',timeout=180)
version=run(*command,'exec','-T','postgres','sh','-c',
'exec psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "SELECT version_num FROM alembic_version"').stdout.strip()
assert version=='008',version
proof['schema_revision']=version
proof['public_smoke']=run(sys.executable,'tests/smoke.py','--base-url',f'http://127.0.0.1:{port}').stdout.strip()
proof['attachments']=run(sys.executable,'tests/attachments.py','--base-url',f'http://127.0.0.1:{port}').stdout.strip()
proof['reconciler']=json.loads(run(*command,'exec','-T','reconciler','python','-m','src.maintenance.reconcile').stdout)
assert proof['reconciler']['result']=='ok'
rows=run(*command,'ps','--format','json').stdout
proof['services']=[{'service':r['Service'],'state':r['State'],'health':r.get('Health','')}
for r in (json.loads(line) for line in rows.splitlines() if line.strip())]
proof['code_manifest']=manifest
proof['result']='PASS'
except subprocess.CalledProcessError as error:
# Never dump compose config/environment; build/application outputs exclude credentials.
print((error.stdout+error.stderr)[-4000:])
raise
finally:
run(*command,'down','--volumes','--remove-orphans',timeout=90)
proof['temporary_project_and_volumes_removed']=True
run('docker','image','rm',environment['APP_IMAGE'],environment['MINIO_IMAGE'],timeout=60)
proof['temporary_image_tags_removed']=True
(ROOT/'evidence/lab4-clean-start.json').write_text(json.dumps(proof,ensure_ascii=False,indent=2)+'\n')
print(json.dumps({k:v for k,v in proof.items() if k!='code_manifest'},ensure_ascii=False,indent=2))
+8
View File
@@ -0,0 +1,8 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# Сохраняет обычный TTL и восстанавливает API даже при падении проверки.
restore() { docker compose up -d --wait --force-recreate api; }
trap restore EXIT
SESSION_TTL_SECONDS=2 docker compose up -d --wait --force-recreate api
"${PYTHON:-python3}" tests/expiry.py
+46
View File
@@ -0,0 +1,46 @@
"""Исходный persistence-сценарий плюс доказательство рестарта API и всех хранилищ."""
import argparse
import json
from pathlib import Path
import subprocess
import sys
ROOT = Path(__file__).resolve().parents[1]
def states():
result = {}
for service in ('api', 'postgres', 'redis', 'minio', 'reconciler', 'lb'):
ids = subprocess.check_output(['docker', 'compose', 'ps', '-q', service], cwd=ROOT, text=True).split()
if not ids:
raise RuntimeError('Не запущен сервис ' + service)
result[service] = {}
for container in ids:
state = json.loads(subprocess.check_output(['docker', 'inspect', '--format', '{{json .State}}', container], cwd=ROOT, text=True))
result[service][container] = {'StartedAt': state['StartedAt'], 'Pid': state['Pid']}
return result
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--action-script', required=True)
parser.add_argument('--ca-file')
args = parser.parse_args()
before = states()
print('ДО: ' + json.dumps(before), flush=True)
command = [sys.executable, str(ROOT / 'tests/resilience.py'), '--mode', 'persistence',
'--action-script', args.action_script, '--base-url', args.base_url]
if args.ca_file:
command += ['--ca-file', args.ca_file]
subprocess.run(command, cwd=ROOT, check=True)
after = states()
print('ПОСЛЕ: ' + json.dumps(after), flush=True)
for service in before:
if before[service] == after[service]:
raise AssertionError('Сервис не перезапущен: ' + service)
print('PASS: StartedAt изменился у API, PostgreSQL, Redis, MinIO, reconciler и LB; прежний токен и данные сохранены')
if __name__ == '__main__':
main()
+7
View File
@@ -0,0 +1,7 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# Pass the DB address via environment, without printing it or placing it in argv.
api_container_id=$(docker compose ps -q api | head -n 1)
test_database_url=$(docker exec "$api_container_id" python -c 'import os; print(os.environ["DATABASE_URL"])')
DATABASE_URL="$test_database_url" docker compose run --rm --no-deps -T -e DATABASE_URL s3-bootstrap python < tests/reconciliation.py
+11
View File
@@ -0,0 +1,11 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
# Temporary profile: 5 admitted uploads per replica, serialized image decoding.
# Restore the ordinary project configuration even when the experiment fails.
restore_upload_profile() {
docker compose up -d --wait api >/dev/null 2>&1
}
trap restore_upload_profile EXIT INT TERM
UPLOAD_CONCURRENCY=5 docker compose up -d --wait api >/dev/null 2>&1
"${PYTHON:-python3}" tests/upload_memory.py
+51
View File
@@ -0,0 +1,51 @@
"""Sequential acceptance checks, reviewer cases, research and reproducible clean startup."""
import json
import os
import subprocess
import sys
import time
from pathlib import Path
from dotenv import dotenv_values
ROOT=Path(__file__).resolve().parents[1]
EVIDENCE=ROOT/'evidence'
EVIDENCE.mkdir(exist_ok=True)
python=sys.executable
port=os.environ.get('APP_PORT') or dotenv_values(ROOT/'.env').get('APP_PORT') or '8080'
base_url=os.environ.get('BASE_URL') or 'http://127.0.0.1:'+str(port)
api=subprocess.check_output(['docker','compose','ps','-q','api'],cwd=ROOT,text=True).split()[0]
checks=[
('course-files',[python,'scripts/check-course-files.py'],None),
('quality',[python,'-m','ruff','check','src'],None),
('format',[python,'-m','ruff','format','--check','src'],None),
('structure',[python,'scripts/check_contract.py'],None),
('openapi',[python,'scripts/validate_openapi.py'],None),
('request-deadline',['docker','exec','-i',api,'python'],'tests/request_deadline.py'),
('readiness-budget',['docker','exec','-i',api,'python'],'tests/readiness.py'),
('public-smoke',[python,'tests/smoke.py'],None),
('attachments',[python,'tests/attachments.py'],None),
('idempotency',[python,'tests/idempotency.py'],None),
('reviewer-cases',[python,'tests/lab4_review.py'],None),
('reconciliation',['scripts/test-reconciliation.sh'],None),
('upload-memory',['scripts/test-upload-memory.sh'],None),
('uploader-failover',[python,'tests/attachment_failover.py'],None),
('persistence-proof',[python,'scripts/test-persistence.py','--action-script','scripts/restart.sh','--base-url',base_url],None),
('clean-start',[python,'scripts/test-clean-lab4.py'],None),
]
summary=[]
for name,command,input_file in checks:
print('Проверка: '+name,flush=True)
started=time.perf_counter()
source=open(ROOT/input_file,'rb') if input_file else None
try:
with (EVIDENCE/('lab4-final-'+name+'.txt')).open('wb') as output:
result=subprocess.run(command,cwd=ROOT,env={**os.environ,'PYTHON':python,'BASE_URL':base_url},stdin=source,
stdout=output,stderr=subprocess.STDOUT,timeout=1000 if name=='clean-start' else 180)
summary.append({'check':name,'exit_code':result.returncode,'seconds':round(time.perf_counter()-started,3),
'evidence':'evidence/lab4-final-'+name+'.txt'})
if result.returncode:
raise RuntimeError('Не прошла проверка '+name+'; см. '+summary[-1]['evidence'])
finally:
if source:source.close()
(EVIDENCE/'lab4-final-summary.json').write_text(json.dumps({'result':'PASS','checks':summary},ensure_ascii=False,indent=2)+'\n')
print('PASS: все итоговые проверки лабораторной 4',flush=True)