This commit is contained in:
2026-10-30 22:16:54 +03:00
commit bd342f3333
68 changed files with 6942 additions and 0 deletions
+54
View File
@@ -0,0 +1,54 @@
#!/usr/bin/env python3
"""Offline structural checks; full schema validation is a separate target."""
import ast
import json
from pathlib import Path
import re
ROOT = Path(__file__).resolve().parents[1]
def main():
lab = json.loads((ROOT/'lab.json').read_text())['number']
spec = json.loads((ROOT/'contracts/openapi.json').read_text())
assert spec['openapi']=='3.1.0'
assert spec['info']['version']=='1.%d.0'%lab
operation_ids = set()
def walk(value):
if isinstance(value,dict):
if '$ref' in value:
pointer = value['$ref']
assert pointer.startswith('#/'), 'unexpected external reference'
target = spec
for part in pointer[2:].split('/'):
target = target[part.replace('~1','/').replace('~0','~')]
for v in value.values():
walk(v)
elif isinstance(value,list):
for v in value:
walk(v)
walk(spec)
for route,methods in spec['paths'].items():
placeholders = set(re.findall(r'\{([^}]+)\}',route))
for method,op in methods.items():
assert op['operationId'] not in operation_ids, 'duplicate operationId'
operation_ids.add(op['operationId'])
params = {p['name'] for p in op.get('parameters',[]) if p['in']=='path' and p.get('required')}
assert placeholders==params, 'path parameter mismatch: '+route
for response in op['responses'].values():
assert 'X-Request-Id' in response['headers']
if lab>=3:
assert 'X-Instance-Id' in response['headers']
assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
ast.parse(file.read_text(),filename=str(file))
for file in ROOT.rglob('*.json'):
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
continue
json.loads(file.read_text())
for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
assert (ROOT/required).is_file(), 'missing '+required
print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
if __name__=='__main__':
main()
+57
View File
@@ -0,0 +1,57 @@
#!/bin/sh
# Создаёт локальные синтетические секреты для учебного стенда. Идемпотентен.
set -eu
umask 077
cd "$(dirname "$0")/.."
mkdir -p secrets
gen() {
# 32 случайных байта → base64 → только URL-safe символы.
head -c 32 /dev/urandom | base64 | tr -d '/+=\n' | cut -c1-32
}
ensure() {
f="secrets/$1"
if [ ! -s "$f" ]; then
printf '%s' "$2" > "$f"
chmod 600 "$f"
echo "created $f"
fi
}
PG_PASS=$(cat secrets/postgres_password 2>/dev/null || gen)
VALKEY_PASS=$(cat secrets/valkey_password 2>/dev/null || gen)
MINIO_ROOT_USER_VAL="minioadmin"
MINIO_ROOT_PASS=$(cat secrets/minio_root_password 2>/dev/null || gen)
S3_KEY=$(cat secrets/s3_access_key_id 2>/dev/null || "app$(gen | cut -c1-20)")
S3_SECRET=$(cat secrets/s3_secret_access_key 2>/dev/null || gen)
ensure postgres_password "$PG_PASS"
ensure valkey_password "$VALKEY_PASS"
ensure minio_root_user "$MINIO_ROOT_USER_VAL"
ensure minio_root_password "$MINIO_ROOT_PASS"
ensure s3_access_key_id "$S3_KEY"
ensure s3_secret_access_key "$S3_SECRET"
ensure database_url "postgresql://messenger:${PG_PASS}@postgres:5432/messenger"
ensure cache_url "redis://:${VALKEY_PASS}@valkey:6379/0"
if [ ! -f .env ]; then
cat > .env <<'EOF'
APP_ENV=lab
SESSION_TTL_SECONDS=3600
SESSION_STORE=postgres+cache
SESSION_CACHE_TTL_SECONDS=60
IDEMPOTENCY_TTL_SECONDS=86400
API_REPLICAS=2
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
S3_ORPHAN_GRACE_SECONDS=3600
EOF
echo "created .env"
fi
echo "secrets ready in ./secrets (не коммить)"
+115
View File
@@ -0,0 +1,115 @@
#!/usr/bin/env python3
"""Сверка S3 и метаданных PostgreSQL.
Классы проблем:
orphan_object — объект в S3 без строки attachments (или с pending > grace)
missing_object — attachments.status=ready, но объекта в S3 нет
failed_referenced — attachments.status=failed, объект всё ещё лежит в S3
Действия:
orphan_object → DELETE объекта (только старше grace)
missing_object → UPDATE status='failed', error_code='object_missing'
failed_referenced → DELETE объекта
Использование:
python scripts/reconcile_s3.py --dry-run
python scripts/reconcile_s3.py --apply
"""
import argparse
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from app import config, db, repo, s3 # noqa: E402
def _paginate(client, prefix):
"""Возвращает {key: last_modified}."""
token = None
out = {}
while True:
kwargs = {"Bucket": config.S3_BUCKET, "Prefix": prefix}
if token:
kwargs["ContinuationToken"] = token
resp = client.list_objects_v2(**kwargs)
for obj in resp.get("Contents", []):
out[obj["Key"]] = obj["LastModified"]
if not resp.get("IsTruncated"):
return out
token = resp["NextContinuationToken"]
def reconcile(dry_run: bool, grace_seconds: int):
client = s3._get_client()
now = time.time()
report = {"orphans_deleted": [], "orphans_skipped_fresh": [],
"missing_marked_failed": [], "failed_objects_deleted": []}
with db.connection() as conn:
rows = conn.execute(
"""SELECT id, object_key, status, created_at,
extract(epoch FROM created_at) AS created_epoch
FROM attachments""").fetchall()
by_key = {r["object_key"]: r for r in rows}
referenced = set(by_key)
for key, last_modified in _paginate(client, "originals/").items():
if key in referenced:
continue
age = now - last_modified.timestamp()
if age < grace_seconds:
report["orphans_skipped_fresh"].append(key)
continue
report["orphans_deleted"].append(key)
if not dry_run:
s3.delete_object(key)
for row in rows:
key = row["object_key"]
if row["status"] == "ready" and key not in _paginate(client, key):
report["missing_marked_failed"].append(key)
if not dry_run:
with db.connection() as conn:
conn.execute(
"""UPDATE attachments
SET status='failed', error_code='object_missing',
updated_at=clock_timestamp()
WHERE id=%s AND status='ready'""", (row["id"],))
elif row["status"] == "failed":
if key in _paginate(client, key):
report["failed_objects_deleted"].append(key)
if not dry_run:
s3.delete_object(key)
elif row["status"] == "pending":
age = now - float(row["created_epoch"])
if age >= grace_seconds and key in _paginate(client, key):
# pending дольше grace, а объект есть → API упал между S3 PUT и COMMIT
report["orphans_deleted"].append(key)
if not dry_run:
with db.connection() as conn:
repo.mark_attachment_failed(conn, row["id"], "orphaned_pending")
s3.delete_object(key)
import json
print(json.dumps(report, ensure_ascii=False, indent=2, default=str))
print("dry-run" if dry_run else "applied")
def main():
p = argparse.ArgumentParser()
p.add_argument("--apply", action="store_true")
p.add_argument("--dry-run", action="store_true")
p.add_argument("--grace-seconds", type=int, default=config.S3_ORPHAN_GRACE_SECONDS)
args = p.parse_args()
if args.apply and args.dry_run:
sys.exit("use --apply or --dry-run, not both")
if not (args.apply or args.dry_run):
sys.exit("choose --dry-run or --apply")
reconcile(dry_run=not args.apply, grace_seconds=args.grace_seconds)
if __name__ == "__main__":
main()
+7
View File
@@ -0,0 +1,7 @@
@echo off
REM Перезапуск стенда с сохранением volumes. Возврат 0 = ОК.
cd /d "%~dp0\.."
docker compose --env-file .env stop
timeout /t 2 /nobreak >nul
docker compose --env-file .env up -d
exit /b 0
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
set -eu
# TODO: copy to restart.sh and implement using YOUR compose/stack services.
# Restart API and ALL used state stores (DB, sessions, S3, broker as applicable).
# Preserve persistent volumes and schema/data; never down -v / flush / re-seed.
# Allow operator unseal if required in lab7; do not print credentials.
# Exit 0 only after the intended action; test waits for API readiness separately.
printf '%s
' 'TODO: implement restart.sh for your infrastructure' >&2
exit 2
+23
View File
@@ -0,0 +1,23 @@
#!/bin/sh
# Перезапуск всего стенда с сохранением volumes и данных.
# Никаких down -v / reset / seed. Останавливает по dependency-порядку,
# поднимает в обратном. Ждёт /health/ready до 90 секунд.
set -eu
cd "$(dirname "$0")/.."
# Compose сам соблюдает зависимость: stop в обратном порядке, up в прямом.
docker compose --env-file .env stop
sleep 1
docker compose --env-file .env up -d
for i in $(seq 1 90); do
if curl -fs -o /dev/null http://127.0.0.1:8080/health/ready 2>/dev/null; then
echo "stack ready after ${i}s"
exit 0
fi
sleep 1
done
echo "stack did not become ready in 90s" >&2
exit 1
+19
View File
@@ -0,0 +1,19 @@
@echo off
REM Остановить одну API-реплику по её X-Instance-Id (== Hostname контейнера).
setlocal
if "%TARGET_INSTANCE_ID%"=="" (
echo TARGET_INSTANCE_ID is not set
exit /b 1
)
for /f "tokens=*" %%c in ('docker ps -q --filter "name=messenger-lab4-api"') do (
for /f "tokens=*" %%h in ('docker inspect -f "{{.Config.Hostname}}" %%c') do (
if "%%h"=="%TARGET_INSTANCE_ID%" (
echo Stopping container %%c
docker stop %%c
exit /b 0
)
)
)
echo Container with instance ID %TARGET_INSTANCE_ID% not found
exit /b 1
+10
View File
@@ -0,0 +1,10 @@
#!/bin/sh
set -eu
# TODO: copy to stop-one.sh, map TARGET_INSTANCE_ID to ONE API container/task.
# Stop that instance and leave it stopped; no stop of DB/LB/volumes.
# Return quickly (within 10 s); the test sends requests during this script.
# Restore the replica explicitly after the test. Never print credentials.
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
printf '%s
' 'TODO: implement stop-one.sh for your infrastructure' >&2
exit 2
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
# Останавливает конкретную реплику API по её X-Instance-Id.
set -eu
if [ -z "${TARGET_INSTANCE_ID:-}" ]; then
echo "TARGET_INSTANCE_ID is not set" >&2
exit 1
fi
# INSTANCE_ID = socket.gethostname(), в Docker это короткий ID контейнера.
cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" || true)
if [ -z "$cid" ]; then
# Fallback: искать по имени проекта и сверять hostname
for c in $(docker ps -q --filter "name=messenger-lab4"); do
h=$(docker inspect -f '{{.Config.Hostname}}' "$c")
if [ "$h" = "$TARGET_INSTANCE_ID" ]; then
cid="$c"
break
fi
done
fi
if [ -n "$cid" ]; then
echo "Stopping container $cid (Instance ID: $TARGET_INSTANCE_ID)"
docker stop "$cid"
else
echo "Container with instance ID $TARGET_INSTANCE_ID not found" >&2
exit 1
fi
+23
View File
@@ -0,0 +1,23 @@
#!/usr/bin/env python3
"""Full OpenAPI and JSON Schema checks; install requirements-dev.txt first."""
import json
from pathlib import Path
from openapi_spec_validator import validate
from jsonschema import Draft202012Validator
root = Path(__file__).resolve().parents[1]
spec = json.loads((root/'contracts/openapi.json').read_text())
validate(spec)
from referencing import Registry, Resource
# The base URI resolves local #/components references inside examples.
resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource))
for name, example in json.loads((root/'contracts/examples.json').read_text()).items():
schema = {'$ref': 'urn:mtusi:openapi#/components/schemas/' + name}
Draft202012Validator(schema, registry=registry).validate(example)
for path in (root/'contracts').glob('*.schema.json'):
schema = json.loads(path.read_text())
Draft202012Validator.check_schema(schema)
for example in schema.get('examples',[]):
Draft202012Validator(schema).validate(example)
print('PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.')