forked from mtusi2026/mtusi2026lab4
4
This commit is contained in:
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Offline structural checks; full schema validation is a separate target."""
|
||||
import ast
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
def main():
|
||||
lab = json.loads((ROOT/'lab.json').read_text())['number']
|
||||
spec = json.loads((ROOT/'contracts/openapi.json').read_text())
|
||||
assert spec['openapi']=='3.1.0'
|
||||
assert spec['info']['version']=='1.%d.0'%lab
|
||||
operation_ids = set()
|
||||
def walk(value):
|
||||
if isinstance(value,dict):
|
||||
if '$ref' in value:
|
||||
pointer = value['$ref']
|
||||
assert pointer.startswith('#/'), 'unexpected external reference'
|
||||
target = spec
|
||||
for part in pointer[2:].split('/'):
|
||||
target = target[part.replace('~1','/').replace('~0','~')]
|
||||
for v in value.values():
|
||||
walk(v)
|
||||
elif isinstance(value,list):
|
||||
for v in value:
|
||||
walk(v)
|
||||
walk(spec)
|
||||
for route,methods in spec['paths'].items():
|
||||
placeholders = set(re.findall(r'\{([^}]+)\}',route))
|
||||
for method,op in methods.items():
|
||||
assert op['operationId'] not in operation_ids, 'duplicate operationId'
|
||||
operation_ids.add(op['operationId'])
|
||||
params = {p['name'] for p in op.get('parameters',[]) if p['in']=='path' and p.get('required')}
|
||||
assert placeholders==params, 'path parameter mismatch: '+route
|
||||
for response in op['responses'].values():
|
||||
assert 'X-Request-Id' in response['headers']
|
||||
if lab>=3:
|
||||
assert 'X-Instance-Id' in response['headers']
|
||||
assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
|
||||
assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
|
||||
for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
|
||||
ast.parse(file.read_text(),filename=str(file))
|
||||
for file in ROOT.rglob('*.json'):
|
||||
if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
|
||||
continue
|
||||
json.loads(file.read_text())
|
||||
for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
|
||||
assert (ROOT/required).is_file(), 'missing '+required
|
||||
print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/bin/sh
|
||||
# Создаёт локальные синтетические секреты для учебного стенда. Рдемпотентен.
|
||||
set -eu
|
||||
|
||||
umask 077
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p secrets
|
||||
|
||||
gen() {
|
||||
# 32 случайных байта → base64 → только URL-safe символы.
|
||||
head -c 32 /dev/urandom | base64 | tr -d '/+=\n' | cut -c1-32
|
||||
}
|
||||
|
||||
ensure() {
|
||||
f="secrets/$1"
|
||||
if [ ! -s "$f" ]; then
|
||||
printf '%s' "$2" > "$f"
|
||||
chmod 600 "$f"
|
||||
echo "created $f"
|
||||
fi
|
||||
}
|
||||
|
||||
PG_PASS=$(cat secrets/postgres_password 2>/dev/null || gen)
|
||||
VALKEY_PASS=$(cat secrets/valkey_password 2>/dev/null || gen)
|
||||
MINIO_ROOT_USER_VAL="minioadmin"
|
||||
MINIO_ROOT_PASS=$(cat secrets/minio_root_password 2>/dev/null || gen)
|
||||
S3_KEY=$(cat secrets/s3_access_key_id 2>/dev/null || "app$(gen | cut -c1-20)")
|
||||
S3_SECRET=$(cat secrets/s3_secret_access_key 2>/dev/null || gen)
|
||||
|
||||
ensure postgres_password "$PG_PASS"
|
||||
ensure valkey_password "$VALKEY_PASS"
|
||||
ensure minio_root_user "$MINIO_ROOT_USER_VAL"
|
||||
ensure minio_root_password "$MINIO_ROOT_PASS"
|
||||
ensure s3_access_key_id "$S3_KEY"
|
||||
ensure s3_secret_access_key "$S3_SECRET"
|
||||
|
||||
ensure database_url "postgresql://messenger:${PG_PASS}@postgres:5432/messenger"
|
||||
ensure cache_url "redis://:${VALKEY_PASS}@valkey:6379/0"
|
||||
|
||||
if [ ! -f .env ]; then
|
||||
cat > .env <<'EOF'
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
SESSION_STORE=postgres+cache
|
||||
SESSION_CACHE_TTL_SECONDS=60
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
API_REPLICAS=2
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
S3_ORPHAN_GRACE_SECONDS=3600
|
||||
EOF
|
||||
echo "created .env"
|
||||
fi
|
||||
|
||||
echo "secrets ready in ./secrets (не коммить)"
|
||||
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Сверка S3 и метаданных PostgreSQL.
|
||||
|
||||
Классы проблем:
|
||||
orphan_object — объект в S3 без строки attachments (или с pending > grace)
|
||||
missing_object — attachments.status=ready, но объекта в S3 нет
|
||||
failed_referenced — attachments.status=failed, объект всё ещё лежит в S3
|
||||
|
||||
Действия:
|
||||
orphan_object → DELETE объекта (только старше grace)
|
||||
missing_object → UPDATE status='failed', error_code='object_missing'
|
||||
failed_referenced → DELETE объекта
|
||||
|
||||
Использование:
|
||||
python scripts/reconcile_s3.py --dry-run
|
||||
python scripts/reconcile_s3.py --apply
|
||||
"""
|
||||
import argparse
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app import config, db, repo, s3 # noqa: E402
|
||||
|
||||
|
||||
def _paginate(client, prefix):
|
||||
"""Возвращает {key: last_modified}."""
|
||||
token = None
|
||||
out = {}
|
||||
while True:
|
||||
kwargs = {"Bucket": config.S3_BUCKET, "Prefix": prefix}
|
||||
if token:
|
||||
kwargs["ContinuationToken"] = token
|
||||
resp = client.list_objects_v2(**kwargs)
|
||||
for obj in resp.get("Contents", []):
|
||||
out[obj["Key"]] = obj["LastModified"]
|
||||
if not resp.get("IsTruncated"):
|
||||
return out
|
||||
token = resp["NextContinuationToken"]
|
||||
|
||||
|
||||
def reconcile(dry_run: bool, grace_seconds: int):
|
||||
client = s3._get_client()
|
||||
now = time.time()
|
||||
report = {"orphans_deleted": [], "orphans_skipped_fresh": [],
|
||||
"missing_marked_failed": [], "failed_objects_deleted": []}
|
||||
|
||||
with db.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"""SELECT id, object_key, status, created_at,
|
||||
extract(epoch FROM created_at) AS created_epoch
|
||||
FROM attachments""").fetchall()
|
||||
by_key = {r["object_key"]: r for r in rows}
|
||||
referenced = set(by_key)
|
||||
|
||||
for key, last_modified in _paginate(client, "originals/").items():
|
||||
if key in referenced:
|
||||
continue
|
||||
age = now - last_modified.timestamp()
|
||||
if age < grace_seconds:
|
||||
report["orphans_skipped_fresh"].append(key)
|
||||
continue
|
||||
report["orphans_deleted"].append(key)
|
||||
if not dry_run:
|
||||
s3.delete_object(key)
|
||||
|
||||
for row in rows:
|
||||
key = row["object_key"]
|
||||
if row["status"] == "ready" and key not in _paginate(client, key):
|
||||
report["missing_marked_failed"].append(key)
|
||||
if not dry_run:
|
||||
with db.connection() as conn:
|
||||
conn.execute(
|
||||
"""UPDATE attachments
|
||||
SET status='failed', error_code='object_missing',
|
||||
updated_at=clock_timestamp()
|
||||
WHERE id=%s AND status='ready'""", (row["id"],))
|
||||
elif row["status"] == "failed":
|
||||
if key in _paginate(client, key):
|
||||
report["failed_objects_deleted"].append(key)
|
||||
if not dry_run:
|
||||
s3.delete_object(key)
|
||||
elif row["status"] == "pending":
|
||||
age = now - float(row["created_epoch"])
|
||||
if age >= grace_seconds and key in _paginate(client, key):
|
||||
# pending дольше grace, а объект есть → API упал между S3 PUT и COMMIT
|
||||
report["orphans_deleted"].append(key)
|
||||
if not dry_run:
|
||||
with db.connection() as conn:
|
||||
repo.mark_attachment_failed(conn, row["id"], "orphaned_pending")
|
||||
s3.delete_object(key)
|
||||
|
||||
import json
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2, default=str))
|
||||
print("dry-run" if dry_run else "applied")
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--apply", action="store_true")
|
||||
p.add_argument("--dry-run", action="store_true")
|
||||
p.add_argument("--grace-seconds", type=int, default=config.S3_ORPHAN_GRACE_SECONDS)
|
||||
args = p.parse_args()
|
||||
if args.apply and args.dry_run:
|
||||
sys.exit("use --apply or --dry-run, not both")
|
||||
if not (args.apply or args.dry_run):
|
||||
sys.exit("choose --dry-run or --apply")
|
||||
reconcile(dry_run=not args.apply, grace_seconds=args.grace_seconds)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,7 @@
|
||||
@echo off
|
||||
REM Перезапуск стенда с сохранением volumes. Возврат 0 = ОК.
|
||||
cd /d "%~dp0\.."
|
||||
docker compose --env-file .env stop
|
||||
timeout /t 2 /nobreak >nul
|
||||
docker compose --env-file .env up -d
|
||||
exit /b 0
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# TODO: copy to restart.sh and implement using YOUR compose/stack services.
|
||||
# Restart API and ALL used state stores (DB, sessions, S3, broker as applicable).
|
||||
# Preserve persistent volumes and schema/data; never down -v / flush / re-seed.
|
||||
# Allow operator unseal if required in lab7; do not print credentials.
|
||||
# Exit 0 only after the intended action; test waits for API readiness separately.
|
||||
printf '%s
|
||||
' 'TODO: implement restart.sh for your infrastructure' >&2
|
||||
exit 2
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/bin/sh
|
||||
# Перезапуск всего стенда с сохранением volumes и данных.
|
||||
# Никаких down -v / reset / seed. Останавливает по dependency-порядку,
|
||||
# поднимает в обратном. Ждёт /health/ready до 90 секунд.
|
||||
set -eu
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# Compose сам соблюдает зависимость: stop в обратном порядке, up в прямом.
|
||||
docker compose --env-file .env stop
|
||||
sleep 1
|
||||
docker compose --env-file .env up -d
|
||||
|
||||
for i in $(seq 1 90); do
|
||||
if curl -fs -o /dev/null http://127.0.0.1:8080/health/ready 2>/dev/null; then
|
||||
echo "stack ready after ${i}s"
|
||||
exit 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "stack did not become ready in 90s" >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,19 @@
|
||||
@echo off
|
||||
REM Остановить одну API-реплику по её X-Instance-Id (== Hostname контейнера).
|
||||
setlocal
|
||||
if "%TARGET_INSTANCE_ID%"=="" (
|
||||
echo TARGET_INSTANCE_ID is not set
|
||||
exit /b 1
|
||||
)
|
||||
|
||||
for /f "tokens=*" %%c in ('docker ps -q --filter "name=messenger-lab4-api"') do (
|
||||
for /f "tokens=*" %%h in ('docker inspect -f "{{.Config.Hostname}}" %%c') do (
|
||||
if "%%h"=="%TARGET_INSTANCE_ID%" (
|
||||
echo Stopping container %%c
|
||||
docker stop %%c
|
||||
exit /b 0
|
||||
)
|
||||
)
|
||||
)
|
||||
echo Container with instance ID %TARGET_INSTANCE_ID% not found
|
||||
exit /b 1
|
||||
@@ -0,0 +1,10 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# TODO: copy to stop-one.sh, map TARGET_INSTANCE_ID to ONE API container/task.
|
||||
# Stop that instance and leave it stopped; no stop of DB/LB/volumes.
|
||||
# Return quickly (within 10 s); the test sends requests during this script.
|
||||
# Restore the replica explicitly after the test. Never print credentials.
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
printf '%s
|
||||
' 'TODO: implement stop-one.sh for your infrastructure' >&2
|
||||
exit 2
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/bin/sh
|
||||
# Останавливает конкретную реплику API по ецX-Instance-Id.
|
||||
set -eu
|
||||
|
||||
if [ -z "${TARGET_INSTANCE_ID:-}" ]; then
|
||||
echo "TARGET_INSTANCE_ID is not set" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# INSTANCE_ID = socket.gethostname(), в Docker это короткий ID контейнера.
|
||||
cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" || true)
|
||||
|
||||
if [ -z "$cid" ]; then
|
||||
# Fallback: искать Р С—Р С• РёРСени проекта Р С‘ сверять hostname
|
||||
for c in $(docker ps -q --filter "name=messenger-lab4"); do
|
||||
h=$(docker inspect -f '{{.Config.Hostname}}' "$c")
|
||||
if [ "$h" = "$TARGET_INSTANCE_ID" ]; then
|
||||
cid="$c"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -n "$cid" ]; then
|
||||
echo "Stopping container $cid (Instance ID: $TARGET_INSTANCE_ID)"
|
||||
docker stop "$cid"
|
||||
else
|
||||
echo "Container with instance ID $TARGET_INSTANCE_ID not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Full OpenAPI and JSON Schema checks; install requirements-dev.txt first."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
from openapi_spec_validator import validate
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
spec = json.loads((root/'contracts/openapi.json').read_text())
|
||||
validate(spec)
|
||||
from referencing import Registry, Resource
|
||||
# The base URI resolves local #/components references inside examples.
|
||||
resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
|
||||
registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource))
|
||||
for name, example in json.loads((root/'contracts/examples.json').read_text()).items():
|
||||
schema = {'$ref': 'urn:mtusi:openapi#/components/schemas/' + name}
|
||||
Draft202012Validator(schema, registry=registry).validate(example)
|
||||
for path in (root/'contracts').glob('*.schema.json'):
|
||||
schema = json.loads(path.read_text())
|
||||
Draft202012Validator.check_schema(schema)
|
||||
for example in schema.get('examples',[]):
|
||||
Draft202012Validator(schema).validate(example)
|
||||
print('PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.')
|
||||
Reference in New Issue
Block a user