ЛР 2: PostgreSQL, Bearer-сессии, Valkey и исследование восстановления

This commit is contained in:
Алихан Абдурашидов
2026-10-30 14:09:36 +03:00
parent 65003b6119
commit cddd65fe4e
40 changed files with 7890 additions and 35 deletions
+104
View File
@@ -0,0 +1,104 @@
"""Lab2 entry point, retaining the HTTP transport from the Python lab1."""
import argparse
import logging
import os
import re
import signal
import threading
from urllib.parse import urlsplit
from urllib.request import urlopen
from http_transport import APIError, Handler, Server, invalid, page_query
class API(Handler):
def authorization(self):
values = self.headers.get_all('Authorization',[])
return values[0] if len(values) == 1 else None
def dispatch(self):
parsed = urlsplit(self.path)
path, method, store = parsed.path, self.command, self.server.store
if method == 'GET' and path == '/health/live':
return 200, {'status':'ok'}
if self.server.stopping.is_set():
raise APIError(503,'unavailable','Сервер завершает работу')
if method == 'GET' and path == '/health/ready':
store.ready()
return 200, {'status':'ok'}
if method == 'POST' and path == '/api/v1/users':
return 201, store.user(self.json_body())
if method == 'POST' and path == '/api/v1/auth/sessions':
if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length','0') != '0':
raise invalid('Вход не принимает тело запроса')
return 201, store.login(self.authorization())
if not path.startswith('/api/v1/'):
raise APIError(404,'not_found','Объект не найден')
actor = store.authenticate(self.authorization())
if method == 'DELETE' and path == '/api/v1/auth/sessions/current':
store.logout(self.authorization())
return 204, None
if method == 'GET' and path == '/api/v1/users/me':
return 200, store.profile(actor)
if path == '/api/v1/chats':
if method == 'POST':
return 201, store.chat(actor,self.json_body())
if method == 'GET':
return 200, store.list_chats(actor,*page_query(parsed.query))
match = re.fullmatch(r'/api/v1/chats/([^/]+)(/messages)?',path)
if match:
cid, messages = match.groups()
chat = store.accessible(actor,cid)
if method == 'GET' and not messages:
return 200, chat
if method == 'POST' and messages:
return 201, store.message(actor,cid,self.json_body())
if method == 'GET' and messages:
return 200, store.history(actor,cid,*page_query(parsed.query))
raise APIError(404,'not_found','Объект не найден')
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--healthcheck',action='store_true')
parser.add_argument('--migrate',action='store_true')
args = parser.parse_args()
host = os.environ.get('APP_HOST','0.0.0.0')
port = int(os.environ.get('APP_PORT','8080'))
if args.healthcheck:
target = '127.0.0.1' if host == '0.0.0.0' else host
with urlopen(f'http://{target}:{port}/health/ready',timeout=3) as response:
if response.status != 200:
raise SystemExit(1)
return
logging.basicConfig(level=logging.INFO,format='%(asctime)s %(levelname)s %(message)s')
from migrate import migrate
migrate()
if args.migrate:
print('PASS: migrations applied')
return
from store import Store
store = Store()
server = Server((host,port),store,API)
deadline = threading.Timer(7.5,server.close_active)
deadline.daemon = True
def stop(signum,_frame):
if not server.stopping.is_set():
server.stopping.set()
logging.info('shutdown_started signal=%s grace_seconds=8',signum)
deadline.start()
threading.Thread(target=server.shutdown,daemon=True).start()
signal.signal(signal.SIGTERM,stop)
signal.signal(signal.SIGINT,stop)
logging.info('started host=%s port=%s',host,port)
try:
server.serve_forever(poll_interval=.1)
finally:
server.server_close()
deadline.cancel()
store.close()
logging.info('shutdown_complete')
if __name__ == '__main__':
main()
+14
View File
@@ -0,0 +1,14 @@
"""Only read secret values from files; never print DSNs or credentials."""
import os
from pathlib import Path
def secret(name):
return Path(os.environ[name + '_FILE']).read_text().strip()
def integer(name, default, minimum=1, maximum=86400):
value = int(os.environ.get(name, str(default)))
if not minimum <= value <= maximum:
raise ValueError('Invalid configuration: ' + name)
return value
+198
View File
@@ -0,0 +1,198 @@
"""Reusable strict JSON/HTTP transport carried over from the Python lab1."""
from __future__ import annotations
import json
import logging
import re
import socket
import threading
import time
import uuid
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs
LOG = logging.getLogger('messenger')
MAX_BODY_BYTES = 65536
class APIError(Exception):
def __init__(self, status: int, code: str, message: str, headers=None):
self.status, self.code, self.message = status, code, message
self.headers = headers or {}
def invalid(message='Неверный запрос'):
return APIError(400, 'invalid_request', message)
def identifier(value):
if not isinstance(value, str) or not re.fullmatch(
r'[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}', value
):
raise invalid('Ожидался UUID')
return str(uuid.UUID(value))
def text(value, maximum):
if not isinstance(value, str) or not 1 <= len(value) <= maximum or not value.strip():
raise invalid('Неверная длина или тип строки')
# Reject unpaired JSON surrogate escapes; valid supplementary Unicode is preserved.
try:
value.encode('utf-8')
except UnicodeError:
raise invalid('Неверная строка Unicode') from None
return value
def fields(body, required):
if not isinstance(body, dict) or set(body) != set(required):
raise invalid('Неверный набор полей JSON')
def page_query(query):
try:
values = parse_qs(query, keep_blank_values=True, strict_parsing=True)
except ValueError:
raise invalid('Неверная строка запроса') from None
if set(values) - {'limit', 'cursor'} or any(len(v) != 1 for v in values.values()):
raise invalid('Неизвестные или повторяющиеся параметры')
raw_limit = values.get('limit', ['50'])[0]
if not re.fullmatch(r'[0-9]+', raw_limit) or len(raw_limit) > 3:
raise invalid('Limit должен быть целым от 1 до 100')
limit = int(raw_limit)
if not 1 <= limit <= 100:
raise invalid('Limit должен быть целым от 1 до 100')
cursor = values.get('cursor', [None])[0]
if cursor == '':
raise invalid('Cursor не может быть пустым')
return limit, cursor
class Server(ThreadingHTTPServer):
daemon_threads = False
block_on_close = True
request_queue_size = 128
allow_reuse_address = True
def __init__(self, address, store, handler=None, request_logging=True):
self.store = store
self.stopping = threading.Event()
self.request_logging = request_logging
self.connections = set()
self.connections_lock = threading.Lock()
super().__init__(address, handler or Handler)
def get_request(self):
conn, address = super().get_request()
conn.settimeout(3)
with self.connections_lock:
self.connections.add(conn)
return conn, address
def shutdown_request(self, request):
with self.connections_lock:
self.connections.discard(request)
super().shutdown_request(request)
def close_active(self):
"""Enforce the overall shutdown deadline even for trickling clients."""
with self.connections_lock:
for conn in list(self.connections):
try:
conn.shutdown(socket.SHUT_RDWR)
except OSError:
pass
conn.close()
class Handler(BaseHTTPRequestHandler):
# Close each connection: no unbounded idle keep-alive worker lifetime.
protocol_version = 'HTTP/1.0'
server_version = 'Messenger/1'
sys_version = ''
def log_message(self, *_args):
pass
def send_error(self, code, message=None, explain=None):
# Covers errors generated by the HTTP parser as well as business errors.
self.request_id = getattr(self, 'request_id', str(uuid.uuid4()))
mapped = {400: 'invalid_request', 404: 'not_found', 413: 'payload_too_large', 415: 'unsupported_media_type'}
self.respond(code, {'error': {'code': mapped.get(code, 'invalid_request'),
'message': 'Неверный HTTP-запрос', 'request_id': self.request_id}})
def respond(self, status, body):
payload = b'' if status == 204 else json.dumps(body, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
self.send_response(status)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.send_header('Content-Length', str(len(payload)))
self.send_header('X-Request-Id', self.request_id)
self.send_header('Connection', 'close')
for key, value in getattr(self, 'response_headers', {}).items():
self.send_header(key, value)
self.end_headers()
self.wfile.write(payload)
self.close_connection = True
def json_body(self):
media = self.headers.get_content_type()
if media != 'application/json' or self.headers.get_content_charset('utf-8').lower() != 'utf-8':
raise APIError(415, 'unsupported_media_type', 'Требуется application/json UTF-8')
lengths = self.headers.get_all('Content-Length', [])
if self.headers.get('Transfer-Encoding') or len(lengths) != 1 or not re.fullmatch(r'[0-9]+', lengths[0]):
raise invalid('Требуется один Content-Length; chunked не поддерживается')
if len(lengths[0]) > 8 or int(lengths[0]) > MAX_BODY_BYTES:
raise APIError(413, 'payload_too_large', 'JSON превышает 65536 байт')
length = int(lengths[0])
raw = self.rfile.read(length)
if len(raw) != length:
raise invalid('Неполное тело JSON')
def pairs(values):
result = {}
for key, value in values:
if key in result:
raise ValueError('duplicate key')
result[key] = value
return result
def constant(_value):
raise ValueError('nonstandard number')
try:
return json.loads(raw.decode('utf-8'), object_pairs_hook=pairs, parse_constant=constant)
except (ValueError, UnicodeError, RecursionError):
raise invalid('Неверный JSON') from None
def dispatch(self):
raise NotImplementedError
def handle_api(self):
self.request_id = str(uuid.uuid4())
self.response_headers = {}
started = time.monotonic()
status = 500
try:
status, body = self.dispatch()
except APIError as exc:
status = exc.status
self.response_headers = exc.headers
body = {'error': {'code': exc.code, 'message': exc.message, 'request_id': self.request_id}}
except (TimeoutError, ConnectionError):
status = 400
body = {'error': {'code': 'invalid_request', 'message': 'Передача тела запроса прервана', 'request_id': self.request_id}}
except Exception:
# Never log exception text/locals: they can contain the message body.
LOG.error('internal_error request_id=%s', self.request_id)
body = {'error': {'code': 'internal_error', 'message': 'Внутренняя ошибка', 'request_id': self.request_id}}
try:
self.respond(status, body)
except (BrokenPipeError, ConnectionResetError, TimeoutError):
pass
finally:
if self.server.request_logging:
LOG.info('request request_id=%s method=%s status=%s duration_ms=%.3f',
self.request_id, self.command, status, (time.monotonic() - started) * 1000)
do_GET = do_POST = do_PUT = do_DELETE = do_PATCH = do_OPTIONS = do_HEAD = handle_api
+30
View File
@@ -0,0 +1,30 @@
"""Versioned, transactional migrations; safe under concurrent API startup."""
import hashlib
from pathlib import Path
import secrets
import psycopg
from config import secret
ROOT = Path(__file__).resolve().parents[1]
def migrate():
with psycopg.connect(secret('DATABASE_URL'), connect_timeout=2) as connection:
connection.execute('SELECT pg_advisory_xact_lock(20260201)')
connection.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT clock_timestamp())')
for file in sorted((ROOT / 'migrations').glob('*.sql')):
sql = file.read_text()
digest = hashlib.sha256(sql.encode()).hexdigest()
prior = connection.execute('SELECT sha256 FROM schema_migrations WHERE version=%s', (file.name,)).fetchone()
if prior:
if prior[0] != digest:
raise RuntimeError('Applied migration checksum changed: ' + file.name)
continue
connection.execute(sql)
connection.execute('INSERT INTO schema_migrations(version,sha256) VALUES (%s,%s)', (file.name,digest))
connection.execute("INSERT INTO app_settings(name,value) VALUES ('cursor_key',%s) ON CONFLICT DO NOTHING", (secrets.token_hex(32),))
if __name__ == '__main__':
migrate()
print('PASS: migrations applied; no duplicate data')
+265
View File
@@ -0,0 +1,265 @@
"""PostgreSQL is authoritative; Valkey caches only immutable public profiles."""
from contextlib import contextmanager
import base64
import datetime as dt
import hashlib
import hmac
import json
import logging
import os
import re
import secrets
import uuid
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout, TooManyRequests
import redis
from redis.backoff import NoBackoff
from redis.retry import Retry
from config import integer, secret
from http_transport import APIError, fields, identifier, invalid, text
LOG = logging.getLogger('messenger')
USER_COLUMNS = 'id,username,display_name,created_at'
CHAT_COLUMNS = 'c.id,c.title,c.created_at,ARRAY(SELECT user_id FROM chat_members WHERE chat_id=c.id ORDER BY user_id) AS member_ids'
def public(row):
result = {}
for key, value in row.items():
if isinstance(value, uuid.UUID):
value = str(value)
elif isinstance(value, dt.datetime):
value = value.astimezone(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00','Z')
elif isinstance(value, list):
value = [str(item) for item in value]
result[key] = value
return result
def password_hash(password, salt=None):
salt = salt if salt is not None else secrets.token_bytes(16)
derived = hashlib.scrypt(password.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=32, maxmem=64*1024*1024)
return 'scrypt$16384$8$1$' + salt.hex() + '$' + derived.hex()
def password_valid(password, encoded):
parts = encoded.split('$')
return hmac.compare_digest(password_hash(password, bytes.fromhex(parts[4])), encoded)
def unauthorized(basic=False):
challenge = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"'
return APIError(401, 'unauthorized', 'Неверные или отсутствующие учётные данные', {'WWW-Authenticate': challenge})
class Store:
def __init__(self):
if os.environ.get('SESSION_STORE','postgres') != 'postgres':
raise ValueError('SESSION_STORE must be postgres')
self.ttl = integer('SESSION_TTL_SECONDS', 3600)
self.cache_ttl = integer('PROFILE_CACHE_TTL_SECONDS', 300)
self.pool = ConnectionPool(secret('DATABASE_URL'), min_size=1,
max_size=integer('DB_POOL_MAX', 8, maximum=32),
max_waiting=32, timeout=2, reconnect_timeout=3,
check=ConnectionPool.check_connection,
kwargs={'connect_timeout': 2, 'row_factory': dict_row,
'options': '-c statement_timeout=2000 -c lock_timeout=1000 -c idle_in_transaction_session_timeout=5000'},
open=True)
self.cache = redis.Redis.from_url(secret('CACHE_URL'), decode_responses=True,
socket_connect_timeout=.2, socket_timeout=.2,
max_connections=16, retry=Retry(NoBackoff(), 0))
with self.connection() as connection:
self.cursor_key = bytes.fromhex(connection.execute("SELECT value FROM app_settings WHERE name='cursor_key'").fetchone()['value'])
self.dummy_hash = password_hash('not-an-existing-password')
@contextmanager
def connection(self):
try:
with self.pool.connection() as connection:
yield connection
except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TooManyRequests,
psycopg.errors.QueryCanceled, psycopg.errors.LockNotAvailable):
raise APIError(503, 'unavailable', 'Хранилище временно недоступно') from None
def close(self):
self.pool.close()
self.cache.close()
def ready(self):
with self.connection() as connection:
connection.execute('SELECT 1')
def user(self, body):
fields(body, ('username','display_name','password'))
name = body['username']
if not isinstance(name, str) or not re.fullmatch(r'[a-z0-9_]{3,32}', name):
raise invalid('Неверный username')
display = text(body['display_name'], 100)
password = text(body['password'], 128)
if len(password) < 12:
raise invalid('Пароль должен содержать 12–128 символов')
encoded = password_hash(password)
try:
with self.connection() as connection:
user = connection.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING ' + USER_COLUMNS,
(uuid.uuid4(),name,display,encoded)).fetchone()
return public(user)
except psycopg.errors.UniqueViolation:
raise APIError(409, 'conflict', 'Username занят') from None
def profile(self, actor):
key = 'profile:v1:' + actor
try:
value = self.cache.get(key)
if value is not None:
result = json.loads(value)
if set(result) == {'id','username','display_name','created_at'} and result['id'] == actor:
return result
except (redis.RedisError, ValueError, TypeError):
LOG.info('profile_cache_fallback')
with self.connection() as connection:
result = public(connection.execute('SELECT ' + USER_COLUMNS + ' FROM users WHERE id=%s', (actor,)).fetchone())
try:
self.cache.setex(key, self.cache_ttl, json.dumps(result))
except redis.RedisError:
LOG.info('profile_cache_write_skipped')
return result
def login(self, authorization):
try:
scheme, value = authorization.split(' ', 1)
if scheme.lower() != 'basic':
raise ValueError
decoded = base64.b64decode(value, validate=True).decode('utf-8')
username, separator, password = decoded.partition(':')
if not separator or not re.fullmatch(r'[a-z0-9_]{3,32}',username) or not 12 <= len(password) <= 128:
raise ValueError
password.encode('utf-8')
except (ValueError, AttributeError, UnicodeError):
raise unauthorized(True) from None
with self.connection() as connection:
user = connection.execute('SELECT id,password_hash FROM users WHERE username=%s', (username,)).fetchone()
valid = password_valid(password, user['password_hash'] if user else self.dummy_hash)
if not valid or user is None:
raise unauthorized(True)
token = secrets.token_urlsafe(32)
with self.connection() as connection:
session = connection.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,clock_timestamp() + %s * interval '1 second') RETURNING expires_at",
(hashlib.sha256(token.encode()).digest(),user['id'],self.ttl)).fetchone()
return {'token': token, 'token_type': 'Bearer', 'expires_at': public(session)['expires_at'], 'user': self.profile(str(user['id']))}
@staticmethod
def token_hash(authorization):
try:
scheme, token = authorization.split(' ', 1)
if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
raise ValueError
return hashlib.sha256(token.encode()).digest()
except (ValueError, AttributeError):
raise unauthorized() from None
def authenticate(self, authorization):
digest = self.token_hash(authorization)
with self.connection() as connection:
session = connection.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()', (digest,)).fetchone()
if session is None:
raise unauthorized()
return str(session['user_id'])
def logout(self, authorization):
digest = self.token_hash(authorization)
with self.connection() as connection:
revoked = connection.execute('DELETE FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp() RETURNING user_id', (digest,)).fetchone()
if revoked is None:
raise unauthorized()
def chat(self, actor, body):
fields(body, ('title','member_ids'))
title = text(body['title'],100)
members = body['member_ids']
if not isinstance(members,list) or not 1 <= len(members) <= 100:
raise invalid('Ожидался список участников длиной 1–100')
members = [identifier(member) for member in members]
if actor not in members or len(set(members)) != len(members):
raise invalid('Неверный список участников')
cid = uuid.uuid4()
try:
with self.connection() as connection:
# Deliberately rely on FK inside the SAME transaction: any bad
# member rolls back the chat and all earlier memberships.
connection.execute('INSERT INTO chats(id,title) VALUES (%s,%s)', (cid,title))
with connection.cursor() as cursor:
cursor.executemany('INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)', [(cid,member) for member in members])
return public(connection.execute('SELECT ' + CHAT_COLUMNS + ' FROM chats c WHERE c.id=%s', (cid,)).fetchone())
except psycopg.errors.ForeignKeyViolation:
raise invalid('Неверный список участников') from None
def accessible(self, actor, cid, connection=None):
cid = identifier(cid)
if connection is None:
with self.connection() as connection:
return self.accessible(actor,cid,connection)
chat = connection.execute('SELECT ' + CHAT_COLUMNS + ' FROM chats c WHERE c.id=%s AND EXISTS (SELECT 1 FROM chat_members WHERE chat_id=c.id AND user_id=%s)', (cid,actor)).fetchone()
if chat is None:
raise APIError(404, 'not_found', 'Объект не найден')
return public(chat)
def message(self, actor, cid, body):
fields(body, ('text',))
value = text(body['text'],4000)
with self.connection() as connection:
self.accessible(actor,cid,connection)
timestamp = connection.execute("UPDATE chats SET last_message_at=greatest(clock_timestamp(),last_message_at + interval '1 microsecond') WHERE id=%s RETURNING last_message_at", (cid,)).fetchone()['last_message_at']
result = connection.execute('INSERT INTO messages(id,chat_id,sender_id,text,created_at) VALUES (%s,%s,%s,%s,%s) RETURNING id,chat_id,sender_id,text,created_at', (uuid.uuid4(),cid,actor,value,timestamp)).fetchone()
return public(result)
def encode_cursor(self, scope, row):
raw = json.dumps([scope,row['created_at'],row['id']], separators=(',',':')).encode()
return base64.urlsafe_b64encode(raw + hmac.digest(self.cursor_key, raw, 'sha256')).decode().rstrip('=')
def decode_cursor(self, scope, cursor):
try:
if not cursor or len(cursor) > 2048:
raise ValueError
data = base64.b64decode(cursor + '=' * (-len(cursor) % 4), altchars=b'-_',validate=True)
raw, signature = data[:-32], data[-32:]
if not hmac.compare_digest(hmac.digest(self.cursor_key,raw,'sha256'),signature):
raise ValueError
resource, timestamp, uid = json.loads(raw)
if resource != scope or not isinstance(timestamp,str):
raise ValueError
return dt.datetime.fromisoformat(timestamp.replace('Z','+00:00')), identifier(uid)
except (ValueError, TypeError, UnicodeError, APIError):
raise invalid('Неверный cursor для данного ресурса') from None
def history(self, actor, cid, limit, cursor):
scope = 'messages:' + identifier(cid)
after = self.decode_cursor(scope,cursor) if cursor is not None else None
with self.connection() as connection:
self.accessible(actor,cid,connection)
query = 'SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s'
values = [cid]
if after:
query += ' AND (created_at,id) > (%s,%s)'
values.extend(after)
query += ' ORDER BY created_at,id LIMIT %s'
rows = connection.execute(query, (*values,limit+1)).fetchall()
items = [public(row) for row in rows[:limit]]
return {'items':items, 'next_cursor':self.encode_cursor(scope,items[-1]) if len(rows)>limit else None}
def list_chats(self, actor, limit, cursor):
scope = 'chats:' + actor
after = self.decode_cursor(scope,cursor) if cursor is not None else None
query = 'SELECT ' + CHAT_COLUMNS + ' FROM chats c JOIN chat_members mine ON mine.chat_id=c.id WHERE mine.user_id=%s'
values = [actor]
if after:
query += ' AND (c.created_at,c.id) > (%s,%s)'
values.extend(after)
query += ' ORDER BY c.created_at,c.id LIMIT %s'
with self.connection() as connection:
rows = connection.execute(query, (*values,limit+1)).fetchall()
items = [public(row) for row in rows[:limit]]
return {'items':items, 'next_cursor':self.encode_cursor(scope,items[-1]) if len(rows)>limit else None}