Реализовать лабораторную 3 на основе lab2: балансировка и идемпотентность

This commit is contained in:
Владислав Гавриленко
2026-11-13 01:30:14 +03:00
parent 0ab3a341aa
commit 4322edcbec
21 changed files with 870 additions and 31 deletions
+27
View File
@@ -0,0 +1,27 @@
"""One transaction per connection; versioned migrations run as a separate job."""
import os
from pathlib import Path
import psycopg
from psycopg.rows import dict_row
def connect():
dsn = Path(os.environ['DATABASE_URL_FILE']).read_text().strip()
return psycopg.connect(dsn, connect_timeout=3, row_factory=dict_row,
options='-c statement_timeout=5000 -c timezone=UTC')
def migrate():
with connect() as db:
db.execute('SELECT pg_advisory_xact_lock(2303002)')
db.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY)')
for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')):
if not db.execute('SELECT 1 FROM schema_migrations WHERE version=%s', (path.name,)).fetchone():
db.execute(path.read_text())
db.execute('INSERT INTO schema_migrations VALUES (%s)', (path.name,))
if __name__ == '__main__':
migrate()
print('Migrations applied')
+265
View File
@@ -0,0 +1,265 @@
"""Shared PostgreSQL state and atomic message retries behind a load balancer."""
import base64
import binascii
import hashlib
import hmac
import json
import os
import re
import secrets
import signal
import threading
import uuid
from datetime import datetime, timezone
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from urllib.parse import parse_qs, urlsplit
import psycopg
from db import connect
INSTANCE_ID = os.environ.get('HOSTNAME') or uuid.uuid4().hex
IDEMPOTENCY_TTL = int(os.getenv('IDEMPOTENCY_TTL_SECONDS', '86400'))
if IDEMPOTENCY_TTL < 86400:
raise ValueError('IDEMPOTENCY_TTL_SECONDS must be at least 86400')
USER_FIELDS = 'id, username, display_name, created_at'
SESSION_TTL = int(os.getenv('SESSION_TTL_SECONDS', '3600'))
if SESSION_TTL <= 0:
raise ValueError('SESSION_TTL_SECONDS must be positive')
class APIError(Exception):
def __init__(self, status, code):
self.status, self.code = status, code
def require(condition, status=400, code='invalid_request'):
if not condition:
raise APIError(status, code)
def identifier(value):
try:
return str(uuid.UUID(value))
except (ValueError, TypeError, AttributeError):
raise APIError(400, 'invalid_request')
def text(value, maximum):
require(isinstance(value, str) and 0 < len(value) <= maximum and bool(value.strip()))
try:
value.encode('utf-8')
except UnicodeEncodeError:
raise APIError(400, 'invalid_request')
require('\x00' not in value)
def password_hash(password, salt):
return hashlib.scrypt(password.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=64)
def serial(value):
if isinstance(value, datetime):
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
if isinstance(value, uuid.UUID):
return str(value)
raise TypeError('Unsupported JSON value')
class Handler(BaseHTTPRequestHandler):
def setup(self):
super().setup()
self.connection.settimeout(5)
def body(self, fields, optional=()):
require(self.headers.get_content_type() == 'application/json', 415, 'unsupported_media_type')
require('Transfer-Encoding' not in self.headers)
try:
length = int(self.headers.get('Content-Length', '0'))
require(0 < length <= 65536)
value = json.loads(self.rfile.read(length).decode('utf-8'))
except (ValueError, UnicodeDecodeError):
raise APIError(400, 'invalid_request')
require(type(value) is dict and set(fields) <= set(value) <= set(fields) | set(optional))
return value
def credentials(self, scheme):
parts = self.headers.get('Authorization', '').split()
require(len(parts) == 2 and parts[0].lower() == scheme.lower(), 401, 'unauthorized')
return parts[1]
def chat(self, db, chat_id):
row = db.execute('SELECT * FROM chats WHERE id=%s', (chat_id,)).fetchone()
members = db.execute('SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (chat_id,)).fetchall()
row['member_ids'] = [x['user_id'] for x in members]
return row
def page(self, db, table, scope, query, where, params):
require(set(query) <= {'limit', 'cursor'} and all(len(v) == 1 for v in query.values()))
raw = query.get('limit', ['50'])[0]
require(re.fullmatch(r'[0-9]{1,3}', raw) is not None)
limit = int(raw)
require(1 <= limit <= 100)
if 'cursor' in query:
try:
cursor = json.loads(base64.b64decode(query['cursor'][0], altchars=b'-_', validate=True))
require(type(cursor) is list and len(cursor) == 3 and cursor[0] == scope)
stamp = datetime.fromisoformat(cursor[1].replace('Z', '+00:00'))
require(stamp.tzinfo is not None)
last_id = identifier(cursor[2])
except (ValueError, TypeError, AttributeError, UnicodeError, binascii.Error):
raise APIError(400, 'invalid_request')
require(db.execute(f'SELECT 1 FROM {table} WHERE {where} AND created_at=%s AND id=%s',
(*params, stamp, last_id)).fetchone() is not None)
where += ' AND (created_at, id) > (%s, %s)'
params = (*params, stamp, last_id)
# table and where are internal constants; every client value is a parameter.
rows = db.execute(f'SELECT * FROM {table} WHERE {where} ORDER BY created_at, id LIMIT %s',
(*params, limit + 1)).fetchall()
selected = rows[:limit]
cursor = None
if len(rows) > limit:
last = selected[-1]
cursor = base64.urlsafe_b64encode(json.dumps([scope, last['created_at'], last['id']], default=serial).encode()).decode()
if table == 'chats':
selected = [self.chat(db, row['id']) for row in selected]
return {'items': selected, 'next_cursor': cursor}
def dispatch(self, db, path, query):
if self.command == 'GET' and path == '/health/ready':
db.execute('SELECT 1')
return 200, {'status': 'ok'}
if self.command == 'POST' and path == '/api/v1/users':
body = self.body(('username', 'display_name', 'password'))
require(isinstance(body['username'], str) and re.fullmatch(r'[a-z0-9_]{3,32}', body['username']) is not None)
text(body['display_name'], 100)
text(body['password'], 128)
require(len(body['password']) >= 12)
salt = secrets.token_bytes(16)
user = db.execute(f'INSERT INTO users (id, username, display_name, password_salt, password_hash) VALUES (%s,%s,%s,%s,%s) RETURNING {USER_FIELDS}',
(uuid.uuid4(), body['username'], body['display_name'], salt, password_hash(body['password'], salt))).fetchone()
return 201, user
if self.command == 'POST' and path == '/api/v1/auth/sessions':
credential = self.credentials('Basic')
try:
username, password = base64.b64decode(credential, validate=True).decode('utf-8').split(':', 1)
require(re.fullmatch(r'[a-z0-9_]{3,32}', username) is not None and 12 <= len(password) <= 128, 401, 'unauthorized')
except (ValueError, UnicodeError, binascii.Error):
raise APIError(401, 'unauthorized')
require(self.headers.get('Content-Length', '0') == '0' and 'Transfer-Encoding' not in self.headers)
user = db.execute('SELECT * FROM users WHERE username=%s', (username,)).fetchone()
# Run the same KDF for unknown users to avoid the cheap username timing oracle.
digest = password_hash(password, user['password_salt'] if user else bytes(16))
require(user is not None and hmac.compare_digest(digest, user['password_hash']), 401, 'unauthorized')
token = secrets.token_urlsafe(32)
session = db.execute("INSERT INTO sessions VALUES (%s,%s,clock_timestamp() + %s * interval '1 second') RETURNING expires_at",
(hashlib.sha256(token.encode()).digest(), user['id'], SESSION_TTL)).fetchone()
return 201, dict(session, token=token, token_type='Bearer', user={k: user[k] for k in ('id', 'username', 'display_name', 'created_at')})
token = self.credentials('Bearer')
token_digest = hashlib.sha256(token.encode()).digest()
actor = db.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()', (token_digest,)).fetchone()
require(actor is not None, 401, 'unauthorized')
actor = actor['user_id']
if self.command == 'DELETE' and path == '/api/v1/auth/sessions/current':
db.execute('DELETE FROM sessions WHERE token_hash=%s', (token_digest,))
return 204, None
if self.command == 'GET' and path == '/api/v1/users/me':
return 200, db.execute(f'SELECT {USER_FIELDS} FROM users WHERE id=%s', (actor,)).fetchone()
if path == '/api/v1/chats':
if self.command == 'POST':
body = self.body(('title', 'member_ids'))
text(body['title'], 100)
members = body['member_ids']
require(type(members) is list and 1 <= len(members) <= 100)
members = [identifier(x) for x in members]
require(len(set(members)) == len(members) and str(actor) in members)
existing = db.execute('SELECT id FROM users WHERE id = ANY(%s::uuid[])', (members,)).fetchall()
require(len(existing) == len(members))
chat_id = uuid.uuid4()
db.execute('INSERT INTO chats (id, title) VALUES (%s,%s)', (chat_id, body['title']))
with db.cursor() as cur:
cur.executemany('INSERT INTO chat_members VALUES (%s,%s)', [(chat_id, member) for member in members])
return 201, self.chat(db, chat_id)
if self.command == 'GET':
return 200, self.page(db, 'chats', 'chats:' + str(actor), query,
'id IN (SELECT chat_id FROM chat_members WHERE user_id=%s)', (actor,))
match = re.fullmatch(r'/api/v1/chats/([^/]+)(/messages)?', path)
if match:
chat_id = identifier(match[1])
require(db.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id, actor)).fetchone() is not None, 404, 'not_found')
if not match[2] and self.command == 'GET':
return 200, self.chat(db, chat_id)
if match[2]:
if self.command == 'GET':
return 200, self.page(db, 'messages', 'messages:' + chat_id, query, 'chat_id=%s', (chat_id,))
if self.command == 'POST':
key = self.headers.get('Idempotency-Key', '')
require(re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key) is not None)
body = self.body(('text',), ('attachment_ids',))
text(body['text'], 4000)
require(type(body.get('attachment_ids', [])) is list and body.get('attachment_ids', []) == [])
# Serialize the same scope across replicas; lock, effect and key commit together.
scope = json.dumps([str(actor), chat_id, key])
db.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (scope,))
db.execute('DELETE FROM message_keys WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at <= clock_timestamp()',
(actor, chat_id, key))
previous = db.execute('SELECT m.* FROM message_keys k JOIN messages m ON m.id=k.message_id WHERE k.user_id=%s AND k.chat_id=%s AND k.key=%s',
(actor, chat_id, key)).fetchone()
if previous:
require(previous['text'] == body['text'], 409, 'conflict')
return 201, previous
message = db.execute('INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *',
(uuid.uuid4(), chat_id, actor, body['text'])).fetchone()
db.execute("INSERT INTO message_keys VALUES (%s,%s,%s,%s,clock_timestamp() + %s * interval '1 second')",
(actor, chat_id, key, message['id'], IDEMPOTENCY_TTL))
return 201, message
raise APIError(404, 'not_found')
def handle_request(self):
request_id = uuid.uuid4().hex
path = urlsplit(self.path)
try:
if self.command == 'GET' and path.path == '/health/live':
status, body = 200, {'status': 'ok'}
else:
with connect() as db:
status, body = self.dispatch(db, path.path, parse_qs(path.query, keep_blank_values=True))
except psycopg.errors.UniqueViolation:
status, body = 409, 'conflict'
except (psycopg.OperationalError, psycopg.errors.QueryCanceled):
status, body = 503, 'unavailable'
except APIError as exc:
status, body = exc.status, exc.code
except Exception:
status, body = 500, 'internal_error'
if status >= 400:
body = {'error': {'code': body, 'message': body, 'request_id': request_id}}
payload = b'' if status == 204 else json.dumps(body, ensure_ascii=False, default=serial).encode('utf-8')
self.send_response(status)
if status != 204:
self.send_header('Content-Type', 'application/json; charset=utf-8')
if status == 401:
challenge = 'Basic realm="messenger", charset="UTF-8"' if path.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"'
self.send_header('WWW-Authenticate', challenge)
self.send_header('Content-Length', str(len(payload)))
self.send_header('X-Request-Id', request_id)
self.send_header('X-Instance-Id', INSTANCE_ID)
self.end_headers()
self.wfile.write(payload)
do_GET = do_POST = do_DELETE = do_PUT = do_PATCH = handle_request
def log_message(self, format, *args):
# Paths may contain cursors; log only method and response status.
print(self.command, args[1] if len(args) > 1 else '', flush=True)
if __name__ == '__main__':
# Non-daemon request threads finish their transactions before process exit.
with ThreadingHTTPServer((os.getenv('APP_HOST', '0.0.0.0'), int(os.getenv('APP_PORT', '8080'))), Handler) as server:
server.daemon_threads = False
def stop(signum, frame):
threading.Thread(target=server.shutdown, daemon=True).start()
signal.signal(signal.SIGTERM, stop)
signal.signal(signal.SIGINT, stop)
server.serve_forever(poll_interval=0.1)