Выполнена лабораторная работа 4

This commit is contained in:
2026-11-16 20:06:52 +03:00
parent ec540a1a72
commit 0cd7200b1f
51 changed files with 2133 additions and 38 deletions
+1
View File
@@ -0,0 +1 @@
+143
View File
@@ -0,0 +1,143 @@
import asyncio
from contextlib import suppress
import hashlib
import tempfile
import unicodedata
import uuid
import warnings
from urllib.parse import quote
from botocore.exceptions import ClientError
from fastapi import Request
from fastapi.responses import JSONResponse, StreamingResponse
from PIL import Image, UnidentifiedImageError
from starlette.concurrency import run_in_threadpool
from src.storage import client, BUCKET, PREFIX
def install(app, api):
def view(row):
return {key: value for key, value in row.items() if key != 'object_key'} | {'variants': []}
def authorize(request, chat_id):
with api.POOL.connection() as db:
user, _ = api.authenticate(request, db)
api.access(db, api.identifier(chat_id), user['id'])
return user['id']
def inspect_image(file, content_type):
try:
with warnings.catch_warnings():
warnings.simplefilter('error', Image.DecompressionBombWarning)
file.seek(0)
with Image.open(file) as image:
if image.width * image.height > api.MAX_PIXELS:
raise api.Problem(413, 'payload_too_large')
if {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(image.format) != content_type:
raise api.Problem(415, 'unsupported_media_type')
image.verify()
file.seek(0)
with Image.open(file) as image:
image.load()
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
raise api.Problem(413, 'payload_too_large')
except (UnidentifiedImageError, OSError, SyntaxError, ValueError):
raise api.Problem(415, 'unsupported_media_type')
finally:
file.seek(0)
@app.post('/api/v1/chats/{chat_id}/attachments')
async def upload(request: Request, chat_id: str):
owner_id = await run_in_threadpool(authorize, request, chat_id)
name = request.query_params.get('filename', '')
if set(request.query_params) != {'filename'} or not 1 <= len(name) <= 255 or not name.strip() or any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in name):
api.invalid()
mime = request.headers.get('content-type', '').split(';')[0].lower().strip()
if mime not in ('image/png', 'image/jpeg'):
raise api.Problem(415, 'unsupported_media_type')
declared = request.headers.get('content-length')
if declared is not None:
if not declared.isdigit():
api.invalid()
if int(declared) > api.MAX_BYTES:
raise api.Problem(413, 'payload_too_large')
try:
await asyncio.wait_for(api.UPLOAD_SLOTS.acquire(), timeout=2)
except TimeoutError:
raise api.Problem(503, 'unavailable')
try:
with tempfile.TemporaryFile() as file:
digest = hashlib.sha256()
size = 0
async with asyncio.timeout(30):
async for chunk in request.stream():
size += len(chunk)
if size > api.MAX_BYTES:
raise api.Problem(413, 'payload_too_large')
digest.update(chunk)
await run_in_threadpool(file.write, chunk)
await run_in_threadpool(inspect_image, file, mime)
attachment_id = uuid.uuid4()
key = PREFIX + str(attachment_id)
def save():
s3 = client()
with api.POOL.connection() as db:
db.execute('SELECT pg_advisory_xact_lock_shared(202604)')
s3.put_object(Bucket=BUCKET, Key=key, Body=file, ContentLength=size, ContentType=mime)
try:
row = db.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) VALUES(%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *', (attachment_id, api.identifier(chat_id), owner_id, name, key, mime, size, digest.hexdigest())).fetchone()
except Exception:
with suppress(Exception):
s3.delete_object(Bucket=BUCKET, Key=key)
raise
db.commit()
return row
row = await run_in_threadpool(save)
return JSONResponse(api.clean(view(row)), status_code=201)
except TimeoutError:
raise api.Problem(400, 'invalid_request', 'Upload timeout')
finally:
api.UPLOAD_SLOTS.release()
def find(request, attachment_id):
with api.POOL.connection() as db:
user, _ = api.authenticate(request, db)
row = db.execute('SELECT * FROM attachments WHERE id=%s', (api.identifier(attachment_id),)).fetchone()
if not row:
raise api.Problem(404, 'not_found')
api.access(db, row['chat_id'], user['id'])
if row['status'] != 'ready':
raise api.Problem(503, 'unavailable')
return row
@app.get('/api/v1/attachments/{attachment_id}')
async def metadata(request: Request, attachment_id: str):
return JSONResponse(api.clean(view(await run_in_threadpool(find, request, attachment_id))))
@app.get('/api/v1/attachments/{attachment_id}/content')
async def content(request: Request, attachment_id: str):
row = await run_in_threadpool(find, request, attachment_id)
if row['status'] != 'ready':
raise api.Problem(409, 'conflict')
if set(request.query_params) - {'variant'}:
api.invalid()
if request.query_params.get('variant', 'original') != 'original':
raise api.Problem(404, 'not_found')
try:
result = await run_in_threadpool(lambda: client().get_object(Bucket=BUCKET, Key=row['object_key']))
except ClientError as exc:
if exc.response['Error']['Code'] in ('NoSuchKey', '404'):
raise api.Problem(503, 'unavailable')
raise
stream = result['Body']
def chunks():
try:
while chunk := stream.read(65536):
yield chunk
finally:
stream.close()
return StreamingResponse(chunks(), media_type=row['content_type'], headers={
'Content-Length': str(row['size_bytes']),
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(row['filename'], safe=''),
'X-Content-Type-Options': 'nosniff'})
+32
View File
@@ -0,0 +1,32 @@
import time
from pathlib import Path
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from src.storage import BUCKET
def main():
s3 = boto3.client('s3', endpoint_url='http://s3:9000', region_name='us-east-1',
aws_access_key_id=Path('/run/secrets/minio_root_user').read_text().strip(),
aws_secret_access_key=Path('/run/secrets/minio_root_password').read_text().strip(),
config=Config(connect_timeout=2, read_timeout=3, retries={'total_max_attempts': 1}))
for _ in range(45):
try:
try:
s3.head_bucket(Bucket=BUCKET)
except ClientError as exc:
if exc.response['Error']['Code'] not in ('404', 'NoSuchBucket', 'NotFound'):
raise
s3.create_bucket(Bucket=BUCKET)
print('Private bucket prepared; identities loaded from S3 configuration')
return
except (BotoCoreError, ClientError):
time.sleep(1)
raise RuntimeError('S3 bootstrap timeout')
if __name__ == '__main__':
main()
+9
View File
@@ -0,0 +1,9 @@
import socket
with socket.create_connection(('127.0.0.1', 8080), timeout=1) as connection:
connection.sendall(b'GET /health/ready HTTP/1.0\r\nHost: localhost\r\nConnection: close\r\n\r\n')
with connection.makefile('rb') as stream:
status = stream.readline(128)
if status.split(b' ', 2)[1:2] != [b'200']:
raise SystemExit(1)
+367
View File
@@ -0,0 +1,367 @@
import base64
import asyncio
from contextlib import asynccontextmanager, suppress
from datetime import datetime, timezone
import hashlib
import json
import logging
import os
import re
import secrets
import uuid
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse, Response
from PIL import Image
from botocore.exceptions import BotoCoreError, ClientError
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import AsyncConnectionPool, ConnectionPool, PoolTimeout
from starlette.concurrency import run_in_threadpool
from src.migrate import database_url
from src.storage import client, BUCKET
INSTANCE = os.environ.get('HOSTNAME', str(uuid.uuid4()))
PASSWORDS = PasswordHasher()
DUMMY_HASH = PASSWORDS.hash(secrets.token_hex(32))
POOL = ConnectionPool(database_url(), min_size=1, max_size=int(os.getenv('DB_POOL_SIZE', '1')), open=False, timeout=2, kwargs={'row_factory': dict_row, 'connect_timeout': 2, 'options': '-c statement_timeout=3000 -c lock_timeout=2000'})
HEALTH_POOL = AsyncConnectionPool(database_url(), min_size=1, max_size=1, open=False, timeout=0.5, kwargs={'connect_timeout': 1, 'options': '-c statement_timeout=500 -c lock_timeout=500'})
MAX_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760)
MAX_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000)
UPLOAD_SLOTS = asyncio.Semaphore(2)
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
@asynccontextmanager
async def lifespan(app):
POOL.open()
POOL.wait(timeout=20)
await HEALTH_POOL.open()
await HEALTH_POOL.wait(timeout=20)
async def record_pool():
while True:
await asyncio.sleep(5)
print(json.dumps({'event': 'pool_metrics', 'instance': INSTANCE, 'at': datetime.now(timezone.utc).isoformat(), 'stats': POOL.get_stats()}), flush=True)
recorder = asyncio.create_task(record_pool())
yield
recorder.cancel()
with suppress(asyncio.CancelledError):
await recorder
await HEALTH_POOL.close()
POOL.close()
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
class Problem(Exception):
def __init__(self, status, code, message='Request cannot be completed'):
self.status, self.code, self.message = status, code, message
def invalid():
raise Problem(400, 'invalid_request')
def clean(value):
if isinstance(value, dict):
return {key: clean(item) for key, item in value.items()}
if isinstance(value, list):
return [clean(item) for item in value]
if isinstance(value, uuid.UUID):
return str(value)
if isinstance(value, datetime):
return value.astimezone(timezone.utc).isoformat().replace('+00:00', 'Z')
return value
@app.middleware('http')
async def headers(request, call_next):
request.state.request_id = str(uuid.uuid4())
try:
response = await call_next(request)
except Exception:
logging.exception('Unhandled request failure')
response = error(request, Problem(500, 'internal_error'))
response.headers['X-Request-Id'] = request.state.request_id
response.headers['X-Instance-Id'] = INSTANCE
return response
def error(request, exc):
headers = {}
if exc.status == 401:
headers['WWW-Authenticate'] = 'Basic realm="messenger", charset="UTF-8"' if request.url.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"'
return JSONResponse({'error': {'code': exc.code, 'message': exc.message, 'request_id': request.state.request_id}}, status_code=exc.status, headers=headers)
@app.exception_handler(Problem)
async def problem_handler(request, exc):
return error(request, exc)
@app.exception_handler(psycopg.Error)
@app.exception_handler(PoolTimeout)
@app.exception_handler(BotoCoreError)
@app.exception_handler(ClientError)
async def database_handler(request, exc):
return error(request, Problem(503, 'unavailable'))
@app.exception_handler(404)
@app.exception_handler(405)
async def missing_handler(request, exc):
return error(request, Problem(404, 'not_found'))
async def body(request, fields, empty_attachments=False):
if request.headers.get('content-type', '').split(';')[0].strip().lower() != 'application/json':
raise Problem(415, 'unsupported_media_type')
raw = bytearray()
async for chunk in request.stream():
if len(raw) + len(chunk) > 65536:
raise Problem(413, 'payload_too_large')
raw.extend(chunk)
try:
value = json.loads(raw)
except (ValueError, UnicodeError):
invalid()
if empty_attachments and isinstance(value, dict) and 'attachment_ids' in value:
fields = (*fields, 'attachment_ids')
if not isinstance(value, dict) or set(value) != set(fields):
invalid()
if empty_attachments:
value.setdefault('attachment_ids', [])
return value
def text(value, maximum, minimum=1):
if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip():
invalid()
return value
def identifier(value):
try:
return uuid.UUID(value)
except (ValueError, TypeError, AttributeError):
invalid()
USER_COLUMNS = 'id, username, display_name, created_at'
def authenticate(request, db):
authorization = request.headers.get('authorization', '')
if not authorization.startswith('Bearer ') or not authorization[7:]:
raise Problem(401, 'unauthorized')
digest = hashlib.sha256(authorization[7:].encode()).hexdigest()
user = db.execute('SELECT u.* FROM users u JOIN sessions s ON s.user_id=u.id WHERE s.token_hash=%s AND s.expires_at>clock_timestamp()', (digest,)).fetchone()
if not user:
raise Problem(401, 'unauthorized')
return user, digest
def user_view(user):
return {key: user[key] for key in ('id', 'username', 'display_name', 'created_at')}
def access(db, chat_id, user_id):
if not db.execute('SELECT 1 FROM members WHERE chat_id=%s AND user_id=%s', (chat_id, user_id)).fetchone():
raise Problem(404, 'not_found')
def chat_view(db, chat):
chat['member_ids'] = [row['user_id'] for row in db.execute('SELECT user_id FROM members WHERE chat_id=%s ORDER BY user_id', (chat['id'],))]
return chat
def pagination(request, scope):
if set(request.query_params) - {'limit', 'cursor'}:
invalid()
raw_limit = request.query_params.get('limit', '50')
if not re.fullmatch('[0-9]+', raw_limit) or not 1 <= int(raw_limit) <= 100:
invalid()
cursor = request.query_params.get('cursor')
position = None
if cursor is not None:
try:
resource, when, item = json.loads(base64.urlsafe_b64decode(cursor + '=' * (-len(cursor) % 4)))
if resource != scope:
invalid()
position = (datetime.fromisoformat(when.replace('Z', '+00:00')), identifier(item))
if position[0].tzinfo is None:
invalid()
except (ValueError, TypeError, UnicodeError, OverflowError):
invalid()
return int(raw_limit), position
def page(rows, limit, scope):
selected = rows[:limit]
cursor = None
if len(rows) > limit:
last = clean(selected[-1])
cursor = base64.urlsafe_b64encode(json.dumps([scope, last['created_at'], last['id']]).encode()).decode().rstrip('=')
return {'items': selected, 'next_cursor': cursor}
@app.get('/health/live')
def live():
return {'status': 'ok'}
@app.get('/health/ready')
async def ready():
async with HEALTH_POOL.connection() as db:
await db.execute('SELECT 1 FROM users LIMIT 1')
await run_in_threadpool(lambda: client().head_bucket(Bucket=BUCKET))
return {'status': 'ok'}
@app.post('/api/v1/users')
async def register(request: Request):
data = await body(request, ('username', 'display_name', 'password'))
if not isinstance(data['username'], str) or not re.fullmatch('[a-z0-9_]{3,32}', data['username']):
invalid()
text(data['display_name'], 100)
text(data['password'], 128, 12)
def create():
hashed = PASSWORDS.hash(data['password'])
try:
with POOL.connection() as db:
return db.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES(%s,%s,%s,%s) RETURNING ' + USER_COLUMNS, (uuid.uuid4(), data['username'], data['display_name'], hashed)).fetchone()
except psycopg.errors.UniqueViolation:
raise Problem(409, 'conflict')
return JSONResponse(clean(await run_in_threadpool(create)), status_code=201)
@app.post('/api/v1/auth/sessions')
async def login(request: Request):
authorization = request.headers.get('authorization', '')
try:
if not authorization.startswith('Basic '):
raise ValueError()
username, password = base64.b64decode(authorization[6:], validate=True).decode('utf-8').split(':', 1)
except (ValueError, UnicodeError):
raise Problem(401, 'unauthorized')
if await request.body():
invalid()
def create():
with POOL.connection() as db:
user = db.execute('SELECT * FROM users WHERE username=%s', (username,)).fetchone()
try:
PASSWORDS.verify(user['password_hash'] if user else DUMMY_HASH, password)
except VerificationError:
raise Problem(401, 'unauthorized')
if not user:
raise Problem(401, 'unauthorized')
token = secrets.token_urlsafe(32)
expiry = db.execute("INSERT INTO sessions VALUES(%s,%s,clock_timestamp()+(%s * interval '1 second')) RETURNING expires_at", (hashlib.sha256(token.encode()).hexdigest(), user['id'], int(os.getenv('SESSION_TTL_SECONDS', '3600')))).fetchone()['expires_at']
return {'token': token, 'token_type': 'Bearer', 'expires_at': expiry, 'user': user_view(user)}
return JSONResponse(clean(await run_in_threadpool(create)), status_code=201)
from src.attachments import install
import sys
install(app, sys.modules[__name__])
@app.api_route('/api/v1/{path:path}', methods=['GET', 'POST', 'DELETE'])
async def business(request: Request, path: str):
data = None
if request.method == 'POST':
def verify_identity():
with POOL.connection() as db:
authenticate(request, db)
await run_in_threadpool(verify_identity)
data = await body(request, ('title', 'member_ids') if path == 'chats' else ('text',), empty_attachments=path.endswith('/messages'))
def execute():
with POOL.connection() as db:
user, digest = authenticate(request, db)
if path == 'users/me' and request.method == 'GET':
return user_view(user)
if path == 'auth/sessions/current' and request.method == 'DELETE':
db.execute('DELETE FROM sessions WHERE token_hash=%s', (digest,))
return None
if path == 'chats' and request.method == 'POST':
text(data['title'], 100)
ids = data['member_ids']
if not isinstance(ids, list) or not 1 <= len(ids) <= 100:
invalid()
ids = [identifier(value) for value in ids]
if len(set(ids)) != len(ids) or user['id'] not in ids:
invalid()
if db.execute('SELECT count(*) AS n FROM users WHERE id=ANY(%s)', (ids,)).fetchone()['n'] != len(ids):
invalid()
chat = db.execute('INSERT INTO chats(id,title) VALUES(%s,%s) RETURNING *', (uuid.uuid4(), data['title'])).fetchone()
for member in ids:
db.execute('INSERT INTO members VALUES(%s,%s)', (chat['id'], member))
return chat_view(db, chat)
if path == 'chats' and request.method == 'GET':
scope = 'chats:' + str(user['id'])
limit, position = pagination(request, scope)
query = 'SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s'
values = [user['id']]
if position:
query += ' AND (c.created_at,c.id)>(%s,%s)'
values.extend(position)
rows = db.execute(query + ' ORDER BY c.created_at,c.id LIMIT %s', values + [limit + 1]).fetchall()
return page([chat_view(db, row) for row in rows], limit, scope)
parts = path.split('/')
if len(parts) not in (2, 3) or parts[0] != 'chats':
raise Problem(404, 'not_found')
chat_id = identifier(parts[1])
access(db, chat_id, user['id'])
if len(parts) == 2 and request.method == 'GET':
return chat_view(db, db.execute('SELECT * FROM chats WHERE id=%s', (chat_id,)).fetchone())
if len(parts) != 3 or parts[2] != 'messages':
raise Problem(404, 'not_found')
if request.method == 'GET':
scope = 'messages:' + str(chat_id)
limit, position = pagination(request, scope)
query = 'SELECT * FROM messages WHERE chat_id=%s'
values = [chat_id]
if position:
query += ' AND (created_at,id)>(%s,%s)'
values.extend(position)
rows = db.execute(query + ' ORDER BY created_at,id LIMIT %s', values + [limit + 1]).fetchall()
return page(rows, limit, scope)
if request.method != 'POST':
raise Problem(404, 'not_found')
attachments = data['attachment_ids']
if not isinstance(attachments, list) or len(attachments) > 10:
invalid()
attachments = [identifier(item) for item in attachments]
if len(set(attachments)) != len(attachments):
invalid()
if not isinstance(data['text'], str) or len(data['text']) > 4000 or (not data['text'].strip() and not attachments):
invalid()
for item in attachments:
row = db.execute('SELECT status FROM attachments WHERE id=%s AND chat_id=%s FOR SHARE', (item, chat_id)).fetchone()
if not row:
raise Problem(404, 'not_found')
if row['status'] != 'ready':
raise Problem(409, 'conflict')
canonical = json.dumps({'text': data['text'], 'attachment_ids': [str(item) for item in attachments]}, sort_keys=True)
key = request.headers.get('idempotency-key', '')
if not re.fullmatch('[A-Za-z0-9._:-]{1,128}', key):
invalid()
lock = str(user['id']) + ':' + str(chat_id) + ':' + key
db.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))', (lock,))
db.execute('DELETE FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at<=clock_timestamp()', (user['id'], chat_id, key))
previous = db.execute('SELECT * FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s', (user['id'], chat_id, key)).fetchone()
if previous:
if previous['body'] != canonical:
raise Problem(409, 'conflict')
return db.execute('SELECT * FROM messages WHERE id=%s', (previous['message_id'],)).fetchone()
message = db.execute('INSERT INTO messages(id,chat_id,sender_id,text,attachment_ids) VALUES(%s,%s,%s,%s,%s) RETURNING *', (uuid.uuid4(), chat_id, user['id'], data['text'], attachments)).fetchone()
db.execute("INSERT INTO idempotency VALUES(%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours')", (user['id'], chat_id, key, canonical, message['id']))
return message
result = await run_in_threadpool(execute)
return Response(status_code=204) if result is None else JSONResponse(clean(result), status_code=201 if request.method == 'POST' else 200)
+18
View File
@@ -0,0 +1,18 @@
from pathlib import Path
import os
import psycopg
def database_url():
return Path(os.environ.get('DATABASE_URL_FILE', '/run/secrets/database_url')).read_text().strip()
def main():
with psycopg.connect(database_url()) as connection:
connection.execute('SELECT pg_advisory_xact_lock(202603)')
for migration in sorted(Path('migrations').glob('*.sql')):
connection.execute(migration.read_text())
if __name__ == '__main__':
main()
+71
View File
@@ -0,0 +1,71 @@
import argparse
from datetime import datetime, timedelta, timezone
import json
import time
from botocore.exceptions import ClientError
import psycopg
from psycopg.rows import dict_row
from src.migrate import database_url
from src.storage import client, BUCKET, PREFIX
def run(apply=False, grace_seconds=3600):
if grace_seconds < 60:
raise ValueError('Grace period must be at least 60 seconds')
started = time.monotonic()
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
report = dict(apply=apply, grace_seconds=grace_seconds, objects=0, metadata=0,
list_requests=0, head_requests=0, orphan_candidates=0, missing_candidates=0,
deleted=0, marked_failed=0, fresh_preserved=0, referenced_preserved=0)
s3 = client()
with psycopg.connect(database_url(), row_factory=dict_row) as db:
db.execute("SET lock_timeout='5s'")
db.execute('SELECT pg_advisory_xact_lock(202604)')
for page in s3.get_paginator('list_objects_v2').paginate(Bucket=BUCKET, Prefix=PREFIX):
report['list_requests'] += 1
for obj in page.get('Contents', []):
report['objects'] += 1
if db.execute('SELECT 1 FROM attachments WHERE object_key=%s', (obj['Key'],)).fetchone():
report['referenced_preserved'] += 1
continue
if obj['LastModified'] >= cutoff:
report['fresh_preserved'] += 1
continue
report['head_requests'] += 1
current = s3.head_object(Bucket=BUCKET, Key=obj['Key'])
if current['LastModified'] >= cutoff:
report['fresh_preserved'] += 1
continue
report['orphan_candidates'] += 1
if apply:
s3.delete_object(Bucket=BUCKET, Key=obj['Key'])
report['deleted'] += 1
with db.cursor(name='attachments_scan') as cursor:
cursor.execute("SELECT id,object_key,created_at FROM attachments WHERE status='ready' ORDER BY id")
while rows := cursor.fetchmany(100):
for row in rows:
report['metadata'] += 1
if row['created_at'] >= cutoff:
continue
report['head_requests'] += 1
try:
s3.head_object(Bucket=BUCKET, Key=row['object_key'])
except ClientError as exc:
if exc.response['Error']['Code'] not in ('NoSuchKey', '404', 'NotFound'):
raise
report['missing_candidates'] += 1
if apply:
db.execute("UPDATE attachments SET status='missing',error_code='original_missing',updated_at=clock_timestamp() WHERE id=%s AND status='ready'", (row['id'],))
report['marked_failed'] += 1
report['elapsed_seconds'] = round(time.monotonic() - started, 6)
return report
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--apply', action='store_true')
parser.add_argument('--grace-seconds', type=int, default=3600)
args = parser.parse_args()
print(json.dumps(run(args.apply, args.grace_seconds)))
+21
View File
@@ -0,0 +1,21 @@
import os
from pathlib import Path
import boto3
from botocore.config import Config
BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
PREFIX = 'originals/'
def secret(name):
return Path(os.getenv(name + '_FILE', '/run/secrets/' + name.lower())).read_text().strip()
def client():
return boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
region_name='us-east-1', aws_access_key_id=secret('S3_ACCESS_KEY_ID'),
aws_secret_access_key=secret('S3_SECRET_ACCESS_KEY'),
config=Config(connect_timeout=2, read_timeout=5, retries={'total_max_attempts': 1}, request_checksum_calculation='when_required', response_checksum_validation='when_required',
s3={'addressing_style': 'path'}))