110 lines
3.5 KiB
Python
110 lines
3.5 KiB
Python
"""Small bounded S3 client. API credentials never contain root/admin permissions."""
|
|
from contextlib import contextmanager
|
|
import threading
|
|
|
|
import boto3
|
|
from botocore.config import Config
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
from .db import setting
|
|
from .errors import ApiError
|
|
|
|
PREFIX = 'originals/'
|
|
|
|
|
|
def unavailable():
|
|
return ApiError(503,'unavailable','Хранилище файлов временно недоступно')
|
|
|
|
|
|
class OpenObject:
|
|
def __init__(self,body,release,length):
|
|
self.body=body
|
|
self.release=release
|
|
self.length=length
|
|
self.closed=False
|
|
self.close_lock=threading.Lock()
|
|
|
|
def chunks(self):
|
|
try:
|
|
while True:
|
|
data=self.body.read(65536)
|
|
if not data:
|
|
break
|
|
yield data
|
|
finally:
|
|
self.close()
|
|
|
|
def close(self):
|
|
with self.close_lock:
|
|
if self.closed:
|
|
return
|
|
self.closed=True
|
|
try:
|
|
self.body.close()
|
|
finally:
|
|
self.release()
|
|
|
|
|
|
class ObjectStore:
|
|
def __init__(self):
|
|
self.bucket=setting('S3_BUCKET','messenger-attachments')
|
|
self.gate=threading.BoundedSemaphore(8)
|
|
self.client=boto3.client('s3',endpoint_url=setting('S3_ENDPOINT','http://minio:9000'),
|
|
region_name=setting('S3_REGION','us-east-1'),
|
|
aws_access_key_id=setting('S3_ACCESS_KEY_ID'),aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'),
|
|
config=Config(signature_version='s3v4',connect_timeout=1,read_timeout=3,max_pool_connections=8,
|
|
retries={'total_max_attempts':1,'mode':'standard'},s3={'addressing_style':'path'},
|
|
request_checksum_calculation='when_required',response_checksum_validation='when_required'))
|
|
|
|
@contextmanager
|
|
def operation(self):
|
|
if not self.gate.acquire(timeout=1):
|
|
raise unavailable()
|
|
try:
|
|
yield
|
|
except (BotoCoreError,ClientError):
|
|
raise unavailable() from None
|
|
finally:
|
|
self.gate.release()
|
|
|
|
def check(self):
|
|
with self.operation():
|
|
self.client.list_objects_v2(Bucket=self.bucket,Prefix=PREFIX,MaxKeys=1)
|
|
|
|
def put(self,key,body,length,mime):
|
|
with self.operation():
|
|
body.seek(0)
|
|
self.client.put_object(Bucket=self.bucket,Key=key,Body=body,ContentLength=length,ContentType=mime)
|
|
|
|
def delete(self,key):
|
|
with self.operation():
|
|
self.client.delete_object(Bucket=self.bucket,Key=key)
|
|
|
|
def get(self,key):
|
|
if not self.gate.acquire(timeout=1):
|
|
raise unavailable()
|
|
try:
|
|
result=self.client.get_object(Bucket=self.bucket,Key=key)
|
|
return OpenObject(result['Body'],self.gate.release,result['ContentLength'])
|
|
except (BotoCoreError,ClientError):
|
|
self.gate.release()
|
|
raise unavailable() from None
|
|
except BaseException:
|
|
self.gate.release()
|
|
raise
|
|
|
|
def pages(self):
|
|
token=None
|
|
while True:
|
|
params={'Bucket':self.bucket,'Prefix':PREFIX,'MaxKeys':1000}
|
|
if token:
|
|
params['ContinuationToken']=token
|
|
with self.operation():
|
|
result=self.client.list_objects_v2(**params)
|
|
yield result
|
|
if not result.get('IsTruncated'):
|
|
break
|
|
token=result['NextContinuationToken']
|
|
|
|
def close(self):
|
|
self.client.close()
|