This commit is contained in:
Getlles
2026-11-09 23:42:13 +03:00
parent 9a09b38387
commit e1d42e069f
16 changed files with 467 additions and 13 deletions
+29
View File
@@ -0,0 +1,29 @@
import os
import boto3
from botocore.client import Config
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
S3_ROOT_ACCESS_KEY = os.getenv("S3_ROOT_ACCESS_KEY", "minioadmin")
S3_ROOT_SECRET_KEY = os.getenv("S3_ROOT_SECRET_KEY", "minioadminpassword")
s3_client = boto3.client(
"s3",
endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ROOT_ACCESS_KEY,
aws_secret_access_key=S3_ROOT_SECRET_KEY,
config=Config(signature_version="s3v4"),
region_name="us-east-1"
)
def init_bucket():
print(f"Checking/Creating bucket: {S3_BUCKET}")
try:
s3_client.head_bucket(Bucket=S3_BUCKET)
print("Bucket already exists.")
except Exception:
s3_client.create_bucket(Bucket=S3_BUCKET)
print("Bucket successfully created.")
if __name__ == "__main__":
init_bucket()
+11
View File
@@ -0,0 +1,11 @@
import os, sys, urllib.request, urllib.error
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
KEY = os.getenv("S3_KEY")
URL = f"http://s3:8333/{BUCKET}/{KEY}"
try:
urllib.request.urlopen(URL)
print("FAIL: object accessible without auth"); sys.exit(1)
except urllib.error.HTTPError as e:
if e.code in (401, 403):
print(f"PASS: anonymous GET -> {e.code}"); sys.exit(0)
print(f"FAIL: unexpected {e.code}"); sys.exit(1)
+81
View File
@@ -0,0 +1,81 @@
import os
import sys
import boto3
from botocore.client import Config
import psycopg2
import psycopg2.extras
from datetime import datetime, timezone, timedelta
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
s3_client = boto3.client(
"s3",
endpoint_url=S3_ENDPOINT,
aws_access_key_id=S3_ACCESS_KEY,
aws_secret_access_key=S3_SECRET_KEY,
config=Config(signature_version="s3v4"),
region_name="us-east-1"
)
def run_reconciliation(dry_run: bool = False):
print(f"Starting reconciliation (dry_run={dry_run})...")
conn = psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
threshold = datetime.now(timezone.utc) - timedelta(hours=1)
with conn.cursor() as cur:
cur.execute("""
SELECT id, chat_id, filename, created_at FROM attachments
WHERE status = 'pending' AND created_at < %s
""", (threshold,))
stale_pending = cur.fetchall()
for row in stale_pending:
att_id = str(row['id'])
chat_id = str(row['chat_id'])
filename = row['filename']
s3_key = f"attachments/{chat_id}/{att_id}/{filename}"
print(f"[DB Stale Pending] Found stale attachment {att_id}, marking as failed and cleaning S3.")
if not dry_run:
try:
s3_client.delete_object(Bucket=S3_BUCKET, Key=s3_key)
except Exception as e:
print(f"Failed to delete S3 key {s3_key}: {e}")
with conn.cursor() as cur:
cur.execute("UPDATE attachments SET status = 'failed', error_code = 'timeout_stale' WHERE id = %s", (att_id,))
conn.commit()
with conn.cursor() as cur:
cur.execute("SELECT id, chat_id, filename FROM attachments")
all_db_attachments = cur.fetchall()
valid_s3_keys = set()
for row in all_db_attachments:
valid_s3_keys.add(f"attachments/{str(row['chat_id'])}/{str(row['id'])}/{row['filename']}")
paginator = s3_client.get_paginator('list_objects_v2')
for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="attachments/"):
for obj in page.get('Contents', []):
key = obj['Key']
last_modified = obj['LastModified']
if key not in valid_s3_keys and last_modified < threshold:
print(f"[S3 Orphan] Found orphan S3 object {key} (Modified: {last_modified})")
if not dry_run:
try:
s3_client.delete_object(Bucket=S3_BUCKET, Key=key)
print(f"Deleted orphan object {key} from S3.")
except Exception as e:
print(f"Failed to delete orphan {key}: {e}")
conn.close()
print("Reconciliation completed successfully.")
if __name__ == "__main__":
dry_run_mode = "--dry-run" in sys.argv
run_reconciliation(dry_run=dry_run_mode)
+54
View File
@@ -0,0 +1,54 @@
import base64, concurrent.futures, json, pathlib, time, urllib.request, uuid
BASE = "http://localhost:8080"
def _req(method, path, body=None, headers=None, token=None):
h = {"Accept": "application/json"}
if token: h["Authorization"] = f"Bearer {token}"
h.update(headers or {})
req = urllib.request.Request(BASE + path, data=body, headers=h, method=method)
with urllib.request.urlopen(req) as r:
return r.status, r.read()
def register_and_login():
u = f"u_{uuid.uuid4().hex[:10]}"
p = "T3st-" + uuid.uuid4().hex
_req("POST", "/api/v1/users",
json.dumps({"username": u, "display_name": "Load", "password": p}).encode(),
{"Content-Type": "application/json"})
basic = base64.b64encode(f"{u}:{p}".encode()).decode()
_, body = _req("POST", "/api/v1/auth/sessions",
headers={"Authorization": f"Basic {basic}"})
return json.loads(body)["token"]
def make_chat(token):
_, me = _req("GET", "/api/v1/users/me", token=token)
uid = json.loads(me)["id"]
_, chat = _req("POST", "/api/v1/chats",
json.dumps({"title": "load", "member_ids": [uid]}).encode(),
{"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex},
token=token)
return json.loads(chat)["id"]
def main():
token = register_and_login()
chat = make_chat(token)
png = pathlib.Path("tests/fixtures/sample.png").read_bytes()
def one(_):
try:
status, _ = _req("POST",
f"/api/v1/chats/{chat}/attachments?filename=f.png",
png, {"Content-Type": "image/png"}, token)
return status
except Exception as e:
return repr(e)
t0 = time.monotonic()
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as ex:
results = list(ex.map(one, range(10)))
print(json.dumps({"results": results,
"elapsed_s": round(time.monotonic() - t0, 3)}))
if __name__ == "__main__":
main()