forked from mtusi2026/mtusi2026lab4
grade4
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
import os
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
|
||||
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
S3_ROOT_ACCESS_KEY = os.getenv("S3_ROOT_ACCESS_KEY", "minioadmin")
|
||||
S3_ROOT_SECRET_KEY = os.getenv("S3_ROOT_SECRET_KEY", "minioadminpassword")
|
||||
|
||||
s3_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=S3_ENDPOINT,
|
||||
aws_access_key_id=S3_ROOT_ACCESS_KEY,
|
||||
aws_secret_access_key=S3_ROOT_SECRET_KEY,
|
||||
config=Config(signature_version="s3v4"),
|
||||
region_name="us-east-1"
|
||||
)
|
||||
|
||||
def init_bucket():
|
||||
print(f"Checking/Creating bucket: {S3_BUCKET}")
|
||||
try:
|
||||
s3_client.head_bucket(Bucket=S3_BUCKET)
|
||||
print("Bucket already exists.")
|
||||
except Exception:
|
||||
s3_client.create_bucket(Bucket=S3_BUCKET)
|
||||
print("Bucket successfully created.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
init_bucket()
|
||||
@@ -0,0 +1,11 @@
|
||||
import os, sys, urllib.request, urllib.error
|
||||
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
KEY = os.getenv("S3_KEY")
|
||||
URL = f"http://s3:8333/{BUCKET}/{KEY}"
|
||||
try:
|
||||
urllib.request.urlopen(URL)
|
||||
print("FAIL: object accessible without auth"); sys.exit(1)
|
||||
except urllib.error.HTTPError as e:
|
||||
if e.code in (401, 403):
|
||||
print(f"PASS: anonymous GET -> {e.code}"); sys.exit(0)
|
||||
print(f"FAIL: unexpected {e.code}"); sys.exit(1)
|
||||
@@ -0,0 +1,81 @@
|
||||
import os
|
||||
import sys
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:postgres@postgres:5432/messenger")
|
||||
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:8333")
|
||||
S3_BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY_ID", "appuser")
|
||||
S3_SECRET_KEY = os.getenv("S3_SECRET_ACCESS_KEY", "appsecretpassword")
|
||||
|
||||
s3_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=S3_ENDPOINT,
|
||||
aws_access_key_id=S3_ACCESS_KEY,
|
||||
aws_secret_access_key=S3_SECRET_KEY,
|
||||
config=Config(signature_version="s3v4"),
|
||||
region_name="us-east-1"
|
||||
)
|
||||
|
||||
def run_reconciliation(dry_run: bool = False):
|
||||
print(f"Starting reconciliation (dry_run={dry_run})...")
|
||||
conn = psycopg2.connect(DATABASE_URL, cursor_factory=psycopg2.extras.RealDictCursor)
|
||||
|
||||
threshold = datetime.now(timezone.utc) - timedelta(hours=1)
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT id, chat_id, filename, created_at FROM attachments
|
||||
WHERE status = 'pending' AND created_at < %s
|
||||
""", (threshold,))
|
||||
stale_pending = cur.fetchall()
|
||||
|
||||
for row in stale_pending:
|
||||
att_id = str(row['id'])
|
||||
chat_id = str(row['chat_id'])
|
||||
filename = row['filename']
|
||||
s3_key = f"attachments/{chat_id}/{att_id}/{filename}"
|
||||
print(f"[DB Stale Pending] Found stale attachment {att_id}, marking as failed and cleaning S3.")
|
||||
|
||||
if not dry_run:
|
||||
try:
|
||||
s3_client.delete_object(Bucket=S3_BUCKET, Key=s3_key)
|
||||
except Exception as e:
|
||||
print(f"Failed to delete S3 key {s3_key}: {e}")
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("UPDATE attachments SET status = 'failed', error_code = 'timeout_stale' WHERE id = %s", (att_id,))
|
||||
conn.commit()
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id, chat_id, filename FROM attachments")
|
||||
all_db_attachments = cur.fetchall()
|
||||
|
||||
valid_s3_keys = set()
|
||||
for row in all_db_attachments:
|
||||
valid_s3_keys.add(f"attachments/{str(row['chat_id'])}/{str(row['id'])}/{row['filename']}")
|
||||
|
||||
paginator = s3_client.get_paginator('list_objects_v2')
|
||||
for page in paginator.paginate(Bucket=S3_BUCKET, Prefix="attachments/"):
|
||||
for obj in page.get('Contents', []):
|
||||
key = obj['Key']
|
||||
last_modified = obj['LastModified']
|
||||
|
||||
if key not in valid_s3_keys and last_modified < threshold:
|
||||
print(f"[S3 Orphan] Found orphan S3 object {key} (Modified: {last_modified})")
|
||||
if not dry_run:
|
||||
try:
|
||||
s3_client.delete_object(Bucket=S3_BUCKET, Key=key)
|
||||
print(f"Deleted orphan object {key} from S3.")
|
||||
except Exception as e:
|
||||
print(f"Failed to delete orphan {key}: {e}")
|
||||
|
||||
conn.close()
|
||||
print("Reconciliation completed successfully.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
dry_run_mode = "--dry-run" in sys.argv
|
||||
run_reconciliation(dry_run=dry_run_mode)
|
||||
@@ -0,0 +1,54 @@
|
||||
import base64, concurrent.futures, json, pathlib, time, urllib.request, uuid
|
||||
|
||||
BASE = "http://localhost:8080"
|
||||
|
||||
def _req(method, path, body=None, headers=None, token=None):
|
||||
h = {"Accept": "application/json"}
|
||||
if token: h["Authorization"] = f"Bearer {token}"
|
||||
h.update(headers or {})
|
||||
req = urllib.request.Request(BASE + path, data=body, headers=h, method=method)
|
||||
with urllib.request.urlopen(req) as r:
|
||||
return r.status, r.read()
|
||||
|
||||
def register_and_login():
|
||||
u = f"u_{uuid.uuid4().hex[:10]}"
|
||||
p = "T3st-" + uuid.uuid4().hex
|
||||
_req("POST", "/api/v1/users",
|
||||
json.dumps({"username": u, "display_name": "Load", "password": p}).encode(),
|
||||
{"Content-Type": "application/json"})
|
||||
basic = base64.b64encode(f"{u}:{p}".encode()).decode()
|
||||
_, body = _req("POST", "/api/v1/auth/sessions",
|
||||
headers={"Authorization": f"Basic {basic}"})
|
||||
return json.loads(body)["token"]
|
||||
|
||||
def make_chat(token):
|
||||
_, me = _req("GET", "/api/v1/users/me", token=token)
|
||||
uid = json.loads(me)["id"]
|
||||
_, chat = _req("POST", "/api/v1/chats",
|
||||
json.dumps({"title": "load", "member_ids": [uid]}).encode(),
|
||||
{"Content-Type": "application/json", "Idempotency-Key": uuid.uuid4().hex},
|
||||
token=token)
|
||||
return json.loads(chat)["id"]
|
||||
|
||||
def main():
|
||||
token = register_and_login()
|
||||
chat = make_chat(token)
|
||||
png = pathlib.Path("tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
def one(_):
|
||||
try:
|
||||
status, _ = _req("POST",
|
||||
f"/api/v1/chats/{chat}/attachments?filename=f.png",
|
||||
png, {"Content-Type": "image/png"}, token)
|
||||
return status
|
||||
except Exception as e:
|
||||
return repr(e)
|
||||
|
||||
t0 = time.monotonic()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=10) as ex:
|
||||
results = list(ex.map(one, range(10)))
|
||||
print(json.dumps({"results": results,
|
||||
"elapsed_s": round(time.monotonic() - t0, 3)}))
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user