forked from mtusi2026/mtusi2026lab4
ЛР 4 — вложения S3 и reconciler
This commit is contained in:
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
if [ -e .env ]; then
|
||||
echo '.env already exists; refusing to overwrite it.' >&2
|
||||
exit 2
|
||||
fi
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import os, secrets
|
||||
source=Path('.env.example').read_text(encoding='utf-8')
|
||||
values={
|
||||
'CHANGE-ME-postgres-long-random-secret':secrets.token_urlsafe(32),
|
||||
'CHANGE-ME-random-32-byte-secret':secrets.token_urlsafe(32),
|
||||
'CHANGE-ME-rustfs-root-long-secret':secrets.token_urlsafe(32),
|
||||
'CHANGE-ME-s3-api-long-secret':secrets.token_urlsafe(32),
|
||||
'MESSENGERADMIN':'ROOT'+secrets.token_hex(10).upper(),
|
||||
'MESSENGERAPI':'API'+secrets.token_hex(10).upper(),
|
||||
}
|
||||
for old,new in values.items(): source=source.replace(old,new)
|
||||
p=Path('.env'); p.write_text(source,encoding='utf-8'); os.chmod(p,0o600)
|
||||
PY
|
||||
printf '%s\n' 'Created .env with fresh local credentials (mode 0600).'
|
||||
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Reconcile the private S3 bucket with attachment metadata.
|
||||
|
||||
Dry-run is the default. Apply mode only removes old untracked objects and old
|
||||
unreferenced attachment records after the grace period. Missing tracked objects
|
||||
are marked failed, never silently recreated. Referenced attachments are never
|
||||
removed by this tool.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
import argparse, json, os, sys, time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
import boto3
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from botocore.config import Config
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p=argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument("--apply",action="store_true",help="apply safe repairs; default only reports")
|
||||
p.add_argument("--grace-seconds",type=int,default=86400)
|
||||
args=p.parse_args()
|
||||
if args.grace_seconds<3600:
|
||||
p.error("--grace-seconds must be at least 3600 (one hour)")
|
||||
start=time.monotonic(); cutoff=datetime.now(timezone.utc)-timedelta(seconds=args.grace_seconds)
|
||||
s3=boto3.client("s3",endpoint_url=os.getenv("S3_ENDPOINT","http://localhost:9000"),region_name=os.getenv("S3_REGION","us-east-1"),aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"],aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],config=Config(connect_timeout=3,read_timeout=15,retries={"max_attempts":2},s3={"addressing_style":"path"}))
|
||||
conn=psycopg.connect(os.environ["DATABASE_URL"],row_factory=dict_row)
|
||||
report={"mode":"apply" if args.apply else "dry-run","grace_seconds":args.grace_seconds,"untracked_objects":[],"missing_objects":[],"stale_unreferenced_metadata":[],"deferred_fresh_metadata":[],"actions":[],"elapsed_seconds":None}
|
||||
try:
|
||||
attachments=conn.execute("SELECT a.id,a.object_key,a.created_at,a.status,EXISTS(SELECT 1 FROM message_attachments ma WHERE ma.attachment_id=a.id) AS referenced FROM attachments a").fetchall()
|
||||
by_key={a["object_key"]:a for a in attachments}; seen={}; token=None
|
||||
while True:
|
||||
params={"Bucket":os.getenv("S3_BUCKET","messenger-attachments"),"Prefix":"attachments/"}
|
||||
if token: params["ContinuationToken"]=token
|
||||
page=s3.list_objects_v2(**params)
|
||||
for obj in page.get("Contents",[]):
|
||||
key=obj["Key"]; seen[key]=obj["LastModified"].astimezone(timezone.utc)
|
||||
if key not in by_key:
|
||||
record={"key":key,"last_modified":obj["LastModified"].astimezone(timezone.utc).isoformat(),"size_bytes":obj["Size"]}
|
||||
report["untracked_objects"].append(record)
|
||||
if args.apply and obj["LastModified"].astimezone(timezone.utc)<cutoff:
|
||||
s3.delete_object(Bucket=params["Bucket"],Key=key); report["actions"].append({"action":"delete_untracked_object","key":key})
|
||||
if not page.get("IsTruncated"): break
|
||||
token=page["NextContinuationToken"]
|
||||
for a in attachments:
|
||||
if a["object_key"] not in seen:
|
||||
report["missing_objects"].append({"attachment_id":str(a["id"]),"key":a["object_key"],"status":a["status"]})
|
||||
if args.apply and a["status"]!="failed":
|
||||
conn.execute("UPDATE attachments SET status='failed',error_code='object_missing',updated_at=now() WHERE id=%s",(a["id"],)); report["actions"].append({"action":"mark_missing_failed","attachment_id":str(a["id"])})
|
||||
object_is_old=a["object_key"] not in seen or seen[a["object_key"]]<cutoff
|
||||
if not a["referenced"] and a["created_at"].astimezone(timezone.utc)<cutoff and not object_is_old:
|
||||
report["deferred_fresh_metadata"].append({"attachment_id":str(a["id"]),"key":a["object_key"],"reason":"object_is_within_grace_period"})
|
||||
if not a["referenced"] and a["created_at"].astimezone(timezone.utc)<cutoff and object_is_old:
|
||||
report["stale_unreferenced_metadata"].append({"attachment_id":str(a["id"]),"key":a["object_key"],"status":a["status"]})
|
||||
if args.apply:
|
||||
# Recheck reference and age inside the write transaction to avoid racing a message send.
|
||||
with conn.transaction():
|
||||
row=conn.execute("SELECT status,created_at,NOT EXISTS(SELECT 1 FROM message_attachments WHERE attachment_id=%s) AS unused FROM attachments WHERE id=%s FOR UPDATE",(a["id"],a["id"])).fetchone()
|
||||
if row and row["unused"] and row["created_at"].astimezone(timezone.utc)<cutoff:
|
||||
if a["object_key"] in seen: s3.delete_object(Bucket=os.getenv("S3_BUCKET","messenger-attachments"),Key=a["object_key"])
|
||||
conn.execute("DELETE FROM attachments WHERE id=%s",(a["id"],)); report["actions"].append({"action":"delete_stale_unreferenced","attachment_id":str(a["id"])})
|
||||
if args.apply: conn.commit()
|
||||
report["elapsed_seconds"]=round(time.monotonic()-start,4)
|
||||
print(json.dumps(report,ensure_ascii=False,sort_keys=True,indent=2,default=str))
|
||||
return 0
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
if __name__=="__main__": sys.exit(main())
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Restart every API and stateful store while preserving all named volumes.
|
||||
docker compose --env-file .env restart api haproxy postgres valkey rustfs
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
# X-Instance-Id is the container hostname (the Docker short ID). Resolve it to
|
||||
# exactly one scaled Compose container and stop only that process.
|
||||
for id in $(docker compose --env-file .env ps -q api); do
|
||||
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id")
|
||||
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
|
||||
docker stop --time 3 "$id" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
echo "No running api container matches TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
Reference in New Issue
Block a user