ЛР 4 — вложения S3 и reconciler

This commit is contained in:
2026-11-15 01:40:45 +03:00
parent ec540a1a72
commit 007da711ba
30 changed files with 1250 additions and 80 deletions
+23
View File
@@ -0,0 +1,23 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
if [ -e .env ]; then
echo '.env already exists; refusing to overwrite it.' >&2
exit 2
fi
python3 - <<'PY'
from pathlib import Path
import os, secrets
source=Path('.env.example').read_text(encoding='utf-8')
values={
'CHANGE-ME-postgres-long-random-secret':secrets.token_urlsafe(32),
'CHANGE-ME-random-32-byte-secret':secrets.token_urlsafe(32),
'CHANGE-ME-rustfs-root-long-secret':secrets.token_urlsafe(32),
'CHANGE-ME-s3-api-long-secret':secrets.token_urlsafe(32),
'MESSENGERADMIN':'ROOT'+secrets.token_hex(10).upper(),
'MESSENGERAPI':'API'+secrets.token_hex(10).upper(),
}
for old,new in values.items(): source=source.replace(old,new)
p=Path('.env'); p.write_text(source,encoding='utf-8'); os.chmod(p,0o600)
PY
printf '%s\n' 'Created .env with fresh local credentials (mode 0600).'
+69
View File
@@ -0,0 +1,69 @@
#!/usr/bin/env python3
"""Reconcile the private S3 bucket with attachment metadata.
Dry-run is the default. Apply mode only removes old untracked objects and old
unreferenced attachment records after the grace period. Missing tracked objects
are marked failed, never silently recreated. Referenced attachments are never
removed by this tool.
"""
from __future__ import annotations
import argparse, json, os, sys, time
from datetime import datetime, timedelta, timezone
import boto3
import psycopg
from psycopg.rows import dict_row
from botocore.config import Config
def main() -> int:
p=argparse.ArgumentParser(description=__doc__)
p.add_argument("--apply",action="store_true",help="apply safe repairs; default only reports")
p.add_argument("--grace-seconds",type=int,default=86400)
args=p.parse_args()
if args.grace_seconds<3600:
p.error("--grace-seconds must be at least 3600 (one hour)")
start=time.monotonic(); cutoff=datetime.now(timezone.utc)-timedelta(seconds=args.grace_seconds)
s3=boto3.client("s3",endpoint_url=os.getenv("S3_ENDPOINT","http://localhost:9000"),region_name=os.getenv("S3_REGION","us-east-1"),aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"],aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"],config=Config(connect_timeout=3,read_timeout=15,retries={"max_attempts":2},s3={"addressing_style":"path"}))
conn=psycopg.connect(os.environ["DATABASE_URL"],row_factory=dict_row)
report={"mode":"apply" if args.apply else "dry-run","grace_seconds":args.grace_seconds,"untracked_objects":[],"missing_objects":[],"stale_unreferenced_metadata":[],"deferred_fresh_metadata":[],"actions":[],"elapsed_seconds":None}
try:
attachments=conn.execute("SELECT a.id,a.object_key,a.created_at,a.status,EXISTS(SELECT 1 FROM message_attachments ma WHERE ma.attachment_id=a.id) AS referenced FROM attachments a").fetchall()
by_key={a["object_key"]:a for a in attachments}; seen={}; token=None
while True:
params={"Bucket":os.getenv("S3_BUCKET","messenger-attachments"),"Prefix":"attachments/"}
if token: params["ContinuationToken"]=token
page=s3.list_objects_v2(**params)
for obj in page.get("Contents",[]):
key=obj["Key"]; seen[key]=obj["LastModified"].astimezone(timezone.utc)
if key not in by_key:
record={"key":key,"last_modified":obj["LastModified"].astimezone(timezone.utc).isoformat(),"size_bytes":obj["Size"]}
report["untracked_objects"].append(record)
if args.apply and obj["LastModified"].astimezone(timezone.utc)<cutoff:
s3.delete_object(Bucket=params["Bucket"],Key=key); report["actions"].append({"action":"delete_untracked_object","key":key})
if not page.get("IsTruncated"): break
token=page["NextContinuationToken"]
for a in attachments:
if a["object_key"] not in seen:
report["missing_objects"].append({"attachment_id":str(a["id"]),"key":a["object_key"],"status":a["status"]})
if args.apply and a["status"]!="failed":
conn.execute("UPDATE attachments SET status='failed',error_code='object_missing',updated_at=now() WHERE id=%s",(a["id"],)); report["actions"].append({"action":"mark_missing_failed","attachment_id":str(a["id"])})
object_is_old=a["object_key"] not in seen or seen[a["object_key"]]<cutoff
if not a["referenced"] and a["created_at"].astimezone(timezone.utc)<cutoff and not object_is_old:
report["deferred_fresh_metadata"].append({"attachment_id":str(a["id"]),"key":a["object_key"],"reason":"object_is_within_grace_period"})
if not a["referenced"] and a["created_at"].astimezone(timezone.utc)<cutoff and object_is_old:
report["stale_unreferenced_metadata"].append({"attachment_id":str(a["id"]),"key":a["object_key"],"status":a["status"]})
if args.apply:
# Recheck reference and age inside the write transaction to avoid racing a message send.
with conn.transaction():
row=conn.execute("SELECT status,created_at,NOT EXISTS(SELECT 1 FROM message_attachments WHERE attachment_id=%s) AS unused FROM attachments WHERE id=%s FOR UPDATE",(a["id"],a["id"])).fetchone()
if row and row["unused"] and row["created_at"].astimezone(timezone.utc)<cutoff:
if a["object_key"] in seen: s3.delete_object(Bucket=os.getenv("S3_BUCKET","messenger-attachments"),Key=a["object_key"])
conn.execute("DELETE FROM attachments WHERE id=%s",(a["id"],)); report["actions"].append({"action":"delete_stale_unreferenced","attachment_id":str(a["id"])})
if args.apply: conn.commit()
report["elapsed_seconds"]=round(time.monotonic()-start,4)
print(json.dumps(report,ensure_ascii=False,sort_keys=True,indent=2,default=str))
return 0
finally:
conn.close()
if __name__=="__main__": sys.exit(main())
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
# Restart every API and stateful store while preserving all named volumes.
docker compose --env-file .env restart api haproxy postgres valkey rustfs
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
# X-Instance-Id is the container hostname (the Docker short ID). Resolve it to
# exactly one scaled Compose container and stop only that process.
for id in $(docker compose --env-file .env ps -q api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id")
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
docker stop --time 3 "$id" >/dev/null
exit 0
fi
done
echo "No running api container matches TARGET_INSTANCE_ID" >&2
exit 1