forked from mtusi2026/mtusi2026lab4
56 lines
2.5 KiB
Python
56 lines
2.5 KiB
Python
"""Сборщик неучтённых объектов; default dry-run, grace >= 1 час, повторный запуск безопасен.
|
|
|
|
Оператор: docker compose --profile tools run --rm cleanup [--apply]
|
|
Только originals/. Свежий объект и любая запись БД защищены. Проверка БД и удаление
|
|
делаются под той же advisory lock, которую upload держит от PUT до COMMIT.
|
|
При недоступности БД/S3 обход прерывается, никаких delete по неполному снимку.
|
|
"""
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import time
|
|
|
|
from .config import read_secret
|
|
from .db import create_pool
|
|
from .objects import ObjectStore
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--apply', action='store_true')
|
|
parser.add_argument('--grace-seconds', type=int, default=3600)
|
|
args = parser.parse_args()
|
|
if args.grace_seconds < 3600:
|
|
parser.error('grace не меньше 3600 секунд')
|
|
store = ObjectStore(admin=True)
|
|
pool = create_pool(read_secret('DATABASE_URL'), 1)
|
|
pool.open(wait=True)
|
|
cutoff = dt.datetime.now(dt.timezone.utc) - dt.timedelta(seconds=args.grace_seconds)
|
|
counts = dict(scanned=0, fresh=0, referenced=0, orphan=0, deleted=0)
|
|
start = time.monotonic()
|
|
try:
|
|
pages = store.client.get_paginator('list_objects_v2').paginate(Bucket=store.bucket, Prefix='originals/')
|
|
for page in pages:
|
|
for obj in page.get('Contents', []):
|
|
counts['scanned'] += 1
|
|
if obj['LastModified'] >= cutoff:
|
|
counts['fresh'] += 1
|
|
continue
|
|
with pool.connection() as conn:
|
|
conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))', (obj['Key'],))
|
|
row = conn.execute('SELECT 1 FROM attachments WHERE object_key=%s', (obj['Key'],)).fetchone()
|
|
if row:
|
|
counts['referenced'] += 1
|
|
else:
|
|
counts['orphan'] += 1
|
|
if args.apply:
|
|
store.call('delete_object', Key=obj['Key'])
|
|
counts['deleted'] += 1
|
|
finally:
|
|
pool.close()
|
|
print(json.dumps(dict(counts, apply=args.apply, seconds=round(time.monotonic()-start, 3))))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|