Files
qrtx dc46ac213d feat(storage): add RustFS attachments and presigned uploads
Add two API replicas, safe object reconciliation, upload validation, regression tests and documented load experiments.
2026-11-12 10:16:39 +03:00

135 lines
7.5 KiB
Python

"""Positive and negative presigned-upload scenarios on the pinned Compose stack."""
import hashlib
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import struct
import subprocess
import sys
import time
from threading import Barrier
from urllib.parse import urlsplit, urlunsplit
import zlib
from direct_client import ROOT, Failure, check, fixture, grant, put, complete, verify, cleanup_staging
def main():
report = {'status': 'failed', 'checks': []}
def passed(name):
report['checks'].append(name)
print('PASS: ' + name, flush=True)
try:
with fixture() as (client, owner, reader, chat, foreign_chat):
image = (ROOT / 'tests/fixtures/sample.png').read_bytes()
cap = grant(client, owner, chat, image)
path = f"/api/v1/chats/{chat['id']}/attachment-uploads"
fields = {'filename': 'direct.png', 'content_type': 'image/png', 'size_bytes': len(image),
'sha256': hashlib.sha256(image).hexdigest()}
client.call('POST', path, 401, body=fields)
client.call('POST', f"/api/v1/chats/{foreign_chat['id']}/attachment-uploads", 404, actor=owner, body=fields)
client.call('POST', path, 413, actor=owner, body={**fields, 'size_bytes': 10485761})
client.call('POST', path, 400, actor=owner, body={**fields, 'filename': '../escape.png'})
client.call('POST', path, 400, actor=owner, body={**fields, 'expires_in': 301})
complete(client, reader, cap, 404)
complete(client, owner, cap, 404, path=cap['complete_url'].replace(chat['id'], foreign_chat['id']))
complete(client, owner, cap, 409)
passed('grant ACL, strict input validation, owner-only complete and missing-upload handling')
parsed = urlsplit(cap['url'])
unsigned = urlunsplit((parsed.scheme, parsed.netloc, parsed.path, '', ''))
put(client, cap, image, 403, url=unsigned)
put(client, cap, image, 403, headers={**cap['headers'], 'Content-Type': 'image/jpeg'})
put(client, cap, image + b'x', 403, headers={**cap['headers'], 'Content-Length': str(len(image) + 1)})
put(client, cap, image, 403, headers={**cap['headers'], 'If-None-Match': 'not-the-signed-value'})
altered = bytes([image[0] ^ 1]) + image[1:]
put(client, cap, altered, 400)
passed('unsigned/tampered requests denied; wrong payload checksum rejected by RustFS')
put(client, cap, image)
put(client, cap, image, 412)
attachment = complete(client, owner, cap)['json']
check(attachment['id'] == cap['upload_id'], 'Unexpected attachment ID')
verify(client, reader, attachment, image)
check(complete(client, owner, cap)['json'] == attachment, 'Complete replay changed its result')
# URLs may be reused until expiry, but only in staging; the original must stay immutable.
put(client, cap, image)
check(complete(client, owner, cap)['json'] == attachment, 'Late PUT changed publication')
verify(client, reader, attachment, image)
client.message(owner, chat, '', attachments=[attachment['id']])
passed('create-only PUT, publication, byte integrity, message binding and replay-safe immutable original')
jpeg = subprocess.run(['node', '-e', "require('sharp')('tests/fixtures/sample.png').jpeg().toBuffer().then(b=>process.stdout.write(b))"],
cwd=ROOT, capture_output=True, check=True).stdout
jpg_cap = grant(client, owner, chat, jpeg, filename='direct.jpg', content_type='image/jpeg')
put(client, jpg_cap, jpeg)
verify(client, reader, complete(client, owner, jpg_cap)['json'], jpeg)
passed('JPEG publication preserves the exact original bytes')
racing = grant(client, owner, chat, image)
put(client, racing, image)
barrier = Barrier(4)
def finish(_):
barrier.wait(timeout=30)
return complete(client, owner, racing, [201, 409])
with ThreadPoolExecutor(max_workers=4) as pool:
replies = list(pool.map(finish, range(4)))
published = complete(client, owner, racing)['json']
check(any(reply['status'] == 201 for reply in replies), 'No concurrent complete succeeded')
for reply in replies:
if reply['status'] == 201:
check(json.loads(reply['bytes']) == published, 'Concurrent publication returned different metadata')
verify(client, reader, published, image)
passed('simultaneous complete publishes one stable attachment across replicas')
for payload, mime, expected in [(b'not an image', 'image/png', 415), (image, 'image/jpeg', 415)]:
invalid = grant(client, owner, chat, payload, content_type=mime)
put(client, invalid, payload)
complete(client, owner, invalid, expected)
complete(client, owner, invalid, 409)
client.call('GET', '/api/v1/attachments/' + invalid['upload_id'], 404, actor=owner)
passed('corrupt image and MIME spoof rejected without ready metadata')
def chunk(kind, value):
return struct.pack('>I', len(value)) + kind + value + struct.pack('>I', zlib.crc32(kind + value))
huge = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 10000, 5000, 8, 2, 0, 0, 0))
huge += chunk(b'IDAT', zlib.compress(b'\x00\x00\x00\x00')) + chunk(b'IEND', b'')
invalid = grant(client, owner, chat, huge)
put(client, invalid, huge)
complete(client, owner, invalid, 413)
passed('40-million-pixel limit enforced before full decoding')
expired = grant(client, owner, chat, image, expires_in=3)
put(client, expired, image)
expired_empty = grant(client, owner, chat, image, expires_in=3)
time.sleep(4.2)
# Test expiry on a missing key: an existing object may yield 412 first.
put(client, expired_empty, image, 403)
put(client, expired, image, [403, 412])
complete(client, owner, expired, 409)
swept = cleanup_staging(chat['id'])
check(swept['stagingDeleted'] >= 1, 'Expired staging object was not cleaned')
verify(client, reader, attachment, image)
passed('real URL expiry, expired-intent cleanup and published-object protection')
proxy = client.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=proxy.png", 201,
actor=owner, raw=image, headers={'Content-Type': 'image/png'}, schema='Attachment')['json']
verify(client, reader, proxy, image)
passed('mandatory proxy upload remains compatible')
report['status'] = 'passed'
report['fixturesRemoved'] = True
except BaseException as error:
report['errorType'] = type(error).__name__
if isinstance(error, Failure): report['failure'] = str(error)
raise
finally:
(ROOT / 'evidence/direct-uploads.json').write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8')
if __name__ == '__main__':
try:
main()
except (Failure, ValueError, OSError, subprocess.SubprocessError) as error:
print('FAIL: ' + (str(error) if isinstance(error, Failure) else type(error).__name__), file=sys.stderr)
sys.exit(1)