From d9e47a47d4e2f5f8fea33034e90ce57cce1f0206 Mon Sep 17 00:00:00 2001 From: gera Date: Fri, 6 Nov 2026 01:59:07 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9D=D0=B0=D0=BF=D0=B8=D1=81=D0=B0=D0=BB=20?= =?UTF-8?q?=D1=82=D0=B5=D1=81=D1=82=D1=8B=20=D0=B2=D0=B0=D1=80=D0=B8=D0=B0?= =?UTF-8?q?=D0=BD=D1=82=D0=BE=D0=B2,=20=D0=B2=D0=BE=D1=81=D1=81=D1=82?= =?UTF-8?q?=D0=B0=D0=BD=D0=BE=D0=B2=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=BE?= =?UTF-8?q?=D1=87=D0=B5=D1=80=D0=B5=D0=B4=D0=B8=20=D0=B8=20=D1=81=D0=B1?= =?UTF-8?q?=D0=BE=D0=B5=D0=B2=20worker?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/attachments.py | 89 +++++++++++++++++++++++ tests/faults.py | 157 ++++++++++++++++++++++++++++++++++++++++ tests/queue_recovery.py | 68 +++++++++++++++++ tests/stack.py | 92 +++++++++++++++++++++++ tests/variants.py | 76 +++++++++++++++++++ tests/worker_load.py | 84 +++++++++++++++++++++ 6 files changed, 566 insertions(+) create mode 100644 tests/attachments.py create mode 100755 tests/faults.py create mode 100755 tests/queue_recovery.py create mode 100644 tests/stack.py create mode 100755 tests/variants.py create mode 100755 tests/worker_load.py diff --git a/tests/attachments.py b/tests/attachments.py new file mode 100644 index 0000000..871480c --- /dev/null +++ b/tests/attachments.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +import hashlib +import uuid +from pathlib import Path +from client import Client, arguments, check, run +from image_gen import png, png_header_only +from stack import wait_status + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() +MAX_BYTES = 10 * 1024 * 1024 + + +def upload(c, actor, chat, data, expected, content_type='image/png', filename='file.png'): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, expected, + actor=actor, raw=data, headers={'Content-Type': content_type}, + schema='Attachment' if expected == 202 else None) + + +def post_message(c, actor, chat, ids, expected, text=''): + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', expected, actor=actor, + body={'text': text, 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message' if expected == 201 else None) + + +def main(): + args = arguments('Attachment validation and access checks') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'second', 'member_ids': [alice['id'], bob['id']]}, schema='Chat')['json'] + eve_chat = c.call('POST', '/api/v1/chats', 201, actor=eve, body={'title': 'eve', 'member_ids': [eve['id'], bob['id']]}, schema='Chat')['json'] + + upload(c, alice, chat, b'x' * (MAX_BYTES + 1), 413) + upload(c, alice, chat, SAMPLE + b'\0' * MAX_BYTES, 413) + upload(c, alice, chat, png_header_only(8000, 8000), 413) + upload(c, alice, chat, SAMPLE[:len(SAMPLE) // 2], 415) + upload(c, alice, chat, SAMPLE[:-1], 415) + upload(c, alice, chat, b'\x89PNG\r\n\x1a\n' + b'garbage' * 100, 415) + upload(c, alice, chat, b'', 415) + upload(c, alice, chat, b'GIF89a' + b'\0' * 64, 415) + upload(c, alice, chat, SAMPLE, 415, content_type='image/jpeg') + upload(c, alice, chat, SAMPLE, 415, content_type='text/plain') + upload(c, alice, chat, SAMPLE, 415, content_type='application/octet-stream') + + for name in ('..%2Fx.png', 'a%5Cb.png', 'a%00b.png', 'a%0Ab.png', 'a' * 256, ''): + upload(c, alice, chat, SAMPLE, 400, filename=name) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments', 400, actor=alice, raw=SAMPLE, headers={'Content-Type': 'image/png'}) + upload(c, eve, chat, SAMPLE, 404) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=a.png', 401, raw=SAMPLE, headers={'Content-Type': 'image/png'}) + + ok = upload(c, alice, chat, SAMPLE, 202)['json'] + check(ok['sha256'] == hashlib.sha256(SAMPLE).hexdigest(), 'checksum mismatch') + check(ok['filename'] == 'file.png' and ok['status'] == 'queued' and ok['variants'] == [], 'attachment fields') + unicode_name = upload(c, alice, chat, SAMPLE, 202, filename='%D1%84%D0%BE%D1%82%D0%BE.png')['json'] + check(unicode_name['filename'] == 'фото.png', 'unicode filename changed') + + post_message(c, alice, other, [ok['id']], 404) + post_message(c, alice, chat, [str(uuid.uuid4())], 404) + post_message(c, alice, chat, [ok['id'], ok['id']], 400) + post_message(c, alice, chat, ['not-a-uuid'], 400) + post_message(c, alice, chat, [str(uuid.uuid4()) for _ in range(11)], 400) + post_message(c, alice, chat, [], 400) + post_message(c, eve, chat, [ok['id']], 404) + post_message(c, eve, eve_chat, [ok['id']], 404) + + shared = post_message(c, bob, chat, [ok['id'], unicode_name['id']], 201, text='from bob')['json'] + check(shared['attachment_ids'] == [ok['id'], unicode_name['id']], 'attachment order changed') + history = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=alice, schema='MessagePage')['json']['items'] + check(any(m['id'] == shared['id'] and m['attachment_ids'] == shared['attachment_ids'] for m in history), 'history lost attachment_ids') + + for suffix in ('', '/content'): + c.call('GET', '/api/v1/attachments/' + ok['id'] + suffix, 404, actor=eve) + c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()) + suffix, 404, actor=alice) + c.call('GET', '/api/v1/attachments/bad' + suffix, 400, actor=alice) + c.call('GET', '/api/v1/attachments/' + ok['id'] + suffix, 401) + c.call('GET', '/api/v1/attachments/' + ok['id'] + '/content?variant=bogus', 400, actor=bob) + wait_status(c, bob, ok['id'], ('ready',), 60) + download = c.call('GET', '/api/v1/attachments/' + ok['id'] + '/content', actor=bob) + check(download['bytes'] == SAMPLE, 'download differs') + + big = png(1800, 1800, noise=True) + check(len(big) < MAX_BYTES, 'generated image exceeds limit') + large = upload(c, alice, chat, big, 202)['json'] + wait_status(c, bob, large['id'], ('ready',), 60) + check(c.call('GET', '/api/v1/attachments/' + large['id'] + '/content', actor=bob)['bytes'] == big, 'large download differs') + print('PASS: attachment validation, limits and access control') + + +if __name__ == '__main__': + run(main) diff --git a/tests/faults.py b/tests/faults.py new file mode 100755 index 0000000..d4edb60 --- /dev/null +++ b/tests/faults.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +import json +import re +import time +import uuid +from client import Client, Failure, arguments, check, run +from stack import (SAMPLE, broker, compose, job_row, logs, now_iso, psql, restore_api, restore_worker, upload, + variant_count, wait_api, wait_log, wait_status) + + +def job_id_of(att_id): + return psql("select id from jobs where attachment_id='%s'" % att_id) + + +def worker_with(fault, **extra): + compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env=dict({'WORKER_FAULT_POINT': fault}, **extra)) + + +def wait_s3(timeout=90): + deadline = time.monotonic() + timeout + while '(healthy)' not in compose('ps', 's3', '--format', '{{.Status}}').stdout: + check(time.monotonic() < deadline, 's3 did not recover') + time.sleep(1) + + +def ops(*args): + out = compose('run', '--rm', '--no-deps', '-T', 'worker', *args).stdout + return [json.loads(line) for line in out.splitlines() if line.startswith('{')] + + +def crash_after_commit(c, alice, bob, chat): + since = now_iso() + compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': 'after_commit'}) + compose('restart', 'lb') + wait_api(c) + name = 'crash-' + uuid.uuid4().hex[:12] + '.png' + answered = None + try: + answered = upload(c, alice, chat, filename=name) + except Failure: + pass + check(answered is None, 'api answered 202 despite injected crash') + wait_log('api', since, '"point":"after_commit"', 20) + att_id = psql("select id from attachments where filename='%s'" % name) + check(att_id, 'committed attachment not found') + restore_api(c) + ready = wait_status(c, bob, att_id, ('ready', 'failed'), 60) + check(ready['status'] == 'ready' and job_row(att_id)[:2] == ('ready', 1), 'outbox did not recover the job') + published = psql("select count(*) from outbox where job_id='%s' and published_at is not null" % job_id_of(att_id)) + print('1) api crashed after COMMIT before 202/publish: attachment=%s recovered by outbox relay (published rows=%s) -> ready attempt=1' + % (att_id, published)) + + +def crash_after_store(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + since = now_iso() + worker_with('after_store', JOB_TIMEOUT_SECONDS='10') + wait_log('worker', since, '"point":"after_store"', 60) + status, attempt, _ = job_row(att['id']) + check(status == 'processing' and attempt == 1, 'job must stay processing under expired-to-be lease') + since = now_iso() + worker_with('', JOB_TIMEOUT_SECONDS='10') + wait_log('worker', since, '"lease expired","job_id":"%s"' % job_id_of(att['id']), 90) + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + status, attempt, _ = job_row(att['id']) + check(ready['status'] == 'ready' and status == 'ready' and attempt >= 2, 'job did not finish after lease expiry') + check(variant_count(att['id']) == 3 and len(ready['variants']) == 3, 'retry duplicated variants') + thumb = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0] + body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)['bytes'] + check(len(body) == thumb['size_bytes'], 'thumbnail differs from metadata') + restore_worker() + print('2) worker crashed after S3 write before COMMIT: lease expired, reaper requeued, attempt=%d ready, variants=3' % attempt) + + +def crash_before_ack(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + job_id = job_id_of(att['id']) + since = now_iso() + worker_with('before_ack') + wait_log('worker', since, '"point":"before_ack"', 60) + wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 60) + status, attempt, _ = job_row(att['id']) + check(status == 'ready' and attempt == 1 and variant_count(att['id']) == 3, 'redelivery after commit changed result') + check(broker('XPENDING', 'attachments.jobs', 'workers').splitlines()[0] == '0', 'redelivered message not acked') + check(wait_status(c, bob, att['id'], ('ready',), 10)['status'] == 'ready', 'attachment not ready') + restore_worker() + print('3) worker crashed after COMMIT before ack: message redelivered after restart, skipped as no-op, attempt=1, variants=3') + + +def retries_and_replay(c, alice, bob, chat): + compose('stop', 'worker') + att = upload(c, alice, chat)['json'] + job_id = job_id_of(att['id']) + compose('stop', 's3') + since = now_iso() + try: + compose('up', '-d', '--no-deps', 'worker') + check('(healthy)' not in compose('ps', '-a', 's3', '--format', '{{.Status}}').stdout, 's3 must stay stopped') + failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 120) + check(failed['status'] == 'failed' and failed['error_code'] == 'retry_exhausted', 'expected retry_exhausted') + text = logs('worker', since) + delays = [int(d) for d in re.findall(r'"job retry scheduled","job_id":"%s".*?"delay_ms":(\d+)' % job_id, text)] + check(len(delays) == 4, 'expected 4 scheduled retries, got %d' % len(delays)) + for n, d in enumerate(delays): + base = 1000 * 2 ** n + check(base // 2 <= d <= base, 'delay %d outside jittered window for retry %d' % (d, n + 1)) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob) + c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, body={'text': '', 'attachment_ids': [att['id']]}, + headers={'Idempotency-Key': uuid.uuid4().hex}) + check(any(x['attachment_id'] == att['id'] for x in ops('failed')), 'failed registry misses job') + print('4) s3 down: transient errors retried with backoff+jitter delays_ms=%s, then failed retry_exhausted, listed in registry' % delays) + finally: + compose('start', 's3') + wait_s3() + check(ops('replay', att['id'])[0]['result'] == 'queued', 'replay did not queue job') + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 120) + check(ready['status'] == 'ready' and variant_count(att['id']) == 3, 'replayed job did not finish') + check(ops('replay', att['id'])[0]['result'] == 'skipped_ready', 'replay of ready job must be a no-op') + check(psql("select count(*) from job_replays where job_id='%s'" % job_id) == '1', 'replay audit missing') + check(variant_count(att['id']) == 3, 'second replay created new effects') + print(' replay: failed -> queued -> ready (audit rows=1); replay of ready job skipped, variants=3') + + +def permanent_error(c, alice, bob, chat): + data = bytearray(SAMPLE) + mid = len(data) // 2 + data[mid:mid + 64] = b'\x00' * 64 + att = upload(c, alice, chat, bytes(data))['json'] + failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 60) + status, attempt, _ = job_row(att['id']) + check(failed['status'] == 'failed' and failed['error_code'] == 'invalid_image' and attempt == 1, 'corrupt image must fail without retries') + check(failed['variants'] == [], 'failed attachment has variants') + print('5) corrupted IDAT (passes header/tail check in API): failed invalid_image on attempt 1, no retries') + + +def main(): + args = arguments('Fault injection: COMMIT/publish gap, crash before COMMIT and before ack, retries, DLQ replay') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + try: + crash_after_commit(c, alice, bob, chat) + crash_after_store(c, alice, bob, chat) + crash_before_ack(c, alice, bob, chat) + retries_and_replay(c, alice, bob, chat) + permanent_error(c, alice, bob, chat) + finally: + compose('up', '-d', '--no-deps', 's3') + restore_worker() + if 'after_commit' in compose('exec', '-T', 'api', 'printenv', 'FAULT_POINT', check_rc=False).stdout: + restore_api(c) + print('PASS: outbox recovery, lease recovery, redelivery after commit, backoff with jitter, retry_exhausted, replay, permanent error') + + +if __name__ == '__main__': + run(main) diff --git a/tests/queue_recovery.py b/tests/queue_recovery.py new file mode 100755 index 0000000..0c420d8 --- /dev/null +++ b/tests/queue_recovery.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +import json +import time +from client import Client, arguments, check, run +from image_probe import jpeg_size +from stack import broker, compose, job_row, logs, now_iso, psql, upload, variant_count, wait_api, wait_log, wait_status + + +def stream_has(job_id): + return job_id in broker('XRANGE', 'attachments.jobs', '-', '+') + + +def main(): + args = arguments('Worker stop, API/broker restart and redelivery scenario') + c = Client(args.base_url, args.ca_file, timeout=15) + alice, bob, eve, chat = c.bootstrap() + + compose('stop', 'worker') + try: + att = upload(c, alice, chat)['json'] + check(att['status'] == 'queued' and att['variants'] == [], 'upload must return queued snapshot') + msg = c.message(alice, chat, '', attachments=[att['id']])['json'] + check(msg['attachment_ids'] == [att['id']], 'message lost queued attachment') + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 409, actor=bob) + c.message(bob, chat, 'text works while worker is down') + job_id = psql("select id from jobs where attachment_id='%s'" % att['id']) + deadline = time.monotonic() + 10 + while not stream_has(job_id): + check(time.monotonic() < deadline, 'job was not published to broker') + time.sleep(0.5) + check(psql("select count(*) from outbox where job_id='%s' and published_at is null" % job_id) == '0', 'outbox not marked') + print('accepted: attachment=%s job=%s status=queued, job in stream, worker stopped' % (att['id'], job_id)) + + compose('restart', 'broker', 'api') + compose('up', '-d', '--wait', '--no-deps', 'broker', 'api') + compose('restart', 'lb') + wait_api(c) + check(stream_has(job_id), 'job lost after broker restart') + check(c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status'] == 'queued', 'status changed without worker') + print('restarted broker and api: job still in stream, attachment still queued') + finally: + compose('start', 'worker') + + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + check(ready['status'] == 'ready', 'job did not finish after worker start') + thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob) + check(jpeg_size(thumb['bytes']) == (256, 160), 'thumbnail size') + check(job_row(att['id'])[:2] == ('ready', 1) and variant_count(att['id']) == 3, 'unexpected job state or variants') + print('worker started: attachment ready, attempt=1, variants=3') + + since = now_iso() + before = psql("select updated_at from attachments where id='%s'" % att['id']) + payload = json.dumps({'schema_version': 1, 'job_id': job_id, 'type': 'attachment.process.v1', 'attachment_id': att['id'], + 'pipeline_version': 1, 'attempt': 1, 'created_at': ready['created_at']}) + broker('XADD', 'attachments.jobs', '*', 'job', payload) + wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 30) + time.sleep(1) + check(psql("select updated_at from attachments where id='%s'" % att['id']) == before, 'duplicate delivery changed attachment') + check(variant_count(att['id']) == 3 and job_row(att['id'])[:2] == ('ready', 1), 'duplicate delivery created new effects') + pending = broker('XPENDING', 'attachments.jobs', 'workers') + check(pending.splitlines()[0] == '0', 'duplicate message was not acked') + print('duplicate delivery: skipped as no-op, acked, variants=3, updated_at unchanged') + print('PASS: worker stop, broker/api restart, redelivery idempotency') + + +if __name__ == '__main__': + run(main) diff --git a/tests/stack.py b/tests/stack.py new file mode 100644 index 0000000..f0b645f --- /dev/null +++ b/tests/stack.py @@ -0,0 +1,92 @@ +import os +import subprocess +import time +import uuid +from pathlib import Path +from client import Failure, check + +ROOT = Path(__file__).resolve().parent.parent +SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes() + + +def compose(*args, env=None, check_rc=True): + full = dict(os.environ, **(env or {})) + return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=check_rc, + capture_output=True, text=True, env=full) + + +def psql(sql): + return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', sql).stdout.strip() + + +def broker(*args): + cmd = 'valkey-cli --no-auth-warning -a "$(sed -n "s/^requirepass //p" /run/secrets/broker_conf)" ' + ' '.join("'%s'" % a for a in args) + return compose('exec', '-T', 'broker', 'sh', '-c', cmd).stdout.strip() + + +def logs(service, since): + return compose('logs', '--no-color', '--since', since, service).stdout + + +def now_iso(): + return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) + + +def upload(c, actor, chat, data=SAMPLE, content_type='image/png', filename=None, expected=202): + filename = filename or 'f-' + uuid.uuid4().hex[:12] + '.png' + return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, expected, actor=actor, + raw=data, headers={'Content-Type': content_type}, schema='Attachment' if expected == 202 else None) + + +def wait_status(c, actor, att_id, statuses, timeout=120): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + att = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json'] + if att['status'] in statuses: + return att + except Failure: + pass + time.sleep(0.5) + raise Failure('attachment %s did not reach %s' % (att_id, statuses)) + + +def wait_api(c, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + c.call('GET', '/health/ready') + return + except Failure: + time.sleep(1) + raise Failure('api did not become ready') + + +def wait_log(service, since, needle, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if needle in logs(service, since): + return + time.sleep(1) + raise Failure('%s did not log %r' % (service, needle)) + + +def job_row(att_id): + out = psql("select status, attempt, coalesce(last_error, '') from jobs where attachment_id='%s'" % att_id) + check(out, 'job not found') + status, attempt, error = out.split('|', 2) + return status, int(attempt), error + + +def variant_count(att_id): + return int(psql("select count(*) from attachment_variants where attachment_id='%s'" % att_id)) + + +def restore_worker(): + compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'WORKER_FAULT_POINT': ''}) + + +def restore_api(c): + compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': ''}) + compose('restart', 'lb') + wait_api(c) diff --git a/tests/variants.py b/tests/variants.py new file mode 100755 index 0000000..4c00905 --- /dev/null +++ b/tests/variants.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +import hashlib +import io +from PIL import Image +from client import Client, arguments, check, run +from stack import SAMPLE, upload, wait_status + +EXPECTED = {'thumbnail': ('image/jpeg', 'JPEG'), 'crop': ('image/jpeg', 'JPEG'), 'optimized': ('image/webp', 'WEBP')} + + +def encode(img, fmt, **kw): + buf = io.BytesIO() + img.save(buf, fmt, **kw) + return buf.getvalue() + + +def halves(w, h): + img = Image.new('RGB', (w, h), (0, 0, 255)) + img.paste((255, 0, 0), (0, 0, w // 2, h)) + return img + + +def process(c, alice, bob, chat, data, content_type='image/png'): + att = upload(c, alice, chat, data, content_type)['json'] + ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60) + check(ready['status'] == 'ready' and ready['error_code'] is None, 'attachment failed: %s' % ready['error_code']) + check(sorted(v['name'] for v in ready['variants']) == sorted(EXPECTED), 'unexpected variant set') + original = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] + check(original == data and ready['sha256'] == hashlib.sha256(data).hexdigest(), 'original changed') + out = {} + for v in ready['variants']: + r = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + v['name'], actor=bob) + body = r['bytes'] + check(r['headers'].get_content_type() == v['content_type'] == EXPECTED[v['name']][0], v['name'] + ' content type') + check(len(body) == v['size_bytes'] and hashlib.sha256(body).hexdigest() == v['sha256'], v['name'] + ' checksum') + img = Image.open(io.BytesIO(body)) + check(img.format == EXPECTED[v['name']][1] and img.size == (v['width'], v['height']), v['name'] + ' real size/format') + check('exif' not in img.info, v['name'] + ' keeps EXIF') + img.load() + out[v['name']] = img.convert('RGB') + return att, out + + +def main(): + args = arguments('Thumbnail, crop and optimized variant rules') + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + + att, v = process(c, alice, bob, chat, SAMPLE) + check(v['thumbnail'].size == (256, 160) and v['crop'].size == (128, 128) and v['optimized'].size == (320, 200), 'fixture sizes') + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=bogus', 400, actor=bob) + c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 404, actor=eve) + + _, v = process(c, alice, bob, chat, encode(halves(2000, 1000), 'PNG')) + check(v['thumbnail'].size == (256, 128) and v['optimized'].size == (1280, 640) and v['crop'].size == (128, 128), 'large image sizes') + + _, v = process(c, alice, bob, chat, encode(halves(100, 50), 'PNG')) + check(v['thumbnail'].size == (100, 50) and v['optimized'].size == (100, 50), 'small image must not be upscaled') + check(v['crop'].size == (128, 128), 'crop must be 128x128') + + _, v = process(c, alice, bob, chat, encode(Image.new('RGBA', (300, 300), (0, 0, 0, 0)), 'PNG')) + for name, img in v.items(): + check(min(img.getpixel((img.width // 2, img.height // 2))) >= 250, name + ': alpha not composited on white') + + exif = Image.Exif() + exif[0x0112] = 6 + _, v = process(c, alice, bob, chat, encode(halves(300, 200), 'JPEG', quality=95, exif=exif.tobytes()), 'image/jpeg') + t = v['thumbnail'] + check(t.size == (170, 256) and v['optimized'].size == (200, 300), 'EXIF orientation not applied to dimensions') + top, bottom = t.getpixel((85, 20)), t.getpixel((85, 236)) + check(top[0] > 200 and top[2] < 60 and bottom[2] > 200 and bottom[0] < 60, 'EXIF rotation direction wrong') + print('PASS: thumbnail/crop/optimized sizes, formats, checksums, no upscale, alpha on white, EXIF orientation, no EXIF in variants') + + +if __name__ == '__main__': + run(main) diff --git a/tests/worker_load.py b/tests/worker_load.py new file mode 100755 index 0000000..97bdfd7 --- /dev/null +++ b/tests/worker_load.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +import json +import subprocess +import threading +import time +from pathlib import Path +from client import Client, arguments, check, run +from image_gen import png +from stack import SAMPLE, compose, job_row, psql, restore_worker, upload, wait_status + +LIMIT_MIB = 768 + + +def sample(stop, state): + while not stop.is_set(): + out = compose('exec', '-T', 'worker', 'grep', 'VmRSS', '/proc/1/status', check_rc=False).stdout.split() + if len(out) > 1: + state['rss'] = max(state['rss'], int(out[1]) / 1024) + n = psql("select count(*) from jobs where status = 'processing'") + state['processing'] = max(state['processing'], int(n or 0)) + time.sleep(0.2) + + +def main(): + def configure(p): + p.add_argument('--huge', type=int, default=6) + p.add_argument('--small', type=int, default=20) + p.add_argument('--output') + args = arguments('Worker concurrency, memory and time limits', configure) + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + concurrency = int(compose('exec', '-T', 'worker', 'printenv', 'WORKER_CONCURRENCY').stdout.strip()) + huge = png(6320, 6320) + + state, stop = {'rss': 0, 'processing': 0}, threading.Event() + t = threading.Thread(target=sample, args=(stop, state)) + t.start() + started = time.monotonic() + ids = [upload(c, alice, chat, huge)['json']['id'] for _ in range(args.huge)] + ids += [upload(c, alice, chat, SAMPLE)['json']['id'] for _ in range(args.small)] + accepted = time.monotonic() - started + results = [wait_status(c, bob, i, ('ready', 'failed'), 300)['status'] for i in ids] + elapsed = time.monotonic() - started + stop.set() + t.join() + container = compose('ps', '-q', 'worker').stdout.strip() + oom = subprocess.run(['docker', 'inspect', '-f', '{{.RestartCount}} {{.State.OOMKilled}}', container], + capture_output=True, text=True).stdout.split() + + compose('stop', 'worker') + slow = upload(c, alice, chat, huge)['json'] + compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'JOB_TIMEOUT_SECONDS': '1'}) + timed = wait_status(c, bob, slow['id'], ('ready', 'failed'), 120) + _, attempts, last_error = job_row(slow['id']) + restore_worker() + + report = { + 'huge_images': {'count': args.huge, 'pixels': 6320 * 6320, 'bytes': len(huge)}, + 'small_images': args.small, + 'worker_concurrency': concurrency, + 'max_processing_jobs_observed': state['processing'], + 'all_ready': results.count('ready') == len(results), + 'accept_seconds': round(accepted, 2), + 'all_ready_seconds': round(elapsed, 1), + 'worker_peak_rss_mib': round(state['rss'], 1), + 'worker_memory_limit_mib': LIMIT_MIB, + 'worker_restarts_oom': oom, + 'timeout_case': {'job_timeout_seconds': 1, 'status': timed['status'], 'error_code': timed['error_code'], + 'attempts': attempts, 'last_error': last_error}, + } + text = json.dumps(report, indent=2) + if args.output: + Path(args.output).write_text(text + '\n') + print(text) + check(report['all_ready'], 'some jobs failed under load') + check(state['processing'] <= concurrency, 'more jobs processing than WORKER_CONCURRENCY') + check(state['rss'] < LIMIT_MIB and oom == ['0', 'false'], 'worker reached memory limit') + check(timed['status'] == 'failed' and timed['error_code'] == 'retry_exhausted' and 'deadline' in last_error, + 'timeout must be a bounded transient failure') + print('PASS: bounded worker concurrency, memory below limit, job timeout enforced') + + +if __name__ == '__main__': + run(main)