Написал тесты вариантов, восстановления очереди и сбоев worker

This commit is contained in:
gera
2026-11-06 01:59:07 +03:00
parent 60f0f2bf13
commit d9e47a47d4
6 changed files with 566 additions and 0 deletions
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env python3
import hashlib
import uuid
from pathlib import Path
from client import Client, arguments, check, run
from image_gen import png, png_header_only
from stack import wait_status
ROOT = Path(__file__).resolve().parent.parent
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
MAX_BYTES = 10 * 1024 * 1024
def upload(c, actor, chat, data, expected, content_type='image/png', filename='file.png'):
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, expected,
actor=actor, raw=data, headers={'Content-Type': content_type},
schema='Attachment' if expected == 202 else None)
def post_message(c, actor, chat, ids, expected, text=''):
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', expected, actor=actor,
body={'text': text, 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message' if expected == 201 else None)
def main():
args = arguments('Attachment validation and access checks')
c = Client(args.base_url, args.ca_file)
alice, bob, eve, chat = c.bootstrap()
other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'second', 'member_ids': [alice['id'], bob['id']]}, schema='Chat')['json']
eve_chat = c.call('POST', '/api/v1/chats', 201, actor=eve, body={'title': 'eve', 'member_ids': [eve['id'], bob['id']]}, schema='Chat')['json']
upload(c, alice, chat, b'x' * (MAX_BYTES + 1), 413)
upload(c, alice, chat, SAMPLE + b'\0' * MAX_BYTES, 413)
upload(c, alice, chat, png_header_only(8000, 8000), 413)
upload(c, alice, chat, SAMPLE[:len(SAMPLE) // 2], 415)
upload(c, alice, chat, SAMPLE[:-1], 415)
upload(c, alice, chat, b'\x89PNG\r\n\x1a\n' + b'garbage' * 100, 415)
upload(c, alice, chat, b'', 415)
upload(c, alice, chat, b'GIF89a' + b'\0' * 64, 415)
upload(c, alice, chat, SAMPLE, 415, content_type='image/jpeg')
upload(c, alice, chat, SAMPLE, 415, content_type='text/plain')
upload(c, alice, chat, SAMPLE, 415, content_type='application/octet-stream')
for name in ('..%2Fx.png', 'a%5Cb.png', 'a%00b.png', 'a%0Ab.png', 'a' * 256, ''):
upload(c, alice, chat, SAMPLE, 400, filename=name)
c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments', 400, actor=alice, raw=SAMPLE, headers={'Content-Type': 'image/png'})
upload(c, eve, chat, SAMPLE, 404)
c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=a.png', 401, raw=SAMPLE, headers={'Content-Type': 'image/png'})
ok = upload(c, alice, chat, SAMPLE, 202)['json']
check(ok['sha256'] == hashlib.sha256(SAMPLE).hexdigest(), 'checksum mismatch')
check(ok['filename'] == 'file.png' and ok['status'] == 'queued' and ok['variants'] == [], 'attachment fields')
unicode_name = upload(c, alice, chat, SAMPLE, 202, filename='%D1%84%D0%BE%D1%82%D0%BE.png')['json']
check(unicode_name['filename'] == 'фото.png', 'unicode filename changed')
post_message(c, alice, other, [ok['id']], 404)
post_message(c, alice, chat, [str(uuid.uuid4())], 404)
post_message(c, alice, chat, [ok['id'], ok['id']], 400)
post_message(c, alice, chat, ['not-a-uuid'], 400)
post_message(c, alice, chat, [str(uuid.uuid4()) for _ in range(11)], 400)
post_message(c, alice, chat, [], 400)
post_message(c, eve, chat, [ok['id']], 404)
post_message(c, eve, eve_chat, [ok['id']], 404)
shared = post_message(c, bob, chat, [ok['id'], unicode_name['id']], 201, text='from bob')['json']
check(shared['attachment_ids'] == [ok['id'], unicode_name['id']], 'attachment order changed')
history = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=alice, schema='MessagePage')['json']['items']
check(any(m['id'] == shared['id'] and m['attachment_ids'] == shared['attachment_ids'] for m in history), 'history lost attachment_ids')
for suffix in ('', '/content'):
c.call('GET', '/api/v1/attachments/' + ok['id'] + suffix, 404, actor=eve)
c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()) + suffix, 404, actor=alice)
c.call('GET', '/api/v1/attachments/bad' + suffix, 400, actor=alice)
c.call('GET', '/api/v1/attachments/' + ok['id'] + suffix, 401)
c.call('GET', '/api/v1/attachments/' + ok['id'] + '/content?variant=bogus', 400, actor=bob)
wait_status(c, bob, ok['id'], ('ready',), 60)
download = c.call('GET', '/api/v1/attachments/' + ok['id'] + '/content', actor=bob)
check(download['bytes'] == SAMPLE, 'download differs')
big = png(1800, 1800, noise=True)
check(len(big) < MAX_BYTES, 'generated image exceeds limit')
large = upload(c, alice, chat, big, 202)['json']
wait_status(c, bob, large['id'], ('ready',), 60)
check(c.call('GET', '/api/v1/attachments/' + large['id'] + '/content', actor=bob)['bytes'] == big, 'large download differs')
print('PASS: attachment validation, limits and access control')
if __name__ == '__main__':
run(main)
+157
View File
@@ -0,0 +1,157 @@
#!/usr/bin/env python3
import json
import re
import time
import uuid
from client import Client, Failure, arguments, check, run
from stack import (SAMPLE, broker, compose, job_row, logs, now_iso, psql, restore_api, restore_worker, upload,
variant_count, wait_api, wait_log, wait_status)
def job_id_of(att_id):
return psql("select id from jobs where attachment_id='%s'" % att_id)
def worker_with(fault, **extra):
compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env=dict({'WORKER_FAULT_POINT': fault}, **extra))
def wait_s3(timeout=90):
deadline = time.monotonic() + timeout
while '(healthy)' not in compose('ps', 's3', '--format', '{{.Status}}').stdout:
check(time.monotonic() < deadline, 's3 did not recover')
time.sleep(1)
def ops(*args):
out = compose('run', '--rm', '--no-deps', '-T', 'worker', *args).stdout
return [json.loads(line) for line in out.splitlines() if line.startswith('{')]
def crash_after_commit(c, alice, bob, chat):
since = now_iso()
compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': 'after_commit'})
compose('restart', 'lb')
wait_api(c)
name = 'crash-' + uuid.uuid4().hex[:12] + '.png'
answered = None
try:
answered = upload(c, alice, chat, filename=name)
except Failure:
pass
check(answered is None, 'api answered 202 despite injected crash')
wait_log('api', since, '"point":"after_commit"', 20)
att_id = psql("select id from attachments where filename='%s'" % name)
check(att_id, 'committed attachment not found')
restore_api(c)
ready = wait_status(c, bob, att_id, ('ready', 'failed'), 60)
check(ready['status'] == 'ready' and job_row(att_id)[:2] == ('ready', 1), 'outbox did not recover the job')
published = psql("select count(*) from outbox where job_id='%s' and published_at is not null" % job_id_of(att_id))
print('1) api crashed after COMMIT before 202/publish: attachment=%s recovered by outbox relay (published rows=%s) -> ready attempt=1'
% (att_id, published))
def crash_after_store(c, alice, bob, chat):
compose('stop', 'worker')
att = upload(c, alice, chat)['json']
since = now_iso()
worker_with('after_store', JOB_TIMEOUT_SECONDS='10')
wait_log('worker', since, '"point":"after_store"', 60)
status, attempt, _ = job_row(att['id'])
check(status == 'processing' and attempt == 1, 'job must stay processing under expired-to-be lease')
since = now_iso()
worker_with('', JOB_TIMEOUT_SECONDS='10')
wait_log('worker', since, '"lease expired","job_id":"%s"' % job_id_of(att['id']), 90)
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60)
status, attempt, _ = job_row(att['id'])
check(ready['status'] == 'ready' and status == 'ready' and attempt >= 2, 'job did not finish after lease expiry')
check(variant_count(att['id']) == 3 and len(ready['variants']) == 3, 'retry duplicated variants')
thumb = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0]
body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)['bytes']
check(len(body) == thumb['size_bytes'], 'thumbnail differs from metadata')
restore_worker()
print('2) worker crashed after S3 write before COMMIT: lease expired, reaper requeued, attempt=%d ready, variants=3' % attempt)
def crash_before_ack(c, alice, bob, chat):
compose('stop', 'worker')
att = upload(c, alice, chat)['json']
job_id = job_id_of(att['id'])
since = now_iso()
worker_with('before_ack')
wait_log('worker', since, '"point":"before_ack"', 60)
wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 60)
status, attempt, _ = job_row(att['id'])
check(status == 'ready' and attempt == 1 and variant_count(att['id']) == 3, 'redelivery after commit changed result')
check(broker('XPENDING', 'attachments.jobs', 'workers').splitlines()[0] == '0', 'redelivered message not acked')
check(wait_status(c, bob, att['id'], ('ready',), 10)['status'] == 'ready', 'attachment not ready')
restore_worker()
print('3) worker crashed after COMMIT before ack: message redelivered after restart, skipped as no-op, attempt=1, variants=3')
def retries_and_replay(c, alice, bob, chat):
compose('stop', 'worker')
att = upload(c, alice, chat)['json']
job_id = job_id_of(att['id'])
compose('stop', 's3')
since = now_iso()
try:
compose('up', '-d', '--no-deps', 'worker')
check('(healthy)' not in compose('ps', '-a', 's3', '--format', '{{.Status}}').stdout, 's3 must stay stopped')
failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 120)
check(failed['status'] == 'failed' and failed['error_code'] == 'retry_exhausted', 'expected retry_exhausted')
text = logs('worker', since)
delays = [int(d) for d in re.findall(r'"job retry scheduled","job_id":"%s".*?"delay_ms":(\d+)' % job_id, text)]
check(len(delays) == 4, 'expected 4 scheduled retries, got %d' % len(delays))
for n, d in enumerate(delays):
base = 1000 * 2 ** n
check(base // 2 <= d <= base, 'delay %d outside jittered window for retry %d' % (d, n + 1))
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob)
c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, body={'text': '', 'attachment_ids': [att['id']]},
headers={'Idempotency-Key': uuid.uuid4().hex})
check(any(x['attachment_id'] == att['id'] for x in ops('failed')), 'failed registry misses job')
print('4) s3 down: transient errors retried with backoff+jitter delays_ms=%s, then failed retry_exhausted, listed in registry' % delays)
finally:
compose('start', 's3')
wait_s3()
check(ops('replay', att['id'])[0]['result'] == 'queued', 'replay did not queue job')
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 120)
check(ready['status'] == 'ready' and variant_count(att['id']) == 3, 'replayed job did not finish')
check(ops('replay', att['id'])[0]['result'] == 'skipped_ready', 'replay of ready job must be a no-op')
check(psql("select count(*) from job_replays where job_id='%s'" % job_id) == '1', 'replay audit missing')
check(variant_count(att['id']) == 3, 'second replay created new effects')
print(' replay: failed -> queued -> ready (audit rows=1); replay of ready job skipped, variants=3')
def permanent_error(c, alice, bob, chat):
data = bytearray(SAMPLE)
mid = len(data) // 2
data[mid:mid + 64] = b'\x00' * 64
att = upload(c, alice, chat, bytes(data))['json']
failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 60)
status, attempt, _ = job_row(att['id'])
check(failed['status'] == 'failed' and failed['error_code'] == 'invalid_image' and attempt == 1, 'corrupt image must fail without retries')
check(failed['variants'] == [], 'failed attachment has variants')
print('5) corrupted IDAT (passes header/tail check in API): failed invalid_image on attempt 1, no retries')
def main():
args = arguments('Fault injection: COMMIT/publish gap, crash before COMMIT and before ack, retries, DLQ replay')
c = Client(args.base_url, args.ca_file, timeout=15)
alice, bob, eve, chat = c.bootstrap()
try:
crash_after_commit(c, alice, bob, chat)
crash_after_store(c, alice, bob, chat)
crash_before_ack(c, alice, bob, chat)
retries_and_replay(c, alice, bob, chat)
permanent_error(c, alice, bob, chat)
finally:
compose('up', '-d', '--no-deps', 's3')
restore_worker()
if 'after_commit' in compose('exec', '-T', 'api', 'printenv', 'FAULT_POINT', check_rc=False).stdout:
restore_api(c)
print('PASS: outbox recovery, lease recovery, redelivery after commit, backoff with jitter, retry_exhausted, replay, permanent error')
if __name__ == '__main__':
run(main)
+68
View File
@@ -0,0 +1,68 @@
#!/usr/bin/env python3
import json
import time
from client import Client, arguments, check, run
from image_probe import jpeg_size
from stack import broker, compose, job_row, logs, now_iso, psql, upload, variant_count, wait_api, wait_log, wait_status
def stream_has(job_id):
return job_id in broker('XRANGE', 'attachments.jobs', '-', '+')
def main():
args = arguments('Worker stop, API/broker restart and redelivery scenario')
c = Client(args.base_url, args.ca_file, timeout=15)
alice, bob, eve, chat = c.bootstrap()
compose('stop', 'worker')
try:
att = upload(c, alice, chat)['json']
check(att['status'] == 'queued' and att['variants'] == [], 'upload must return queued snapshot')
msg = c.message(alice, chat, '', attachments=[att['id']])['json']
check(msg['attachment_ids'] == [att['id']], 'message lost queued attachment')
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob)
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 409, actor=bob)
c.message(bob, chat, 'text works while worker is down')
job_id = psql("select id from jobs where attachment_id='%s'" % att['id'])
deadline = time.monotonic() + 10
while not stream_has(job_id):
check(time.monotonic() < deadline, 'job was not published to broker')
time.sleep(0.5)
check(psql("select count(*) from outbox where job_id='%s' and published_at is null" % job_id) == '0', 'outbox not marked')
print('accepted: attachment=%s job=%s status=queued, job in stream, worker stopped' % (att['id'], job_id))
compose('restart', 'broker', 'api')
compose('up', '-d', '--wait', '--no-deps', 'broker', 'api')
compose('restart', 'lb')
wait_api(c)
check(stream_has(job_id), 'job lost after broker restart')
check(c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status'] == 'queued', 'status changed without worker')
print('restarted broker and api: job still in stream, attachment still queued')
finally:
compose('start', 'worker')
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60)
check(ready['status'] == 'ready', 'job did not finish after worker start')
thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)
check(jpeg_size(thumb['bytes']) == (256, 160), 'thumbnail size')
check(job_row(att['id'])[:2] == ('ready', 1) and variant_count(att['id']) == 3, 'unexpected job state or variants')
print('worker started: attachment ready, attempt=1, variants=3')
since = now_iso()
before = psql("select updated_at from attachments where id='%s'" % att['id'])
payload = json.dumps({'schema_version': 1, 'job_id': job_id, 'type': 'attachment.process.v1', 'attachment_id': att['id'],
'pipeline_version': 1, 'attempt': 1, 'created_at': ready['created_at']})
broker('XADD', 'attachments.jobs', '*', 'job', payload)
wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 30)
time.sleep(1)
check(psql("select updated_at from attachments where id='%s'" % att['id']) == before, 'duplicate delivery changed attachment')
check(variant_count(att['id']) == 3 and job_row(att['id'])[:2] == ('ready', 1), 'duplicate delivery created new effects')
pending = broker('XPENDING', 'attachments.jobs', 'workers')
check(pending.splitlines()[0] == '0', 'duplicate message was not acked')
print('duplicate delivery: skipped as no-op, acked, variants=3, updated_at unchanged')
print('PASS: worker stop, broker/api restart, redelivery idempotency')
if __name__ == '__main__':
run(main)
+92
View File
@@ -0,0 +1,92 @@
import os
import subprocess
import time
import uuid
from pathlib import Path
from client import Failure, check
ROOT = Path(__file__).resolve().parent.parent
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
def compose(*args, env=None, check_rc=True):
full = dict(os.environ, **(env or {}))
return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=check_rc,
capture_output=True, text=True, env=full)
def psql(sql):
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', sql).stdout.strip()
def broker(*args):
cmd = 'valkey-cli --no-auth-warning -a "$(sed -n "s/^requirepass //p" /run/secrets/broker_conf)" ' + ' '.join("'%s'" % a for a in args)
return compose('exec', '-T', 'broker', 'sh', '-c', cmd).stdout.strip()
def logs(service, since):
return compose('logs', '--no-color', '--since', since, service).stdout
def now_iso():
return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
def upload(c, actor, chat, data=SAMPLE, content_type='image/png', filename=None, expected=202):
filename = filename or 'f-' + uuid.uuid4().hex[:12] + '.png'
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, expected, actor=actor,
raw=data, headers={'Content-Type': content_type}, schema='Attachment' if expected == 202 else None)
def wait_status(c, actor, att_id, statuses, timeout=120):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
att = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json']
if att['status'] in statuses:
return att
except Failure:
pass
time.sleep(0.5)
raise Failure('attachment %s did not reach %s' % (att_id, statuses))
def wait_api(c, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
c.call('GET', '/health/ready')
return
except Failure:
time.sleep(1)
raise Failure('api did not become ready')
def wait_log(service, since, needle, timeout=60):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
if needle in logs(service, since):
return
time.sleep(1)
raise Failure('%s did not log %r' % (service, needle))
def job_row(att_id):
out = psql("select status, attempt, coalesce(last_error, '') from jobs where attachment_id='%s'" % att_id)
check(out, 'job not found')
status, attempt, error = out.split('|', 2)
return status, int(attempt), error
def variant_count(att_id):
return int(psql("select count(*) from attachment_variants where attachment_id='%s'" % att_id))
def restore_worker():
compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'WORKER_FAULT_POINT': ''})
def restore_api(c):
compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': ''})
compose('restart', 'lb')
wait_api(c)
+76
View File
@@ -0,0 +1,76 @@
#!/usr/bin/env python3
import hashlib
import io
from PIL import Image
from client import Client, arguments, check, run
from stack import SAMPLE, upload, wait_status
EXPECTED = {'thumbnail': ('image/jpeg', 'JPEG'), 'crop': ('image/jpeg', 'JPEG'), 'optimized': ('image/webp', 'WEBP')}
def encode(img, fmt, **kw):
buf = io.BytesIO()
img.save(buf, fmt, **kw)
return buf.getvalue()
def halves(w, h):
img = Image.new('RGB', (w, h), (0, 0, 255))
img.paste((255, 0, 0), (0, 0, w // 2, h))
return img
def process(c, alice, bob, chat, data, content_type='image/png'):
att = upload(c, alice, chat, data, content_type)['json']
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60)
check(ready['status'] == 'ready' and ready['error_code'] is None, 'attachment failed: %s' % ready['error_code'])
check(sorted(v['name'] for v in ready['variants']) == sorted(EXPECTED), 'unexpected variant set')
original = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes']
check(original == data and ready['sha256'] == hashlib.sha256(data).hexdigest(), 'original changed')
out = {}
for v in ready['variants']:
r = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + v['name'], actor=bob)
body = r['bytes']
check(r['headers'].get_content_type() == v['content_type'] == EXPECTED[v['name']][0], v['name'] + ' content type')
check(len(body) == v['size_bytes'] and hashlib.sha256(body).hexdigest() == v['sha256'], v['name'] + ' checksum')
img = Image.open(io.BytesIO(body))
check(img.format == EXPECTED[v['name']][1] and img.size == (v['width'], v['height']), v['name'] + ' real size/format')
check('exif' not in img.info, v['name'] + ' keeps EXIF')
img.load()
out[v['name']] = img.convert('RGB')
return att, out
def main():
args = arguments('Thumbnail, crop and optimized variant rules')
c = Client(args.base_url, args.ca_file, timeout=30)
alice, bob, eve, chat = c.bootstrap()
att, v = process(c, alice, bob, chat, SAMPLE)
check(v['thumbnail'].size == (256, 160) and v['crop'].size == (128, 128) and v['optimized'].size == (320, 200), 'fixture sizes')
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=bogus', 400, actor=bob)
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 404, actor=eve)
_, v = process(c, alice, bob, chat, encode(halves(2000, 1000), 'PNG'))
check(v['thumbnail'].size == (256, 128) and v['optimized'].size == (1280, 640) and v['crop'].size == (128, 128), 'large image sizes')
_, v = process(c, alice, bob, chat, encode(halves(100, 50), 'PNG'))
check(v['thumbnail'].size == (100, 50) and v['optimized'].size == (100, 50), 'small image must not be upscaled')
check(v['crop'].size == (128, 128), 'crop must be 128x128')
_, v = process(c, alice, bob, chat, encode(Image.new('RGBA', (300, 300), (0, 0, 0, 0)), 'PNG'))
for name, img in v.items():
check(min(img.getpixel((img.width // 2, img.height // 2))) >= 250, name + ': alpha not composited on white')
exif = Image.Exif()
exif[0x0112] = 6
_, v = process(c, alice, bob, chat, encode(halves(300, 200), 'JPEG', quality=95, exif=exif.tobytes()), 'image/jpeg')
t = v['thumbnail']
check(t.size == (170, 256) and v['optimized'].size == (200, 300), 'EXIF orientation not applied to dimensions')
top, bottom = t.getpixel((85, 20)), t.getpixel((85, 236))
check(top[0] > 200 and top[2] < 60 and bottom[2] > 200 and bottom[0] < 60, 'EXIF rotation direction wrong')
print('PASS: thumbnail/crop/optimized sizes, formats, checksums, no upscale, alpha on white, EXIF orientation, no EXIF in variants')
if __name__ == '__main__':
run(main)
+84
View File
@@ -0,0 +1,84 @@
#!/usr/bin/env python3
import json
import subprocess
import threading
import time
from pathlib import Path
from client import Client, arguments, check, run
from image_gen import png
from stack import SAMPLE, compose, job_row, psql, restore_worker, upload, wait_status
LIMIT_MIB = 768
def sample(stop, state):
while not stop.is_set():
out = compose('exec', '-T', 'worker', 'grep', 'VmRSS', '/proc/1/status', check_rc=False).stdout.split()
if len(out) > 1:
state['rss'] = max(state['rss'], int(out[1]) / 1024)
n = psql("select count(*) from jobs where status = 'processing'")
state['processing'] = max(state['processing'], int(n or 0))
time.sleep(0.2)
def main():
def configure(p):
p.add_argument('--huge', type=int, default=6)
p.add_argument('--small', type=int, default=20)
p.add_argument('--output')
args = arguments('Worker concurrency, memory and time limits', configure)
c = Client(args.base_url, args.ca_file, timeout=30)
alice, bob, eve, chat = c.bootstrap()
concurrency = int(compose('exec', '-T', 'worker', 'printenv', 'WORKER_CONCURRENCY').stdout.strip())
huge = png(6320, 6320)
state, stop = {'rss': 0, 'processing': 0}, threading.Event()
t = threading.Thread(target=sample, args=(stop, state))
t.start()
started = time.monotonic()
ids = [upload(c, alice, chat, huge)['json']['id'] for _ in range(args.huge)]
ids += [upload(c, alice, chat, SAMPLE)['json']['id'] for _ in range(args.small)]
accepted = time.monotonic() - started
results = [wait_status(c, bob, i, ('ready', 'failed'), 300)['status'] for i in ids]
elapsed = time.monotonic() - started
stop.set()
t.join()
container = compose('ps', '-q', 'worker').stdout.strip()
oom = subprocess.run(['docker', 'inspect', '-f', '{{.RestartCount}} {{.State.OOMKilled}}', container],
capture_output=True, text=True).stdout.split()
compose('stop', 'worker')
slow = upload(c, alice, chat, huge)['json']
compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'JOB_TIMEOUT_SECONDS': '1'})
timed = wait_status(c, bob, slow['id'], ('ready', 'failed'), 120)
_, attempts, last_error = job_row(slow['id'])
restore_worker()
report = {
'huge_images': {'count': args.huge, 'pixels': 6320 * 6320, 'bytes': len(huge)},
'small_images': args.small,
'worker_concurrency': concurrency,
'max_processing_jobs_observed': state['processing'],
'all_ready': results.count('ready') == len(results),
'accept_seconds': round(accepted, 2),
'all_ready_seconds': round(elapsed, 1),
'worker_peak_rss_mib': round(state['rss'], 1),
'worker_memory_limit_mib': LIMIT_MIB,
'worker_restarts_oom': oom,
'timeout_case': {'job_timeout_seconds': 1, 'status': timed['status'], 'error_code': timed['error_code'],
'attempts': attempts, 'last_error': last_error},
}
text = json.dumps(report, indent=2)
if args.output:
Path(args.output).write_text(text + '\n')
print(text)
check(report['all_ready'], 'some jobs failed under load')
check(state['processing'] <= concurrency, 'more jobs processing than WORKER_CONCURRENCY')
check(state['rss'] < LIMIT_MIB and oom == ['0', 'false'], 'worker reached memory limit')
check(timed['status'] == 'failed' and timed['error_code'] == 'retry_exhausted' and 'deadline' in last_error,
'timeout must be a bounded transient failure')
print('PASS: bounded worker concurrency, memory below limit, job timeout enforced')
if __name__ == '__main__':
run(main)