forked from mtusi2026/mtusi2026lab5
Написал тесты вариантов, восстановления очереди и сбоев worker
This commit is contained in:
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from client import Client, arguments, check, run
|
||||
from image_gen import png, png_header_only
|
||||
from stack import wait_status
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
|
||||
|
||||
def upload(c, actor, chat, data, expected, content_type='image/png', filename='file.png'):
|
||||
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, expected,
|
||||
actor=actor, raw=data, headers={'Content-Type': content_type},
|
||||
schema='Attachment' if expected == 202 else None)
|
||||
|
||||
|
||||
def post_message(c, actor, chat, ids, expected, text=''):
|
||||
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', expected, actor=actor,
|
||||
body={'text': text, 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex}, schema='Message' if expected == 201 else None)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Attachment validation and access checks')
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'second', 'member_ids': [alice['id'], bob['id']]}, schema='Chat')['json']
|
||||
eve_chat = c.call('POST', '/api/v1/chats', 201, actor=eve, body={'title': 'eve', 'member_ids': [eve['id'], bob['id']]}, schema='Chat')['json']
|
||||
|
||||
upload(c, alice, chat, b'x' * (MAX_BYTES + 1), 413)
|
||||
upload(c, alice, chat, SAMPLE + b'\0' * MAX_BYTES, 413)
|
||||
upload(c, alice, chat, png_header_only(8000, 8000), 413)
|
||||
upload(c, alice, chat, SAMPLE[:len(SAMPLE) // 2], 415)
|
||||
upload(c, alice, chat, SAMPLE[:-1], 415)
|
||||
upload(c, alice, chat, b'\x89PNG\r\n\x1a\n' + b'garbage' * 100, 415)
|
||||
upload(c, alice, chat, b'', 415)
|
||||
upload(c, alice, chat, b'GIF89a' + b'\0' * 64, 415)
|
||||
upload(c, alice, chat, SAMPLE, 415, content_type='image/jpeg')
|
||||
upload(c, alice, chat, SAMPLE, 415, content_type='text/plain')
|
||||
upload(c, alice, chat, SAMPLE, 415, content_type='application/octet-stream')
|
||||
|
||||
for name in ('..%2Fx.png', 'a%5Cb.png', 'a%00b.png', 'a%0Ab.png', 'a' * 256, ''):
|
||||
upload(c, alice, chat, SAMPLE, 400, filename=name)
|
||||
c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments', 400, actor=alice, raw=SAMPLE, headers={'Content-Type': 'image/png'})
|
||||
upload(c, eve, chat, SAMPLE, 404)
|
||||
c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=a.png', 401, raw=SAMPLE, headers={'Content-Type': 'image/png'})
|
||||
|
||||
ok = upload(c, alice, chat, SAMPLE, 202)['json']
|
||||
check(ok['sha256'] == hashlib.sha256(SAMPLE).hexdigest(), 'checksum mismatch')
|
||||
check(ok['filename'] == 'file.png' and ok['status'] == 'queued' and ok['variants'] == [], 'attachment fields')
|
||||
unicode_name = upload(c, alice, chat, SAMPLE, 202, filename='%D1%84%D0%BE%D1%82%D0%BE.png')['json']
|
||||
check(unicode_name['filename'] == 'фото.png', 'unicode filename changed')
|
||||
|
||||
post_message(c, alice, other, [ok['id']], 404)
|
||||
post_message(c, alice, chat, [str(uuid.uuid4())], 404)
|
||||
post_message(c, alice, chat, [ok['id'], ok['id']], 400)
|
||||
post_message(c, alice, chat, ['not-a-uuid'], 400)
|
||||
post_message(c, alice, chat, [str(uuid.uuid4()) for _ in range(11)], 400)
|
||||
post_message(c, alice, chat, [], 400)
|
||||
post_message(c, eve, chat, [ok['id']], 404)
|
||||
post_message(c, eve, eve_chat, [ok['id']], 404)
|
||||
|
||||
shared = post_message(c, bob, chat, [ok['id'], unicode_name['id']], 201, text='from bob')['json']
|
||||
check(shared['attachment_ids'] == [ok['id'], unicode_name['id']], 'attachment order changed')
|
||||
history = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', actor=alice, schema='MessagePage')['json']['items']
|
||||
check(any(m['id'] == shared['id'] and m['attachment_ids'] == shared['attachment_ids'] for m in history), 'history lost attachment_ids')
|
||||
|
||||
for suffix in ('', '/content'):
|
||||
c.call('GET', '/api/v1/attachments/' + ok['id'] + suffix, 404, actor=eve)
|
||||
c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()) + suffix, 404, actor=alice)
|
||||
c.call('GET', '/api/v1/attachments/bad' + suffix, 400, actor=alice)
|
||||
c.call('GET', '/api/v1/attachments/' + ok['id'] + suffix, 401)
|
||||
c.call('GET', '/api/v1/attachments/' + ok['id'] + '/content?variant=bogus', 400, actor=bob)
|
||||
wait_status(c, bob, ok['id'], ('ready',), 60)
|
||||
download = c.call('GET', '/api/v1/attachments/' + ok['id'] + '/content', actor=bob)
|
||||
check(download['bytes'] == SAMPLE, 'download differs')
|
||||
|
||||
big = png(1800, 1800, noise=True)
|
||||
check(len(big) < MAX_BYTES, 'generated image exceeds limit')
|
||||
large = upload(c, alice, chat, big, 202)['json']
|
||||
wait_status(c, bob, large['id'], ('ready',), 60)
|
||||
check(c.call('GET', '/api/v1/attachments/' + large['id'] + '/content', actor=bob)['bytes'] == big, 'large download differs')
|
||||
print('PASS: attachment validation, limits and access control')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Executable
+157
@@ -0,0 +1,157 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from client import Client, Failure, arguments, check, run
|
||||
from stack import (SAMPLE, broker, compose, job_row, logs, now_iso, psql, restore_api, restore_worker, upload,
|
||||
variant_count, wait_api, wait_log, wait_status)
|
||||
|
||||
|
||||
def job_id_of(att_id):
|
||||
return psql("select id from jobs where attachment_id='%s'" % att_id)
|
||||
|
||||
|
||||
def worker_with(fault, **extra):
|
||||
compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env=dict({'WORKER_FAULT_POINT': fault}, **extra))
|
||||
|
||||
|
||||
def wait_s3(timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while '(healthy)' not in compose('ps', 's3', '--format', '{{.Status}}').stdout:
|
||||
check(time.monotonic() < deadline, 's3 did not recover')
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def ops(*args):
|
||||
out = compose('run', '--rm', '--no-deps', '-T', 'worker', *args).stdout
|
||||
return [json.loads(line) for line in out.splitlines() if line.startswith('{')]
|
||||
|
||||
|
||||
def crash_after_commit(c, alice, bob, chat):
|
||||
since = now_iso()
|
||||
compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': 'after_commit'})
|
||||
compose('restart', 'lb')
|
||||
wait_api(c)
|
||||
name = 'crash-' + uuid.uuid4().hex[:12] + '.png'
|
||||
answered = None
|
||||
try:
|
||||
answered = upload(c, alice, chat, filename=name)
|
||||
except Failure:
|
||||
pass
|
||||
check(answered is None, 'api answered 202 despite injected crash')
|
||||
wait_log('api', since, '"point":"after_commit"', 20)
|
||||
att_id = psql("select id from attachments where filename='%s'" % name)
|
||||
check(att_id, 'committed attachment not found')
|
||||
restore_api(c)
|
||||
ready = wait_status(c, bob, att_id, ('ready', 'failed'), 60)
|
||||
check(ready['status'] == 'ready' and job_row(att_id)[:2] == ('ready', 1), 'outbox did not recover the job')
|
||||
published = psql("select count(*) from outbox where job_id='%s' and published_at is not null" % job_id_of(att_id))
|
||||
print('1) api crashed after COMMIT before 202/publish: attachment=%s recovered by outbox relay (published rows=%s) -> ready attempt=1'
|
||||
% (att_id, published))
|
||||
|
||||
|
||||
def crash_after_store(c, alice, bob, chat):
|
||||
compose('stop', 'worker')
|
||||
att = upload(c, alice, chat)['json']
|
||||
since = now_iso()
|
||||
worker_with('after_store', JOB_TIMEOUT_SECONDS='10')
|
||||
wait_log('worker', since, '"point":"after_store"', 60)
|
||||
status, attempt, _ = job_row(att['id'])
|
||||
check(status == 'processing' and attempt == 1, 'job must stay processing under expired-to-be lease')
|
||||
since = now_iso()
|
||||
worker_with('', JOB_TIMEOUT_SECONDS='10')
|
||||
wait_log('worker', since, '"lease expired","job_id":"%s"' % job_id_of(att['id']), 90)
|
||||
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60)
|
||||
status, attempt, _ = job_row(att['id'])
|
||||
check(ready['status'] == 'ready' and status == 'ready' and attempt >= 2, 'job did not finish after lease expiry')
|
||||
check(variant_count(att['id']) == 3 and len(ready['variants']) == 3, 'retry duplicated variants')
|
||||
thumb = [v for v in ready['variants'] if v['name'] == 'thumbnail'][0]
|
||||
body = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)['bytes']
|
||||
check(len(body) == thumb['size_bytes'], 'thumbnail differs from metadata')
|
||||
restore_worker()
|
||||
print('2) worker crashed after S3 write before COMMIT: lease expired, reaper requeued, attempt=%d ready, variants=3' % attempt)
|
||||
|
||||
|
||||
def crash_before_ack(c, alice, bob, chat):
|
||||
compose('stop', 'worker')
|
||||
att = upload(c, alice, chat)['json']
|
||||
job_id = job_id_of(att['id'])
|
||||
since = now_iso()
|
||||
worker_with('before_ack')
|
||||
wait_log('worker', since, '"point":"before_ack"', 60)
|
||||
wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 60)
|
||||
status, attempt, _ = job_row(att['id'])
|
||||
check(status == 'ready' and attempt == 1 and variant_count(att['id']) == 3, 'redelivery after commit changed result')
|
||||
check(broker('XPENDING', 'attachments.jobs', 'workers').splitlines()[0] == '0', 'redelivered message not acked')
|
||||
check(wait_status(c, bob, att['id'], ('ready',), 10)['status'] == 'ready', 'attachment not ready')
|
||||
restore_worker()
|
||||
print('3) worker crashed after COMMIT before ack: message redelivered after restart, skipped as no-op, attempt=1, variants=3')
|
||||
|
||||
|
||||
def retries_and_replay(c, alice, bob, chat):
|
||||
compose('stop', 'worker')
|
||||
att = upload(c, alice, chat)['json']
|
||||
job_id = job_id_of(att['id'])
|
||||
compose('stop', 's3')
|
||||
since = now_iso()
|
||||
try:
|
||||
compose('up', '-d', '--no-deps', 'worker')
|
||||
check('(healthy)' not in compose('ps', '-a', 's3', '--format', '{{.Status}}').stdout, 's3 must stay stopped')
|
||||
failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 120)
|
||||
check(failed['status'] == 'failed' and failed['error_code'] == 'retry_exhausted', 'expected retry_exhausted')
|
||||
text = logs('worker', since)
|
||||
delays = [int(d) for d in re.findall(r'"job retry scheduled","job_id":"%s".*?"delay_ms":(\d+)' % job_id, text)]
|
||||
check(len(delays) == 4, 'expected 4 scheduled retries, got %d' % len(delays))
|
||||
for n, d in enumerate(delays):
|
||||
base = 1000 * 2 ** n
|
||||
check(base // 2 <= d <= base, 'delay %d outside jittered window for retry %d' % (d, n + 1))
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob)
|
||||
c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, body={'text': '', 'attachment_ids': [att['id']]},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
check(any(x['attachment_id'] == att['id'] for x in ops('failed')), 'failed registry misses job')
|
||||
print('4) s3 down: transient errors retried with backoff+jitter delays_ms=%s, then failed retry_exhausted, listed in registry' % delays)
|
||||
finally:
|
||||
compose('start', 's3')
|
||||
wait_s3()
|
||||
check(ops('replay', att['id'])[0]['result'] == 'queued', 'replay did not queue job')
|
||||
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 120)
|
||||
check(ready['status'] == 'ready' and variant_count(att['id']) == 3, 'replayed job did not finish')
|
||||
check(ops('replay', att['id'])[0]['result'] == 'skipped_ready', 'replay of ready job must be a no-op')
|
||||
check(psql("select count(*) from job_replays where job_id='%s'" % job_id) == '1', 'replay audit missing')
|
||||
check(variant_count(att['id']) == 3, 'second replay created new effects')
|
||||
print(' replay: failed -> queued -> ready (audit rows=1); replay of ready job skipped, variants=3')
|
||||
|
||||
|
||||
def permanent_error(c, alice, bob, chat):
|
||||
data = bytearray(SAMPLE)
|
||||
mid = len(data) // 2
|
||||
data[mid:mid + 64] = b'\x00' * 64
|
||||
att = upload(c, alice, chat, bytes(data))['json']
|
||||
failed = wait_status(c, bob, att['id'], ('failed', 'ready'), 60)
|
||||
status, attempt, _ = job_row(att['id'])
|
||||
check(failed['status'] == 'failed' and failed['error_code'] == 'invalid_image' and attempt == 1, 'corrupt image must fail without retries')
|
||||
check(failed['variants'] == [], 'failed attachment has variants')
|
||||
print('5) corrupted IDAT (passes header/tail check in API): failed invalid_image on attempt 1, no retries')
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Fault injection: COMMIT/publish gap, crash before COMMIT and before ack, retries, DLQ replay')
|
||||
c = Client(args.base_url, args.ca_file, timeout=15)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
try:
|
||||
crash_after_commit(c, alice, bob, chat)
|
||||
crash_after_store(c, alice, bob, chat)
|
||||
crash_before_ack(c, alice, bob, chat)
|
||||
retries_and_replay(c, alice, bob, chat)
|
||||
permanent_error(c, alice, bob, chat)
|
||||
finally:
|
||||
compose('up', '-d', '--no-deps', 's3')
|
||||
restore_worker()
|
||||
if 'after_commit' in compose('exec', '-T', 'api', 'printenv', 'FAULT_POINT', check_rc=False).stdout:
|
||||
restore_api(c)
|
||||
print('PASS: outbox recovery, lease recovery, redelivery after commit, backoff with jitter, retry_exhausted, replay, permanent error')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Executable
+68
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import time
|
||||
from client import Client, arguments, check, run
|
||||
from image_probe import jpeg_size
|
||||
from stack import broker, compose, job_row, logs, now_iso, psql, upload, variant_count, wait_api, wait_log, wait_status
|
||||
|
||||
|
||||
def stream_has(job_id):
|
||||
return job_id in broker('XRANGE', 'attachments.jobs', '-', '+')
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Worker stop, API/broker restart and redelivery scenario')
|
||||
c = Client(args.base_url, args.ca_file, timeout=15)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
att = upload(c, alice, chat)['json']
|
||||
check(att['status'] == 'queued' and att['variants'] == [], 'upload must return queued snapshot')
|
||||
msg = c.message(alice, chat, '', attachments=[att['id']])['json']
|
||||
check(msg['attachment_ids'] == [att['id']], 'message lost queued attachment')
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', 409, actor=bob)
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 409, actor=bob)
|
||||
c.message(bob, chat, 'text works while worker is down')
|
||||
job_id = psql("select id from jobs where attachment_id='%s'" % att['id'])
|
||||
deadline = time.monotonic() + 10
|
||||
while not stream_has(job_id):
|
||||
check(time.monotonic() < deadline, 'job was not published to broker')
|
||||
time.sleep(0.5)
|
||||
check(psql("select count(*) from outbox where job_id='%s' and published_at is null" % job_id) == '0', 'outbox not marked')
|
||||
print('accepted: attachment=%s job=%s status=queued, job in stream, worker stopped' % (att['id'], job_id))
|
||||
|
||||
compose('restart', 'broker', 'api')
|
||||
compose('up', '-d', '--wait', '--no-deps', 'broker', 'api')
|
||||
compose('restart', 'lb')
|
||||
wait_api(c)
|
||||
check(stream_has(job_id), 'job lost after broker restart')
|
||||
check(c.call('GET', '/api/v1/attachments/' + att['id'], actor=bob, schema='Attachment')['json']['status'] == 'queued', 'status changed without worker')
|
||||
print('restarted broker and api: job still in stream, attachment still queued')
|
||||
finally:
|
||||
compose('start', 'worker')
|
||||
|
||||
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60)
|
||||
check(ready['status'] == 'ready', 'job did not finish after worker start')
|
||||
thumb = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', actor=bob)
|
||||
check(jpeg_size(thumb['bytes']) == (256, 160), 'thumbnail size')
|
||||
check(job_row(att['id'])[:2] == ('ready', 1) and variant_count(att['id']) == 3, 'unexpected job state or variants')
|
||||
print('worker started: attachment ready, attempt=1, variants=3')
|
||||
|
||||
since = now_iso()
|
||||
before = psql("select updated_at from attachments where id='%s'" % att['id'])
|
||||
payload = json.dumps({'schema_version': 1, 'job_id': job_id, 'type': 'attachment.process.v1', 'attachment_id': att['id'],
|
||||
'pipeline_version': 1, 'attempt': 1, 'created_at': ready['created_at']})
|
||||
broker('XADD', 'attachments.jobs', '*', 'job', payload)
|
||||
wait_log('worker', since, '"job delivery skipped","job_id":"%s"' % job_id, 30)
|
||||
time.sleep(1)
|
||||
check(psql("select updated_at from attachments where id='%s'" % att['id']) == before, 'duplicate delivery changed attachment')
|
||||
check(variant_count(att['id']) == 3 and job_row(att['id'])[:2] == ('ready', 1), 'duplicate delivery created new effects')
|
||||
pending = broker('XPENDING', 'attachments.jobs', 'workers')
|
||||
check(pending.splitlines()[0] == '0', 'duplicate message was not acked')
|
||||
print('duplicate delivery: skipped as no-op, acked, variants=3, updated_at unchanged')
|
||||
print('PASS: worker stop, broker/api restart, redelivery idempotency')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,92 @@
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from client import Failure, check
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SAMPLE = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
|
||||
|
||||
def compose(*args, env=None, check_rc=True):
|
||||
full = dict(os.environ, **(env or {}))
|
||||
return subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, check=check_rc,
|
||||
capture_output=True, text=True, env=full)
|
||||
|
||||
|
||||
def psql(sql):
|
||||
return compose('exec', '-T', 'postgres', 'psql', '-U', 'messenger', '-d', 'messenger', '-Atc', sql).stdout.strip()
|
||||
|
||||
|
||||
def broker(*args):
|
||||
cmd = 'valkey-cli --no-auth-warning -a "$(sed -n "s/^requirepass //p" /run/secrets/broker_conf)" ' + ' '.join("'%s'" % a for a in args)
|
||||
return compose('exec', '-T', 'broker', 'sh', '-c', cmd).stdout.strip()
|
||||
|
||||
|
||||
def logs(service, since):
|
||||
return compose('logs', '--no-color', '--since', since, service).stdout
|
||||
|
||||
|
||||
def now_iso():
|
||||
return time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())
|
||||
|
||||
|
||||
def upload(c, actor, chat, data=SAMPLE, content_type='image/png', filename=None, expected=202):
|
||||
filename = filename or 'f-' + uuid.uuid4().hex[:12] + '.png'
|
||||
return c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + filename, expected, actor=actor,
|
||||
raw=data, headers={'Content-Type': content_type}, schema='Attachment' if expected == 202 else None)
|
||||
|
||||
|
||||
def wait_status(c, actor, att_id, statuses, timeout=120):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
att = c.call('GET', '/api/v1/attachments/' + att_id, actor=actor, schema='Attachment')['json']
|
||||
if att['status'] in statuses:
|
||||
return att
|
||||
except Failure:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise Failure('attachment %s did not reach %s' % (att_id, statuses))
|
||||
|
||||
|
||||
def wait_api(c, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
c.call('GET', '/health/ready')
|
||||
return
|
||||
except Failure:
|
||||
time.sleep(1)
|
||||
raise Failure('api did not become ready')
|
||||
|
||||
|
||||
def wait_log(service, since, needle, timeout=60):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if needle in logs(service, since):
|
||||
return
|
||||
time.sleep(1)
|
||||
raise Failure('%s did not log %r' % (service, needle))
|
||||
|
||||
|
||||
def job_row(att_id):
|
||||
out = psql("select status, attempt, coalesce(last_error, '') from jobs where attachment_id='%s'" % att_id)
|
||||
check(out, 'job not found')
|
||||
status, attempt, error = out.split('|', 2)
|
||||
return status, int(attempt), error
|
||||
|
||||
|
||||
def variant_count(att_id):
|
||||
return int(psql("select count(*) from attachment_variants where attachment_id='%s'" % att_id))
|
||||
|
||||
|
||||
def restore_worker():
|
||||
compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'WORKER_FAULT_POINT': ''})
|
||||
|
||||
|
||||
def restore_api(c):
|
||||
compose('up', '-d', '--force-recreate', '--no-deps', '--wait', 'api', env={'API_FAULT_POINT': ''})
|
||||
compose('restart', 'lb')
|
||||
wait_api(c)
|
||||
Executable
+76
@@ -0,0 +1,76 @@
|
||||
#!/usr/bin/env python3
|
||||
import hashlib
|
||||
import io
|
||||
from PIL import Image
|
||||
from client import Client, arguments, check, run
|
||||
from stack import SAMPLE, upload, wait_status
|
||||
|
||||
EXPECTED = {'thumbnail': ('image/jpeg', 'JPEG'), 'crop': ('image/jpeg', 'JPEG'), 'optimized': ('image/webp', 'WEBP')}
|
||||
|
||||
|
||||
def encode(img, fmt, **kw):
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, fmt, **kw)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def halves(w, h):
|
||||
img = Image.new('RGB', (w, h), (0, 0, 255))
|
||||
img.paste((255, 0, 0), (0, 0, w // 2, h))
|
||||
return img
|
||||
|
||||
|
||||
def process(c, alice, bob, chat, data, content_type='image/png'):
|
||||
att = upload(c, alice, chat, data, content_type)['json']
|
||||
ready = wait_status(c, bob, att['id'], ('ready', 'failed'), 60)
|
||||
check(ready['status'] == 'ready' and ready['error_code'] is None, 'attachment failed: %s' % ready['error_code'])
|
||||
check(sorted(v['name'] for v in ready['variants']) == sorted(EXPECTED), 'unexpected variant set')
|
||||
original = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes']
|
||||
check(original == data and ready['sha256'] == hashlib.sha256(data).hexdigest(), 'original changed')
|
||||
out = {}
|
||||
for v in ready['variants']:
|
||||
r = c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=' + v['name'], actor=bob)
|
||||
body = r['bytes']
|
||||
check(r['headers'].get_content_type() == v['content_type'] == EXPECTED[v['name']][0], v['name'] + ' content type')
|
||||
check(len(body) == v['size_bytes'] and hashlib.sha256(body).hexdigest() == v['sha256'], v['name'] + ' checksum')
|
||||
img = Image.open(io.BytesIO(body))
|
||||
check(img.format == EXPECTED[v['name']][1] and img.size == (v['width'], v['height']), v['name'] + ' real size/format')
|
||||
check('exif' not in img.info, v['name'] + ' keeps EXIF')
|
||||
img.load()
|
||||
out[v['name']] = img.convert('RGB')
|
||||
return att, out
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Thumbnail, crop and optimized variant rules')
|
||||
c = Client(args.base_url, args.ca_file, timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
|
||||
att, v = process(c, alice, bob, chat, SAMPLE)
|
||||
check(v['thumbnail'].size == (256, 160) and v['crop'].size == (128, 128) and v['optimized'].size == (320, 200), 'fixture sizes')
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=bogus', 400, actor=bob)
|
||||
c.call('GET', '/api/v1/attachments/' + att['id'] + '/content?variant=thumbnail', 404, actor=eve)
|
||||
|
||||
_, v = process(c, alice, bob, chat, encode(halves(2000, 1000), 'PNG'))
|
||||
check(v['thumbnail'].size == (256, 128) and v['optimized'].size == (1280, 640) and v['crop'].size == (128, 128), 'large image sizes')
|
||||
|
||||
_, v = process(c, alice, bob, chat, encode(halves(100, 50), 'PNG'))
|
||||
check(v['thumbnail'].size == (100, 50) and v['optimized'].size == (100, 50), 'small image must not be upscaled')
|
||||
check(v['crop'].size == (128, 128), 'crop must be 128x128')
|
||||
|
||||
_, v = process(c, alice, bob, chat, encode(Image.new('RGBA', (300, 300), (0, 0, 0, 0)), 'PNG'))
|
||||
for name, img in v.items():
|
||||
check(min(img.getpixel((img.width // 2, img.height // 2))) >= 250, name + ': alpha not composited on white')
|
||||
|
||||
exif = Image.Exif()
|
||||
exif[0x0112] = 6
|
||||
_, v = process(c, alice, bob, chat, encode(halves(300, 200), 'JPEG', quality=95, exif=exif.tobytes()), 'image/jpeg')
|
||||
t = v['thumbnail']
|
||||
check(t.size == (170, 256) and v['optimized'].size == (200, 300), 'EXIF orientation not applied to dimensions')
|
||||
top, bottom = t.getpixel((85, 20)), t.getpixel((85, 236))
|
||||
check(top[0] > 200 and top[2] < 60 and bottom[2] > 200 and bottom[0] < 60, 'EXIF rotation direction wrong')
|
||||
print('PASS: thumbnail/crop/optimized sizes, formats, checksums, no upscale, alpha on white, EXIF orientation, no EXIF in variants')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Executable
+84
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from client import Client, arguments, check, run
|
||||
from image_gen import png
|
||||
from stack import SAMPLE, compose, job_row, psql, restore_worker, upload, wait_status
|
||||
|
||||
LIMIT_MIB = 768
|
||||
|
||||
|
||||
def sample(stop, state):
|
||||
while not stop.is_set():
|
||||
out = compose('exec', '-T', 'worker', 'grep', 'VmRSS', '/proc/1/status', check_rc=False).stdout.split()
|
||||
if len(out) > 1:
|
||||
state['rss'] = max(state['rss'], int(out[1]) / 1024)
|
||||
n = psql("select count(*) from jobs where status = 'processing'")
|
||||
state['processing'] = max(state['processing'], int(n or 0))
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
def main():
|
||||
def configure(p):
|
||||
p.add_argument('--huge', type=int, default=6)
|
||||
p.add_argument('--small', type=int, default=20)
|
||||
p.add_argument('--output')
|
||||
args = arguments('Worker concurrency, memory and time limits', configure)
|
||||
c = Client(args.base_url, args.ca_file, timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
concurrency = int(compose('exec', '-T', 'worker', 'printenv', 'WORKER_CONCURRENCY').stdout.strip())
|
||||
huge = png(6320, 6320)
|
||||
|
||||
state, stop = {'rss': 0, 'processing': 0}, threading.Event()
|
||||
t = threading.Thread(target=sample, args=(stop, state))
|
||||
t.start()
|
||||
started = time.monotonic()
|
||||
ids = [upload(c, alice, chat, huge)['json']['id'] for _ in range(args.huge)]
|
||||
ids += [upload(c, alice, chat, SAMPLE)['json']['id'] for _ in range(args.small)]
|
||||
accepted = time.monotonic() - started
|
||||
results = [wait_status(c, bob, i, ('ready', 'failed'), 300)['status'] for i in ids]
|
||||
elapsed = time.monotonic() - started
|
||||
stop.set()
|
||||
t.join()
|
||||
container = compose('ps', '-q', 'worker').stdout.strip()
|
||||
oom = subprocess.run(['docker', 'inspect', '-f', '{{.RestartCount}} {{.State.OOMKilled}}', container],
|
||||
capture_output=True, text=True).stdout.split()
|
||||
|
||||
compose('stop', 'worker')
|
||||
slow = upload(c, alice, chat, huge)['json']
|
||||
compose('up', '-d', '--force-recreate', '--no-deps', 'worker', env={'JOB_TIMEOUT_SECONDS': '1'})
|
||||
timed = wait_status(c, bob, slow['id'], ('ready', 'failed'), 120)
|
||||
_, attempts, last_error = job_row(slow['id'])
|
||||
restore_worker()
|
||||
|
||||
report = {
|
||||
'huge_images': {'count': args.huge, 'pixels': 6320 * 6320, 'bytes': len(huge)},
|
||||
'small_images': args.small,
|
||||
'worker_concurrency': concurrency,
|
||||
'max_processing_jobs_observed': state['processing'],
|
||||
'all_ready': results.count('ready') == len(results),
|
||||
'accept_seconds': round(accepted, 2),
|
||||
'all_ready_seconds': round(elapsed, 1),
|
||||
'worker_peak_rss_mib': round(state['rss'], 1),
|
||||
'worker_memory_limit_mib': LIMIT_MIB,
|
||||
'worker_restarts_oom': oom,
|
||||
'timeout_case': {'job_timeout_seconds': 1, 'status': timed['status'], 'error_code': timed['error_code'],
|
||||
'attempts': attempts, 'last_error': last_error},
|
||||
}
|
||||
text = json.dumps(report, indent=2)
|
||||
if args.output:
|
||||
Path(args.output).write_text(text + '\n')
|
||||
print(text)
|
||||
check(report['all_ready'], 'some jobs failed under load')
|
||||
check(state['processing'] <= concurrency, 'more jobs processing than WORKER_CONCURRENCY')
|
||||
check(state['rss'] < LIMIT_MIB and oom == ['0', 'false'], 'worker reached memory limit')
|
||||
check(timed['status'] == 'failed' and timed['error_code'] == 'retry_exhausted' and 'deadline' in last_error,
|
||||
'timeout must be a bounded transient failure')
|
||||
print('PASS: bounded worker concurrency, memory below limit, job timeout enforced')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Reference in New Issue
Block a user