{ "hypothesis": "With fixed worker CPU, more simultaneous image processes need not improve throughput and can increase RSS; bounded admission protects the queue when the worker stops.", "profile": { "settings": [ [ 1, 1 ], [ 2, 2 ], [ 4, 4 ] ], "repeats": 3, "batch": 8, "upload_clients": 4, "worker_cpu": 1, "worker_memory_limit_mib": 1024, "warmup_images": 1, "image_sizes_bytes": { "small": 111817, "large": 3319965 } }, "runs": [ { "repeat": 1, "image": "small", "concurrency": 1, "prefetch": 1, "successful_jobs": 8, "errors": 0, "seconds": 1.7385662909946404, "throughput": 4.601492644507199, "p95_ready_seconds": 1.355825, "p95_queue_lag_seconds": 1.221817, "peak_rss_bytes": 152162304, "peak_cgroup_memory_bytes": 53309440, "worker_cpu_seconds": 1.329851, "requests": [ { "upload_seconds": 0.04113916699861875, "status": 202 }, { "upload_seconds": 0.10876779200043529, "status": 202 }, { "upload_seconds": 0.12739412499649916, "status": 202 }, { "upload_seconds": 0.1493414579963428, "status": 202 }, { "upload_seconds": 0.11392083399550756, "status": 202 }, { "upload_seconds": 0.05546583300019847, "status": 202 }, { "upload_seconds": 0.04235250000056112, "status": 202 }, { "upload_seconds": 0.02778862499690149, "status": 202 } ], "jobs": [ { "id": "d2b88786-66d5-44f9-8398-afd799333b0d", "ready_seconds": 0.27748, "queue_lag_seconds": 0.115927, "attempts": 1 }, { "id": "746abbed-2f2a-44a7-8054-8e2fb4e40d60", "ready_seconds": 0.405159, "queue_lag_seconds": 0.269961, "attempts": 1 }, { "id": "a4fbcad3-77a5-402f-8ce9-a286c3c50998", "ready_seconds": 0.52426, "queue_lag_seconds": 0.395915, "attempts": 1 }, { "id": "bbc83e93-f179-4f84-a5a1-4bfb334ecf5d", "ready_seconds": 0.629692, "queue_lag_seconds": 0.459954, "attempts": 1 }, { "id": "8d5f7573-9a93-4f75-9871-fe57beaafa15", "ready_seconds": 0.909593, "queue_lag_seconds": 0.620331, "attempts": 1 }, { "id": "8280ed8d-da55-40a0-bde3-36ee21893e72", "ready_seconds": 1.068921, "queue_lag_seconds": 0.902328, "attempts": 1 }, { "id": "d8dbae97-6243-4f95-b9cc-4a3e8fdfefbb", "ready_seconds": 1.225459, "queue_lag_seconds": 1.068507, "attempts": 1 }, { "id": "dcd72436-5215-4aca-900d-74767f0385b2", "ready_seconds": 1.355825, "queue_lag_seconds": 1.221817, "attempts": 1 } ], "samples": [ { "time": 1794262118.743776, "memory_bytes": 42041344, "cpu_usec": 795694, "rss_bytes": 103636992 }, { "time": 1794262119.0674748, "memory_bytes": 48332800, "cpu_usec": 1134009, "rss_bytes": 127852544 }, { "time": 1794262119.545117, "memory_bytes": 50176000, "cpu_usec": 1544976, "rss_bytes": 152162304 }, { "time": 1794262119.89533, "memory_bytes": 53309440, "cpu_usec": 1889921, "rss_bytes": 147939328 }, { "time": 1794262120.2800698, "memory_bytes": 46026752, "cpu_usec": 2125545, "rss_bytes": 107089920 } ] }, { "repeat": 1, "image": "small", "concurrency": 2, "prefetch": 2, "successful_jobs": 8, "errors": 0, "seconds": 1.969336124995607, "throughput": 4.062282663919469, "p95_ready_seconds": 1.525365, "p95_queue_lag_seconds": 1.016688, "peak_rss_bytes": 219119616, "peak_cgroup_memory_bytes": 62783488, "worker_cpu_seconds": 1.534243, "requests": [ { "upload_seconds": 0.03250100000150269, "status": 202 }, { "upload_seconds": 0.044958750004298054, "status": 202 }, { "upload_seconds": 0.05604845799825853, "status": 202 }, { "upload_seconds": 0.06452524999622256, "status": 202 }, { "upload_seconds": 0.0502204169970355, "status": 202 }, { "upload_seconds": 0.04115625000122236, "status": 202 }, { "upload_seconds": 0.03952566699445015, "status": 202 }, { "upload_seconds": 0.0422521250002319, "status": 202 } ], "jobs": [ { "id": "9353ef12-3388-4f17-a52e-7788d9631d5e", "ready_seconds": 0.394761, "queue_lag_seconds": 0.116528, "attempts": 1 }, { "id": "5ff9fdf3-e9e6-446e-95c7-978c9b3fd22b", "ready_seconds": 0.404182, "queue_lag_seconds": 0.129876, "attempts": 1 }, { "id": "314ef166-fd5e-4c85-abf7-15185aea7576", "ready_seconds": 0.692893, "queue_lag_seconds": 0.388414, "attempts": 1 }, { "id": "52a91f37-b649-4896-8051-5f39802fb3ad", "ready_seconds": 0.751849, "queue_lag_seconds": 0.382157, "attempts": 1 }, { "id": "052cd532-0568-47b6-a95c-e35bc68c6d81", "ready_seconds": 0.976998, "queue_lag_seconds": 0.669662, "attempts": 1 }, { "id": "5df94c7d-97a6-4ca1-a3bb-c663ea5a7e41", "ready_seconds": 1.032436, "queue_lag_seconds": 0.738607, "attempts": 1 }, { "id": "926d6d55-a9b6-408f-a7e2-2bf2b98871b0", "ready_seconds": 1.523118, "queue_lag_seconds": 1.005413, "attempts": 1 }, { "id": "dad6cf86-7200-4468-b927-aca54ae14db9", "ready_seconds": 1.525365, "queue_lag_seconds": 1.016688, "attempts": 1 } ], "samples": [ { "time": 1794262122.505246, "memory_bytes": 42008576, "cpu_usec": 782744, "rss_bytes": 103116800 }, { "time": 1794262122.863539, "memory_bytes": 54919168, "cpu_usec": 1204770, "rss_bytes": 161333248 }, { "time": 1794262123.2467399, "memory_bytes": 57008128, "cpu_usec": 1566113, "rss_bytes": 175915008 }, { "time": 1794262123.8038552, "memory_bytes": 62783488, "cpu_usec": 2036485, "rss_bytes": 219119616 }, { "time": 1794262124.115168, "memory_bytes": 45998080, "cpu_usec": 2316987, "rss_bytes": 106729472 } ] }, { "repeat": 1, "image": "small", "concurrency": 4, "prefetch": 4, "successful_jobs": 8, "errors": 0, "seconds": 9.549317416000122, "throughput": 0.8377562135065066, "p95_ready_seconds": 9.209337, "p95_queue_lag_seconds": 7.286448, "peak_rss_bytes": 289599488, "peak_cgroup_memory_bytes": 87416832, "worker_cpu_seconds": 5.107425, "requests": [ { "upload_seconds": 0.03872108399809804, "status": 202 }, { "upload_seconds": 0.06579587499436457, "status": 202 }, { "upload_seconds": 0.08007283299957635, "status": 202 }, { "upload_seconds": 0.09368379199440824, "status": 202 }, { "upload_seconds": 0.040345542001887225, "status": 202 }, { "upload_seconds": 0.07635662500251783, "status": 202 }, { "upload_seconds": 0.032701916999940295, "status": 202 }, { "upload_seconds": 0.05977749999874504, "status": 202 } ], "jobs": [ { "id": "c70a41eb-b3da-40f7-9669-fbc0adf914d1", "ready_seconds": 9.074966, "queue_lag_seconds": 7.278132, "attempts": 1 }, { "id": "e799ce82-e00a-4e15-aacc-daa6fa757335", "ready_seconds": 9.150763, "queue_lag_seconds": 7.224741, "attempts": 1 }, { "id": "d248b714-1eeb-4668-b497-4a30b3b0aad8", "ready_seconds": 9.16808, "queue_lag_seconds": 7.267137, "attempts": 1 }, { "id": "a540b150-be0e-4d38-9077-92802293c12d", "ready_seconds": 9.209337, "queue_lag_seconds": 7.286448, "attempts": 1 }, { "id": "d62a309a-a54e-4a7d-aebc-864fd251bed4", "ready_seconds": 7.164243, "queue_lag_seconds": 0.116455, "attempts": 1 }, { "id": "063093ae-38a4-4fc4-a841-d3e5db5624b3", "ready_seconds": 7.164926, "queue_lag_seconds": 0.135483, "attempts": 1 }, { "id": "06613954-2472-4efe-b2a7-42644a1d8a30", "ready_seconds": 7.149471, "queue_lag_seconds": 0.127605, "attempts": 1 }, { "id": "f593fbcd-f166-4e3e-9d89-09faeaa592bb", "ready_seconds": 7.127115, "queue_lag_seconds": 0.132045, "attempts": 1 } ], "samples": [ { "time": 1794262126.910836, "memory_bytes": 44044288, "cpu_usec": 1063223, "rss_bytes": 143413248 }, { "time": 1794262127.9235861, "memory_bytes": 87416832, "cpu_usec": 1873587, "rss_bytes": 284831744 }, { "time": 1794262130.018199, "memory_bytes": 42848256, "cpu_usec": 3399041, "rss_bytes": 106557440 }, { "time": 1794262130.52949, "memory_bytes": 43167744, "cpu_usec": 3541243, "rss_bytes": 106573824 }, { "time": 1794262131.362358, "memory_bytes": 42827776, "cpu_usec": 3715963, "rss_bytes": 106573824 }, { "time": 1794262133.449707, "memory_bytes": 44789760, "cpu_usec": 4203236, "rss_bytes": 108687360 }, { "time": 1794262133.8703759, "memory_bytes": 47022080, "cpu_usec": 4272996, "rss_bytes": 108298240 }, { "time": 1794262134.622085, "memory_bytes": 77737984, "cpu_usec": 5000530, "rss_bytes": 289599488 }, { "time": 1794262135.440066, "memory_bytes": 55914496, "cpu_usec": 5808140, "rss_bytes": 108367872 }, { "time": 1794262135.956406, "memory_bytes": 44863488, "cpu_usec": 6170648, "rss_bytes": 109416448 } ] }, { "repeat": 1, "image": "large", "concurrency": 1, "prefetch": 1, "successful_jobs": 8, "errors": 0, "seconds": 9.295377750000625, "throughput": 0.8606428071198572, "p95_ready_seconds": 7.433391, "p95_queue_lag_seconds": 6.388493, "peak_rss_bytes": 193171456, "peak_cgroup_memory_bytes": 102318080, "worker_cpu_seconds": 7.074789, "requests": [ { "upload_seconds": 0.7639207920001354, "status": 202 }, { "upload_seconds": 0.9345893749996321, "status": 202 }, { "upload_seconds": 1.1469821249993402, "status": 202 }, { "upload_seconds": 1.436994916999538, "status": 202 }, { "upload_seconds": 0.7003262910002377, "status": 202 }, { "upload_seconds": 0.9740857910001068, "status": 202 }, { "upload_seconds": 0.39281854200089583, "status": 202 }, { "upload_seconds": 0.8782962080003927, "status": 202 } ], "jobs": [ { "id": "2be2644d-861c-43c4-8085-5fbba90a4d64", "ready_seconds": 1.712623, "queue_lag_seconds": 0.363585, "attempts": 1 }, { "id": "ad0083b2-3bf8-4f42-8043-4a57ad7b84a9", "ready_seconds": 2.629908, "queue_lag_seconds": 1.699148, "attempts": 1 }, { "id": "407c2a2e-e5aa-4437-ab5b-49a2ec05829b", "ready_seconds": 3.021709, "queue_lag_seconds": 2.407417, "attempts": 1 }, { "id": "854309f9-5d4f-4305-ad65-bcb005b3844f", "ready_seconds": 3.852419, "queue_lag_seconds": 2.844126, "attempts": 1 }, { "id": "fae78450-5058-4101-8cd9-4dcbcecd9e2c", "ready_seconds": 4.70857, "queue_lag_seconds": 3.753701, "attempts": 1 }, { "id": "fe283bf1-cdff-4ee2-867d-369338758dd4", "ready_seconds": 5.265006, "queue_lag_seconds": 4.502767, "attempts": 1 }, { "id": "86228db3-70a0-4118-bd9c-1f0aeb13ccf4", "ready_seconds": 6.477127, "queue_lag_seconds": 5.104128, "attempts": 1 }, { "id": "3fea2d31-ad7b-4282-b815-23adaa068c7a", "ready_seconds": 7.433391, "queue_lag_seconds": 6.388493, "attempts": 1 } ], "samples": [ { "time": 1794262140.0249028, "memory_bytes": 41619456, "cpu_usec": 1623573, "rss_bytes": 106139648 }, { "time": 1794262140.4498188, "memory_bytes": 41545728, "cpu_usec": 1659252, "rss_bytes": 106139648 }, { "time": 1794262140.9149399, "memory_bytes": 60805120, "cpu_usec": 2022910, "rss_bytes": 150097920 }, { "time": 1794262141.32444, "memory_bytes": 88137728, "cpu_usec": 2454146, "rss_bytes": 191004672 }, { "time": 1794262141.7649221, "memory_bytes": 54976512, "cpu_usec": 2788341, "rss_bytes": 106950656 }, { "time": 1794262142.153526, "memory_bytes": 82583552, "cpu_usec": 3145199, "rss_bytes": 180039680 }, { "time": 1794262142.4781618, "memory_bytes": 98701312, "cpu_usec": 3485405, "rss_bytes": 192589824 }, { "time": 1794262142.818176, "memory_bytes": 54738944, "cpu_usec": 3670691, "rss_bytes": 106897408 }, { "time": 1794262143.150275, "memory_bytes": 92516352, "cpu_usec": 4041841, "rss_bytes": 187322368 }, { "time": 1794262143.4721, "memory_bytes": 58019840, "cpu_usec": 4330075, "rss_bytes": 106909696 }, { "time": 1794262143.816778, "memory_bytes": 91193344, "cpu_usec": 4691490, "rss_bytes": 185057280 }, { "time": 1794262144.319894, "memory_bytes": 54292480, "cpu_usec": 4990675, "rss_bytes": 106909696 }, { "time": 1794262144.7009401, "memory_bytes": 66936832, "cpu_usec": 5267047, "rss_bytes": 147943424 }, { "time": 1794262145.049203, "memory_bytes": 98369536, "cpu_usec": 5612259, "rss_bytes": 192471040 }, { "time": 1794262145.4414349, "memory_bytes": 63508480, "cpu_usec": 5901715, "rss_bytes": 109088768 }, { "time": 1794262145.81682, "memory_bytes": 92336128, "cpu_usec": 6171315, "rss_bytes": 182304768 }, { "time": 1794262146.1910982, "memory_bytes": 75481088, "cpu_usec": 6539191, "rss_bytes": 109092864 }, { "time": 1794262146.612237, "memory_bytes": 73048064, "cpu_usec": 6912398, "rss_bytes": 154517504 }, { "time": 1794262147.0016918, "memory_bytes": 96288768, "cpu_usec": 7288823, "rss_bytes": 190681088 }, { "time": 1794262147.491589, "memory_bytes": 92078080, "cpu_usec": 7759923, "rss_bytes": 179814400 }, { "time": 1794262147.9121048, "memory_bytes": 64536576, "cpu_usec": 8079635, "rss_bytes": 128561152 }, { "time": 1794262148.251782, "memory_bytes": 102318080, "cpu_usec": 8385623, "rss_bytes": 193171456 }, { "time": 1794262148.6732469, "memory_bytes": 67047424, "cpu_usec": 8698362, "rss_bytes": 115306496 } ] } ] }