{ "configurations": [ { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 7.935087719298245, "avg_output_len": 615.0537109375, "avg_power_watts": 759.9467920298994, "data_parallel": 1, "energy_per_request_joules": 430.1127103864672, "energy_per_token_joules": 0.6993091867226764, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 7.15609549661167, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 1086.7107231801174, "p90_itl_ms": 8.386991303996183, "p95_itl_ms": 9.683607009355912, "p99_itl_ms": 17.90369909140282, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 15.948170731707316, "avg_output_len": 626.7978515625, "avg_power_watts": 761.6265567745648, "data_parallel": 1, "energy_per_request_joules": 249.56838433485814, "energy_per_token_joules": 0.3981640710999994, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 7.8423559898510575, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 1912.846015137517, "p90_itl_ms": 11.547983792843299, "p95_itl_ms": 14.749675209168343, "p99_itl_ms": 21.02427156176417, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 31.91346153846154, "avg_output_len": 604.6357421875, "avg_power_watts": 699.7358859610536, "data_parallel": 1, "energy_per_request_joules": 147.60976251438845, "energy_per_token_joules": 0.24413006412812105, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 9.416910004802048, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 2866.24217488358, "p90_itl_ms": 18.61438018968329, "p95_itl_ms": 23.177931003738202, "p99_itl_ms": 34.41342919832098, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 63.75912408759124, "avg_output_len": 609.0732421875, "avg_power_watts": 696.9625164938583, "data_parallel": 1, "energy_per_request_joules": 101.64963425431539, "energy_per_token_joules": 0.16689229999538066, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 12.726755507173948, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 4176.121465838442, "p90_itl_ms": 25.543338389252316, "p95_itl_ms": 30.718190365587354, "p99_itl_ms": 44.96450745529728, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 127.70212765957447, "avg_output_len": 605.9296875, "avg_power_watts": 708.7273838663926, "data_parallel": 1, "energy_per_request_joules": 74.2123020033151, "energy_per_token_joules": 0.12247675519829204, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 19.06669449817855, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 5786.627696977687, "p90_itl_ms": 34.60722669551615, "p95_itl_ms": 40.30590570473578, "p99_itl_ms": 54.33177557948516, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 255.5681818181818, "avg_output_len": 623.7705078125, "avg_power_watts": 955.0710709996039, "data_parallel": 1, "energy_per_request_joules": 55.631687252929815, "energy_per_token_joules": 0.08918614547523977, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 21.165275014936924, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 10708.738065877671, "p90_itl_ms": 32.557443191763014, "p95_itl_ms": 37.41190079599618, "p99_itl_ms": 53.248244108399376, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 511.2625, "avg_output_len": 620.720703125, "avg_power_watts": 913.3277617016568, "data_parallel": 1, "energy_per_request_joules": 47.80931954311802, "energy_per_token_joules": 0.07702227314543146, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 512, "median_itl_ms": 39.11911048635375, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 11857.969447060268, "p90_itl_ms": 56.06306250265334, "p95_itl_ms": 59.9696527569904, "p99_itl_ms": 71.64305746671742, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 1022.0232558139535, "avg_output_len": 613.32763671875, "avg_power_watts": 932.265569899849, "data_parallel": 1, "energy_per_request_joules": 37.99201956234111, "energy_per_token_joules": 0.06194408549009, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 1024, "median_itl_ms": 61.50677800178528, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 15050.114349480478, "p90_itl_ms": 78.1277182046324, "p95_itl_ms": 85.33365190087349, "p99_itl_ms": 150.3133866604185, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 2044.5882352941176, "avg_output_len": 613.1474609375, "avg_power_watts": 869.4923250228219, "data_parallel": 1, "energy_per_request_joules": 33.58277147726921, "energy_per_token_joules": 0.05477111725443875, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 2048, "median_itl_ms": 112.645222019637, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 15875.015311146617, "p90_itl_ms": 142.40702299866825, "p95_itl_ms": 153.51142458384857, "p99_itl_ms": 215.13447568984702, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 7.986908358509567, "avg_output_len": 619.5361328125, "avg_power_watts": 443.0121237999165, "data_parallel": 1, "energy_per_request_joules": 437.4502427837893, "energy_per_token_joules": 0.706093187491587, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 12.535519897937775, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 627.4131115380502, "p90_itl_ms": 13.173650950193405, "p95_itl_ms": 13.427985832095146, "p99_itl_ms": 14.364334940910336, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 15.97373358348968, "avg_output_len": 621.626953125, "avg_power_watts": 465.2511720330447, "data_parallel": 1, "energy_per_request_joules": 247.5777242734419, "energy_per_token_joules": 0.3982737927125526, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 13.344110921025276, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 1168.169185485001, "p90_itl_ms": 14.119432680308819, "p95_itl_ms": 14.563228283077477, "p99_itl_ms": 17.94641043990854, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 31.923333333333332, "avg_output_len": 611.408203125, "avg_power_watts": 493.87015235470756, "data_parallel": 1, "energy_per_request_joules": 149.24495943199702, "energy_per_token_joules": 0.24410035499880997, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 15.01433365046978, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 2023.225866906729, "p90_itl_ms": 17.230525985360146, "p95_itl_ms": 17.838258296251297, "p99_itl_ms": 28.449818044900816, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 63.844919786096256, "avg_output_len": 617.2373046875, "avg_power_watts": 524.979298008556, "data_parallel": 1, "energy_per_request_joules": 103.18576467353151, "energy_per_token_joules": 0.16717357147713754, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 18.936093896627426, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 3140.3247138280562, "p90_itl_ms": 21.735174953937534, "p95_itl_ms": 23.127018660306913, "p99_itl_ms": 44.97944485396147, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 95.88235294117646, "avg_output_len": 622.826171875, "avg_power_watts": 541.7770177610251, "data_parallel": 1, "energy_per_request_joules": 89.39130796863861, "energy_per_token_joules": 0.14352529165485883, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 23.49620033055544, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 3774.7843011798827, "p90_itl_ms": 26.171179115772247, "p95_itl_ms": 29.94294641539451, "p99_itl_ms": 60.39297431707382, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 127.71311475409836, "avg_output_len": 613.51953125, "avg_power_watts": 572.0829869523203, "data_parallel": 1, "energy_per_request_joules": 78.44358044476974, "energy_per_token_joules": 0.12785832634365663, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 26.163955219089985, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 4474.350660704568, "p90_itl_ms": 29.976078681647778, "p95_itl_ms": 36.86737883836031, "p99_itl_ms": 72.38292438909403, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 191.65625, "avg_output_len": 612.6025390625, "avg_power_watts": 612.8136444798971, "data_parallel": 1, "energy_per_request_joules": 70.7114051695591, "energy_per_token_joules": 0.11542786825169338, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 33.36937911808491, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 5309.061440376266, "p90_itl_ms": 39.20198418200016, "p95_itl_ms": 47.289641574025154, "p99_itl_ms": 90.8597562462091, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 14.0, "architecture": "Dense Transformer", "avg_batch_size": 253.40963855421685, "avg_output_len": 624.564453125, "avg_power_watts": 644.6719210104968, "data_parallel": 1, "energy_per_request_joules": 69.89933984987961, "energy_per_token_joules": 0.11191693587449492, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 256, "median_itl_ms": 38.40287588536739, "model_id": "Qwen/Qwen3-14B", "nickname": "Qwen 3 14B", "num_gpus": 1, "output_throughput_tokens_per_sec": 5760.271365304712, "p90_itl_ms": 49.61365107446908, "p95_itl_ms": 67.32841385528444, "p99_itl_ms": 137.34881250187755, "tensor_parallel": 1, "total_params_billions": 14.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 7.990445859872612, "avg_output_len": 864.9384765625, "avg_power_watts": 2562.1769211252954, "data_parallel": 1, "energy_per_request_joules": 5584.70889877075, "energy_per_token_joules": 6.456770105737344, "expert_parallel": 4, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 19.666570995468646, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 396.8202180295379, "p90_itl_ms": 20.312772403121926, "p95_itl_ms": 20.50538700132165, "p99_itl_ms": 21.42864028399345, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 15.983870967741936, "avg_output_len": 869.3984375, "avg_power_watts": 2896.7329268203844, "data_parallel": 1, "energy_per_request_joules": 3956.1002725542494, "energy_per_token_joules": 4.550388063648032, "expert_parallel": 4, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 24.541911014239304, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 636.5903053327902, "p90_itl_ms": 25.69366459356388, "p95_itl_ms": 26.0902940921369, "p99_itl_ms": 56.299414734530714, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 31.96969696969697, "avg_output_len": 871.380859375, "avg_power_watts": 3185.865439767311, "data_parallel": 1, "energy_per_request_joules": 2660.1117799463236, "energy_per_token_joules": 3.05275443145985, "expert_parallel": 4, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 29.83295900048688, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 1043.6035754909399, "p90_itl_ms": 30.65786920778919, "p95_itl_ms": 31.018828594824296, "p99_itl_ms": 53.95943550916854, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 63.91256830601093, "avg_output_len": 876.064453125, "avg_power_watts": 3122.5641096018644, "data_parallel": 1, "energy_per_request_joules": 1789.8288810233641, "energy_per_token_joules": 2.043033334635237, "expert_parallel": 4, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 40.821517002768815, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 1528.3960651378047, "p90_itl_ms": 42.09522349992767, "p95_itl_ms": 43.59326000849251, "p99_itl_ms": 67.2282963452744, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 127.87, "avg_output_len": 867.787109375, "avg_power_watts": 3142.9811977702625, "data_parallel": 1, "energy_per_request_joules": 1047.6840546893295, "energy_per_token_joules": 1.2073053901940252, "expert_parallel": 4, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 47.08670650143176, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 2603.3025473903963, "p90_itl_ms": 50.35464620159473, "p95_itl_ms": 53.13555959728546, "p99_itl_ms": 85.30445821001186, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 255.7062937062937, "avg_output_len": 878.83203125, "avg_power_watts": 3203.8406083650552, "data_parallel": 1, "energy_per_request_joules": 624.8232723189357, "energy_per_token_joules": 0.710970071755604, "expert_parallel": 4, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 53.255411505233496, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 4506.294618638146, "p90_itl_ms": 58.379866098403, "p95_itl_ms": 64.27994945697716, "p99_itl_ms": 105.64801411645021, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 511.3775510204082, "avg_output_len": 879.8984375, "avg_power_watts": 3012.2719003895086, "data_parallel": 1, "energy_per_request_joules": 407.37904725739253, "energy_per_token_joules": 0.4629841694171579, "expert_parallel": 4, "gpu_model": "B200", "max_num_seqs": 512, "median_itl_ms": 67.11570649349596, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 6506.209281802445, "p90_itl_ms": 106.40274759498425, "p95_itl_ms": 114.1425818655989, "p99_itl_ms": 130.3036870845244, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 1022.5977011494252, "avg_output_len": 868.17919921875, "avg_power_watts": 3002.1073530617564, "data_parallel": 1, "energy_per_request_joules": 298.38028333669223, "energy_per_token_joules": 0.34368513275277296, "expert_parallel": 4, "gpu_model": "B200", "max_num_seqs": 1024, "median_itl_ms": 103.86629600543529, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 8735.051554357742, "p90_itl_ms": 128.0457684013527, "p95_itl_ms": 135.17154659202788, "p99_itl_ms": 182.001227917498, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 1411.1935483870968, "avg_output_len": 880.4085286458334, "avg_power_watts": 3006.796685734427, "data_parallel": 1, "energy_per_request_joules": 278.39042896426184, "energy_per_token_joules": 0.3162059656469451, "expert_parallel": 4, "gpu_model": "B200", "max_num_seqs": 1536, "median_itl_ms": 135.5150669987779, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 9508.981525957734, "p90_itl_ms": 169.60162940667945, "p95_itl_ms": 188.24133549642283, "p99_itl_ms": 286.6105389723092, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 2044.8076923076924, "avg_output_len": 866.607421875, "avg_power_watts": 3268.479266859503, "data_parallel": 1, "energy_per_request_joules": 231.9495459068565, "energy_per_token_joules": 0.2676523879809479, "expert_parallel": 4, "gpu_model": "B200", "max_num_seqs": 2048, "median_itl_ms": 143.62831650942098, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 12211.657409505946, "p90_itl_ms": 177.43998000514694, "p95_itl_ms": 194.96807576797437, "p99_itl_ms": 353.6779921996638, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 3046.5263157894738, "avg_output_len": 868.5126953125, "avg_power_watts": 3231.4177508472008, "data_parallel": 1, "energy_per_request_joules": 213.28545466493532, "energy_per_token_joules": 0.24557551756706672, "expert_parallel": 4, "gpu_model": "B200", "max_num_seqs": 3072, "median_itl_ms": 118.4986879961798, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 13158.550098401807, "p90_itl_ms": 222.9848100047093, "p95_itl_ms": 238.56627569039105, "p99_itl_ms": 360.99415537231835, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 3764.6470588235293, "avg_output_len": 873.809375, "avg_power_watts": 3196.0657904719633, "data_parallel": 1, "energy_per_request_joules": 209.92863171593132, "energy_per_token_joules": 0.24024534151505447, "expert_parallel": 4, "gpu_model": "B200", "max_num_seqs": 4096, "median_itl_ms": 67.7747610170627, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 13303.341368938422, "p90_itl_ms": 215.12070119788407, "p95_itl_ms": 282.5096203494467, "p99_itl_ms": 393.15440698177474, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 7.989968321013728, "avg_output_len": 882.6181640625, "avg_power_watts": 3941.7836755253757, "data_parallel": 1, "energy_per_request_joules": 7410.486528426251, "energy_per_token_joules": 8.396027671034311, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 16.562483004236128, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 469.48197766477654, "p90_itl_ms": 17.01701099955244, "p95_itl_ms": 17.150035199301783, "p99_itl_ms": 17.91131647914881, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 15.98239110287303, "avg_output_len": 888.0126953125, "avg_power_watts": 4603.751957820395, "data_parallel": 1, "energy_per_request_joules": 4956.073287800221, "energy_per_token_joules": 5.5810838222939285, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 18.702683999435976, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 824.884933537545, "p90_itl_ms": 19.14035200024955, "p95_itl_ms": 19.311960000777617, "p99_itl_ms": 58.10142100381199, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 31.9735516372796, "avg_output_len": 867.7587890625, "avg_power_watts": 4530.013731280813, "data_parallel": 1, "energy_per_request_joules": 3645.1335604803685, "energy_per_token_joules": 4.200629952038238, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 28.806008995161392, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 1078.4129482966598, "p90_itl_ms": 29.66216999629978, "p95_itl_ms": 30.026355001609772, "p99_itl_ms": 62.37182980694343, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 63.93736017897092, "avg_output_len": 868.8349609375, "avg_power_watts": 4770.273283734266, "data_parallel": 1, "energy_per_request_joules": 2208.2986392148973, "energy_per_token_joules": 2.5416779233101696, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 32.98562100098934, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 1876.8205208005554, "p90_itl_ms": 33.926146395970136, "p95_itl_ms": 35.452592905494384, "p99_itl_ms": 57.160001784504864, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 127.82773109243698, "avg_output_len": 880.1298828125, "avg_power_watts": 4999.513810745349, "data_parallel": 1, "energy_per_request_joules": 1331.0017335729428, "energy_per_token_joules": 1.5122787665380237, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 36.91444099240471, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 3305.9472376184053, "p90_itl_ms": 39.362635600264184, "p95_itl_ms": 41.41223899787292, "p99_itl_ms": 82.56698048324324, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 255.66101694915255, "avg_output_len": 872.8271484375, "avg_power_watts": 5077.562470171535, "data_parallel": 1, "energy_per_request_joules": 820.2126944940134, "energy_per_token_joules": 0.9397195034117868, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 43.46898099174723, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 5403.2745428148655, "p90_itl_ms": 48.61373399035074, "p95_itl_ms": 60.02770250051981, "p99_itl_ms": 96.95789399847854, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 511.37142857142857, "avg_output_len": 864.42431640625, "avg_power_watts": 4867.028033186916, "data_parallel": 1, "energy_per_request_joules": 582.6883909444114, "energy_per_token_joules": 0.6740768160790235, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 512, "median_itl_ms": 58.895011999993585, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 7220.286942217494, "p90_itl_ms": 98.12347019906156, "p95_itl_ms": 106.29511259903666, "p99_itl_ms": 120.94796748075176, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 1022.6304347826087, "avg_output_len": 861.154296875, "avg_power_watts": 4508.793870120932, "data_parallel": 1, "energy_per_request_joules": 466.90285722453757, "energy_per_token_joules": 0.542182578567927, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 1024, "median_itl_ms": 112.50159650080604, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 8316.006541615669, "p90_itl_ms": 135.53495049563935, "p95_itl_ms": 139.58224974703626, "p99_itl_ms": 170.15029240646984, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 1534.0267857142858, "avg_output_len": 874.529296875, "avg_power_watts": 4779.828628691161, "data_parallel": 1, "energy_per_request_joules": 409.482366537498, "energy_per_token_joules": 0.46823173105889326, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 1536, "median_itl_ms": 143.55941200483358, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 10208.254399764211, "p90_itl_ms": 171.39873139676638, "p95_itl_ms": 178.11441884259693, "p99_itl_ms": 231.99295172540582, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 2044.982142857143, "avg_output_len": 873.5260416666666, "avg_power_watts": 4924.190101961018, "data_parallel": 1, "energy_per_request_joules": 366.7888721248703, "energy_per_token_joules": 0.41989460488784736, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 2048, "median_itl_ms": 155.6095345003996, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 11727.20498106008, "p90_itl_ms": 187.33278021099977, "p95_itl_ms": 200.78389109912675, "p99_itl_ms": 286.1253915645647, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 4090.3977272727275, "avg_output_len": 873.5543619791666, "avg_power_watts": 5395.589402297423, "data_parallel": 1, "energy_per_request_joules": 326.0231812181162, "energy_per_token_joules": 0.37321453066694377, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 4096, "median_itl_ms": 240.9158169903094, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 14457.072163442748, "p90_itl_ms": 320.58046300080605, "p95_itl_ms": 367.319661048532, "p99_itl_ms": 789.0634731343014, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 7.990679094540613, "avg_output_len": 864.4228515625, "avg_power_watts": 2828.5444663296767, "data_parallel": 1, "energy_per_request_joules": 6303.880679088534, "energy_per_token_joules": 7.292589116187597, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 20.238544999301666, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 387.8656018136363, "p90_itl_ms": 20.947102600439393, "p95_itl_ms": 21.136431999184424, "p99_itl_ms": 21.77302635951491, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 15.974395448079658, "avg_output_len": 865.3212890625, "avg_power_watts": 3188.597383478894, "data_parallel": 1, "energy_per_request_joules": 4480.900425440654, "energy_per_token_joules": 5.17830831400822, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 25.347841999973753, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 615.7604356722419, "p90_itl_ms": 26.363165998918703, "p95_itl_ms": 26.64854619943071, "p99_itl_ms": 55.88809064021912, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 31.966101694915253, "avg_output_len": 864.8896484375, "avg_power_watts": 3472.184599429809, "data_parallel": 1, "energy_per_request_joules": 3308.61788935041, "energy_per_token_joules": 3.825479811589516, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 34.71215199897415, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 907.6468235201821, "p90_itl_ms": 35.663523800394614, "p95_itl_ms": 35.957775101269355, "p99_itl_ms": 52.50825485960378, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 63.93071161048689, "avg_output_len": 879.2333984375, "avg_power_watts": 3704.701103562126, "data_parallel": 1, "energy_per_request_joules": 2104.846923791714, "energy_per_token_joules": 2.3939569715302804, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 40.34211900034279, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 1547.5220096349449, "p90_itl_ms": 40.97743200145487, "p95_itl_ms": 41.59834450001654, "p99_itl_ms": 72.15075349886321, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 95.9023746701847, "avg_output_len": 871.859375, "avg_power_watts": 3845.520596375327, "data_parallel": 1, "energy_per_request_joules": 1586.01676502642, "energy_per_token_joules": 1.819119929778148, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 43.67961599928094, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 2113.9456137146, "p90_itl_ms": 44.70363039945369, "p95_itl_ms": 46.96414245099729, "p99_itl_ms": 84.55999325944505, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 127.84406779661018, "avg_output_len": 862.9716796875, "avg_power_watts": 3903.4543908845462, "data_parallel": 1, "energy_per_request_joules": 1282.582425670985, "energy_per_token_joules": 1.4862393006169505, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 46.639504998893244, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 2626.396966668954, "p90_itl_ms": 48.37928880042455, "p95_itl_ms": 53.2070759001726, "p99_itl_ms": 89.28590384100973, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 191.8106796116505, "avg_output_len": 880.40625, "avg_power_watts": 3987.424328613464, "data_parallel": 1, "energy_per_request_joules": 1011.1754127417912, "energy_per_token_joules": 1.148532751490339, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 52.36333249922609, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 3471.755005191948, "p90_itl_ms": 54.93876010059467, "p95_itl_ms": 67.49959940143513, "p99_itl_ms": 101.43425735997883, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 228.4659090909091, "avg_output_len": 874.19921875, "avg_power_watts": 4038.402679101194, "data_parallel": 1, "energy_per_request_joules": 955.2463326192346, "energy_per_token_joules": 1.0927101193079563, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 256, "median_itl_ms": 55.47962600030587, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507", "nickname": "Qwen 3 235B A22B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 3695.767621936938, "p90_itl_ms": 61.82613250075519, "p95_itl_ms": 88.42033074961364, "p99_itl_ms": 184.1327575995819, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 7.990046127700898, "avg_output_len": 874.828125, "avg_power_watts": 1805.668383628926, "data_parallel": 1, "energy_per_request_joules": 7367.011627281942, "energy_per_token_joules": 8.421096003608643, "expert_parallel": 2, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 36.719368494232185, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 2, "output_throughput_tokens_per_sec": 214.4220161906661, "p90_itl_ms": 37.812683993251994, "p95_itl_ms": 37.99658724456094, "p99_itl_ms": 38.57398231921252, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 15.979510022271715, "avg_output_len": 871.1796875, "avg_power_watts": 1818.2945061663515, "data_parallel": 1, "energy_per_request_joules": 4068.3596537103085, "energy_per_token_joules": 4.669943195513622, "expert_parallel": 2, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 40.43045798607636, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 2, "output_throughput_tokens_per_sec": 389.361161376261, "p90_itl_ms": 41.137090703705326, "p95_itl_ms": 41.47334825247526, "p99_itl_ms": 82.23042580037145, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 31.96029776674938, "avg_output_len": 882.130859375, "avg_power_watts": 1894.6462274541648, "data_parallel": 1, "energy_per_request_joules": 2317.5088691133965, "energy_per_token_joules": 2.6271712915194674, "expert_parallel": 2, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 43.2236600026954, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 2, "output_throughput_tokens_per_sec": 721.1734665227577, "p90_itl_ms": 44.146505699609406, "p95_itl_ms": 44.5199030888034, "p99_itl_ms": 90.64704591408372, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 63.93798449612403, "avg_output_len": 869.4404296875, "avg_power_watts": 1892.042070500693, "data_parallel": 1, "energy_per_request_joules": 1279.8051087597003, "energy_per_token_joules": 1.4719871138494172, "expert_parallel": 2, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 47.80994201428257, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 2, "output_throughput_tokens_per_sec": 1285.3659197822617, "p90_itl_ms": 49.54287161817774, "p95_itl_ms": 51.0372142191045, "p99_itl_ms": 93.26520285278093, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 127.85227272727273, "avg_output_len": 884.7900390625, "avg_power_watts": 1853.3838545799715, "data_parallel": 1, "energy_per_request_joules": 727.0552650013819, "energy_per_token_joules": 0.8217263225202562, "expert_parallel": 2, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 53.917051991447806, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 2, "output_throughput_tokens_per_sec": 2255.4758242325674, "p90_itl_ms": 57.904711982700974, "p95_itl_ms": 61.82281699148007, "p99_itl_ms": 108.06301402044483, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 255.74937965260546, "avg_output_len": 876.0244140625, "avg_power_watts": 1890.8444703429448, "data_parallel": 1, "energy_per_request_joules": 438.619907732891, "energy_per_token_joules": 0.5006937029286922, "expert_parallel": 2, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 64.47548398864456, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 2, "output_throughput_tokens_per_sec": 3776.449472567533, "p90_itl_ms": 70.13620478683151, "p95_itl_ms": 80.76567870884868, "p99_itl_ms": 119.52163472480606, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 7.9914574165156615, "avg_output_len": 855.27734375, "avg_power_watts": 2120.09324571957, "data_parallel": 1, "energy_per_request_joules": 8100.649098460837, "energy_per_token_joules": 9.47137107940121, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 35.24416498839855, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 4, "output_throughput_tokens_per_sec": 223.84227457104387, "p90_itl_ms": 36.37699596583843, "p95_itl_ms": 36.59535786136985, "p99_itl_ms": 37.07177681848407, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 15.981186685962374, "avg_output_len": 871.4091796875, "avg_power_watts": 2209.765938093048, "data_parallel": 1, "energy_per_request_joules": 4577.523088122597, "energy_per_token_joules": 5.2530122413493086, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 37.27450594305992, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 4, "output_throughput_tokens_per_sec": 420.6664360495454, "p90_itl_ms": 37.85925358533859, "p95_itl_ms": 38.39360661804676, "p99_itl_ms": 67.92267538607122, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 31.96929460580913, "avg_output_len": 889.3369140625, "avg_power_watts": 2201.57691316768, "data_parallel": 1, "energy_per_request_joules": 2676.3674279469665, "energy_per_token_joules": 3.009396535359466, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 42.75384545326233, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 4, "output_throughput_tokens_per_sec": 731.5675708733766, "p90_itl_ms": 43.42821389436722, "p95_itl_ms": 43.75521503388882, "p99_itl_ms": 69.76336650550363, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 63.91310975609756, "avg_output_len": 862.9033203125, "avg_power_watts": 2198.09276787414, "data_parallel": 1, "energy_per_request_joules": 1502.6156065506154, "energy_per_token_joules": 1.7413487365032319, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 49.0025170147419, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 4, "output_throughput_tokens_per_sec": 1262.2932568280878, "p90_itl_ms": 49.943381920456886, "p95_itl_ms": 51.18645802140236, "p99_itl_ms": 90.44502444565296, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 95.8923076923077, "avg_output_len": 863.1259765625, "avg_power_watts": 2222.2427318286004, "data_parallel": 1, "energy_per_request_joules": 1094.4910049232049, "energy_per_token_joules": 1.2680547621589875, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 52.73650772869587, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 4, "output_throughput_tokens_per_sec": 1752.4816736187438, "p90_itl_ms": 53.88541780412197, "p95_itl_ms": 58.11721198260784, "p99_itl_ms": 100.46623401343822, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 127.89090909090909, "avg_output_len": 862.5478515625, "avg_power_watts": 2526.2738006461814, "data_parallel": 1, "energy_per_request_joules": 949.5175501898179, "energy_per_token_joules": 1.100828839199788, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 53.126478000194766, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 4, "output_throughput_tokens_per_sec": 2294.883373951735, "p90_itl_ms": 56.17911879380699, "p95_itl_ms": 60.4096467985073, "p99_itl_ms": 105.0155172747327, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 191.75, "avg_output_len": 863.916015625, "avg_power_watts": 2533.7933939862187, "data_parallel": 1, "energy_per_request_joules": 725.8982138983607, "energy_per_token_joules": 0.8402416447543337, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 60.29347349976888, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 4, "output_throughput_tokens_per_sec": 3015.5532159168783, "p90_itl_ms": 63.27185699774418, "p95_itl_ms": 79.10008025646675, "p99_itl_ms": 119.78105250091176, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 7.992145662263478, "avg_output_len": 876.666015625, "avg_power_watts": 3596.4456629623637, "data_parallel": 1, "energy_per_request_joules": 9984.831726816567, "energy_per_token_joules": 11.389550351963965, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 24.736017920076847, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 315.76713318996025, "p90_itl_ms": 24.99899808317423, "p95_itl_ms": 25.11235298588872, "p99_itl_ms": 25.919590368866917, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 15.984137475214805, "avg_output_len": 872.8037109375, "avg_power_watts": 3638.3735050517316, "data_parallel": 1, "energy_per_request_joules": 5495.996818900514, "energy_per_token_joules": 6.296944834248159, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 26.786433532834053, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 577.7998062271646, "p90_itl_ms": 27.224621549248695, "p95_itl_ms": 27.381783351302147, "p99_itl_ms": 82.417909540236, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 31.97048406139315, "avg_output_len": 872.3017578125, "avg_power_watts": 3705.948513878366, "data_parallel": 1, "energy_per_request_joules": 3179.0357930716477, "energy_per_token_joules": 3.6444220874251374, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 30.15841171145439, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 1016.882354726562, "p90_itl_ms": 30.648254230618477, "p95_itl_ms": 30.96773363649845, "p99_itl_ms": 81.87093801796436, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 63.937106918238996, "avg_output_len": 876.3876953125, "avg_power_watts": 3725.2104772419757, "data_parallel": 1, "energy_per_request_joules": 1854.4103294013923, "energy_per_token_joules": 2.1159702941061393, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 34.669145941734314, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 1760.52116025364, "p90_itl_ms": 35.32791547477245, "p95_itl_ms": 36.43390350043774, "p99_itl_ms": 83.94805938005449, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 95.9093567251462, "avg_output_len": 872.818359375, "avg_power_watts": 3817.2735790542147, "data_parallel": 1, "energy_per_request_joules": 1419.7867944188235, "energy_per_token_joules": 1.6266692596103178, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 38.73824793845415, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 2346.6808366248188, "p90_itl_ms": 39.73771445453167, "p95_itl_ms": 41.582258231937885, "p99_itl_ms": 89.86550176516175, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 127.82608695652173, "avg_output_len": 878.6328125, "avg_power_watts": 3874.1341684941376, "data_parallel": 1, "energy_per_request_joules": 1198.584199997726, "energy_per_token_joules": 1.3641468688010399, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 42.144618928432465, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 2839.968523256698, "p90_itl_ms": 44.59398053586483, "p95_itl_ms": 48.33376640453935, "p99_itl_ms": 101.21062705293299, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 191.76960784313727, "avg_output_len": 886.9775390625, "avg_power_watts": 3984.6582773008904, "data_parallel": 1, "energy_per_request_joules": 979.1941073359543, "energy_per_token_joules": 1.1039671967014222, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 49.6076624840498, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 3609.3991644016, "p90_itl_ms": 53.009020164608955, "p95_itl_ms": 59.90309827029705, "p99_itl_ms": 116.0414159297943, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 255.72972972972974, "avg_output_len": 862.06640625, "avg_power_watts": 4107.038743075696, "data_parallel": 1, "energy_per_request_joules": 819.2440372120919, "energy_per_token_joules": 0.9503259044460555, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 256, "median_itl_ms": 54.93493750691414, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 4321.716080621508, "p90_itl_ms": 59.989769384264946, "p95_itl_ms": 67.8470803424716, "p99_itl_ms": 123.05417845025649, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 383.46, "avg_output_len": 873.591796875, "avg_power_watts": 4148.866789614736, "data_parallel": 1, "energy_per_request_joules": 721.5324594315263, "energy_per_token_joules": 0.8259377686610403, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 384, "median_itl_ms": 66.64565298706293, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 5023.219602053827, "p90_itl_ms": 75.68840105086566, "p95_itl_ms": 103.03171053528784, "p99_itl_ms": 160.3875942155719, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 22.0, "architecture": "MoE", "avg_batch_size": 511.18666666666667, "avg_output_len": 881.4365234375, "avg_power_watts": 3960.90471917865, "data_parallel": 1, "energy_per_request_joules": 653.5586701886854, "energy_per_token_joules": 0.7414699218950929, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 512, "median_itl_ms": 75.83757489919662, "model_id": "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8", "nickname": "Qwen 3 235B A22B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 5341.962771807566, "p90_itl_ms": 115.92580489814281, "p95_itl_ms": 130.9320306405425, "p99_itl_ms": 197.445574700832, "tensor_parallel": 1, "total_params_billions": 235.0, "weight_precision": "fp8" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 7.93570722057369, "avg_output_len": 914.9296875, "avg_power_watts": 656.7634883648834, "data_parallel": 1, "energy_per_request_joules": 660.287494543943, "energy_per_token_joules": 0.7216811341515716, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 8.242472002166323, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 910.0466359523072, "p90_itl_ms": 11.42477749090176, "p95_itl_ms": 12.892912994720971, "p99_itl_ms": 17.201489711005706, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 15.908594815825376, "avg_output_len": 911.40625, "avg_power_watts": 659.3575560299904, "data_parallel": 1, "energy_per_request_joules": 487.79045890114133, "energy_per_token_joules": 0.5352064009887373, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 10.813061991939321, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 1231.968741053726, "p90_itl_ms": 20.17996278591454, "p95_itl_ms": 24.632791586918753, "p99_itl_ms": 57.74696251959539, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 31.924568965517242, "avg_output_len": 914.544921875, "avg_power_watts": 742.5519660462408, "data_parallel": 1, "energy_per_request_joules": 349.9358104000504, "energy_per_token_joules": 0.38263381276297725, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 14.283012496889569, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 1940.6334235971326, "p90_itl_ms": 23.598392892745327, "p95_itl_ms": 28.098717705870452, "p99_itl_ms": 67.64560521638487, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 63.82214765100671, "avg_output_len": 929.154296875, "avg_power_watts": 724.6018312032227, "data_parallel": 1, "energy_per_request_joules": 231.63294551851206, "energy_per_token_joules": 0.24929438124276776, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 19.317286991281435, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 2906.6111622371113, "p90_itl_ms": 31.938829791033644, "p95_itl_ms": 37.97192979836833, "p99_itl_ms": 83.05495763168435, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 95.734375, "avg_output_len": 920.421875, "avg_power_watts": 825.2291682923685, "data_parallel": 1, "energy_per_request_joules": 172.60925807376233, "energy_per_token_joules": 0.18753276379243197, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 96, "median_itl_ms": 18.95634800894186, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 4400.453294688078, "p90_itl_ms": 28.834016789915054, "p95_itl_ms": 36.271817538363265, "p99_itl_ms": 79.3119276460493, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 127.78378378378379, "avg_output_len": 898.48828125, "avg_power_watts": 828.9865970187229, "data_parallel": 1, "energy_per_request_joules": 141.16048666000196, "energy_per_token_joules": 0.15710887899797188, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 20.9287769976072, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 5276.510164835588, "p90_itl_ms": 30.844363593496386, "p95_itl_ms": 38.85331525525537, "p99_itl_ms": 82.43987930385622, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 191.71844660194174, "avg_output_len": 901.298828125, "avg_power_watts": 837.8363290346232, "data_parallel": 1, "energy_per_request_joules": 104.70586761853554, "energy_per_token_joules": 0.11617219988664404, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 192, "median_itl_ms": 23.712188500212505, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 7212.020860861279, "p90_itl_ms": 32.67976699862629, "p95_itl_ms": 43.808169750263914, "p99_itl_ms": 73.9106492110294, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 255.62569832402235, "avg_output_len": 913.5546875, "avg_power_watts": 910.2569463412497, "data_parallel": 1, "energy_per_request_joules": 93.72985029724424, "energy_per_token_joules": 0.10259905792147143, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 26.267984503647313, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 8871.981524800683, "p90_itl_ms": 35.47000579710584, "p95_itl_ms": 40.975108358543366, "p99_itl_ms": 78.78310811793199, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 511.1168831168831, "avg_output_len": 923.59423828125, "avg_power_watts": 703.0843707736994, "data_parallel": 1, "energy_per_request_joules": 73.45899660008183, "energy_per_token_joules": 0.07953600569962882, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 512, "median_itl_ms": 38.00327400676906, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 8839.82498981566, "p90_itl_ms": 95.90098900371231, "p95_itl_ms": 101.6370452067349, "p99_itl_ms": 113.81626507500187, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 767.0610687022901, "avg_output_len": 922.40185546875, "avg_power_watts": 547.2774521494814, "data_parallel": 1, "energy_per_request_joules": 62.88193259087031, "energy_per_token_joules": 0.06817194937114986, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 768, "median_itl_ms": 86.87586602172814, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 8027.8979433304485, "p90_itl_ms": 112.73916799109429, "p95_itl_ms": 116.789551495458, "p99_itl_ms": 134.5816407061647, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 1022.6375, "avg_output_len": 928.0029296875, "avg_power_watts": 600.6375753585329, "data_parallel": 1, "energy_per_request_joules": 52.3470536595582, "energy_per_token_joules": 0.05640828491477477, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 1024, "median_itl_ms": 88.67768199706916, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 10648.03824945244, "p90_itl_ms": 114.47294919344131, "p95_itl_ms": 120.49813674530014, "p99_itl_ms": 141.11808279762045, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 1478.2241379310344, "avg_output_len": 918.9970703125, "avg_power_watts": 606.2796373072633, "data_parallel": 1, "energy_per_request_joules": 52.3208391000015, "energy_per_token_joules": 0.056932541778626215, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 1536, "median_itl_ms": 123.30110900802538, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 10649.087821595815, "p90_itl_ms": 161.50045840768144, "p95_itl_ms": 173.22267589042895, "p99_itl_ms": 195.9992899087957, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 7.970568999345978, "avg_output_len": 927.435546875, "avg_power_watts": 399.25222684608104, "data_parallel": 1, "energy_per_request_joules": 605.7397168287757, "energy_per_token_joules": 0.6531340305747603, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 12.851867824792862, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 611.2868234636888, "p90_itl_ms": 13.437983579933643, "p95_itl_ms": 13.635628949850798, "p99_itl_ms": 14.225561413913951, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 15.9658203125, "avg_output_len": 924.6201171875, "avg_power_watts": 439.1560224761966, "data_parallel": 1, "energy_per_request_joules": 450.1035967267058, "energy_per_token_joules": 0.4867984033224654, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 17.55353808403015, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 902.1311891717331, "p90_itl_ms": 18.70672106742859, "p95_itl_ms": 19.133332185447216, "p99_itl_ms": 34.12651613354683, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 31.92604006163328, "avg_output_len": 921.689453125, "avg_power_watts": 467.8673544859396, "data_parallel": 1, "energy_per_request_joules": 309.39814384987744, "energy_per_token_joules": 0.33568588942930727, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 22.570966742932796, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 1393.7653300868599, "p90_itl_ms": 24.07204359769821, "p95_itl_ms": 25.483781471848488, "p99_itl_ms": 36.4373036660254, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 63.916256157635466, "avg_output_len": 918.94140625, "avg_power_watts": 479.9613345455722, "data_parallel": 1, "energy_per_request_joules": 203.82810980197172, "energy_per_token_joules": 0.22180751505555713, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 28.9388382807374, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 2163.8641703611984, "p90_itl_ms": 30.66201340407133, "p95_itl_ms": 34.05419941991567, "p99_itl_ms": 44.07660672441129, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 88.50769230769231, "avg_output_len": 931.109375, "avg_power_watts": 494.18756722559067, "data_parallel": 1, "energy_per_request_joules": 173.0671825776575, "energy_per_token_joules": 0.1858720223686476, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 32.06684719771147, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 2658.7517633258876, "p90_itl_ms": 35.17773486673832, "p95_itl_ms": 39.93274671956897, "p99_itl_ms": 64.75569777190682, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 7.991780821917808, "avg_output_len": 900.919921875, "avg_power_watts": 683.7088844029176, "data_parallel": 1, "energy_per_request_joules": 741.4139384512273, "energy_per_token_joules": 0.8229520964617945, "expert_parallel": 2, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 9.360754862427711, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 830.8003434737697, "p90_itl_ms": 9.758817590773106, "p95_itl_ms": 9.877575561404228, "p99_itl_ms": 10.439677517861114, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 15.983827493261456, "avg_output_len": 926.1552734375, "avg_power_watts": 759.4100917703079, "data_parallel": 1, "energy_per_request_joules": 561.3317906644965, "energy_per_token_joules": 0.6060882087093973, "expert_parallel": 2, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 12.550568208098412, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 1252.9695857099643, "p90_itl_ms": 12.935153394937517, "p95_itl_ms": 13.052436150610447, "p99_itl_ms": 29.591265879571473, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 31.969026548672566, "avg_output_len": 926.5751953125, "avg_power_watts": 833.105057733547, "data_parallel": 1, "energy_per_request_joules": 381.0216699760385, "energy_per_token_joules": 0.4112150550798349, "expert_parallel": 2, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 15.388114377856255, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 2025.959525173038, "p90_itl_ms": 16.06137752532959, "p95_itl_ms": 16.301700845360756, "p99_itl_ms": 30.816579908132553, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 63.92537313432836, "avg_output_len": 918.298828125, "avg_power_watts": 894.57664740362, "data_parallel": 1, "energy_per_request_joules": 249.67658244884063, "energy_per_token_joules": 0.27189034201277923, "expert_parallel": 2, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 18.69724877178669, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 3290.21119610631, "p90_itl_ms": 19.593029096722603, "p95_itl_ms": 22.987046465277672, "p99_itl_ms": 34.24973435699939, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 95.89545454545454, "avg_output_len": 920.6142578125, "avg_power_watts": 892.1903220809265, "data_parallel": 1, "energy_per_request_joules": 215.347572397768, "energy_per_token_joules": 0.23391726835674045, "expert_parallel": 2, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 24.592196568846703, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 3814.1276543990457, "p90_itl_ms": 25.715524330735207, "p95_itl_ms": 26.3423640280962, "p99_itl_ms": 42.27248825132847, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 127.86338797814207, "avg_output_len": 918.5947265625, "avg_power_watts": 899.6847345336954, "data_parallel": 1, "energy_per_request_joules": 181.7891692443168, "energy_per_token_joules": 0.1978992084187064, "expert_parallel": 2, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 27.418648824095726, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 4546.176519464303, "p90_itl_ms": 28.79050448536873, "p95_itl_ms": 29.795746505260464, "p99_itl_ms": 46.54600627720355, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 191.7795275590551, "avg_output_len": 914.654296875, "avg_power_watts": 930.269055905984, "data_parallel": 1, "energy_per_request_joules": 142.40252279564615, "energy_per_token_joules": 0.15568999487804014, "expert_parallel": 2, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 31.257938593626022, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 5975.137044835225, "p90_itl_ms": 33.031379617750645, "p95_itl_ms": 34.63115952908993, "p99_itl_ms": 52.51390812918544, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 255.80208333333334, "avg_output_len": 920.416015625, "avg_power_watts": 954.8779613627548, "data_parallel": 1, "energy_per_request_joules": 122.8502984414401, "energy_per_token_joules": 0.13347257800378423, "expert_parallel": 2, "gpu_model": "H100", "max_num_seqs": 256, "median_itl_ms": 34.58299767225981, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 7154.113418980205, "p90_itl_ms": 37.12718449532986, "p95_itl_ms": 39.88573867827654, "p99_itl_ms": 56.68182469904422, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 255.7037037037037, "avg_output_len": 917.5947265625, "avg_power_watts": 1093.4015243177876, "data_parallel": 1, "energy_per_request_joules": 134.90642313299892, "energy_per_token_joules": 0.14702179429298418, "expert_parallel": 2, "gpu_model": "H100", "max_num_seqs": 256, "median_itl_ms": 33.661473979009315, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 7437.0029938477255, "p90_itl_ms": 35.56408200529404, "p95_itl_ms": 37.31281879299786, "p99_itl_ms": 54.12989009055295, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 383.530303030303, "avg_output_len": 940.3955078125, "avg_power_watts": 939.3746176972072, "data_parallel": 1, "energy_per_request_joules": 105.32033932199157, "energy_per_token_joules": 0.11199579160791863, "expert_parallel": 2, "gpu_model": "H100", "max_num_seqs": 384, "median_itl_ms": 42.92529448866844, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 8387.58853534269, "p90_itl_ms": 48.14500920474529, "p95_itl_ms": 53.5843838006258, "p99_itl_ms": 69.31480057537554, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 511.0681818181818, "avg_output_len": 927.0078125, "avg_power_watts": 953.1709777253628, "data_parallel": 1, "energy_per_request_joules": 91.56399003446357, "energy_per_token_joules": 0.09877369834406177, "expert_parallel": 2, "gpu_model": "H100", "max_num_seqs": 512, "median_itl_ms": 47.386035323143005, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 9650.048481582111, "p90_itl_ms": 55.81157021224499, "p95_itl_ms": 61.83922197669744, "p99_itl_ms": 82.14658012613653, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 511.33561643835617, "avg_output_len": 926.48828125, "avg_power_watts": 1100.593056693011, "data_parallel": 1, "energy_per_request_joules": 108.86985721181036, "energy_per_token_joules": 0.11750807799201222, "expert_parallel": 2, "gpu_model": "H100", "max_num_seqs": 512, "median_itl_ms": 50.20220347796567, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 9366.105509510804, "p90_itl_ms": 69.28075756295587, "p95_itl_ms": 75.94001314428169, "p99_itl_ms": 83.91718219208998, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 767.1616161616162, "avg_output_len": 924.3076171875, "avg_power_watts": 1081.8927539729257, "data_parallel": 1, "energy_per_request_joules": 93.71700460697053, "energy_per_token_joules": 0.10139157447618395, "expert_parallel": 2, "gpu_model": "H100", "max_num_seqs": 768, "median_itl_ms": 64.9999399902299, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 10670.440414425691, "p90_itl_ms": 87.23299278644845, "p95_itl_ms": 89.56072178843897, "p99_itl_ms": 112.6711294124834, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 3.0, "architecture": "MoE", "avg_batch_size": 1018.84375, "avg_output_len": 920.4560546875, "avg_power_watts": 1123.3771247831066, "data_parallel": 1, "energy_per_request_joules": 81.80765510798567, "energy_per_token_joules": 0.08887730673438812, "expert_parallel": 2, "gpu_model": "H100", "max_num_seqs": 1024, "median_itl_ms": 74.8217350046616, "model_id": "Qwen/Qwen3-30B-A3B-Instruct-2507", "nickname": "Qwen 3 30B A3B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 12639.639589218708, "p90_itl_ms": 94.91471797809936, "p95_itl_ms": 98.34421574487351, "p99_itl_ms": 134.33715586143083, "tensor_parallel": 1, "total_params_billions": 30.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 7.973481608212147, "avg_output_len": 645.7470703125, "avg_power_watts": 859.6469411324731, "data_parallel": 1, "energy_per_request_joules": 997.7141677615186, "energy_per_token_joules": 1.5450541142659606, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 14.126725000096485, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 1, "output_throughput_tokens_per_sec": 556.3862994797969, "p90_itl_ms": 15.713961998699233, "p95_itl_ms": 17.302533000474796, "p99_itl_ms": 27.10405021207407, "tensor_parallel": 1, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 15.96742671009772, "avg_output_len": 628.697265625, "avg_power_watts": 836.3809406652724, "data_parallel": 1, "energy_per_request_joules": 512.9694499505373, "energy_per_token_joules": 0.8159244170412998, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 15.11033899441827, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 1, "output_throughput_tokens_per_sec": 1025.0715914326377, "p90_itl_ms": 17.966197305941026, "p95_itl_ms": 20.78293744998517, "p99_itl_ms": 31.63132181711269, "tensor_parallel": 1, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 31.902298850574713, "avg_output_len": 633.6572265625, "avg_power_watts": 848.7690635227195, "data_parallel": 1, "energy_per_request_joules": 300.0351809182488, "energy_per_token_joules": 0.4734976077616866, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 16.813653986901045, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 1, "output_throughput_tokens_per_sec": 1792.5519571999796, "p90_itl_ms": 24.107908585574478, "p95_itl_ms": 27.652266106451858, "p99_itl_ms": 40.08290835598017, "tensor_parallel": 1, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 63.91, "avg_output_len": 630.9091796875, "avg_power_watts": 945.4371052002505, "data_parallel": 1, "energy_per_request_joules": 197.24272318925327, "energy_per_token_joules": 0.31263251437703116, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 19.583309011068195, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 1, "output_throughput_tokens_per_sec": 3024.1163721699922, "p90_itl_ms": 27.58910280535929, "p95_itl_ms": 31.90371829841751, "p99_itl_ms": 48.932129096356185, "tensor_parallel": 1, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 95.79577464788733, "avg_output_len": 628.6318359375, "avg_power_watts": 970.4547244114405, "data_parallel": 1, "energy_per_request_joules": 148.71996008309438, "energy_per_token_joules": 0.23657720080514735, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 96, "median_itl_ms": 21.336754987714812, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 1, "output_throughput_tokens_per_sec": 4102.063601685516, "p90_itl_ms": 28.89867899939418, "p95_itl_ms": 33.98535900050772, "p99_itl_ms": 50.91743976459812, "tensor_parallel": 1, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 127.78048780487805, "avg_output_len": 634.07421875, "avg_power_watts": 975.5957432445775, "data_parallel": 1, "energy_per_request_joules": 132.2973764829926, "energy_per_token_joules": 0.20864651577192456, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 24.72050450160168, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 1, "output_throughput_tokens_per_sec": 4675.830505173734, "p90_itl_ms": 32.939433003775775, "p95_itl_ms": 38.228705797519076, "p99_itl_ms": 65.6113175497738, "tensor_parallel": 1, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 191.58163265306123, "avg_output_len": 635.3955078125, "avg_power_watts": 969.4150830337443, "data_parallel": 1, "energy_per_request_joules": 112.34719675593938, "energy_per_token_joules": 0.17681459087226048, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 192, "median_itl_ms": 31.635102990549058, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 1, "output_throughput_tokens_per_sec": 5482.66451457107, "p90_itl_ms": 41.79628597921692, "p95_itl_ms": 51.09449400333688, "p99_itl_ms": 83.85509598883802, "tensor_parallel": 1, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 255.62285714285716, "avg_output_len": 627.4912109375, "avg_power_watts": 971.2567983598984, "data_parallel": 1, "energy_per_request_joules": 94.57739042768381, "energy_per_token_joules": 0.15072305201913658, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 36.022864005644806, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 1, "output_throughput_tokens_per_sec": 6443.983089173265, "p90_itl_ms": 48.6644494056236, "p95_itl_ms": 56.358281818393145, "p99_itl_ms": 97.60897816129719, "tensor_parallel": 1, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 383.34640522875816, "avg_output_len": 634.96435546875, "avg_power_watts": 971.6747776603263, "data_parallel": 1, "energy_per_request_joules": 89.26404865431735, "energy_per_token_joules": 0.1405811962285976, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 384, "median_itl_ms": 50.2491410006769, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 1, "output_throughput_tokens_per_sec": 6911.840301033549, "p90_itl_ms": 66.56363700749351, "p95_itl_ms": 77.11441750579975, "p99_itl_ms": 131.57422270800444, "tensor_parallel": 1, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 455.73026315789474, "avg_output_len": 627.43017578125, "avg_power_watts": 968.8827415172412, "data_parallel": 1, "energy_per_request_joules": 94.60777040155742, "energy_per_token_joules": 0.15078613374588773, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 512, "median_itl_ms": 57.57595301838592, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 1, "output_throughput_tokens_per_sec": 6425.542703747883, "p90_itl_ms": 93.01012900541537, "p95_itl_ms": 131.57266860362097, "p99_itl_ms": 250.04708052263587, "tensor_parallel": 1, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 7.9854651162790695, "avg_output_len": 637.71484375, "avg_power_watts": 538.4142363571873, "data_parallel": 1, "energy_per_request_joules": 1100.4790985603745, "energy_per_token_joules": 1.7256601588401939, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 25.205765850842, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 1, "output_throughput_tokens_per_sec": 312.00479051394007, "p90_itl_ms": 26.041114702820778, "p95_itl_ms": 26.380172930657864, "p99_itl_ms": 27.392359357327226, "tensor_parallel": 1, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 15.969953051643193, "avg_output_len": 625.513671875, "avg_power_watts": 566.219327648272, "data_parallel": 1, "energy_per_request_joules": 601.3437909556953, "energy_per_token_joules": 0.9613599478219963, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 26.353154331445694, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 1, "output_throughput_tokens_per_sec": 588.9774469292871, "p90_itl_ms": 27.748262882232666, "p95_itl_ms": 28.254767972975966, "p99_itl_ms": 33.39976480230688, "tensor_parallel": 1, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 29.26, "avg_output_len": 629.3203125, "avg_power_watts": 599.3695473965292, "data_parallel": 1, "energy_per_request_joules": 392.6896857517892, "energy_per_token_joules": 0.6239901651860145, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 28.807315975427628, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 1, "output_throughput_tokens_per_sec": 960.5432598731013, "p90_itl_ms": 29.899904504418373, "p95_itl_ms": 30.685177911072966, "p99_itl_ms": 67.19147257506849, "tensor_parallel": 1, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 7.990155865463494, "avg_output_len": 631.0400390625, "avg_power_watts": 850.1373193376241, "data_parallel": 1, "energy_per_request_joules": 1026.7776792422917, "energy_per_token_joules": 1.6271197003089002, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 15.06047509610653, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 2, "output_throughput_tokens_per_sec": 522.4798883427138, "p90_itl_ms": 15.467118471860886, "p95_itl_ms": 15.633875504136086, "p99_itl_ms": 17.32978187501431, "tensor_parallel": 2, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 15.970496894409937, "avg_output_len": 625.572265625, "avg_power_watts": 882.7766718244878, "data_parallel": 1, "energy_per_request_joules": 569.230646536766, "energy_per_token_joules": 0.9099358744238063, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 16.036865301430225, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 2, "output_throughput_tokens_per_sec": 970.1526191430618, "p90_itl_ms": 16.648652032017708, "p95_itl_ms": 16.839390527457, "p99_itl_ms": 21.17397813126445, "tensor_parallel": 2, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 31.935656836461128, "avg_output_len": 641.69140625, "avg_power_watts": 944.462703925736, "data_parallel": 1, "energy_per_request_joules": 355.2177431451657, "energy_per_token_joules": 0.5535647504164556, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 17.91422814130783, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 2, "output_throughput_tokens_per_sec": 1706.1467573851144, "p90_itl_ms": 18.73834375292063, "p95_itl_ms": 19.087451603263617, "p99_itl_ms": 33.89401728287314, "tensor_parallel": 2, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 63.8646288209607, "avg_output_len": 635.0771484375, "avg_power_watts": 974.1410465427495, "data_parallel": 1, "energy_per_request_joules": 233.11824260529224, "energy_per_token_joules": 0.3670707459382538, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 22.44568057358265, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 2, "output_throughput_tokens_per_sec": 2653.823703800719, "p90_itl_ms": 23.600613698363304, "p95_itl_ms": 25.688573531806398, "p99_itl_ms": 64.99680075794453, "tensor_parallel": 2, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 95.8135593220339, "avg_output_len": 634.9384765625, "avg_power_watts": 1021.968645687438, "data_parallel": 1, "energy_per_request_joules": 193.37327853381154, "energy_per_token_joules": 0.30455435553491284, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 26.096563786268234, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 2, "output_throughput_tokens_per_sec": 3355.6198659266383, "p90_itl_ms": 28.729891031980515, "p95_itl_ms": 32.105561345815666, "p99_itl_ms": 78.12223881483078, "tensor_parallel": 2, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 127.84027777777777, "avg_output_len": 624.1328125, "avg_power_watts": 1073.7054290819397, "data_parallel": 1, "energy_per_request_joules": 171.5957529632542, "energy_per_token_joules": 0.27493467660499615, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 29.656008817255497, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 2, "output_throughput_tokens_per_sec": 3905.311044574245, "p90_itl_ms": 33.37810207158327, "p95_itl_ms": 42.72646140307189, "p99_itl_ms": 97.02063553035259, "tensor_parallel": 2, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 191.74358974358975, "avg_output_len": 632.0390625, "avg_power_watts": 1143.1306846891766, "data_parallel": 1, "energy_per_request_joules": 157.02174784007383, "energy_per_token_joules": 0.24843677733933386, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 37.01616916805506, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 2, "output_throughput_tokens_per_sec": 4601.294127752276, "p90_itl_ms": 43.76935381442312, "p95_itl_ms": 61.56245293095707, "p99_itl_ms": 128.9956690557327, "tensor_parallel": 2, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 255.58762886597938, "avg_output_len": 634.4560546875, "avg_power_watts": 1177.9230310696903, "data_parallel": 1, "energy_per_request_joules": 144.96092761983886, "energy_per_token_joules": 0.22848064345644722, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 256, "median_itl_ms": 44.45189796388149, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 2, "output_throughput_tokens_per_sec": 5155.460932051449, "p90_itl_ms": 55.59205114841466, "p95_itl_ms": 74.43105783313511, "p99_itl_ms": 148.15355882048607, "tensor_parallel": 2, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 32.0, "architecture": "Dense Transformer", "avg_batch_size": 340.18518518518516, "avg_output_len": 627.5693359375, "avg_power_watts": 1179.9335260193766, "data_parallel": 1, "energy_per_request_joules": 146.0310064197816, "energy_per_token_joules": 0.23269302379414677, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 384, "median_itl_ms": 54.89538051187992, "model_id": "Qwen/Qwen3-32B", "nickname": "Qwen 3 32B", "num_gpus": 2, "output_throughput_tokens_per_sec": 5070.773101746323, "p90_itl_ms": 82.19895176589489, "p95_itl_ms": 111.24950386583802, "p99_itl_ms": 240.1536944136023, "tensor_parallel": 2, "total_params_billions": 32.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 7.948717948717949, "avg_output_len": 638.6728515625, "avg_power_watts": 646.3385532103331, "data_parallel": 1, "energy_per_request_joules": 274.88167193717175, "energy_per_token_joules": 0.43039510958494537, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 4.971045011188835, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 1501.7330327791929, "p90_itl_ms": 7.418043198413215, "p95_itl_ms": 8.840273588430135, "p99_itl_ms": 16.70140288886577, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 15.941747572815533, "avg_output_len": 646.443359375, "avg_power_watts": 532.3667484570893, "data_parallel": 1, "energy_per_request_joules": 164.9530298410289, "energy_per_token_joules": 0.2551701203961786, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 5.781241008662619, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 2086.3208734256727, "p90_itl_ms": 14.502711396198729, "p95_itl_ms": 19.645376897824438, "p99_itl_ms": 29.29614132444839, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 31.931623931623932, "avg_output_len": 643.1826171875, "avg_power_watts": 494.07917148633146, "data_parallel": 1, "energy_per_request_joules": 118.890970278417, "energy_per_token_joules": 0.1848479220385367, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 10.273801002767868, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 2672.89545934591, "p90_itl_ms": 21.252272714627917, "p95_itl_ms": 25.5326471538865, "p99_itl_ms": 34.70710960653379, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 63.82608695652174, "avg_output_len": 642.6279296875, "avg_power_watts": 542.9719727131063, "data_parallel": 1, "energy_per_request_joules": 78.73551093089202, "energy_per_token_joules": 0.12252114683091954, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 12.89358499343507, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 4431.659242158526, "p90_itl_ms": 23.25513940304518, "p95_itl_ms": 27.942761490703564, "p99_itl_ms": 37.58133703260676, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 127.67692307692307, "avg_output_len": 657.71484375, "avg_power_watts": 747.8724226373723, "data_parallel": 1, "energy_per_request_joules": 53.94159600340641, "energy_per_token_joules": 0.0820136515330188, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 12.078401006874628, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 9118.877268088447, "p90_itl_ms": 22.3784514964791, "p95_itl_ms": 26.97687374893576, "p99_itl_ms": 36.58603523945203, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 255.35643564356437, "avg_output_len": 654.41552734375, "avg_power_watts": 830.6249908866325, "data_parallel": 1, "energy_per_request_joules": 47.42141919949401, "energy_per_token_joules": 0.07246377449504585, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 20.050100021762773, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 11462.623865162035, "p90_itl_ms": 32.296393398428336, "p95_itl_ms": 37.01951170514807, "p99_itl_ms": 49.14332657412161, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 510.9753086419753, "avg_output_len": 655.1123046875, "avg_power_watts": 752.1679936463977, "data_parallel": 1, "energy_per_request_joules": 40.51024704674587, "energy_per_token_joules": 0.06183710297743524, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 512, "median_itl_ms": 34.60888950212393, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 12163.700390700202, "p90_itl_ms": 60.08085860521533, "p95_itl_ms": 65.76630160270724, "p99_itl_ms": 77.67809892975502, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 1022.3878787878788, "avg_output_len": 640.7451171875, "avg_power_watts": 722.8552752357476, "data_parallel": 1, "energy_per_request_joules": 39.222798162107956, "energy_per_token_joules": 0.06121435358613941, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 1024, "median_itl_ms": 79.63052600098308, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 11808.591170019668, "p90_itl_ms": 110.10884499410167, "p95_itl_ms": 122.377487263293, "p99_itl_ms": 157.1214812764083, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 1426.7027027027027, "avg_output_len": 648.70556640625, "avg_power_watts": 801.1575857379834, "data_parallel": 1, "energy_per_request_joules": 34.41909628921428, "energy_per_token_joules": 0.053058117691038, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 1536, "median_itl_ms": 91.19931398890913, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 15099.623217001274, "p90_itl_ms": 113.74230188957881, "p95_itl_ms": 121.03408360417234, "p99_itl_ms": 169.20664652658155, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 7.971976401179941, "avg_output_len": 646.5693359375, "avg_power_watts": 427.0172598265621, "data_parallel": 1, "energy_per_request_joules": 287.08839845754466, "energy_per_token_joules": 0.44401796141674094, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 8.203724399209023, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 961.7116804555965, "p90_itl_ms": 8.684439584612846, "p95_itl_ms": 8.83135162293911, "p99_itl_ms": 13.145123086869724, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 15.955263157894738, "avg_output_len": 660.125, "avg_power_watts": 451.34471023541903, "data_parallel": 1, "energy_per_request_joules": 170.74922433101034, "energy_per_token_joules": 0.25866195694907834, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 8.904065936803818, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 1744.9211146433615, "p90_itl_ms": 9.840286523103716, "p95_itl_ms": 10.73058359324932, "p99_itl_ms": 15.232820529490677, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 31.940366972477065, "avg_output_len": 646.841796875, "avg_power_watts": 478.0785440142732, "data_parallel": 1, "energy_per_request_joules": 105.61778198132008, "energy_per_token_joules": 0.16328224689804693, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 10.569130070507526, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 2927.9272737640877, "p90_itl_ms": 11.781345307826996, "p95_itl_ms": 12.624603137373905, "p99_itl_ms": 17.41659682244062, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 63.88194444444444, "avg_output_len": 644.5478515625, "avg_power_watts": 514.3934412982122, "data_parallel": 1, "energy_per_request_joules": 76.84254986536409, "energy_per_token_joules": 0.11921930959677225, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 13.950487598776817, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 4314.68226949151, "p90_itl_ms": 16.530530527234077, "p95_itl_ms": 17.652933299541473, "p99_itl_ms": 26.652423888444886, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 95.82142857142857, "avg_output_len": 642.03125, "avg_power_watts": 543.8369227988641, "data_parallel": 1, "energy_per_request_joules": 66.44737967775771, "energy_per_token_joules": 0.10349555364751747, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 17.802705988287926, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 5254.688763258855, "p90_itl_ms": 20.21986525505781, "p95_itl_ms": 21.761623676866293, "p99_itl_ms": 32.362725958228104, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 127.80412371134021, "avg_output_len": 641.4453125, "avg_power_watts": 573.0065316962111, "data_parallel": 1, "energy_per_request_joules": 61.97866199722847, "energy_per_token_joules": 0.09662345454777717, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 20.701183937489986, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 5930.304752381607, "p90_itl_ms": 23.542076349258423, "p95_itl_ms": 26.694484520703554, "p99_itl_ms": 40.73789641261099, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 191.6875, "avg_output_len": 658.55078125, "avg_power_watts": 600.1541052930601, "data_parallel": 1, "energy_per_request_joules": 57.014618638630466, "energy_per_token_joules": 0.08657588793746566, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 26.345420628786087, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 6932.116084406264, "p90_itl_ms": 30.825260654091835, "p95_itl_ms": 35.26492286473511, "p99_itl_ms": 56.26332165673371, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 255.4923076923077, "avg_output_len": 654.0283203125, "avg_power_watts": 632.8068089048945, "data_parallel": 1, "energy_per_request_joules": 53.61870897628265, "energy_per_token_joules": 0.08198224344576271, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 256, "median_itl_ms": 31.35814145207405, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 7718.827666914761, "p90_itl_ms": 36.36135905981064, "p95_itl_ms": 42.25611686706543, "p99_itl_ms": 72.22084887325764, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 382.98039215686276, "avg_output_len": 648.2626953125, "avg_power_watts": 621.6589983960096, "data_parallel": 1, "energy_per_request_joules": 48.80366717705554, "energy_per_token_joules": 0.07528378160574141, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 384, "median_itl_ms": 42.677098885178566, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 8257.542131074346, "p90_itl_ms": 50.742095708847046, "p95_itl_ms": 59.25782434642315, "p99_itl_ms": 89.2347013950348, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 474.4, "avg_output_len": 645.0908203125, "avg_power_watts": 615.5932690290543, "data_parallel": 1, "energy_per_request_joules": 46.529870604677456, "energy_per_token_joules": 0.07212917800029628, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 512, "median_itl_ms": 47.158295288681984, "model_id": "Qwen/Qwen3-8B", "nickname": "Qwen 3 8B", "num_gpus": 1, "output_throughput_tokens_per_sec": 8534.59426678238, "p90_itl_ms": 63.71635422110558, "p95_itl_ms": 71.94560617208478, "p99_itl_ms": 141.98259882628923, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 37.0, "architecture": "MoE", "avg_batch_size": 7.991587802313354, "avg_output_len": 846.3876953125, "avg_power_watts": 3853.81992416062, "data_parallel": 1, "energy_per_request_joules": 7249.766839401403, "energy_per_token_joules": 8.565539030815744, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 16.83571099420078, "model_id": "deepseek-ai/DeepSeek-V3.1", "nickname": "DeepSeek V3.1", "num_gpus": 8, "output_throughput_tokens_per_sec": 449.9214714095581, "p90_itl_ms": 17.286713790963404, "p95_itl_ms": 17.45043160044588, "p99_itl_ms": 18.071177114616148, "tensor_parallel": 1, "total_params_billions": 671.0, "weight_precision": "fp8" }, { "activated_params_billions": 37.0, "architecture": "MoE", "avg_batch_size": 15.981912144702843, "avg_output_len": 840.4033203125, "avg_power_watts": 4421.671082109635, "data_parallel": 1, "energy_per_request_joules": 5098.238066839519, "energy_per_token_joules": 6.066418282288274, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 20.335174005595036, "model_id": "deepseek-ai/DeepSeek-V3.1", "nickname": "DeepSeek V3.1", "num_gpus": 8, "output_throughput_tokens_per_sec": 728.8767236870725, "p90_itl_ms": 21.167887997580692, "p95_itl_ms": 21.435712999664247, "p99_itl_ms": 109.65111887722743, "tensor_parallel": 1, "total_params_billions": 671.0, "weight_precision": "fp8" }, { "activated_params_billions": 37.0, "architecture": "MoE", "avg_batch_size": 31.95736434108527, "avg_output_len": 850.46875, "avg_power_watts": 4790.504846711863, "data_parallel": 1, "energy_per_request_joules": 3733.007527773207, "energy_per_token_joules": 4.389352963025634, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 26.71557999565266, "model_id": "deepseek-ai/DeepSeek-V3.1", "nickname": "DeepSeek V3.1", "num_gpus": 8, "output_throughput_tokens_per_sec": 1091.3920313689493, "p90_itl_ms": 28.485231501690578, "p95_itl_ms": 28.994712258281652, "p99_itl_ms": 121.63755995570547, "tensor_parallel": 1, "total_params_billions": 671.0, "weight_precision": "fp8" }, { "activated_params_billions": 37.0, "architecture": "MoE", "avg_batch_size": 63.93899782135076, "avg_output_len": 848.5703125, "avg_power_watts": 5185.741514447487, "data_parallel": 1, "energy_per_request_joules": 2477.5543545664264, "energy_per_token_joules": 2.9196806888839, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 32.58884750539437, "model_id": "deepseek-ai/DeepSeek-V3.1", "nickname": "DeepSeek V3.1", "num_gpus": 8, "output_throughput_tokens_per_sec": 1776.1331005103264, "p90_itl_ms": 35.75396790110972, "p95_itl_ms": 39.66121470148207, "p99_itl_ms": 127.48976287257393, "tensor_parallel": 1, "total_params_billions": 671.0, "weight_precision": "fp8" }, { "activated_params_billions": 37.0, "architecture": "MoE", "avg_batch_size": 127.86666666666666, "avg_output_len": 849.7783203125, "avg_power_watts": 5628.633143216881, "data_parallel": 1, "energy_per_request_joules": 1645.4981296655167, "energy_per_token_joules": 1.9363851610857714, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 38.835546001791954, "model_id": "deepseek-ai/DeepSeek-V3.1", "nickname": "DeepSeek V3.1", "num_gpus": 8, "output_throughput_tokens_per_sec": 2906.7735367589726, "p90_itl_ms": 45.542008604388684, "p95_itl_ms": 57.851680801832075, "p99_itl_ms": 130.0896560336696, "tensor_parallel": 1, "total_params_billions": 671.0, "weight_precision": "fp8" }, { "activated_params_billions": 37.0, "architecture": "MoE", "avg_batch_size": 255.69285714285715, "avg_output_len": 835.6826171875, "avg_power_watts": 5935.4727918451335, "data_parallel": 1, "energy_per_request_joules": 1093.9860157824724, "energy_per_token_joules": 1.3090927025194032, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 48.59261299134232, "model_id": "deepseek-ai/DeepSeek-V3.1", "nickname": "DeepSeek V3.1", "num_gpus": 8, "output_throughput_tokens_per_sec": 4534.035504454398, "p90_itl_ms": 62.114378809928894, "p95_itl_ms": 103.47795409325045, "p99_itl_ms": 131.67373138101533, "tensor_parallel": 1, "total_params_billions": 671.0, "weight_precision": "fp8" }, { "activated_params_billions": 37.0, "architecture": "MoE", "avg_batch_size": 511.4634146341463, "avg_output_len": 849.19921875, "avg_power_watts": 5383.808505403248, "data_parallel": 1, "energy_per_request_joules": 818.1863505330515, "energy_per_token_joules": 0.9634798672299785, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 512, "median_itl_ms": 64.13963149680058, "model_id": "deepseek-ai/DeepSeek-V3.1", "nickname": "DeepSeek V3.1", "num_gpus": 8, "output_throughput_tokens_per_sec": 5587.878572784081, "p90_itl_ms": 128.64145500498125, "p95_itl_ms": 140.48716849356424, "p99_itl_ms": 160.8727269434894, "tensor_parallel": 1, "total_params_billions": 671.0, "weight_precision": "fp8" }, { "activated_params_billions": 37.0, "architecture": "MoE", "avg_batch_size": 767.1405405405405, "avg_output_len": 848.89892578125, "avg_power_watts": 5265.756284829935, "data_parallel": 1, "energy_per_request_joules": 767.9969383479814, "energy_per_token_joules": 0.904697738474797, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 768, "median_itl_ms": 113.01947399624623, "model_id": "deepseek-ai/DeepSeek-V3.1", "nickname": "DeepSeek V3.1", "num_gpus": 8, "output_throughput_tokens_per_sec": 5820.45921072746, "p90_itl_ms": 162.24553100473716, "p95_itl_ms": 168.0515918007586, "p99_itl_ms": 199.70160615514033, "tensor_parallel": 1, "total_params_billions": 671.0, "weight_precision": "fp8" }, { "activated_params_billions": 37.0, "architecture": "MoE", "avg_batch_size": 1022.4545454545455, "avg_output_len": 844.32177734375, "avg_power_watts": 5634.6521511230185, "data_parallel": 1, "energy_per_request_joules": 651.8722566819248, "energy_per_token_joules": 0.7720661413385848, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 1024, "median_itl_ms": 127.63650799752213, "model_id": "deepseek-ai/DeepSeek-V3.1", "nickname": "DeepSeek V3.1", "num_gpus": 8, "output_throughput_tokens_per_sec": 7298.146945485565, "p90_itl_ms": 167.937457410153, "p95_itl_ms": 175.6417997050448, "p99_itl_ms": 209.52899652125782, "tensor_parallel": 1, "total_params_billions": 671.0, "weight_precision": "fp8" }, { "activated_params_billions": 37.0, "architecture": "MoE", "avg_batch_size": 2042.2179487179487, "avg_output_len": 844.9951171875, "avg_power_watts": 5861.563812319105, "data_parallel": 1, "energy_per_request_joules": 532.4669908422877, "energy_per_token_joules": 0.630142091962096, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 2048, "median_itl_ms": 187.29518799227662, "model_id": "deepseek-ai/DeepSeek-V3.1", "nickname": "DeepSeek V3.1", "num_gpus": 8, "output_throughput_tokens_per_sec": 9301.971550682709, "p90_itl_ms": 241.26953700033485, "p95_itl_ms": 267.2704561962746, "p99_itl_ms": 459.060942842625, "tensor_parallel": 1, "total_params_billions": 671.0, "weight_precision": "fp8" }, { "activated_params_billions": 37.0, "architecture": "MoE", "avg_batch_size": 3007.5081967213114, "avg_output_len": 841.7314453125, "avg_power_watts": 5469.903991899082, "data_parallel": 1, "energy_per_request_joules": 510.3790437450712, "energy_per_token_joules": 0.6063442759413469, "expert_parallel": 8, "gpu_model": "B200", "max_num_seqs": 3072, "median_itl_ms": 185.5040895025013, "model_id": "deepseek-ai/DeepSeek-V3.1", "nickname": "DeepSeek V3.1", "num_gpus": 8, "output_throughput_tokens_per_sec": 9021.119203949074, "p90_itl_ms": 331.0587235027924, "p95_itl_ms": 357.32441850268515, "p99_itl_ms": 627.1504627031377, "tensor_parallel": 1, "total_params_billions": 671.0, "weight_precision": "fp8" }, { "activated_params_billions": 12.0, "architecture": "Dense Transformer", "avg_batch_size": 7.962962962962963, "avg_output_len": 724.2236328125, "avg_power_watts": 667.9603321280928, "data_parallel": 1, "energy_per_request_joules": 484.71031294552694, "energy_per_token_joules": 0.6692826510827456, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 7.7277939999476075, "model_id": "google/gemma-3-12b-it", "nickname": "Gemma 3 12B", "num_gpus": 1, "output_throughput_tokens_per_sec": 998.0242742696025, "p90_itl_ms": 10.196260001976043, "p95_itl_ms": 11.869198002386838, "p99_itl_ms": 18.957180011784473, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Dense Transformer", "avg_batch_size": 15.939613526570048, "avg_output_len": 717.4931640625, "avg_power_watts": 680.5389114542013, "data_parallel": 1, "energy_per_request_joules": 282.85409780724746, "energy_per_token_joules": 0.3942254950635437, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 8.316921012010425, "model_id": "google/gemma-3-12b-it", "nickname": "Gemma 3 12B", "num_gpus": 1, "output_throughput_tokens_per_sec": 1726.268138351904, "p90_itl_ms": 12.58522779098712, "p95_itl_ms": 16.907912207534537, "p99_itl_ms": 28.344203702872615, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Dense Transformer", "avg_batch_size": 31.928571428571427, "avg_output_len": 725.390625, "avg_power_watts": 624.9568836091115, "data_parallel": 1, "energy_per_request_joules": 178.84274416564705, "energy_per_token_joules": 0.24654680940444612, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 11.29119799588807, "model_id": "google/gemma-3-12b-it", "nickname": "Gemma 3 12B", "num_gpus": 1, "output_throughput_tokens_per_sec": 2534.8406865160646, "p90_itl_ms": 18.826704210368927, "p95_itl_ms": 24.09385959617794, "p99_itl_ms": 34.22033947543244, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Dense Transformer", "avg_batch_size": 63.855345911949684, "avg_output_len": 722.0390625, "avg_power_watts": 751.0496334359677, "data_parallel": 1, "energy_per_request_joules": 123.95665510799999, "energy_per_token_joules": 0.17167582966884148, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 12.957806000486016, "model_id": "google/gemma-3-12b-it", "nickname": "Gemma 3 12B", "num_gpus": 1, "output_throughput_tokens_per_sec": 4374.8128952382185, "p90_itl_ms": 21.188866085140045, "p95_itl_ms": 26.04149694670923, "p99_itl_ms": 36.92552951222747, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Dense Transformer", "avg_batch_size": 127.6826923076923, "avg_output_len": 728.93359375, "avg_power_watts": 834.2599107877865, "data_parallel": 1, "energy_per_request_joules": 98.03654737603547, "energy_per_token_joules": 0.13449311187825258, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 18.888652994064614, "model_id": "google/gemma-3-12b-it", "nickname": "Gemma 3 12B", "num_gpus": 1, "output_throughput_tokens_per_sec": 6202.993589314707, "p90_itl_ms": 27.22457039926667, "p95_itl_ms": 32.53021871350938, "p99_itl_ms": 41.70991236518602, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Dense Transformer", "avg_batch_size": 255.62686567164178, "avg_output_len": 724.1064453125, "avg_power_watts": 943.4010862886412, "data_parallel": 1, "energy_per_request_joules": 81.97458288074554, "energy_per_token_joules": 0.11320791771901445, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 28.038521006237715, "model_id": "google/gemma-3-12b-it", "nickname": "Gemma 3 12B", "num_gpus": 1, "output_throughput_tokens_per_sec": 8333.348985626533, "p90_itl_ms": 40.02958402270451, "p95_itl_ms": 44.482655997853726, "p99_itl_ms": 54.81611241120845, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Dense Transformer", "avg_batch_size": 486.64102564102564, "avg_output_len": 723.3974609375, "avg_power_watts": 928.0759099365115, "data_parallel": 1, "energy_per_request_joules": 68.98002693481025, "energy_per_token_joules": 0.0953556387181873, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 512, "median_itl_ms": 43.514438992133364, "model_id": "google/gemma-3-12b-it", "nickname": "Gemma 3 12B", "num_gpus": 1, "output_throughput_tokens_per_sec": 9732.784787686589, "p90_itl_ms": 60.27450519613921, "p95_itl_ms": 66.80109159497079, "p99_itl_ms": 105.54652150487546, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Dense Transformer", "avg_batch_size": 7.982716049382716, "avg_output_len": 728.0185546875, "avg_power_watts": 424.23657242266853, "data_parallel": 1, "energy_per_request_joules": 511.1280065604447, "energy_per_token_joules": 0.7020810160255394, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 13.095855712890625, "model_id": "google/gemma-3-12b-it", "nickname": "Gemma 3 12B", "num_gpus": 1, "output_throughput_tokens_per_sec": 604.2558661167905, "p90_itl_ms": 13.650080561637878, "p95_itl_ms": 13.927154429256914, "p99_itl_ms": 17.139788120985024, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Dense Transformer", "avg_batch_size": 15.970238095238095, "avg_output_len": 723.6044921875, "avg_power_watts": 452.29953029252124, "data_parallel": 1, "energy_per_request_joules": 302.39851981188093, "energy_per_token_joules": 0.41790580776759967, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 14.531975612044334, "model_id": "google/gemma-3-12b-it", "nickname": "Gemma 3 12B", "num_gpus": 1, "output_throughput_tokens_per_sec": 1082.3001783790671, "p90_itl_ms": 15.338726341724396, "p95_itl_ms": 15.611305460333824, "p99_itl_ms": 20.028106123209007, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Dense Transformer", "avg_batch_size": 31.940298507462686, "avg_output_len": 725.19140625, "avg_power_watts": 483.82148451154814, "data_parallel": 1, "energy_per_request_joules": 196.38382039807516, "energy_per_token_joules": 0.27080274077375716, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 17.485393211245537, "model_id": "google/gemma-3-12b-it", "nickname": "Gemma 3 12B", "num_gpus": 1, "output_throughput_tokens_per_sec": 1786.6196004114968, "p90_itl_ms": 18.48367191851139, "p95_itl_ms": 19.314130954444405, "p99_itl_ms": 26.807269845158196, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Dense Transformer", "avg_batch_size": 63.890625, "avg_output_len": 717.6943359375, "avg_power_watts": 540.5247225006851, "data_parallel": 1, "energy_per_request_joules": 144.13966692079484, "energy_per_token_joules": 0.20083712480816784, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 22.891703993082047, "model_id": "google/gemma-3-12b-it", "nickname": "Gemma 3 12B", "num_gpus": 1, "output_throughput_tokens_per_sec": 2691.3585972562305, "p90_itl_ms": 24.700338020920753, "p95_itl_ms": 26.22461207211016, "p99_itl_ms": 42.21526157110929, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Dense Transformer", "avg_batch_size": 95.83809523809524, "avg_output_len": 729.0927734375, "avg_power_watts": 575.4938655265942, "data_parallel": 1, "energy_per_request_joules": 130.1285082972949, "energy_per_token_joules": 0.17848004127618733, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 28.476839885115623, "model_id": "google/gemma-3-12b-it", "nickname": "Gemma 3 12B", "num_gpus": 1, "output_throughput_tokens_per_sec": 3224.415802526913, "p90_itl_ms": 30.828069895505905, "p95_itl_ms": 34.062905050814145, "p99_itl_ms": 55.01849431544541, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Dense Transformer", "avg_batch_size": 127.78142076502732, "avg_output_len": 723.0419921875, "avg_power_watts": 588.9711776970237, "data_parallel": 1, "energy_per_request_joules": 120.10687928402726, "energy_per_token_joules": 0.16611328329721825, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 34.7265861928463, "model_id": "google/gemma-3-12b-it", "nickname": "Gemma 3 12B", "num_gpus": 1, "output_throughput_tokens_per_sec": 3545.5995210401493, "p90_itl_ms": 37.31006197631359, "p95_itl_ms": 41.08920879662037, "p99_itl_ms": 66.15548413246871, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 27.0, "architecture": "Dense Transformer", "avg_batch_size": 7.974527526705012, "avg_output_len": 717.814453125, "avg_power_watts": 804.576084108009, "data_parallel": 1, "energy_per_request_joules": 972.0061511013283, "energy_per_token_joules": 1.3541189465741552, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 13.200574001530185, "model_id": "google/gemma-3-27b-it", "nickname": "Gemma 3 27B", "num_gpus": 1, "output_throughput_tokens_per_sec": 594.1694310854606, "p90_itl_ms": 15.534032200230286, "p95_itl_ms": 17.338150605792176, "p99_itl_ms": 25.748170475708182, "tensor_parallel": 1, "total_params_billions": 27.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 27.0, "architecture": "Dense Transformer", "avg_batch_size": 15.939157566302653, "avg_output_len": 721.1865234375, "avg_power_watts": 799.9626972468399, "data_parallel": 1, "energy_per_request_joules": 512.7246606174074, "energy_per_token_joules": 0.7109459813163599, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 13.531176999094896, "model_id": "google/gemma-3-27b-it", "nickname": "Gemma 3 27B", "num_gpus": 1, "output_throughput_tokens_per_sec": 1125.2088319926363, "p90_itl_ms": 18.042510224040598, "p95_itl_ms": 21.11968814861028, "p99_itl_ms": 35.80351640412114, "tensor_parallel": 1, "total_params_billions": 27.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 27.0, "architecture": "Dense Transformer", "avg_batch_size": 31.92207792207792, "avg_output_len": 717.419921875, "avg_power_watts": 859.7463831809043, "data_parallel": 1, "energy_per_request_joules": 334.4961274728931, "energy_per_token_joules": 0.4662487300306307, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 15.783169510541484, "model_id": "google/gemma-3-27b-it", "nickname": "Gemma 3 27B", "num_gpus": 1, "output_throughput_tokens_per_sec": 1843.9650937481854, "p90_itl_ms": 23.05839310283773, "p95_itl_ms": 28.190243343124163, "p99_itl_ms": 44.58779252308886, "tensor_parallel": 1, "total_params_billions": 27.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 27.0, "architecture": "Dense Transformer", "avg_batch_size": 63.83783783783784, "avg_output_len": 711.7890625, "avg_power_watts": 899.3569074838913, "data_parallel": 1, "energy_per_request_joules": 209.05826375885408, "energy_per_token_joules": 0.2937081711042084, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 19.319453509524465, "model_id": "google/gemma-3-27b-it", "nickname": "Gemma 3 27B", "num_gpus": 1, "output_throughput_tokens_per_sec": 3062.0765643077643, "p90_itl_ms": 28.405169697362, "p95_itl_ms": 35.142206303135026, "p99_itl_ms": 47.441880211117734, "tensor_parallel": 1, "total_params_billions": 27.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 27.0, "architecture": "Dense Transformer", "avg_batch_size": 127.81944444444444, "avg_output_len": 713.556640625, "avg_power_watts": 957.4702142580207, "data_parallel": 1, "energy_per_request_joules": 153.91765615400791, "energy_per_token_joules": 0.2157048892701669, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 26.413496991153806, "model_id": "google/gemma-3-27b-it", "nickname": "Gemma 3 27B", "num_gpus": 1, "output_throughput_tokens_per_sec": 4438.79699480898, "p90_itl_ms": 35.59680680336897, "p95_itl_ms": 40.885346494906116, "p99_itl_ms": 60.39131319092144, "tensor_parallel": 1, "total_params_billions": 27.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 27.0, "architecture": "Dense Transformer", "avg_batch_size": 247.31, "avg_output_len": 716.5390625, "avg_power_watts": 973.9345699315605, "data_parallel": 1, "energy_per_request_joules": 126.64505420566941, "energy_per_token_joules": 0.17674549907133558, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 39.601704498636536, "model_id": "google/gemma-3-27b-it", "nickname": "Gemma 3 27B", "num_gpus": 1, "output_throughput_tokens_per_sec": 5510.378340884791, "p90_itl_ms": 54.347466814215295, "p95_itl_ms": 63.79778845002875, "p99_itl_ms": 111.33141885075047, "tensor_parallel": 1, "total_params_billions": 27.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 27.0, "architecture": "Dense Transformer", "avg_batch_size": 7.988339552238806, "avg_output_len": 719.109375, "avg_power_watts": 510.4211657968816, "data_parallel": 1, "energy_per_request_joules": 1086.455440311296, "energy_per_token_joules": 1.5108347604441896, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 23.425721999956295, "model_id": "google/gemma-3-27b-it", "nickname": "Gemma 3 27B", "num_gpus": 1, "output_throughput_tokens_per_sec": 337.8404966316875, "p90_itl_ms": 23.847885405120905, "p95_itl_ms": 24.02262069954304, "p99_itl_ms": 24.892170185485135, "tensor_parallel": 1, "total_params_billions": 27.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 27.0, "architecture": "Dense Transformer", "avg_batch_size": 15.966552315608919, "avg_output_len": 725.525390625, "avg_power_watts": 539.235305519997, "data_parallel": 1, "energy_per_request_joules": 626.4096583623079, "energy_per_token_joules": 0.8633876449488428, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 25.104925502091646, "model_id": "google/gemma-3-27b-it", "nickname": "Gemma 3 27B", "num_gpus": 1, "output_throughput_tokens_per_sec": 624.5575885579735, "p90_itl_ms": 25.871247697796207, "p95_itl_ms": 26.233697752468288, "p99_itl_ms": 29.62442388670752, "tensor_parallel": 1, "total_params_billions": 27.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 27.0, "architecture": "Dense Transformer", "avg_batch_size": 31.734567901234566, "avg_output_len": 714.3095703125, "avg_power_watts": 571.4471831334777, "data_parallel": 1, "energy_per_request_joules": 375.1188860253971, "energy_per_token_joules": 0.5251489012827982, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 28.28457900614012, "model_id": "google/gemma-3-27b-it", "nickname": "Gemma 3 27B", "num_gpus": 1, "output_throughput_tokens_per_sec": 1088.1621988308177, "p90_itl_ms": 29.135124199092388, "p95_itl_ms": 29.889975202968344, "p99_itl_ms": 47.08953475928857, "tensor_parallel": 1, "total_params_billions": 27.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 27.0, "architecture": "Dense Transformer", "avg_batch_size": 7.9897810218978105, "avg_output_len": 711.2275390625, "avg_power_watts": 812.9914005533786, "data_parallel": 1, "energy_per_request_joules": 1101.4679192488024, "energy_per_token_joules": 1.5486857000794645, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 15.06505161523819, "model_id": "google/gemma-3-27b-it", "nickname": "Gemma 3 27B", "num_gpus": 2, "output_throughput_tokens_per_sec": 524.9557095488537, "p90_itl_ms": 15.434471890330315, "p95_itl_ms": 15.643206611275673, "p99_itl_ms": 16.760235279798508, "tensor_parallel": 2, "total_params_billions": 27.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 27.0, "architecture": "Dense Transformer", "avg_batch_size": 15.979310344827587, "avg_output_len": 706.646484375, "avg_power_watts": 871.8307328742193, "data_parallel": 1, "energy_per_request_joules": 629.7643038286432, "energy_per_token_joules": 0.89120135421836, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 16.05406031012535, "model_id": "google/gemma-3-27b-it", "nickname": "Gemma 3 27B", "num_gpus": 2, "output_throughput_tokens_per_sec": 978.2645961516408, "p90_itl_ms": 16.38836171478033, "p95_itl_ms": 16.533141303807497, "p99_itl_ms": 19.79724021628478, "tensor_parallel": 2, "total_params_billions": 27.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 27.0, "architecture": "Dense Transformer", "avg_batch_size": 31.96135265700483, "avg_output_len": 709.8515625, "avg_power_watts": 935.1903041557412, "data_parallel": 1, "energy_per_request_joules": 393.541856231028, "energy_per_token_joules": 0.5544002112850572, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 18.39852798730135, "model_id": "google/gemma-3-27b-it", "nickname": "Gemma 3 27B", "num_gpus": 2, "output_throughput_tokens_per_sec": 1686.8505551035803, "p90_itl_ms": 18.943024426698685, "p95_itl_ms": 19.220177456736565, "p99_itl_ms": 28.431920204311563, "tensor_parallel": 2, "total_params_billions": 27.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 27.0, "architecture": "Dense Transformer", "avg_batch_size": 63.91119691119691, "avg_output_len": 710.787109375, "avg_power_watts": 1029.7000798888814, "data_parallel": 1, "energy_per_request_joules": 275.8018528898932, "energy_per_token_joules": 0.3880231496212807, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 22.762875072658062, "model_id": "google/gemma-3-27b-it", "nickname": "Gemma 3 27B", "num_gpus": 2, "output_throughput_tokens_per_sec": 2653.7078545285035, "p90_itl_ms": 23.736554197967052, "p95_itl_ms": 25.15928894281387, "p99_itl_ms": 51.5003892965615, "tensor_parallel": 2, "total_params_billions": 27.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 27.0, "architecture": "Dense Transformer", "avg_batch_size": 95.84924623115577, "avg_output_len": 710.5146484375, "avg_power_watts": 1101.2112240966244, "data_parallel": 1, "energy_per_request_joules": 233.97439485570175, "energy_per_token_joules": 0.32930270385028265, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 27.00553648173809, "model_id": "google/gemma-3-27b-it", "nickname": "Gemma 3 27B", "num_gpus": 2, "output_throughput_tokens_per_sec": 3344.069791170891, "p90_itl_ms": 28.48411053419113, "p95_itl_ms": 32.965153269469745, "p99_itl_ms": 63.57433378696441, "tensor_parallel": 2, "total_params_billions": 27.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 27.0, "architecture": "Dense Transformer", "avg_batch_size": 127.80357142857143, "avg_output_len": 705.3154296875, "avg_power_watts": 1137.067654131935, "data_parallel": 1, "energy_per_request_joules": 215.93595124162627, "energy_per_token_joules": 0.3061551500968861, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 31.995583325624466, "model_id": "google/gemma-3-27b-it", "nickname": "Gemma 3 27B", "num_gpus": 2, "output_throughput_tokens_per_sec": 3714.024257870879, "p90_itl_ms": 34.478471428155906, "p95_itl_ms": 40.37652723491191, "p99_itl_ms": 90.45042231678961, "tensor_parallel": 2, "total_params_billions": 27.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 27.0, "architecture": "Dense Transformer", "avg_batch_size": 186.993006993007, "avg_output_len": 714.279296875, "avg_power_watts": 1210.4252130316443, "data_parallel": 1, "energy_per_request_joules": 206.79777106974888, "energy_per_token_joules": 0.2895194806492324, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 39.78635370731354, "model_id": "google/gemma-3-27b-it", "nickname": "Gemma 3 27B", "num_gpus": 2, "output_throughput_tokens_per_sec": 4180.807489421191, "p90_itl_ms": 45.55294439196587, "p95_itl_ms": 62.64864569529892, "p99_itl_ms": 141.18140656501055, "tensor_parallel": 2, "total_params_billions": 27.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 7.979517190929042, "avg_output_len": 363.43359375, "avg_power_watts": 5877.9501427961595, "data_parallel": 1, "energy_per_request_joules": 7946.248051989536, "energy_per_token_joules": 21.864374093759835, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 29.06466599961277, "model_id": "meta-llama/Llama-3.1-405B-Instruct", "nickname": "Llama 3.1 405B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 268.83688129328823, "p90_itl_ms": 29.44802339770831, "p95_itl_ms": 29.688990950671723, "p99_itl_ms": 37.66600416842266, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 15.947666195190948, "avg_output_len": 368.111328125, "avg_power_watts": 6077.188012596595, "data_parallel": 1, "energy_per_request_joules": 4277.752356504871, "energy_per_token_joules": 11.620811503666276, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 29.41388850013027, "model_id": "meta-llama/Llama-3.1-405B-Instruct", "nickname": "Llama 3.1 405B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 522.9572832051608, "p90_itl_ms": 29.948657004570123, "p95_itl_ms": 30.634326539438916, "p99_itl_ms": 51.67656740581149, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 31.91860465116279, "avg_output_len": 363.52734375, "avg_power_watts": 6734.613874207607, "data_parallel": 1, "energy_per_request_joules": 2326.384235718299, "energy_per_token_joules": 6.399475240900085, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 28.005737498460803, "model_id": "meta-llama/Llama-3.1-405B-Instruct", "nickname": "Llama 3.1 405B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 1052.3697054354705, "p90_itl_ms": 29.523408997920342, "p95_itl_ms": 35.40738280717051, "p99_itl_ms": 81.10821318012295, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 63.80829015544042, "avg_output_len": 365.146484375, "avg_power_watts": 7349.563787335136, "data_parallel": 1, "energy_per_request_joules": 1490.585274632415, "energy_per_token_joules": 4.082156992922342, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 30.750189995160326, "model_id": "meta-llama/Llama-3.1-405B-Instruct", "nickname": "Llama 3.1 405B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 1800.4118411119011, "p90_itl_ms": 34.505488503782544, "p95_itl_ms": 48.42060674127424, "p99_itl_ms": 128.18236754392274, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 127.62931034482759, "avg_output_len": 372.146484375, "avg_power_watts": 7574.999604415671, "data_parallel": 1, "energy_per_request_joules": 1005.0429903073642, "energy_per_token_joules": 2.7006650136579413, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 36.52045650233049, "model_id": "meta-llama/Llama-3.1-405B-Instruct", "nickname": "Llama 3.1 405B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 2804.8645670999526, "p90_itl_ms": 46.83387330442201, "p95_itl_ms": 81.05555580186768, "p99_itl_ms": 192.61761015091906, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 255.1038961038961, "avg_output_len": 366.001953125, "avg_power_watts": 7642.860124607908, "data_parallel": 1, "energy_per_request_joules": 779.8473435026905, "energy_per_token_joules": 2.1307190763442474, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 49.297641999146435, "model_id": "meta-llama/Llama-3.1-405B-Instruct", "nickname": "Llama 3.1 405B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 3586.9862946555313, "p90_itl_ms": 95.97202859731625, "p95_itl_ms": 135.0864815089153, "p99_itl_ms": 319.0174898816622, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 510.45454545454544, "avg_output_len": 358.28125, "avg_power_watts": 7654.599729808096, "data_parallel": 1, "energy_per_request_joules": 602.6229605587456, "energy_per_token_joules": 1.6819829688512744, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 512, "median_itl_ms": 77.24275349755771, "model_id": "meta-llama/Llama-3.1-405B-Instruct", "nickname": "Llama 3.1 405B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 4550.937715520315, "p90_itl_ms": 136.37292749626795, "p95_itl_ms": 185.58262625447242, "p99_itl_ms": 849.8948430991736, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 765.6511627906976, "avg_output_len": 371.802734375, "avg_power_watts": 7511.669297413645, "data_parallel": 1, "energy_per_request_joules": 508.16597794635663, "energy_per_token_joules": 1.3667623472446568, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 768, "median_itl_ms": 122.20522700226866, "model_id": "meta-llama/Llama-3.1-405B-Instruct", "nickname": "Llama 3.1 405B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 5495.958615304919, "p90_itl_ms": 151.11171999742515, "p95_itl_ms": 196.6257254520314, "p99_itl_ms": 851.2015087372855, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 1021.3015873015873, "avg_output_len": 372.1220703125, "avg_power_watts": 7710.24963406389, "data_parallel": 1, "energy_per_request_joules": 444.0431271669371, "energy_per_token_joules": 1.1932727526589308, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 1024, "median_itl_ms": 145.99897049629362, "model_id": "meta-llama/Llama-3.1-405B-Instruct", "nickname": "Llama 3.1 405B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 6461.431065850948, "p90_itl_ms": 162.19715449697105, "p95_itl_ms": 172.50488176068762, "p99_itl_ms": 856.4103093843734, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 2042.7924528301887, "avg_output_len": 362.244140625, "avg_power_watts": 7680.174988637808, "data_parallel": 1, "energy_per_request_joules": 387.8542420386501, "energy_per_token_joules": 1.070698455934894, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 2048, "median_itl_ms": 239.57511500339024, "model_id": "meta-llama/Llama-3.1-405B-Instruct", "nickname": "Llama 3.1 405B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 7173.051334917417, "p90_itl_ms": 291.83659920236096, "p95_itl_ms": 303.05594674719026, "p99_itl_ms": 870.0112116293167, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 3062.8809523809523, "avg_output_len": 371.100830078125, "avg_power_watts": 7484.256349090715, "data_parallel": 1, "energy_per_request_joules": 387.1649734985235, "energy_per_token_joules": 1.0432878132259005, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 3072, "median_itl_ms": 325.5319989984855, "model_id": "meta-llama/Llama-3.1-405B-Instruct", "nickname": "Llama 3.1 405B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 7173.721627159626, "p90_itl_ms": 428.6404837912414, "p95_itl_ms": 465.4160458012483, "p99_itl_ms": 913.9620979560992, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 3588.6633663366338, "avg_output_len": 370.17919921875, "avg_power_watts": 7530.190882529053, "data_parallel": 1, "energy_per_request_joules": 375.74735377651336, "energy_per_token_joules": 1.0150417813035275, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 4096, "median_itl_ms": 272.3095130058937, "model_id": "meta-llama/Llama-3.1-405B-Instruct", "nickname": "Llama 3.1 405B Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 7418.601895242875, "p90_itl_ms": 544.915402593324, "p95_itl_ms": 616.2923471885733, "p99_itl_ms": 977.1967455570126, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 7.973407977606718, "avg_output_len": 375.546875, "avg_power_watts": 3451.197983196121, "data_parallel": 1, "energy_per_request_joules": 4876.598144607552, "energy_per_token_joules": 12.98532478697247, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 29.202338017057627, "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "nickname": "Llama 3.1 405B Instruct FP8", "num_gpus": 4, "output_throughput_tokens_per_sec": 265.77679340439266, "p90_itl_ms": 29.674868521397002, "p95_itl_ms": 29.974431738082785, "p99_itl_ms": 41.281948534015115, "tensor_parallel": 4, "total_params_billions": 405.0, "weight_precision": "fp8" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 15.94638069705094, "avg_output_len": 359.220703125, "avg_power_watts": 3392.955404670849, "data_parallel": 1, "energy_per_request_joules": 2541.7184328078824, "energy_per_token_joules": 7.0756457261413095, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 31.588607496814802, "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "nickname": "Llama 3.1 405B Instruct FP8", "num_gpus": 4, "output_throughput_tokens_per_sec": 479.52590279293014, "p90_itl_ms": 32.44399398972746, "p95_itl_ms": 33.968448425002855, "p99_itl_ms": 70.30166173586625, "tensor_parallel": 4, "total_params_billions": 405.0, "weight_precision": "fp8" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 31.898058252427184, "avg_output_len": 362.28515625, "avg_power_watts": 3544.362952014253, "data_parallel": 1, "energy_per_request_joules": 1483.8283189412648, "energy_per_token_joules": 4.095746936750917, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 33.71466898533981, "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "nickname": "Llama 3.1 405B Instruct FP8", "num_gpus": 4, "output_throughput_tokens_per_sec": 865.3764519020633, "p90_itl_ms": 36.17100650444627, "p95_itl_ms": 42.246173521562014, "p99_itl_ms": 108.67303599661682, "tensor_parallel": 4, "total_params_billions": 405.0, "weight_precision": "fp8" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 63.855421686746986, "avg_output_len": 371.6298828125, "avg_power_watts": 3674.4747424311345, "data_parallel": 1, "energy_per_request_joules": 951.8614341537074, "energy_per_token_joules": 2.56131564811206, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 38.19187398767099, "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "nickname": "Llama 3.1 405B Instruct FP8", "num_gpus": 4, "output_throughput_tokens_per_sec": 1434.6044171242936, "p90_itl_ms": 42.57957520894707, "p95_itl_ms": 58.78698981832713, "p99_itl_ms": 169.6888421662152, "tensor_parallel": 4, "total_params_billions": 405.0, "weight_precision": "fp8" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 127.66887417218543, "avg_output_len": 368.5947265625, "avg_power_watts": 3849.82022158273, "data_parallel": 1, "energy_per_request_joules": 653.8476995329744, "energy_per_token_joules": 1.7738932556923221, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 45.75715999817476, "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "nickname": "Llama 3.1 405B Instruct FP8", "num_gpus": 4, "output_throughput_tokens_per_sec": 2170.266000633847, "p90_itl_ms": 61.618857010034866, "p95_itl_ms": 110.84766660351308, "p99_itl_ms": 250.4983342648484, "tensor_parallel": 4, "total_params_billions": 405.0, "weight_precision": "fp8" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 255.36458333333334, "avg_output_len": 374.6845703125, "avg_power_watts": 3898.317361100596, "data_parallel": 1, "energy_per_request_joules": 507.5081740501037, "energy_per_token_joules": 1.3544944581700393, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 63.40548800653778, "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "nickname": "Llama 3.1 405B Instruct FP8", "num_gpus": 4, "output_throughput_tokens_per_sec": 2878.0607684193365, "p90_itl_ms": 115.09766280651093, "p95_itl_ms": 151.11726260511202, "p99_itl_ms": 431.80344728636555, "tensor_parallel": 4, "total_params_billions": 405.0, "weight_precision": "fp8" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 510.4655172413793, "avg_output_len": 379.5888671875, "avg_power_watts": 3883.077089471107, "data_parallel": 1, "energy_per_request_joules": 435.27251516948945, "energy_per_token_joules": 1.1466946288350555, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 512, "median_itl_ms": 98.1649300083518, "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "nickname": "Llama 3.1 405B Instruct FP8", "num_gpus": 4, "output_throughput_tokens_per_sec": 3386.3218609613477, "p90_itl_ms": 186.06416878756139, "p95_itl_ms": 248.80010780179873, "p99_itl_ms": 959.0856144862483, "tensor_parallel": 4, "total_params_billions": 405.0, "weight_precision": "fp8" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 7.981012658227848, "avg_output_len": 365.66015625, "avg_power_watts": 3260.4333583354814, "data_parallel": 1, "energy_per_request_joules": 5110.052686327374, "energy_per_token_joules": 13.97486873804664, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 33.19642134010792, "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "nickname": "Llama 3.1 405B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 233.30690394672098, "p90_itl_ms": 33.49717278033495, "p95_itl_ms": 33.59378566965461, "p99_itl_ms": 42.53464464098218, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "fp8" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 15.94773519163763, "avg_output_len": 380.203125, "avg_power_watts": 3384.257712479697, "data_parallel": 1, "energy_per_request_joules": 2885.4491030407085, "energy_per_token_joules": 7.589230370057344, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 34.0373395010829, "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "nickname": "Llama 3.1 405B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 445.928974014545, "p90_itl_ms": 34.33295153081417, "p95_itl_ms": 34.65995714068413, "p99_itl_ms": 85.39401005953532, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "fp8" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 31.914893617021278, "avg_output_len": 372.1015625, "avg_power_watts": 3744.284581648396, "data_parallel": 1, "energy_per_request_joules": 1773.1699180981968, "energy_per_token_joules": 4.76528479532573, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 36.33137606084347, "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "nickname": "Llama 3.1 405B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 785.742036933693, "p90_itl_ms": 36.93876639008522, "p95_itl_ms": 43.746870290488005, "p99_itl_ms": 136.96959275752303, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "fp8" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 63.86381322957198, "avg_output_len": 359.689453125, "avg_power_watts": 4162.080045275077, "data_parallel": 1, "energy_per_request_joules": 1108.7333771979386, "energy_per_token_joules": 3.082473971825438, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 39.31737877428532, "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "nickname": "Llama 3.1 405B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 1350.2401263781953, "p90_itl_ms": 43.910397589206696, "p95_itl_ms": 92.15272851288307, "p99_itl_ms": 190.17528364434835, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "fp8" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 95.71, "avg_output_len": 368.140625, "avg_power_watts": 4474.554460755058, "data_parallel": 1, "energy_per_request_joules": 979.5898748564199, "energy_per_token_joules": 2.660912185001098, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 44.08974479883909, "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "nickname": "Llama 3.1 405B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 1681.586670156577, "p90_itl_ms": 57.852825708687305, "p95_itl_ms": 104.21531414613128, "p99_itl_ms": 280.5608754046261, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "fp8" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 127.65868263473054, "avg_output_len": 360.8701171875, "avg_power_watts": 4606.148580022177, "data_parallel": 1, "energy_per_request_joules": 876.7384976636154, "energy_per_token_joules": 2.4295126027519807, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 48.04321005940437, "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "nickname": "Llama 3.1 405B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 1895.9146681538743, "p90_itl_ms": 99.37790222465993, "p95_itl_ms": 142.75479894131425, "p99_itl_ms": 353.72009355574846, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "fp8" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 191.45323741007195, "avg_output_len": 359.1103515625, "avg_power_watts": 4812.870137887257, "data_parallel": 1, "energy_per_request_joules": 802.2131915627757, "energy_per_token_joules": 2.2338904686883065, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 62.19174154102802, "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "nickname": "Llama 3.1 405B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 2154.4790155773726, "p90_itl_ms": 113.54916319251075, "p95_itl_ms": 182.43902027606964, "p99_itl_ms": 424.72660258412367, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "fp8" }, { "activated_params_billions": 405.0, "architecture": "Dense Transformer", "avg_batch_size": 255.20833333333334, "avg_output_len": 367.150390625, "avg_power_watts": 4874.811811328656, "data_parallel": 1, "energy_per_request_joules": 757.2790860738882, "energy_per_token_joules": 2.0625855382715845, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 256, "median_itl_ms": 66.90175086259842, "model_id": "meta-llama/Llama-3.1-405B-Instruct-FP8", "nickname": "Llama 3.1 405B Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 2363.447101162978, "p90_itl_ms": 148.1746520847082, "p95_itl_ms": 194.30709956213832, "p99_itl_ms": 569.3191265128553, "tensor_parallel": 8, "total_params_billions": 405.0, "weight_precision": "fp8" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 7.981258366800535, "avg_output_len": 352.0458984375, "avg_power_watts": 1698.02001433363, "data_parallel": 1, "energy_per_request_joules": 1256.5454567843215, "energy_per_token_joules": 3.5692660029879617, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 16.356827021809295, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 475.7336698671816, "p90_itl_ms": 16.857134003657848, "p95_itl_ms": 17.089632994611748, "p99_itl_ms": 22.177567987819188, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 15.944162436548224, "avg_output_len": 359.6552734375, "avg_power_watts": 1732.844097605657, "data_parallel": 1, "energy_per_request_joules": 684.3001762506891, "energy_per_token_joules": 1.9026557561920612, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 16.777443001046777, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 910.7501932318739, "p90_itl_ms": 17.612792609725147, "p95_itl_ms": 18.330938415601846, "p99_itl_ms": 31.258267909870483, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 31.934579439252335, "avg_output_len": 346.1669921875, "avg_power_watts": 1735.0251401691617, "data_parallel": 1, "energy_per_request_joules": 377.8432809428633, "energy_per_token_joules": 1.0915058034713083, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 18.678036984056234, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 1589.5702383361345, "p90_itl_ms": 20.279276999644935, "p95_itl_ms": 23.937293502967805, "p99_itl_ms": 49.35216449666768, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 63.76984126984127, "avg_output_len": 345.671875, "avg_power_watts": 1810.8450919667694, "data_parallel": 1, "energy_per_request_joules": 240.9596658118521, "energy_per_token_joules": 0.6970762831423647, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 21.745624995674007, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 2597.771772988206, "p90_itl_ms": 25.51190589438193, "p95_itl_ms": 34.09519464912589, "p99_itl_ms": 80.05626511498122, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 127.675, "avg_output_len": 352.810546875, "avg_power_watts": 1913.0743598882357, "data_parallel": 1, "energy_per_request_joules": 169.69262046743322, "energy_per_token_joules": 0.4809737746517962, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 27.29597048892174, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 3977.502435082281, "p90_itl_ms": 35.82712191273459, "p95_itl_ms": 50.09463485621353, "p99_itl_ms": 103.27662374067584, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 255.28, "avg_output_len": 361.1494140625, "avg_power_watts": 1941.7341341746157, "data_parallel": 1, "energy_per_request_joules": 130.52984687087246, "energy_per_token_joules": 0.3614289315952847, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 35.835087997838855, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 5372.381578873992, "p90_itl_ms": 59.97263340395875, "p95_itl_ms": 82.05338139669036, "p99_itl_ms": 179.9102551862598, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 510.52, "avg_output_len": 351.6748046875, "avg_power_watts": 1931.7732530547523, "data_parallel": 1, "energy_per_request_joules": 95.22793592138136, "energy_per_token_joules": 0.27078407281977845, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 512, "median_itl_ms": 63.735153002198786, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 7133.998809230012, "p90_itl_ms": 83.2075481011998, "p95_itl_ms": 103.6826805502642, "p99_itl_ms": 225.53748196834923, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 1021.4772727272727, "avg_output_len": 350.72900390625, "avg_power_watts": 1938.3873829344943, "data_parallel": 1, "energy_per_request_joules": 79.50134838645519, "energy_per_token_joules": 0.22667457638520802, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 1024, "median_itl_ms": 105.35636899294332, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 8551.410633896685, "p90_itl_ms": 128.1254446017556, "p95_itl_ms": 153.76852200133723, "p99_itl_ms": 516.3436320715109, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 1407.2539682539682, "avg_output_len": 349.5397135416667, "avg_power_watts": 1918.7437543923984, "data_parallel": 1, "energy_per_request_joules": 77.24768575278874, "energy_per_token_joules": 0.22099830937688422, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 1536, "median_itl_ms": 134.78756349650212, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 8682.164853669661, "p90_itl_ms": 184.36961160332433, "p95_itl_ms": 224.77936413924897, "p99_itl_ms": 536.4187428011792, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 2041.9333333333334, "avg_output_len": 350.83984375, "avg_power_watts": 1907.0860701591266, "data_parallel": 1, "energy_per_request_joules": 66.19452074640198, "energy_per_token_joules": 0.18867446763991436, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 2048, "median_itl_ms": 140.47316349751782, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 10107.81211689334, "p90_itl_ms": 211.69149940833452, "p95_itl_ms": 251.77556945272934, "p99_itl_ms": 558.5651662404416, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 7.986666666666666, "avg_output_len": 349.986328125, "avg_power_watts": 1765.4278811426893, "data_parallel": 1, "energy_per_request_joules": 1315.4984708778682, "energy_per_token_joules": 3.7587138844121615, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 16.57271385192871, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 469.6893499832831, "p90_itl_ms": 16.90893229097128, "p95_itl_ms": 17.02127968892455, "p99_itl_ms": 21.116143707185984, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 15.953771289537713, "avg_output_len": 354.4248046875, "avg_power_watts": 1803.80598644041, "data_parallel": 1, "energy_per_request_joules": 741.1058743374467, "energy_per_token_joules": 2.0910101791292157, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 17.69743673503399, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 862.648113550356, "p90_itl_ms": 18.106621503829956, "p95_itl_ms": 18.409153632819653, "p99_itl_ms": 32.13571004569532, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 31.875536480686694, "avg_output_len": 350.5712890625, "avg_power_watts": 1794.2248144020984, "data_parallel": 1, "energy_per_request_joules": 423.09579469199866, "energy_per_token_joules": 1.206875200257968, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 20.04694938659668, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 1486.6697186408214, "p90_itl_ms": 20.663706585764885, "p95_itl_ms": 23.068370297551155, "p99_itl_ms": 56.29271231591702, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 63.791044776119406, "avg_output_len": 351.9580078125, "avg_power_watts": 2001.5016204568867, "data_parallel": 1, "energy_per_request_joules": 280.5980262398846, "energy_per_token_joules": 0.797248592193898, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 22.432271391153336, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 2510.511326146191, "p90_itl_ms": 24.364156648516655, "p95_itl_ms": 31.98671154677868, "p99_itl_ms": 90.53552784025669, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 95.72549019607843, "avg_output_len": 357.466796875, "avg_power_watts": 2101.546572444986, "data_parallel": 1, "energy_per_request_joules": 231.55182448905188, "energy_per_token_joules": 0.6477575722089276, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 25.356492958962917, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 3244.3411896806874, "p90_itl_ms": 28.38642876595259, "p95_itl_ms": 44.49358126148583, "p99_itl_ms": 114.1463346965611, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 127.60975609756098, "avg_output_len": 358.9296875, "avg_power_watts": 2183.755339171335, "data_parallel": 1, "energy_per_request_joules": 202.20864846476806, "energy_per_token_joules": 0.5633656270485234, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 27.867806144058704, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 3876.266556431646, "p90_itl_ms": 34.00429822504521, "p95_itl_ms": 54.682417679578066, "p99_itl_ms": 121.33453141897917, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 191.62121212121212, "avg_output_len": 357.2021484375, "avg_power_watts": 2293.0811079459736, "data_parallel": 1, "energy_per_request_joules": 183.60242082890318, "energy_per_token_joules": 0.5140014460496121, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 33.70159678161144, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 4461.234740815577, "p90_itl_ms": 56.644922122359276, "p95_itl_ms": 77.61731185019016, "p99_itl_ms": 161.5235060453415, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 255.3148148148148, "avg_output_len": 349.048828125, "avg_power_watts": 2354.298369521534, "data_parallel": 1, "energy_per_request_joules": 167.19181572219904, "energy_per_token_joules": 0.47899262868266945, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 256, "median_itl_ms": 39.83728773891926, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 4915.103549706704, "p90_itl_ms": 68.45140736550097, "p95_itl_ms": 92.88467960432172, "p99_itl_ms": 240.35985104739666, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 383.0232558139535, "avg_output_len": 363.86328125, "avg_power_watts": 2343.5247258308104, "data_parallel": 1, "energy_per_request_joules": 160.3236142090118, "energy_per_token_joules": 0.44061498499723045, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 384, "median_itl_ms": 55.124069564044476, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 5318.758566156212, "p90_itl_ms": 92.43539813905954, "p95_itl_ms": 114.86738631501794, "p99_itl_ms": 436.4945012889802, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 510.5056179775281, "avg_output_len": 350.13671875, "avg_power_watts": 2492.926569137417, "data_parallel": 1, "energy_per_request_joules": 147.41104261576268, "energy_per_token_joules": 0.4210099504617085, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 512, "median_itl_ms": 73.88564106076956, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 5921.300830071837, "p90_itl_ms": 103.65471374243498, "p95_itl_ms": 136.37098474428058, "p99_itl_ms": 299.11049067973937, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 766.3235294117648, "avg_output_len": 351.4052734375, "avg_power_watts": 2535.3280376865096, "data_parallel": 1, "energy_per_request_joules": 136.90855273166184, "energy_per_token_joules": 0.3896030113390203, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 768, "median_itl_ms": 103.63952443003654, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 6507.4651989287295, "p90_itl_ms": 135.2294901385903, "p95_itl_ms": 152.2790588438511, "p99_itl_ms": 561.7390470393002, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 898.4528301886793, "avg_output_len": 350.78271484375, "avg_power_watts": 2547.51808188361, "data_parallel": 1, "energy_per_request_joules": 130.89176771326385, "energy_per_token_joules": 0.3731420112064737, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 1024, "median_itl_ms": 105.61014339327812, "model_id": "meta-llama/Llama-3.1-70B-Instruct", "nickname": "Llama 3.1 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 6827.207887009997, "p90_itl_ms": 151.6728650778532, "p95_itl_ms": 219.92053147405286, "p99_itl_ms": 592.8732616081833, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 7.957983193277311, "avg_output_len": 387.9423828125, "avg_power_watts": 639.3908265351646, "data_parallel": 1, "energy_per_request_joules": 151.50076862712154, "energy_per_token_joules": 0.3905238905034637, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 4.574391001369804, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 1637.2643059323757, "p90_itl_ms": 6.00760200759396, "p95_itl_ms": 7.697059109341346, "p99_itl_ms": 14.804588232655046, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 15.899328859060402, "avg_output_len": 408.970703125, "avg_power_watts": 666.6801912464827, "data_parallel": 1, "energy_per_request_joules": 99.11245254574936, "energy_per_token_joules": 0.2423460942028801, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 5.083433003164828, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 2750.9425866313777, "p90_itl_ms": 8.795807996648364, "p95_itl_ms": 12.222974241012707, "p99_itl_ms": 18.040224708965983, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 31.766990291262136, "avg_output_len": 397.7666015625, "avg_power_watts": 575.5530724733363, "data_parallel": 1, "energy_per_request_joules": 60.090312522947194, "energy_per_token_joules": 0.1510692760199108, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 6.345150992274284, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 3809.8618570031335, "p90_itl_ms": 14.910607208730656, "p95_itl_ms": 18.325782212195918, "p99_itl_ms": 25.094780172803443, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 63.803030303030305, "avg_output_len": 389.7119140625, "avg_power_watts": 616.5121449284105, "data_parallel": 1, "energy_per_request_joules": 42.986272638295084, "energy_per_token_joules": 0.11030269049306295, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 8.997416487545706, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 5589.275675620837, "p90_itl_ms": 20.032751504913904, "p95_itl_ms": 24.230182738392614, "p99_itl_ms": 34.18971085484391, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 127.54268292682927, "avg_output_len": 395.64306640625, "avg_power_watts": 449.32570995413784, "data_parallel": 1, "energy_per_request_joules": 38.89831770775215, "energy_per_token_joules": 0.09831669252055335, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 26.657679496565834, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 4570.187405970814, "p90_itl_ms": 42.8361739992397, "p95_itl_ms": 48.31662949582096, "p99_itl_ms": 58.44129902019631, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 255.23076923076923, "avg_output_len": 399.49609375, "avg_power_watts": 540.1979487399086, "data_parallel": 1, "energy_per_request_joules": 28.15810902362383, "energy_per_token_joules": 0.07048406596246932, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 31.692558506620117, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 7664.1144542874135, "p90_itl_ms": 48.46516041143332, "p95_itl_ms": 54.20180945366155, "p99_itl_ms": 65.51385449856748, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 510.5533980582524, "avg_output_len": 396.1354166666667, "avg_power_watts": 576.5352555778485, "data_parallel": 1, "energy_per_request_joules": 23.86120572476089, "energy_per_token_joules": 0.0602349719839345, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 512, "median_itl_ms": 50.231036002514884, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 9571.437266238263, "p90_itl_ms": 71.37475250056013, "p95_itl_ms": 77.39006950578187, "p99_itl_ms": 93.56033818767166, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 1021.3333333333334, "avg_output_len": 395.5830078125, "avg_power_watts": 599.8901004576812, "data_parallel": 1, "energy_per_request_joules": 21.726446649141838, "energy_per_token_joules": 0.054922598342342914, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 1024, "median_itl_ms": 87.12003598338924, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 10922.463950420793, "p90_itl_ms": 113.56322859646755, "p95_itl_ms": 121.82107815460766, "p99_itl_ms": 153.47145174542663, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 2042.921052631579, "avg_output_len": 395.582763671875, "avg_power_watts": 691.0648132414816, "data_parallel": 1, "energy_per_request_joules": 18.472040700505755, "energy_per_token_joules": 0.04669576735104617, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 2048, "median_itl_ms": 124.85829350771382, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 14799.303072722694, "p90_itl_ms": 175.23864297836553, "p95_itl_ms": 190.32727355806853, "p99_itl_ms": 287.23442446033016, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 7.978201634877384, "avg_output_len": 388.677734375, "avg_power_watts": 486.4380535594687, "data_parallel": 1, "energy_per_request_joules": 176.76995773999624, "energy_per_token_joules": 0.4547982611462042, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 7.316111586987972, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 1069.5688508868182, "p90_itl_ms": 7.704039663076401, "p95_itl_ms": 7.996502518653869, "p99_itl_ms": 10.866769570857286, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 15.951690821256038, "avg_output_len": 404.2314453125, "avg_power_watts": 513.280830756726, "data_parallel": 1, "energy_per_request_joules": 106.2076803873686, "energy_per_token_joules": 0.26273977845850766, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 7.912369444966316, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 1953.571072367271, "p90_itl_ms": 8.581877872347832, "p95_itl_ms": 9.243009611964226, "p99_itl_ms": 13.290748298168158, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 31.911504424778762, "avg_output_len": 390.3291015625, "avg_power_watts": 559.7134507541415, "data_parallel": 1, "energy_per_request_joules": 64.425811784883, "energy_per_token_joules": 0.16505510741316592, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 8.906599134206772, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 3391.070167571773, "p90_itl_ms": 10.384867712855337, "p95_itl_ms": 11.403440311551089, "p99_itl_ms": 18.416078612208267, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 63.917808219178085, "avg_output_len": 395.541015625, "avg_power_watts": 610.3476218460647, "data_parallel": 1, "energy_per_request_joules": 46.90311454458103, "energy_per_token_joules": 0.11857964835952284, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 11.500172317028046, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 5147.153245011704, "p90_itl_ms": 13.176492601633074, "p95_itl_ms": 14.754379913210862, "p99_itl_ms": 25.68335087969897, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 95.70175438596492, "avg_output_len": 392.8330078125, "avg_power_watts": 643.9532367685447, "data_parallel": 1, "energy_per_request_joules": 40.40897607889865, "energy_per_token_joules": 0.102865531346047, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 14.077862724661827, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 6260.145923927034, "p90_itl_ms": 16.678499430418015, "p95_itl_ms": 20.120452716946595, "p99_itl_ms": 35.39796203374864, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 127.62, "avg_output_len": 404.58984375, "avg_power_watts": 669.2439022030471, "data_parallel": 1, "energy_per_request_joules": 38.48050460996364, "energy_per_token_joules": 0.09510991243206074, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 16.482017002999783, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 7036.531577937304, "p90_itl_ms": 20.042716525495052, "p95_itl_ms": 24.74002493545413, "p99_itl_ms": 46.433303505182266, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 191.65853658536585, "avg_output_len": 396.7734375, "avg_power_watts": 688.2545655688424, "data_parallel": 1, "energy_per_request_joules": 33.20802896960434, "energy_per_token_joules": 0.08369519184258484, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 20.908080972731113, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 8223.34653182135, "p90_itl_ms": 26.95829514414072, "p95_itl_ms": 33.54309350252143, "p99_itl_ms": 59.01624012738466, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 255.18987341772151, "avg_output_len": 404.18310546875, "avg_power_watts": 690.5083838667351, "data_parallel": 1, "energy_per_request_joules": 30.97711994863476, "energy_per_token_joules": 0.0766413032348523, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 256, "median_itl_ms": 26.12623802269809, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 9009.611720077452, "p90_itl_ms": 32.539849379099905, "p95_itl_ms": 38.75030499766581, "p99_itl_ms": 59.66428812243974, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 383.01428571428573, "avg_output_len": 394.34423828125, "avg_power_watts": 686.5186027909551, "data_parallel": 1, "energy_per_request_joules": 29.27382187823128, "energy_per_token_joules": 0.07423418180476347, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 384, "median_itl_ms": 37.70148498006165, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 9248.01198181863, "p90_itl_ms": 47.82290779403411, "p95_itl_ms": 54.311841214075685, "p99_itl_ms": 96.15210728254148, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 510.8688524590164, "avg_output_len": 396.67138671875, "avg_power_watts": 688.2983232359837, "data_parallel": 1, "energy_per_request_joules": 27.844149422286517, "energy_per_token_joules": 0.07019449941326047, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 512, "median_itl_ms": 47.40760399727151, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 9805.587745326337, "p90_itl_ms": 59.06887900782748, "p95_itl_ms": 65.01795589574611, "p99_itl_ms": 106.78900223167149, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 8.0, "architecture": "Dense Transformer", "avg_batch_size": 690.8681318681319, "avg_output_len": 397.3541666666667, "avg_power_watts": 685.140945777979, "data_parallel": 1, "energy_per_request_joules": 28.35051624785673, "energy_per_token_joules": 0.07134822942888497, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 768, "median_itl_ms": 62.930971500463784, "model_id": "meta-llama/Llama-3.1-8B-Instruct", "nickname": "Llama 3.1 8B Instruct", "num_gpus": 1, "output_throughput_tokens_per_sec": 9602.774326178349, "p90_itl_ms": 87.81373900419567, "p95_itl_ms": 106.97549971955596, "p99_itl_ms": 192.04428087599808, "tensor_parallel": 1, "total_params_billions": 8.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 7.984090909090909, "avg_output_len": 411.193359375, "avg_power_watts": 1701.4827217894435, "data_parallel": 1, "energy_per_request_joules": 1487.2153646902502, "energy_per_token_joules": 3.61682729251943, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 16.615877000731416, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 470.4351588223653, "p90_itl_ms": 17.130404198542237, "p95_itl_ms": 17.365419460111298, "p99_itl_ms": 21.898801030183677, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 15.964285714285714, "avg_output_len": 405.6298828125, "avg_power_watts": 1745.5327379190146, "data_parallel": 1, "energy_per_request_joules": 779.4169809995365, "energy_per_token_joules": 1.9214979320441667, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 16.940318018896505, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 908.4229073627193, "p90_itl_ms": 17.743828997481614, "p95_itl_ms": 18.33818800514564, "p99_itl_ms": 28.917519008973606, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 31.916334661354583, "avg_output_len": 406.7421875, "avg_power_watts": 1757.6593043836017, "data_parallel": 1, "energy_per_request_joules": 446.7558867279323, "energy_per_token_joules": 1.09837607324156, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 18.645607007783838, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 1600.2345163950497, "p90_itl_ms": 20.242558198515326, "p95_itl_ms": 23.322992264002092, "p99_itl_ms": 46.5622446453199, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 63.847682119205295, "avg_output_len": 415.2265625, "avg_power_watts": 1815.1244370249738, "data_parallel": 1, "energy_per_request_joules": 283.8446104335905, "energy_per_token_joules": 0.6835897220173396, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 21.814916501170956, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 2655.2833937707046, "p90_itl_ms": 24.840720393694934, "p95_itl_ms": 31.674871589348182, "p99_itl_ms": 69.46604490774922, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 127.72527472527473, "avg_output_len": 409.75, "avg_power_watts": 1911.5402435916485, "data_parallel": 1, "energy_per_request_joules": 197.12534417158176, "energy_per_token_joules": 0.48108686802094386, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 27.55990250443574, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 3973.378553139868, "p90_itl_ms": 33.62744760524947, "p95_itl_ms": 50.744753259641584, "p99_itl_ms": 107.72431701538142, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 255.51724137931035, "avg_output_len": 410.296875, "avg_power_watts": 1942.152033355922, "data_parallel": 1, "energy_per_request_joules": 144.4835481095945, "energy_per_token_joules": 0.35214391557233893, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 36.69821401126683, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 5515.22246295054, "p90_itl_ms": 56.227579014375834, "p95_itl_ms": 78.17061689711406, "p99_itl_ms": 170.2859676859225, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 510.625, "avg_output_len": 413.69189453125, "avg_power_watts": 1930.1302457264087, "data_parallel": 1, "energy_per_request_joules": 112.8012327090528, "energy_per_token_joules": 0.2726696708352643, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 512, "median_itl_ms": 63.53210502129514, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 7078.639292055747, "p90_itl_ms": 83.94049727648962, "p95_itl_ms": 104.08187189023009, "p99_itl_ms": 183.94360792590302, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 1021.5576923076923, "avg_output_len": 408.65869140625, "avg_power_watts": 1935.3379848233947, "data_parallel": 1, "energy_per_request_joules": 89.32705003728988, "energy_per_token_joules": 0.2185859542835205, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 1024, "median_itl_ms": 105.5690009961836, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 2, "output_throughput_tokens_per_sec": 8853.899104208374, "p90_itl_ms": 123.76886698184535, "p95_itl_ms": 138.31367259263013, "p99_itl_ms": 514.3569973506965, "tensor_parallel": 2, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 7.976430976430977, "avg_output_len": 408.8447265625, "avg_power_watts": 1787.3856485130575, "data_parallel": 1, "energy_per_request_joules": 1572.7957049116321, "energy_per_token_joules": 3.8469267248117465, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 16.86663366854191, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 464.62690255700807, "p90_itl_ms": 17.01442338526249, "p95_itl_ms": 17.139077931642532, "p99_itl_ms": 20.57395108044147, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 15.970212765957447, "avg_output_len": 409.775390625, "avg_power_watts": 1838.807782685267, "data_parallel": 1, "energy_per_request_joules": 861.2021161923748, "energy_per_token_joules": 2.1016443053811678, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 17.56241451948881, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 874.9376752179618, "p90_itl_ms": 17.893794924020767, "p95_itl_ms": 18.158955965191126, "p99_itl_ms": 28.45144355669618, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 31.911439114391143, "avg_output_len": 412.9306640625, "avg_power_watts": 1895.6477994442098, "data_parallel": 1, "energy_per_request_joules": 515.7798529031772, "energy_per_token_joules": 1.2490713279290644, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 19.864235073328018, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 1517.6457557369092, "p90_itl_ms": 20.36301977932453, "p95_itl_ms": 21.960968896746593, "p99_itl_ms": 51.30415171384808, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 63.857142857142854, "avg_output_len": 408.841796875, "avg_power_watts": 2108.3171636085754, "data_parallel": 1, "energy_per_request_joules": 336.2015351773509, "energy_per_token_joules": 0.8223267233123469, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 22.33504317700863, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 2563.8436692367673, "p90_itl_ms": 23.909232392907143, "p95_itl_ms": 29.054128192365162, "p99_itl_ms": 84.97802283614882, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 95.86324786324786, "avg_output_len": 420.3291015625, "avg_power_watts": 2223.797428056266, "data_parallel": 1, "energy_per_request_joules": 281.94130570658655, "energy_per_token_joules": 0.6707632297133818, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 25.379663333296776, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 3315.3239914574606, "p90_itl_ms": 27.717379480600354, "p95_itl_ms": 39.72181230783462, "p99_itl_ms": 100.1505321264267, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 127.7127659574468, "avg_output_len": 411.4013671875, "avg_power_watts": 2351.335081311273, "data_parallel": 1, "energy_per_request_joules": 248.33036375914227, "energy_per_token_joules": 0.6036206575024905, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 27.75922231376171, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 3895.3853750466974, "p90_itl_ms": 32.05261752009392, "p95_itl_ms": 54.11705840379, "p99_itl_ms": 123.06474521756172, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 191.53424657534248, "avg_output_len": 406.4072265625, "avg_power_watts": 2461.337558890742, "data_parallel": 1, "energy_per_request_joules": 219.46648076861783, "energy_per_token_joules": 0.5400161867812329, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 33.60642120242119, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 4557.89589116124, "p90_itl_ms": 48.582508787512985, "p95_itl_ms": 75.1825701445341, "p99_itl_ms": 171.24340519309044, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 255.4516129032258, "avg_output_len": 413.3115234375, "avg_power_watts": 2519.352730636792, "data_parallel": 1, "energy_per_request_joules": 202.9719263454225, "energy_per_token_joules": 0.4910870247635751, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 256, "median_itl_ms": 39.05790485441685, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 5130.155356577968, "p90_itl_ms": 59.90947894752055, "p95_itl_ms": 85.33287849277252, "p99_itl_ms": 181.4057970046997, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 382.9166666666667, "avg_output_len": 408.2890625, "avg_power_watts": 2510.6066406517143, "data_parallel": 1, "energy_per_request_joules": 181.17673323083628, "energy_per_token_joules": 0.4437462324400039, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 384, "median_itl_ms": 53.8472905755043, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 5657.753141579986, "p90_itl_ms": 81.16421084851027, "p95_itl_ms": 104.58444682881232, "p99_itl_ms": 268.8867459446191, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 510.8, "avg_output_len": 413.61328125, "avg_power_watts": 2490.545119063151, "data_parallel": 1, "energy_per_request_joules": 175.83417338370057, "energy_per_token_joules": 0.42511732904780986, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 512, "median_itl_ms": 73.5875191166997, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 5858.488819172689, "p90_itl_ms": 104.21592239290477, "p95_itl_ms": 134.7207486629486, "p99_itl_ms": 277.25199323147484, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 70.0, "architecture": "Dense Transformer", "avg_batch_size": 724.2444444444444, "avg_output_len": 413.41943359375, "avg_power_watts": 2546.049766678004, "data_parallel": 1, "energy_per_request_joules": 174.91534994349365, "energy_per_token_joules": 0.42309416473966643, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 768, "median_itl_ms": 97.54406102001667, "model_id": "meta-llama/Llama-3.3-70B-Instruct", "nickname": "Llama 3.3 70B Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 6017.690573077533, "p90_itl_ms": 149.4576372206211, "p95_itl_ms": 193.7341911718247, "p99_itl_ms": 570.3238400816917, "tensor_parallel": 4, "total_params_billions": 70.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 7.984301412872841, "avg_output_len": 414.544921875, "avg_power_watts": 1845.3941329316724, "data_parallel": 1, "energy_per_request_joules": 1162.6047671829551, "energy_per_token_joules": 2.8045326473291636, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 11.49036269634962, "model_id": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "nickname": "Llama 4 Maverick 17B 128E Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 658.0041543424691, "p90_itl_ms": 11.708237417042255, "p95_itl_ms": 11.806785874068735, "p99_itl_ms": 46.78823893889785, "tensor_parallel": 1, "total_params_billions": 400.0, "weight_precision": "fp8" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 15.973474801061007, "avg_output_len": 418.2998046875, "avg_power_watts": 2014.707172835016, "data_parallel": 1, "energy_per_request_joules": 759.546383212392, "energy_per_token_joules": 1.8157942573743913, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 13.283604755997658, "model_id": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "nickname": "Llama 4 Maverick 17B 128E Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 1109.5459546987718, "p90_itl_ms": 13.624375686049461, "p95_itl_ms": 13.8603825122118, "p99_itl_ms": 48.573178388178306, "tensor_parallel": 1, "total_params_billions": 400.0, "weight_precision": "fp8" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 31.9070796460177, "avg_output_len": 412.380859375, "avg_power_watts": 2284.2672979381214, "data_parallel": 1, "energy_per_request_joules": 521.8813493378412, "energy_per_token_joules": 1.2655324258473077, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 16.148020513355732, "model_id": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "nickname": "Llama 4 Maverick 17B 128E Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 1804.985199339119, "p90_itl_ms": 16.708863899111748, "p95_itl_ms": 18.226942885667086, "p99_itl_ms": 50.50701173022389, "tensor_parallel": 1, "total_params_billions": 400.0, "weight_precision": "fp8" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 63.86231884057971, "avg_output_len": 406.4326171875, "avg_power_watts": 2655.646101321286, "data_parallel": 1, "energy_per_request_joules": 383.64430886159573, "energy_per_token_joules": 0.9439309067180714, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 20.67001536488533, "model_id": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "nickname": "Llama 4 Maverick 17B 128E Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 2813.3903471331737, "p90_itl_ms": 22.415068745613098, "p95_itl_ms": 47.64300063252449, "p99_itl_ms": 53.07567436248064, "tensor_parallel": 1, "total_params_billions": 400.0, "weight_precision": "fp8" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 95.79629629629629, "avg_output_len": 407.57421875, "avg_power_watts": 3009.7765411308883, "data_parallel": 1, "energy_per_request_joules": 345.9427115013843, "energy_per_token_joules": 0.8487845785790009, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 24.52058345079422, "model_id": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "nickname": "Llama 4 Maverick 17B 128E Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 3545.9840071196018, "p90_itl_ms": 26.628663763403893, "p95_itl_ms": 50.2775227651, "p99_itl_ms": 56.6649042069912, "tensor_parallel": 1, "total_params_billions": 400.0, "weight_precision": "fp8" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 127.72222222222223, "avg_output_len": 405.5849609375, "avg_power_watts": 3140.5181505218916, "data_parallel": 1, "energy_per_request_joules": 307.16558047394597, "energy_per_token_joules": 0.7573396700014222, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 27.628473937511444, "model_id": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "nickname": "Llama 4 Maverick 17B 128E Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 4146.7762417832855, "p90_itl_ms": 31.287204846739783, "p95_itl_ms": 52.70781237632036, "p99_itl_ms": 59.35958638787269, "tensor_parallel": 1, "total_params_billions": 400.0, "weight_precision": "fp8" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 191.56716417910448, "avg_output_len": 415.068359375, "avg_power_watts": 3343.0232559162782, "data_parallel": 1, "energy_per_request_joules": 266.3089596057839, "energy_per_token_joules": 0.6416026507218848, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 33.2093695178628, "model_id": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "nickname": "Llama 4 Maverick 17B 128E Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 5210.42619159218, "p90_itl_ms": 38.075342774391174, "p95_itl_ms": 56.61759898066521, "p99_itl_ms": 66.56867032870652, "tensor_parallel": 1, "total_params_billions": 400.0, "weight_precision": "fp8" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 255.42307692307693, "avg_output_len": 409.4130859375, "avg_power_watts": 3507.338192747269, "data_parallel": 1, "energy_per_request_joules": 229.47161977802995, "energy_per_token_joules": 0.5604892165392596, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 256, "median_itl_ms": 36.97950020432472, "model_id": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "nickname": "Llama 4 Maverick 17B 128E Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 6257.637237703389, "p90_itl_ms": 41.91430062055588, "p95_itl_ms": 60.561890713870525, "p99_itl_ms": 83.9682464301582, "tensor_parallel": 1, "total_params_billions": 400.0, "weight_precision": "fp8" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 383.3170731707317, "avg_output_len": 407.5556640625, "avg_power_watts": 3461.5403094992657, "data_parallel": 1, "energy_per_request_joules": 197.33553739574418, "energy_per_token_joules": 0.48419284725112327, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 384, "median_itl_ms": 47.900935634970665, "model_id": "meta-llama/Llama-4-Maverick-17B-128E-Instruct-FP8", "nickname": "Llama 4 Maverick 17B 128E Instruct FP8", "num_gpus": 8, "output_throughput_tokens_per_sec": 7149.094269259128, "p90_itl_ms": 66.18660315871239, "p95_itl_ms": 69.31344084441656, "p99_itl_ms": 112.54705876111986, "tensor_parallel": 1, "total_params_billions": 400.0, "weight_precision": "fp8" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 7.981466559226431, "avg_output_len": 396.13671875, "avg_power_watts": 1280.568613682407, "data_parallel": 1, "energy_per_request_joules": 1575.5517849207724, "energy_per_token_joules": 3.97729296565183, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 24.152799509465694, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 321.9698988084317, "p90_itl_ms": 24.903803132474422, "p95_itl_ms": 25.16512768343091, "p99_itl_ms": 55.34326063469052, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 15.955026455026456, "avg_output_len": 403.30078125, "avg_power_watts": 1423.0150897004576, "data_parallel": 1, "energy_per_request_joules": 1078.197533845436, "energy_per_token_joules": 2.673432792526821, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 28.983852826058865, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 532.2801058168667, "p90_itl_ms": 29.74723931401968, "p95_itl_ms": 30.112036038190126, "p99_itl_ms": 56.53170496225357, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 31.915690866510538, "avg_output_len": 401.2099609375, "avg_power_watts": 1606.0245726119294, "data_parallel": 1, "energy_per_request_joules": 693.7291179986164, "energy_per_token_joules": 1.7290924591642545, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 32.988084480166435, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 928.8251557052021, "p90_itl_ms": 33.73837023973465, "p95_itl_ms": 34.76462010294199, "p99_itl_ms": 59.271730110049006, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 63.86899563318777, "avg_output_len": 403.7783203125, "avg_power_watts": 1749.6014271210397, "data_parallel": 1, "energy_per_request_joules": 427.7302317272289, "energy_per_token_joules": 1.059319458747046, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 36.49779595434666, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 1651.6277622148593, "p90_itl_ms": 37.53687329590321, "p95_itl_ms": 54.0454238653183, "p99_itl_ms": 69.62598972022533, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 95.74534161490683, "avg_output_len": 409.7841796875, "avg_power_watts": 1819.437956710615, "data_parallel": 1, "energy_per_request_joules": 325.2695546028074, "energy_per_token_joules": 0.7937582042597565, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 38.966670632362366, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 2292.1816076312402, "p90_itl_ms": 41.05184823274613, "p95_itl_ms": 58.421021327376366, "p99_itl_ms": 79.91059008985758, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 127.64341085271317, "avg_output_len": 402.7412109375, "avg_power_watts": 1834.7612222231176, "data_parallel": 1, "energy_per_request_joules": 263.38188021585, "energy_per_token_joules": 0.6539730056498323, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 42.131686583161354, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 2805.5610955990046, "p90_itl_ms": 45.737025141716, "p95_itl_ms": 60.89247018098831, "p99_itl_ms": 94.84558720141649, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 191.55789473684212, "avg_output_len": 414.373046875, "avg_power_watts": 1866.6037123827937, "data_parallel": 1, "energy_per_request_joules": 213.9707172912882, "energy_per_token_joules": 0.5163721890334115, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 47.56573308259249, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 3614.8416820759808, "p90_itl_ms": 55.65968342125416, "p95_itl_ms": 72.53987705335021, "p99_itl_ms": 124.22373022884138, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 255.31944444444446, "avg_output_len": 397.69921875, "avg_power_watts": 1879.1558955956496, "data_parallel": 1, "energy_per_request_joules": 172.79005693462804, "energy_per_token_joules": 0.4344742176706326, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 256, "median_itl_ms": 52.11937241256237, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 4325.126369224987, "p90_itl_ms": 69.86898817121984, "p95_itl_ms": 82.65670957043767, "p99_itl_ms": 158.14138049259782, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 383.08, "avg_output_len": 407.8466796875, "avg_power_watts": 1863.0366513178105, "data_parallel": 1, "energy_per_request_joules": 146.38039430942285, "energy_per_token_joules": 0.35891034940282546, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 384, "median_itl_ms": 64.68318961560726, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 5190.813400665744, "p90_itl_ms": 82.68126100301743, "p95_itl_ms": 97.68766351044178, "p99_itl_ms": 213.87550570070744, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 472.8205128205128, "avg_output_len": 402.279296875, "avg_power_watts": 1881.3809101037816, "data_parallel": 1, "energy_per_request_joules": 135.24559872171807, "energy_per_token_joules": 0.3361982577088546, "expert_parallel": 4, "gpu_model": "H100", "max_num_seqs": 512, "median_itl_ms": 69.11092065274715, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 4, "output_throughput_tokens_per_sec": 5596.045984667312, "p90_itl_ms": 99.41024724394083, "p95_itl_ms": 127.44442112743853, "p99_itl_ms": 335.4441526159644, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 7.989833641404806, "avg_output_len": 405.7119140625, "avg_power_watts": 1999.7234778327174, "data_parallel": 1, "energy_per_request_joules": 2139.4616926262606, "energy_per_token_joules": 5.2733518993890725, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 20.288575440645218, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 379.21297800444324, "p90_itl_ms": 20.527634024620056, "p95_itl_ms": 20.659492164850235, "p99_itl_ms": 59.29509520530702, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 15.96938775510204, "avg_output_len": 403.5986328125, "avg_power_watts": 2292.3401072451134, "data_parallel": 1, "energy_per_request_joules": 1344.41622832889, "energy_per_token_joules": 3.3310723055731115, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 21.932996809482574, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 688.1688228171665, "p90_itl_ms": 22.307416424155235, "p95_itl_ms": 22.554870694875717, "p99_itl_ms": 60.24247668683532, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 31.91358024691358, "avg_output_len": 408.994140625, "avg_power_watts": 2559.4351312278013, "data_parallel": 1, "energy_per_request_joules": 842.121869722538, "energy_per_token_joules": 2.0590071741264033, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 23.91286753118038, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 1243.0433285467884, "p90_itl_ms": 24.39761534333229, "p95_itl_ms": 24.990255944430828, "p99_itl_ms": 61.95208542048931, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 63.84393063583815, "avg_output_len": 402.0166015625, "avg_power_watts": 2797.4258387931445, "data_parallel": 1, "energy_per_request_joules": 507.1400061022294, "energy_per_token_joules": 1.261490207447033, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 26.28241293132305, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 2217.556523450541, "p90_itl_ms": 27.173882350325584, "p95_itl_ms": 57.4591439217329, "p99_itl_ms": 66.1684792488813, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 95.76612903225806, "avg_output_len": 414.76171875, "avg_power_watts": 2932.5972254129206, "data_parallel": 1, "energy_per_request_joules": 399.670534041373, "energy_per_token_joules": 0.9636148081503074, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 28.28422375023365, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 3043.329347586661, "p90_itl_ms": 29.647081159055233, "p95_itl_ms": 59.40303606912494, "p99_itl_ms": 70.17477327957747, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 127.58947368421053, "avg_output_len": 397.2763671875, "avg_power_watts": 2991.3189947245887, "data_parallel": 1, "energy_per_request_joules": 320.9536894339825, "energy_per_token_joules": 0.8078851800477325, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 30.28608113527298, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 3702.6536302446493, "p90_itl_ms": 32.941509410738945, "p95_itl_ms": 61.36552169919013, "p99_itl_ms": 78.7656581774354, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 191.4927536231884, "avg_output_len": 404.7431640625, "avg_power_watts": 3125.0239997132066, "data_parallel": 1, "energy_per_request_joules": 261.0016560025629, "energy_per_token_joules": 0.6448574779690641, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 34.25414115190506, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 4846.069257900618, "p90_itl_ms": 40.279205143451684, "p95_itl_ms": 65.93217179179192, "p99_itl_ms": 105.29956094920635, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 255.42592592592592, "avg_output_len": 412.951171875, "avg_power_watts": 3197.3768254564466, "data_parallel": 1, "energy_per_request_joules": 231.81280855027526, "energy_per_token_joules": 0.5613564613407728, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 256, "median_itl_ms": 37.449986673891544, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 5695.80479722219, "p90_itl_ms": 66.18961375206709, "p95_itl_ms": 69.67382710427046, "p99_itl_ms": 110.03914296627045, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 383.25, "avg_output_len": 410.4990234375, "avg_power_watts": 3188.6593800182022, "data_parallel": 1, "energy_per_request_joules": 192.97864898724157, "energy_per_token_joules": 0.4701074496383626, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 384, "median_itl_ms": 47.76894859969616, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 6782.830994214466, "p90_itl_ms": 75.32594613730907, "p95_itl_ms": 78.70527002960439, "p99_itl_ms": 182.18950707465407, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 17.0, "architecture": "MoE", "avg_batch_size": 510.90625, "avg_output_len": 403.3076171875, "avg_power_watts": 3098.2878789821207, "data_parallel": 1, "energy_per_request_joules": 182.34410212744567, "energy_per_token_joules": 0.45212164203353705, "expert_parallel": 8, "gpu_model": "H100", "max_num_seqs": 512, "median_itl_ms": 55.76958321034908, "model_id": "meta-llama/Llama-4-Scout-17B-16E-Instruct", "nickname": "Llama 4 Scout 17B 16E Instruct", "num_gpus": 8, "output_throughput_tokens_per_sec": 6852.774985613935, "p90_itl_ms": 81.86518400907516, "p95_itl_ms": 87.10338044911623, "p99_itl_ms": 237.28490564972162, "tensor_parallel": 1, "total_params_billions": 109.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 7.955950540958269, "avg_output_len": 1338.4833984375, "avg_power_watts": 677.5667436886174, "data_parallel": 1, "energy_per_request_joules": 878.8845938401172, "energy_per_token_joules": 0.6566271907937724, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 7.196357488282956, "model_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2", "nickname": "NVIDIA Nemotron Nano 12B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 1031.8895610605646, "p90_itl_ms": 10.500433313427497, "p95_itl_ms": 12.215735411155038, "p99_itl_ms": 16.57573381176917, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 15.939309056956116, "avg_output_len": 1330.5908203125, "avg_power_watts": 510.2741532840746, "data_parallel": 1, "energy_per_request_joules": 559.8601625589888, "energy_per_token_joules": 0.420760577941986, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 10.752652015071362, "model_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2", "nickname": "NVIDIA Nemotron Nano 12B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 1212.7423053269752, "p90_itl_ms": 22.949242484173737, "p95_itl_ms": 27.950524745392613, "p99_itl_ms": 50.69378914340633, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 31.92829705505762, "avg_output_len": 1402.1123046875, "avg_power_watts": 518.6896311880196, "data_parallel": 1, "energy_per_request_joules": 423.5923270329613, "energy_per_token_joules": 0.3021101274247577, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 15.963502984959632, "model_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2", "nickname": "NVIDIA Nemotron Nano 12B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 1716.889253628885, "p90_itl_ms": 31.53710138867609, "p95_itl_ms": 37.65478119021279, "p99_itl_ms": 56.877935969969116, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 63.8372591006424, "avg_output_len": 1359.98046875, "avg_power_watts": 565.9963988572973, "data_parallel": 1, "energy_per_request_joules": 278.49116362380187, "energy_per_token_joules": 0.2047758552589889, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 20.077912515262142, "model_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2", "nickname": "NVIDIA Nemotron Nano 12B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 2763.9801486432916, "p90_itl_ms": 35.47910900088027, "p95_itl_ms": 45.731572841759764, "p99_itl_ms": 78.18046510219585, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 127.74460431654676, "avg_output_len": 1406.4287109375, "avg_power_watts": 719.0804003733156, "data_parallel": 1, "energy_per_request_joules": 221.7610323131786, "energy_per_token_joules": 0.1576766960092536, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 23.154423019150272, "model_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2", "nickname": "NVIDIA Nemotron Nano 12B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 4560.473542210161, "p90_itl_ms": 40.37621141760609, "p95_itl_ms": 66.97639439080375, "p99_itl_ms": 88.08277491712941, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 255.61928934010152, "avg_output_len": 1464.31640625, "avg_power_watts": 884.8116469596908, "data_parallel": 1, "energy_per_request_joules": 207.8776530829472, "energy_per_token_joules": 0.14196225091495468, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 33.855443994980305, "model_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2", "nickname": "NVIDIA Nemotron Nano 12B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 6232.724835349045, "p90_itl_ms": 68.01143300253898, "p95_itl_ms": 84.35218151134904, "p99_itl_ms": 107.42822773754584, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 7.90084985835694, "avg_output_len": 466.2724609375, "avg_power_watts": 497.97820634482736, "data_parallel": 1, "energy_per_request_joules": 348.47257551306757, "energy_per_token_joules": 0.7473582609026902, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 11.14979200065136, "model_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2", "nickname": "NVIDIA Nemotron Nano 12B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 666.317925948057, "p90_itl_ms": 11.312101781368256, "p95_itl_ms": 11.4034753292799, "p99_itl_ms": 53.9729418605566, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 15.858910891089108, "avg_output_len": 460.1650390625, "avg_power_watts": 516.6568164274356, "data_parallel": 1, "energy_per_request_joules": 208.3708668895252, "energy_per_token_joules": 0.4528176832252223, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 12.23376952111721, "model_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2", "nickname": "NVIDIA Nemotron Nano 12B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 1140.9819792096348, "p90_itl_ms": 12.520584464073181, "p95_itl_ms": 12.980451062321663, "p99_itl_ms": 56.04606762528419, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 31.768924302788843, "avg_output_len": 460.8447265625, "avg_power_watts": 559.5918240402709, "data_parallel": 1, "energy_per_request_joules": 142.67637624757475, "energy_per_token_joules": 0.30959750220386845, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 14.496257528662682, "model_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2", "nickname": "NVIDIA Nemotron Nano 12B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 1807.481714344654, "p90_itl_ms": 15.625864267349243, "p95_itl_ms": 55.9028796851635, "p99_itl_ms": 62.74990998208523, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 63.7093023255814, "avg_output_len": 458.6630859375, "avg_power_watts": 628.9821341616944, "data_parallel": 1, "energy_per_request_joules": 113.00696845218933, "energy_per_token_joules": 0.24638339538749862, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 19.291361793875694, "model_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2", "nickname": "NVIDIA Nemotron Nano 12B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 2552.8592670478665, "p90_itl_ms": 54.48200739920139, "p95_itl_ms": 63.501773029565804, "p99_itl_ms": 90.03322672098875, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 95.48591549295774, "avg_output_len": 454.115234375, "avg_power_watts": 648.1298844528815, "data_parallel": 1, "energy_per_request_joules": 99.74627113651152, "energy_per_token_joules": 0.21964969150130492, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 24.107911624014378, "model_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2", "nickname": "NVIDIA Nemotron Nano 12B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 2950.743431611107, "p90_itl_ms": 65.90593066066504, "p95_itl_ms": 71.27436995506287, "p99_itl_ms": 130.53565505892036, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 127.57142857142857, "avg_output_len": 459.5390625, "avg_power_watts": 674.5290034452222, "data_parallel": 1, "energy_per_request_joules": 95.98195301833874, "energy_per_token_joules": 0.20886571099347784, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 29.165392741560936, "model_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2", "nickname": "NVIDIA Nemotron Nano 12B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 3229.4865453826715, "p90_itl_ms": 73.33428356796503, "p95_itl_ms": 77.04649427905679, "p99_itl_ms": 174.3089305609465, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 191.31730769230768, "avg_output_len": 462.1357421875, "avg_power_watts": 691.9084429902782, "data_parallel": 1, "energy_per_request_joules": 86.82471754982495, "energy_per_token_joules": 0.1878770881015258, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 38.656579330563545, "model_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2", "nickname": "NVIDIA Nemotron Nano 12B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 3682.77180566255, "p90_itl_ms": 82.72171914577484, "p95_itl_ms": 87.65082247555256, "p99_itl_ms": 236.39661975204925, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 12.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 241.1573033707865, "avg_output_len": 461.5576171875, "avg_power_watts": 693.5924849065933, "data_parallel": 1, "energy_per_request_joules": 83.22016916377432, "energy_per_token_joules": 0.1803028832475481, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 256, "median_itl_ms": 45.98557949066162, "model_id": "nvidia/NVIDIA-Nemotron-Nano-12B-v2", "nickname": "NVIDIA Nemotron Nano 12B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 3846.818599979462, "p90_itl_ms": 92.31475181877613, "p95_itl_ms": 104.63101789355278, "p99_itl_ms": 285.7917774468661, "tensor_parallel": 1, "total_params_billions": 12.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 9.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 7.947916666666667, "avg_output_len": 1078.91796875, "avg_power_watts": 636.7976028882379, "data_parallel": 1, "energy_per_request_joules": 556.9241430490553, "energy_per_token_joules": 0.516187661323585, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 8, "median_itl_ms": 5.901763011934236, "model_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "nickname": "NVIDIA Nemotron Nano 9B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 1233.6552199938108, "p90_itl_ms": 8.14572680974379, "p95_itl_ms": 9.750498400535442, "p99_itl_ms": 14.144910586765016, "tensor_parallel": 1, "total_params_billions": 9.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 9.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 15.919786096256685, "avg_output_len": 1051.029296875, "avg_power_watts": 592.2426120376942, "data_parallel": 1, "energy_per_request_joules": 345.1031805460658, "energy_per_token_joules": 0.32834782205610513, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 16, "median_itl_ms": 7.11653899634257, "model_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "nickname": "NVIDIA Nemotron Nano 9B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 1803.705011134495, "p90_itl_ms": 14.086036605294794, "p95_itl_ms": 17.208993987878785, "p99_itl_ms": 45.85801072535106, "tensor_parallel": 1, "total_params_billions": 9.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 9.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 31.854721549636803, "avg_output_len": 1019.115234375, "avg_power_watts": 546.609468787951, "data_parallel": 1, "energy_per_request_joules": 246.55884708120877, "energy_per_token_joules": 0.24193421780454263, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 32, "median_itl_ms": 10.740508500020951, "model_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "nickname": "NVIDIA Nemotron Nano 9B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 2259.3309609042317, "p90_itl_ms": 27.64044899959117, "p95_itl_ms": 37.33884826215217, "p99_itl_ms": 58.475531692965916, "tensor_parallel": 1, "total_params_billions": 9.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 9.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 63.807817589576544, "avg_output_len": 1094.93359375, "avg_power_watts": 575.9831359663104, "data_parallel": 1, "energy_per_request_joules": 209.51729309523571, "energy_per_token_joules": 0.19135159820758374, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 64, "median_itl_ms": 16.043806506786495, "model_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "nickname": "NVIDIA Nemotron Nano 9B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 3010.077477071643, "p90_itl_ms": 33.93647558987141, "p95_itl_ms": 47.784045957087066, "p99_itl_ms": 68.23242199607193, "tensor_parallel": 1, "total_params_billions": 9.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 9.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 127.75113122171946, "avg_output_len": 1114.8984375, "avg_power_watts": 641.7001737751592, "data_parallel": 1, "energy_per_request_joules": 177.17358644256166, "energy_per_token_joules": 0.15891455264736762, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 128, "median_itl_ms": 24.563489510910586, "model_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "nickname": "NVIDIA Nemotron Nano 9B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 4038.020200699277, "p90_itl_ms": 52.835576294455684, "p95_itl_ms": 65.90449873765465, "p99_itl_ms": 85.10220431315243, "tensor_parallel": 1, "total_params_billions": 9.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 9.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 255.30769230769232, "avg_output_len": 1241.8701171875, "avg_power_watts": 799.5262572910163, "data_parallel": 1, "energy_per_request_joules": 183.01198589066843, "energy_per_token_joules": 0.1473680567378021, "expert_parallel": 1, "gpu_model": "B200", "max_num_seqs": 256, "median_itl_ms": 40.74862599372864, "model_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "nickname": "NVIDIA Nemotron Nano 9B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 5425.370158158067, "p90_itl_ms": 70.74995161383413, "p95_itl_ms": 81.06225341325613, "p99_itl_ms": 102.12520816014153, "tensor_parallel": 1, "total_params_billions": 9.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 9.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 7.936236391912908, "avg_output_len": 506.1953125, "avg_power_watts": 467.2015190264626, "data_parallel": 1, "energy_per_request_joules": 296.5471140543948, "energy_per_token_joules": 0.5858353618286317, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 8, "median_itl_ms": 9.139970876276493, "model_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "nickname": "NVIDIA Nemotron Nano 9B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 797.4962753496744, "p90_itl_ms": 9.268289431929588, "p95_itl_ms": 9.342021029442549, "p99_itl_ms": 49.64682737365365, "tensor_parallel": 1, "total_params_billions": 9.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 9.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 15.847645429362881, "avg_output_len": 503.765625, "avg_power_watts": 495.2273554020826, "data_parallel": 1, "energy_per_request_joules": 178.39352182101797, "energy_per_token_joules": 0.354120076813534, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 16, "median_itl_ms": 9.988353587687016, "model_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "nickname": "NVIDIA Nemotron Nano 9B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 1398.4729695595606, "p90_itl_ms": 10.147353261709213, "p95_itl_ms": 10.4744735173881, "p99_itl_ms": 52.39917958155275, "tensor_parallel": 1, "total_params_billions": 9.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 9.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 31.75862068965517, "avg_output_len": 501.591796875, "avg_power_watts": 545.0429946089819, "data_parallel": 1, "energy_per_request_joules": 127.15043914563014, "energy_per_token_joules": 0.2534938568329834, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 32, "median_itl_ms": 12.150208465754986, "model_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "nickname": "NVIDIA Nemotron Nano 9B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 2150.1230894446808, "p90_itl_ms": 13.254364021122456, "p95_itl_ms": 47.185920644551516, "p99_itl_ms": 56.61105271428824, "tensor_parallel": 1, "total_params_billions": 9.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 9.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 63.592592592592595, "avg_output_len": 502.5732421875, "avg_power_watts": 617.5559167488011, "data_parallel": 1, "energy_per_request_joules": 104.18586509367923, "energy_per_token_joules": 0.20730483907221142, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 64, "median_itl_ms": 16.765335574746132, "model_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "nickname": "NVIDIA Nemotron Nano 9B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 2978.9749217271537, "p90_itl_ms": 50.713999196887016, "p95_itl_ms": 59.038255363702774, "p99_itl_ms": 68.72391104698187, "tensor_parallel": 1, "total_params_billions": 9.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 9.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 95.63636363636364, "avg_output_len": 500.966796875, "avg_power_watts": 641.6266384074639, "data_parallel": 1, "energy_per_request_joules": 93.45563835795397, "energy_per_token_joules": 0.18655056371185572, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 96, "median_itl_ms": 21.002349443733692, "model_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "nickname": "NVIDIA Nemotron Nano 9B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 3439.42481674038, "p90_itl_ms": 57.588535360991955, "p95_itl_ms": 64.92515467107296, "p99_itl_ms": 91.56877128407345, "tensor_parallel": 1, "total_params_billions": 9.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 9.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 127.4873949579832, "avg_output_len": 502.75, "avg_power_watts": 663.0256114403097, "data_parallel": 1, "energy_per_request_joules": 89.06497904734755, "energy_per_token_joules": 0.1771556022821433, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 128, "median_itl_ms": 25.85348580032587, "model_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "nickname": "NVIDIA Nemotron Nano 9B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 3742.617240996733, "p90_itl_ms": 63.7841984629631, "p95_itl_ms": 70.4051981680095, "p99_itl_ms": 140.63350588083262, "tensor_parallel": 1, "total_params_billions": 9.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 9.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 191.2828282828283, "avg_output_len": 508.7529296875, "avg_power_watts": 691.3507118198783, "data_parallel": 1, "energy_per_request_joules": 83.76542661391157, "energy_per_token_joules": 0.1646485390567957, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 192, "median_itl_ms": 34.82525423169136, "model_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "nickname": "NVIDIA Nemotron Nano 9B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 4198.948352535313, "p90_itl_ms": 76.76311247050762, "p95_itl_ms": 79.32483814656734, "p99_itl_ms": 170.12767519801855, "tensor_parallel": 1, "total_params_billions": 9.0, "weight_precision": "bfloat16" }, { "activated_params_billions": 9.0, "architecture": "Mamba-Transformer Hybrid", "avg_batch_size": 255.34883720930233, "avg_output_len": 510.1044921875, "avg_power_watts": 692.9838955224471, "data_parallel": 1, "energy_per_request_joules": 79.17199580690063, "energy_per_token_joules": 0.1552074075399423, "expert_parallel": 1, "gpu_model": "H100", "max_num_seqs": 256, "median_itl_ms": 43.37812401354313, "model_id": "nvidia/NVIDIA-Nemotron-Nano-9B-v2", "nickname": "NVIDIA Nemotron Nano 9B V2", "num_gpus": 1, "output_throughput_tokens_per_sec": 4464.889314925959, "p90_itl_ms": 86.51775121688843, "p95_itl_ms": 97.61630296707153, "p99_itl_ms": 242.5210145488381, "tensor_parallel": 1, "total_params_billions": 9.0, "weight_precision": "bfloat16" } ], "task": "lm-arena-chat", "task_display_name": "LLM Chat (LM Arena)" }