model,topology,nodes,quant,context_tokens,prompt_tps,generation_tps,peak_memory_gb,ttft_seconds,ttft_minutes,feasibility,kv_cache_gb,node
Llama-3.1-405B-Instruct,SINGLE,1,Q2,1024,26.95,5.0,128.2,38.0,0.63,practical,0.5,muladhara
Llama-3.1-405B-Instruct,SINGLE,1,Q2,4096,25.06,4.71,130.2,163.4,2.72,practical,2.0,muladhara
Llama-3.1-405B-Instruct,SINGLE,1,Q2,8192,25.39,4.63,132.3,322.6,5.38,practical,3.9,muladhara
Llama-3.1-405B-Instruct,SINGLE,1,Q2,16384,24.15,4.22,136.3,678.4,11.31,batch_only,7.9,muladhara
Llama-3.1-405B-Instruct,SINGLE,1,Q2,32768,22.35,3.72,144.8,1466.1,24.44,batch_only,15.8,muladhara
Llama-3.1-405B-Instruct,SINGLE,1,Q2,65536,20.01,2.96,161.8,3275.2,54.59,batch_only,31.5,muladhara
Llama-3.1-405B-Instruct,SINGLE,1,Q2,1024,28.24,5.2,128.2,36.3,0.6,practical,0.5,vishuddha
Llama-3.1-405B-Instruct,SINGLE,1,Q2,8192,24.98,4.53,132.3,327.9,5.47,practical,3.9,vishuddha
Llama-3.1-405B-Instruct,SINGLE,1,Q2,32768,21.73,3.58,144.8,1508.0,25.13,batch_only,15.8,vishuddha
Llama-3.1-405B-Instruct,SINGLE,1,Q3,1024,27.87,3.71,178.9,36.7,0.61,practical,0.5,svadhisthana
Llama-3.1-405B-Instruct,SINGLE,1,Q3,4096,25.7,3.58,180.9,159.4,2.66,practical,2.0,svadhisthana
Llama-3.1-405B-Instruct,SINGLE,1,Q3,8192,25.38,3.46,183.0,322.8,5.38,practical,3.9,svadhisthana
Llama-3.1-405B-Instruct,SINGLE,1,Q3,16384,24.27,3.23,187.1,675.1,11.25,batch_only,7.9,svadhisthana
Llama-3.1-405B-Instruct,SINGLE,1,Q3,32768,22.59,2.92,195.5,1450.6,24.18,batch_only,15.8,svadhisthana
Llama-3.1-405B-Instruct,SINGLE,1,Q3,65536,20.23,2.44,212.5,3239.5,53.99,batch_only,31.5,svadhisthana
Llama-3.1-405B-Instruct,SINGLE,1,Q4,1024,27.77,2.99,229.6,36.9,0.61,practical,0.5,muladhara
Llama-3.1-405B-Instruct,SINGLE,1,Q4,4096,26.37,2.94,231.6,155.3,2.59,practical,2.0,muladhara
Llama-3.1-405B-Instruct,SINGLE,1,Q4,8192,25.88,2.86,233.8,316.5,5.28,practical,3.9,muladhara
Llama-3.1-405B-Instruct,SINGLE,1,Q4,16384,25.05,2.71,237.8,654.1,10.9,batch_only,7.9,muladhara
Llama-3.1-405B-Instruct,SINGLE,1,Q4,32768,23.43,2.5,246.3,1398.5,23.31,batch_only,15.8,muladhara
Llama-3.1-405B-Instruct,SINGLE,1,Q4,65536,19.3,2.09,263.2,3395.6,56.59,batch_only,31.5,muladhara
Llama-3.1-405B-Instruct,SINGLE,1,Q4,1024,28.03,2.99,229.6,36.5,0.61,practical,0.5,svadhisthana
Llama-3.1-405B-Instruct,SINGLE,1,Q4,4096,27.32,2.94,231.6,149.9,2.5,practical,2.0,svadhisthana
Llama-3.1-405B-Instruct,SINGLE,1,Q4,8192,24.18,2.83,233.8,338.8,5.65,practical,3.9,manipura
Llama-3.1-405B-Instruct,SINGLE,1,Q4,16384,23.07,2.67,237.8,710.2,11.84,batch_only,7.9,manipura
Llama-3.1-405B-Instruct,SINGLE,1,Q4,32768,23.94,2.5,246.3,1368.8,22.81,batch_only,15.8,anahata
Llama-3.1-405B-Instruct,SINGLE,1,Q4,65536,19.15,2.09,263.2,3422.2,57.04,batch_only,31.5,vishuddha
Llama-3.1-405B-Instruct,SINGLE,1,Q5,1024,25.12,2.34,280.4,40.8,0.68,practical,0.5,manipura
Llama-3.1-405B-Instruct,SINGLE,1,Q5,4096,22.23,2.2,282.4,184.3,3.07,practical,2.0,manipura
Llama-3.1-405B-Instruct,SINGLE,1,Q5,8192,22.29,2.18,284.5,367.5,6.13,practical,3.9,manipura
Llama-3.1-405B-Instruct,SINGLE,1,Q5,16384,21.26,2.07,288.5,770.6,12.84,batch_only,7.9,manipura
Llama-3.1-405B-Instruct,SINGLE,1,Q5,32768,19.81,1.95,297.0,1654.1,27.57,batch_only,15.8,manipura
Llama-3.1-405B-Instruct,SINGLE,1,Q5,65536,17.9,1.76,314.0,3661.2,61.02,batch_only,31.5,manipura
Llama-3.1-405B-Instruct,SINGLE,1,Q6,1024,26.62,2.08,331.1,38.5,0.64,practical,0.5,anahata
Llama-3.1-405B-Instruct,SINGLE,1,Q6,4096,25.97,2.05,333.1,157.7,2.63,practical,2.0,anahata
Llama-3.1-405B-Instruct,SINGLE,1,Q6,8192,24.85,2.01,335.2,329.7,5.49,practical,3.9,anahata
Llama-3.1-405B-Instruct,SINGLE,1,Q6,16384,23.58,1.93,339.3,694.8,11.58,batch_only,7.9,anahata
Llama-3.1-405B-Instruct,SINGLE,1,Q6,32768,22.16,1.82,347.7,1478.7,24.65,batch_only,15.8,anahata
Llama-3.1-405B-Instruct,TP2,2,Q2,1024,52.18,6.23,66.0,19.6,0.33,practical,0.5,muladhara+svadhisthana
Llama-3.1-405B-Instruct,TP2,2,Q2,4096,52.36,6.12,66.0,78.2,1.3,practical,2.0,muladhara+svadhisthana
Llama-3.1-405B-Instruct,TP2,2,Q2,16384,49.86,5.62,69.2,328.6,5.48,practical,7.9,muladhara+svadhisthana
Llama-3.1-405B-Instruct,TP2,2,Q3,1024,51.79,5.0,91.7,19.8,0.33,practical,0.5,muladhara+svadhisthana
Llama-3.1-405B-Instruct,TP2,2,Q3,16384,47.99,4.42,94.8,341.4,5.69,practical,7.9,muladhara+svadhisthana
Llama-3.1-405B-Instruct,TP2,2,Q4,1024,51.04,4.31,117.9,20.1,0.33,practical,0.5,muladhara+svadhisthana
Llama-3.1-405B-Instruct,TP2,2,Q4,4096,51.3,4.23,118.4,79.8,1.33,practical,2.0,muladhara+svadhisthana
Llama-3.1-405B-Instruct,TP2,2,Q4,8192,51.06,4.0,118.3,160.4,2.67,practical,3.9,muladhara+svadhisthana
Llama-3.1-405B-Instruct,TP2,2,Q4,16384,48.11,3.89,120.5,340.6,5.68,practical,7.9,muladhara+svadhisthana
Llama-3.1-405B-Instruct,TP2,2,Q4,32768,44.31,3.72,124.5,739.5,12.33,batch_only,15.8,muladhara+svadhisthana
Llama-3.1-405B-Instruct,TP2,2,Q4,65536,39.23,3.29,133.0,1670.6,27.84,batch_only,31.5,muladhara+svadhisthana
Llama-3.1-405B-Instruct,TP2,2,Q4,131072,31.84,2.7,150.0,4116.6,68.61,batch_only,63.0,muladhara+svadhisthana
Llama-3.1-405B-Instruct,TP2,2,Q5,1024,50.2,3.64,144.1,20.4,0.34,practical,0.5,muladhara+svadhisthana
Llama-3.1-405B-Instruct,TP2,2,Q5,16384,46.23,3.39,146.1,354.4,5.91,practical,7.9,muladhara+svadhisthana
Llama-3.1-405B-Instruct,TP2,2,Q6,1024,49.97,3.2,170.3,20.5,0.34,practical,0.5,muladhara+svadhisthana
Llama-3.1-405B-Instruct,TP2,2,Q6,16384,45.39,3.09,171.7,361.0,6.02,practical,7.9,muladhara+svadhisthana
Llama-3.1-405B-Instruct,TP4,4,Q2,1024,95.59,8.43,35.4,10.7,0.18,practical,0.5,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q2,16384,89.67,7.73,35.5,182.7,3.05,practical,7.9,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q2,65536,74.13,6.13,41.8,884.1,14.73,batch_only,31.5,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q3,1024,95.16,7.21,48.8,10.8,0.18,practical,0.5,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q3,16384,92.68,6.69,48.8,176.8,2.95,practical,7.9,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q3,65536,72.56,5.49,54.9,903.2,15.05,batch_only,31.5,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q4,1024,95.08,6.44,61.9,10.8,0.18,practical,0.5,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q4,4096,96.61,6.31,61.9,42.4,0.71,practical,2.0,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q4,8192,93.03,6.19,61.9,88.1,1.47,practical,3.9,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q4,16384,91.87,5.98,61.9,178.3,2.97,practical,7.9,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q4,32768,84.41,5.63,63.8,388.2,6.47,practical,15.8,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q4,65536,74.71,5.04,68.0,877.2,14.62,batch_only,31.5,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q4,131072,60.75,4.3,76.4,2157.6,35.96,batch_only,63.0,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q5,1024,92.53,5.71,75.4,11.1,0.18,practical,0.5,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q5,16384,90.51,5.35,75.4,181.0,3.02,practical,7.9,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q5,65536,70.18,4.58,81.0,933.8,15.56,batch_only,31.5,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q6,1024,92.05,5.14,89.1,11.1,0.19,practical,0.5,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q6,16384,89.71,4.82,89.1,182.6,3.04,practical,7.9,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,TP4,4,Q6,65536,70.14,4.23,94.1,934.4,15.57,batch_only,31.5,muladhara+svadhisthana+anahata+manipura
Llama-3.1-405B-Instruct,PP4,4,Q4,1024,,,,,,gpu_timeout,,Metal GPU timeout — 405B layers too heavy for PP
Llama-3.1-405B-Instruct,PP4,4,Q4,4096,,,,,,gpu_timeout,,Metal GPU timeout — 405B layers too heavy for PP
Llama-3.1-405B-Instruct,PP4,4,Q4,16384,,,,,,gpu_timeout,,Metal GPU timeout — 405B layers too heavy for PP