{"generated_from":"results/processed","devices":[{"gpu":"A100","cc":"sm80","arch":"Ampere","sm":108,"mem_gb":85.09,"l2_mb":41.943,"bw_gbs":1732.8,"event_us":9.552,"reps":5},{"gpu":"A10","cc":"sm86","arch":"Ampere","sm":72,"mem_gb":23.69,"l2_mb":6.291,"bw_gbs":483.3,"event_us":10.272,"reps":5},{"gpu":"L40","cc":"sm89","arch":"Ada","sm":142,"mem_gb":47.67,"l2_mb":100.663,"bw_gbs":644.4,"event_us":6.624,"reps":2},{"gpu":"L40S","cc":"sm89","arch":"Ada","sm":142,"mem_gb":47.68,"l2_mb":100.663,"bw_gbs":644.4,"event_us":6.656,"reps":5},{"gpu":"H100","cc":"sm90","arch":"Hopper","sm":132,"mem_gb":85.02,"l2_mb":52.429,"bw_gbs":2979.7,"event_us":5.088,"reps":5},{"gpu":"RTX 5090","cc":"sm120","arch":"Blackwell","sm":170,"mem_gb":33.67,"l2_mb":100.663,"bw_gbs":1517.5,"event_us":2.08,"reps":5}],"software":{"torch":"2.9.0+cu128","cuda":"12.8","triton":"3.5.0","flash_attn":"2.8.3","flashinfer":"0.6.18","flash_attn_interface":null},"headline":{"cells":1650,"cells_note":"forward prefill + decode GPU/configuration observations","rows":73230,"attempt_records":73230,"ok_rows":54213,"eligible_rows":54098,"devices":6,"dispatch_mismatch_pct":0.449,"probed_pct":66.504,"selector_median_regret":1.018,"selector_p95_regret":1.328},"separation":{"by_regime":{"decode":{"n":1203,"separated":0.34,"median_ratio":1.062},"prefill":{"n":447,"separated":0.286,"median_ratio":1.051}},"population":"forward prefill and decode, forward-plus-backward excluded","practical_threshold":1.1},"pairs":[{"regime":"decode","a":"A100","b":"A10","n":191,"flip":0.419,"n_sep":40,"flip_sep":0.05,"spearman":null},{"regime":"decode","a":"A100","b":"L40","n":192,"flip":0.536,"n_sep":35,"flip_sep":0.029,"spearman":null},{"regime":"decode","a":"A100","b":"L40S","n":192,"flip":0.688,"n_sep":15,"flip_sep":0.067,"spearman":null},{"regime":"decode","a":"A100","b":"H100","n":192,"flip":0.266,"n_sep":63,"flip_sep":0.127,"spearman":null},{"regime":"decode","a":"A100","b":"RTX 5090","n":192,"flip":0.714,"n_sep":18,"flip_sep":0.667,"spearman":null},{"regime":"decode","a":"A10","b":"L40","n":191,"flip":0.482,"n_sep":36,"flip_sep":0.0,"spearman":null},{"regime":"decode","a":"A10","b":"L40S","n":191,"flip":0.654,"n_sep":15,"flip_sep":0.0,"spearman":null},{"regime":"decode","a":"A10","b":"H100","n":191,"flip":0.497,"n_sep":20,"flip_sep":0.2,"spearman":null},{"regime":"decode","a":"A10","b":"RTX 5090","n":191,"flip":0.89,"n_sep":19,"flip_sep":0.632,"spearman":null},{"regime":"decode","a":"L40","b":"L40S","n":192,"flip":0.323,"n_sep":23,"flip_sep":0.043,"spearman":null},{"regime":"decode","a":"L40","b":"H100","n":192,"flip":0.635,"n_sep":27,"flip_sep":0.333,"spearman":null},{"regime":"decode","a":"L40","b":"RTX 5090","n":192,"flip":0.828,"n_sep":25,"flip_sep":0.72,"spearman":null},{"regime":"decode","a":"L40S","b":"H100","n":192,"flip":0.708,"n_sep":15,"flip_sep":0.2,"spearman":null},{"regime":"decode","a":"L40S","b":"RTX 5090","n":192,"flip":0.786,"n_sep":9,"flip_sep":0.889,"spearman":null},{"regime":"decode","a":"H100","b":"RTX 5090","n":192,"flip":0.703,"n_sep":14,"flip_sep":0.929,"spearman":null},{"regime":"prefill","a":"A100","b":"A10","n":72,"flip":0.722,"n_sep":7,"flip_sep":0.143,"spearman":null},{"regime":"prefill","a":"A100","b":"L40","n":72,"flip":0.653,"n_sep":7,"flip_sep":0.0,"spearman":null},{"regime":"prefill","a":"A100","b":"L40S","n":72,"flip":0.778,"n_sep":1,"flip_sep":1.0,"spearman":null},{"regime":"prefill","a":"A100","b":"H100","n":72,"flip":0.875,"n_sep":4,"flip_sep":1.0,"spearman":null},{"regime":"prefill","a":"A100","b":"RTX 5090","n":72,"flip":0.514,"n_sep":3,"flip_sep":0.667,"spearman":null},{"regime":"prefill","a":"A10","b":"L40","n":72,"flip":0.417,"n_sep":5,"flip_sep":0.0,"spearman":null},{"regime":"prefill","a":"A10","b":"L40S","n":72,"flip":0.5,"n_sep":2,"flip_sep":0.5,"spearman":null},{"regime":"prefill","a":"A10","b":"H100","n":72,"flip":0.653,"n_sep":10,"flip_sep":0.3,"spearman":null},{"regime":"prefill","a":"A10","b":"RTX 5090","n":72,"flip":0.611,"n_sep":4,"flip_sep":0.25,"spearman":null},{"regime":"prefill","a":"L40","b":"L40S","n":72,"flip":0.431,"n_sep":8,"flip_sep":0.125,"spearman":null},{"regime":"prefill","a":"L40","b":"H100","n":72,"flip":0.861,"n_sep":8,"flip_sep":1.0,"spearman":null},{"regime":"prefill","a":"L40","b":"RTX 5090","n":72,"flip":0.694,"n_sep":4,"flip_sep":0.75,"spearman":null},{"regime":"prefill","a":"L40S","b":"H100","n":72,"flip":0.708,"n_sep":12,"flip_sep":0.667,"spearman":null},{"regime":"prefill","a":"L40S","b":"RTX 5090","n":72,"flip":0.722,"n_sep":2,"flip_sep":1.0,"spearman":null},{"regime":"prefill","a":"H100","b":"RTX 5090","n":72,"flip":0.75,"n_sep":17,"flip_sep":0.471,"spearman":null}],"winners":{"decode":{"A10":{"D4-flashinfer":128,"D0-naive-kv":22,"D2-sdpa":21,"D3-fa-kvcache":20},"A100":{"D4-flashinfer":133,"D3-fa-kvcache":59},"H100":{"D4-flashinfer":131,"D3-fa-kvcache":78,"D6-fa-prefill-at-1":29,"D2-sdpa":5,"D0-naive-kv":1},"L40":{"D4-flashinfer":87,"D2-sdpa":73,"D3-fa-kvcache":27,"D0-naive-kv":5},"L40S":{"D2-sdpa":106,"D4-flashinfer":49,"D3-fa-kvcache":37},"RTX 5090":{"D3-fa-kvcache":137,"D0-naive-kv":38,"D2-sdpa":17}},"prefill":{"A10":{"P2d-sdpa-cudnn":38,"P3-triton":22,"P2b-sdpa-mem-eff":10,"P2c-sdpa-flash":2},"A100":{"P4-fa2":35,"P2c-sdpa-flash":14,"P2b-sdpa-mem-eff":12,"P2d-sdpa-cudnn":11},"H100":{"P4h-fa3":48,"P2d-sdpa-cudnn":37,"P2b-sdpa-mem-eff":2},"L40":{"P2c-sdpa-flash":27,"P3-triton":20,"P2d-sdpa-cudnn":14,"P2b-sdpa-mem-eff":9,"P4-fa2":2},"L40S":{"P3-triton":24,"P2d-sdpa-cudnn":21,"P2c-sdpa-flash":19,"P2b-sdpa-mem-eff":8},"RTX 5090":{"P2d-sdpa-cudnn":40,"P2c-sdpa-flash":15,"P4-fa2":13,"P2b-sdpa-mem-eff":2,"P3-triton":2}}},"coverage":[{"gpu":"A10","regime":"decode","implementation":"D0-naive-kv","status":"OK","n":955},{"gpu":"A10","regime":"decode","implementation":"D0-naive-kv","status":"OOM","n":5},{"gpu":"A10","regime":"decode","implementation":"D1-inductor","status":"OK","n":30},{"gpu":"A10","regime":"decode","implementation":"D1-inductor","status":"UNSUPPORTED","n":930},{"gpu":"A10","regime":"decode","implementation":"D2-sdpa","status":"OK","n":955},{"gpu":"A10","regime":"decode","implementation":"D2-sdpa","status":"OOM","n":5},{"gpu":"A10","regime":"decode","implementation":"D3-fa-kvcache","status":"OK","n":945},{"gpu":"A10","regime":"decode","implementation":"D3-fa-kvcache","status":"OOM","n":15},{"gpu":"A10","regime":"decode","implementation":"D4-flashinfer","status":"OK","n":945},{"gpu":"A10","regime":"decode","implementation":"D4-flashinfer","status":"OOM","n":15},{"gpu":"A10","regime":"decode","implementation":"D6-fa-prefill-at-1","status":"OK","n":945},{"gpu":"A10","regime":"decode","implementation":"D6-fa-prefill-at-1","status":"OOM","n":15},{"gpu":"A10","regime":"prefill","implementation":"P0-naive","status":"OK","n":480},{"gpu":"A10","regime":"prefill","implementation":"P0-naive","status":"OOM_PREDICTED","n":240},{"gpu":"A10","regime":"prefill","implementation":"P1-inductor","status":"OK","n":480},{"gpu":"A10","regime":"prefill","implementation":"P1-inductor","status":"OOM_PREDICTED","n":240},{"gpu":"A10","regime":"prefill","implementation":"P1-inductor-nofuse","status":"OK","n":10},{"gpu":"A10","regime":"prefill","implementation":"P1-inductor-nofuse","status":"OOM_PREDICTED","n":5},{"gpu":"A10","regime":"prefill","implementation":"P1-inductor-nofuse","status":"UNSUPPORTED","n":705},{"gpu":"A10","regime":"prefill","implementation":"P1-inductor-where","status":"OK","n":10},{"gpu":"A10","regime":"prefill","implementation":"P1-inductor-where","status":"OOM_PREDICTED","n":5},{"gpu":"A10","regime":"prefill","implementation":"P1-inductor-where","status":"UNSUPPORTED","n":705},{"gpu":"A10","regime":"prefill","implementation":"P2b-sdpa-mem-eff","status":"OK","n":720},{"gpu":"A10","regime":"prefill","implementation":"P2c-sdpa-flash","status":"OK","n":720},{"gpu":"A10","regime":"prefill","implementation":"P2d-sdpa-cudnn","status":"OK","n":720},{"gpu":"A10","regime":"prefill","implementation":"P3-triton","status":"OK","n":720},{"gpu":"A10","regime":"prefill","implementation":"P4-fa2","status":"OK","n":720},{"gpu":"A10","regime":"prefill","implementation":"P4h-fa3","status":"UNSUPPORTED","n":720},{"gpu":"A100","regime":"decode","implementation":"D0-naive-kv","status":"OK","n":960},{"gpu":"A100","regime":"decode","implementation":"D1-inductor","status":"OK","n":30},{"gpu":"A100","regime":"decode","implementation":"D1-inductor","status":"UNSUPPORTED","n":930},{"gpu":"A100","regime":"decode","implementation":"D2-sdpa","status":"OK","n":960},{"gpu":"A100","regime":"decode","implementation":"D3-fa-kvcache","status":"OK","n":960},{"gpu":"A100","regime":"decode","implementation":"D4-flashinfer","status":"OK","n":960},{"gpu":"A100","regime":"decode","implementation":"D6-fa-prefill-at-1","status":"OK","n":960},{"gpu":"A100","regime":"prefill","implementation":"P0-naive","status":"OK","n":600},{"gpu":"A100","regime":"prefill","implementation":"P0-naive","status":"OOM_PREDICTED","n":120},{"gpu":"A100","regime":"prefill","implementation":"P1-inductor","status":"OK","n":600},{"gpu":"A100","regime":"prefill","implementation":"P1-inductor","status":"OOM_PREDICTED","n":120},{"gpu":"A100","regime":"prefill","implementation":"P1-inductor-nofuse","status":"OK","n":15},{"gpu":"A100","regime":"prefill","implementation":"P1-inductor-nofuse","status":"UNSUPPORTED","n":705},{"gpu":"A100","regime":"prefill","implementation":"P1-inductor-where","status":"OK","n":15},{"gpu":"A100","regime":"prefill","implementation":"P1-inductor-where","status":"UNSUPPORTED","n":705},{"gpu":"A100","regime":"prefill","implementation":"P2b-sdpa-mem-eff","status":"OK","n":720},{"gpu":"A100","regime":"prefill","implementation":"P2c-sdpa-flash","status":"OK","n":720},{"gpu":"A100","regime":"prefill","implementation":"P2d-sdpa-cudnn","status":"OK","n":720},{"gpu":"A100","regime":"prefill","implementation":"P3-triton","status":"OK","n":720},{"gpu":"A100","regime":"prefill","implementation":"P4-fa2","status":"OK","n":720},{"gpu":"A100","regime":"prefill","implementation":"P4h-fa3","status":"UNSUPPORTED","n":720},{"gpu":"H100","regime":"decode","implementation":"D0-naive-kv","status":"OK","n":1220},{"gpu":"H100","regime":"decode","implementation":"D1-inductor","status":"OK","n":60},{"gpu":"H100","regime":"decode","implementation":"D1-inductor","status":"UNSUPPORTED","n":1160},{"gpu":"H100","regime":"decode","implementation":"D2-sdpa","status":"OK","n":1220},{"gpu":"H100","regime":"decode","implementation":"D3-fa-kvcache","status":"OK","n":1220},{"gpu":"H100","regime":"decode","implementation":"D4-flashinfer","status":"OK","n":1220},{"gpu":"H100","regime":"decode","implementation":"D6-fa-prefill-at-1","status":"OK","n":1220},{"gpu":"H100","regime":"prefill","implementation":"P0-naive","status":"OK","n":675},{"gpu":"H100","regime":"prefill","implementation":"P0-naive","status":"OOM_PREDICTED","n":120},{"gpu":"H100","regime":"prefill","implementation":"P1-inductor","status":"OK","n":675},{"gpu":"H100","regime":"prefill","implementation":"P1-inductor","status":"OOM_PREDICTED","n":120},{"gpu":"H100","regime":"prefill","implementation":"P1-inductor-nofuse","status":"OK","n":60},{"gpu":"H100","regime":"prefill","implementation":"P1-inductor-nofuse","status":"UNSUPPORTED","n":735},{"gpu":"H100","regime":"prefill","implementation":"P1-inductor-where","status":"OK","n":60},{"gpu":"H100","regime":"prefill","implementation":"P1-inductor-where","status":"UNSUPPORTED","n":735},{"gpu":"H100","regime":"prefill","implementation":"P2b-sdpa-mem-eff","status":"OK","n":765},{"gpu":"H100","regime":"prefill","implementation":"P2b-sdpa-mem-eff","status":"UNSUPPORTED","n":30},{"gpu":"H100","regime":"prefill","implementation":"P2c-sdpa-flash","status":"OK","n":795},{"gpu":"H100","regime":"prefill","implementation":"P2d-sdpa-cudnn","status":"OK","n":795},{"gpu":"H100","regime":"prefill","implementation":"P3-triton","status":"OK","n":795},{"gpu":"H100","regime":"prefill","implementation":"P4-fa2","status":"OK","n":795},{"gpu":"H100","regime":"prefill","implementation":"P4h-fa3","status":"OK","n":795},{"gpu":"L40","regime":"decode","implementation":"D0-naive-kv","status":"OK","n":480},{"gpu":"L40","regime":"decode","implementation":"D1-inductor","status":"OK","n":14},{"gpu":"L40","regime":"decode","implementation":"D1-inductor","status":"UNSUPPORTED","n":466},{"gpu":"L40","regime":"decode","implementation":"D2-sdpa","status":"OK","n":480},{"gpu":"L40","regime":"decode","implementation":"D3-fa-kvcache","status":"OK","n":480},{"gpu":"L40","regime":"decode","implementation":"D4-flashinfer","status":"OK","n":480},{"gpu":"L40","regime":"decode","implementation":"D6-fa-prefill-at-1","status":"OK","n":480},{"gpu":"L40","regime":"prefill","implementation":"P0-naive","status":"OK","n":271},{"gpu":"L40","regime":"prefill","implementation":"P0-naive","status":"OOM","n":3},{"gpu":"L40","regime":"prefill","implementation":"P0-naive","status":"OOM_PREDICTED","n":50},{"gpu":"L40","regime":"prefill","implementation":"P1-inductor","status":"OK","n":274},{"gpu":"L40","regime":"prefill","implementation":"P1-inductor","status":"OOM_PREDICTED","n":50},{"gpu":"L40","regime":"prefill","implementation":"P1-inductor-nofuse","status":"OK","n":6},{"gpu":"L40","regime":"prefill","implementation":"P1-inductor-nofuse","status":"UNSUPPORTED","n":318},{"gpu":"L40","regime":"prefill","implementation":"P1-inductor-where","status":"OK","n":6},{"gpu":"L40","regime":"prefill","implementation":"P1-inductor-where","status":"UNSUPPORTED","n":318},{"gpu":"L40","regime":"prefill","implementation":"P2b-sdpa-mem-eff","status":"OK","n":324},{"gpu":"L40","regime":"prefill","implementation":"P2c-sdpa-flash","status":"OK","n":324},{"gpu":"L40","regime":"prefill","implementation":"P2d-sdpa-cudnn","status":"OK","n":324},{"gpu":"L40","regime":"prefill","implementation":"P3-triton","status":"OK","n":324},{"gpu":"L40","regime":"prefill","implementation":"P4-fa2","status":"OK","n":324},{"gpu":"L40","regime":"prefill","implementation":"P4h-fa3","status":"UNSUPPORTED","n":324},{"gpu":"L40S","regime":"decode","implementation":"D0-naive-kv","status":"OK","n":960},{"gpu":"L40S","regime":"decode","implementation":"D1-inductor","status":"OK","n":30},{"gpu":"L40S","regime":"decode","implementation":"D1-inductor","status":"UNSUPPORTED","n":930},{"gpu":"L40S","regime":"decode","implementation":"D2-sdpa","status":"OK","n":960},{"gpu":"L40S","regime":"decode","implementation":"D3-fa-kvcache","status":"OK","n":960},{"gpu":"L40S","regime":"decode","implementation":"D4-flashinfer","status":"OK","n":960},{"gpu":"L40S","regime":"decode","implementation":"D6-fa-prefill-at-1","status":"OK","n":960},{"gpu":"L40S","regime":"prefill","implementation":"P0-naive","status":"OK","n":594},{"gpu":"L40S","regime":"prefill","implementation":"P0-naive","status":"OOM","n":6},{"gpu":"L40S","regime":"prefill","implementation":"P0-naive","status":"OOM_PREDICTED","n":120},{"gpu":"L40S","regime":"prefill","implementation":"P1-inductor","status":"OK","n":600},{"gpu":"L40S","regime":"prefill","implementation":"P1-inductor","status":"OOM_PREDICTED","n":120},{"gpu":"L40S","regime":"prefill","implementation":"P1-inductor-nofuse","status":"OK","n":15},{"gpu":"L40S","regime":"prefill","implementation":"P1-inductor-nofuse","status":"UNSUPPORTED","n":705},{"gpu":"L40S","regime":"prefill","implementation":"P1-inductor-where","status":"OK","n":15},{"gpu":"L40S","regime":"prefill","implementation":"P1-inductor-where","status":"UNSUPPORTED","n":705},{"gpu":"L40S","regime":"prefill","implementation":"P2b-sdpa-mem-eff","status":"OK","n":720},{"gpu":"L40S","regime":"prefill","implementation":"P2c-sdpa-flash","status":"OK","n":720},{"gpu":"L40S","regime":"prefill","implementation":"P2d-sdpa-cudnn","status":"OK","n":720},{"gpu":"L40S","regime":"prefill","implementation":"P3-triton","status":"OK","n":720},{"gpu":"L40S","regime":"prefill","implementation":"P4-fa2","status":"OK","n":720},{"gpu":"L40S","regime":"prefill","implementation":"P4h-fa3","status":"UNSUPPORTED","n":720},{"gpu":"RTX 5090","regime":"decode","implementation":"D0-naive-kv","status":"OK","n":955},{"gpu":"RTX 5090","regime":"decode","implementation":"D0-naive-kv","status":"OOM","n":5},{"gpu":"RTX 5090","regime":"decode","implementation":"D1-inductor","status":"OK","n":30},{"gpu":"RTX 5090","regime":"decode","implementation":"D1-inductor","status":"UNSUPPORTED","n":930},{"gpu":"RTX 5090","regime":"decode","implementation":"D2-sdpa","status":"OK","n":960},{"gpu":"RTX 5090","regime":"decode","implementation":"D3-fa-kvcache","status":"OK","n":950},{"gpu":"RTX 5090","regime":"decode","implementation":"D3-fa-kvcache","status":"OOM","n":10},{"gpu":"RTX 5090","regime":"decode","implementation":"D4-flashinfer","status":"ERROR","n":950},{"gpu":"RTX 5090","regime":"decode","implementation":"D4-flashinfer","status":"OOM","n":10},{"gpu":"RTX 5090","regime":"decode","implementation":"D6-fa-prefill-at-1","status":"OK","n":950},{"gpu":"RTX 5090","regime":"decode","implementation":"D6-fa-prefill-at-1","status":"OOM","n":10},{"gpu":"RTX 5090","regime":"prefill","implementation":"P0-naive","status":"OK","n":528},{"gpu":"RTX 5090","regime":"prefill","implementation":"P0-naive","status":"OOM","n":52},{"gpu":"RTX 5090","regime":"prefill","implementation":"P0-naive","status":"OOM_PREDICTED","n":140},{"gpu":"RTX 5090","regime":"prefill","implementation":"P1-inductor","status":"OK","n":580},{"gpu":"RTX 5090","regime":"prefill","implementation":"P1-inductor","status":"OOM_PREDICTED","n":140},{"gpu":"RTX 5090","regime":"prefill","implementation":"P1-inductor-nofuse","status":"OK","n":15},{"gpu":"RTX 5090","regime":"prefill","implementation":"P1-inductor-nofuse","status":"UNSUPPORTED","n":705},{"gpu":"RTX 5090","regime":"prefill","implementation":"P1-inductor-where","status":"OK","n":15},{"gpu":"RTX 5090","regime":"prefill","implementation":"P1-inductor-where","status":"UNSUPPORTED","n":705},{"gpu":"RTX 5090","regime":"prefill","implementation":"P2b-sdpa-mem-eff","status":"OK","n":720},{"gpu":"RTX 5090","regime":"prefill","implementation":"P2c-sdpa-flash","status":"OK","n":720},{"gpu":"RTX 5090","regime":"prefill","implementation":"P2d-sdpa-cudnn","status":"OK","n":720},{"gpu":"RTX 5090","regime":"prefill","implementation":"P3-triton","status":"OK","n":720},{"gpu":"RTX 5090","regime":"prefill","implementation":"P4-fa2","status":"OK","n":720},{"gpu":"RTX 5090","regime":"prefill","implementation":"P4h-fa3","status":"UNSUPPORTED","n":720}],"dispatch":{"flows":[{"impl":"D0-naive-kv","observed":"unfused-gemm","n":5509},{"impl":"D0-naive-kv","observed":"unknown","n":21},{"impl":"D1-inductor","observed":"triton","n":187},{"impl":"D1-inductor","observed":"unprobed","n":7},{"impl":"D2-sdpa","observed":"pytorch-flash","n":4525},{"impl":"D2-sdpa","observed":"unprobed","n":1010},{"impl":"D3-fa-kvcache","observed":"flash-attn","n":4516},{"impl":"D3-fa-kvcache","observed":"unprobed","n":999},{"impl":"D4-flashinfer","observed":"flashinfer","n":3489},{"impl":"D4-flashinfer","observed":"unprobed","n":1076},{"impl":"D6-fa-prefill-at-1","observed":"flash-attn","n":4531},{"impl":"D6-fa-prefill-at-1","observed":"unprobed","n":984},{"impl":"P0-naive","observed":"unfused-gemm","n":2792},{"impl":"P0-naive","observed":"unknown","n":41},{"impl":"P0-naive","observed":"unprobed","n":315},{"impl":"P1-inductor","observed":"pytorch-flash","n":30},{"impl":"P1-inductor","observed":"triton","n":2047},{"impl":"P1-inductor","observed":"unfused-gemm","n":4},{"impl":"P1-inductor","observed":"unknown","n":32},{"impl":"P1-inductor","observed":"unprobed","n":1096},{"impl":"P1-inductor-nofuse","observed":"triton","n":88},{"impl":"P1-inductor-nofuse","observed":"unprobed","n":33},{"impl":"P1-inductor-where","observed":"pytorch-flash","n":15},{"impl":"P1-inductor-where","observed":"triton","n":73},{"impl":"P1-inductor-where","observed":"unprobed","n":33},{"impl":"P2b-sdpa-mem-eff","observed":"cutlass-fmha","n":1645},{"impl":"P2b-sdpa-mem-eff","observed":"unknown","n":110},{"impl":"P2b-sdpa-mem-eff","observed":"unprobed","n":2214},{"impl":"P2c-sdpa-flash","observed":"pytorch-flash","n":1650},{"impl":"P2c-sdpa-flash","observed":"unprobed","n":2349},{"impl":"P2d-sdpa-cudnn","observed":"cudnn","n":1988},{"impl":"P2d-sdpa-cudnn","observed":"unprobed","n":2011},{"impl":"P3-triton","observed":"triton","n":1591},{"impl":"P3-triton","observed":"unprobed","n":2408},{"impl":"P4-fa2","observed":"flash-attn","n":932},{"impl":"P4-fa2","observed":"unprobed","n":3067},{"impl":"P4h-fa3","observed":"flash-attn","n":238},{"impl":"P4h-fa3","observed":"unprobed","n":557}],"by_gpu":{"A10":{"probed":56.312,"mismatch":0.285},"A100":{"probed":67.267,"mismatch":0.077},"H100":{"probed":75.804,"mismatch":0.821},"L40":{"probed":56.545,"mismatch":0.462},"L40S":{"probed":80.868,"mismatch":0.333},"RTX 5090":{"probed":52.522,"mismatch":0.599}},"inductor_counter_fired":0,"inductor_rows":3645,"attn_kernel_in_trace":45},"correctness":[{"impl":"D0-naive-kv","n":355,"max_abs":0.007,"max_rel":474.434,"min_cos":1.0,"vs_baseline":1.0,"fail":0},{"impl":"D1-inductor","n":50,"max_abs":0.001,"max_rel":0.015,"min_cos":1.0,"vs_baseline":0.484,"fail":0},{"impl":"D2-sdpa","n":360,"max_abs":0.001,"max_rel":220.787,"min_cos":1.0,"vs_baseline":0.739,"fail":0},{"impl":"D3-fa-kvcache","n":360,"max_abs":0.001,"max_rel":220.787,"min_cos":1.0,"vs_baseline":0.739,"fail":0},{"impl":"D4-flashinfer","n":320,"max_abs":0.001,"max_rel":0.084,"min_cos":1.0,"vs_baseline":0.599,"fail":0},{"impl":"D6-fa-prefill-at-1","n":360,"max_abs":0.001,"max_rel":220.787,"min_cos":1.0,"vs_baseline":0.739,"fail":0},{"impl":"P0-naive","n":415,"max_abs":0.025,"max_rel":3203.779,"min_cos":1.0,"vs_baseline":1.0,"fail":0},{"impl":"P1-inductor","n":415,"max_abs":0.02,"max_rel":2517.196,"min_cos":1.0,"vs_baseline":1.081,"fail":0},{"impl":"P1-inductor-nofuse","n":49,"max_abs":0.02,"max_rel":1571.193,"min_cos":1.0,"vs_baseline":1.069,"fail":0},{"impl":"P1-inductor-where","n":49,"max_abs":0.02,"max_rel":1571.193,"min_cos":1.0,"vs_baseline":1.069,"fail":0},{"impl":"P2b-sdpa-mem-eff","n":405,"max_abs":0.012,"max_rel":2060.056,"min_cos":1.0,"vs_baseline":0.775,"fail":0},{"impl":"P2c-sdpa-flash","n":415,"max_abs":0.012,"max_rel":2060.056,"min_cos":1.0,"vs_baseline":0.775,"fail":0},{"impl":"P2d-sdpa-cudnn","n":415,"max_abs":0.012,"max_rel":2060.056,"min_cos":1.0,"vs_baseline":0.775,"fail":0},{"impl":"P3-triton","n":415,"max_abs":0.012,"max_rel":2060.056,"min_cos":1.0,"vs_baseline":0.775,"fail":0},{"impl":"P4-fa2","n":415,"max_abs":0.012,"max_rel":2060.056,"min_cos":1.0,"vs_baseline":0.775,"fail":0},{"impl":"P4h-fa3","n":65,"max_abs":0.011,"max_rel":792.961,"min_cos":1.0,"vs_baseline":0.664,"fail":0}],"prefill_slice":{"batch":4,"head_dim":128,"dtype":"bf16","hq":32,"hkv":32,"mode":"fwd","launch":"eager","cache":"warm","causal":true},"prefill_latency":{"A10":{"P0-naive":{"x":[256,512,1024,2048],"y":[781.008,2807.424,10838.848,43078.88]},"P1-inductor":{"x":[256,512,1024,2048],"y":[297.392,791.568,2625.92,11648.656]},"P1-inductor-nofuse":{"x":[512,2048],"y":[790.368,12402.256]},"P1-inductor-where":{"x":[512,2048],"y":[796.8,12264.576]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[112.384,309.568,1160.912,3972.928,14568.816,55807.264]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[152.496,262.736,657.008,2576.4,9496.0,40406.256]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[107.936,213.264,636.672,2403.888,8667.152,32273.008]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[213.952,331.2,808.912,2536.288,8530.16,32156.736]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[178.544,313.984,676.944,2644.304,9965.952,42793.713]}},"A100":{"P0-naive":{"x":[256,512,1024,2048,4096],"y":[293.008,1001.872,3620.384,14180.096,59714.161]},"P1-inductor":{"x":[256,512,1024,2048,4096],"y":[162.08,315.552,901.968,4110.672,16872.607]},"P1-inductor-nofuse":{"x":[512,2048,4096],"y":[314.048,3647.808,14737.872]},"P1-inductor-where":{"x":[512,2048,4096],"y":[314.416,4132.048,16310.207]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[71.456,180.704,456.672,1495.968,5434.0,20742.56]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[82.912,115.392,299.232,873.824,3098.896,11501.28]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[86.904,107.576,289.52,906.048,3275.488,12296.96]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[173.504,230.352,413.44,1061.056,3511.392,12860.048]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[109.816,164.928,312.48,870.576,2994.512,11032.432]}},"RTX 5090":{"P0-naive":{"x":[256,512,1024,2048,4096],"y":[132.992,760.24,3610.416,13887.552,54426.704]},"P1-inductor":{"x":[256,512,1024,2048,4096],"y":[43.58,140.576,797.856,2974.464,11569.088]},"P1-inductor-nofuse":{"x":[512,2048,4096],"y":[141.968,2938.192,11514.528]},"P1-inductor-where":{"x":[512,2048,4096],"y":[140.72,2971.504,11525.36]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[50.048,147.696,468.912,1644.72,6139.392,23718.096]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[35.053,81.936,243.184,779.136,2789.808,10585.568]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[26.19,63.675,205.712,678.384,2497.648,9710.944]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[59.888,106.84,299.024,860.528,2940.528,10806.432]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[35.635,82.312,244.48,772.8,2762.352,10425.296]}},"H100":{"P0-naive":{"x":[256,512,1024,2048,4096],"y":[289.968,607.232,2117.328,8429.12,36609.552]},"P1-inductor":{"x":[256,512,1024,2048,4096],"y":[263.92,256.544,525.488,2346.56,9176.736]},"P1-inductor-nofuse":{"x":[512,2048,4096],"y":[274.8,2114.224,7317.12]},"P1-inductor-where":{"x":[512,2048,4096],"y":[280.672,2345.056,9176.832]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[101.248,166.016,340.192,1105.168,3733.616,14115.504]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[121.424,143.552,233.776,618.176,2001.872,7139.488]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[36.202,46.18,125.136,333.6,1082.528,3662.976]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[291.584,279.088,295.936,490.624,1320.144,4509.504]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[181.344,157.536,227.168,585.888,1919.28,6634.336]},"P4h-fa3":{"x":[256,512,1024,2048,4096,8192],"y":[84.128,124.208,108.296,290.272,909.84,3311.264]}},"L40":{"P0-naive":{"x":[256,512,1024,2048,4096],"y":[239.552,1602.256,7724.464,30146.696,117889.479]},"P1-inductor":{"x":[256,512,1024,2048,4096],"y":[162.464,273.912,1889.984,7093.184,32128.72]},"P1-inductor-nofuse":{"x":[512,2048,4096],"y":[275.76,7094.36,31399.256]},"P1-inductor-where":{"x":[512,2048,4096],"y":[273.872,7036.64,31923.84]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[49.378,146.656,449.856,1798.256,6344.784,25549.584]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[47.58,94.008,327.088,1311.168,4696.864,19771.992]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[69.548,95.204,318.272,1356.8,4782.328,19868.344]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[129.104,192.008,412.232,1256.472,4292.296,16666.008]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[98.024,101.24,342.848,1131.168,4728.8,19746.208]}},"L40S":{"P0-naive":{"x":[256,512,1024,2048,4096],"y":[333.616,1608.576,7767.392,30222.8,118154.831]},"P1-inductor":{"x":[256,512,1024,2048,4096],"y":[221.872,273.024,1912.736,6839.856,26611.616]},"P1-inductor-nofuse":{"x":[512,2048,4096],"y":[276.832,6827.056,26872.496]},"P1-inductor-where":{"x":[512,2048,4096],"y":[285.424,6885.824,26589.36]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[81.72,119.984,383.2,1283.072,4965.856,19235.472]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[99.72,79.584,296.576,960.736,3611.728,13413.776]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[53.936,80.075,294.0,953.664,3674.256,13462.096]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[216.288,194.72,350.496,921.808,3224.992,11769.68]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[176.432,112.12,310.576,970.32,3607.536,13531.616]}}},"prefill_tflops":{"A10":{"P0-naive":{"x":[256,512,1024,2048],"y":[2.75,3.06,3.17,3.19]},"P1-inductor":{"x":[256,512,1024,2048],"y":[7.221,10.852,13.085,11.799]},"P1-inductor-nofuse":{"x":[512,2048],"y":[10.868,11.082]},"P1-inductor-where":{"x":[512,2048],"y":[10.781,11.206]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[19.108,27.748,29.597,34.594,37.735,39.404]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[14.082,32.694,52.297,53.345,57.893,54.423]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[19.896,40.278,53.968,57.174,63.43,68.138]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[10.037,25.936,42.476,54.189,64.448,68.385]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[12.028,27.358,50.757,51.975,55.163,51.387]}},"A100":{"P0-naive":{"x":[256,512,1024,2048,4096],"y":[7.329,8.574,9.491,9.692,9.206]},"P1-inductor":{"x":[256,512,1024,2048,4096],"y":[13.25,27.222,38.094,33.435,32.583]},"P1-inductor-nofuse":{"x":[512,2048,4096],"y":[27.352,37.677,37.302]},"P1-inductor-where":{"x":[512,2048,4096],"y":[27.32,33.262,33.706]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[30.053,47.536,75.239,91.873,101.17,106.015]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[25.901,74.441,114.826,157.284,177.404,191.198]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[24.711,79.85,118.678,151.691,167.839,178.827]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[12.377,37.29,83.107,129.53,156.563,170.997]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[19.555,52.083,109.958,157.871,183.588,199.324]}},"RTX 5090":{"P0-naive":{"x":[256,512,1024,2048,4096],"y":[16.147,11.299,9.517,9.897,10.101]},"P1-inductor":{"x":[256,512,1024,2048,4096],"y":[49.277,61.105,43.065,46.206,47.519]},"P1-inductor-nofuse":{"x":[512,2048,4096],"y":[60.506,46.777,47.745]},"P1-inductor-where":{"x":[512,2048,4096],"y":[61.043,46.252,47.7]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[42.908,58.16,73.275,83.564,89.546,92.715]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[61.264,104.837,141.291,176.399,197.059,207.738]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[81.997,134.903,167.028,202.598,220.109,226.448]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[35.858,80.4,114.906,159.715,186.958,203.492]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[60.263,104.358,140.542,177.845,199.017,210.931]}},"H100":{"P0-naive":{"x":[256,512,1024,2048,4096],"y":[7.406,14.146,16.228,16.305,15.017]},"P1-inductor":{"x":[256,512,1024,2048,4096],"y":[8.137,33.483,65.386,58.57,59.908]},"P1-inductor-nofuse":{"x":[512,2048,4096],"y":[31.259,65.007,75.133]},"P1-inductor-where":{"x":[512,2048,4096],"y":[30.605,58.608,59.907]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[21.21,51.742,101.001,124.36,147.245,155.788]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[17.686,59.838,146.977,222.33,274.621,308.009]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[59.32,186.01,274.579,411.987,507.844,600.338]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[7.365,30.779,116.105,280.131,416.436,487.642]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[11.842,54.527,151.253,234.582,286.439,331.461]},"P4h-fa3":{"x":[256,512,1024,2048,4096,8192],"y":[25.526,69.158,317.276,473.483,604.234,664.104]}},"L40":{"P0-naive":{"x":[256,512,1024,2048,4096],"y":[8.965,5.361,4.448,4.559,4.663]},"P1-inductor":{"x":[256,512,1024,2048,4096],"y":[14.143,31.361,18.18,19.376,17.111]},"P1-inductor-nofuse":{"x":[512,2048,4096],"y":[31.15,19.373,17.509]},"P1-inductor-where":{"x":[512,2048,4096],"y":[31.365,19.532,17.221]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[43.496,58.572,76.389,76.432,86.651,86.088]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[45.136,91.375,105.058,104.832,117.051,111.226]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[33.648,90.227,107.964,101.318,114.964,110.68]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[16.637,44.738,83.357,109.388,128.086,131.949]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[25.079,84.848,100.232,123.176,116.26,111.367]}},"L40S":{"P0-naive":{"x":[256,512,1024,2048,4096],"y":[6.437,5.34,4.424,4.548,4.653]},"P1-inductor":{"x":[256,512,1024,2048,4096],"y":[9.679,31.462,17.964,20.094,20.658]},"P1-inductor-nofuse":{"x":[512,2048,4096],"y":[31.029,20.132,20.458]},"P1-inductor-where":{"x":[512,2048,4096],"y":[30.095,19.96,20.676]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[26.279,71.592,89.665,107.117,110.707,114.321]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[21.535,107.935,115.855,143.056,152.214,163.938]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[39.815,107.274,116.87,144.117,149.624,163.349]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[9.929,44.114,98.032,149.097,170.467,186.838]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[12.172,76.614,110.632,141.643,152.391,162.51]}}},"prefill_memory":{"A10":{"P0-naive":{"x":[256,512,1024,2048],"y":[128.312,400.5,1425.25,5396.25]},"P1-inductor":{"x":[256,512,1024,2048],"y":[72.312,160.5,433.25,1364.25]},"P1-inductor-nofuse":{"x":[512,2048],"y":[160.5,1364.25]},"P1-inductor-where":{"x":[512,2048],"y":[160.5,1364.25]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[56.25,96.25,176.25,336.25,656.25,1296.25]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[56.376,96.501,176.751,337.251,658.251,1300.251]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[56.251,96.251,176.251,336.251,656.251,1296.251]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[56.375,96.5,176.75,338.25,660.25,1304.25]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[56.375,96.5,176.75,337.25,658.25,1300.25]}},"A100":{"P0-naive":{"x":[256,512,1024,2048,4096],"y":[128.312,400.5,1425.25,5396.25,21024.25]},"P1-inductor":{"x":[256,512,1024,2048,4096],"y":[72.312,160.5,433.25,1364.25,4768.25]},"P1-inductor-nofuse":{"x":[512,2048,4096],"y":[160.5,1364.25,4768.25]},"P1-inductor-where":{"x":[512,2048,4096],"y":[160.5,1364.25,4768.25]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[56.25,96.25,176.25,336.25,656.25,1296.25]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[56.376,96.501,176.751,337.251,658.251,1300.251]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[56.251,96.251,176.251,336.251,656.251,1296.251]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[56.375,96.5,176.75,338.25,660.25,1300.25]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[56.375,96.5,176.75,337.25,658.25,1300.25]}},"RTX 5090":{"P0-naive":{"x":[256,512,1024,2048,4096],"y":[128.312,400.5,1425.25,5396.25,21024.25]},"P1-inductor":{"x":[256,512,1024,2048,4096],"y":[72.312,160.5,433.25,1364.25,4768.25]},"P1-inductor-nofuse":{"x":[512,2048,4096],"y":[160.5,1364.25,4768.25]},"P1-inductor-where":{"x":[512,2048,4096],"y":[160.5,1364.25,4768.25]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[56.25,96.25,176.25,336.25,656.25,1296.25]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[56.376,96.501,176.751,337.251,658.251,1300.251]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[56.251,96.251,176.251,336.251,656.251,1296.251]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[56.375,96.5,176.75,338.25,658.25,1300.25]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[56.375,96.5,176.75,337.25,658.25,1300.25]}},"H100":{"P0-naive":{"x":[256,512,1024,2048,4096],"y":[176.062,448.25,1473.0,5444.0,21072.0]},"P1-inductor":{"x":[256,512,1024,2048,4096],"y":[120.062,208.25,481.0,1412.0,4816.0]},"P1-inductor-nofuse":{"x":[512,2048,4096],"y":[208.25,1412.0,4816.0]},"P1-inductor-where":{"x":[512,2048,4096],"y":[208.25,1412.0,4816.0]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[104.0,144.0,224.0,384.0,704.0,1344.0]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[104.126,144.251,224.501,385.001,706.001,1348.001]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[104.001,144.001,224.001,384.001,704.001,1344.001]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[104.125,144.25,224.5,386.0,708.0,1348.0]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[104.125,144.25,224.5,385.0,706.0,1348.0]},"P4h-fa3":{"x":[256,512,1024,2048,4096,8192],"y":[104.125,144.25,224.5,385.0,706.0,1348.0]}},"L40":{"P0-naive":{"x":[256,512,1024,2048,4096],"y":[128.312,400.5,1425.25,5396.25,21024.25]},"P1-inductor":{"x":[256,512,1024,2048,4096],"y":[72.312,160.5,433.25,1364.25,4768.25]},"P1-inductor-nofuse":{"x":[512,2048,4096],"y":[160.5,1364.25,4768.25]},"P1-inductor-where":{"x":[512,2048,4096],"y":[160.5,1364.25,4768.25]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[56.25,96.25,176.25,336.25,656.25,1296.25]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[56.376,96.501,176.751,337.251,658.251,1300.251]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[56.251,96.251,176.251,336.251,656.251,1296.251]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[56.438,96.5,176.75,337.75,658.25,1302.25]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[56.375,96.5,176.75,337.25,658.25,1300.25]}},"L40S":{"P0-naive":{"x":[256,512,1024,2048,4096],"y":[128.312,400.5,1425.25,5396.25,21024.25]},"P1-inductor":{"x":[256,512,1024,2048,4096],"y":[72.312,160.5,433.25,1364.25,4768.25]},"P1-inductor-nofuse":{"x":[512,2048,4096],"y":[160.5,1364.25,4768.25]},"P1-inductor-where":{"x":[512,2048,4096],"y":[160.5,1364.25,4768.25]},"P2b-sdpa-mem-eff":{"x":[256,512,1024,2048,4096,8192],"y":[56.25,96.25,176.25,336.25,656.25,1296.25]},"P2c-sdpa-flash":{"x":[256,512,1024,2048,4096,8192],"y":[56.376,96.501,176.751,337.251,658.251,1300.251]},"P2d-sdpa-cudnn":{"x":[256,512,1024,2048,4096,8192],"y":[56.251,96.251,176.251,336.251,656.251,1296.251]},"P3-triton":{"x":[256,512,1024,2048,4096,8192],"y":[56.375,96.5,176.75,337.25,660.25,1300.25]},"P4-fa2":{"x":[256,512,1024,2048,4096,8192],"y":[56.375,96.5,176.75,337.25,658.25,1300.25]}}},"prefill_speedup":[{"gpu":"A10","B":1,"N":256,"impl":"P0-naive","speedup":0.352,"us":246.592,"base_us":86.912},{"gpu":"A10","B":1,"N":256,"impl":"P1-inductor","speedup":0.471,"us":184.496,"base_us":86.912},{"gpu":"A10","B":1,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":1.211,"us":71.771,"base_us":86.912},{"gpu":"A10","B":1,"N":256,"impl":"P2d-sdpa-cudnn","speedup":0.935,"us":92.992,"base_us":86.912},{"gpu":"A10","B":1,"N":256,"impl":"P3-triton","speedup":0.522,"us":166.432,"base_us":86.912},{"gpu":"A10","B":1,"N":256,"impl":"P4-fa2","speedup":0.659,"us":131.984,"base_us":86.912},{"gpu":"A10","B":1,"N":512,"impl":"P0-naive","speedup":0.127,"us":746.128,"base_us":94.88},{"gpu":"A10","B":1,"N":512,"impl":"P1-inductor","speedup":0.356,"us":266.272,"base_us":94.88},{"gpu":"A10","B":1,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":0.947,"us":100.184,"base_us":94.88},{"gpu":"A10","B":1,"N":512,"impl":"P2d-sdpa-cudnn","speedup":0.969,"us":97.928,"base_us":94.88},{"gpu":"A10","B":1,"N":512,"impl":"P3-triton","speedup":0.475,"us":199.936,"base_us":94.88},{"gpu":"A10","B":1,"N":512,"impl":"P4-fa2","speedup":0.678,"us":139.92,"base_us":94.88},{"gpu":"A10","B":1,"N":1024,"impl":"P0-naive","speedup":0.076,"us":2770.24,"base_us":210.288},{"gpu":"A10","B":1,"N":1024,"impl":"P1-inductor","speedup":0.282,"us":745.888,"base_us":210.288},{"gpu":"A10","B":1,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.747,"us":281.456,"base_us":210.288},{"gpu":"A10","B":1,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":1.026,"us":205.008,"base_us":210.288},{"gpu":"A10","B":1,"N":1024,"impl":"P3-triton","speedup":0.703,"us":299.136,"base_us":210.288},{"gpu":"A10","B":1,"N":1024,"impl":"P4-fa2","speedup":0.883,"us":238.224,"base_us":210.288},{"gpu":"A10","B":1,"N":2048,"impl":"P0-naive","speedup":0.056,"us":10877.456,"base_us":610.976},{"gpu":"A10","B":1,"N":2048,"impl":"P1-inductor","speedup":0.189,"us":3227.44,"base_us":610.976},{"gpu":"A10","B":1,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.586,"us":1041.744,"base_us":610.976},{"gpu":"A10","B":1,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":1.05,"us":581.856,"base_us":610.976},{"gpu":"A10","B":1,"N":2048,"impl":"P3-triton","speedup":0.875,"us":698.08,"base_us":610.976},{"gpu":"A10","B":1,"N":2048,"impl":"P4-fa2","speedup":0.979,"us":624.4,"base_us":610.976},{"gpu":"A10","B":1,"N":4096,"impl":"P0-naive","speedup":0.058,"us":42711.391,"base_us":2491.52},{"gpu":"A10","B":1,"N":4096,"impl":"P1-inductor","speedup":0.214,"us":11620.08,"base_us":2491.52},{"gpu":"A10","B":1,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.666,"us":3741.376,"base_us":2491.52},{"gpu":"A10","B":1,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":1.09,"us":2286.224,"base_us":2491.52},{"gpu":"A10","B":1,"N":4096,"impl":"P3-triton","speedup":1.048,"us":2377.92,"base_us":2491.52},{"gpu":"A10","B":1,"N":4096,"impl":"P4-fa2","speedup":0.975,"us":2555.344,"base_us":2491.52},{"gpu":"A10","B":1,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.697,"us":14315.248,"base_us":9977.104},{"gpu":"A10","B":1,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":1.186,"us":8408.928,"base_us":9977.104},{"gpu":"A10","B":1,"N":8192,"impl":"P3-triton","speedup":1.185,"us":8422.304,"base_us":9977.104},{"gpu":"A10","B":1,"N":8192,"impl":"P4-fa2","speedup":0.955,"us":10446.736,"base_us":9977.104},{"gpu":"A10","B":4,"N":256,"impl":"P0-naive","speedup":0.195,"us":781.008,"base_us":152.496},{"gpu":"A10","B":4,"N":256,"impl":"P1-inductor","speedup":0.513,"us":297.392,"base_us":152.496},{"gpu":"A10","B":4,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":1.357,"us":112.384,"base_us":152.496},{"gpu":"A10","B":4,"N":256,"impl":"P2d-sdpa-cudnn","speedup":1.413,"us":107.936,"base_us":152.496},{"gpu":"A10","B":4,"N":256,"impl":"P3-triton","speedup":0.713,"us":213.952,"base_us":152.496},{"gpu":"A10","B":4,"N":256,"impl":"P4-fa2","speedup":0.854,"us":178.544,"base_us":152.496},{"gpu":"A10","B":4,"N":512,"impl":"P0-naive","speedup":0.094,"us":2807.424,"base_us":262.736},{"gpu":"A10","B":4,"N":512,"impl":"P1-inductor","speedup":0.332,"us":791.568,"base_us":262.736},{"gpu":"A10","B":4,"N":512,"impl":"P1-inductor-nofuse","speedup":0.332,"us":790.368,"base_us":262.736},{"gpu":"A10","B":4,"N":512,"impl":"P1-inductor-where","speedup":0.33,"us":796.8,"base_us":262.736},{"gpu":"A10","B":4,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":0.849,"us":309.568,"base_us":262.736},{"gpu":"A10","B":4,"N":512,"impl":"P2d-sdpa-cudnn","speedup":1.232,"us":213.264,"base_us":262.736},{"gpu":"A10","B":4,"N":512,"impl":"P3-triton","speedup":0.793,"us":331.2,"base_us":262.736},{"gpu":"A10","B":4,"N":512,"impl":"P4-fa2","speedup":0.837,"us":313.984,"base_us":262.736},{"gpu":"A10","B":4,"N":1024,"impl":"P0-naive","speedup":0.061,"us":10838.848,"base_us":657.008},{"gpu":"A10","B":4,"N":1024,"impl":"P1-inductor","speedup":0.25,"us":2625.92,"base_us":657.008},{"gpu":"A10","B":4,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.566,"us":1160.912,"base_us":657.008},{"gpu":"A10","B":4,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":1.032,"us":636.672,"base_us":657.008},{"gpu":"A10","B":4,"N":1024,"impl":"P3-triton","speedup":0.812,"us":808.912,"base_us":657.008},{"gpu":"A10","B":4,"N":1024,"impl":"P4-fa2","speedup":0.971,"us":676.944,"base_us":657.008},{"gpu":"A10","B":4,"N":2048,"impl":"P0-naive","speedup":0.06,"us":43078.88,"base_us":2576.4},{"gpu":"A10","B":4,"N":2048,"impl":"P1-inductor","speedup":0.221,"us":11648.656,"base_us":2576.4},{"gpu":"A10","B":4,"N":2048,"impl":"P1-inductor-nofuse","speedup":0.208,"us":12402.256,"base_us":2576.4},{"gpu":"A10","B":4,"N":2048,"impl":"P1-inductor-where","speedup":0.21,"us":12264.576,"base_us":2576.4},{"gpu":"A10","B":4,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.648,"us":3972.928,"base_us":2576.4},{"gpu":"A10","B":4,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":1.072,"us":2403.888,"base_us":2576.4},{"gpu":"A10","B":4,"N":2048,"impl":"P3-triton","speedup":1.016,"us":2536.288,"base_us":2576.4},{"gpu":"A10","B":4,"N":2048,"impl":"P4-fa2","speedup":0.974,"us":2644.304,"base_us":2576.4},{"gpu":"A10","B":4,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.652,"us":14568.816,"base_us":9496.0},{"gpu":"A10","B":4,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":1.096,"us":8667.152,"base_us":9496.0},{"gpu":"A10","B":4,"N":4096,"impl":"P3-triton","speedup":1.113,"us":8530.16,"base_us":9496.0},{"gpu":"A10","B":4,"N":4096,"impl":"P4-fa2","speedup":0.953,"us":9965.952,"base_us":9496.0},{"gpu":"A10","B":4,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.724,"us":55807.264,"base_us":40406.256},{"gpu":"A10","B":4,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":1.252,"us":32273.008,"base_us":40406.256},{"gpu":"A10","B":4,"N":8192,"impl":"P3-triton","speedup":1.257,"us":32156.736,"base_us":40406.256},{"gpu":"A10","B":4,"N":8192,"impl":"P4-fa2","speedup":0.944,"us":42793.713,"base_us":40406.256},{"gpu":"A10","B":16,"N":256,"impl":"P0-naive","speedup":0.151,"us":2877.504,"base_us":434.816},{"gpu":"A10","B":16,"N":256,"impl":"P1-inductor","speedup":0.478,"us":910.416,"base_us":434.816},{"gpu":"A10","B":16,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":1.167,"us":372.688,"base_us":434.816},{"gpu":"A10","B":16,"N":256,"impl":"P2d-sdpa-cudnn","speedup":1.287,"us":337.792,"base_us":434.816},{"gpu":"A10","B":16,"N":256,"impl":"P3-triton","speedup":1.044,"us":416.304,"base_us":434.816},{"gpu":"A10","B":16,"N":256,"impl":"P4-fa2","speedup":0.887,"us":490.08,"base_us":434.816},{"gpu":"A10","B":16,"N":512,"impl":"P0-naive","speedup":0.078,"us":10972.08,"base_us":856.704},{"gpu":"A10","B":16,"N":512,"impl":"P1-inductor","speedup":0.29,"us":2951.104,"base_us":856.704},{"gpu":"A10","B":16,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":0.627,"us":1365.52,"base_us":856.704},{"gpu":"A10","B":16,"N":512,"impl":"P2d-sdpa-cudnn","speedup":1.043,"us":821.504,"base_us":856.704},{"gpu":"A10","B":16,"N":512,"impl":"P3-triton","speedup":0.921,"us":929.856,"base_us":856.704},{"gpu":"A10","B":16,"N":512,"impl":"P4-fa2","speedup":0.886,"us":967.008,"base_us":856.704},{"gpu":"A10","B":16,"N":1024,"impl":"P0-naive","speedup":0.067,"us":43108.225,"base_us":2877.104},{"gpu":"A10","B":16,"N":1024,"impl":"P1-inductor","speedup":0.28,"us":10273.072,"base_us":2877.104},{"gpu":"A10","B":16,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.659,"us":4368.48,"base_us":2877.104},{"gpu":"A10","B":16,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":1.042,"us":2762.064,"base_us":2877.104},{"gpu":"A10","B":16,"N":1024,"impl":"P3-triton","speedup":0.978,"us":2943.024,"base_us":2877.104},{"gpu":"A10","B":16,"N":1024,"impl":"P4-fa2","speedup":0.961,"us":2994.048,"base_us":2877.104},{"gpu":"A10","B":16,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.646,"us":15513.856,"base_us":10017.296},{"gpu":"A10","B":16,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":1.072,"us":9343.952,"base_us":10017.296},{"gpu":"A10","B":16,"N":2048,"impl":"P3-triton","speedup":1.085,"us":9231.84,"base_us":10017.296},{"gpu":"A10","B":16,"N":2048,"impl":"P4-fa2","speedup":0.95,"us":10539.024,"base_us":10017.296},{"gpu":"A10","B":16,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.669,"us":58970.112,"base_us":39432.096},{"gpu":"A10","B":16,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":1.134,"us":34780.224,"base_us":39432.096},{"gpu":"A10","B":16,"N":4096,"impl":"P3-triton","speedup":1.146,"us":34421.265,"base_us":39432.096},{"gpu":"A10","B":16,"N":4096,"impl":"P4-fa2","speedup":0.922,"us":42786.081,"base_us":39432.096},{"gpu":"A10","B":16,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.741,"us":227088.89,"base_us":168180.801},{"gpu":"A10","B":16,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":1.263,"us":133115.044,"base_us":168180.801},{"gpu":"A10","B":16,"N":8192,"impl":"P3-triton","speedup":1.297,"us":129675.911,"base_us":168180.801},{"gpu":"A10","B":16,"N":8192,"impl":"P4-fa2","speedup":0.932,"us":180359.009,"base_us":168180.801},{"gpu":"A100","B":1,"N":256,"impl":"P0-naive","speedup":0.364,"us":226.592,"base_us":82.592},{"gpu":"A100","B":1,"N":256,"impl":"P1-inductor","speedup":0.516,"us":159.936,"base_us":82.592},{"gpu":"A100","B":1,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":1.221,"us":67.664,"base_us":82.592},{"gpu":"A100","B":1,"N":256,"impl":"P2d-sdpa-cudnn","speedup":0.95,"us":86.944,"base_us":82.592},{"gpu":"A100","B":1,"N":256,"impl":"P3-triton","speedup":0.521,"us":158.56,"base_us":82.592},{"gpu":"A100","B":1,"N":256,"impl":"P4-fa2","speedup":0.751,"us":109.904,"base_us":82.592},{"gpu":"A100","B":1,"N":512,"impl":"P0-naive","speedup":0.287,"us":288.576,"base_us":82.92},{"gpu":"A100","B":1,"N":512,"impl":"P1-inductor","speedup":0.512,"us":161.84,"base_us":82.92},{"gpu":"A100","B":1,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":1.172,"us":70.779,"base_us":82.92},{"gpu":"A100","B":1,"N":512,"impl":"P2d-sdpa-cudnn","speedup":0.957,"us":86.656,"base_us":82.92},{"gpu":"A100","B":1,"N":512,"impl":"P3-triton","speedup":0.47,"us":176.416,"base_us":82.92},{"gpu":"A100","B":1,"N":512,"impl":"P4-fa2","speedup":0.754,"us":109.984,"base_us":82.92},{"gpu":"A100","B":1,"N":1024,"impl":"P0-naive","speedup":0.114,"us":998.24,"base_us":113.984},{"gpu":"A100","B":1,"N":1024,"impl":"P1-inductor","speedup":0.375,"us":303.616,"base_us":113.984},{"gpu":"A100","B":1,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.63,"us":180.96,"base_us":113.984},{"gpu":"A100","B":1,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":1.074,"us":106.144,"base_us":113.984},{"gpu":"A100","B":1,"N":1024,"impl":"P3-triton","speedup":0.497,"us":229.424,"base_us":113.984},{"gpu":"A100","B":1,"N":1024,"impl":"P4-fa2","speedup":0.711,"us":160.32,"base_us":113.984},{"gpu":"A100","B":1,"N":2048,"impl":"P0-naive","speedup":0.083,"us":3653.872,"base_us":303.888},{"gpu":"A100","B":1,"N":2048,"impl":"P1-inductor","speedup":0.28,"us":1086.768,"base_us":303.888},{"gpu":"A100","B":1,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.667,"us":455.824,"base_us":303.888},{"gpu":"A100","B":1,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":1.065,"us":285.36,"base_us":303.888},{"gpu":"A100","B":1,"N":2048,"impl":"P3-triton","speedup":0.725,"us":418.992,"base_us":303.888},{"gpu":"A100","B":1,"N":2048,"impl":"P4-fa2","speedup":0.954,"us":318.48,"base_us":303.888},{"gpu":"A100","B":1,"N":4096,"impl":"P0-naive","speedup":0.058,"us":15070.288,"base_us":873.584},{"gpu":"A100","B":1,"N":4096,"impl":"P1-inductor","speedup":0.205,"us":4265.888,"base_us":873.584},{"gpu":"A100","B":1,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.595,"us":1468.272,"base_us":873.584},{"gpu":"A100","B":1,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":0.99,"us":882.576,"base_us":873.584},{"gpu":"A100","B":1,"N":4096,"impl":"P3-triton","speedup":0.825,"us":1058.704,"base_us":873.584},{"gpu":"A100","B":1,"N":4096,"impl":"P4-fa2","speedup":1.018,"us":857.968,"base_us":873.584},{"gpu":"A100","B":1,"N":8192,"impl":"P0-naive","speedup":0.05,"us":61484.272,"base_us":3074.464},{"gpu":"A100","B":1,"N":8192,"impl":"P1-inductor","speedup":0.184,"us":16685.536,"base_us":3074.464},{"gpu":"A100","B":1,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.572,"us":5373.12,"base_us":3074.464},{"gpu":"A100","B":1,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":0.956,"us":3216.256,"base_us":3074.464},{"gpu":"A100","B":1,"N":8192,"impl":"P3-triton","speedup":0.881,"us":3490.336,"base_us":3074.464},{"gpu":"A100","B":1,"N":8192,"impl":"P4-fa2","speedup":1.037,"us":2965.248,"base_us":3074.464},{"gpu":"A100","B":4,"N":256,"impl":"P0-naive","speedup":0.283,"us":293.008,"base_us":82.912},{"gpu":"A100","B":4,"N":256,"impl":"P1-inductor","speedup":0.512,"us":162.08,"base_us":82.912},{"gpu":"A100","B":4,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":1.16,"us":71.456,"base_us":82.912},{"gpu":"A100","B":4,"N":256,"impl":"P2d-sdpa-cudnn","speedup":0.954,"us":86.904,"base_us":82.912},{"gpu":"A100","B":4,"N":256,"impl":"P3-triton","speedup":0.478,"us":173.504,"base_us":82.912},{"gpu":"A100","B":4,"N":256,"impl":"P4-fa2","speedup":0.755,"us":109.816,"base_us":82.912},{"gpu":"A100","B":4,"N":512,"impl":"P0-naive","speedup":0.115,"us":1001.872,"base_us":115.392},{"gpu":"A100","B":4,"N":512,"impl":"P1-inductor","speedup":0.366,"us":315.552,"base_us":115.392},{"gpu":"A100","B":4,"N":512,"impl":"P1-inductor-nofuse","speedup":0.367,"us":314.048,"base_us":115.392},{"gpu":"A100","B":4,"N":512,"impl":"P1-inductor-where","speedup":0.367,"us":314.416,"base_us":115.392},{"gpu":"A100","B":4,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":0.639,"us":180.704,"base_us":115.392},{"gpu":"A100","B":4,"N":512,"impl":"P2d-sdpa-cudnn","speedup":1.073,"us":107.576,"base_us":115.392},{"gpu":"A100","B":4,"N":512,"impl":"P3-triton","speedup":0.501,"us":230.352,"base_us":115.392},{"gpu":"A100","B":4,"N":512,"impl":"P4-fa2","speedup":0.7,"us":164.928,"base_us":115.392},{"gpu":"A100","B":4,"N":1024,"impl":"P0-naive","speedup":0.083,"us":3620.384,"base_us":299.232},{"gpu":"A100","B":4,"N":1024,"impl":"P1-inductor","speedup":0.332,"us":901.968,"base_us":299.232},{"gpu":"A100","B":4,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.655,"us":456.672,"base_us":299.232},{"gpu":"A100","B":4,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":1.034,"us":289.52,"base_us":299.232},{"gpu":"A100","B":4,"N":1024,"impl":"P3-triton","speedup":0.724,"us":413.44,"base_us":299.232},{"gpu":"A100","B":4,"N":1024,"impl":"P4-fa2","speedup":0.958,"us":312.48,"base_us":299.232},{"gpu":"A100","B":4,"N":2048,"impl":"P0-naive","speedup":0.062,"us":14180.096,"base_us":873.824},{"gpu":"A100","B":4,"N":2048,"impl":"P1-inductor","speedup":0.213,"us":4110.672,"base_us":873.824},{"gpu":"A100","B":4,"N":2048,"impl":"P1-inductor-nofuse","speedup":0.24,"us":3647.808,"base_us":873.824},{"gpu":"A100","B":4,"N":2048,"impl":"P1-inductor-where","speedup":0.211,"us":4132.048,"base_us":873.824},{"gpu":"A100","B":4,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.584,"us":1495.968,"base_us":873.824},{"gpu":"A100","B":4,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":0.964,"us":906.048,"base_us":873.824},{"gpu":"A100","B":4,"N":2048,"impl":"P3-triton","speedup":0.824,"us":1061.056,"base_us":873.824},{"gpu":"A100","B":4,"N":2048,"impl":"P4-fa2","speedup":1.004,"us":870.576,"base_us":873.824},{"gpu":"A100","B":4,"N":4096,"impl":"P0-naive","speedup":0.052,"us":59714.161,"base_us":3098.896},{"gpu":"A100","B":4,"N":4096,"impl":"P1-inductor","speedup":0.184,"us":16872.607,"base_us":3098.896},{"gpu":"A100","B":4,"N":4096,"impl":"P1-inductor-nofuse","speedup":0.21,"us":14737.872,"base_us":3098.896},{"gpu":"A100","B":4,"N":4096,"impl":"P1-inductor-where","speedup":0.19,"us":16310.207,"base_us":3098.896},{"gpu":"A100","B":4,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.57,"us":5434.0,"base_us":3098.896},{"gpu":"A100","B":4,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":0.946,"us":3275.488,"base_us":3098.896},{"gpu":"A100","B":4,"N":4096,"impl":"P3-triton","speedup":0.883,"us":3511.392,"base_us":3098.896},{"gpu":"A100","B":4,"N":4096,"impl":"P4-fa2","speedup":1.035,"us":2994.512,"base_us":3098.896},{"gpu":"A100","B":4,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.554,"us":20742.56,"base_us":11501.28},{"gpu":"A100","B":4,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":0.935,"us":12296.96,"base_us":11501.28},{"gpu":"A100","B":4,"N":8192,"impl":"P3-triton","speedup":0.894,"us":12860.048,"base_us":11501.28},{"gpu":"A100","B":4,"N":8192,"impl":"P4-fa2","speedup":1.042,"us":11032.432,"base_us":11501.28},{"gpu":"A100","B":16,"N":256,"impl":"P0-naive","speedup":0.156,"us":1046.256,"base_us":162.72},{"gpu":"A100","B":16,"N":256,"impl":"P1-inductor","speedup":0.48,"us":339.136,"base_us":162.72},{"gpu":"A100","B":16,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":0.802,"us":202.848,"base_us":162.72},{"gpu":"A100","B":16,"N":256,"impl":"P2d-sdpa-cudnn","speedup":0.935,"us":174.096,"base_us":162.72},{"gpu":"A100","B":16,"N":256,"impl":"P3-triton","speedup":0.612,"us":265.68,"base_us":162.72},{"gpu":"A100","B":16,"N":256,"impl":"P4-fa2","speedup":0.871,"us":186.864,"base_us":162.72},{"gpu":"A100","B":16,"N":512,"impl":"P0-naive","speedup":0.088,"us":3679.184,"base_us":325.488},{"gpu":"A100","B":16,"N":512,"impl":"P1-inductor","speedup":0.343,"us":950.128,"base_us":325.488},{"gpu":"A100","B":16,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":0.645,"us":504.288,"base_us":325.488},{"gpu":"A100","B":16,"N":512,"impl":"P2d-sdpa-cudnn","speedup":1.008,"us":322.96,"base_us":325.488},{"gpu":"A100","B":16,"N":512,"impl":"P3-triton","speedup":0.725,"us":448.784,"base_us":325.488},{"gpu":"A100","B":16,"N":512,"impl":"P4-fa2","speedup":0.936,"us":347.68,"base_us":325.488},{"gpu":"A100","B":16,"N":1024,"impl":"P0-naive","speedup":0.068,"us":14198.24,"base_us":968.032},{"gpu":"A100","B":16,"N":1024,"impl":"P1-inductor","speedup":0.291,"us":3326.032,"base_us":968.032},{"gpu":"A100","B":16,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.6,"us":1613.392,"base_us":968.032},{"gpu":"A100","B":16,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":0.977,"us":990.416,"base_us":968.032},{"gpu":"A100","B":16,"N":1024,"impl":"P3-triton","speedup":0.835,"us":1158.864,"base_us":968.032},{"gpu":"A100","B":16,"N":1024,"impl":"P4-fa2","speedup":1.0,"us":967.936,"base_us":968.032},{"gpu":"A100","B":16,"N":2048,"impl":"P0-naive","speedup":0.057,"us":56779.312,"base_us":3264.288},{"gpu":"A100","B":16,"N":2048,"impl":"P1-inductor","speedup":0.2,"us":16340.704,"base_us":3264.288},{"gpu":"A100","B":16,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.578,"us":5645.44,"base_us":3264.288},{"gpu":"A100","B":16,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":0.952,"us":3429.968,"base_us":3264.288},{"gpu":"A100","B":16,"N":2048,"impl":"P3-triton","speedup":0.884,"us":3693.504,"base_us":3264.288},{"gpu":"A100","B":16,"N":2048,"impl":"P4-fa2","speedup":1.026,"us":3181.744,"base_us":3264.288},{"gpu":"A100","B":16,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.556,"us":21148.96,"base_us":11753.888},{"gpu":"A100","B":16,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":0.935,"us":12572.912,"base_us":11753.888},{"gpu":"A100","B":16,"N":4096,"impl":"P3-triton","speedup":0.891,"us":13191.184,"base_us":11753.888},{"gpu":"A100","B":16,"N":4096,"impl":"P4-fa2","speedup":1.04,"us":11304.4,"base_us":11753.888},{"gpu":"A100","B":16,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.552,"us":82422.413,"base_us":45502.735},{"gpu":"A100","B":16,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":0.927,"us":49094.353,"base_us":45502.735},{"gpu":"A100","B":16,"N":8192,"impl":"P3-triton","speedup":0.901,"us":50512.945,"base_us":45502.735},{"gpu":"A100","B":16,"N":8192,"impl":"P4-fa2","speedup":1.044,"us":43568.048,"base_us":45502.735},{"gpu":"RTX 5090","B":1,"N":256,"impl":"P0-naive","speedup":0.41,"us":52.432,"base_us":21.499},{"gpu":"RTX 5090","B":1,"N":256,"impl":"P1-inductor","speedup":0.673,"us":31.963,"base_us":21.499},{"gpu":"RTX 5090","B":1,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":1.112,"us":19.334,"base_us":21.499},{"gpu":"RTX 5090","B":1,"N":256,"impl":"P2d-sdpa-cudnn","speedup":1.247,"us":17.24,"base_us":21.499},{"gpu":"RTX 5090","B":1,"N":256,"impl":"P3-triton","speedup":0.395,"us":54.443,"base_us":21.499},{"gpu":"RTX 5090","B":1,"N":256,"impl":"P4-fa2","speedup":0.884,"us":24.313,"base_us":21.499},{"gpu":"RTX 5090","B":1,"N":512,"impl":"P0-naive","speedup":0.243,"us":134.24,"base_us":32.613},{"gpu":"RTX 5090","B":1,"N":512,"impl":"P1-inductor","speedup":0.803,"us":40.627,"base_us":32.613},{"gpu":"RTX 5090","B":1,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":0.619,"us":52.709,"base_us":32.613},{"gpu":"RTX 5090","B":1,"N":512,"impl":"P2d-sdpa-cudnn","speedup":1.056,"us":30.877,"base_us":32.613},{"gpu":"RTX 5090","B":1,"N":512,"impl":"P3-triton","speedup":0.554,"us":58.885,"base_us":32.613},{"gpu":"RTX 5090","B":1,"N":512,"impl":"P4-fa2","speedup":0.987,"us":33.027,"base_us":32.613},{"gpu":"RTX 5090","B":1,"N":1024,"impl":"P0-naive","speedup":0.12,"us":760.88,"base_us":91.664},{"gpu":"RTX 5090","B":1,"N":1024,"impl":"P1-inductor","speedup":0.67,"us":136.736,"base_us":91.664},{"gpu":"RTX 5090","B":1,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.588,"us":155.936,"base_us":91.664},{"gpu":"RTX 5090","B":1,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":1.216,"us":75.384,"base_us":91.664},{"gpu":"RTX 5090","B":1,"N":1024,"impl":"P3-triton","speedup":0.793,"us":115.624,"base_us":91.664},{"gpu":"RTX 5090","B":1,"N":1024,"impl":"P4-fa2","speedup":0.995,"us":92.104,"base_us":91.664},{"gpu":"RTX 5090","B":1,"N":2048,"impl":"P0-naive","speedup":0.074,"us":3469.872,"base_us":256.288},{"gpu":"RTX 5090","B":1,"N":2048,"impl":"P1-inductor","speedup":0.343,"us":746.624,"base_us":256.288},{"gpu":"RTX 5090","B":1,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.539,"us":475.328,"base_us":256.288},{"gpu":"RTX 5090","B":1,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":1.169,"us":219.184,"base_us":256.288},{"gpu":"RTX 5090","B":1,"N":2048,"impl":"P3-triton","speedup":0.834,"us":307.328,"base_us":256.288},{"gpu":"RTX 5090","B":1,"N":2048,"impl":"P4-fa2","speedup":1.001,"us":256.08,"base_us":256.288},{"gpu":"RTX 5090","B":1,"N":4096,"impl":"P0-naive","speedup":0.06,"us":13641.088,"base_us":812.736},{"gpu":"RTX 5090","B":1,"N":4096,"impl":"P1-inductor","speedup":0.28,"us":2900.528,"base_us":812.736},{"gpu":"RTX 5090","B":1,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.49,"us":1657.408,"base_us":812.736},{"gpu":"RTX 5090","B":1,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":1.133,"us":717.216,"base_us":812.736},{"gpu":"RTX 5090","B":1,"N":4096,"impl":"P3-triton","speedup":0.851,"us":955.504,"base_us":812.736},{"gpu":"RTX 5090","B":1,"N":4096,"impl":"P4-fa2","speedup":1.011,"us":803.664,"base_us":812.736},{"gpu":"RTX 5090","B":1,"N":8192,"impl":"P0-naive","speedup":0.051,"us":55841.103,"base_us":2860.944},{"gpu":"RTX 5090","B":1,"N":8192,"impl":"P1-inductor","speedup":0.229,"us":12472.448,"base_us":2860.944},{"gpu":"RTX 5090","B":1,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.464,"us":6165.376,"base_us":2860.944},{"gpu":"RTX 5090","B":1,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":1.116,"us":2563.296,"base_us":2860.944},{"gpu":"RTX 5090","B":1,"N":8192,"impl":"P3-triton","speedup":0.92,"us":3108.56,"base_us":2860.944},{"gpu":"RTX 5090","B":1,"N":8192,"impl":"P4-fa2","speedup":1.014,"us":2820.672,"base_us":2860.944},{"gpu":"RTX 5090","B":4,"N":256,"impl":"P0-naive","speedup":0.264,"us":132.992,"base_us":35.053},{"gpu":"RTX 5090","B":4,"N":256,"impl":"P1-inductor","speedup":0.804,"us":43.58,"base_us":35.053},{"gpu":"RTX 5090","B":4,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":0.7,"us":50.048,"base_us":35.053},{"gpu":"RTX 5090","B":4,"N":256,"impl":"P2d-sdpa-cudnn","speedup":1.338,"us":26.19,"base_us":35.053},{"gpu":"RTX 5090","B":4,"N":256,"impl":"P3-triton","speedup":0.585,"us":59.888,"base_us":35.053},{"gpu":"RTX 5090","B":4,"N":256,"impl":"P4-fa2","speedup":0.984,"us":35.635,"base_us":35.053},{"gpu":"RTX 5090","B":4,"N":512,"impl":"P0-naive","speedup":0.108,"us":760.24,"base_us":81.936},{"gpu":"RTX 5090","B":4,"N":512,"impl":"P1-inductor","speedup":0.583,"us":140.576,"base_us":81.936},{"gpu":"RTX 5090","B":4,"N":512,"impl":"P1-inductor-nofuse","speedup":0.577,"us":141.968,"base_us":81.936},{"gpu":"RTX 5090","B":4,"N":512,"impl":"P1-inductor-where","speedup":0.582,"us":140.72,"base_us":81.936},{"gpu":"RTX 5090","B":4,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":0.555,"us":147.696,"base_us":81.936},{"gpu":"RTX 5090","B":4,"N":512,"impl":"P2d-sdpa-cudnn","speedup":1.287,"us":63.675,"base_us":81.936},{"gpu":"RTX 5090","B":4,"N":512,"impl":"P3-triton","speedup":0.767,"us":106.84,"base_us":81.936},{"gpu":"RTX 5090","B":4,"N":512,"impl":"P4-fa2","speedup":0.995,"us":82.312,"base_us":81.936},{"gpu":"RTX 5090","B":4,"N":1024,"impl":"P0-naive","speedup":0.067,"us":3610.416,"base_us":243.184},{"gpu":"RTX 5090","B":4,"N":1024,"impl":"P1-inductor","speedup":0.305,"us":797.856,"base_us":243.184},{"gpu":"RTX 5090","B":4,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.519,"us":468.912,"base_us":243.184},{"gpu":"RTX 5090","B":4,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":1.182,"us":205.712,"base_us":243.184},{"gpu":"RTX 5090","B":4,"N":1024,"impl":"P3-triton","speedup":0.813,"us":299.024,"base_us":243.184},{"gpu":"RTX 5090","B":4,"N":1024,"impl":"P4-fa2","speedup":0.995,"us":244.48,"base_us":243.184},{"gpu":"RTX 5090","B":4,"N":2048,"impl":"P0-naive","speedup":0.056,"us":13887.552,"base_us":779.136},{"gpu":"RTX 5090","B":4,"N":2048,"impl":"P1-inductor","speedup":0.262,"us":2974.464,"base_us":779.136},{"gpu":"RTX 5090","B":4,"N":2048,"impl":"P1-inductor-nofuse","speedup":0.265,"us":2938.192,"base_us":779.136},{"gpu":"RTX 5090","B":4,"N":2048,"impl":"P1-inductor-where","speedup":0.262,"us":2971.504,"base_us":779.136},{"gpu":"RTX 5090","B":4,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.474,"us":1644.72,"base_us":779.136},{"gpu":"RTX 5090","B":4,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":1.149,"us":678.384,"base_us":779.136},{"gpu":"RTX 5090","B":4,"N":2048,"impl":"P3-triton","speedup":0.905,"us":860.528,"base_us":779.136},{"gpu":"RTX 5090","B":4,"N":2048,"impl":"P4-fa2","speedup":1.008,"us":772.8,"base_us":779.136},{"gpu":"RTX 5090","B":4,"N":4096,"impl":"P0-naive","speedup":0.051,"us":54426.704,"base_us":2789.808},{"gpu":"RTX 5090","B":4,"N":4096,"impl":"P1-inductor","speedup":0.241,"us":11569.088,"base_us":2789.808},{"gpu":"RTX 5090","B":4,"N":4096,"impl":"P1-inductor-nofuse","speedup":0.242,"us":11514.528,"base_us":2789.808},{"gpu":"RTX 5090","B":4,"N":4096,"impl":"P1-inductor-where","speedup":0.242,"us":11525.36,"base_us":2789.808},{"gpu":"RTX 5090","B":4,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.454,"us":6139.392,"base_us":2789.808},{"gpu":"RTX 5090","B":4,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":1.117,"us":2497.648,"base_us":2789.808},{"gpu":"RTX 5090","B":4,"N":4096,"impl":"P3-triton","speedup":0.949,"us":2940.528,"base_us":2789.808},{"gpu":"RTX 5090","B":4,"N":4096,"impl":"P4-fa2","speedup":1.01,"us":2762.352,"base_us":2789.808},{"gpu":"RTX 5090","B":4,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.446,"us":23718.096,"base_us":10585.568},{"gpu":"RTX 5090","B":4,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":1.09,"us":9710.944,"base_us":10585.568},{"gpu":"RTX 5090","B":4,"N":8192,"impl":"P3-triton","speedup":0.98,"us":10806.432,"base_us":10585.568},{"gpu":"RTX 5090","B":4,"N":8192,"impl":"P4-fa2","speedup":1.015,"us":10425.296,"base_us":10585.568},{"gpu":"RTX 5090","B":16,"N":256,"impl":"P0-naive","speedup":0.125,"us":777.872,"base_us":97.544},{"gpu":"RTX 5090","B":16,"N":256,"impl":"P1-inductor","speedup":0.573,"us":170.272,"base_us":97.544},{"gpu":"RTX 5090","B":16,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":0.574,"us":169.904,"base_us":97.544},{"gpu":"RTX 5090","B":16,"N":256,"impl":"P2d-sdpa-cudnn","speedup":1.237,"us":78.872,"base_us":97.544},{"gpu":"RTX 5090","B":16,"N":256,"impl":"P3-triton","speedup":0.633,"us":154.096,"base_us":97.544},{"gpu":"RTX 5090","B":16,"N":256,"impl":"P4-fa2","speedup":1.011,"us":96.44,"base_us":97.544},{"gpu":"RTX 5090","B":16,"N":512,"impl":"P0-naive","speedup":0.064,"us":4012.128,"base_us":258.08},{"gpu":"RTX 5090","B":16,"N":512,"impl":"P1-inductor","speedup":0.294,"us":877.936,"base_us":258.08},{"gpu":"RTX 5090","B":16,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":0.51,"us":505.696,"base_us":258.08},{"gpu":"RTX 5090","B":16,"N":512,"impl":"P2d-sdpa-cudnn","speedup":1.23,"us":209.904,"base_us":258.08},{"gpu":"RTX 5090","B":16,"N":512,"impl":"P3-triton","speedup":0.794,"us":325.152,"base_us":258.08},{"gpu":"RTX 5090","B":16,"N":512,"impl":"P4-fa2","speedup":0.993,"us":259.84,"base_us":258.08},{"gpu":"RTX 5090","B":16,"N":1024,"impl":"P0-naive","speedup":0.056,"us":14413.184,"base_us":808.032},{"gpu":"RTX 5090","B":16,"N":1024,"impl":"P1-inductor","speedup":0.252,"us":3203.696,"base_us":808.032},{"gpu":"RTX 5090","B":16,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.469,"us":1722.096,"base_us":808.032},{"gpu":"RTX 5090","B":16,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":1.18,"us":684.752,"base_us":808.032},{"gpu":"RTX 5090","B":16,"N":1024,"impl":"P3-triton","speedup":0.888,"us":910.08,"base_us":808.032},{"gpu":"RTX 5090","B":16,"N":1024,"impl":"P4-fa2","speedup":1.007,"us":802.688,"base_us":808.032},{"gpu":"RTX 5090","B":16,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.455,"us":6296.544,"base_us":2867.904},{"gpu":"RTX 5090","B":16,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":1.102,"us":2602.704,"base_us":2867.904},{"gpu":"RTX 5090","B":16,"N":2048,"impl":"P3-triton","speedup":0.935,"us":3066.688,"base_us":2867.904},{"gpu":"RTX 5090","B":16,"N":2048,"impl":"P4-fa2","speedup":1.014,"us":2828.976,"base_us":2867.904},{"gpu":"RTX 5090","B":16,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.446,"us":24046.737,"base_us":10726.608},{"gpu":"RTX 5090","B":16,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":1.088,"us":9861.552,"base_us":10726.608},{"gpu":"RTX 5090","B":16,"N":4096,"impl":"P3-triton","speedup":0.991,"us":10827.984,"base_us":10726.608},{"gpu":"RTX 5090","B":16,"N":4096,"impl":"P4-fa2","speedup":1.015,"us":10565.936,"base_us":10726.608},{"gpu":"RTX 5090","B":16,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.442,"us":93941.116,"base_us":41476.255},{"gpu":"RTX 5090","B":16,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":1.083,"us":38284.576,"base_us":41476.255},{"gpu":"RTX 5090","B":16,"N":8192,"impl":"P3-triton","speedup":0.993,"us":41771.488,"base_us":41476.255},{"gpu":"RTX 5090","B":16,"N":8192,"impl":"P4-fa2","speedup":1.016,"us":40832.529,"base_us":41476.255},{"gpu":"H100","B":1,"N":256,"impl":"P0-naive","speedup":0.465,"us":252.192,"base_us":117.152},{"gpu":"H100","B":1,"N":256,"impl":"P1-inductor","speedup":0.503,"us":232.928,"base_us":117.152},{"gpu":"H100","B":1,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":1.303,"us":89.904,"base_us":117.152},{"gpu":"H100","B":1,"N":256,"impl":"P2d-sdpa-cudnn","speedup":3.268,"us":35.843,"base_us":117.152},{"gpu":"H100","B":1,"N":256,"impl":"P3-triton","speedup":0.498,"us":235.408,"base_us":117.152},{"gpu":"H100","B":1,"N":256,"impl":"P4-fa2","speedup":0.639,"us":183.408,"base_us":117.152},{"gpu":"H100","B":1,"N":256,"impl":"P4h-fa3","speedup":1.442,"us":81.24,"base_us":117.152},{"gpu":"H100","B":1,"N":512,"impl":"P0-naive","speedup":0.449,"us":256.752,"base_us":115.352},{"gpu":"H100","B":1,"N":512,"impl":"P1-inductor","speedup":0.458,"us":251.872,"base_us":115.352},{"gpu":"H100","B":1,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":1.163,"us":99.152,"base_us":115.352},{"gpu":"H100","B":1,"N":512,"impl":"P2d-sdpa-cudnn","speedup":3.241,"us":35.597,"base_us":115.352},{"gpu":"H100","B":1,"N":512,"impl":"P3-triton","speedup":0.53,"us":217.824,"base_us":115.352},{"gpu":"H100","B":1,"N":512,"impl":"P4-fa2","speedup":0.867,"us":133.12,"base_us":115.352},{"gpu":"H100","B":1,"N":512,"impl":"P4h-fa3","speedup":1.188,"us":97.128,"base_us":115.352},{"gpu":"H100","B":1,"N":1024,"impl":"P0-naive","speedup":0.14,"us":585.792,"base_us":82.064},{"gpu":"H100","B":1,"N":1024,"impl":"P1-inductor","speedup":0.388,"us":211.36,"base_us":82.064},{"gpu":"H100","B":1,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.6,"us":136.752,"base_us":82.064},{"gpu":"H100","B":1,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":1.885,"us":43.536,"base_us":82.064},{"gpu":"H100","B":1,"N":1024,"impl":"P3-triton","speedup":0.461,"us":178.08,"base_us":82.064},{"gpu":"H100","B":1,"N":1024,"impl":"P4-fa2","speedup":0.821,"us":99.912,"base_us":82.064},{"gpu":"H100","B":1,"N":1024,"impl":"P4h-fa3","speedup":1.18,"us":69.557,"base_us":82.064},{"gpu":"H100","B":1,"N":2048,"impl":"P0-naive","speedup":0.109,"us":2162.992,"base_us":235.2},{"gpu":"H100","B":1,"N":2048,"impl":"P1-inductor","speedup":0.351,"us":669.984,"base_us":235.2},{"gpu":"H100","B":1,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.666,"us":353.312,"base_us":235.2},{"gpu":"H100","B":1,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":1.902,"us":123.664,"base_us":235.2},{"gpu":"H100","B":1,"N":2048,"impl":"P3-triton","speedup":0.947,"us":248.4,"base_us":235.2},{"gpu":"H100","B":1,"N":2048,"impl":"P4-fa2","speedup":0.955,"us":246.224,"base_us":235.2},{"gpu":"H100","B":1,"N":2048,"impl":"P4h-fa3","speedup":2.476,"us":94.984,"base_us":235.2},{"gpu":"H100","B":1,"N":4096,"impl":"P0-naive","speedup":0.07,"us":9205.232,"base_us":648.544},{"gpu":"H100","B":1,"N":4096,"impl":"P1-inductor","speedup":0.267,"us":2426.0,"base_us":648.544},{"gpu":"H100","B":1,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.584,"us":1110.448,"base_us":648.544},{"gpu":"H100","B":1,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":1.971,"us":329.12,"base_us":648.544},{"gpu":"H100","B":1,"N":4096,"impl":"P3-triton","speedup":1.371,"us":473.024,"base_us":648.544},{"gpu":"H100","B":1,"N":4096,"impl":"P4-fa2","speedup":1.051,"us":616.864,"base_us":648.544},{"gpu":"H100","B":1,"N":4096,"impl":"P4h-fa3","speedup":2.307,"us":281.104,"base_us":648.544},{"gpu":"H100","B":1,"N":8192,"impl":"P0-naive","speedup":0.052,"us":39312.88,"base_us":2062.576},{"gpu":"H100","B":1,"N":8192,"impl":"P1-inductor","speedup":0.229,"us":9021.024,"base_us":2062.576},{"gpu":"H100","B":1,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.548,"us":3766.368,"base_us":2062.576},{"gpu":"H100","B":1,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":1.971,"us":1046.416,"base_us":2062.576},{"gpu":"H100","B":1,"N":8192,"impl":"P3-triton","speedup":1.48,"us":1393.984,"base_us":2062.576},{"gpu":"H100","B":1,"N":8192,"impl":"P4-fa2","speedup":1.073,"us":1921.824,"base_us":2062.576},{"gpu":"H100","B":1,"N":8192,"impl":"P4h-fa3","speedup":2.388,"us":863.712,"base_us":2062.576},{"gpu":"H100","B":4,"N":256,"impl":"P0-naive","speedup":0.419,"us":289.968,"base_us":121.424},{"gpu":"H100","B":4,"N":256,"impl":"P1-inductor","speedup":0.46,"us":263.92,"base_us":121.424},{"gpu":"H100","B":4,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":1.199,"us":101.248,"base_us":121.424},{"gpu":"H100","B":4,"N":256,"impl":"P2d-sdpa-cudnn","speedup":3.354,"us":36.202,"base_us":121.424},{"gpu":"H100","B":4,"N":256,"impl":"P3-triton","speedup":0.416,"us":291.584,"base_us":121.424},{"gpu":"H100","B":4,"N":256,"impl":"P4-fa2","speedup":0.67,"us":181.344,"base_us":121.424},{"gpu":"H100","B":4,"N":256,"impl":"P4h-fa3","speedup":1.443,"us":84.128,"base_us":121.424},{"gpu":"H100","B":4,"N":512,"impl":"P0-naive","speedup":0.236,"us":607.232,"base_us":143.552},{"gpu":"H100","B":4,"N":512,"impl":"P1-inductor","speedup":0.56,"us":256.544,"base_us":143.552},{"gpu":"H100","B":4,"N":512,"impl":"P1-inductor-nofuse","speedup":0.522,"us":274.8,"base_us":143.552},{"gpu":"H100","B":4,"N":512,"impl":"P1-inductor-where","speedup":0.511,"us":280.672,"base_us":143.552},{"gpu":"H100","B":4,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":0.865,"us":166.016,"base_us":143.552},{"gpu":"H100","B":4,"N":512,"impl":"P2d-sdpa-cudnn","speedup":3.109,"us":46.18,"base_us":143.552},{"gpu":"H100","B":4,"N":512,"impl":"P3-triton","speedup":0.514,"us":279.088,"base_us":143.552},{"gpu":"H100","B":4,"N":512,"impl":"P4-fa2","speedup":0.911,"us":157.536,"base_us":143.552},{"gpu":"H100","B":4,"N":512,"impl":"P4h-fa3","speedup":1.156,"us":124.208,"base_us":143.552},{"gpu":"H100","B":4,"N":1024,"impl":"P0-naive","speedup":0.11,"us":2117.328,"base_us":233.776},{"gpu":"H100","B":4,"N":1024,"impl":"P1-inductor","speedup":0.445,"us":525.488,"base_us":233.776},{"gpu":"H100","B":4,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.687,"us":340.192,"base_us":233.776},{"gpu":"H100","B":4,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":1.868,"us":125.136,"base_us":233.776},{"gpu":"H100","B":4,"N":1024,"impl":"P3-triton","speedup":0.79,"us":295.936,"base_us":233.776},{"gpu":"H100","B":4,"N":1024,"impl":"P4-fa2","speedup":1.029,"us":227.168,"base_us":233.776},{"gpu":"H100","B":4,"N":1024,"impl":"P4h-fa3","speedup":2.159,"us":108.296,"base_us":233.776},{"gpu":"H100","B":4,"N":2048,"impl":"P0-naive","speedup":0.073,"us":8429.12,"base_us":618.176},{"gpu":"H100","B":4,"N":2048,"impl":"P1-inductor","speedup":0.263,"us":2346.56,"base_us":618.176},{"gpu":"H100","B":4,"N":2048,"impl":"P1-inductor-nofuse","speedup":0.292,"us":2114.224,"base_us":618.176},{"gpu":"H100","B":4,"N":2048,"impl":"P1-inductor-where","speedup":0.264,"us":2345.056,"base_us":618.176},{"gpu":"H100","B":4,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.559,"us":1105.168,"base_us":618.176},{"gpu":"H100","B":4,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":1.853,"us":333.6,"base_us":618.176},{"gpu":"H100","B":4,"N":2048,"impl":"P3-triton","speedup":1.26,"us":490.624,"base_us":618.176},{"gpu":"H100","B":4,"N":2048,"impl":"P4-fa2","speedup":1.055,"us":585.888,"base_us":618.176},{"gpu":"H100","B":4,"N":2048,"impl":"P4h-fa3","speedup":2.13,"us":290.272,"base_us":618.176},{"gpu":"H100","B":4,"N":4096,"impl":"P0-naive","speedup":0.055,"us":36609.552,"base_us":2001.872},{"gpu":"H100","B":4,"N":4096,"impl":"P1-inductor","speedup":0.218,"us":9176.736,"base_us":2001.872},{"gpu":"H100","B":4,"N":4096,"impl":"P1-inductor-nofuse","speedup":0.274,"us":7317.12,"base_us":2001.872},{"gpu":"H100","B":4,"N":4096,"impl":"P1-inductor-where","speedup":0.218,"us":9176.832,"base_us":2001.872},{"gpu":"H100","B":4,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.536,"us":3733.616,"base_us":2001.872},{"gpu":"H100","B":4,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":1.849,"us":1082.528,"base_us":2001.872},{"gpu":"H100","B":4,"N":4096,"impl":"P3-triton","speedup":1.516,"us":1320.144,"base_us":2001.872},{"gpu":"H100","B":4,"N":4096,"impl":"P4-fa2","speedup":1.043,"us":1919.28,"base_us":2001.872},{"gpu":"H100","B":4,"N":4096,"impl":"P4h-fa3","speedup":2.2,"us":909.84,"base_us":2001.872},{"gpu":"H100","B":4,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.506,"us":14115.504,"base_us":7139.488},{"gpu":"H100","B":4,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":1.949,"us":3662.976,"base_us":7139.488},{"gpu":"H100","B":4,"N":8192,"impl":"P3-triton","speedup":1.583,"us":4509.504,"base_us":7139.488},{"gpu":"H100","B":4,"N":8192,"impl":"P4-fa2","speedup":1.076,"us":6634.336,"base_us":7139.488},{"gpu":"H100","B":4,"N":8192,"impl":"P4h-fa3","speedup":2.156,"us":3311.264,"base_us":7139.488},{"gpu":"H100","B":16,"N":256,"impl":"P0-naive","speedup":0.26,"us":624.96,"base_us":162.192},{"gpu":"H100","B":16,"N":256,"impl":"P1-inductor","speedup":0.589,"us":275.568,"base_us":162.192},{"gpu":"H100","B":16,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":0.946,"us":171.424,"base_us":162.192},{"gpu":"H100","B":16,"N":256,"impl":"P2d-sdpa-cudnn","speedup":2.652,"us":61.163,"base_us":162.192},{"gpu":"H100","B":16,"N":256,"impl":"P3-triton","speedup":0.564,"us":287.824,"base_us":162.192},{"gpu":"H100","B":16,"N":256,"impl":"P4-fa2","speedup":0.961,"us":168.752,"base_us":162.192},{"gpu":"H100","B":16,"N":256,"impl":"P4h-fa3","speedup":1.456,"us":111.424,"base_us":162.192},{"gpu":"H100","B":16,"N":512,"impl":"P0-naive","speedup":0.119,"us":2155.728,"base_us":256.416},{"gpu":"H100","B":16,"N":512,"impl":"P1-inductor","speedup":0.462,"us":554.88,"base_us":256.416},{"gpu":"H100","B":16,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":0.686,"us":373.696,"base_us":256.416},{"gpu":"H100","B":16,"N":512,"impl":"P2d-sdpa-cudnn","speedup":1.773,"us":144.624,"base_us":256.416},{"gpu":"H100","B":16,"N":512,"impl":"P3-triton","speedup":0.844,"us":303.984,"base_us":256.416},{"gpu":"H100","B":16,"N":512,"impl":"P4-fa2","speedup":1.026,"us":249.952,"base_us":256.416},{"gpu":"H100","B":16,"N":512,"impl":"P4h-fa3","speedup":1.565,"us":163.872,"base_us":256.416},{"gpu":"H100","B":16,"N":1024,"impl":"P0-naive","speedup":0.078,"us":8246.992,"base_us":641.36},{"gpu":"H100","B":16,"N":1024,"impl":"P1-inductor","speedup":0.377,"us":1703.248,"base_us":641.36},{"gpu":"H100","B":16,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.555,"us":1156.4,"base_us":641.36},{"gpu":"H100","B":16,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":1.743,"us":368.0,"base_us":641.36},{"gpu":"H100","B":16,"N":1024,"impl":"P3-triton","speedup":1.221,"us":525.328,"base_us":641.36},{"gpu":"H100","B":16,"N":1024,"impl":"P4-fa2","speedup":1.06,"us":605.056,"base_us":641.36},{"gpu":"H100","B":16,"N":1024,"impl":"P4h-fa3","speedup":1.812,"us":353.936,"base_us":641.36},{"gpu":"H100","B":16,"N":2048,"impl":"P0-naive","speedup":0.061,"us":33597.887,"base_us":2043.072},{"gpu":"H100","B":16,"N":2048,"impl":"P1-inductor","speedup":0.227,"us":9002.448,"base_us":2043.072},{"gpu":"H100","B":16,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.531,"us":3848.16,"base_us":2043.072},{"gpu":"H100","B":16,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":1.819,"us":1123.312,"base_us":2043.072},{"gpu":"H100","B":16,"N":2048,"impl":"P3-triton","speedup":1.36,"us":1502.336,"base_us":2043.072},{"gpu":"H100","B":16,"N":2048,"impl":"P4-fa2","speedup":1.027,"us":1990.192,"base_us":2043.072},{"gpu":"H100","B":16,"N":2048,"impl":"P4h-fa3","speedup":2.013,"us":1015.104,"base_us":2043.072},{"gpu":"H100","B":16,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.505,"us":14360.064,"base_us":7256.768},{"gpu":"H100","B":16,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":1.889,"us":3841.056,"base_us":7256.768},{"gpu":"H100","B":16,"N":4096,"impl":"P3-triton","speedup":1.505,"us":4821.68,"base_us":7256.768},{"gpu":"H100","B":16,"N":4096,"impl":"P4-fa2","speedup":1.074,"us":6759.072,"base_us":7256.768},{"gpu":"H100","B":16,"N":4096,"impl":"P4h-fa3","speedup":2.079,"us":3490.944,"base_us":7256.768},{"gpu":"H100","B":16,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.499,"us":55645.985,"base_us":27776.959},{"gpu":"H100","B":16,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":1.761,"us":15768.96,"base_us":27776.959},{"gpu":"H100","B":16,"N":8192,"impl":"P3-triton","speedup":1.57,"us":17693.199,"base_us":27776.959},{"gpu":"H100","B":16,"N":8192,"impl":"P4-fa2","speedup":1.077,"us":25781.856,"base_us":27776.959},{"gpu":"H100","B":16,"N":8192,"impl":"P4h-fa3","speedup":1.968,"us":14113.904,"base_us":27776.959},{"gpu":"L40","B":1,"N":256,"impl":"P0-naive","speedup":0.465,"us":122.0,"base_us":56.709},{"gpu":"L40","B":1,"N":256,"impl":"P1-inductor","speedup":0.475,"us":119.488,"base_us":56.709},{"gpu":"L40","B":1,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":1.406,"us":40.332,"base_us":56.709},{"gpu":"L40","B":1,"N":256,"impl":"P2d-sdpa-cudnn","speedup":1.193,"us":47.552,"base_us":56.709},{"gpu":"L40","B":1,"N":256,"impl":"P3-triton","speedup":0.547,"us":103.608,"base_us":56.709},{"gpu":"L40","B":1,"N":256,"impl":"P4-fa2","speedup":0.798,"us":71.029,"base_us":56.709},{"gpu":"L40","B":1,"N":512,"impl":"P0-naive","speedup":0.301,"us":158.608,"base_us":47.676},{"gpu":"L40","B":1,"N":512,"impl":"P1-inductor","speedup":0.473,"us":100.74,"base_us":47.676},{"gpu":"L40","B":1,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":0.97,"us":49.128,"base_us":47.676},{"gpu":"L40","B":1,"N":512,"impl":"P2d-sdpa-cudnn","speedup":0.96,"us":49.642,"base_us":47.676},{"gpu":"L40","B":1,"N":512,"impl":"P3-triton","speedup":0.364,"us":131.08,"base_us":47.676},{"gpu":"L40","B":1,"N":512,"impl":"P4-fa2","speedup":0.754,"us":63.235,"base_us":47.676},{"gpu":"L40","B":1,"N":1024,"impl":"P0-naive","speedup":0.062,"us":1646.88,"base_us":102.828},{"gpu":"L40","B":1,"N":1024,"impl":"P1-inductor","speedup":0.418,"us":246.04,"base_us":102.828},{"gpu":"L40","B":1,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.692,"us":148.592,"base_us":102.828},{"gpu":"L40","B":1,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":0.985,"us":104.364,"base_us":102.828},{"gpu":"L40","B":1,"N":1024,"impl":"P3-triton","speedup":0.514,"us":200.136,"base_us":102.828},{"gpu":"L40","B":1,"N":1024,"impl":"P4-fa2","speedup":0.879,"us":116.932,"base_us":102.828},{"gpu":"L40","B":1,"N":2048,"impl":"P0-naive","speedup":0.041,"us":7629.552,"base_us":314.32},{"gpu":"L40","B":1,"N":2048,"impl":"P1-inductor","speedup":0.175,"us":1797.224,"base_us":314.32},{"gpu":"L40","B":1,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.739,"us":425.216,"base_us":314.32},{"gpu":"L40","B":1,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":1.011,"us":310.816,"base_us":314.32},{"gpu":"L40","B":1,"N":2048,"impl":"P3-triton","speedup":0.732,"us":429.28,"base_us":314.32},{"gpu":"L40","B":1,"N":2048,"impl":"P4-fa2","speedup":0.965,"us":325.832,"base_us":314.32},{"gpu":"L40","B":1,"N":4096,"impl":"P0-naive","speedup":0.034,"us":29536.264,"base_us":1018.768},{"gpu":"L40","B":1,"N":4096,"impl":"P1-inductor","speedup":0.133,"us":7641.616,"base_us":1018.768},{"gpu":"L40","B":1,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.631,"us":1613.376,"base_us":1018.768},{"gpu":"L40","B":1,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":0.997,"us":1022.048,"base_us":1018.768},{"gpu":"L40","B":1,"N":4096,"impl":"P3-triton","speedup":0.855,"us":1191.176,"base_us":1018.768},{"gpu":"L40","B":1,"N":4096,"impl":"P4-fa2","speedup":0.926,"us":1100.576,"base_us":1018.768},{"gpu":"L40","B":1,"N":8192,"impl":"P0-naive","speedup":0.038,"us":121054.888,"base_us":4581.296},{"gpu":"L40","B":1,"N":8192,"impl":"P1-inductor","speedup":0.145,"us":31546.912,"base_us":4581.296},{"gpu":"L40","B":1,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.732,"us":6260.152,"base_us":4581.296},{"gpu":"L40","B":1,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":0.986,"us":4647.248,"base_us":4581.296},{"gpu":"L40","B":1,"N":8192,"impl":"P3-triton","speedup":1.119,"us":4094.072,"base_us":4581.296},{"gpu":"L40","B":1,"N":8192,"impl":"P4-fa2","speedup":0.985,"us":4653.096,"base_us":4581.296},{"gpu":"L40","B":4,"N":256,"impl":"P0-naive","speedup":0.199,"us":239.552,"base_us":47.58},{"gpu":"L40","B":4,"N":256,"impl":"P1-inductor","speedup":0.293,"us":162.464,"base_us":47.58},{"gpu":"L40","B":4,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":0.964,"us":49.378,"base_us":47.58},{"gpu":"L40","B":4,"N":256,"impl":"P2d-sdpa-cudnn","speedup":0.684,"us":69.548,"base_us":47.58},{"gpu":"L40","B":4,"N":256,"impl":"P3-triton","speedup":0.369,"us":129.104,"base_us":47.58},{"gpu":"L40","B":4,"N":256,"impl":"P4-fa2","speedup":0.485,"us":98.024,"base_us":47.58},{"gpu":"L40","B":4,"N":512,"impl":"P0-naive","speedup":0.059,"us":1602.256,"base_us":94.008},{"gpu":"L40","B":4,"N":512,"impl":"P1-inductor","speedup":0.343,"us":273.912,"base_us":94.008},{"gpu":"L40","B":4,"N":512,"impl":"P1-inductor-nofuse","speedup":0.341,"us":275.76,"base_us":94.008},{"gpu":"L40","B":4,"N":512,"impl":"P1-inductor-where","speedup":0.343,"us":273.872,"base_us":94.008},{"gpu":"L40","B":4,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":0.641,"us":146.656,"base_us":94.008},{"gpu":"L40","B":4,"N":512,"impl":"P2d-sdpa-cudnn","speedup":0.987,"us":95.204,"base_us":94.008},{"gpu":"L40","B":4,"N":512,"impl":"P3-triton","speedup":0.49,"us":192.008,"base_us":94.008},{"gpu":"L40","B":4,"N":512,"impl":"P4-fa2","speedup":0.929,"us":101.24,"base_us":94.008},{"gpu":"L40","B":4,"N":1024,"impl":"P0-naive","speedup":0.042,"us":7724.464,"base_us":327.088},{"gpu":"L40","B":4,"N":1024,"impl":"P1-inductor","speedup":0.173,"us":1889.984,"base_us":327.088},{"gpu":"L40","B":4,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.727,"us":449.856,"base_us":327.088},{"gpu":"L40","B":4,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":1.028,"us":318.272,"base_us":327.088},{"gpu":"L40","B":4,"N":1024,"impl":"P3-triton","speedup":0.793,"us":412.232,"base_us":327.088},{"gpu":"L40","B":4,"N":1024,"impl":"P4-fa2","speedup":0.954,"us":342.848,"base_us":327.088},{"gpu":"L40","B":4,"N":2048,"impl":"P0-naive","speedup":0.043,"us":30146.696,"base_us":1311.168},{"gpu":"L40","B":4,"N":2048,"impl":"P1-inductor","speedup":0.185,"us":7093.184,"base_us":1311.168},{"gpu":"L40","B":4,"N":2048,"impl":"P1-inductor-nofuse","speedup":0.185,"us":7094.36,"base_us":1311.168},{"gpu":"L40","B":4,"N":2048,"impl":"P1-inductor-where","speedup":0.186,"us":7036.64,"base_us":1311.168},{"gpu":"L40","B":4,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.729,"us":1798.256,"base_us":1311.168},{"gpu":"L40","B":4,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":0.966,"us":1356.8,"base_us":1311.168},{"gpu":"L40","B":4,"N":2048,"impl":"P3-triton","speedup":1.044,"us":1256.472,"base_us":1311.168},{"gpu":"L40","B":4,"N":2048,"impl":"P4-fa2","speedup":1.159,"us":1131.168,"base_us":1311.168},{"gpu":"L40","B":4,"N":4096,"impl":"P0-naive","speedup":0.04,"us":117889.479,"base_us":4696.864},{"gpu":"L40","B":4,"N":4096,"impl":"P1-inductor","speedup":0.146,"us":32128.72,"base_us":4696.864},{"gpu":"L40","B":4,"N":4096,"impl":"P1-inductor-nofuse","speedup":0.15,"us":31399.256,"base_us":4696.864},{"gpu":"L40","B":4,"N":4096,"impl":"P1-inductor-where","speedup":0.147,"us":31923.84,"base_us":4696.864},{"gpu":"L40","B":4,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.74,"us":6344.784,"base_us":4696.864},{"gpu":"L40","B":4,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":0.982,"us":4782.328,"base_us":4696.864},{"gpu":"L40","B":4,"N":4096,"impl":"P3-triton","speedup":1.094,"us":4292.296,"base_us":4696.864},{"gpu":"L40","B":4,"N":4096,"impl":"P4-fa2","speedup":0.993,"us":4728.8,"base_us":4696.864},{"gpu":"L40","B":4,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.774,"us":25549.584,"base_us":19771.992},{"gpu":"L40","B":4,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":0.995,"us":19868.344,"base_us":19771.992},{"gpu":"L40","B":4,"N":8192,"impl":"P3-triton","speedup":1.186,"us":16666.008,"base_us":19771.992},{"gpu":"L40","B":4,"N":8192,"impl":"P4-fa2","speedup":1.001,"us":19746.208,"base_us":19771.992},{"gpu":"L40","B":16,"N":256,"impl":"P0-naive","speedup":0.137,"us":1710.624,"base_us":234.152},{"gpu":"L40","B":16,"N":256,"impl":"P1-inductor","speedup":0.528,"us":443.448,"base_us":234.152},{"gpu":"L40","B":16,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":1.057,"us":221.552,"base_us":234.152},{"gpu":"L40","B":16,"N":256,"impl":"P2d-sdpa-cudnn","speedup":1.045,"us":224.12,"base_us":234.152},{"gpu":"L40","B":16,"N":256,"impl":"P3-triton","speedup":0.822,"us":284.816,"base_us":234.152},{"gpu":"L40","B":16,"N":256,"impl":"P4-fa2","speedup":0.893,"us":262.32,"base_us":234.152},{"gpu":"L40","B":16,"N":512,"impl":"P0-naive","speedup":0.055,"us":7864.344,"base_us":432.856},{"gpu":"L40","B":16,"N":512,"impl":"P1-inductor","speedup":0.206,"us":2100.696,"base_us":432.856},{"gpu":"L40","B":16,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":0.886,"us":488.28,"base_us":432.856},{"gpu":"L40","B":16,"N":512,"impl":"P2d-sdpa-cudnn","speedup":1.048,"us":413.08,"base_us":432.856},{"gpu":"L40","B":16,"N":512,"impl":"P3-triton","speedup":0.878,"us":492.88,"base_us":432.856},{"gpu":"L40","B":16,"N":512,"impl":"P4-fa2","speedup":0.96,"us":451.008,"base_us":432.856},{"gpu":"L40","B":16,"N":1024,"impl":"P0-naive","speedup":0.035,"us":30534.64,"base_us":1064.688},{"gpu":"L40","B":16,"N":1024,"impl":"P1-inductor","speedup":0.145,"us":7319.824,"base_us":1064.688},{"gpu":"L40","B":16,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.526,"us":2022.84,"base_us":1064.688},{"gpu":"L40","B":16,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":0.648,"us":1642.472,"base_us":1064.688},{"gpu":"L40","B":16,"N":1024,"impl":"P3-triton","speedup":0.678,"us":1569.296,"base_us":1064.688},{"gpu":"L40","B":16,"N":1024,"impl":"P4-fa2","speedup":0.68,"us":1565.944,"base_us":1064.688},{"gpu":"L40","B":16,"N":2048,"impl":"P0-naive","speedup":0.045,"us":120883.856,"base_us":5486.448},{"gpu":"L40","B":16,"N":2048,"impl":"P1-inductor","speedup":0.175,"us":31414.232,"base_us":5486.448},{"gpu":"L40","B":16,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.731,"us":7509.672,"base_us":5486.448},{"gpu":"L40","B":16,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":0.981,"us":5595.104,"base_us":5486.448},{"gpu":"L40","B":16,"N":2048,"impl":"P3-triton","speedup":1.081,"us":5073.92,"base_us":5486.448},{"gpu":"L40","B":16,"N":2048,"impl":"P4-fa2","speedup":0.968,"us":5667.432,"base_us":5486.448},{"gpu":"L40","B":16,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.769,"us":26532.168,"base_us":20407.144},{"gpu":"L40","B":16,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":0.958,"us":21302.711,"base_us":20407.144},{"gpu":"L40","B":16,"N":4096,"impl":"P3-triton","speedup":1.134,"us":18001.568,"base_us":20407.144},{"gpu":"L40","B":16,"N":4096,"impl":"P4-fa2","speedup":0.974,"us":20960.352,"base_us":20407.144},{"gpu":"L40","B":16,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.724,"us":102402.506,"base_us":74168.896},{"gpu":"L40","B":16,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":0.993,"us":74668.831,"base_us":74168.896},{"gpu":"L40","B":16,"N":8192,"impl":"P3-triton","speedup":1.119,"us":66310.328,"base_us":74168.896},{"gpu":"L40","B":16,"N":8192,"impl":"P4-fa2","speedup":0.99,"us":74950.575,"base_us":74168.896},{"gpu":"L40S","B":1,"N":256,"impl":"P0-naive","speedup":0.572,"us":219.568,"base_us":125.504},{"gpu":"L40S","B":1,"N":256,"impl":"P1-inductor","speedup":0.569,"us":220.384,"base_us":125.504},{"gpu":"L40S","B":1,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":1.51,"us":83.12,"base_us":125.504},{"gpu":"L40S","B":1,"N":256,"impl":"P2d-sdpa-cudnn","speedup":2.312,"us":54.277,"base_us":125.504},{"gpu":"L40S","B":1,"N":256,"impl":"P3-triton","speedup":1.124,"us":111.664,"base_us":125.504},{"gpu":"L40S","B":1,"N":256,"impl":"P4-fa2","speedup":0.81,"us":154.896,"base_us":125.504},{"gpu":"L40S","B":1,"N":512,"impl":"P0-naive","speedup":0.302,"us":330.08,"base_us":99.544},{"gpu":"L40S","B":1,"N":512,"impl":"P1-inductor","speedup":0.674,"us":147.648,"base_us":99.544},{"gpu":"L40S","B":1,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":1.173,"us":84.872,"base_us":99.544},{"gpu":"L40S","B":1,"N":512,"impl":"P2d-sdpa-cudnn","speedup":1.858,"us":53.568,"base_us":99.544},{"gpu":"L40S","B":1,"N":512,"impl":"P3-triton","speedup":0.465,"us":214.128,"base_us":99.544},{"gpu":"L40S","B":1,"N":512,"impl":"P4-fa2","speedup":0.556,"us":178.992,"base_us":99.544},{"gpu":"L40S","B":1,"N":1024,"impl":"P0-naive","speedup":0.056,"us":1638.208,"base_us":91.216},{"gpu":"L40S","B":1,"N":1024,"impl":"P1-inductor","speedup":0.372,"us":245.04,"base_us":91.216},{"gpu":"L40S","B":1,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.804,"us":113.52,"base_us":91.216},{"gpu":"L40S","B":1,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":0.99,"us":92.112,"base_us":91.216},{"gpu":"L40S","B":1,"N":1024,"impl":"P3-triton","speedup":0.464,"us":196.592,"base_us":91.216},{"gpu":"L40S","B":1,"N":1024,"impl":"P4-fa2","speedup":0.507,"us":179.856,"base_us":91.216},{"gpu":"L40S","B":1,"N":2048,"impl":"P0-naive","speedup":0.032,"us":7687.776,"base_us":243.872},{"gpu":"L40S","B":1,"N":2048,"impl":"P1-inductor","speedup":0.136,"us":1796.992,"base_us":243.872},{"gpu":"L40S","B":1,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.717,"us":340.272,"base_us":243.872},{"gpu":"L40S","B":1,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":0.984,"us":247.952,"base_us":243.872},{"gpu":"L40S","B":1,"N":2048,"impl":"P3-triton","speedup":0.647,"us":377.008,"base_us":243.872},{"gpu":"L40S","B":1,"N":2048,"impl":"P4-fa2","speedup":0.798,"us":305.616,"base_us":243.872},{"gpu":"L40S","B":1,"N":4096,"impl":"P0-naive","speedup":0.03,"us":29687.28,"base_us":884.016},{"gpu":"L40S","B":1,"N":4096,"impl":"P1-inductor","speedup":0.132,"us":6705.952,"base_us":884.016},{"gpu":"L40S","B":1,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.73,"us":1211.776,"base_us":884.016},{"gpu":"L40S","B":1,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":1.002,"us":882.592,"base_us":884.016},{"gpu":"L40S","B":1,"N":4096,"impl":"P3-triton","speedup":1.013,"us":872.304,"base_us":884.016},{"gpu":"L40S","B":1,"N":4096,"impl":"P4-fa2","speedup":0.969,"us":912.0,"base_us":884.016},{"gpu":"L40S","B":1,"N":8192,"impl":"P0-naive","speedup":0.028,"us":121182.419,"base_us":3335.136},{"gpu":"L40S","B":1,"N":8192,"impl":"P1-inductor","speedup":0.117,"us":28533.567,"base_us":3335.136},{"gpu":"L40S","B":1,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.74,"us":4508.64,"base_us":3335.136},{"gpu":"L40S","B":1,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":0.976,"us":3417.136,"base_us":3335.136},{"gpu":"L40S","B":1,"N":8192,"impl":"P3-triton","speedup":1.111,"us":3001.056,"base_us":3335.136},{"gpu":"L40S","B":1,"N":8192,"impl":"P4-fa2","speedup":0.982,"us":3396.88,"base_us":3335.136},{"gpu":"L40S","B":4,"N":256,"impl":"P0-naive","speedup":0.299,"us":333.616,"base_us":99.72},{"gpu":"L40S","B":4,"N":256,"impl":"P1-inductor","speedup":0.449,"us":221.872,"base_us":99.72},{"gpu":"L40S","B":4,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":1.22,"us":81.72,"base_us":99.72},{"gpu":"L40S","B":4,"N":256,"impl":"P2d-sdpa-cudnn","speedup":1.849,"us":53.936,"base_us":99.72},{"gpu":"L40S","B":4,"N":256,"impl":"P3-triton","speedup":0.461,"us":216.288,"base_us":99.72},{"gpu":"L40S","B":4,"N":256,"impl":"P4-fa2","speedup":0.565,"us":176.432,"base_us":99.72},{"gpu":"L40S","B":4,"N":512,"impl":"P0-naive","speedup":0.049,"us":1608.576,"base_us":79.584},{"gpu":"L40S","B":4,"N":512,"impl":"P1-inductor","speedup":0.291,"us":273.024,"base_us":79.584},{"gpu":"L40S","B":4,"N":512,"impl":"P1-inductor-nofuse","speedup":0.287,"us":276.832,"base_us":79.584},{"gpu":"L40S","B":4,"N":512,"impl":"P1-inductor-where","speedup":0.279,"us":285.424,"base_us":79.584},{"gpu":"L40S","B":4,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":0.663,"us":119.984,"base_us":79.584},{"gpu":"L40S","B":4,"N":512,"impl":"P2d-sdpa-cudnn","speedup":0.994,"us":80.075,"base_us":79.584},{"gpu":"L40S","B":4,"N":512,"impl":"P3-triton","speedup":0.409,"us":194.72,"base_us":79.584},{"gpu":"L40S","B":4,"N":512,"impl":"P4-fa2","speedup":0.71,"us":112.12,"base_us":79.584},{"gpu":"L40S","B":4,"N":1024,"impl":"P0-naive","speedup":0.038,"us":7767.392,"base_us":296.576},{"gpu":"L40S","B":4,"N":1024,"impl":"P1-inductor","speedup":0.155,"us":1912.736,"base_us":296.576},{"gpu":"L40S","B":4,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.774,"us":383.2,"base_us":296.576},{"gpu":"L40S","B":4,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":1.009,"us":294.0,"base_us":296.576},{"gpu":"L40S","B":4,"N":1024,"impl":"P3-triton","speedup":0.846,"us":350.496,"base_us":296.576},{"gpu":"L40S","B":4,"N":1024,"impl":"P4-fa2","speedup":0.955,"us":310.576,"base_us":296.576},{"gpu":"L40S","B":4,"N":2048,"impl":"P0-naive","speedup":0.032,"us":30222.8,"base_us":960.736},{"gpu":"L40S","B":4,"N":2048,"impl":"P1-inductor","speedup":0.14,"us":6839.856,"base_us":960.736},{"gpu":"L40S","B":4,"N":2048,"impl":"P1-inductor-nofuse","speedup":0.141,"us":6827.056,"base_us":960.736},{"gpu":"L40S","B":4,"N":2048,"impl":"P1-inductor-where","speedup":0.14,"us":6885.824,"base_us":960.736},{"gpu":"L40S","B":4,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.749,"us":1283.072,"base_us":960.736},{"gpu":"L40S","B":4,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":1.007,"us":953.664,"base_us":960.736},{"gpu":"L40S","B":4,"N":2048,"impl":"P3-triton","speedup":1.042,"us":921.808,"base_us":960.736},{"gpu":"L40S","B":4,"N":2048,"impl":"P4-fa2","speedup":0.99,"us":970.32,"base_us":960.736},{"gpu":"L40S","B":4,"N":4096,"impl":"P0-naive","speedup":0.031,"us":118154.831,"base_us":3611.728},{"gpu":"L40S","B":4,"N":4096,"impl":"P1-inductor","speedup":0.136,"us":26611.616,"base_us":3611.728},{"gpu":"L40S","B":4,"N":4096,"impl":"P1-inductor-nofuse","speedup":0.134,"us":26872.496,"base_us":3611.728},{"gpu":"L40S","B":4,"N":4096,"impl":"P1-inductor-where","speedup":0.136,"us":26589.36,"base_us":3611.728},{"gpu":"L40S","B":4,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.727,"us":4965.856,"base_us":3611.728},{"gpu":"L40S","B":4,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":0.983,"us":3674.256,"base_us":3611.728},{"gpu":"L40S","B":4,"N":4096,"impl":"P3-triton","speedup":1.12,"us":3224.992,"base_us":3611.728},{"gpu":"L40S","B":4,"N":4096,"impl":"P4-fa2","speedup":1.001,"us":3607.536,"base_us":3611.728},{"gpu":"L40S","B":4,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.697,"us":19235.472,"base_us":13413.776},{"gpu":"L40S","B":4,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":0.996,"us":13462.096,"base_us":13413.776},{"gpu":"L40S","B":4,"N":8192,"impl":"P3-triton","speedup":1.14,"us":11769.68,"base_us":13413.776},{"gpu":"L40S","B":4,"N":8192,"impl":"P4-fa2","speedup":0.991,"us":13531.616,"base_us":13413.776},{"gpu":"L40S","B":16,"N":256,"impl":"P0-naive","speedup":0.143,"us":1756.368,"base_us":251.456},{"gpu":"L40S","B":16,"N":256,"impl":"P1-inductor","speedup":0.597,"us":421.312,"base_us":251.456},{"gpu":"L40S","B":16,"N":256,"impl":"P2b-sdpa-mem-eff","speedup":1.103,"us":227.888,"base_us":251.456},{"gpu":"L40S","B":16,"N":256,"impl":"P2d-sdpa-cudnn","speedup":1.102,"us":228.272,"base_us":251.456},{"gpu":"L40S","B":16,"N":256,"impl":"P3-triton","speedup":0.673,"us":373.6,"base_us":251.456},{"gpu":"L40S","B":16,"N":256,"impl":"P4-fa2","speedup":0.984,"us":255.456,"base_us":251.456},{"gpu":"L40S","B":16,"N":512,"impl":"P0-naive","speedup":0.053,"us":7900.256,"base_us":419.36},{"gpu":"L40S","B":16,"N":512,"impl":"P1-inductor","speedup":0.201,"us":2091.392,"base_us":419.36},{"gpu":"L40S","B":16,"N":512,"impl":"P2b-sdpa-mem-eff","speedup":0.926,"us":453.024,"base_us":419.36},{"gpu":"L40S","B":16,"N":512,"impl":"P2d-sdpa-cudnn","speedup":1.013,"us":413.952,"base_us":419.36},{"gpu":"L40S","B":16,"N":512,"impl":"P3-triton","speedup":0.843,"us":497.744,"base_us":419.36},{"gpu":"L40S","B":16,"N":512,"impl":"P4-fa2","speedup":0.834,"us":502.736,"base_us":419.36},{"gpu":"L40S","B":16,"N":1024,"impl":"P0-naive","speedup":0.035,"us":30545.6,"base_us":1055.92},{"gpu":"L40S","B":16,"N":1024,"impl":"P1-inductor","speedup":0.143,"us":7363.696,"base_us":1055.92},{"gpu":"L40S","B":16,"N":1024,"impl":"P2b-sdpa-mem-eff","speedup":0.721,"us":1465.2,"base_us":1055.92},{"gpu":"L40S","B":16,"N":1024,"impl":"P2d-sdpa-cudnn","speedup":0.985,"us":1072.24,"base_us":1055.92},{"gpu":"L40S","B":16,"N":1024,"impl":"P3-triton","speedup":0.977,"us":1081.168,"base_us":1055.92},{"gpu":"L40S","B":16,"N":1024,"impl":"P4-fa2","speedup":0.996,"us":1060.592,"base_us":1055.92},{"gpu":"L40S","B":16,"N":2048,"impl":"P0-naive","speedup":0.033,"us":121031.521,"base_us":3950.368},{"gpu":"L40S","B":16,"N":2048,"impl":"P1-inductor","speedup":0.147,"us":26850.096,"base_us":3950.368},{"gpu":"L40S","B":16,"N":2048,"impl":"P2b-sdpa-mem-eff","speedup":0.744,"us":5306.8,"base_us":3950.368},{"gpu":"L40S","B":16,"N":2048,"impl":"P2d-sdpa-cudnn","speedup":0.971,"us":4069.04,"base_us":3950.368},{"gpu":"L40S","B":16,"N":2048,"impl":"P3-triton","speedup":1.1,"us":3591.52,"base_us":3950.368},{"gpu":"L40S","B":16,"N":2048,"impl":"P4-fa2","speedup":0.988,"us":3996.416,"base_us":3950.368},{"gpu":"L40S","B":16,"N":4096,"impl":"P2b-sdpa-mem-eff","speedup":0.71,"us":20646.561,"base_us":14654.336},{"gpu":"L40S","B":16,"N":4096,"impl":"P2d-sdpa-cudnn","speedup":0.989,"us":14820.544,"base_us":14654.336},{"gpu":"L40S","B":16,"N":4096,"impl":"P3-triton","speedup":1.147,"us":12770.976,"base_us":14654.336},{"gpu":"L40S","B":16,"N":4096,"impl":"P4-fa2","speedup":0.944,"us":15524.448,"base_us":14654.336},{"gpu":"L40S","B":16,"N":8192,"impl":"P2b-sdpa-mem-eff","speedup":0.731,"us":75090.561,"base_us":54859.858},{"gpu":"L40S","B":16,"N":8192,"impl":"P2d-sdpa-cudnn","speedup":0.98,"us":55965.408,"base_us":54859.858},{"gpu":"L40S","B":16,"N":8192,"impl":"P3-triton","speedup":1.136,"us":48310.608,"base_us":54859.858},{"gpu":"L40S","B":16,"N":8192,"impl":"P4-fa2","speedup":0.983,"us":55782.255,"base_us":54859.858}],"decode_slice":{"hkv":8,"head_dim":128,"dtype":"bf16","launch":"eager","cache":"warm"},"decode_latency_b1":{"A10":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[186.528,184.128,200.384,239.136,333.536,597.856]},"D1-inductor":{"x":[512,2048,8192],"y":[193.248,189.696,352.384]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[62.587,62.096,64.549,66.229,102.688,188.416]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[49.644,52.36,51.072,62.288,103.488,189.776]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[38.8,67.685,144.592,250.496,464.32,886.192]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[157.136,155.216,157.472,155.76,174.576,241.104]}},"A100":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[271.248,192.128,228.128,219.696,233.376,271.248]},"D1-inductor":{"x":[512,2048,8192],"y":[197.264,211.6,242.816]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[77.232,60.512,60.741,61.509,63.349,83.352]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[82.939,49.728,50.716,51.004,58.768,74.597]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[61.184,90.088,199.536,342.464,634.656,1202.928]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[174.32,160.88,165.104,167.296,193.2,184.464]}},"RTX 5090":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[31.085,30.355,32.421,49.364,132.112,201.408]},"D1-inductor":{"x":[512,2048,8192],"y":[37.594,37.968,99.072]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[14.094,14.12,14.673,21.552,28.112,43.336]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[11.585,11.825,12.608,21.429,27.849,41.412]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[25.307,25.799,25.125,27.031,29.485,43.82]}},"H100":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[263.936,279.104,269.744,296.528,260.528,290.448]},"D1-inductor":{"x":[512,2048,8192],"y":[306.736,311.84,313.856]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[140.848,140.736,140.48,140.816,125.664,142.32]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[102.096,102.352,100.792,102.216,103.568,103.784]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[78.128,89.784,165.328,268.992,456.672,859.712]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[269.728,295.888,289.952,285.008,259.6,268.496]}},"L40":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[79.944,82.32,86.188,87.748,245.192,484.464]},"D1-inductor":{"x":[512,2048,8192],"y":[120.816,120.448,231.448]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[34.899,34.893,35.246,35.974,37.642,50.046]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[27.614,28.333,28.093,29.273,32.16,49.02]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[22.894,43.444,84.428,166.776,312.632,605.816]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[80.6,80.816,81.028,81.96,81.304,85.916]}},"L40S":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[140.704,303.584,228.048,232.512,312.512,506.704]},"D1-inductor":{"x":[512,2048,8192],"y":[244.544,240.448,287.024]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[67.632,44.92,66.619,67.584,41.628,67.995]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[54.672,55.563,55.717,55.803,33.243,56.928]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[40.198,47.44,92.792,183.232,329.888,606.176]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[215.472,213.84,219.312,211.264,213.088,198.912]}}},"decode_bw_b1":{"A10":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[44.972,91.117,167.451,280.631,402.409,448.997]},"D1-inductor":{"x":[512,2048,8192],"y":[43.409,176.885,380.885]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[33.508,67.545,129.957,253.32,326.761,356.174]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[42.244,80.105,164.251,269.349,324.235,353.621]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[54.05,61.968,58.016,66.976,72.266,75.727]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[13.346,27.022,53.27,107.712,192.205,278.34]}},"A100":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[30.926,87.323,147.086,305.462,575.114,989.631]},"D1-inductor":{"x":[512,2048,8192],"y":[42.525,158.575,552.755]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[27.154,69.314,138.104,272.759,529.673,805.126]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[25.286,84.345,165.404,328.939,570.964,899.615]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[34.276,46.558,42.041,48.99,52.87,55.788]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[12.03,26.071,50.808,100.285,173.677,363.805]}},"RTX 5090":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[269.857,552.706,1034.949,1359.47,1015.939,1332.794]},"D1-inductor":{"x":[512,2048,8192],"y":[223.139,883.756,1354.749]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[148.802,297.047,571.695,778.453,1193.598,1548.571]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[181.021,354.696,665.34,782.909,1204.864,1620.517]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[82.867,162.577,333.881,620.669,1138.004,1531.467]}},"H100":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[31.783,60.111,124.394,226.315,515.176,924.212]},"D1-inductor":{"x":[512,2048,8192],"y":[27.348,107.601,427.641]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[14.889,29.803,59.714,119.143,267.017,471.535]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[20.541,40.979,83.227,164.135,323.985,646.621]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[26.843,46.715,50.739,62.371,73.476,78.06]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[7.775,14.175,28.931,58.866,129.254,249.944]}},"L40":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[104.931,203.805,390.853,764.792,547.401,554.088]},"D1-inductor":{"x":[512,2048,8192],"y":[69.433,278.617,579.906]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[60.092,120.205,238.0,466.37,891.428,1340.944]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[75.946,148.034,298.625,573.123,1043.36,1369.01]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[91.603,96.545,99.359,100.598,107.329,110.775]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[26.019,51.899,103.537,204.717,412.703,781.099]}},"L40S":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[59.619,55.264,147.138,288.625,429.48,529.768]},"D1-inductor":{"x":[512,2048,8192],"y":[34.303,139.55,467.618]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[31.008,93.373,125.92,248.242,806.054,986.972]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[38.359,75.488,150.557,300.653,1009.378,1178.838]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[52.17,88.413,90.402,91.563,101.715,110.709]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[9.733,19.614,38.25,79.414,157.467,337.38]}}},"decode_latency_b32":{"A10":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[593.184,1131.52,2261.248,4468.784,8900.944,17872.144]},"D1-inductor":{"x":[512,2048,8192],"y":[601.872,2175.264,8487.392]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[176.592,304.688,611.024,1377.968,2697.2,5040.368]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[172.4,305.104,578.656,1317.856,2634.4,4975.152]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[174.832,306.176,575.456,1107.056,2174.08,4299.824]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[223.632,358.08,643.792,1344.368,2650.448,5023.312]}},"A100":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[251.488,402.736,719.888,1385.824,2763.584,5376.96]},"D1-inductor":{"x":[512,2048,8192],"y":[307.36,894.208,3208.832]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[92.44,173.536,272.48,495.728,958.88,1833.216]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[89.6,160.96,264.992,483.856,932.368,1793.456]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[82.408,156.992,248.384,443.76,844.144,1638.192]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[161.552,222.192,318.336,537.664,990.864,1850.768]}},"RTX 5090":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[189.232,359.088,684.672,1351.12,2653.744,5283.648]},"D1-inductor":{"x":[512,2048,8192],"y":[184.32,661.248,2558.064]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[60.133,119.84,184.144,347.136,658.656,1294.816]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[57.803,115.792,182.224,343.936,657.392,1293.664]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[61.093,126.016,192.176,353.888,667.52,1304.64]}},"H100":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[281.84,330.48,437.552,785.344,1515.712,2963.264]},"D1-inductor":{"x":[512,2048,8192],"y":[299.728,543.136,1788.48]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[100.76,110.624,173.376,300.528,530.96,932.144]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[78.544,92.072,135.792,277.984,469.952,863.696]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[82.096,102.496,166.688,329.2,569.168,1120.032]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[254.4,267.52,222.576,367.936,507.392,936.064]}},"L40":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[423.688,805.408,1564.328,3109.36,6064.896,12259.944]},"D1-inductor":{"x":[512,2048,8192],"y":[403.288,1448.312,5616.0]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[69.109,205.792,381.768,739.536,1448.416,2868.288]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[66.536,211.64,390.424,759.888,1499.264,2985.712]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[29.983,205.48,383.76,748.112,1474.576,2917.504]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[87.084,243.704,445.16,791.264,1528.24,3018.632]}},"L40S":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[461.168,826.4,1582.624,3131.696,6109.856,12307.536]},"D1-inductor":{"x":[512,2048,8192],"y":[428.512,1429.52,5671.856]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[57.104,221.632,386.528,742.688,1450.56,2869.248]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[49.512,220.848,389.184,763.792,1499.424,2982.24]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[32.36,211.952,390.256,755.68,1479.36,2932.304]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[109.056,285.328,449.248,819.952,1548.208,3033.152]}}},"decode_bw_b32":{"A10":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[452.533,474.469,474.845,480.552,482.529,480.633]},"D1-inductor":{"x":[512,2048,8192],"y":[446.001,493.614,506.041]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[380.022,440.509,439.321,389.611,398.095,426.057]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[389.263,439.908,463.895,407.382,407.585,431.642]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[383.848,438.368,466.474,484.954,493.883,499.435]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[300.086,374.826,416.96,399.348,405.117,427.504]}},"A100":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[1067.389,1333.059,1491.54,1549.608,1554.129,1597.545]},"D1-inductor":{"x":[512,2048,8192],"y":[873.358,1200.774,1338.483]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[725.972,773.429,985.157,1082.995,1119.787,1171.43]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[748.983,833.858,1012.995,1109.568,1151.629,1197.4]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[814.349,854.934,1080.728,1209.823,1271.989,1310.886]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[415.401,604.062,843.246,998.525,1083.642,1160.32]}},"RTX 5090":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[1418.552,1495.096,1568.257,1589.41,1618.456,1625.758]},"D1-inductor":{"x":[512,2048,8192],"y":[1456.356,1623.811,1678.991]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[1116.001,1119.974,1457.748,1546.572,1630.201,1658.524]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[1160.999,1159.128,1473.107,1560.962,1633.336,1660.001]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[1098.465,1065.085,1396.821,1517.064,1608.554,1646.035]}},"H100":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[952.439,1624.519,2453.975,2734.45,2833.63,2898.808]},"D1-inductor":{"x":[512,2048,8192],"y":[895.597,1976.93,2401.462]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[666.027,1213.279,1548.285,1786.426,2022.265,2303.811]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[854.411,1457.748,1976.813,1931.302,2284.79,2486.388]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[817.444,1309.492,1610.407,1630.835,1886.511,1917.341]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[263.793,501.711,1206.04,1459.142,2116.198,2294.163]}},"L40":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[633.574,666.586,686.392,690.651,708.168,700.65]},"D1-inductor":{"x":[512,2048,8192],"y":[665.619,741.375,764.773]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[971.054,652.244,703.138,725.956,741.321,748.7]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[1008.61,634.203,687.549,706.513,716.179,719.254]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[2238.274,653.209,699.488,717.634,728.17,736.069]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[770.625,550.762,604.687,678.498,702.6,711.411]}},"L40S":{"D0-naive-kv":{"x":[512,1024,2048,4096,8192,16384],"y":[582.077,649.65,678.457,685.725,702.957,697.941]},"D1-inductor":{"x":[512,2048,8192],"y":[626.436,751.121,757.242]},"D2-sdpa":{"x":[512,1024,2048,4096,8192,16384],"y":[1175.204,605.588,694.479,722.875,740.226,748.448]},"D3-fa-kvcache":{"x":[512,1024,2048,4096,8192,16384],"y":[1355.406,607.738,689.739,702.902,716.103,720.091]},"D4-flashinfer":{"x":[512,1024,2048,4096,8192,16384],"y":[2073.822,633.246,687.845,710.447,725.815,732.354]},"D6-fa-prefill-at-1":{"x":[512,1024,2048,4096,8192,16384],"y":[615.362,470.398,597.522,654.759,693.538,708.004]}}},"decode_winner_map":{"A10":[{"N":512,"y":1,"impl":"D4-flashinfer","us":38.8},{"N":512,"y":8,"impl":"D4-flashinfer","us":47.98},{"N":512,"y":32,"impl":"D3-fa-kvcache","us":172.4},{"N":512,"y":64,"impl":"D4-flashinfer","us":308.272},{"N":1024,"y":1,"impl":"D3-fa-kvcache","us":52.36},{"N":1024,"y":8,"impl":"D4-flashinfer","us":91.096},{"N":1024,"y":32,"impl":"D2-sdpa","us":304.688},{"N":1024,"y":64,"impl":"D4-flashinfer","us":575.088},{"N":2048,"y":1,"impl":"D3-fa-kvcache","us":51.072},{"N":2048,"y":8,"impl":"D4-flashinfer","us":175.968},{"N":2048,"y":32,"impl":"D4-flashinfer","us":575.456},{"N":2048,"y":64,"impl":"D4-flashinfer","us":1117.424},{"N":4096,"y":1,"impl":"D3-fa-kvcache","us":62.288},{"N":4096,"y":8,"impl":"D2-sdpa","us":310.176},{"N":4096,"y":32,"impl":"D4-flashinfer","us":1107.056},{"N":4096,"y":64,"impl":"D4-flashinfer","us":2193.136},{"N":8192,"y":1,"impl":"D2-sdpa","us":102.688},{"N":8192,"y":8,"impl":"D2-sdpa","us":566.896},{"N":8192,"y":32,"impl":"D4-flashinfer","us":2174.08},{"N":8192,"y":64,"impl":"D4-flashinfer","us":4324.672},{"N":16384,"y":1,"impl":"D2-sdpa","us":188.416},{"N":16384,"y":8,"impl":"D2-sdpa","us":1078.672},{"N":16384,"y":32,"impl":"D4-flashinfer","us":4299.824},{"N":16384,"y":64,"impl":"D4-flashinfer","us":8625.44}],"A100":[{"N":512,"y":1,"impl":"D4-flashinfer","us":61.184},{"N":512,"y":8,"impl":"D3-fa-kvcache","us":49.484},{"N":512,"y":32,"impl":"D4-flashinfer","us":82.408},{"N":512,"y":64,"impl":"D4-flashinfer","us":111.024},{"N":1024,"y":1,"impl":"D3-fa-kvcache","us":49.728},{"N":1024,"y":8,"impl":"D3-fa-kvcache","us":49.696},{"N":1024,"y":32,"impl":"D4-flashinfer","us":156.992},{"N":1024,"y":64,"impl":"D4-flashinfer","us":217.568},{"N":2048,"y":1,"impl":"D3-fa-kvcache","us":50.716},{"N":2048,"y":8,"impl":"D3-fa-kvcache","us":67.536},{"N":2048,"y":32,"impl":"D4-flashinfer","us":248.384},{"N":2048,"y":64,"impl":"D4-flashinfer","us":367.744},{"N":4096,"y":1,"impl":"D3-fa-kvcache","us":51.004},{"N":4096,"y":8,"impl":"D3-fa-kvcache","us":111.968},{"N":4096,"y":32,"impl":"D4-flashinfer","us":443.76},{"N":4096,"y":64,"impl":"D4-flashinfer","us":683.424},{"N":8192,"y":1,"impl":"D3-fa-kvcache","us":58.768},{"N":8192,"y":8,"impl":"D3-fa-kvcache","us":210.08},{"N":8192,"y":32,"impl":"D4-flashinfer","us":844.144},{"N":8192,"y":64,"impl":"D4-flashinfer","us":1324.944},{"N":16384,"y":1,"impl":"D3-fa-kvcache","us":74.597},{"N":16384,"y":8,"impl":"D3-fa-kvcache","us":356.352},{"N":16384,"y":32,"impl":"D4-flashinfer","us":1638.192},{"N":16384,"y":64,"impl":"D4-flashinfer","us":2650.032}],"RTX 5090":[{"N":512,"y":1,"impl":"D3-fa-kvcache","us":11.585},{"N":512,"y":8,"impl":"D3-fa-kvcache","us":17.145},{"N":512,"y":32,"impl":"D3-fa-kvcache","us":57.803},{"N":512,"y":64,"impl":"D3-fa-kvcache","us":117.792},{"N":1024,"y":1,"impl":"D3-fa-kvcache","us":11.825},{"N":1024,"y":8,"impl":"D3-fa-kvcache","us":23.584},{"N":1024,"y":32,"impl":"D3-fa-kvcache","us":115.792},{"N":1024,"y":64,"impl":"D3-fa-kvcache","us":222.24},{"N":2048,"y":1,"impl":"D3-fa-kvcache","us":12.608},{"N":2048,"y":8,"impl":"D3-fa-kvcache","us":36.55},{"N":2048,"y":32,"impl":"D3-fa-kvcache","us":182.224},{"N":2048,"y":64,"impl":"D3-fa-kvcache","us":429.136},{"N":4096,"y":1,"impl":"D3-fa-kvcache","us":21.429},{"N":4096,"y":8,"impl":"D3-fa-kvcache","us":94.48},{"N":4096,"y":32,"impl":"D3-fa-kvcache","us":343.936},{"N":4096,"y":64,"impl":"D3-fa-kvcache","us":843.024},{"N":8192,"y":1,"impl":"D3-fa-kvcache","us":27.849},{"N":8192,"y":8,"impl":"D3-fa-kvcache","us":180.208},{"N":8192,"y":32,"impl":"D3-fa-kvcache","us":657.392},{"N":8192,"y":64,"impl":"D3-fa-kvcache","us":1662.016},{"N":16384,"y":1,"impl":"D3-fa-kvcache","us":41.412},{"N":16384,"y":8,"impl":"D3-fa-kvcache","us":340.592},{"N":16384,"y":32,"impl":"D3-fa-kvcache","us":1293.664},{"N":16384,"y":64,"impl":"D3-fa-kvcache","us":3302.656}],"H100":[{"N":512,"y":1,"impl":"D4-flashinfer","us":78.128},{"N":512,"y":8,"impl":"D4-flashinfer","us":47.8},{"N":512,"y":32,"impl":"D3-fa-kvcache","us":78.544},{"N":512,"y":64,"impl":"D3-fa-kvcache","us":95.592},{"N":1024,"y":1,"impl":"D4-flashinfer","us":89.784},{"N":1024,"y":8,"impl":"D4-flashinfer","us":89.664},{"N":1024,"y":32,"impl":"D3-fa-kvcache","us":92.072},{"N":1024,"y":64,"impl":"D4-flashinfer","us":146.848},{"N":2048,"y":1,"impl":"D3-fa-kvcache","us":100.792},{"N":2048,"y":8,"impl":"D3-fa-kvcache","us":57.152},{"N":2048,"y":32,"impl":"D3-fa-kvcache","us":135.792},{"N":2048,"y":64,"impl":"D3-fa-kvcache","us":266.944},{"N":4096,"y":1,"impl":"D3-fa-kvcache","us":102.216},{"N":4096,"y":8,"impl":"D3-fa-kvcache","us":64.987},{"N":4096,"y":32,"impl":"D3-fa-kvcache","us":277.984},{"N":4096,"y":64,"impl":"D3-fa-kvcache","us":460.464},{"N":8192,"y":1,"impl":"D3-fa-kvcache","us":103.568},{"N":8192,"y":8,"impl":"D3-fa-kvcache","us":161.888},{"N":8192,"y":32,"impl":"D3-fa-kvcache","us":469.952},{"N":8192,"y":64,"impl":"D3-fa-kvcache","us":893.456},{"N":16384,"y":1,"impl":"D3-fa-kvcache","us":103.784},{"N":16384,"y":8,"impl":"D3-fa-kvcache","us":220.096},{"N":16384,"y":32,"impl":"D3-fa-kvcache","us":863.696},{"N":16384,"y":64,"impl":"D3-fa-kvcache","us":1706.128}],"L40":[{"N":512,"y":1,"impl":"D4-flashinfer","us":22.894},{"N":512,"y":8,"impl":"D4-flashinfer","us":23.038},{"N":512,"y":32,"impl":"D4-flashinfer","us":29.983},{"N":512,"y":64,"impl":"D2-sdpa","us":206.008},{"N":1024,"y":1,"impl":"D3-fa-kvcache","us":28.333},{"N":1024,"y":8,"impl":"D3-fa-kvcache","us":28.823},{"N":1024,"y":32,"impl":"D4-flashinfer","us":205.48},{"N":1024,"y":64,"impl":"D2-sdpa","us":385.088},{"N":2048,"y":1,"impl":"D3-fa-kvcache","us":28.093},{"N":2048,"y":8,"impl":"D3-fa-kvcache","us":44.932},{"N":2048,"y":32,"impl":"D2-sdpa","us":381.768},{"N":2048,"y":64,"impl":"D2-sdpa","us":783.944},{"N":4096,"y":1,"impl":"D3-fa-kvcache","us":29.273},{"N":4096,"y":8,"impl":"D4-flashinfer","us":209.744},{"N":4096,"y":32,"impl":"D2-sdpa","us":739.536},{"N":4096,"y":64,"impl":"D2-sdpa","us":1451.392},{"N":8192,"y":1,"impl":"D3-fa-kvcache","us":32.16},{"N":8192,"y":8,"impl":"D4-flashinfer","us":392.72},{"N":8192,"y":32,"impl":"D2-sdpa","us":1448.416},{"N":8192,"y":64,"impl":"D2-sdpa","us":2872.848},{"N":16384,"y":1,"impl":"D3-fa-kvcache","us":49.02},{"N":16384,"y":8,"impl":"D2-sdpa","us":757.696},{"N":16384,"y":32,"impl":"D2-sdpa","us":2868.288},{"N":16384,"y":64,"impl":"D2-sdpa","us":5714.6}],"L40S":[{"N":512,"y":1,"impl":"D4-flashinfer","us":40.198},{"N":512,"y":8,"impl":"D4-flashinfer","us":40.204},{"N":512,"y":32,"impl":"D4-flashinfer","us":32.36},{"N":512,"y":64,"impl":"D2-sdpa","us":222.624},{"N":1024,"y":1,"impl":"D2-sdpa","us":44.92},{"N":1024,"y":8,"impl":"D4-flashinfer","us":43.656},{"N":1024,"y":32,"impl":"D4-flashinfer","us":211.952},{"N":1024,"y":64,"impl":"D2-sdpa","us":382.24},{"N":2048,"y":1,"impl":"D3-fa-kvcache","us":55.717},{"N":2048,"y":8,"impl":"D3-fa-kvcache","us":34.938},{"N":2048,"y":32,"impl":"D2-sdpa","us":386.528},{"N":2048,"y":64,"impl":"D2-sdpa","us":738.576},{"N":4096,"y":1,"impl":"D3-fa-kvcache","us":55.803},{"N":4096,"y":8,"impl":"D3-fa-kvcache","us":235.04},{"N":4096,"y":32,"impl":"D2-sdpa","us":742.688},{"N":4096,"y":64,"impl":"D2-sdpa","us":1449.392},{"N":8192,"y":1,"impl":"D3-fa-kvcache","us":33.243},{"N":8192,"y":8,"impl":"D3-fa-kvcache","us":406.256},{"N":8192,"y":32,"impl":"D2-sdpa","us":1450.56},{"N":8192,"y":64,"impl":"D2-sdpa","us":2865.136},{"N":16384,"y":1,"impl":"D3-fa-kvcache","us":56.928},{"N":16384,"y":8,"impl":"D2-sdpa","us":758.336},{"N":16384,"y":32,"impl":"D2-sdpa","us":2869.248},{"N":16384,"y":64,"impl":"D2-sdpa","us":5705.184}]},"mha_vs_gqa_slice":{"batch":1,"head_dim":128,"dtype":"bf16","launch":"eager","cache":"warm"},"mha_vs_gqa":[{"gpu":"A10","hkv":8,"batch":1,"N":512,"impl":"D0-naive-kv","us":186.528,"kv_mb":8.389},{"gpu":"A10","hkv":8,"batch":1,"N":512,"impl":"D1-inductor","us":193.248,"kv_mb":8.389},{"gpu":"A10","hkv":8,"batch":1,"N":512,"impl":"D2-sdpa","us":62.587,"kv_mb":2.097},{"gpu":"A10","hkv":8,"batch":1,"N":512,"impl":"D3-fa-kvcache","us":49.644,"kv_mb":2.097},{"gpu":"A10","hkv":8,"batch":1,"N":512,"impl":"D4-flashinfer","us":38.8,"kv_mb":2.097},{"gpu":"A10","hkv":8,"batch":1,"N":512,"impl":"D6-fa-prefill-at-1","us":157.136,"kv_mb":2.097},{"gpu":"A10","hkv":8,"batch":1,"N":1024,"impl":"D0-naive-kv","us":184.128,"kv_mb":16.777},{"gpu":"A10","hkv":8,"batch":1,"N":1024,"impl":"D2-sdpa","us":62.096,"kv_mb":4.194},{"gpu":"A10","hkv":8,"batch":1,"N":1024,"impl":"D3-fa-kvcache","us":52.36,"kv_mb":4.194},{"gpu":"A10","hkv":8,"batch":1,"N":1024,"impl":"D4-flashinfer","us":67.685,"kv_mb":4.194},{"gpu":"A10","hkv":8,"batch":1,"N":1024,"impl":"D6-fa-prefill-at-1","us":155.216,"kv_mb":4.194},{"gpu":"A10","hkv":8,"batch":1,"N":2048,"impl":"D0-naive-kv","us":200.384,"kv_mb":33.554},{"gpu":"A10","hkv":8,"batch":1,"N":2048,"impl":"D1-inductor","us":189.696,"kv_mb":33.554},{"gpu":"A10","hkv":8,"batch":1,"N":2048,"impl":"D2-sdpa","us":64.549,"kv_mb":8.389},{"gpu":"A10","hkv":8,"batch":1,"N":2048,"impl":"D3-fa-kvcache","us":51.072,"kv_mb":8.389},{"gpu":"A10","hkv":8,"batch":1,"N":2048,"impl":"D4-flashinfer","us":144.592,"kv_mb":8.389},{"gpu":"A10","hkv":8,"batch":1,"N":2048,"impl":"D6-fa-prefill-at-1","us":157.472,"kv_mb":8.389},{"gpu":"A10","hkv":8,"batch":1,"N":4096,"impl":"D0-naive-kv","us":239.136,"kv_mb":67.109},{"gpu":"A10","hkv":8,"batch":1,"N":4096,"impl":"D2-sdpa","us":66.229,"kv_mb":16.777},{"gpu":"A10","hkv":8,"batch":1,"N":4096,"impl":"D3-fa-kvcache","us":62.288,"kv_mb":16.777},{"gpu":"A10","hkv":8,"batch":1,"N":4096,"impl":"D4-flashinfer","us":250.496,"kv_mb":16.777},{"gpu":"A10","hkv":8,"batch":1,"N":4096,"impl":"D6-fa-prefill-at-1","us":155.76,"kv_mb":16.777},{"gpu":"A10","hkv":8,"batch":1,"N":8192,"impl":"D0-naive-kv","us":333.536,"kv_mb":134.218},{"gpu":"A10","hkv":8,"batch":1,"N":8192,"impl":"D1-inductor","us":352.384,"kv_mb":134.218},{"gpu":"A10","hkv":8,"batch":1,"N":8192,"impl":"D2-sdpa","us":102.688,"kv_mb":33.554},{"gpu":"A10","hkv":8,"batch":1,"N":8192,"impl":"D3-fa-kvcache","us":103.488,"kv_mb":33.554},{"gpu":"A10","hkv":8,"batch":1,"N":8192,"impl":"D4-flashinfer","us":464.32,"kv_mb":33.554},{"gpu":"A10","hkv":8,"batch":1,"N":8192,"impl":"D6-fa-prefill-at-1","us":174.576,"kv_mb":33.554},{"gpu":"A10","hkv":8,"batch":1,"N":16384,"impl":"D0-naive-kv","us":597.856,"kv_mb":268.435},{"gpu":"A10","hkv":8,"batch":1,"N":16384,"impl":"D2-sdpa","us":188.416,"kv_mb":67.109},{"gpu":"A10","hkv":8,"batch":1,"N":16384,"impl":"D3-fa-kvcache","us":189.776,"kv_mb":67.109},{"gpu":"A10","hkv":8,"batch":1,"N":16384,"impl":"D4-flashinfer","us":886.192,"kv_mb":67.109},{"gpu":"A10","hkv":8,"batch":1,"N":16384,"impl":"D6-fa-prefill-at-1","us":241.104,"kv_mb":67.109},{"gpu":"A10","hkv":32,"batch":1,"N":512,"impl":"D0-naive-kv","us":186.64,"kv_mb":8.389},{"gpu":"A10","hkv":32,"batch":1,"N":512,"impl":"D2-sdpa","us":54.528,"kv_mb":8.389},{"gpu":"A10","hkv":32,"batch":1,"N":512,"impl":"D3-fa-kvcache","us":40.438,"kv_mb":8.389},{"gpu":"A10","hkv":32,"batch":1,"N":512,"impl":"D4-flashinfer","us":37.088,"kv_mb":8.389},{"gpu":"A10","hkv":32,"batch":1,"N":512,"impl":"D6-fa-prefill-at-1","us":142.192,"kv_mb":8.389},{"gpu":"A10","hkv":32,"batch":1,"N":1024,"impl":"D0-naive-kv","us":184.56,"kv_mb":16.777},{"gpu":"A10","hkv":32,"batch":1,"N":1024,"impl":"D2-sdpa","us":60.091,"kv_mb":16.777},{"gpu":"A10","hkv":32,"batch":1,"N":1024,"impl":"D3-fa-kvcache","us":52.768,"kv_mb":16.777},{"gpu":"A10","hkv":32,"batch":1,"N":1024,"impl":"D4-flashinfer","us":45.412,"kv_mb":16.777},{"gpu":"A10","hkv":32,"batch":1,"N":1024,"impl":"D6-fa-prefill-at-1","us":142.512,"kv_mb":16.777},{"gpu":"A10","hkv":32,"batch":1,"N":2048,"impl":"D0-naive-kv","us":201.984,"kv_mb":33.554},{"gpu":"A10","hkv":32,"batch":1,"N":2048,"impl":"D2-sdpa","us":95.88,"kv_mb":33.554},{"gpu":"A10","hkv":32,"batch":1,"N":2048,"impl":"D3-fa-kvcache","us":94.432,"kv_mb":33.554},{"gpu":"A10","hkv":32,"batch":1,"N":2048,"impl":"D4-flashinfer","us":87.048,"kv_mb":33.554},{"gpu":"A10","hkv":32,"batch":1,"N":2048,"impl":"D6-fa-prefill-at-1","us":161.408,"kv_mb":33.554},{"gpu":"A10","hkv":32,"batch":1,"N":4096,"impl":"D0-naive-kv","us":234.64,"kv_mb":67.109},{"gpu":"A10","hkv":32,"batch":1,"N":4096,"impl":"D2-sdpa","us":179.728,"kv_mb":67.109},{"gpu":"A10","hkv":32,"batch":1,"N":4096,"impl":"D3-fa-kvcache","us":176.592,"kv_mb":67.109},{"gpu":"A10","hkv":32,"batch":1,"N":4096,"impl":"D4-flashinfer","us":168.48,"kv_mb":67.109},{"gpu":"A10","hkv":32,"batch":1,"N":4096,"impl":"D6-fa-prefill-at-1","us":227.68,"kv_mb":67.109},{"gpu":"A10","hkv":32,"batch":1,"N":8192,"impl":"D0-naive-kv","us":333.296,"kv_mb":134.218},{"gpu":"A10","hkv":32,"batch":1,"N":8192,"impl":"D2-sdpa","us":306.528,"kv_mb":134.218},{"gpu":"A10","hkv":32,"batch":1,"N":8192,"impl":"D3-fa-kvcache","us":308.256,"kv_mb":134.218},{"gpu":"A10","hkv":32,"batch":1,"N":8192,"impl":"D4-flashinfer","us":297.488,"kv_mb":134.218},{"gpu":"A10","hkv":32,"batch":1,"N":8192,"impl":"D6-fa-prefill-at-1","us":360.416,"kv_mb":134.218},{"gpu":"A10","hkv":32,"batch":1,"N":16384,"impl":"D0-naive-kv","us":602.64,"kv_mb":268.435},{"gpu":"A10","hkv":32,"batch":1,"N":16384,"impl":"D2-sdpa","us":564.288,"kv_mb":268.435},{"gpu":"A10","hkv":32,"batch":1,"N":16384,"impl":"D3-fa-kvcache","us":575.072,"kv_mb":268.435},{"gpu":"A10","hkv":32,"batch":1,"N":16384,"impl":"D4-flashinfer","us":555.248,"kv_mb":268.435},{"gpu":"A10","hkv":32,"batch":1,"N":16384,"impl":"D6-fa-prefill-at-1","us":627.184,"kv_mb":268.435},{"gpu":"A100","hkv":8,"batch":1,"N":512,"impl":"D0-naive-kv","us":271.248,"kv_mb":8.389},{"gpu":"A100","hkv":8,"batch":1,"N":512,"impl":"D1-inductor","us":197.264,"kv_mb":8.389},{"gpu":"A100","hkv":8,"batch":1,"N":512,"impl":"D2-sdpa","us":77.232,"kv_mb":2.097},{"gpu":"A100","hkv":8,"batch":1,"N":512,"impl":"D3-fa-kvcache","us":82.939,"kv_mb":2.097},{"gpu":"A100","hkv":8,"batch":1,"N":512,"impl":"D4-flashinfer","us":61.184,"kv_mb":2.097},{"gpu":"A100","hkv":8,"batch":1,"N":512,"impl":"D6-fa-prefill-at-1","us":174.32,"kv_mb":2.097},{"gpu":"A100","hkv":8,"batch":1,"N":1024,"impl":"D0-naive-kv","us":192.128,"kv_mb":16.777},{"gpu":"A100","hkv":8,"batch":1,"N":1024,"impl":"D2-sdpa","us":60.512,"kv_mb":4.194},{"gpu":"A100","hkv":8,"batch":1,"N":1024,"impl":"D3-fa-kvcache","us":49.728,"kv_mb":4.194},{"gpu":"A100","hkv":8,"batch":1,"N":1024,"impl":"D4-flashinfer","us":90.088,"kv_mb":4.194},{"gpu":"A100","hkv":8,"batch":1,"N":1024,"impl":"D6-fa-prefill-at-1","us":160.88,"kv_mb":4.194},{"gpu":"A100","hkv":8,"batch":1,"N":2048,"impl":"D0-naive-kv","us":228.128,"kv_mb":33.554},{"gpu":"A100","hkv":8,"batch":1,"N":2048,"impl":"D1-inductor","us":211.6,"kv_mb":33.554},{"gpu":"A100","hkv":8,"batch":1,"N":2048,"impl":"D2-sdpa","us":60.741,"kv_mb":8.389},{"gpu":"A100","hkv":8,"batch":1,"N":2048,"impl":"D3-fa-kvcache","us":50.716,"kv_mb":8.389},{"gpu":"A100","hkv":8,"batch":1,"N":2048,"impl":"D4-flashinfer","us":199.536,"kv_mb":8.389},{"gpu":"A100","hkv":8,"batch":1,"N":2048,"impl":"D6-fa-prefill-at-1","us":165.104,"kv_mb":8.389},{"gpu":"A100","hkv":8,"batch":1,"N":4096,"impl":"D0-naive-kv","us":219.696,"kv_mb":67.109},{"gpu":"A100","hkv":8,"batch":1,"N":4096,"impl":"D2-sdpa","us":61.509,"kv_mb":16.777},{"gpu":"A100","hkv":8,"batch":1,"N":4096,"impl":"D3-fa-kvcache","us":51.004,"kv_mb":16.777},{"gpu":"A100","hkv":8,"batch":1,"N":4096,"impl":"D4-flashinfer","us":342.464,"kv_mb":16.777},{"gpu":"A100","hkv":8,"batch":1,"N":4096,"impl":"D6-fa-prefill-at-1","us":167.296,"kv_mb":16.777},{"gpu":"A100","hkv":8,"batch":1,"N":8192,"impl":"D0-naive-kv","us":233.376,"kv_mb":134.218},{"gpu":"A100","hkv":8,"batch":1,"N":8192,"impl":"D1-inductor","us":242.816,"kv_mb":134.218},{"gpu":"A100","hkv":8,"batch":1,"N":8192,"impl":"D2-sdpa","us":63.349,"kv_mb":33.554},{"gpu":"A100","hkv":8,"batch":1,"N":8192,"impl":"D3-fa-kvcache","us":58.768,"kv_mb":33.554},{"gpu":"A100","hkv":8,"batch":1,"N":8192,"impl":"D4-flashinfer","us":634.656,"kv_mb":33.554},{"gpu":"A100","hkv":8,"batch":1,"N":8192,"impl":"D6-fa-prefill-at-1","us":193.2,"kv_mb":33.554},{"gpu":"A100","hkv":8,"batch":1,"N":16384,"impl":"D0-naive-kv","us":271.248,"kv_mb":268.435},{"gpu":"A100","hkv":8,"batch":1,"N":16384,"impl":"D2-sdpa","us":83.352,"kv_mb":67.109},{"gpu":"A100","hkv":8,"batch":1,"N":16384,"impl":"D3-fa-kvcache","us":74.597,"kv_mb":67.109},{"gpu":"A100","hkv":8,"batch":1,"N":16384,"impl":"D4-flashinfer","us":1202.928,"kv_mb":67.109},{"gpu":"A100","hkv":8,"batch":1,"N":16384,"impl":"D6-fa-prefill-at-1","us":184.464,"kv_mb":67.109},{"gpu":"A100","hkv":32,"batch":1,"N":512,"impl":"D0-naive-kv","us":198.336,"kv_mb":8.389},{"gpu":"A100","hkv":32,"batch":1,"N":512,"impl":"D2-sdpa","us":55.733,"kv_mb":8.389},{"gpu":"A100","hkv":32,"batch":1,"N":512,"impl":"D3-fa-kvcache","us":38.842,"kv_mb":8.389},{"gpu":"A100","hkv":32,"batch":1,"N":512,"impl":"D4-flashinfer","us":39.05,"kv_mb":8.389},{"gpu":"A100","hkv":32,"batch":1,"N":512,"impl":"D6-fa-prefill-at-1","us":150.912,"kv_mb":8.389},{"gpu":"A100","hkv":32,"batch":1,"N":1024,"impl":"D0-naive-kv","us":194.368,"kv_mb":16.777},{"gpu":"A100","hkv":32,"batch":1,"N":1024,"impl":"D2-sdpa","us":55.877,"kv_mb":16.777},{"gpu":"A100","hkv":32,"batch":1,"N":1024,"impl":"D3-fa-kvcache","us":39.12,"kv_mb":16.777},{"gpu":"A100","hkv":32,"batch":1,"N":1024,"impl":"D4-flashinfer","us":38.771,"kv_mb":16.777},{"gpu":"A100","hkv":32,"batch":1,"N":1024,"impl":"D6-fa-prefill-at-1","us":147.408,"kv_mb":16.777},{"gpu":"A100","hkv":32,"batch":1,"N":2048,"impl":"D0-naive-kv","us":208.576,"kv_mb":33.554},{"gpu":"A100","hkv":32,"batch":1,"N":2048,"impl":"D2-sdpa","us":53.792,"kv_mb":33.554},{"gpu":"A100","hkv":32,"batch":1,"N":2048,"impl":"D3-fa-kvcache","us":39.798,"kv_mb":33.554},{"gpu":"A100","hkv":32,"batch":1,"N":2048,"impl":"D4-flashinfer","us":47.8,"kv_mb":33.554},{"gpu":"A100","hkv":32,"batch":1,"N":2048,"impl":"D6-fa-prefill-at-1","us":159.28,"kv_mb":33.554},{"gpu":"A100","hkv":32,"batch":1,"N":4096,"impl":"D0-naive-kv","us":198.288,"kv_mb":67.109},{"gpu":"A100","hkv":32,"batch":1,"N":4096,"impl":"D2-sdpa","us":68.053,"kv_mb":67.109},{"gpu":"A100","hkv":32,"batch":1,"N":4096,"impl":"D3-fa-kvcache","us":65.659,"kv_mb":67.109},{"gpu":"A100","hkv":32,"batch":1,"N":4096,"impl":"D4-flashinfer","us":88.904,"kv_mb":67.109},{"gpu":"A100","hkv":32,"batch":1,"N":4096,"impl":"D6-fa-prefill-at-1","us":153.312,"kv_mb":67.109},{"gpu":"A100","hkv":32,"batch":1,"N":8192,"impl":"D0-naive-kv","us":213.328,"kv_mb":134.218},{"gpu":"A100","hkv":32,"batch":1,"N":8192,"impl":"D2-sdpa","us":129.52,"kv_mb":134.218},{"gpu":"A100","hkv":32,"batch":1,"N":8192,"impl":"D3-fa-kvcache","us":108.912,"kv_mb":134.218},{"gpu":"A100","hkv":32,"batch":1,"N":8192,"impl":"D4-flashinfer","us":170.928,"kv_mb":134.218},{"gpu":"A100","hkv":32,"batch":1,"N":8192,"impl":"D6-fa-prefill-at-1","us":173.36,"kv_mb":134.218},{"gpu":"A100","hkv":32,"batch":1,"N":16384,"impl":"D0-naive-kv","us":244.304,"kv_mb":268.435},{"gpu":"A100","hkv":32,"batch":1,"N":16384,"impl":"D2-sdpa","us":216.256,"kv_mb":268.435},{"gpu":"A100","hkv":32,"batch":1,"N":16384,"impl":"D3-fa-kvcache","us":201.648,"kv_mb":268.435},{"gpu":"A100","hkv":32,"batch":1,"N":16384,"impl":"D4-flashinfer","us":327.088,"kv_mb":268.435},{"gpu":"A100","hkv":32,"batch":1,"N":16384,"impl":"D6-fa-prefill-at-1","us":254.096,"kv_mb":268.435},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":512,"impl":"D0-naive-kv","us":31.085,"kv_mb":8.389},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":512,"impl":"D1-inductor","us":37.594,"kv_mb":8.389},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":512,"impl":"D2-sdpa","us":14.094,"kv_mb":2.097},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":512,"impl":"D3-fa-kvcache","us":11.585,"kv_mb":2.097},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":512,"impl":"D6-fa-prefill-at-1","us":25.307,"kv_mb":2.097},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":1024,"impl":"D0-naive-kv","us":30.355,"kv_mb":16.777},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":1024,"impl":"D2-sdpa","us":14.12,"kv_mb":4.194},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":1024,"impl":"D3-fa-kvcache","us":11.825,"kv_mb":4.194},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":1024,"impl":"D6-fa-prefill-at-1","us":25.799,"kv_mb":4.194},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":2048,"impl":"D0-naive-kv","us":32.421,"kv_mb":33.554},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":2048,"impl":"D1-inductor","us":37.968,"kv_mb":33.554},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":2048,"impl":"D2-sdpa","us":14.673,"kv_mb":8.389},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":2048,"impl":"D3-fa-kvcache","us":12.608,"kv_mb":8.389},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":2048,"impl":"D6-fa-prefill-at-1","us":25.125,"kv_mb":8.389},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":4096,"impl":"D0-naive-kv","us":49.364,"kv_mb":67.109},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":4096,"impl":"D2-sdpa","us":21.552,"kv_mb":16.777},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":4096,"impl":"D3-fa-kvcache","us":21.429,"kv_mb":16.777},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":4096,"impl":"D6-fa-prefill-at-1","us":27.031,"kv_mb":16.777},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":8192,"impl":"D0-naive-kv","us":132.112,"kv_mb":134.218},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":8192,"impl":"D1-inductor","us":99.072,"kv_mb":134.218},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":8192,"impl":"D2-sdpa","us":28.112,"kv_mb":33.554},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":8192,"impl":"D3-fa-kvcache","us":27.849,"kv_mb":33.554},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":8192,"impl":"D6-fa-prefill-at-1","us":29.485,"kv_mb":33.554},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":16384,"impl":"D0-naive-kv","us":201.408,"kv_mb":268.435},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":16384,"impl":"D2-sdpa","us":43.336,"kv_mb":67.109},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":16384,"impl":"D3-fa-kvcache","us":41.412,"kv_mb":67.109},{"gpu":"RTX 5090","hkv":8,"batch":1,"N":16384,"impl":"D6-fa-prefill-at-1","us":43.82,"kv_mb":67.109},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":512,"impl":"D0-naive-kv","us":30.085,"kv_mb":8.389},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":512,"impl":"D2-sdpa","us":13.12,"kv_mb":8.389},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":512,"impl":"D3-fa-kvcache","us":10.189,"kv_mb":8.389},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":512,"impl":"D6-fa-prefill-at-1","us":24.024,"kv_mb":8.389},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":1024,"impl":"D0-naive-kv","us":30.373,"kv_mb":16.777},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":1024,"impl":"D2-sdpa","us":17.251,"kv_mb":16.777},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":1024,"impl":"D3-fa-kvcache","us":17.069,"kv_mb":16.777},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":1024,"impl":"D6-fa-prefill-at-1","us":23.748,"kv_mb":16.777},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":2048,"impl":"D0-naive-kv","us":32.6,"kv_mb":33.554},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":2048,"impl":"D2-sdpa","us":23.726,"kv_mb":33.554},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":2048,"impl":"D3-fa-kvcache","us":23.482,"kv_mb":33.554},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":2048,"impl":"D6-fa-prefill-at-1","us":25.607,"kv_mb":33.554},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":4096,"impl":"D0-naive-kv","us":47.776,"kv_mb":67.109},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":4096,"impl":"D2-sdpa","us":45.364,"kv_mb":67.109},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":4096,"impl":"D3-fa-kvcache","us":43.828,"kv_mb":67.109},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":4096,"impl":"D6-fa-prefill-at-1","us":45.932,"kv_mb":67.109},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":8192,"impl":"D0-naive-kv","us":132.16,"kv_mb":134.218},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":8192,"impl":"D2-sdpa","us":94.72,"kv_mb":134.218},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":8192,"impl":"D3-fa-kvcache","us":94.232,"kv_mb":134.218},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":8192,"impl":"D6-fa-prefill-at-1","us":99.272,"kv_mb":134.218},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":16384,"impl":"D0-naive-kv","us":201.344,"kv_mb":268.435},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":16384,"impl":"D2-sdpa","us":182.544,"kv_mb":268.435},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":16384,"impl":"D3-fa-kvcache","us":180.32,"kv_mb":268.435},{"gpu":"RTX 5090","hkv":32,"batch":1,"N":16384,"impl":"D6-fa-prefill-at-1","us":190.288,"kv_mb":268.435},{"gpu":"H100","hkv":8,"batch":1,"N":512,"impl":"D0-naive-kv","us":263.936,"kv_mb":8.389},{"gpu":"H100","hkv":8,"batch":1,"N":512,"impl":"D1-inductor","us":306.736,"kv_mb":8.389},{"gpu":"H100","hkv":8,"batch":1,"N":512,"impl":"D2-sdpa","us":140.848,"kv_mb":2.097},{"gpu":"H100","hkv":8,"batch":1,"N":512,"impl":"D3-fa-kvcache","us":102.096,"kv_mb":2.097},{"gpu":"H100","hkv":8,"batch":1,"N":512,"impl":"D4-flashinfer","us":78.128,"kv_mb":2.097},{"gpu":"H100","hkv":8,"batch":1,"N":512,"impl":"D6-fa-prefill-at-1","us":269.728,"kv_mb":2.097},{"gpu":"H100","hkv":8,"batch":1,"N":1024,"impl":"D0-naive-kv","us":279.104,"kv_mb":16.777},{"gpu":"H100","hkv":8,"batch":1,"N":1024,"impl":"D2-sdpa","us":140.736,"kv_mb":4.194},{"gpu":"H100","hkv":8,"batch":1,"N":1024,"impl":"D3-fa-kvcache","us":102.352,"kv_mb":4.194},{"gpu":"H100","hkv":8,"batch":1,"N":1024,"impl":"D4-flashinfer","us":89.784,"kv_mb":4.194},{"gpu":"H100","hkv":8,"batch":1,"N":1024,"impl":"D6-fa-prefill-at-1","us":295.888,"kv_mb":4.194},{"gpu":"H100","hkv":8,"batch":1,"N":2048,"impl":"D0-naive-kv","us":269.744,"kv_mb":33.554},{"gpu":"H100","hkv":8,"batch":1,"N":2048,"impl":"D1-inductor","us":311.84,"kv_mb":33.554},{"gpu":"H100","hkv":8,"batch":1,"N":2048,"impl":"D2-sdpa","us":140.48,"kv_mb":8.389},{"gpu":"H100","hkv":8,"batch":1,"N":2048,"impl":"D3-fa-kvcache","us":100.792,"kv_mb":8.389},{"gpu":"H100","hkv":8,"batch":1,"N":2048,"impl":"D4-flashinfer","us":165.328,"kv_mb":8.389},{"gpu":"H100","hkv":8,"batch":1,"N":2048,"impl":"D6-fa-prefill-at-1","us":289.952,"kv_mb":8.389},{"gpu":"H100","hkv":8,"batch":1,"N":4096,"impl":"D0-naive-kv","us":296.528,"kv_mb":67.109},{"gpu":"H100","hkv":8,"batch":1,"N":4096,"impl":"D2-sdpa","us":140.816,"kv_mb":16.777},{"gpu":"H100","hkv":8,"batch":1,"N":4096,"impl":"D3-fa-kvcache","us":102.216,"kv_mb":16.777},{"gpu":"H100","hkv":8,"batch":1,"N":4096,"impl":"D4-flashinfer","us":268.992,"kv_mb":16.777},{"gpu":"H100","hkv":8,"batch":1,"N":4096,"impl":"D6-fa-prefill-at-1","us":285.008,"kv_mb":16.777},{"gpu":"H100","hkv":8,"batch":1,"N":8192,"impl":"D0-naive-kv","us":260.528,"kv_mb":134.218},{"gpu":"H100","hkv":8,"batch":1,"N":8192,"impl":"D1-inductor","us":313.856,"kv_mb":134.218},{"gpu":"H100","hkv":8,"batch":1,"N":8192,"impl":"D2-sdpa","us":125.664,"kv_mb":33.554},{"gpu":"H100","hkv":8,"batch":1,"N":8192,"impl":"D3-fa-kvcache","us":103.568,"kv_mb":33.554},{"gpu":"H100","hkv":8,"batch":1,"N":8192,"impl":"D4-flashinfer","us":456.672,"kv_mb":33.554},{"gpu":"H100","hkv":8,"batch":1,"N":8192,"impl":"D6-fa-prefill-at-1","us":259.6,"kv_mb":33.554},{"gpu":"H100","hkv":8,"batch":1,"N":16384,"impl":"D0-naive-kv","us":290.448,"kv_mb":268.435},{"gpu":"H100","hkv":8,"batch":1,"N":16384,"impl":"D2-sdpa","us":142.32,"kv_mb":67.109},{"gpu":"H100","hkv":8,"batch":1,"N":16384,"impl":"D3-fa-kvcache","us":103.784,"kv_mb":67.109},{"gpu":"H100","hkv":8,"batch":1,"N":16384,"impl":"D4-flashinfer","us":859.712,"kv_mb":67.109},{"gpu":"H100","hkv":8,"batch":1,"N":16384,"impl":"D6-fa-prefill-at-1","us":268.496,"kv_mb":67.109},{"gpu":"H100","hkv":32,"batch":1,"N":512,"impl":"D0-naive-kv","us":270.896,"kv_mb":8.389},{"gpu":"H100","hkv":32,"batch":1,"N":512,"impl":"D2-sdpa","us":130.288,"kv_mb":8.389},{"gpu":"H100","hkv":32,"batch":1,"N":512,"impl":"D3-fa-kvcache","us":86.752,"kv_mb":8.389},{"gpu":"H100","hkv":32,"batch":1,"N":512,"impl":"D4-flashinfer","us":78.84,"kv_mb":8.389},{"gpu":"H100","hkv":32,"batch":1,"N":512,"impl":"D6-fa-prefill-at-1","us":272.912,"kv_mb":8.389},{"gpu":"H100","hkv":32,"batch":1,"N":1024,"impl":"D0-naive-kv","us":262.304,"kv_mb":16.777},{"gpu":"H100","hkv":32,"batch":1,"N":1024,"impl":"D2-sdpa","us":91.68,"kv_mb":16.777},{"gpu":"H100","hkv":32,"batch":1,"N":1024,"impl":"D3-fa-kvcache","us":84.904,"kv_mb":16.777},{"gpu":"H100","hkv":32,"batch":1,"N":1024,"impl":"D4-flashinfer","us":76.904,"kv_mb":16.777},{"gpu":"H100","hkv":32,"batch":1,"N":1024,"impl":"D6-fa-prefill-at-1","us":241.904,"kv_mb":16.777},{"gpu":"H100","hkv":32,"batch":1,"N":2048,"impl":"D0-naive-kv","us":267.6,"kv_mb":33.554},{"gpu":"H100","hkv":32,"batch":1,"N":2048,"impl":"D2-sdpa","us":131.104,"kv_mb":33.554},{"gpu":"H100","hkv":32,"batch":1,"N":2048,"impl":"D3-fa-kvcache","us":86.096,"kv_mb":33.554},{"gpu":"H100","hkv":32,"batch":1,"N":2048,"impl":"D4-flashinfer","us":80.136,"kv_mb":33.554},{"gpu":"H100","hkv":32,"batch":1,"N":2048,"impl":"D6-fa-prefill-at-1","us":273.68,"kv_mb":33.554},{"gpu":"H100","hkv":32,"batch":1,"N":4096,"impl":"D0-naive-kv","us":289.392,"kv_mb":67.109},{"gpu":"H100","hkv":32,"batch":1,"N":4096,"impl":"D2-sdpa","us":131.392,"kv_mb":67.109},{"gpu":"H100","hkv":32,"batch":1,"N":4096,"impl":"D3-fa-kvcache","us":86.664,"kv_mb":67.109},{"gpu":"H100","hkv":32,"batch":1,"N":4096,"impl":"D4-flashinfer","us":95.184,"kv_mb":67.109},{"gpu":"H100","hkv":32,"batch":1,"N":4096,"impl":"D6-fa-prefill-at-1","us":274.592,"kv_mb":67.109},{"gpu":"H100","hkv":32,"batch":1,"N":8192,"impl":"D0-naive-kv","us":180.944,"kv_mb":134.218},{"gpu":"H100","hkv":32,"batch":1,"N":8192,"impl":"D2-sdpa","us":71.637,"kv_mb":134.218},{"gpu":"H100","hkv":32,"batch":1,"N":8192,"impl":"D3-fa-kvcache","us":68.528,"kv_mb":134.218},{"gpu":"H100","hkv":32,"batch":1,"N":8192,"impl":"D4-flashinfer","us":171.76,"kv_mb":134.218},{"gpu":"H100","hkv":32,"batch":1,"N":8192,"impl":"D6-fa-prefill-at-1","us":218.688,"kv_mb":134.218},{"gpu":"H100","hkv":32,"batch":1,"N":16384,"impl":"D0-naive-kv","us":233.168,"kv_mb":268.435},{"gpu":"H100","hkv":32,"batch":1,"N":16384,"impl":"D2-sdpa","us":135.872,"kv_mb":268.435},{"gpu":"H100","hkv":32,"batch":1,"N":16384,"impl":"D3-fa-kvcache","us":129.488,"kv_mb":268.435},{"gpu":"H100","hkv":32,"batch":1,"N":16384,"impl":"D4-flashinfer","us":280.288,"kv_mb":268.435},{"gpu":"H100","hkv":32,"batch":1,"N":16384,"impl":"D6-fa-prefill-at-1","us":157.824,"kv_mb":268.435},{"gpu":"L40","hkv":8,"batch":1,"N":512,"impl":"D0-naive-kv","us":79.944,"kv_mb":8.389},{"gpu":"L40","hkv":8,"batch":1,"N":512,"impl":"D1-inductor","us":120.816,"kv_mb":8.389},{"gpu":"L40","hkv":8,"batch":1,"N":512,"impl":"D2-sdpa","us":34.899,"kv_mb":2.097},{"gpu":"L40","hkv":8,"batch":1,"N":512,"impl":"D3-fa-kvcache","us":27.614,"kv_mb":2.097},{"gpu":"L40","hkv":8,"batch":1,"N":512,"impl":"D4-flashinfer","us":22.894,"kv_mb":2.097},{"gpu":"L40","hkv":8,"batch":1,"N":512,"impl":"D6-fa-prefill-at-1","us":80.6,"kv_mb":2.097},{"gpu":"L40","hkv":8,"batch":1,"N":1024,"impl":"D0-naive-kv","us":82.32,"kv_mb":16.777},{"gpu":"L40","hkv":8,"batch":1,"N":1024,"impl":"D2-sdpa","us":34.893,"kv_mb":4.194},{"gpu":"L40","hkv":8,"batch":1,"N":1024,"impl":"D3-fa-kvcache","us":28.333,"kv_mb":4.194},{"gpu":"L40","hkv":8,"batch":1,"N":1024,"impl":"D4-flashinfer","us":43.444,"kv_mb":4.194},{"gpu":"L40","hkv":8,"batch":1,"N":1024,"impl":"D6-fa-prefill-at-1","us":80.816,"kv_mb":4.194},{"gpu":"L40","hkv":8,"batch":1,"N":2048,"impl":"D0-naive-kv","us":86.188,"kv_mb":33.554},{"gpu":"L40","hkv":8,"batch":1,"N":2048,"impl":"D1-inductor","us":120.448,"kv_mb":33.554},{"gpu":"L40","hkv":8,"batch":1,"N":2048,"impl":"D2-sdpa","us":35.246,"kv_mb":8.389},{"gpu":"L40","hkv":8,"batch":1,"N":2048,"impl":"D3-fa-kvcache","us":28.093,"kv_mb":8.389},{"gpu":"L40","hkv":8,"batch":1,"N":2048,"impl":"D4-flashinfer","us":84.428,"kv_mb":8.389},{"gpu":"L40","hkv":8,"batch":1,"N":2048,"impl":"D6-fa-prefill-at-1","us":81.028,"kv_mb":8.389},{"gpu":"L40","hkv":8,"batch":1,"N":4096,"impl":"D0-naive-kv","us":87.748,"kv_mb":67.109},{"gpu":"L40","hkv":8,"batch":1,"N":4096,"impl":"D2-sdpa","us":35.974,"kv_mb":16.777},{"gpu":"L40","hkv":8,"batch":1,"N":4096,"impl":"D3-fa-kvcache","us":29.273,"kv_mb":16.777},{"gpu":"L40","hkv":8,"batch":1,"N":4096,"impl":"D4-flashinfer","us":166.776,"kv_mb":16.777},{"gpu":"L40","hkv":8,"batch":1,"N":4096,"impl":"D6-fa-prefill-at-1","us":81.96,"kv_mb":16.777},{"gpu":"L40","hkv":8,"batch":1,"N":8192,"impl":"D0-naive-kv","us":245.192,"kv_mb":134.218},{"gpu":"L40","hkv":8,"batch":1,"N":8192,"impl":"D1-inductor","us":231.448,"kv_mb":134.218},{"gpu":"L40","hkv":8,"batch":1,"N":8192,"impl":"D2-sdpa","us":37.642,"kv_mb":33.554},{"gpu":"L40","hkv":8,"batch":1,"N":8192,"impl":"D3-fa-kvcache","us":32.16,"kv_mb":33.554},{"gpu":"L40","hkv":8,"batch":1,"N":8192,"impl":"D4-flashinfer","us":312.632,"kv_mb":33.554},{"gpu":"L40","hkv":8,"batch":1,"N":8192,"impl":"D6-fa-prefill-at-1","us":81.304,"kv_mb":33.554},{"gpu":"L40","hkv":8,"batch":1,"N":16384,"impl":"D0-naive-kv","us":484.464,"kv_mb":268.435},{"gpu":"L40","hkv":8,"batch":1,"N":16384,"impl":"D2-sdpa","us":50.046,"kv_mb":67.109},{"gpu":"L40","hkv":8,"batch":1,"N":16384,"impl":"D3-fa-kvcache","us":49.02,"kv_mb":67.109},{"gpu":"L40","hkv":8,"batch":1,"N":16384,"impl":"D4-flashinfer","us":605.816,"kv_mb":67.109},{"gpu":"L40","hkv":8,"batch":1,"N":16384,"impl":"D6-fa-prefill-at-1","us":85.916,"kv_mb":67.109},{"gpu":"L40","hkv":32,"batch":1,"N":512,"impl":"D0-naive-kv","us":80.008,"kv_mb":8.389},{"gpu":"L40","hkv":32,"batch":1,"N":512,"impl":"D2-sdpa","us":32.304,"kv_mb":8.389},{"gpu":"L40","hkv":32,"batch":1,"N":512,"impl":"D3-fa-kvcache","us":22.878,"kv_mb":8.389},{"gpu":"L40","hkv":32,"batch":1,"N":512,"impl":"D4-flashinfer","us":19.136,"kv_mb":8.389},{"gpu":"L40","hkv":32,"batch":1,"N":512,"impl":"D6-fa-prefill-at-1","us":74.784,"kv_mb":8.389},{"gpu":"L40","hkv":32,"batch":1,"N":1024,"impl":"D0-naive-kv","us":81.832,"kv_mb":16.777},{"gpu":"L40","hkv":32,"batch":1,"N":1024,"impl":"D2-sdpa","us":32.251,"kv_mb":16.777},{"gpu":"L40","hkv":32,"batch":1,"N":1024,"impl":"D3-fa-kvcache","us":23.412,"kv_mb":16.777},{"gpu":"L40","hkv":32,"batch":1,"N":1024,"impl":"D4-flashinfer","us":19.36,"kv_mb":16.777},{"gpu":"L40","hkv":32,"batch":1,"N":1024,"impl":"D6-fa-prefill-at-1","us":71.029,"kv_mb":16.777},{"gpu":"L40","hkv":32,"batch":1,"N":2048,"impl":"D0-naive-kv","us":82.208,"kv_mb":33.554},{"gpu":"L40","hkv":32,"batch":1,"N":2048,"impl":"D2-sdpa","us":33.749,"kv_mb":33.554},{"gpu":"L40","hkv":32,"batch":1,"N":2048,"impl":"D3-fa-kvcache","us":27.824,"kv_mb":33.554},{"gpu":"L40","hkv":32,"batch":1,"N":2048,"impl":"D4-flashinfer","us":25.465,"kv_mb":33.554},{"gpu":"L40","hkv":32,"batch":1,"N":2048,"impl":"D6-fa-prefill-at-1","us":70.773,"kv_mb":33.554},{"gpu":"L40","hkv":32,"batch":1,"N":4096,"impl":"D0-naive-kv","us":87.62,"kv_mb":67.109},{"gpu":"L40","hkv":32,"batch":1,"N":4096,"impl":"D2-sdpa","us":55.107,"kv_mb":67.109},{"gpu":"L40","hkv":32,"batch":1,"N":4096,"impl":"D3-fa-kvcache","us":51.992,"kv_mb":67.109},{"gpu":"L40","hkv":32,"batch":1,"N":4096,"impl":"D4-flashinfer","us":47.402,"kv_mb":67.109},{"gpu":"L40","hkv":32,"batch":1,"N":4096,"impl":"D6-fa-prefill-at-1","us":83.336,"kv_mb":67.109},{"gpu":"L40","hkv":32,"batch":1,"N":8192,"impl":"D0-naive-kv","us":244.768,"kv_mb":134.218},{"gpu":"L40","hkv":32,"batch":1,"N":8192,"impl":"D2-sdpa","us":211.856,"kv_mb":134.218},{"gpu":"L40","hkv":32,"batch":1,"N":8192,"impl":"D3-fa-kvcache","us":212.448,"kv_mb":134.218},{"gpu":"L40","hkv":32,"batch":1,"N":8192,"impl":"D4-flashinfer","us":199.264,"kv_mb":134.218},{"gpu":"L40","hkv":32,"batch":1,"N":8192,"impl":"D6-fa-prefill-at-1","us":244.88,"kv_mb":134.218},{"gpu":"L40","hkv":32,"batch":1,"N":16384,"impl":"D0-naive-kv","us":475.648,"kv_mb":268.435},{"gpu":"L40","hkv":32,"batch":1,"N":16384,"impl":"D2-sdpa","us":393.92,"kv_mb":268.435},{"gpu":"L40","hkv":32,"batch":1,"N":16384,"impl":"D3-fa-kvcache","us":397.536,"kv_mb":268.435},{"gpu":"L40","hkv":32,"batch":1,"N":16384,"impl":"D4-flashinfer","us":379.696,"kv_mb":268.435},{"gpu":"L40","hkv":32,"batch":1,"N":16384,"impl":"D6-fa-prefill-at-1","us":429.808,"kv_mb":268.435},{"gpu":"L40S","hkv":8,"batch":1,"N":512,"impl":"D0-naive-kv","us":140.704,"kv_mb":8.389},{"gpu":"L40S","hkv":8,"batch":1,"N":512,"impl":"D1-inductor","us":244.544,"kv_mb":8.389},{"gpu":"L40S","hkv":8,"batch":1,"N":512,"impl":"D2-sdpa","us":67.632,"kv_mb":2.097},{"gpu":"L40S","hkv":8,"batch":1,"N":512,"impl":"D3-fa-kvcache","us":54.672,"kv_mb":2.097},{"gpu":"L40S","hkv":8,"batch":1,"N":512,"impl":"D4-flashinfer","us":40.198,"kv_mb":2.097},{"gpu":"L40S","hkv":8,"batch":1,"N":512,"impl":"D6-fa-prefill-at-1","us":215.472,"kv_mb":2.097},{"gpu":"L40S","hkv":8,"batch":1,"N":1024,"impl":"D0-naive-kv","us":303.584,"kv_mb":16.777},{"gpu":"L40S","hkv":8,"batch":1,"N":1024,"impl":"D2-sdpa","us":44.92,"kv_mb":4.194},{"gpu":"L40S","hkv":8,"batch":1,"N":1024,"impl":"D3-fa-kvcache","us":55.563,"kv_mb":4.194},{"gpu":"L40S","hkv":8,"batch":1,"N":1024,"impl":"D4-flashinfer","us":47.44,"kv_mb":4.194},{"gpu":"L40S","hkv":8,"batch":1,"N":1024,"impl":"D6-fa-prefill-at-1","us":213.84,"kv_mb":4.194},{"gpu":"L40S","hkv":8,"batch":1,"N":2048,"impl":"D0-naive-kv","us":228.048,"kv_mb":33.554},{"gpu":"L40S","hkv":8,"batch":1,"N":2048,"impl":"D1-inductor","us":240.448,"kv_mb":33.554},{"gpu":"L40S","hkv":8,"batch":1,"N":2048,"impl":"D2-sdpa","us":66.619,"kv_mb":8.389},{"gpu":"L40S","hkv":8,"batch":1,"N":2048,"impl":"D3-fa-kvcache","us":55.717,"kv_mb":8.389},{"gpu":"L40S","hkv":8,"batch":1,"N":2048,"impl":"D4-flashinfer","us":92.792,"kv_mb":8.389},{"gpu":"L40S","hkv":8,"batch":1,"N":2048,"impl":"D6-fa-prefill-at-1","us":219.312,"kv_mb":8.389},{"gpu":"L40S","hkv":8,"batch":1,"N":4096,"impl":"D0-naive-kv","us":232.512,"kv_mb":67.109},{"gpu":"L40S","hkv":8,"batch":1,"N":4096,"impl":"D2-sdpa","us":67.584,"kv_mb":16.777},{"gpu":"L40S","hkv":8,"batch":1,"N":4096,"impl":"D3-fa-kvcache","us":55.803,"kv_mb":16.777},{"gpu":"L40S","hkv":8,"batch":1,"N":4096,"impl":"D4-flashinfer","us":183.232,"kv_mb":16.777},{"gpu":"L40S","hkv":8,"batch":1,"N":4096,"impl":"D6-fa-prefill-at-1","us":211.264,"kv_mb":16.777},{"gpu":"L40S","hkv":8,"batch":1,"N":8192,"impl":"D0-naive-kv","us":312.512,"kv_mb":134.218},{"gpu":"L40S","hkv":8,"batch":1,"N":8192,"impl":"D1-inductor","us":287.024,"kv_mb":134.218},{"gpu":"L40S","hkv":8,"batch":1,"N":8192,"impl":"D2-sdpa","us":41.628,"kv_mb":33.554},{"gpu":"L40S","hkv":8,"batch":1,"N":8192,"impl":"D3-fa-kvcache","us":33.243,"kv_mb":33.554},{"gpu":"L40S","hkv":8,"batch":1,"N":8192,"impl":"D4-flashinfer","us":329.888,"kv_mb":33.554},{"gpu":"L40S","hkv":8,"batch":1,"N":8192,"impl":"D6-fa-prefill-at-1","us":213.088,"kv_mb":33.554},{"gpu":"L40S","hkv":8,"batch":1,"N":16384,"impl":"D0-naive-kv","us":506.704,"kv_mb":268.435},{"gpu":"L40S","hkv":8,"batch":1,"N":16384,"impl":"D2-sdpa","us":67.995,"kv_mb":67.109},{"gpu":"L40S","hkv":8,"batch":1,"N":16384,"impl":"D3-fa-kvcache","us":56.928,"kv_mb":67.109},{"gpu":"L40S","hkv":8,"batch":1,"N":16384,"impl":"D4-flashinfer","us":606.176,"kv_mb":67.109},{"gpu":"L40S","hkv":8,"batch":1,"N":16384,"impl":"D6-fa-prefill-at-1","us":198.912,"kv_mb":67.109},{"gpu":"L40S","hkv":32,"batch":1,"N":512,"impl":"D0-naive-kv","us":220.256,"kv_mb":8.389},{"gpu":"L40S","hkv":32,"batch":1,"N":512,"impl":"D2-sdpa","us":61.205,"kv_mb":8.389},{"gpu":"L40S","hkv":32,"batch":1,"N":512,"impl":"D3-fa-kvcache","us":44.012,"kv_mb":8.389},{"gpu":"L40S","hkv":32,"batch":1,"N":512,"impl":"D4-flashinfer","us":39.686,"kv_mb":8.389},{"gpu":"L40S","hkv":32,"batch":1,"N":512,"impl":"D6-fa-prefill-at-1","us":203.184,"kv_mb":8.389},{"gpu":"L40S","hkv":32,"batch":1,"N":1024,"impl":"D0-naive-kv","us":221.856,"kv_mb":16.777},{"gpu":"L40S","hkv":32,"batch":1,"N":1024,"impl":"D2-sdpa","us":59.696,"kv_mb":16.777},{"gpu":"L40S","hkv":32,"batch":1,"N":1024,"impl":"D3-fa-kvcache","us":42.94,"kv_mb":16.777},{"gpu":"L40S","hkv":32,"batch":1,"N":1024,"impl":"D4-flashinfer","us":39.656,"kv_mb":16.777},{"gpu":"L40S","hkv":32,"batch":1,"N":1024,"impl":"D6-fa-prefill-at-1","us":192.272,"kv_mb":16.777},{"gpu":"L40S","hkv":32,"batch":1,"N":2048,"impl":"D0-naive-kv","us":233.216,"kv_mb":33.554},{"gpu":"L40S","hkv":32,"batch":1,"N":2048,"impl":"D2-sdpa","us":60.411,"kv_mb":33.554},{"gpu":"L40S","hkv":32,"batch":1,"N":2048,"impl":"D3-fa-kvcache","us":44.076,"kv_mb":33.554},{"gpu":"L40S","hkv":32,"batch":1,"N":2048,"impl":"D4-flashinfer","us":39.606,"kv_mb":33.554},{"gpu":"L40S","hkv":32,"batch":1,"N":2048,"impl":"D6-fa-prefill-at-1","us":192.16,"kv_mb":33.554},{"gpu":"L40S","hkv":32,"batch":1,"N":4096,"impl":"D0-naive-kv","us":229.008,"kv_mb":67.109},{"gpu":"L40S","hkv":32,"batch":1,"N":4096,"impl":"D2-sdpa","us":39.085,"kv_mb":67.109},{"gpu":"L40S","hkv":32,"batch":1,"N":4096,"impl":"D3-fa-kvcache","us":46.048,"kv_mb":67.109},{"gpu":"L40S","hkv":32,"batch":1,"N":4096,"impl":"D4-flashinfer","us":51.284,"kv_mb":67.109},{"gpu":"L40S","hkv":32,"batch":1,"N":4096,"impl":"D6-fa-prefill-at-1","us":90.4,"kv_mb":67.109},{"gpu":"L40S","hkv":32,"batch":1,"N":8192,"impl":"D0-naive-kv","us":313.92,"kv_mb":134.218},{"gpu":"L40S","hkv":32,"batch":1,"N":8192,"impl":"D2-sdpa","us":217.776,"kv_mb":134.218},{"gpu":"L40S","hkv":32,"batch":1,"N":8192,"impl":"D3-fa-kvcache","us":226.8,"kv_mb":134.218},{"gpu":"L40S","hkv":32,"batch":1,"N":8192,"impl":"D4-flashinfer","us":206.304,"kv_mb":134.218},{"gpu":"L40S","hkv":32,"batch":1,"N":8192,"impl":"D6-fa-prefill-at-1","us":310.512,"kv_mb":134.218},{"gpu":"L40S","hkv":32,"batch":1,"N":16384,"impl":"D0-naive-kv","us":521.072,"kv_mb":268.435},{"gpu":"L40S","hkv":32,"batch":1,"N":16384,"impl":"D2-sdpa","us":418.448,"kv_mb":268.435},{"gpu":"L40S","hkv":32,"batch":1,"N":16384,"impl":"D3-fa-kvcache","us":419.216,"kv_mb":268.435},{"gpu":"L40S","hkv":32,"batch":1,"N":16384,"impl":"D4-flashinfer","us":400.24,"kv_mb":268.435},{"gpu":"L40S","hkv":32,"batch":1,"N":16384,"impl":"D6-fa-prefill-at-1","us":443.104,"kv_mb":268.435}],"roofline_population":"forward only, correctness-qualified rows","roofline":[{"gpu":"A10","regime":"decode","impl":"D0-naive-kv","ai":1.0,"tflops":0.451},{"gpu":"A10","regime":"decode","impl":"D1-inductor","ai":1.0,"tflops":0.412},{"gpu":"A10","regime":"decode","impl":"D2-sdpa","ai":1.0,"tflops":0.437},{"gpu":"A10","regime":"decode","impl":"D3-fa-kvcache","ai":4.0,"tflops":0.436},{"gpu":"A10","regime":"decode","impl":"D4-flashinfer","ai":4.0,"tflops":0.481},{"gpu":"A10","regime":"decode","impl":"D6-fa-prefill-at-1","ai":4.0,"tflops":0.413},{"gpu":"A10","regime":"prefill","impl":"P0-naive","ai":54.044,"tflops":2.112},{"gpu":"A10","regime":"prefill","impl":"P1-inductor","ai":54.044,"tflops":7.057},{"gpu":"A10","regime":"prefill","impl":"P1-inductor-nofuse","ai":83.2,"tflops":10.951},{"gpu":"A10","regime":"prefill","impl":"P1-inductor-where","ai":83.2,"tflops":10.902},{"gpu":"A10","regime":"prefill","impl":"P2b-sdpa-mem-eff","ai":384.0,"tflops":33.411},{"gpu":"A10","regime":"prefill","impl":"P2c-sdpa-flash","ai":384.0,"tflops":48.588},{"gpu":"A10","regime":"prefill","impl":"P2d-sdpa-cudnn","ai":384.0,"tflops":51.545},{"gpu":"A10","regime":"prefill","impl":"P3-triton","ai":384.0,"tflops":46.21},{"gpu":"A10","regime":"prefill","impl":"P4-fa2","ai":384.0,"tflops":43.252},{"gpu":"A100","regime":"decode","impl":"D0-naive-kv","ai":1.0,"tflops":1.244},{"gpu":"A100","regime":"decode","impl":"D1-inductor","ai":1.0,"tflops":0.75},{"gpu":"A100","regime":"decode","impl":"D2-sdpa","ai":2.5,"tflops":1.308},{"gpu":"A100","regime":"decode","impl":"D3-fa-kvcache","ai":2.5,"tflops":1.361},{"gpu":"A100","regime":"decode","impl":"D4-flashinfer","ai":2.5,"tflops":1.6},{"gpu":"A100","regime":"decode","impl":"D6-fa-prefill-at-1","ai":2.5,"tflops":1.143},{"gpu":"A100","regime":"prefill","impl":"P0-naive","ai":58.562,"tflops":6.262},{"gpu":"A100","regime":"prefill","impl":"P1-inductor","ai":58.562,"tflops":19.02},{"gpu":"A100","regime":"prefill","impl":"P1-inductor-nofuse","ai":102.4,"tflops":37.302},{"gpu":"A100","regime":"prefill","impl":"P1-inductor-where","ai":102.4,"tflops":33.239},{"gpu":"A100","regime":"prefill","impl":"P2b-sdpa-mem-eff","ai":384.0,"tflops":68.739},{"gpu":"A100","regime":"prefill","impl":"P2c-sdpa-flash","ai":384.0,"tflops":119.091},{"gpu":"A100","regime":"prefill","impl":"P2d-sdpa-cudnn","ai":384.0,"tflops":118.705},{"gpu":"A100","regime":"prefill","impl":"P3-triton","ai":384.0,"tflops":90.278},{"gpu":"A100","regime":"prefill","impl":"P4-fa2","ai":384.0,"tflops":114.327},{"gpu":"RTX 5090","regime":"decode","impl":"D0-naive-kv","ai":1.0,"tflops":1.436},{"gpu":"RTX 5090","regime":"decode","impl":"D1-inductor","ai":1.0,"tflops":1.404},{"gpu":"RTX 5090","regime":"decode","impl":"D2-sdpa","ai":2.5,"tflops":1.578},{"gpu":"RTX 5090","regime":"decode","impl":"D3-fa-kvcache","ai":4.0,"tflops":1.61},{"gpu":"RTX 5090","regime":"decode","impl":"D6-fa-prefill-at-1","ai":4.0,"tflops":1.57},{"gpu":"RTX 5090","regime":"prefill","impl":"P0-naive","ai":56.889,"tflops":9.447},{"gpu":"RTX 5090","regime":"prefill","impl":"P1-inductor","ai":56.889,"tflops":39.128},{"gpu":"RTX 5090","regime":"prefill","impl":"P1-inductor-nofuse","ai":102.4,"tflops":47.745},{"gpu":"RTX 5090","regime":"prefill","impl":"P1-inductor-where","ai":102.4,"tflops":47.7},{"gpu":"RTX 5090","regime":"prefill","impl":"P2b-sdpa-mem-eff","ai":384.0,"tflops":80.821},{"gpu":"RTX 5090","regime":"prefill","impl":"P2c-sdpa-flash","ai":384.0,"tflops":151.542},{"gpu":"RTX 5090","regime":"prefill","impl":"P2d-sdpa-cudnn","ai":384.0,"tflops":164.644},{"gpu":"RTX 5090","regime":"prefill","impl":"P3-triton","ai":384.0,"tflops":127.936},{"gpu":"RTX 5090","regime":"prefill","impl":"P4-fa2","ai":384.0,"tflops":148.696},{"gpu":"H100","regime":"decode","impl":"D0-naive-kv","ai":1.0,"tflops":1.601},{"gpu":"H100","regime":"decode","impl":"D1-inductor","ai":1.0,"tflops":0.819},{"gpu":"H100","regime":"decode","impl":"D2-sdpa","ai":4.0,"tflops":1.861},{"gpu":"H100","regime":"decode","impl":"D3-fa-kvcache","ai":4.0,"tflops":1.974},{"gpu":"H100","regime":"decode","impl":"D4-flashinfer","ai":4.0,"tflops":2.387},{"gpu":"H100","regime":"decode","impl":"D6-fa-prefill-at-1","ai":4.0,"tflops":1.6},{"gpu":"H100","regime":"prefill","impl":"P0-naive","ai":62.061,"tflops":14.053},{"gpu":"H100","regime":"prefill","impl":"P1-inductor","ai":62.061,"tflops":34.973},{"gpu":"H100","regime":"prefill","impl":"P1-inductor-nofuse","ai":107.31,"tflops":65.787},{"gpu":"H100","regime":"prefill","impl":"P1-inductor-where","ai":107.31,"tflops":59.507},{"gpu":"H100","regime":"prefill","impl":"P2b-sdpa-mem-eff","ai":512.0,"tflops":95.207},{"gpu":"H100","regime":"prefill","impl":"P2c-sdpa-flash","ai":512.0,"tflops":183.608},{"gpu":"H100","regime":"prefill","impl":"P2d-sdpa-cudnn","ai":512.0,"tflops":293.954},{"gpu":"H100","regime":"prefill","impl":"P3-triton","ai":512.0,"tflops":188.732},{"gpu":"H100","regime":"prefill","impl":"P4-fa2","ai":512.0,"tflops":179.849},{"gpu":"H100","regime":"prefill","impl":"P4h-fa3","ai":512.0,"tflops":323.27},{"gpu":"L40","regime":"decode","impl":"D0-naive-kv","ai":1.0,"tflops":0.669},{"gpu":"L40","regime":"decode","impl":"D1-inductor","ai":1.0,"tflops":0.623},{"gpu":"L40","regime":"decode","impl":"D2-sdpa","ai":1.0,"tflops":0.857},{"gpu":"L40","regime":"decode","impl":"D3-fa-kvcache","ai":1.0,"tflops":0.876},{"gpu":"L40","regime":"decode","impl":"D4-flashinfer","ai":1.0,"tflops":0.728},{"gpu":"L40","regime":"decode","impl":"D6-fa-prefill-at-1","ai":1.0,"tflops":0.705},{"gpu":"L40","regime":"prefill","impl":"P0-naive","ai":56.889,"tflops":4.487},{"gpu":"L40","regime":"prefill","impl":"P1-inductor","ai":56.889,"tflops":16.436},{"gpu":"L40","regime":"prefill","impl":"P1-inductor-nofuse","ai":102.4,"tflops":19.373},{"gpu":"L40","regime":"prefill","impl":"P1-inductor-where","ai":102.4,"tflops":19.532},{"gpu":"L40","regime":"prefill","impl":"P2b-sdpa-mem-eff","ai":256.0,"tflops":68.07},{"gpu":"L40","regime":"prefill","impl":"P2c-sdpa-flash","ai":256.0,"tflops":96.177},{"gpu":"L40","regime":"prefill","impl":"P2d-sdpa-cudnn","ai":256.0,"tflops":91.749},{"gpu":"L40","regime":"prefill","impl":"P3-triton","ai":256.0,"tflops":82.637},{"gpu":"L40","regime":"prefill","impl":"P4-fa2","ai":256.0,"tflops":91.107},{"gpu":"L40S","regime":"decode","impl":"D0-naive-kv","ai":1.0,"tflops":0.617},{"gpu":"L40S","regime":"decode","impl":"D1-inductor","ai":1.0,"tflops":0.557},{"gpu":"L40S","regime":"decode","impl":"D2-sdpa","ai":2.5,"tflops":0.75},{"gpu":"L40S","regime":"decode","impl":"D3-fa-kvcache","ai":2.5,"tflops":0.728},{"gpu":"L40S","regime":"decode","impl":"D4-flashinfer","ai":2.5,"tflops":0.723},{"gpu":"L40S","regime":"decode","impl":"D6-fa-prefill-at-1","ai":2.5,"tflops":0.66},{"gpu":"L40S","regime":"prefill","impl":"P0-naive","ai":58.562,"tflops":4.026},{"gpu":"L40S","regime":"prefill","impl":"P1-inductor","ai":58.562,"tflops":15.005},{"gpu":"L40S","regime":"prefill","impl":"P1-inductor-nofuse","ai":102.4,"tflops":20.458},{"gpu":"L40S","regime":"prefill","impl":"P1-inductor-where","ai":102.4,"tflops":20.676},{"gpu":"L40S","regime":"prefill","impl":"P2b-sdpa-mem-eff","ai":384.0,"tflops":85.118},{"gpu":"L40S","regime":"prefill","impl":"P2c-sdpa-flash","ai":384.0,"tflops":128.467},{"gpu":"L40S","regime":"prefill","impl":"P2d-sdpa-cudnn","ai":384.0,"tflops":126.017},{"gpu":"L40S","regime":"prefill","impl":"P3-triton","ai":384.0,"tflops":107.14},{"gpu":"L40S","regime":"prefill","impl":"P4-fa2","ai":384.0,"tflops":115.416}],"roofline_is_modelled":true,"selector":{"accuracy":0.385,"n":1343,"median_regret":1.018,"p95_regret":1.328,"rule":"|--- is_decode <= 0.50\n|   |--- bw_gbs <= 2356.25\n|   |   |--- log_len <= 10.50\n|   |   |   |--- log_batch <= 3.00\n|   |   |   |   |--- class: P2c-sdpa-flash\n|   |   |   |--- log_batch >  3.00\n|   |   |   |   |--- class: P4-fa2\n|   |   |--- log_len >  10.50\n|   |   |   |--- log_len <= 11.50\n|   |   |   |   |--- class: P4-fa2\n|   |   |   |--- log_len >  11.50\n|   |   |   |   |--- class: P4-fa2\n|   |--- bw_gbs >  2356.25\n|   |   |--- log_len <= 10.50\n|   |   |   |--- log_len <= 9.50\n|   |   |   |   |--- class: P2d-sdpa-cudnn\n|   |   |   |--- log_len >  9.50\n|   |   |   |   |--- class: P2d-sdpa-cudnn\n|   |   |--- log_len >  10.50\n|   |   |   |--- log_len <= 11.50\n|   |   |   |   |--- class: P4h-fa3\n|   |   |   |--- log_len >  11.50\n|   |   |   |   |--- class: P4h-fa3\n|--- is_decode >  0.50\n|   |--- log_batch <= 1.50\n|   |   |--- log_len <= 11.50\n|   |   |   |--- bw_gbs <= 2356.25\n|   |   |   |   |--- class: D3-fa-kvcache\n|   |   |   |--- bw_gbs >  2356.25\n|   |   |   |   |--- class: D4-flashinfer\n|   |   |--- log_len >  11.50\n|   |   |   |--- gqa <= 2.50\n|   |   |   |   |--- class: D3-fa-kvcache\n|   |   |   |--- gqa >  2.50\n|   |   |   |   |--- class: D3-fa-kvcache\n|   |--- log_batch >  1.50\n|   |   |--- gqa <= 2.50\n|   |   |   |--- log_len <= 9.50\n|   |   |   |   |--- class: D4-flashinfer\n|   |   |   |--- log_len >  9.50\n|   |   |   |   |--- class: D4-flashinfer\n|   |   |--- gqa >  2.50\n|   |   |   |--- log_batch <= 4.00\n|   |   |   |   |--- class: D3-fa-kvcache\n|   |   |   |--- log_batch >  4.00\n|   |   |   |   |--- class: D4-flashinfer\n","vs_fixed":{"D0-naive-kv":{"median":3.15,"p95":8.53,"coverage":0.553},"D2-sdpa":{"median":1.026,"p95":1.512,"coverage":0.571},"D3-fa-kvcache":{"median":1.0,"p95":1.332,"coverage":0.568},"D6-fa-prefill-at-1":{"median":1.249,"p95":4.227,"coverage":0.568}}},"supplementary_fwd_bwd":{"note":"prefill forward-plus-backward: a training workload, excluded from every portability number on this page","n":432,"separated":0.264},"spread":{"decode":3.14,"prefill":10.71},"status_counts":{"OK":54213,"UNSUPPORTED":16326,"OOM_PREDICTED":1590,"ERROR":950,"OOM":151},"not_collected":["Nsight Compute and Nsight Systems: profiling is blocked by pod security policy on the cluster and declined by the cloud host, so no hardware counters exist. The roofline here uses analytic traffic, not measured.","Paged versus contiguous KV layout: page_size equals the KV length in every decode cell by construction, so the paged and contiguous entries differ by library, not by layout. The page-size sweep was not run."]}