A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 01:15 UTC
[Kernel] [Helion] [6/N] Add num_tokens dimension to silu_mul autotuning and dispatching (#34185)
Signed-off-by: Yanan Cao <gmagogsfm@gmail.com>
Yanan Cao committed
Feb 20, 2026 at 08:36 UTC
a6d0299c75f2c0687334d50d302801ade083c784
3 files changed
+55199
-200
tests/kernels/helion/test_silu_mul_fp8.py
+75
-11
@@ -54,8 +54,8 @@ def reset_config_manager_singleton():
54
class TestSiluMulFp8ConfigPicker:
55
def test_config_picker_exact_match(self):
56
config_keys = [
57
- "intermediate_2048_batchsize_256",
58
- "intermediate_4096_batchsize_256",
57
+ "intermediate_2048_numtokens_256",
58
+ "intermediate_4096_numtokens_256",
59
]
60
61
input_tensor = torch.randn(32, 4096, dtype=torch.bfloat16, device="cuda")
@@ -63,12 +63,12 @@ class TestSiluMulFp8ConfigPicker:
63
args = (input_tensor, scale)
64
65
selected_key = pick_silu_mul_fp8_config(args, config_keys)
66
- assert selected_key == "intermediate_2048_batchsize_256"
66
+ assert selected_key == "intermediate_2048_numtokens_256"
67
68
def test_config_picker_closest_match(self):
69
config_keys = [
70
- "intermediate_2048_batchsize_256",
71
- "intermediate_4096_batchsize_256",
70
+ "intermediate_2048_numtokens_256",
71
+ "intermediate_4096_numtokens_256",
72
]
73
# Use 7000 (intermediate_size=3500) which is closer to 4096 than 2048
74
input_tensor = torch.randn(32, 7000, dtype=torch.bfloat16, device="cuda")
@@ -76,10 +76,10 @@ class TestSiluMulFp8ConfigPicker:
76
args = (input_tensor, scale)
77
78
selected_key = pick_silu_mul_fp8_config(args, config_keys)
79
- assert selected_key == "intermediate_4096_batchsize_256"
79
+ assert selected_key == "intermediate_4096_numtokens_256"
80
81
def test_config_picker_fallback_to_default(self):
82
- config_keys = ["default", "some_other_key"]
82
+ config_keys = ["default"]
83
84
input_tensor = torch.randn(32, 4096, dtype=torch.bfloat16, device="cuda")
85
scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
@@ -101,9 +101,9 @@ class TestSiluMulFp8ConfigPicker:
101
@pytest.mark.parametrize("intermediate_size", [2048, 4096, 5120])
102
def test_config_picker_different_sizes(self, intermediate_size):
103
config_keys = [
104
- "intermediate_2048_batchsize_256",
105
- "intermediate_4096_batchsize_256",
106
- "intermediate_5120_batchsize_256",
104
+ "intermediate_2048_numtokens_256",
105
+ "intermediate_4096_numtokens_256",
106
+ "intermediate_5120_numtokens_256",
107
]
108
109
input_tensor = torch.randn(
@@ -113,9 +113,73 @@ class TestSiluMulFp8ConfigPicker:
113
args = (input_tensor, scale)
114
115
selected_key = pick_silu_mul_fp8_config(args, config_keys)
116
- expected_key = f"intermediate_{intermediate_size}_batchsize_256"
116
+ expected_key = f"intermediate_{intermediate_size}_numtokens_256"
117
assert selected_key == expected_key
118
119
+ def test_config_picker_numtokens_ceiling(self):
120
+ """Pick the smallest numtokens >= input num_tokens."""
121
+ config_keys = [
122
+ "intermediate_4096_numtokens_8",
123
+ "intermediate_4096_numtokens_32",
124
+ "intermediate_4096_numtokens_128",
125
+ "intermediate_4096_numtokens_256",
126
+ ]
127
+ # 20 tokens -> should pick numtokens_32 (smallest >= 20)
128
+ input_tensor = torch.randn(20, 8192, dtype=torch.bfloat16, device="cuda")
129
+ scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
130
+
131
+ selected_key = pick_silu_mul_fp8_config((input_tensor, scale), config_keys)
132
+ assert selected_key == "intermediate_4096_numtokens_32"
133
+
134
+ def test_config_picker_numtokens_exact(self):
135
+ """Exact num_tokens match is preferred over ceiling."""
136
+ config_keys = [
137
+ "intermediate_4096_numtokens_8",
138
+ "intermediate_4096_numtokens_32",
139
+ "intermediate_4096_numtokens_128",
140
+ ]
141
+ input_tensor = torch.randn(32, 8192, dtype=torch.bfloat16, device="cuda")
142
+ scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
143
+
144
+ selected_key = pick_silu_mul_fp8_config((input_tensor, scale), config_keys)
145
+ assert selected_key == "intermediate_4096_numtokens_32"
146
+
147
+ def test_config_picker_numtokens_fallback_to_largest(self):
148
+ """Fall back to the largest numtokens when input exceeds all."""
149
+ config_keys = [
150
+ "intermediate_4096_numtokens_8",
151
+ "intermediate_4096_numtokens_32",
152
+ "intermediate_4096_numtokens_128",
153
+ ]
154
+ # 512 tokens -> exceeds all available, should pick largest (128)
155
+ input_tensor = torch.randn(512, 8192, dtype=torch.bfloat16, device="cuda")
156
+ scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
157
+
158
+ selected_key = pick_silu_mul_fp8_config((input_tensor, scale), config_keys)
159
+ assert selected_key == "intermediate_4096_numtokens_128"
160
+
161
+ def test_config_picker_malformed_key_raises(self):
162
+ """Malformed config keys should raise ValueError."""
163
+ config_keys = ["intermediate_4096_badformat_256"]
164
+ input_tensor = torch.randn(32, 8192, dtype=torch.bfloat16, device="cuda")
165
+ scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
166
+
167
+ with pytest.raises(ValueError, match="Malformed config key"):
168
+ pick_silu_mul_fp8_config((input_tensor, scale), config_keys)
169
+
170
+ def test_config_picker_default_ignored_when_valid_keys_exist(self):
171
+ """'default' is skipped in favor of a real match."""
172
+ config_keys = [
173
+ "default",
174
+ "intermediate_4096_numtokens_32",
175
+ "intermediate_4096_numtokens_128",
176
+ ]
177
+ input_tensor = torch.randn(64, 8192, dtype=torch.bfloat16, device="cuda")
178
+ scale = torch.tensor([0.5], dtype=torch.float32, device="cuda")
179
+
180
+ selected_key = pick_silu_mul_fp8_config((input_tensor, scale), config_keys)
181
+ assert selected_key == "intermediate_4096_numtokens_128"
182
+
183
184
class TestSiluMulFp8Correctness:
185
@pytest.mark.parametrize("batch_size", [1, 8, 32, 128])
vllm/kernels/helion/configs/silu_mul_fp8.json
+55060
-160
@@ -1,9 +1,9 @@
1
{
2
"nvidia_h200": {
3
- "intermediate_2048_batchsize_256": {
3
+ "intermediate_2048_numtokens_256": {
4
"block_sizes": [
5
64,
6
- 128
6
+ 32
7
],
8
"loop_orders": [
9
[
@@ -15,11 +15,12 @@
15
true
16
],
17
"l2_groupings": [
18
- 2
18
+ 1
19
],
20
"range_unroll_factors": [
21
0
22
],
23
+ "range_warp_specializes": [],
24
"range_num_stages": [
25
0
26
],
@@ -34,21 +35,20 @@
35
"",
36
""
37
],
37
- "num_warps": 32,
38
+ "num_warps": 8,
39
"num_stages": 1,
40
"indexing": [
41
"pointer",
41
- "tensor_descriptor",
42
+ "pointer",
43
"pointer",
44
"pointer"
45
],
45
- "pid_type": "flat",
46
- "range_warp_specializes": []
46
+ "pid_type": "flat"
47
},
48
- "intermediate_4096_batchsize_256": {
48
+ "intermediate_4096_numtokens_256": {
49
"block_sizes": [
50
- 16,
51
- 64
50
+ 32,
51
+ 512
52
],
53
"loop_orders": [
54
[
@@ -57,14 +57,15 @@
57
]
58
],
59
"flatten_loops": [
60
- false
60
+ true
61
],
62
"l2_groupings": [
63
- 1
63
+ 2
64
],
65
"range_unroll_factors": [
66
0
67
],
68
+ "range_warp_specializes": [],
69
"range_num_stages": [
70
0
71
],
@@ -80,15 +81,14 @@
81
""
82
],
83
"num_warps": 2,
83
- "num_stages": 1,
84
+ "num_stages": 2,
85
"indexing": [
86
"pointer",
86
- "pointer",
87
+ "tensor_descriptor",
88
"pointer",
89
"pointer"
90
],
90
- "pid_type": "flat",
91
- "range_warp_specializes": []
91
+ "pid_type": "flat"
92
},
93
"default": {
94
"block_sizes": [
@@ -110,6 +110,7 @@
110
"range_unroll_factors": [
111
0
112
],
113
+ "range_warp_specializes": [],
114
"range_num_stages": [
115
0
116
],
@@ -132,31 +133,29 @@
133
"tensor_descriptor",
134
"pointer"
135
],
135
- "pid_type": "flat",
136
- "range_warp_specializes": []
137
- }
138
- },
139
- "nvidia_h100_pcie": {
140
- "intermediate_2048_batchsize_256": {
136
+ "pid_type": "flat"
137
+ },
138
+ "intermediate_8192_numtokens_256": {
139
"block_sizes": [
142
- 1,
143
- 512
140
+ 32,
141
+ 8
142
],
143
"loop_orders": [
144
[
147
- 1,
148
- 0
145
+ 0,
146
+ 1
147
]
148
],
149
"flatten_loops": [
152
- false
150
+ true
151
],
152
"l2_groupings": [
155
- 4
153
+ 1
154
],
155
"range_unroll_factors": [
156
0
157
],
158
+ "range_warp_specializes": [],
159
"range_num_stages": [
160
0
161
],
@@ -171,21 +170,20 @@
170
"first",
171
""
172
],
174
- "num_warps": 8,
175
- "num_stages": 2,
173
+ "num_warps": 4,
174
+ "num_stages": 1,
175
"indexing": [
176
"tensor_descriptor",
178
- "tensor_descriptor",
179
- "tensor_descriptor",
180
- "pointer"
177
+ "pointer",
178
+ "pointer",
179
+ "tensor_descriptor"
180
],
182
- "pid_type": "flat",
183
- "range_warp_specializes": []
181
+ "pid_type": "flat"
182
},
185
- "intermediate_4096_batchsize_256": {
183
+ "intermediate_14336_numtokens_256": {
184
"block_sizes": [
187
- 256,
188
- 128
185
+ 16,
186
+ 32
187
],
188
"loop_orders": [
189
[
@@ -200,53 +198,54 @@
198
1
199
],
200
"range_unroll_factors": [
203
- 2
201
+ 0
202
],
203
+ "range_warp_specializes": [],
204
"range_num_stages": [
206
- 3
205
+ 0
206
],
207
"range_multi_buffers": [
209
- false
208
+ null
209
],
210
"range_flattens": [
212
- true
211
+ null
212
],
213
"load_eviction_policies": [
215
- "last",
216
- "last",
214
+ "",
215
+ "",
216
""
217
],
219
- "num_warps": 32,
220
- "num_stages": 3,
218
+ "num_warps": 1,
219
+ "num_stages": 1,
220
"indexing": [
222
- "tensor_descriptor",
223
- "tensor_descriptor",
224
- "tensor_descriptor",
225
- "pointer"
221
+ "pointer",
222
+ "pointer",
223
+ "pointer",
224
+ "tensor_descriptor"
225
],
227
- "pid_type": "persistent_blocked",
228
- "range_warp_specializes": []
226
+ "pid_type": "flat"
227
},
230
- "default": {
228
+ "intermediate_11008_numtokens_256": {
229
"block_sizes": [
232
- 1,
233
- 512
230
+ 64,
231
+ 32
232
],
233
"loop_orders": [
234
[
237
- 1,
238
- 0
235
+ 0,
236
+ 1
237
]
238
],
239
"flatten_loops": [
242
- false
240
+ true
241
],
242
"l2_groupings": [
245
- 4
243
+ 1
244
],
245
"range_unroll_factors": [
246
0
247
],
248
+ "range_warp_specializes": [],
249
"range_num_stages": [
250
0
251
],
@@ -258,42 +257,40 @@
257
],
258
"load_eviction_policies": [
259
"",
261
- "first",
260
+ "",
261
""
262
],
264
- "num_warps": 8,
265
- "num_stages": 2,
263
+ "num_warps": 2,
264
+ "num_stages": 1,
265
"indexing": [
267
- "tensor_descriptor",
268
- "tensor_descriptor",
269
- "tensor_descriptor",
266
+ "pointer",
267
+ "pointer",
268
+ "pointer",
269
"pointer"
270
],
272
- "pid_type": "flat",
273
- "range_warp_specializes": []
274
- }
275
- },
276
- "nvidia_h100_sxm5": {
277
- "intermediate_2048_batchsize_256": {
271
+ "pid_type": "flat"
272
+ },
273
+ "intermediate_7688_numtokens_256": {
274
"block_sizes": [
279
- 1,
280
- 512
275
+ 8,
276
+ 16
277
],
278
"loop_orders": [
279
[
284
- 1,
285
- 0
280
+ 0,
281
+ 1
282
]
283
],
284
"flatten_loops": [
289
- false
285
+ true
286
],
287
"l2_groupings": [
292
- 4
288
+ 1
289
],
290
"range_unroll_factors": [
291
0
292
],
293
+ "range_warp_specializes": [],
294
"range_num_stages": [
295
0
296
],
@@ -305,24 +302,23 @@
302
],
303
"load_eviction_policies": [
304
"",
308
- "first",
305
+ "last",
306
""
307
],
311
- "num_warps": 8,
312
- "num_stages": 2,
308
+ "num_warps": 1,
309
+ "num_stages": 1,
310
"indexing": [
314
- "tensor_descriptor",
315
- "tensor_descriptor",
316
- "tensor_descriptor",
311
+ "pointer",
312
+ "pointer",
313
+ "pointer",
314
"pointer"
315
],
319
- "pid_type": "flat",
320
- "range_warp_specializes": []
316
+ "pid_type": "flat"
317
},
322
- "intermediate_4096_batchsize_256": {
318
+ "intermediate_2880_numtokens_256": {
319
"block_sizes": [
324
- 256,
325
- 128
320
+ 32,
321
+ 64
322
],
323
"loop_orders": [
324
[
@@ -334,56 +330,57 @@
330
true
331
],
332
"l2_groupings": [
337
- 1
333
+ 2
334
],
335
"range_unroll_factors": [
340
- 2
336
+ 0
337
],
338
+ "range_warp_specializes": [],
339
"range_num_stages": [
343
- 3
340
+ 0
341
],
342
"range_multi_buffers": [
346
- false
343
+ null
344
],
345
"range_flattens": [
349
- true
346
+ null
347
],
348
"load_eviction_policies": [
352
- "last",
353
- "last",
349
+ "",
350
+ "",
351
""
352
],
356
- "num_warps": 32,
357
- "num_stages": 3,
353
+ "num_warps": 8,
354
+ "num_stages": 1,
355
"indexing": [
359
- "tensor_descriptor",
360
- "tensor_descriptor",
361
- "tensor_descriptor",
356
+ "pointer",
357
+ "pointer",
358
+ "pointer",
359
"pointer"
360
],
364
- "pid_type": "persistent_blocked",
365
- "range_warp_specializes": []
361
+ "pid_type": "flat"
362
},
367
- "default": {
363
+ "intermediate_2048_numtokens_1": {
364
"block_sizes": [
365
1,
370
- 512
366
+ 16
367
],
368
"loop_orders": [
369
[
374
- 1,
375
- 0
370
+ 0,
371
+ 1
372
]
373
],
374
"flatten_loops": [
375
false
376
],
377
"l2_groupings": [
382
- 4
378
+ 1
379
],
380
"range_unroll_factors": [
381
0
382
],
383
+ "range_warp_specializes": [],
384
"range_num_stages": [
385
0
386
],
@@ -395,42 +392,40 @@
392
],
393
"load_eviction_policies": [
394
"",
398
- "first",
395
+ "",
396
""
397
],
401
- "num_warps": 8,
398
+ "num_warps": 16,
399
"num_stages": 2,
400
"indexing": [
404
- "tensor_descriptor",
405
- "tensor_descriptor",
406
- "tensor_descriptor",
407
- "pointer"
401
+ "pointer",
402
+ "pointer",
403
+ "pointer",
404
+ "tensor_descriptor"
405
],
409
- "pid_type": "flat",
410
- "range_warp_specializes": []
411
- }
412
- },
413
- "nvidia_h100": {
414
- "intermediate_2048_batchsize_256": {
406
+ "pid_type": "flat"
407
+ },
408
+ "intermediate_2880_numtokens_1": {
409
"block_sizes": [
410
1,
417
- 512
411
+ 1
412
],
413
"loop_orders": [
414
[
421
- 1,
422
- 0
415
+ 0,
416
+ 1
417
]
418
],
419
"flatten_loops": [
420
false
421
],
422
"l2_groupings": [
429
- 4
423
+ 1
424
],
425
"range_unroll_factors": [
426
0
427
],
428
+ "range_warp_specializes": [],
429
"range_num_stages": [
430
0
431
],
@@ -442,85 +437,85 @@
437
],
438
"load_eviction_policies": [
439
"",
445
- "first",
440
+ "",
441
""
442
],
448
- "num_warps": 8,
443
+ "num_warps": 16,
444
"num_stages": 2,
445
"indexing": [
451
- "tensor_descriptor",
452
- "tensor_descriptor",
453
- "tensor_descriptor",
446
+ "pointer",
447
+ "pointer",
448
+ "pointer",
449
"pointer"
450
],
456
- "pid_type": "flat",
457
- "range_warp_specializes": []
451
+ "pid_type": "flat"
452
},
459
- "intermediate_4096_batchsize_256": {
453
+ "intermediate_4096_numtokens_1": {
454
"block_sizes": [
461
- 256,
462
- 128
455
+ 1,
456
+ 32
457
],
458
"loop_orders": [
459
[
466
- 0,
467
- 1
460
+ 1,
461
+ 0
462
]
463
],
464
"flatten_loops": [
471
- true
465
+ false
466
],
467
"l2_groupings": [
468
1
469
],
470
"range_unroll_factors": [
477
- 2
471
+ 0
472
],
473
+ "range_warp_specializes": [],
474
"range_num_stages": [
480
- 3
475
+ 0
476
],
477
"range_multi_buffers": [
483
- false
478
+ null
479
],
480
"range_flattens": [
486
- true
481
+ null
482
],
483
"load_eviction_policies": [
489
- "last",
490
- "last",
484
+ "",
485
+ "",
486
""
487
],
493
- "num_warps": 32,
494
- "num_stages": 3,
488
+ "num_warps": 4,
489
+ "num_stages": 2,
490
"indexing": [
496
- "tensor_descriptor",
497
- "tensor_descriptor",
498
- "tensor_descriptor",
491
+ "pointer",
492
+ "pointer",
493
+ "pointer",
494
"pointer"
495
],
501
- "pid_type": "persistent_blocked",
502
- "range_warp_specializes": []
496
+ "pid_type": "flat"
497
},
504
- "default": {
498
+ "intermediate_8192_numtokens_1": {
499
"block_sizes": [
500
1,
507
- 512
501
+ 32
502
],
503
"loop_orders": [
504
[
511
- 1,
512
- 0
505
+ 0,
506
+ 1
507
]
508
],
509
"flatten_loops": [
510
false
511
],
512
"l2_groupings": [
519
- 4
513
+ 1
514
],
515
"range_unroll_factors": [
516
0
517
],
518
+ "range_warp_specializes": [],
519
"range_num_stages": [
520
0
521
],
@@ -532,19 +527,54924 @@
527
],
528
"load_eviction_policies": [
529
"",
535
- "first",
530
+ "",
531
""
532
],
538
- "num_warps": 8,
533
+ "num_warps": 1,
534
"num_stages": 2,
535
"indexing": [
541
- "tensor_descriptor",
542
- "tensor_descriptor",
543
- "tensor_descriptor",
536
+ "pointer",
537
+ "pointer",
538
+ "pointer",
539
+ "pointer"
540
+ ],
541
+ "pid_type": "flat"
542
+ },
543
+ "intermediate_11008_numtokens_1": {
544
+ "block_sizes": [
545
+ 1,
546
+ 32
547
+ ],
548
+ "loop_orders": [
549
+ [
550
+ 0,
551
+ 1
552
+ ]
553
+ ],
554
+ "flatten_loops": [
555
+ true
556
+ ],
557
+ "l2_groupings": [
558
+ 1
559
+ ],
560
+ "range_unroll_factors": [
561
+ 0
562
+ ],
563
+ "range_warp_specializes": [],
564
+ "range_num_stages": [
565
+ 0
566
+ ],
567
+ "range_multi_buffers": [
568
+ null
569
+ ],
570
+ "range_flattens": [
571
+ null
572
+ ],
573
+ "load_eviction_policies": [
574
+ "first",
575
+ "",
576
+ ""
577
+ ],
578
+ "num_warps": 4,
579
+ "num_stages": 1,
580
+ "indexing": [
581
+ "pointer",
582
+ "pointer",
583
+ "pointer",
584
+ "pointer"
585
+ ],
586
+ "pid_type": "flat"
587
+ },
588
+ "intermediate_14336_numtokens_1": {
589
+ "block_sizes": [
590
+ 1,
591
+ 32
592
+ ],
593
+ "loop_orders": [
594
+ [
595
+ 0,
596
+ 1
597
+ ]
598
+ ],
599
+ "flatten_loops": [
600
+ false
601
+ ],
602
+ "l2_groupings": [
603
+ 2
604
+ ],
605
+ "range_unroll_factors": [
606
+ 0
607
+ ],
608
+ "range_warp_specializes": [],
609
+ "range_num_stages": [
610
+ 0
611
+ ],
612
+ "range_multi_buffers": [
613
+ null
614
+ ],
615
+ "range_flattens": [
616
+ null
617
+ ],
618
+ "load_eviction_policies": [
619
+ "",
620
+ "",
621
+ "first"
622
+ ],
623
+ "num_warps": 8,
624
+ "num_stages": 1,
625
+ "indexing": [
626
+ "pointer",
627
+ "pointer",
628
+ "pointer",
629
+ "pointer"
630
+ ],
631
+ "pid_type": "flat"
632
+ },
633
+ "intermediate_2048_numtokens_2": {
634
+ "block_sizes": [
635
+ 2,
636
+ 16
637
+ ],
638
+ "loop_orders": [
639
+ [
640
+ 0,
641
+ 1
642
+ ]
643
+ ],
644
+ "flatten_loops": [
645
+ true
646
+ ],
647
+ "l2_groupings": [
648
+ 2
649
+ ],
650
+ "range_unroll_factors": [
651
+ 0
652
+ ],
653
+ "range_warp_specializes": [],
654
+ "range_num_stages": [
655
+ 0
656
+ ],
657
+ "range_multi_buffers": [
658
+ null
659
+ ],
660
+ "range_flattens": [
661
+ null
662
+ ],
663
+ "load_eviction_policies": [
664
+ "",
665
+ "",
666
+ ""
667
+ ],
668
+ "num_warps": 2,
669
+ "num_stages": 1,
670
+ "indexing": [
671
+ "pointer",
672
+ "pointer",
673
+ "tensor_descriptor",
674
+ "pointer"
675
+ ],
676
+ "pid_type": "flat"
677
+ },
678
+ "intermediate_2880_numtokens_2": {
679
+ "block_sizes": [
680
+ 1,
681
+ 4
682
+ ],
683
+ "loop_orders": [
684
+ [
685
+ 0,
686
+ 1
687
+ ]
688
+ ],
689
+ "flatten_loops": [
690
+ false
691
+ ],
692
+ "l2_groupings": [
693
+ 1
694
+ ],
695
+ "range_unroll_factors": [
696
+ 0
697
+ ],
698
+ "range_warp_specializes": [],
699
+ "range_num_stages": [
700
+ 0
701
+ ],
702
+ "range_multi_buffers": [
703
+ null
704
+ ],
705
+ "range_flattens": [
706
+ null
707
+ ],
708
+ "load_eviction_policies": [
709
+ "first",
710
+ "",
711
+ ""
712
+ ],
713
+ "num_warps": 32,
714
+ "num_stages": 1,
715
+ "indexing": [
716
+ "pointer",
717
+ "pointer",
718
+ "pointer",
719
+ "pointer"
720
+ ],
721
+ "pid_type": "flat"
722
+ },
723
+ "intermediate_4096_numtokens_2": {
724
+ "block_sizes": [
725
+ 2,
726
+ 32
727
+ ],
728
+ "loop_orders": [
729
+ [
730
+ 0,
731
+ 1
732
+ ]
733
+ ],
734
+ "flatten_loops": [
735
+ true
736
+ ],
737
+ "l2_groupings": [
738
+ 1
739
+ ],
740
+ "range_unroll_factors": [
741
+ 0
742
+ ],
743
+ "range_warp_specializes": [],
744
+ "range_num_stages": [
745
+ 0
746
+ ],
747
+ "range_multi_buffers": [
748
+ null
749
+ ],
750
+ "range_flattens": [
751
+ null
752
+ ],
753
+ "load_eviction_policies": [
754
+ "",
755
+ "",
756
+ ""
757
+ ],
758
+ "num_warps": 8,
759
+ "num_stages": 2,
760
+ "indexing": [
761
+ "pointer",
762
+ "pointer",
763
+ "pointer",
764
+ "pointer"
765
+ ],
766
+ "pid_type": "flat"
767
+ },
768
+ "intermediate_8192_numtokens_2": {
769
+ "block_sizes": [
770
+ 1,
771
+ 32
772
+ ],
773
+ "loop_orders": [
774
+ [
775
+ 0,
776
+ 1
777
+ ]
778
+ ],
779
+ "flatten_loops": [
780
+ false
781
+ ],
782
+ "l2_groupings": [
783
+ 1
784
+ ],
785
+ "range_unroll_factors": [
786
+ 0
787
+ ],
788
+ "range_warp_specializes": [],
789
+ "range_num_stages": [
790
+ 0
791
+ ],
792
+ "range_multi_buffers": [
793
+ null
794
+ ],
795
+ "range_flattens": [
796
+ null
797
+ ],
798
+ "load_eviction_policies": [
799
+ "first",
800
+ "",
801
+ ""
802
+ ],
803
+ "num_warps": 8,
804
+ "num_stages": 2,
805
+ "indexing": [
806
+ "tensor_descriptor",
807
+ "pointer",
808
+ "pointer",
809
+ "pointer"
810
+ ],
811
+ "pid_type": "flat"
812
+ },
813
+ "intermediate_11008_numtokens_2": {
814
+ "block_sizes": [
815
+ 1,
816
+ 256
817
+ ],
818
+ "loop_orders": [
819
+ [
820
+ 0,
821
+ 1
822
+ ]
823
+ ],
824
+ "flatten_loops": [
825
+ false
826
+ ],
827
+ "l2_groupings": [
828
+ 1
829
+ ],
830
+ "range_unroll_factors": [
831
+ 0
832
+ ],
833
+ "range_warp_specializes": [],
834
+ "range_num_stages": [
835
+ 0
836
+ ],
837
+ "range_multi_buffers": [
838
+ null
839
+ ],
840
+ "range_flattens": [
841
+ null
842
+ ],
843
+ "load_eviction_policies": [
844
+ "",
845
+ "first",
846
+ ""
847
+ ],
848
+ "num_warps": 4,
849
+ "num_stages": 2,
850
+ "indexing": [
851
+ "pointer",
852
+ "pointer",
853
+ "pointer",
854
+ "tensor_descriptor"
855
+ ],
856
+ "pid_type": "flat"
857
+ },
858
+ "intermediate_14336_numtokens_2": {
859
+ "block_sizes": [
860
+ 1,
861
+ 64
862
+ ],
863
+ "loop_orders": [
864
+ [
865
+ 0,
866
+ 1
867
+ ]
868
+ ],
869
+ "flatten_loops": [
870
+ false
871
+ ],
872
+ "l2_groupings": [
873
+ 1
874
+ ],
875
+ "range_unroll_factors": [
876
+ 0
877
+ ],
878
+ "range_warp_specializes": [],
879
+ "range_num_stages": [
880
+ 0
881
+ ],
882
+ "range_multi_buffers": [
883
+ null
884
+ ],
885
+ "range_flattens": [
886
+ null
887
+ ],
888
+ "load_eviction_policies": [
889
+ "",
890
+ "",
891
+ ""
892
+ ],
893
+ "num_warps": 8,
894
+ "num_stages": 2,
895
+ "indexing": [
896
+ "pointer",
897
+ "pointer",
898
+ "pointer",
899
+ "pointer"
900
+ ],
901
+ "pid_type": "flat"
902
+ },
903
+ "intermediate_2048_numtokens_4": {
904
+ "block_sizes": [
905
+ 1,
906
+ 256
907
+ ],
908
+ "loop_orders": [
909
+ [
910
+ 0,
911
+ 1
912
+ ]
913
+ ],
914
+ "flatten_loops": [
915
+ true
916
+ ],
917
+ "l2_groupings": [
918
+ 1
919
+ ],
920
+ "range_unroll_factors": [
921
+ 0
922
+ ],
923
+ "range_warp_specializes": [],
924
+ "range_num_stages": [
925
+ 0
926
+ ],
927
+ "range_multi_buffers": [
928
+ null
929
+ ],
930
+ "range_flattens": [
931
+ null
932
+ ],
933
+ "load_eviction_policies": [
934
+ "",
935
+ "",
936
+ ""
937
+ ],
938
+ "num_warps": 32,
939
+ "num_stages": 2,
940
+ "indexing": [
941
+ "pointer",
942
+ "pointer",
943
+ "pointer",
944
+ "pointer"
945
+ ],
946
+ "pid_type": "flat"
947
+ },
948
+ "intermediate_2880_numtokens_4": {
949
+ "block_sizes": [
950
+ 1,
951
+ 8
952
+ ],
953
+ "loop_orders": [
954
+ [
955
+ 0,
956
+ 1
957
+ ]
958
+ ],
959
+ "flatten_loops": [
960
+ true
961
+ ],
962
+ "l2_groupings": [
963
+ 1
964
+ ],
965
+ "range_unroll_factors": [
966
+ 0
967
+ ],
968
+ "range_warp_specializes": [],
969
+ "range_num_stages": [
970
+ 0
971
+ ],
972
+ "range_multi_buffers": [
973
+ null
974
+ ],
975
+ "range_flattens": [
976
+ null
977
+ ],
978
+ "load_eviction_policies": [
979
+ "",
980
+ "",
981
+ ""
982
+ ],
983
+ "num_warps": 1,
984
+ "num_stages": 2,
985
+ "indexing": [
986
+ "pointer",
987
+ "pointer",
988
+ "pointer",
989
+ "pointer"
990
+ ],
991
+ "pid_type": "flat"
992
+ },
993
+ "intermediate_4096_numtokens_4": {
994
+ "block_sizes": [
995
+ 4,
996
+ 16
997
+ ],
998
+ "loop_orders": [
999
+ [
1000
+ 0,
1001
+ 1
1002
+ ]
1003
+ ],
1004
+ "flatten_loops": [
1005
+ true
1006
+ ],
1007
+ "l2_groupings": [
1008
+ 2
1009
+ ],
1010
+ "range_unroll_factors": [
1011
+ 0
1012
+ ],
1013
+ "range_warp_specializes": [],
1014
+ "range_num_stages": [
1015
+ 0
1016
+ ],
1017
+ "range_multi_buffers": [
1018
+ null
1019
+ ],
1020
+ "range_flattens": [
1021
+ null
1022
+ ],
1023
+ "load_eviction_policies": [
1024
+ "last",
1025
+ "",
1026
+ ""
1027
+ ],
1028
+ "num_warps": 4,
1029
+ "num_stages": 1,
1030
+ "indexing": [
1031
+ "pointer",
1032
+ "pointer",
1033
+ "pointer",
1034
+ "pointer"
1035
+ ],
1036
+ "pid_type": "flat"
1037
+ },
1038
+ "intermediate_8192_numtokens_4": {
1039
+ "block_sizes": [
1040
+ 1,
1041
+ 16
1042
+ ],
1043
+ "loop_orders": [
1044
+ [
1045
+ 0,
1046
+ 1
1047
+ ]
1048
+ ],
1049
+ "flatten_loops": [
1050
+ true
1051
+ ],
1052
+ "l2_groupings": [
1053
+ 1
1054
+ ],
1055
+ "range_unroll_factors": [
1056
+ 0
1057
+ ],
1058
+ "range_warp_specializes": [],
1059
+ "range_num_stages": [
1060
+ 0
1061
+ ],
1062
+ "range_multi_buffers": [
1063
+ null
1064
+ ],
1065
+ "range_flattens": [
1066
+ null
1067
+ ],
1068
+ "load_eviction_policies": [
1069
+ "",
1070
+ "",
1071
+ ""
1072
+ ],
1073
+ "num_warps": 4,
1074
+ "num_stages": 1,
1075
+ "indexing": [
1076
+ "pointer",
1077
+ "pointer",
1078
+ "pointer",
1079
+ "pointer"
1080
+ ],
1081
+ "pid_type": "flat"
1082
+ },
1083
+ "intermediate_11008_numtokens_4": {
1084
+ "block_sizes": [
1085
+ 1,
1086
+ 32
1087
+ ],
1088
+ "loop_orders": [
1089
+ [
1090
+ 0,
1091
+ 1
1092
+ ]
1093
+ ],
1094
+ "flatten_loops": [
1095
+ false
1096
+ ],
1097
+ "l2_groupings": [
1098
+ 1
1099
+ ],
1100
+ "range_unroll_factors": [
1101
+ 0
1102
+ ],
1103
+ "range_warp_specializes": [],
1104
+ "range_num_stages": [
1105
+ 0
1106
+ ],
1107
+ "range_multi_buffers": [
1108
+ null
1109
+ ],
1110
+ "range_flattens": [
1111
+ null
1112
+ ],
1113
+ "load_eviction_policies": [
1114
+ "",
1115
+ "first",
1116
+ ""
1117
+ ],
1118
+ "num_warps": 16,
1119
+ "num_stages": 1,
1120
+ "indexing": [
1121
+ "pointer",
1122
+ "pointer",
1123
+ "tensor_descriptor",
1124
+ "pointer"
1125
+ ],
1126
+ "pid_type": "flat"
1127
+ },
1128
+ "intermediate_14336_numtokens_4": {
1129
+ "block_sizes": [
1130
+ 4,
1131
+ 16
1132
+ ],
1133
+ "loop_orders": [
1134
+ [
1135
+ 1,
1136
+ 0
1137
+ ]
1138
+ ],
1139
+ "flatten_loops": [
1140
+ true
1141
+ ],
1142
+ "l2_groupings": [
1143
+ 1
1144
+ ],
1145
+ "range_unroll_factors": [
1146
+ 0
1147
+ ],
1148
+ "range_warp_specializes": [],
1149
+ "range_num_stages": [
1150
+ 0
1151
+ ],
1152
+ "range_multi_buffers": [
1153
+ null
1154
+ ],
1155
+ "range_flattens": [
1156
+ null
1157
+ ],
1158
+ "load_eviction_policies": [
1159
+ "",
1160
+ "",
1161
+ ""
1162
+ ],
1163
+ "num_warps": 16,
1164
+ "num_stages": 1,
1165
+ "indexing": [
1166
+ "pointer",
1167
+ "pointer",
1168
+ "pointer",
1169
+ "pointer"
1170
+ ],
1171
+ "pid_type": "flat"
1172
+ },
1173
+ "intermediate_2048_numtokens_8": {
1174
+ "block_sizes": [
1175
+ 8,
1176
+ 256
1177
+ ],
1178
+ "loop_orders": [
1179
+ [
1180
+ 1,
1181
+ 0
1182
+ ]
1183
+ ],
1184
+ "flatten_loops": [
1185
+ true
1186
+ ],
1187
+ "l2_groupings": [
1188
+ 1
1189
+ ],
1190
+ "range_unroll_factors": [
1191
+ 0
1192
+ ],
1193
+ "range_warp_specializes": [],
1194
+ "range_num_stages": [
1195
+ 0
1196
+ ],
1197
+ "range_multi_buffers": [
1198
+ null
1199
+ ],
1200
+ "range_flattens": [
1201
+ null
1202
+ ],
1203
+ "load_eviction_policies": [
1204
+ "last",
1205
+ "",
1206
+ ""
1207
+ ],
1208
+ "num_warps": 32,
1209
+ "num_stages": 1,
1210
+ "indexing": [
1211
+ "tensor_descriptor",
1212
+ "pointer",
1213
+ "pointer",
1214
+ "pointer"
1215
+ ],
1216
+ "pid_type": "flat"
1217
+ },
1218
+ "intermediate_2880_numtokens_8": {
1219
+ "block_sizes": [
1220
+ 8,
1221
+ 32
1222
+ ],
1223
+ "loop_orders": [
1224
+ [
1225
+ 0,
1226
+ 1
1227
+ ]
1228
+ ],
1229
+ "flatten_loops": [
1230
+ true
1231
+ ],
1232
+ "l2_groupings": [
1233
+ 1
1234
+ ],
1235
+ "range_unroll_factors": [
1236
+ 0
1237
+ ],
1238
+ "range_warp_specializes": [],
1239
+ "range_num_stages": [
1240
+ 0
1241
+ ],
1242
+ "range_multi_buffers": [
1243
+ null
1244
+ ],
1245
+ "range_flattens": [
1246
+ null
1247
+ ],
1248
+ "load_eviction_policies": [
1249
+ "",
1250
+ "",
1251
+ ""
1252
+ ],
1253
+ "num_warps": 4,
1254
+ "num_stages": 1,
1255
+ "indexing": [
1256
+ "pointer",
1257
+ "pointer",
1258
+ "pointer",
1259
+ "pointer"
1260
+ ],
1261
+ "pid_type": "flat"
1262
+ },
1263
+ "intermediate_4096_numtokens_8": {
1264
+ "block_sizes": [
1265
+ 2,
1266
+ 32
1267
+ ],
1268
+ "loop_orders": [
1269
+ [
1270
+ 0,
1271
+ 1
1272
+ ]
1273
+ ],
1274
+ "flatten_loops": [
1275
+ true
1276
+ ],
1277
+ "l2_groupings": [
1278
+ 2
1279
+ ],
1280
+ "range_unroll_factors": [
1281
+ 0
1282
+ ],
1283
+ "range_warp_specializes": [],
1284
+ "range_num_stages": [
1285
+ 0
1286
+ ],
1287
+ "range_multi_buffers": [
1288
+ null
1289
+ ],
1290
+ "range_flattens": [
1291
+ null
1292
+ ],
1293
+ "load_eviction_policies": [
1294
+ "",
1295
+ "",
1296
+ ""
1297
+ ],
1298
+ "num_warps": 32,
1299
+ "num_stages": 1,
1300
+ "indexing": [
1301
+ "pointer",
1302
+ "pointer",
1303
+ "pointer",
1304
+ "pointer"
1305
+ ],
1306
+ "pid_type": "flat"
1307
+ },
1308
+ "intermediate_8192_numtokens_8": {
1309
+ "block_sizes": [
1310
+ 4,
1311
+ 64
1312
+ ],
1313
+ "loop_orders": [
1314
+ [
1315
+ 0,
1316
+ 1
1317
+ ]
1318
+ ],
1319
+ "flatten_loops": [
1320
+ false
1321
+ ],
1322
+ "l2_groupings": [
1323
+ 1
1324
+ ],
1325
+ "range_unroll_factors": [
1326
+ 0
1327
+ ],
1328
+ "range_warp_specializes": [],
1329
+ "range_num_stages": [
1330
+ 0
1331
+ ],
1332
+ "range_multi_buffers": [
1333
+ null
1334
+ ],
1335
+ "range_flattens": [
1336
+ null
1337
+ ],
1338
+ "load_eviction_policies": [
1339
+ "",
1340
+ "",
1341
+ ""
1342
+ ],
1343
+ "num_warps": 1,
1344
+ "num_stages": 1,
1345
+ "indexing": [
1346
+ "pointer",
1347
+ "pointer",
1348
+ "pointer",
1349
+ "pointer"
1350
+ ],
1351
+ "pid_type": "flat"
1352
+ },
1353
+ "intermediate_11008_numtokens_8": {
1354
+ "block_sizes": [
1355
+ 8,
1356
+ 128
1357
+ ],
1358
+ "loop_orders": [
1359
+ [
1360
+ 0,
1361
+ 1
1362
+ ]
1363
+ ],
1364
+ "flatten_loops": [
1365
+ false
1366
+ ],
1367
+ "l2_groupings": [
1368
+ 1
1369
+ ],
1370
+ "range_unroll_factors": [
1371
+ 0
1372
+ ],
1373
+ "range_warp_specializes": [],
1374
+ "range_num_stages": [
1375
+ 0
1376
+ ],
1377
+ "range_multi_buffers": [
1378
+ null
1379
+ ],
1380
+ "range_flattens": [
1381
+ null
1382
+ ],
1383
+ "load_eviction_policies": [
1384
+ "last",
1385
+ "",
1386
+ ""
1387
+ ],
1388
+ "num_warps": 4,
1389
+ "num_stages": 2,
1390
+ "indexing": [
1391
+ "pointer",
1392
+ "pointer",
1393
+ "pointer",
1394
+ "pointer"
1395
+ ],
1396
+ "pid_type": "flat"
1397
+ },
1398
+ "intermediate_14336_numtokens_8": {
1399
+ "block_sizes": [
1400
+ 8,
1401
+ 32
1402
+ ],
1403
+ "loop_orders": [
1404
+ [
1405
+ 0,
1406
+ 1
1407
+ ]
1408
+ ],
1409
+ "flatten_loops": [
1410
+ false
1411
+ ],
1412
+ "l2_groupings": [
1413
+ 1
1414
+ ],
1415
+ "range_unroll_factors": [
1416
+ 0
1417
+ ],
1418
+ "range_warp_specializes": [],
1419
+ "range_num_stages": [
1420
+ 0
1421
+ ],
1422
+ "range_multi_buffers": [
1423
+ null
1424
+ ],
1425
+ "range_flattens": [
1426
+ null
1427
+ ],
1428
+ "load_eviction_policies": [
1429
+ "",
1430
+ "first",
1431
+ ""
1432
+ ],
1433
+ "num_warps": 4,
1434
+ "num_stages": 1,
1435
+ "indexing": [
1436
+ "pointer",
1437
+ "pointer",
1438
+ "pointer",
1439
+ "pointer"
1440
+ ],
1441
+ "pid_type": "flat"
1442
+ },
1443
+ "intermediate_2048_numtokens_16": {
1444
+ "block_sizes": [
1445
+ 16,
1446
+ 64
1447
+ ],
1448
+ "loop_orders": [
1449
+ [
1450
+ 0,
1451
+ 1
1452
+ ]
1453
+ ],
1454
+ "flatten_loops": [
1455
+ false
1456
+ ],
1457
+ "l2_groupings": [
1458
+ 2
1459
+ ],
1460
+ "range_unroll_factors": [
1461
+ 0
1462
+ ],
1463
+ "range_warp_specializes": [],
1464
+ "range_num_stages": [
1465
+ 0
1466
+ ],
1467
+ "range_multi_buffers": [
1468
+ null
1469
+ ],
1470
+ "range_flattens": [
1471
+ null
1472
+ ],
1473
+ "load_eviction_policies": [
1474
+ "",
1475
+ "",
1476
+ ""
1477
+ ],
1478
+ "num_warps": 4,
1479
+ "num_stages": 2,
1480
+ "indexing": [
1481
+ "pointer",
1482
+ "pointer",
1483
+ "pointer",
1484
+ "pointer"
1485
+ ],
1486
+ "pid_type": "flat"
1487
+ },
1488
+ "intermediate_2880_numtokens_16": {
1489
+ "block_sizes": [
1490
+ 16,
1491
+ 32
1492
+ ],
1493
+ "loop_orders": [
1494
+ [
1495
+ 0,
1496
+ 1
1497
+ ]
1498
+ ],
1499
+ "flatten_loops": [
1500
+ true
1501
+ ],
1502
+ "l2_groupings": [
1503
+ 1
1504
+ ],
1505
+ "range_unroll_factors": [
1506
+ 0
1507
+ ],
1508
+ "range_warp_specializes": [],
1509
+ "range_num_stages": [
1510
+ 0
1511
+ ],
1512
+ "range_multi_buffers": [
1513
+ null
1514
+ ],
1515
+ "range_flattens": [
1516
+ null
1517
+ ],
1518
+ "load_eviction_policies": [
1519
+ "",
1520
+ "",
1521
+ ""
1522
+ ],
1523
+ "num_warps": 8,
1524
+ "num_stages": 1,
1525
+ "indexing": [
1526
+ "pointer",
1527
+ "pointer",
1528
+ "pointer",
1529
+ "pointer"
1530
+ ],
1531
+ "pid_type": "flat"
1532
+ },
1533
+ "intermediate_4096_numtokens_16": {
1534
+ "block_sizes": [
1535
+ 16,
1536
+ 32
1537
+ ],
1538
+ "loop_orders": [
1539
+ [
1540
+ 0,
1541
+ 1
1542
+ ]
1543
+ ],
1544
+ "flatten_loops": [
1545
+ true
1546
+ ],
1547
+ "l2_groupings": [
1548
+ 1
1549
+ ],
1550
+ "range_unroll_factors": [
1551
+ 0
1552
+ ],
1553
+ "range_warp_specializes": [],
1554
+ "range_num_stages": [
1555
+ 0
1556
+ ],
1557
+ "range_multi_buffers": [
1558
+ null
1559
+ ],
1560
+ "range_flattens": [
1561
+ null
1562
+ ],
1563
+ "load_eviction_policies": [
1564
+ "",
1565
+ "",
1566
+ ""
1567
+ ],
1568
+ "num_warps": 8,
1569
+ "num_stages": 2,
1570
+ "indexing": [
1571
+ "pointer",
1572
+ "pointer",
1573
+ "pointer",
1574
+ "pointer"
1575
+ ],
1576
+ "pid_type": "flat"
1577
+ },
1578
+ "intermediate_8192_numtokens_16": {
1579
+ "block_sizes": [
1580
+ 4,
1581
+ 32
1582
+ ],
1583
+ "loop_orders": [
1584
+ [
1585
+ 0,
1586
+ 1
1587
+ ]
1588
+ ],
1589
+ "flatten_loops": [
1590
+ false
1591
+ ],
1592
+ "l2_groupings": [
1593
+ 1
1594
+ ],
1595
+ "range_unroll_factors": [
1596
+ 0
1597
+ ],
1598
+ "range_warp_specializes": [],
1599
+ "range_num_stages": [
1600
+ 0
1601
+ ],
1602
+ "range_multi_buffers": [
1603
+ null
1604
+ ],
1605
+ "range_flattens": [
1606
+ null
1607
+ ],
1608
+ "load_eviction_policies": [
1609
+ "",
1610
+ "",
1611
+ ""
1612
+ ],
1613
+ "num_warps": 1,
1614
+ "num_stages": 1,
1615
+ "indexing": [
1616
+ "pointer",
1617
+ "tensor_descriptor",
1618
+ "pointer",
1619
+ "pointer"
1620
+ ],
1621
+ "pid_type": "flat"
1622
+ },
1623
+ "intermediate_11008_numtokens_16": {
1624
+ "block_sizes": [
1625
+ 8,
1626
+ 32
1627
+ ],
1628
+ "loop_orders": [
1629
+ [
1630
+ 0,
1631
+ 1
1632
+ ]
1633
+ ],
1634
+ "flatten_loops": [
1635
+ true
1636
+ ],
1637
+ "l2_groupings": [
1638
+ 1
1639
+ ],
1640
+ "range_unroll_factors": [
1641
+ 0
1642
+ ],
1643
+ "range_warp_specializes": [],
1644
+ "range_num_stages": [
1645
+ 0
1646
+ ],
1647
+ "range_multi_buffers": [
1648
+ null
1649
+ ],
1650
+ "range_flattens": [
1651
+ null
1652
+ ],
1653
+ "load_eviction_policies": [
1654
+ "",
1655
+ "",
1656
+ "first"
1657
+ ],
1658
+ "num_warps": 16,
1659
+ "num_stages": 1,
1660
+ "indexing": [
1661
+ "pointer",
1662
+ "pointer",
1663
+ "pointer",
1664
+ "pointer"
1665
+ ],
1666
+ "pid_type": "flat"
1667
+ },
1668
+ "intermediate_14336_numtokens_16": {
1669
+ "block_sizes": [
1670
+ 16,
1671
+ 32
1672
+ ],
1673
+ "loop_orders": [
1674
+ [
1675
+ 0,
1676
+ 1
1677
+ ]
1678
+ ],
1679
+ "flatten_loops": [
1680
+ true
1681
+ ],
1682
+ "l2_groupings": [
1683
+ 1
1684
+ ],
1685
+ "range_unroll_factors": [
1686
+ 0
1687
+ ],
1688
+ "range_warp_specializes": [],
1689
+ "range_num_stages": [
1690
+ 0
1691
+ ],
1692
+ "range_multi_buffers": [
1693
+ null
1694
+ ],
1695
+ "range_flattens": [
1696
+ null
1697
+ ],
1698
+ "load_eviction_policies": [
1699
+ "",
1700
+ "",
1701
+ ""
1702
+ ],
1703
+ "num_warps": 1,
1704
+ "num_stages": 1,
1705
+ "indexing": [
1706
+ "tensor_descriptor",
1707
+ "pointer",
1708
+ "pointer",
1709
+ "pointer"
1710
+ ],
1711
+ "pid_type": "flat"
1712
+ },
1713
+ "intermediate_2048_numtokens_24": {
1714
+ "block_sizes": [
1715
+ 16,
1716
+ 8
1717
+ ],
1718
+ "loop_orders": [
1719
+ [
1720
+ 0,
1721
+ 1
1722
+ ]
1723
+ ],
1724
+ "flatten_loops": [
1725
+ true
1726
+ ],
1727
+ "l2_groupings": [
1728
+ 1
1729
+ ],
1730
+ "range_unroll_factors": [
1731
+ 0
1732
+ ],
1733
+ "range_warp_specializes": [],
1734
+ "range_num_stages": [
1735
+ 0
1736
+ ],
1737
+ "range_multi_buffers": [
1738
+ null
1739
+ ],
1740
+ "range_flattens": [
1741
+ null
1742
+ ],
1743
+ "load_eviction_policies": [
1744
+ "",
1745
+ "last",
1746
+ "last"
1747
+ ],
1748
+ "num_warps": 1,
1749
+ "num_stages": 1,
1750
+ "indexing": [
1751
+ "pointer",
1752
+ "pointer",
1753
+ "pointer",
1754
+ "pointer"
1755
+ ],
1756
+ "pid_type": "flat"
1757
+ },
1758
+ "intermediate_2880_numtokens_24": {
1759
+ "block_sizes": [
1760
+ 32,
1761
+ 64
1762
+ ],
1763
+ "loop_orders": [
1764
+ [
1765
+ 1,
1766
+ 0
1767
+ ]
1768
+ ],
1769
+ "flatten_loops": [
1770
+ true
1771
+ ],
1772
+ "l2_groupings": [
1773
+ 1
1774
+ ],
1775
+ "range_unroll_factors": [
1776
+ 0
1777
+ ],
1778
+ "range_warp_specializes": [],
1779
+ "range_num_stages": [
1780
+ 0
1781
+ ],
1782
+ "range_multi_buffers": [
1783
+ null
1784
+ ],
1785
+ "range_flattens": [
1786
+ null
1787
+ ],
1788
+ "load_eviction_policies": [
1789
+ "",
1790
+ "",
1791
+ ""
1792
+ ],
1793
+ "num_warps": 4,
1794
+ "num_stages": 1,
1795
+ "indexing": [
1796
+ "tensor_descriptor",
1797
+ "pointer",
1798
+ "pointer",
1799
+ "tensor_descriptor"
1800
+ ],
1801
+ "pid_type": "flat"
1802
+ },
1803
+ "intermediate_4096_numtokens_24": {
1804
+ "block_sizes": [
1805
+ 32,
1806
+ 32
1807
+ ],
1808
+ "loop_orders": [
1809
+ [
1810
+ 0,
1811
+ 1
1812
+ ]
1813
+ ],
1814
+ "flatten_loops": [
1815
+ true
1816
+ ],
1817
+ "l2_groupings": [
1818
+ 2
1819
+ ],
1820
+ "range_unroll_factors": [
1821
+ 0
1822
+ ],
1823
+ "range_warp_specializes": [],
1824
+ "range_num_stages": [
1825
+ 0
1826
+ ],
1827
+ "range_multi_buffers": [
1828
+ null
1829
+ ],
1830
+ "range_flattens": [
1831
+ null
1832
+ ],
1833
+ "load_eviction_policies": [
1834
+ "",
1835
+ "first",
1836
+ ""
1837
+ ],
1838
+ "num_warps": 1,
1839
+ "num_stages": 1,
1840
+ "indexing": [
1841
+ "pointer",
1842
+ "pointer",
1843
+ "pointer",
1844
+ "tensor_descriptor"
1845
+ ],
1846
+ "pid_type": "flat"
1847
+ },
1848
+ "intermediate_8192_numtokens_24": {
1849
+ "block_sizes": [
1850
+ 16,
1851
+ 32
1852
+ ],
1853
+ "loop_orders": [
1854
+ [
1855
+ 0,
1856
+ 1
1857
+ ]
1858
+ ],
1859
+ "flatten_loops": [
1860
+ false
1861
+ ],
1862
+ "l2_groupings": [
1863
+ 1
1864
+ ],
1865
+ "range_unroll_factors": [
1866
+ 0
1867
+ ],
1868
+ "range_warp_specializes": [],
1869
+ "range_num_stages": [
1870
+ 0
1871
+ ],
1872
+ "range_multi_buffers": [
1873
+ null
1874
+ ],
1875
+ "range_flattens": [
1876
+ null
1877
+ ],
1878
+ "load_eviction_policies": [
1879
+ "",
1880
+ "",
1881
+ ""
1882
+ ],
1883
+ "num_warps": 2,
1884
+ "num_stages": 1,
1885
+ "indexing": [
1886
+ "pointer",
1887
+ "pointer",
1888
+ "pointer",
1889
+ "pointer"
1890
+ ],
1891
+ "pid_type": "flat"
1892
+ },
1893
+ "intermediate_11008_numtokens_24": {
1894
+ "block_sizes": [
1895
+ 32,
1896
+ 8
1897
+ ],
1898
+ "loop_orders": [
1899
+ [
1900
+ 0,
1901
+ 1
1902
+ ]
1903
+ ],
1904
+ "flatten_loops": [
1905
+ false
1906
+ ],
1907
+ "l2_groupings": [
1908
+ 1
1909
+ ],
1910
+ "range_unroll_factors": [
1911
+ 0
1912
+ ],
1913
+ "range_warp_specializes": [],
1914
+ "range_num_stages": [
1915
+ 0
1916
+ ],
1917
+ "range_multi_buffers": [
1918
+ null
1919
+ ],
1920
+ "range_flattens": [
1921
+ null
1922
+ ],
1923
+ "load_eviction_policies": [
1924
+ "",
1925
+ "",
1926
+ ""
1927
+ ],
1928
+ "num_warps": 1,
1929
+ "num_stages": 1,
1930
+ "indexing": [
1931
+ "pointer",
1932
+ "pointer",
1933
+ "pointer",
1934
+ "pointer"
1935
+ ],
1936
+ "pid_type": "flat"
1937
+ },
1938
+ "intermediate_14336_numtokens_24": {
1939
+ "block_sizes": [
1940
+ 8,
1941
+ 32
1942
+ ],
1943
+ "loop_orders": [
1944
+ [
1945
+ 1,
1946
+ 0
1947
+ ]
1948
+ ],
1949
+ "flatten_loops": [
1950
+ true
1951
+ ],
1952
+ "l2_groupings": [
1953
+ 1
1954
+ ],
1955
+ "range_unroll_factors": [
1956
+ 0
1957
+ ],
1958
+ "range_warp_specializes": [],
1959
+ "range_num_stages": [
1960
+ 0
1961
+ ],
1962
+ "range_multi_buffers": [
1963
+ null
1964
+ ],
1965
+ "range_flattens": [
1966
+ null
1967
+ ],
1968
+ "load_eviction_policies": [
1969
+ "",
1970
+ "",
1971
+ "last"
1972
+ ],
1973
+ "num_warps": 2,
1974
+ "num_stages": 1,
1975
+ "indexing": [
1976
+ "pointer",
1977
+ "pointer",
1978
+ "pointer",
1979
+ "pointer"
1980
+ ],
1981
+ "pid_type": "flat"
1982
+ },
1983
+ "intermediate_2048_numtokens_32": {
1984
+ "block_sizes": [
1985
+ 32,
1986
+ 64
1987
+ ],
1988
+ "loop_orders": [
1989
+ [
1990
+ 0,
1991
+ 1
1992
+ ]
1993
+ ],
1994
+ "flatten_loops": [
1995
+ true
1996
+ ],
1997
+ "l2_groupings": [
1998
+ 1
1999
+ ],
2000
+ "range_unroll_factors": [
2001
+ 0
2002
+ ],
2003
+ "range_warp_specializes": [],
2004
+ "range_num_stages": [
2005
+ 0
2006
+ ],
2007
+ "range_multi_buffers": [
2008
+ null
2009
+ ],
2010
+ "range_flattens": [
2011
+ null
2012
+ ],
2013
+ "load_eviction_policies": [
2014
+ "",
2015
+ "",
2016
+ ""
2017
+ ],
2018
+ "num_warps": 1,
2019
+ "num_stages": 1,
2020
+ "indexing": [
2021
+ "pointer",
2022
+ "pointer",
2023
+ "pointer",
2024
+ "pointer"
2025
+ ],
2026
+ "pid_type": "flat"
2027
+ },
2028
+ "intermediate_2880_numtokens_32": {
2029
+ "block_sizes": [
2030
+ 32,
2031
+ 64
2032
+ ],
2033
+ "loop_orders": [
2034
+ [
2035
+ 0,
2036
+ 1
2037
+ ]
2038
+ ],
2039
+ "flatten_loops": [
2040
+ true
2041
+ ],
2042
+ "l2_groupings": [
2043
+ 2
2044
+ ],
2045
+ "range_unroll_factors": [
2046
+ 0
2047
+ ],
2048
+ "range_warp_specializes": [],
2049
+ "range_num_stages": [
2050
+ 0
2051
+ ],
2052
+ "range_multi_buffers": [
2053
+ null
2054
+ ],
2055
+ "range_flattens": [
2056
+ null
2057
+ ],
2058
+ "load_eviction_policies": [
2059
+ "",
2060
+ "",
2061
+ ""
2062
+ ],
2063
+ "num_warps": 1,
2064
+ "num_stages": 1,
2065
+ "indexing": [
2066
+ "pointer",
2067
+ "pointer",
2068
+ "pointer",
2069
+ "pointer"
2070
+ ],
2071
+ "pid_type": "flat"
2072
+ },
2073
+ "intermediate_4096_numtokens_32": {
2074
+ "block_sizes": [
2075
+ 32,
2076
+ 16
2077
+ ],
2078
+ "loop_orders": [
2079
+ [
2080
+ 0,
2081
+ 1
2082
+ ]
2083
+ ],
2084
+ "flatten_loops": [
2085
+ true
2086
+ ],
2087
+ "l2_groupings": [
2088
+ 1
2089
+ ],
2090
+ "range_unroll_factors": [
2091
+ 0
2092
+ ],
2093
+ "range_warp_specializes": [],
2094
+ "range_num_stages": [
2095
+ 0
2096
+ ],
2097
+ "range_multi_buffers": [
2098
+ null
2099
+ ],
2100
+ "range_flattens": [
2101
+ null
2102
+ ],
2103
+ "load_eviction_policies": [
2104
+ "",
2105
+ "",
2106
+ ""
2107
+ ],
2108
+ "num_warps": 1,
2109
+ "num_stages": 2,
2110
+ "indexing": [
2111
+ "pointer",
2112
+ "pointer",
2113
+ "pointer",
2114
+ "pointer"
2115
+ ],
2116
+ "pid_type": "flat"
2117
+ },
2118
+ "intermediate_8192_numtokens_32": {
2119
+ "block_sizes": [
2120
+ 32,
2121
+ 128
2122
+ ],
2123
+ "loop_orders": [
2124
+ [
2125
+ 0,
2126
+ 1
2127
+ ]
2128
+ ],
2129
+ "flatten_loops": [
2130
+ true
2131
+ ],
2132
+ "l2_groupings": [
2133
+ 1
2134
+ ],
2135
+ "range_unroll_factors": [
2136
+ 0
2137
+ ],
2138
+ "range_warp_specializes": [],
2139
+ "range_num_stages": [
2140
+ 0
2141
+ ],
2142
+ "range_multi_buffers": [
2143
+ null
2144
+ ],
2145
+ "range_flattens": [
2146
+ null
2147
+ ],
2148
+ "load_eviction_policies": [
2149
+ "",
2150
+ "",
2151
+ "last"
2152
+ ],
2153
+ "num_warps": 2,
2154
+ "num_stages": 2,
2155
+ "indexing": [
2156
+ "pointer",
2157
+ "pointer",
2158
+ "pointer",
2159
+ "pointer"
2160
+ ],
2161
+ "pid_type": "flat"
2162
+ },
2163
+ "intermediate_11008_numtokens_32": {
2164
+ "block_sizes": [
2165
+ 16,
2166
+ 8
2167
+ ],
2168
+ "loop_orders": [
2169
+ [
2170
+ 1,
2171
+ 0
2172
+ ]
2173
+ ],
2174
+ "flatten_loops": [
2175
+ true
2176
+ ],
2177
+ "l2_groupings": [
2178
+ 1
2179
+ ],
2180
+ "range_unroll_factors": [
2181
+ 0
2182
+ ],
2183
+ "range_warp_specializes": [],
2184
+ "range_num_stages": [
2185
+ 0
2186
+ ],
2187
+ "range_multi_buffers": [
2188
+ null
2189
+ ],
2190
+ "range_flattens": [
2191
+ null
2192
+ ],
2193
+ "load_eviction_policies": [
2194
+ "",
2195
+ "",
2196
+ ""
2197
+ ],
2198
+ "num_warps": 16,
2199
+ "num_stages": 1,
2200
+ "indexing": [
2201
+ "pointer",
2202
+ "pointer",
2203
+ "pointer",
2204
+ "tensor_descriptor"
2205
+ ],
2206
+ "pid_type": "flat"
2207
+ },
2208
+ "intermediate_14336_numtokens_32": {
2209
+ "block_sizes": [
2210
+ 32,
2211
+ 8
2212
+ ],
2213
+ "loop_orders": [
2214
+ [
2215
+ 0,
2216
+ 1
2217
+ ]
2218
+ ],
2219
+ "flatten_loops": [
2220
+ true
2221
+ ],
2222
+ "l2_groupings": [
2223
+ 1
2224
+ ],
2225
+ "range_unroll_factors": [
2226
+ 0
2227
+ ],
2228
+ "range_warp_specializes": [],
2229
+ "range_num_stages": [
2230
+ 0
2231
+ ],
2232
+ "range_multi_buffers": [
2233
+ null
2234
+ ],
2235
+ "range_flattens": [
2236
+ null
2237
+ ],
2238
+ "load_eviction_policies": [
2239
+ "",
2240
+ "",
2241
+ ""
2242
+ ],
2243
+ "num_warps": 8,
2244
+ "num_stages": 2,
2245
+ "indexing": [
2246
+ "pointer",
2247
+ "pointer",
2248
+ "pointer",
2249
+ "pointer"
2250
+ ],
2251
+ "pid_type": "flat"
2252
+ },
2253
+ "intermediate_2048_numtokens_40": {
2254
+ "block_sizes": [
2255
+ 64,
2256
+ 32
2257
+ ],
2258
+ "loop_orders": [
2259
+ [
2260
+ 0,
2261
+ 1
2262
+ ]
2263
+ ],
2264
+ "flatten_loops": [
2265
+ true
2266
+ ],
2267
+ "l2_groupings": [
2268
+ 2
2269
+ ],
2270
+ "range_unroll_factors": [
2271
+ 0
2272
+ ],
2273
+ "range_warp_specializes": [],
2274
+ "range_num_stages": [
2275
+ 0
2276
+ ],
2277
+ "range_multi_buffers": [
2278
+ null
2279
+ ],
2280
+ "range_flattens": [
2281
+ null
2282
+ ],
2283
+ "load_eviction_policies": [
2284
+ "",
2285
+ "",
2286
+ ""
2287
+ ],
2288
+ "num_warps": 2,
2289
+ "num_stages": 1,
2290
+ "indexing": [
2291
+ "pointer",
2292
+ "pointer",
2293
+ "pointer",
2294
+ "pointer"
2295
+ ],
2296
+ "pid_type": "flat"
2297
+ },
2298
+ "intermediate_2880_numtokens_40": {
2299
+ "block_sizes": [
2300
+ 64,
2301
+ 32
2302
+ ],
2303
+ "loop_orders": [
2304
+ [
2305
+ 0,
2306
+ 1
2307
+ ]
2308
+ ],
2309
+ "flatten_loops": [
2310
+ true
2311
+ ],
2312
+ "l2_groupings": [
2313
+ 1
2314
+ ],
2315
+ "range_unroll_factors": [
2316
+ 0
2317
+ ],
2318
+ "range_warp_specializes": [],
2319
+ "range_num_stages": [
2320
+ 0
2321
+ ],
2322
+ "range_multi_buffers": [
2323
+ null
2324
+ ],
2325
+ "range_flattens": [
2326
+ null
2327
+ ],
2328
+ "load_eviction_policies": [
2329
+ "",
2330
+ "",
2331
+ "last"
2332
+ ],
2333
+ "num_warps": 8,
2334
+ "num_stages": 2,
2335
+ "indexing": [
2336
+ "pointer",
2337
+ "pointer",
2338
+ "pointer",
2339
+ "pointer"
2340
+ ],
2341
+ "pid_type": "flat"
2342
+ },
2343
+ "intermediate_4096_numtokens_40": {
2344
+ "block_sizes": [
2345
+ 32,
2346
+ 32
2347
+ ],
2348
+ "loop_orders": [
2349
+ [
2350
+ 1,
2351
+ 0
2352
+ ]
2353
+ ],
2354
+ "flatten_loops": [
2355
+ true
2356
+ ],
2357
+ "l2_groupings": [
2358
+ 1
2359
+ ],
2360
+ "range_unroll_factors": [
2361
+ 0
2362
+ ],
2363
+ "range_warp_specializes": [],
2364
+ "range_num_stages": [
2365
+ 0
2366
+ ],
2367
+ "range_multi_buffers": [
2368
+ null
2369
+ ],
2370
+ "range_flattens": [
2371
+ null
2372
+ ],
2373
+ "load_eviction_policies": [
2374
+ "",
2375
+ "",
2376
+ ""
2377
+ ],
2378
+ "num_warps": 8,
2379
+ "num_stages": 1,
2380
+ "indexing": [
2381
+ "tensor_descriptor",
2382
+ "pointer",
2383
+ "pointer",
2384
+ "pointer"
2385
+ ],
2386
+ "pid_type": "flat"
2387
+ },
2388
+ "intermediate_8192_numtokens_40": {
2389
+ "block_sizes": [
2390
+ 64,
2391
+ 128
2392
+ ],
2393
+ "loop_orders": [
2394
+ [
2395
+ 0,
2396
+ 1
2397
+ ]
2398
+ ],
2399
+ "flatten_loops": [
2400
+ true
2401
+ ],
2402
+ "l2_groupings": [
2403
+ 2
2404
+ ],
2405
+ "range_unroll_factors": [
2406
+ 0
2407
+ ],
2408
+ "range_warp_specializes": [],
2409
+ "range_num_stages": [
2410
+ 0
2411
+ ],
2412
+ "range_multi_buffers": [
2413
+ null
2414
+ ],
2415
+ "range_flattens": [
2416
+ null
2417
+ ],
2418
+ "load_eviction_policies": [
2419
+ "",
2420
+ "",
2421
+ ""
2422
+ ],
2423
+ "num_warps": 4,
2424
+ "num_stages": 1,
2425
+ "indexing": [
2426
+ "pointer",
2427
+ "pointer",
2428
+ "pointer",
2429
+ "pointer"
2430
+ ],
2431
+ "pid_type": "flat"
2432
+ },
2433
+ "intermediate_11008_numtokens_40": {
2434
+ "block_sizes": [
2435
+ 64,
2436
+ 32
2437
+ ],
2438
+ "loop_orders": [
2439
+ [
2440
+ 0,
2441
+ 1
2442
+ ]
2443
+ ],
2444
+ "flatten_loops": [
2445
+ true
2446
+ ],
2447
+ "l2_groupings": [
2448
+ 1
2449
+ ],
2450
+ "range_unroll_factors": [
2451
+ 0
2452
+ ],
2453
+ "range_warp_specializes": [],
2454
+ "range_num_stages": [
2455
+ 0
2456
+ ],
2457
+ "range_multi_buffers": [
2458
+ null
2459
+ ],
2460
+ "range_flattens": [
2461
+ null
2462
+ ],
2463
+ "load_eviction_policies": [
2464
+ "",
2465
+ "",
2466
+ ""
2467
+ ],
2468
+ "num_warps": 4,
2469
+ "num_stages": 1,
2470
+ "indexing": [
2471
+ "pointer",
2472
+ "pointer",
2473
+ "pointer",
2474
+ "pointer"
2475
+ ],
2476
+ "pid_type": "flat"
2477
+ },
2478
+ "intermediate_14336_numtokens_40": {
2479
+ "block_sizes": [
2480
+ 32,
2481
+ 32
2482
+ ],
2483
+ "loop_orders": [
2484
+ [
2485
+ 0,
2486
+ 1
2487
+ ]
2488
+ ],
2489
+ "flatten_loops": [
2490
+ true
2491
+ ],
2492
+ "l2_groupings": [
2493
+ 1
2494
+ ],
2495
+ "range_unroll_factors": [
2496
+ 0
2497
+ ],
2498
+ "range_warp_specializes": [],
2499
+ "range_num_stages": [
2500
+ 0
2501
+ ],
2502
+ "range_multi_buffers": [
2503
+ null
2504
+ ],
2505
+ "range_flattens": [
2506
+ null
2507
+ ],
2508
+ "load_eviction_policies": [
2509
+ "",
2510
+ "",
2511
+ ""
2512
+ ],
2513
+ "num_warps": 2,
2514
+ "num_stages": 1,
2515
+ "indexing": [
2516
+ "pointer",
2517
+ "pointer",
2518
+ "pointer",
2519
+ "pointer"
2520
+ ],
2521
+ "pid_type": "flat"
2522
+ },
2523
+ "intermediate_2048_numtokens_48": {
2524
+ "block_sizes": [
2525
+ 32,
2526
+ 8
2527
+ ],
2528
+ "loop_orders": [
2529
+ [
2530
+ 0,
2531
+ 1
2532
+ ]
2533
+ ],
2534
+ "flatten_loops": [
2535
+ true
2536
+ ],
2537
+ "l2_groupings": [
2538
+ 2
2539
+ ],
2540
+ "range_unroll_factors": [
2541
+ 0
2542
+ ],
2543
+ "range_warp_specializes": [],
2544
+ "range_num_stages": [
2545
+ 0
2546
+ ],
2547
+ "range_multi_buffers": [
2548
+ null
2549
+ ],
2550
+ "range_flattens": [
2551
+ null
2552
+ ],
2553
+ "load_eviction_policies": [
2554
+ "",
2555
+ "first",
2556
+ ""
2557
+ ],
2558
+ "num_warps": 16,
2559
+ "num_stages": 2,
2560
+ "indexing": [
2561
+ "pointer",
2562
+ "pointer",
2563
+ "tensor_descriptor",
2564
+ "pointer"
2565
+ ],
2566
+ "pid_type": "flat"
2567
+ },
2568
+ "intermediate_2880_numtokens_48": {
2569
+ "block_sizes": [
2570
+ 8,
2571
+ 8
2572
+ ],
2573
+ "loop_orders": [
2574
+ [
2575
+ 0,
2576
+ 1
2577
+ ]
2578
+ ],
2579
+ "flatten_loops": [
2580
+ true
2581
+ ],
2582
+ "l2_groupings": [
2583
+ 1
2584
+ ],
2585
+ "range_unroll_factors": [
2586
+ 0
2587
+ ],
2588
+ "range_warp_specializes": [],
2589
+ "range_num_stages": [
2590
+ 0
2591
+ ],
2592
+ "range_multi_buffers": [
2593
+ null
2594
+ ],
2595
+ "range_flattens": [
2596
+ null
2597
+ ],
2598
+ "load_eviction_policies": [
2599
+ "",
2600
+ "",
2601
+ ""
2602
+ ],
2603
+ "num_warps": 1,
2604
+ "num_stages": 1,
2605
+ "indexing": [
2606
+ "pointer",
2607
+ "pointer",
2608
+ "tensor_descriptor",
2609
+ "pointer"
2610
+ ],
2611
+ "pid_type": "flat"
2612
+ },
2613
+ "intermediate_4096_numtokens_48": {
2614
+ "block_sizes": [
2615
+ 32,
2616
+ 64
2617
+ ],
2618
+ "loop_orders": [
2619
+ [
2620
+ 0,
2621
+ 1
2622
+ ]
2623
+ ],
2624
+ "flatten_loops": [
2625
+ true
2626
+ ],
2627
+ "l2_groupings": [
2628
+ 1
2629
+ ],
2630
+ "range_unroll_factors": [
2631
+ 0
2632
+ ],
2633
+ "range_warp_specializes": [],
2634
+ "range_num_stages": [
2635
+ 0
2636
+ ],
2637
+ "range_multi_buffers": [
2638
+ null
2639
+ ],
2640
+ "range_flattens": [
2641
+ null
2642
+ ],
2643
+ "load_eviction_policies": [
2644
+ "",
2645
+ "",
2646
+ ""
2647
+ ],
2648
+ "num_warps": 1,
2649
+ "num_stages": 1,
2650
+ "indexing": [
2651
+ "pointer",
2652
+ "pointer",
2653
+ "pointer",
2654
+ "tensor_descriptor"
2655
+ ],
2656
+ "pid_type": "flat"
2657
+ },
2658
+ "intermediate_8192_numtokens_48": {
2659
+ "block_sizes": [
2660
+ 64,
2661
+ 32
2662
+ ],
2663
+ "loop_orders": [
2664
+ [
2665
+ 0,
2666
+ 1
2667
+ ]
2668
+ ],
2669
+ "flatten_loops": [
2670
+ true
2671
+ ],
2672
+ "l2_groupings": [
2673
+ 1
2674
+ ],
2675
+ "range_unroll_factors": [
2676
+ 0
2677
+ ],
2678
+ "range_warp_specializes": [],
2679
+ "range_num_stages": [
2680
+ 0
2681
+ ],
2682
+ "range_multi_buffers": [
2683
+ null
2684
+ ],
2685
+ "range_flattens": [
2686
+ null
2687
+ ],
2688
+ "load_eviction_policies": [
2689
+ "",
2690
+ "",
2691
+ ""
2692
+ ],
2693
+ "num_warps": 4,
2694
+ "num_stages": 1,
2695
+ "indexing": [
2696
+ "pointer",
2697
+ "pointer",
2698
+ "pointer",
2699
+ "pointer"
2700
+ ],
2701
+ "pid_type": "flat"
2702
+ },
2703
+ "intermediate_11008_numtokens_48": {
2704
+ "block_sizes": [
2705
+ 16,
2706
+ 256
2707
+ ],
2708
+ "loop_orders": [
2709
+ [
2710
+ 0,
2711
+ 1
2712
+ ]
2713
+ ],
2714
+ "flatten_loops": [
2715
+ true
2716
+ ],
2717
+ "l2_groupings": [
2718
+ 2
2719
+ ],
2720
+ "range_unroll_factors": [
2721
+ 0
2722
+ ],
2723
+ "range_warp_specializes": [],
2724
+ "range_num_stages": [
2725
+ 0
2726
+ ],
2727
+ "range_multi_buffers": [
2728
+ null
2729
+ ],
2730
+ "range_flattens": [
2731
+ null
2732
+ ],
2733
+ "load_eviction_policies": [
2734
+ "",
2735
+ "",
2736
+ ""
2737
+ ],
2738
+ "num_warps": 32,
2739
+ "num_stages": 1,
2740
+ "indexing": [
2741
+ "pointer",
2742
+ "pointer",
2743
+ "pointer",
2744
+ "tensor_descriptor"
2745
+ ],
2746
+ "pid_type": "flat"
2747
+ },
2748
+ "intermediate_14336_numtokens_48": {
2749
+ "block_sizes": [
2750
+ 64,
2751
+ 4
2752
+ ],
2753
+ "loop_orders": [
2754
+ [
2755
+ 0,
2756
+ 1
2757
+ ]
2758
+ ],
2759
+ "flatten_loops": [
2760
+ true
2761
+ ],
2762
+ "l2_groupings": [
2763
+ 1
2764
+ ],
2765
+ "range_unroll_factors": [
2766
+ 0
2767
+ ],
2768
+ "range_warp_specializes": [],
2769
+ "range_num_stages": [
2770
+ 0
2771
+ ],
2772
+ "range_multi_buffers": [
2773
+ null
2774
+ ],
2775
+ "range_flattens": [
2776
+ null
2777
+ ],
2778
+ "load_eviction_policies": [
2779
+ "",
2780
+ "",
2781
+ ""
2782
+ ],
2783
+ "num_warps": 2,
2784
+ "num_stages": 1,
2785
+ "indexing": [
2786
+ "tensor_descriptor",
2787
+ "pointer",
2788
+ "pointer",
2789
+ "pointer"
2790
+ ],
2791
+ "pid_type": "flat"
2792
+ },
2793
+ "intermediate_2048_numtokens_56": {
2794
+ "block_sizes": [
2795
+ 2,
2796
+ 8
2797
+ ],
2798
+ "loop_orders": [
2799
+ [
2800
+ 0,
2801
+ 1
2802
+ ]
2803
+ ],
2804
+ "flatten_loops": [
2805
+ true
2806
+ ],
2807
+ "l2_groupings": [
2808
+ 1
2809
+ ],
2810
+ "range_unroll_factors": [
2811
+ 0
2812
+ ],
2813
+ "range_warp_specializes": [],
2814
+ "range_num_stages": [
2815
+ 0
2816
+ ],
2817
+ "range_multi_buffers": [
2818
+ null
2819
+ ],
2820
+ "range_flattens": [
2821
+ null
2822
+ ],
2823
+ "load_eviction_policies": [
2824
+ "",
2825
+ "",
2826
+ ""
2827
+ ],
2828
+ "num_warps": 1,
2829
+ "num_stages": 1,
2830
+ "indexing": [
2831
+ "tensor_descriptor",
2832
+ "pointer",
2833
+ "tensor_descriptor",
2834
+ "pointer"
2835
+ ],
2836
+ "pid_type": "flat"
2837
+ },
2838
+ "intermediate_2880_numtokens_56": {
2839
+ "block_sizes": [
2840
+ 8,
2841
+ 32
2842
+ ],
2843
+ "loop_orders": [
2844
+ [
2845
+ 0,
2846
+ 1
2847
+ ]
2848
+ ],
2849
+ "flatten_loops": [
2850
+ true
2851
+ ],
2852
+ "l2_groupings": [
2853
+ 1
2854
+ ],
2855
+ "range_unroll_factors": [
2856
+ 0
2857
+ ],
2858
+ "range_warp_specializes": [],
2859
+ "range_num_stages": [
2860
+ 0
2861
+ ],
2862
+ "range_multi_buffers": [
2863
+ null
2864
+ ],
2865
+ "range_flattens": [
2866
+ null
2867
+ ],
2868
+ "load_eviction_policies": [
2869
+ "",
2870
+ "",
2871
+ ""
2872
+ ],
2873
+ "num_warps": 4,
2874
+ "num_stages": 1,
2875
+ "indexing": [
2876
+ "pointer",
2877
+ "pointer",
2878
+ "pointer",
2879
+ "pointer"
2880
+ ],
2881
+ "pid_type": "flat"
2882
+ },
2883
+ "intermediate_4096_numtokens_56": {
2884
+ "block_sizes": [
2885
+ 32,
2886
+ 4
2887
+ ],
2888
+ "loop_orders": [
2889
+ [
2890
+ 0,
2891
+ 1
2892
+ ]
2893
+ ],
2894
+ "flatten_loops": [
2895
+ true
2896
+ ],
2897
+ "l2_groupings": [
2898
+ 1
2899
+ ],
2900
+ "range_unroll_factors": [
2901
+ 0
2902
+ ],
2903
+ "range_warp_specializes": [],
2904
+ "range_num_stages": [
2905
+ 0
2906
+ ],
2907
+ "range_multi_buffers": [
2908
+ null
2909
+ ],
2910
+ "range_flattens": [
2911
+ null
2912
+ ],
2913
+ "load_eviction_policies": [
2914
+ "",
2915
+ "first",
2916
+ ""
2917
+ ],
2918
+ "num_warps": 4,
2919
+ "num_stages": 1,
2920
+ "indexing": [
2921
+ "pointer",
2922
+ "pointer",
2923
+ "tensor_descriptor",
2924
+ "pointer"
2925
+ ],
2926
+ "pid_type": "flat"
2927
+ },
2928
+ "intermediate_8192_numtokens_56": {
2929
+ "block_sizes": [
2930
+ 32,
2931
+ 128
2932
+ ],
2933
+ "loop_orders": [
2934
+ [
2935
+ 0,
2936
+ 1
2937
+ ]
2938
+ ],
2939
+ "flatten_loops": [
2940
+ true
2941
+ ],
2942
+ "l2_groupings": [
2943
+ 1
2944
+ ],
2945
+ "range_unroll_factors": [
2946
+ 0
2947
+ ],
2948
+ "range_warp_specializes": [],
2949
+ "range_num_stages": [
2950
+ 0
2951
+ ],
2952
+ "range_multi_buffers": [
2953
+ null
2954
+ ],
2955
+ "range_flattens": [
2956
+ null
2957
+ ],
2958
+ "load_eviction_policies": [
2959
+ "",
2960
+ "",
2961
+ ""
2962
+ ],
2963
+ "num_warps": 1,
2964
+ "num_stages": 1,
2965
+ "indexing": [
2966
+ "pointer",
2967
+ "pointer",
2968
+ "pointer",
2969
+ "pointer"
2970
+ ],
2971
+ "pid_type": "flat"
2972
+ },
2973
+ "intermediate_11008_numtokens_56": {
2974
+ "block_sizes": [
2975
+ 32,
2976
+ 8
2977
+ ],
2978
+ "loop_orders": [
2979
+ [
2980
+ 0,
2981
+ 1
2982
+ ]
2983
+ ],
2984
+ "flatten_loops": [
2985
+ true
2986
+ ],
2987
+ "l2_groupings": [
2988
+ 1
2989
+ ],
2990
+ "range_unroll_factors": [
2991
+ 0
2992
+ ],
2993
+ "range_warp_specializes": [],
2994
+ "range_num_stages": [
2995
+ 0
2996
+ ],
2997
+ "range_multi_buffers": [
2998
+ null
2999
+ ],
3000
+ "range_flattens": [
3001
+ null
3002
+ ],
3003
+ "load_eviction_policies": [
3004
+ "",
3005
+ "first",
3006
+ "last"
3007
+ ],
3008
+ "num_warps": 1,
3009
+ "num_stages": 1,
3010
+ "indexing": [
3011
+ "pointer",
3012
+ "pointer",
3013
+ "pointer",
3014
+ "pointer"
3015
+ ],
3016
+ "pid_type": "flat"
3017
+ },
3018
+ "intermediate_14336_numtokens_56": {
3019
+ "block_sizes": [
3020
+ 64,
3021
+ 64
3022
+ ],
3023
+ "loop_orders": [
3024
+ [
3025
+ 0,
3026
+ 1
3027
+ ]
3028
+ ],
3029
+ "flatten_loops": [
3030
+ true
3031
+ ],
3032
+ "l2_groupings": [
3033
+ 2
3034
+ ],
3035
+ "range_unroll_factors": [
3036
+ 0
3037
+ ],
3038
+ "range_warp_specializes": [],
3039
+ "range_num_stages": [
3040
+ 0
3041
+ ],
3042
+ "range_multi_buffers": [
3043
+ null
3044
+ ],
3045
+ "range_flattens": [
3046
+ null
3047
+ ],
3048
+ "load_eviction_policies": [
3049
+ "",
3050
+ "",
3051
+ ""
3052
+ ],
3053
+ "num_warps": 16,
3054
+ "num_stages": 1,
3055
+ "indexing": [
3056
+ "pointer",
3057
+ "pointer",
3058
+ "pointer",
3059
+ "tensor_descriptor"
3060
+ ],
3061
+ "pid_type": "flat"
3062
+ },
3063
+ "intermediate_2048_numtokens_64": {
3064
+ "block_sizes": [
3065
+ 16,
3066
+ 128
3067
+ ],
3068
+ "loop_orders": [
3069
+ [
3070
+ 1,
3071
+ 0
3072
+ ]
3073
+ ],
3074
+ "flatten_loops": [
3075
+ true
3076
+ ],
3077
+ "l2_groupings": [
3078
+ 1
3079
+ ],
3080
+ "range_unroll_factors": [
3081
+ 0
3082
+ ],
3083
+ "range_warp_specializes": [],
3084
+ "range_num_stages": [
3085
+ 0
3086
+ ],
3087
+ "range_multi_buffers": [
3088
+ null
3089
+ ],
3090
+ "range_flattens": [
3091
+ null
3092
+ ],
3093
+ "load_eviction_policies": [
3094
+ "",
3095
+ "",
3096
+ ""
3097
+ ],
3098
+ "num_warps": 8,
3099
+ "num_stages": 1,
3100
+ "indexing": [
3101
+ "tensor_descriptor",
3102
+ "pointer",
3103
+ "pointer",
3104
+ "pointer"
3105
+ ],
3106
+ "pid_type": "flat"
3107
+ },
3108
+ "intermediate_2880_numtokens_64": {
3109
+ "block_sizes": [
3110
+ 4,
3111
+ 64
3112
+ ],
3113
+ "loop_orders": [
3114
+ [
3115
+ 1,
3116
+ 0
3117
+ ]
3118
+ ],
3119
+ "flatten_loops": [
3120
+ true
3121
+ ],
3122
+ "l2_groupings": [
3123
+ 1
3124
+ ],
3125
+ "range_unroll_factors": [
3126
+ 0
3127
+ ],
3128
+ "range_warp_specializes": [],
3129
+ "range_num_stages": [
3130
+ 0
3131
+ ],
3132
+ "range_multi_buffers": [
3133
+ null
3134
+ ],
3135
+ "range_flattens": [
3136
+ null
3137
+ ],
3138
+ "load_eviction_policies": [
3139
+ "",
3140
+ "",
3141
+ ""
3142
+ ],
3143
+ "num_warps": 4,
3144
+ "num_stages": 1,
3145
+ "indexing": [
3146
+ "pointer",
3147
+ "pointer",
3148
+ "pointer",
3149
+ "pointer"
3150
+ ],
3151
+ "pid_type": "flat"
3152
+ },
3153
+ "intermediate_4096_numtokens_64": {
3154
+ "block_sizes": [
3155
+ 2,
3156
+ 16
3157
+ ],
3158
+ "loop_orders": [
3159
+ [
3160
+ 0,
3161
+ 1
3162
+ ]
3163
+ ],
3164
+ "flatten_loops": [
3165
+ true
3166
+ ],
3167
+ "l2_groupings": [
3168
+ 1
3169
+ ],
3170
+ "range_unroll_factors": [
3171
+ 0
3172
+ ],
3173
+ "range_warp_specializes": [],
3174
+ "range_num_stages": [
3175
+ 0
3176
+ ],
3177
+ "range_multi_buffers": [
3178
+ null
3179
+ ],
3180
+ "range_flattens": [
3181
+ null
3182
+ ],
3183
+ "load_eviction_policies": [
3184
+ "first",
3185
+ "",
3186
+ ""
3187
+ ],
3188
+ "num_warps": 1,
3189
+ "num_stages": 1,
3190
+ "indexing": [
3191
+ "pointer",
3192
+ "pointer",
3193
+ "pointer",
3194
+ "pointer"
3195
+ ],
3196
+ "pid_type": "flat"
3197
+ },
3198
+ "intermediate_8192_numtokens_64": {
3199
+ "block_sizes": [
3200
+ 8,
3201
+ 32
3202
+ ],
3203
+ "loop_orders": [
3204
+ [
3205
+ 0,
3206
+ 1
3207
+ ]
3208
+ ],
3209
+ "flatten_loops": [
3210
+ true
3211
+ ],
3212
+ "l2_groupings": [
3213
+ 1
3214
+ ],
3215
+ "range_unroll_factors": [
3216
+ 0
3217
+ ],
3218
+ "range_warp_specializes": [],
3219
+ "range_num_stages": [
3220
+ 0
3221
+ ],
3222
+ "range_multi_buffers": [
3223
+ null
3224
+ ],
3225
+ "range_flattens": [
3226
+ null
3227
+ ],
3228
+ "load_eviction_policies": [
3229
+ "",
3230
+ "",
3231
+ ""
3232
+ ],
3233
+ "num_warps": 16,
3234
+ "num_stages": 2,
3235
+ "indexing": [
3236
+ "pointer",
3237
+ "pointer",
3238
+ "pointer",
3239
+ "pointer"
3240
+ ],
3241
+ "pid_type": "flat"
3242
+ },
3243
+ "intermediate_11008_numtokens_64": {
3244
+ "block_sizes": [
3245
+ 32,
3246
+ 32
3247
+ ],
3248
+ "loop_orders": [
3249
+ [
3250
+ 0,
3251
+ 1
3252
+ ]
3253
+ ],
3254
+ "flatten_loops": [
3255
+ false
3256
+ ],
3257
+ "l2_groupings": [
3258
+ 1
3259
+ ],
3260
+ "range_unroll_factors": [
3261
+ 0
3262
+ ],
3263
+ "range_warp_specializes": [],
3264
+ "range_num_stages": [
3265
+ 0
3266
+ ],
3267
+ "range_multi_buffers": [
3268
+ null
3269
+ ],
3270
+ "range_flattens": [
3271
+ null
3272
+ ],
3273
+ "load_eviction_policies": [
3274
+ "",
3275
+ "",
3276
+ ""
3277
+ ],
3278
+ "num_warps": 2,
3279
+ "num_stages": 1,
3280
+ "indexing": [
3281
+ "pointer",
3282
+ "pointer",
3283
+ "tensor_descriptor",
3284
+ "pointer"
3285
+ ],
3286
+ "pid_type": "flat"
3287
+ },
3288
+ "intermediate_14336_numtokens_64": {
3289
+ "block_sizes": [
3290
+ 32,
3291
+ 8
3292
+ ],
3293
+ "loop_orders": [
3294
+ [
3295
+ 0,
3296
+ 1
3297
+ ]
3298
+ ],
3299
+ "flatten_loops": [
3300
+ true
3301
+ ],
3302
+ "l2_groupings": [
3303
+ 1
3304
+ ],
3305
+ "range_unroll_factors": [
3306
+ 0
3307
+ ],
3308
+ "range_warp_specializes": [],
3309
+ "range_num_stages": [
3310
+ 0
3311
+ ],
3312
+ "range_multi_buffers": [
3313
+ null
3314
+ ],
3315
+ "range_flattens": [
3316
+ null
3317
+ ],
3318
+ "load_eviction_policies": [
3319
+ "",
3320
+ "",
3321
+ ""
3322
+ ],
3323
+ "num_warps": 16,
3324
+ "num_stages": 1,
3325
+ "indexing": [
3326
+ "pointer",
3327
+ "pointer",
3328
+ "pointer",
3329
+ "pointer"
3330
+ ],
3331
+ "pid_type": "flat"
3332
+ },
3333
+ "intermediate_2048_numtokens_72": {
3334
+ "block_sizes": [
3335
+ 4,
3336
+ 16
3337
+ ],
3338
+ "loop_orders": [
3339
+ [
3340
+ 0,
3341
+ 1
3342
+ ]
3343
+ ],
3344
+ "flatten_loops": [
3345
+ true
3346
+ ],
3347
+ "l2_groupings": [
3348
+ 1
3349
+ ],
3350
+ "range_unroll_factors": [
3351
+ 0
3352
+ ],
3353
+ "range_warp_specializes": [],
3354
+ "range_num_stages": [
3355
+ 0
3356
+ ],
3357
+ "range_multi_buffers": [
3358
+ null
3359
+ ],
3360
+ "range_flattens": [
3361
+ null
3362
+ ],
3363
+ "load_eviction_policies": [
3364
+ "",
3365
+ "",
3366
+ "last"
3367
+ ],
3368
+ "num_warps": 1,
3369
+ "num_stages": 1,
3370
+ "indexing": [
3371
+ "pointer",
3372
+ "pointer",
3373
+ "pointer",
3374
+ "pointer"
3375
+ ],
3376
+ "pid_type": "flat"
3377
+ },
3378
+ "intermediate_2880_numtokens_72": {
3379
+ "block_sizes": [
3380
+ 64,
3381
+ 32
3382
+ ],
3383
+ "loop_orders": [
3384
+ [
3385
+ 0,
3386
+ 1
3387
+ ]
3388
+ ],
3389
+ "flatten_loops": [
3390
+ true
3391
+ ],
3392
+ "l2_groupings": [
3393
+ 1
3394
+ ],
3395
+ "range_unroll_factors": [
3396
+ 0
3397
+ ],
3398
+ "range_warp_specializes": [],
3399
+ "range_num_stages": [
3400
+ 0
3401
+ ],
3402
+ "range_multi_buffers": [
3403
+ null
3404
+ ],
3405
+ "range_flattens": [
3406
+ null
3407
+ ],
3408
+ "load_eviction_policies": [
3409
+ "",
3410
+ "",
3411
+ "first"
3412
+ ],
3413
+ "num_warps": 16,
3414
+ "num_stages": 1,
3415
+ "indexing": [
3416
+ "pointer",
3417
+ "pointer",
3418
+ "pointer",
3419
+ "pointer"
3420
+ ],
3421
+ "pid_type": "flat"
3422
+ },
3423
+ "intermediate_4096_numtokens_72": {
3424
+ "block_sizes": [
3425
+ 64,
3426
+ 16
3427
+ ],
3428
+ "loop_orders": [
3429
+ [
3430
+ 1,
3431
+ 0
3432
+ ]
3433
+ ],
3434
+ "flatten_loops": [
3435
+ true
3436
+ ],
3437
+ "l2_groupings": [
3438
+ 1
3439
+ ],
3440
+ "range_unroll_factors": [
3441
+ 0
3442
+ ],
3443
+ "range_warp_specializes": [],
3444
+ "range_num_stages": [
3445
+ 0
3446
+ ],
3447
+ "range_multi_buffers": [
3448
+ null
3449
+ ],
3450
+ "range_flattens": [
3451
+ null
3452
+ ],
3453
+ "load_eviction_policies": [
3454
+ "last",
3455
+ "",
3456
+ ""
3457
+ ],
3458
+ "num_warps": 32,
3459
+ "num_stages": 1,
3460
+ "indexing": [
3461
+ "pointer",
3462
+ "pointer",
3463
+ "pointer",
3464
+ "pointer"
3465
+ ],
3466
+ "pid_type": "flat"
3467
+ },
3468
+ "intermediate_8192_numtokens_72": {
3469
+ "block_sizes": [
3470
+ 32,
3471
+ 8
3472
+ ],
3473
+ "loop_orders": [
3474
+ [
3475
+ 0,
3476
+ 1
3477
+ ]
3478
+ ],
3479
+ "flatten_loops": [
3480
+ true
3481
+ ],
3482
+ "l2_groupings": [
3483
+ 1
3484
+ ],
3485
+ "range_unroll_factors": [
3486
+ 0
3487
+ ],
3488
+ "range_warp_specializes": [],
3489
+ "range_num_stages": [
3490
+ 0
3491
+ ],
3492
+ "range_multi_buffers": [
3493
+ null
3494
+ ],
3495
+ "range_flattens": [
3496
+ null
3497
+ ],
3498
+ "load_eviction_policies": [
3499
+ "",
3500
+ "",
3501
+ ""
3502
+ ],
3503
+ "num_warps": 32,
3504
+ "num_stages": 1,
3505
+ "indexing": [
3506
+ "pointer",
3507
+ "pointer",
3508
+ "pointer",
3509
+ "pointer"
3510
+ ],
3511
+ "pid_type": "flat"
3512
+ },
3513
+ "intermediate_11008_numtokens_72": {
3514
+ "block_sizes": [
3515
+ 32,
3516
+ 32
3517
+ ],
3518
+ "loop_orders": [
3519
+ [
3520
+ 0,
3521
+ 1
3522
+ ]
3523
+ ],
3524
+ "flatten_loops": [
3525
+ true
3526
+ ],
3527
+ "l2_groupings": [
3528
+ 1
3529
+ ],
3530
+ "range_unroll_factors": [
3531
+ 0
3532
+ ],
3533
+ "range_warp_specializes": [],
3534
+ "range_num_stages": [
3535
+ 0
3536
+ ],
3537
+ "range_multi_buffers": [
3538
+ null
3539
+ ],
3540
+ "range_flattens": [
3541
+ null
3542
+ ],
3543
+ "load_eviction_policies": [
3544
+ "",
3545
+ "",
3546
+ ""
3547
+ ],
3548
+ "num_warps": 1,
3549
+ "num_stages": 2,
3550
+ "indexing": [
3551
+ "pointer",
3552
+ "pointer",
3553
+ "tensor_descriptor",
3554
+ "pointer"
3555
+ ],
3556
+ "pid_type": "flat"
3557
+ },
3558
+ "intermediate_14336_numtokens_72": {
3559
+ "block_sizes": [
3560
+ 128,
3561
+ 128
3562
+ ],
3563
+ "loop_orders": [
3564
+ [
3565
+ 0,
3566
+ 1
3567
+ ]
3568
+ ],
3569
+ "flatten_loops": [
3570
+ true
3571
+ ],
3572
+ "l2_groupings": [
3573
+ 1
3574
+ ],
3575
+ "range_unroll_factors": [
3576
+ 0
3577
+ ],
3578
+ "range_warp_specializes": [],
3579
+ "range_num_stages": [
3580
+ 0
3581
+ ],
3582
+ "range_multi_buffers": [
3583
+ null
3584
+ ],
3585
+ "range_flattens": [
3586
+ null
3587
+ ],
3588
+ "load_eviction_policies": [
3589
+ "",
3590
+ "",
3591
+ ""
3592
+ ],
3593
+ "num_warps": 16,
3594
+ "num_stages": 1,
3595
+ "indexing": [
3596
+ "pointer",
3597
+ "pointer",
3598
+ "pointer",
3599
+ "pointer"
3600
+ ],
3601
+ "pid_type": "flat"
3602
+ },
3603
+ "intermediate_2048_numtokens_80": {
3604
+ "block_sizes": [
3605
+ 32,
3606
+ 64
3607
+ ],
3608
+ "loop_orders": [
3609
+ [
3610
+ 0,
3611
+ 1
3612
+ ]
3613
+ ],
3614
+ "flatten_loops": [
3615
+ true
3616
+ ],
3617
+ "l2_groupings": [
3618
+ 1
3619
+ ],
3620
+ "range_unroll_factors": [
3621
+ 0
3622
+ ],
3623
+ "range_warp_specializes": [],
3624
+ "range_num_stages": [
3625
+ 0
3626
+ ],
3627
+ "range_multi_buffers": [
3628
+ null
3629
+ ],
3630
+ "range_flattens": [
3631
+ null
3632
+ ],
3633
+ "load_eviction_policies": [
3634
+ "",
3635
+ "",
3636
+ ""
3637
+ ],
3638
+ "num_warps": 4,
3639
+ "num_stages": 1,
3640
+ "indexing": [
3641
+ "pointer",
3642
+ "pointer",
3643
+ "pointer",
3644
+ "pointer"
3645
+ ],
3646
+ "pid_type": "flat"
3647
+ },
3648
+ "intermediate_2880_numtokens_80": {
3649
+ "block_sizes": [
3650
+ 32,
3651
+ 128
3652
+ ],
3653
+ "loop_orders": [
3654
+ [
3655
+ 0,
3656
+ 1
3657
+ ]
3658
+ ],
3659
+ "flatten_loops": [
3660
+ true
3661
+ ],
3662
+ "l2_groupings": [
3663
+ 1
3664
+ ],
3665
+ "range_unroll_factors": [
3666
+ 0
3667
+ ],
3668
+ "range_warp_specializes": [],
3669
+ "range_num_stages": [
3670
+ 0
3671
+ ],
3672
+ "range_multi_buffers": [
3673
+ null
3674
+ ],
3675
+ "range_flattens": [
3676
+ null
3677
+ ],
3678
+ "load_eviction_policies": [
3679
+ "",
3680
+ "",
3681
+ ""
3682
+ ],
3683
+ "num_warps": 16,
3684
+ "num_stages": 1,
3685
+ "indexing": [
3686
+ "pointer",
3687
+ "pointer",
3688
+ "pointer",
3689
+ "pointer"
3690
+ ],
3691
+ "pid_type": "flat"
3692
+ },
3693
+ "intermediate_4096_numtokens_80": {
3694
+ "block_sizes": [
3695
+ 32,
3696
+ 32
3697
+ ],
3698
+ "loop_orders": [
3699
+ [
3700
+ 0,
3701
+ 1
3702
+ ]
3703
+ ],
3704
+ "flatten_loops": [
3705
+ true
3706
+ ],
3707
+ "l2_groupings": [
3708
+ 1
3709
+ ],
3710
+ "range_unroll_factors": [
3711
+ 0
3712
+ ],
3713
+ "range_warp_specializes": [],
3714
+ "range_num_stages": [
3715
+ 0
3716
+ ],
3717
+ "range_multi_buffers": [
3718
+ null
3719
+ ],
3720
+ "range_flattens": [
3721
+ null
3722
+ ],
3723
+ "load_eviction_policies": [
3724
+ "",
3725
+ "",
3726
+ ""
3727
+ ],
3728
+ "num_warps": 2,
3729
+ "num_stages": 1,
3730
+ "indexing": [
3731
+ "pointer",
3732
+ "pointer",
3733
+ "pointer",
3734
+ "pointer"
3735
+ ],
3736
+ "pid_type": "flat"
3737
+ },
3738
+ "intermediate_8192_numtokens_80": {
3739
+ "block_sizes": [
3740
+ 32,
3741
+ 8
3742
+ ],
3743
+ "loop_orders": [
3744
+ [
3745
+ 0,
3746
+ 1
3747
+ ]
3748
+ ],
3749
+ "flatten_loops": [
3750
+ true
3751
+ ],
3752
+ "l2_groupings": [
3753
+ 4
3754
+ ],
3755
+ "range_unroll_factors": [
3756
+ 0
3757
+ ],
3758
+ "range_warp_specializes": [],
3759
+ "range_num_stages": [
3760
+ 0
3761
+ ],
3762
+ "range_multi_buffers": [
3763
+ null
3764
+ ],
3765
+ "range_flattens": [
3766
+ null
3767
+ ],
3768
+ "load_eviction_policies": [
3769
+ "",
3770
+ "",
3771
+ ""
3772
+ ],
3773
+ "num_warps": 1,
3774
+ "num_stages": 1,
3775
+ "indexing": [
3776
+ "tensor_descriptor",
3777
+ "pointer",
3778
+ "pointer",
3779
+ "pointer"
3780
+ ],
3781
+ "pid_type": "flat"
3782
+ },
3783
+ "intermediate_11008_numtokens_80": {
3784
+ "block_sizes": [
3785
+ 64,
3786
+ 8
3787
+ ],
3788
+ "loop_orders": [
3789
+ [
3790
+ 0,
3791
+ 1
3792
+ ]
3793
+ ],
3794
+ "flatten_loops": [
3795
+ true
3796
+ ],
3797
+ "l2_groupings": [
3798
+ 1
3799
+ ],
3800
+ "range_unroll_factors": [
3801
+ 0
3802
+ ],
3803
+ "range_warp_specializes": [],
3804
+ "range_num_stages": [
3805
+ 0
3806
+ ],
3807
+ "range_multi_buffers": [
3808
+ null
3809
+ ],
3810
+ "range_flattens": [
3811
+ null
3812
+ ],
3813
+ "load_eviction_policies": [
3814
+ "",
3815
+ "",
3816
+ ""
3817
+ ],
3818
+ "num_warps": 8,
3819
+ "num_stages": 1,
3820
+ "indexing": [
3821
+ "pointer",
3822
+ "pointer",
3823
+ "pointer",
3824
+ "pointer"
3825
+ ],
3826
+ "pid_type": "flat"
3827
+ },
3828
+ "intermediate_14336_numtokens_80": {
3829
+ "block_sizes": [
3830
+ 32,
3831
+ 512
3832
+ ],
3833
+ "loop_orders": [
3834
+ [
3835
+ 0,
3836
+ 1
3837
+ ]
3838
+ ],
3839
+ "flatten_loops": [
3840
+ true
3841
+ ],
3842
+ "l2_groupings": [
3843
+ 2
3844
+ ],
3845
+ "range_unroll_factors": [
3846
+ 0
3847
+ ],
3848
+ "range_warp_specializes": [],
3849
+ "range_num_stages": [
3850
+ 0
3851
+ ],
3852
+ "range_multi_buffers": [
3853
+ null
3854
+ ],
3855
+ "range_flattens": [
3856
+ null
3857
+ ],
3858
+ "load_eviction_policies": [
3859
+ "",
3860
+ "first",
3861
+ ""
3862
+ ],
3863
+ "num_warps": 16,
3864
+ "num_stages": 1,
3865
+ "indexing": [
3866
+ "pointer",
3867
+ "tensor_descriptor",
3868
+ "pointer",
3869
+ "pointer"
3870
+ ],
3871
+ "pid_type": "flat"
3872
+ },
3873
+ "intermediate_2048_numtokens_88": {
3874
+ "block_sizes": [
3875
+ 32,
3876
+ 16
3877
+ ],
3878
+ "loop_orders": [
3879
+ [
3880
+ 0,
3881
+ 1
3882
+ ]
3883
+ ],
3884
+ "flatten_loops": [
3885
+ true
3886
+ ],
3887
+ "l2_groupings": [
3888
+ 1
3889
+ ],
3890
+ "range_unroll_factors": [
3891
+ 0
3892
+ ],
3893
+ "range_warp_specializes": [],
3894
+ "range_num_stages": [
3895
+ 0
3896
+ ],
3897
+ "range_multi_buffers": [
3898
+ null
3899
+ ],
3900
+ "range_flattens": [
3901
+ null
3902
+ ],
3903
+ "load_eviction_policies": [
3904
+ "",
3905
+ "",
3906
+ ""
3907
+ ],
3908
+ "num_warps": 4,
3909
+ "num_stages": 2,
3910
+ "indexing": [
3911
+ "pointer",
3912
+ "pointer",
3913
+ "pointer",
3914
+ "tensor_descriptor"
3915
+ ],
3916
+ "pid_type": "flat"
3917
+ },
3918
+ "intermediate_2880_numtokens_88": {
3919
+ "block_sizes": [
3920
+ 16,
3921
+ 128
3922
+ ],
3923
+ "loop_orders": [
3924
+ [
3925
+ 0,
3926
+ 1
3927
+ ]
3928
+ ],
3929
+ "flatten_loops": [
3930
+ true
3931
+ ],
3932
+ "l2_groupings": [
3933
+ 1
3934
+ ],
3935
+ "range_unroll_factors": [
3936
+ 0
3937
+ ],
3938
+ "range_warp_specializes": [],
3939
+ "range_num_stages": [
3940
+ 0
3941
+ ],
3942
+ "range_multi_buffers": [
3943
+ null
3944
+ ],
3945
+ "range_flattens": [
3946
+ null
3947
+ ],
3948
+ "load_eviction_policies": [
3949
+ "",
3950
+ "",
3951
+ ""
3952
+ ],
3953
+ "num_warps": 4,
3954
+ "num_stages": 1,
3955
+ "indexing": [
3956
+ "tensor_descriptor",
3957
+ "pointer",
3958
+ "pointer",
3959
+ "tensor_descriptor"
3960
+ ],
3961
+ "pid_type": "flat"
3962
+ },
3963
+ "intermediate_4096_numtokens_88": {
3964
+ "block_sizes": [
3965
+ 64,
3966
+ 32
3967
+ ],
3968
+ "loop_orders": [
3969
+ [
3970
+ 1,
3971
+ 0
3972
+ ]
3973
+ ],
3974
+ "flatten_loops": [
3975
+ true
3976
+ ],
3977
+ "l2_groupings": [
3978
+ 1
3979
+ ],
3980
+ "range_unroll_factors": [
3981
+ 0
3982
+ ],
3983
+ "range_warp_specializes": [],
3984
+ "range_num_stages": [
3985
+ 0
3986
+ ],
3987
+ "range_multi_buffers": [
3988
+ null
3989
+ ],
3990
+ "range_flattens": [
3991
+ null
3992
+ ],
3993
+ "load_eviction_policies": [
3994
+ "",
3995
+ "",
3996
+ ""
3997
+ ],
3998
+ "num_warps": 32,
3999
+ "num_stages": 1,
4000
+ "indexing": [
4001
+ "tensor_descriptor",
4002
+ "tensor_descriptor",
4003
+ "tensor_descriptor",
4004
+ "pointer"
4005
+ ],
4006
+ "pid_type": "flat"
4007
+ },
4008
+ "intermediate_8192_numtokens_88": {
4009
+ "block_sizes": [
4010
+ 128,
4011
+ 64
4012
+ ],
4013
+ "loop_orders": [
4014
+ [
4015
+ 0,
4016
+ 1
4017
+ ]
4018
+ ],
4019
+ "flatten_loops": [
4020
+ true
4021
+ ],
4022
+ "l2_groupings": [
4023
+ 1
4024
+ ],
4025
+ "range_unroll_factors": [
4026
+ 0
4027
+ ],
4028
+ "range_warp_specializes": [],
4029
+ "range_num_stages": [
4030
+ 0
4031
+ ],
4032
+ "range_multi_buffers": [
4033
+ null
4034
+ ],
4035
+ "range_flattens": [
4036
+ null
4037
+ ],
4038
+ "load_eviction_policies": [
4039
+ "first",
4040
+ "first",
4041
+ "last"
4042
+ ],
4043
+ "num_warps": 2,
4044
+ "num_stages": 1,
4045
+ "indexing": [
4046
+ "pointer",
4047
+ "pointer",
4048
+ "pointer",
4049
+ "pointer"
4050
+ ],
4051
+ "pid_type": "flat"
4052
+ },
4053
+ "intermediate_11008_numtokens_88": {
4054
+ "block_sizes": [
4055
+ 32,
4056
+ 128
4057
+ ],
4058
+ "loop_orders": [
4059
+ [
4060
+ 0,
4061
+ 1
4062
+ ]
4063
+ ],
4064
+ "flatten_loops": [
4065
+ true
4066
+ ],
4067
+ "l2_groupings": [
4068
+ 4
4069
+ ],
4070
+ "range_unroll_factors": [
4071
+ 0
4072
+ ],
4073
+ "range_warp_specializes": [],
4074
+ "range_num_stages": [
4075
+ 0
4076
+ ],
4077
+ "range_multi_buffers": [
4078
+ null
4079
+ ],
4080
+ "range_flattens": [
4081
+ null
4082
+ ],
4083
+ "load_eviction_policies": [
4084
+ "",
4085
+ "",
4086
+ ""
4087
+ ],
4088
+ "num_warps": 32,
4089
+ "num_stages": 1,
4090
+ "indexing": [
4091
+ "pointer",
4092
+ "tensor_descriptor",
4093
+ "pointer",
4094
+ "pointer"
4095
+ ],
4096
+ "pid_type": "flat"
4097
+ },
4098
+ "intermediate_14336_numtokens_88": {
4099
+ "block_sizes": [
4100
+ 16,
4101
+ 128
4102
+ ],
4103
+ "loop_orders": [
4104
+ [
4105
+ 0,
4106
+ 1
4107
+ ]
4108
+ ],
4109
+ "flatten_loops": [
4110
+ true
4111
+ ],
4112
+ "l2_groupings": [
4113
+ 1
4114
+ ],
4115
+ "range_unroll_factors": [
4116
+ 0
4117
+ ],
4118
+ "range_warp_specializes": [],
4119
+ "range_num_stages": [
4120
+ 0
4121
+ ],
4122
+ "range_multi_buffers": [
4123
+ null
4124
+ ],
4125
+ "range_flattens": [
4126
+ null
4127
+ ],
4128
+ "load_eviction_policies": [
4129
+ "first",
4130
+ "last",
4131
+ ""
4132
+ ],
4133
+ "num_warps": 8,
4134
+ "num_stages": 1,
4135
+ "indexing": [
4136
+ "pointer",
4137
+ "pointer",
4138
+ "pointer",
4139
+ "pointer"
4140
+ ],
4141
+ "pid_type": "flat"
4142
+ },
4143
+ "intermediate_2048_numtokens_96": {
4144
+ "block_sizes": [
4145
+ 128,
4146
+ 4
4147
+ ],
4148
+ "loop_orders": [
4149
+ [
4150
+ 0,
4151
+ 1
4152
+ ]
4153
+ ],
4154
+ "flatten_loops": [
4155
+ true
4156
+ ],
4157
+ "l2_groupings": [
4158
+ 2
4159
+ ],
4160
+ "range_unroll_factors": [
4161
+ 0
4162
+ ],
4163
+ "range_warp_specializes": [],
4164
+ "range_num_stages": [
4165
+ 0
4166
+ ],
4167
+ "range_multi_buffers": [
4168
+ null
4169
+ ],
4170
+ "range_flattens": [
4171
+ null
4172
+ ],
4173
+ "load_eviction_policies": [
4174
+ "",
4175
+ "last",
4176
+ ""
4177
+ ],
4178
+ "num_warps": 4,
4179
+ "num_stages": 1,
4180
+ "indexing": [
4181
+ "pointer",
4182
+ "pointer",
4183
+ "pointer",
4184
+ "pointer"
4185
+ ],
4186
+ "pid_type": "flat"
4187
+ },
4188
+ "intermediate_2880_numtokens_96": {
4189
+ "block_sizes": [
4190
+ 32,
4191
+ 128
4192
+ ],
4193
+ "loop_orders": [
4194
+ [
4195
+ 0,
4196
+ 1
4197
+ ]
4198
+ ],
4199
+ "flatten_loops": [
4200
+ true
4201
+ ],
4202
+ "l2_groupings": [
4203
+ 1
4204
+ ],
4205
+ "range_unroll_factors": [
4206
+ 0
4207
+ ],
4208
+ "range_warp_specializes": [],
4209
+ "range_num_stages": [
4210
+ 0
4211
+ ],
4212
+ "range_multi_buffers": [
4213
+ null
4214
+ ],
4215
+ "range_flattens": [
4216
+ null
4217
+ ],
4218
+ "load_eviction_policies": [
4219
+ "",
4220
+ "",
4221
+ ""
4222
+ ],
4223
+ "num_warps": 16,
4224
+ "num_stages": 1,
4225
+ "indexing": [
4226
+ "pointer",
4227
+ "tensor_descriptor",
4228
+ "pointer",
4229
+ "pointer"
4230
+ ],
4231
+ "pid_type": "flat"
4232
+ },
4233
+ "intermediate_4096_numtokens_96": {
4234
+ "block_sizes": [
4235
+ 16,
4236
+ 256
4237
+ ],
4238
+ "loop_orders": [
4239
+ [
4240
+ 0,
4241
+ 1
4242
+ ]
4243
+ ],
4244
+ "flatten_loops": [
4245
+ true
4246
+ ],
4247
+ "l2_groupings": [
4248
+ 2
4249
+ ],
4250
+ "range_unroll_factors": [
4251
+ 0
4252
+ ],
4253
+ "range_warp_specializes": [],
4254
+ "range_num_stages": [
4255
+ 0
4256
+ ],
4257
+ "range_multi_buffers": [
4258
+ null
4259
+ ],
4260
+ "range_flattens": [
4261
+ null
4262
+ ],
4263
+ "load_eviction_policies": [
4264
+ "",
4265
+ "",
4266
+ ""
4267
+ ],
4268
+ "num_warps": 1,
4269
+ "num_stages": 3,
4270
+ "indexing": [
4271
+ "pointer",
4272
+ "pointer",
4273
+ "pointer",
4274
+ "tensor_descriptor"
4275
+ ],
4276
+ "pid_type": "flat"
4277
+ },
4278
+ "intermediate_8192_numtokens_96": {
4279
+ "block_sizes": [
4280
+ 64,
4281
+ 64
4282
+ ],
4283
+ "loop_orders": [
4284
+ [
4285
+ 0,
4286
+ 1
4287
+ ]
4288
+ ],
4289
+ "flatten_loops": [
4290
+ true
4291
+ ],
4292
+ "l2_groupings": [
4293
+ 2
4294
+ ],
4295
+ "range_unroll_factors": [
4296
+ 0
4297
+ ],
4298
+ "range_warp_specializes": [],
4299
+ "range_num_stages": [
4300
+ 0
4301
+ ],
4302
+ "range_multi_buffers": [
4303
+ null
4304
+ ],
4305
+ "range_flattens": [
4306
+ null
4307
+ ],
4308
+ "load_eviction_policies": [
4309
+ "first",
4310
+ "",
4311
+ "last"
4312
+ ],
4313
+ "num_warps": 1,
4314
+ "num_stages": 1,
4315
+ "indexing": [
4316
+ "pointer",
4317
+ "pointer",
4318
+ "pointer",
4319
+ "pointer"
4320
+ ],
4321
+ "pid_type": "flat"
4322
+ },
4323
+ "intermediate_11008_numtokens_96": {
4324
+ "block_sizes": [
4325
+ 64,
4326
+ 256
4327
+ ],
4328
+ "loop_orders": [
4329
+ [
4330
+ 0,
4331
+ 1
4332
+ ]
4333
+ ],
4334
+ "flatten_loops": [
4335
+ true
4336
+ ],
4337
+ "l2_groupings": [
4338
+ 1
4339
+ ],
4340
+ "range_unroll_factors": [
4341
+ 0
4342
+ ],
4343
+ "range_warp_specializes": [],
4344
+ "range_num_stages": [
4345
+ 0
4346
+ ],
4347
+ "range_multi_buffers": [
4348
+ null
4349
+ ],
4350
+ "range_flattens": [
4351
+ null
4352
+ ],
4353
+ "load_eviction_policies": [
4354
+ "first",
4355
+ "",
4356
+ ""
4357
+ ],
4358
+ "num_warps": 2,
4359
+ "num_stages": 2,
4360
+ "indexing": [
4361
+ "pointer",
4362
+ "pointer",
4363
+ "pointer",
4364
+ "pointer"
4365
+ ],
4366
+ "pid_type": "flat"
4367
+ },
4368
+ "intermediate_14336_numtokens_96": {
4369
+ "block_sizes": [
4370
+ 32,
4371
+ 64
4372
+ ],
4373
+ "loop_orders": [
4374
+ [
4375
+ 0,
4376
+ 1
4377
+ ]
4378
+ ],
4379
+ "flatten_loops": [
4380
+ true
4381
+ ],
4382
+ "l2_groupings": [
4383
+ 1
4384
+ ],
4385
+ "range_unroll_factors": [
4386
+ 0
4387
+ ],
4388
+ "range_warp_specializes": [],
4389
+ "range_num_stages": [
4390
+ 0
4391
+ ],
4392
+ "range_multi_buffers": [
4393
+ null
4394
+ ],
4395
+ "range_flattens": [
4396
+ null
4397
+ ],
4398
+ "load_eviction_policies": [
4399
+ "",
4400
+ "",
4401
+ "first"
4402
+ ],
4403
+ "num_warps": 1,
4404
+ "num_stages": 1,
4405
+ "indexing": [
4406
+ "pointer",
4407
+ "pointer",
4408
+ "tensor_descriptor",
4409
+ "pointer"
4410
+ ],
4411
+ "pid_type": "flat"
4412
+ },
4413
+ "intermediate_2048_numtokens_104": {
4414
+ "block_sizes": [
4415
+ 32,
4416
+ 8
4417
+ ],
4418
+ "loop_orders": [
4419
+ [
4420
+ 0,
4421
+ 1
4422
+ ]
4423
+ ],
4424
+ "flatten_loops": [
4425
+ true
4426
+ ],
4427
+ "l2_groupings": [
4428
+ 1
4429
+ ],
4430
+ "range_unroll_factors": [
4431
+ 0
4432
+ ],
4433
+ "range_warp_specializes": [],
4434
+ "range_num_stages": [
4435
+ 0
4436
+ ],
4437
+ "range_multi_buffers": [
4438
+ null
4439
+ ],
4440
+ "range_flattens": [
4441
+ null
4442
+ ],
4443
+ "load_eviction_policies": [
4444
+ "",
4445
+ "",
4446
+ ""
4447
+ ],
4448
+ "num_warps": 16,
4449
+ "num_stages": 2,
4450
+ "indexing": [
4451
+ "tensor_descriptor",
4452
+ "pointer",
4453
+ "pointer",
4454
+ "pointer"
4455
+ ],
4456
+ "pid_type": "flat"
4457
+ },
4458
+ "intermediate_2880_numtokens_104": {
4459
+ "block_sizes": [
4460
+ 64,
4461
+ 64
4462
+ ],
4463
+ "loop_orders": [
4464
+ [
4465
+ 0,
4466
+ 1
4467
+ ]
4468
+ ],
4469
+ "flatten_loops": [
4470
+ true
4471
+ ],
4472
+ "l2_groupings": [
4473
+ 1
4474
+ ],
4475
+ "range_unroll_factors": [
4476
+ 0
4477
+ ],
4478
+ "range_warp_specializes": [],
4479
+ "range_num_stages": [
4480
+ 0
4481
+ ],
4482
+ "range_multi_buffers": [
4483
+ null
4484
+ ],
4485
+ "range_flattens": [
4486
+ null
4487
+ ],
4488
+ "load_eviction_policies": [
4489
+ "",
4490
+ "",
4491
+ ""
4492
+ ],
4493
+ "num_warps": 32,
4494
+ "num_stages": 1,
4495
+ "indexing": [
4496
+ "pointer",
4497
+ "pointer",
4498
+ "pointer",
4499
+ "pointer"
4500
+ ],
4501
+ "pid_type": "flat"
4502
+ },
4503
+ "intermediate_4096_numtokens_104": {
4504
+ "block_sizes": [
4505
+ 32,
4506
+ 32
4507
+ ],
4508
+ "loop_orders": [
4509
+ [
4510
+ 0,
4511
+ 1
4512
+ ]
4513
+ ],
4514
+ "flatten_loops": [
4515
+ false
4516
+ ],
4517
+ "l2_groupings": [
4518
+ 1
4519
+ ],
4520
+ "range_unroll_factors": [
4521
+ 0
4522
+ ],
4523
+ "range_warp_specializes": [],
4524
+ "range_num_stages": [
4525
+ 0
4526
+ ],
4527
+ "range_multi_buffers": [
4528
+ null
4529
+ ],
4530
+ "range_flattens": [
4531
+ null
4532
+ ],
4533
+ "load_eviction_policies": [
4534
+ "",
4535
+ "",
4536
+ ""
4537
+ ],
4538
+ "num_warps": 8,
4539
+ "num_stages": 1,
4540
+ "indexing": [
4541
+ "pointer",
4542
+ "pointer",
4543
+ "pointer",
4544
+ "pointer"
4545
+ ],
4546
+ "pid_type": "flat"
4547
+ },
4548
+ "intermediate_8192_numtokens_104": {
4549
+ "block_sizes": [
4550
+ 8,
4551
+ 8
4552
+ ],
4553
+ "loop_orders": [
4554
+ [
4555
+ 0,
4556
+ 1
4557
+ ]
4558
+ ],
4559
+ "flatten_loops": [
4560
+ true
4561
+ ],
4562
+ "l2_groupings": [
4563
+ 2
4564
+ ],
4565
+ "range_unroll_factors": [
4566
+ 0
4567
+ ],
4568
+ "range_warp_specializes": [],
4569
+ "range_num_stages": [
4570
+ 0
4571
+ ],
4572
+ "range_multi_buffers": [
4573
+ null
4574
+ ],
4575
+ "range_flattens": [
4576
+ null
4577
+ ],
4578
+ "load_eviction_policies": [
4579
+ "",
4580
+ "",
4581
+ ""
4582
+ ],
4583
+ "num_warps": 8,
4584
+ "num_stages": 1,
4585
+ "indexing": [
4586
+ "pointer",
4587
+ "pointer",
4588
+ "pointer",
4589
+ "tensor_descriptor"
4590
+ ],
4591
+ "pid_type": "flat"
4592
+ },
4593
+ "intermediate_11008_numtokens_104": {
4594
+ "block_sizes": [
4595
+ 128,
4596
+ 16
4597
+ ],
4598
+ "loop_orders": [
4599
+ [
4600
+ 0,
4601
+ 1
4602
+ ]
4603
+ ],
4604
+ "flatten_loops": [
4605
+ true
4606
+ ],
4607
+ "l2_groupings": [
4608
+ 1
4609
+ ],
4610
+ "range_unroll_factors": [
4611
+ 0
4612
+ ],
4613
+ "range_warp_specializes": [],
4614
+ "range_num_stages": [
4615
+ 0
4616
+ ],
4617
+ "range_multi_buffers": [
4618
+ null
4619
+ ],
4620
+ "range_flattens": [
4621
+ null
4622
+ ],
4623
+ "load_eviction_policies": [
4624
+ "",
4625
+ "",
4626
+ ""
4627
+ ],
4628
+ "num_warps": 2,
4629
+ "num_stages": 1,
4630
+ "indexing": [
4631
+ "pointer",
4632
+ "tensor_descriptor",
4633
+ "pointer",
4634
+ "pointer"
4635
+ ],
4636
+ "pid_type": "flat"
4637
+ },
4638
+ "intermediate_14336_numtokens_104": {
4639
+ "block_sizes": [
4640
+ 32,
4641
+ 16
4642
+ ],
4643
+ "loop_orders": [
4644
+ [
4645
+ 0,
4646
+ 1
4647
+ ]
4648
+ ],
4649
+ "flatten_loops": [
4650
+ true
4651
+ ],
4652
+ "l2_groupings": [
4653
+ 1
4654
+ ],
4655
+ "range_unroll_factors": [
4656
+ 0
4657
+ ],
4658
+ "range_warp_specializes": [],
4659
+ "range_num_stages": [
4660
+ 0
4661
+ ],
4662
+ "range_multi_buffers": [
4663
+ null
4664
+ ],
4665
+ "range_flattens": [
4666
+ null
4667
+ ],
4668
+ "load_eviction_policies": [
4669
+ "",
4670
+ "",
4671
+ ""
4672
+ ],
4673
+ "num_warps": 2,
4674
+ "num_stages": 1,
4675
+ "indexing": [
4676
+ "pointer",
4677
+ "pointer",
4678
+ "pointer",
4679
+ "tensor_descriptor"
4680
+ ],
4681
+ "pid_type": "flat"
4682
+ },
4683
+ "intermediate_2048_numtokens_112": {
4684
+ "block_sizes": [
4685
+ 32,
4686
+ 1024
4687
+ ],
4688
+ "loop_orders": [
4689
+ [
4690
+ 0,
4691
+ 1
4692
+ ]
4693
+ ],
4694
+ "flatten_loops": [
4695
+ true
4696
+ ],
4697
+ "l2_groupings": [
4698
+ 1
4699
+ ],
4700
+ "range_unroll_factors": [
4701
+ 0
4702
+ ],
4703
+ "range_warp_specializes": [],
4704
+ "range_num_stages": [
4705
+ 0
4706
+ ],
4707
+ "range_multi_buffers": [
4708
+ null
4709
+ ],
4710
+ "range_flattens": [
4711
+ null
4712
+ ],
4713
+ "load_eviction_policies": [
4714
+ "first",
4715
+ "",
4716
+ ""
4717
+ ],
4718
+ "num_warps": 32,
4719
+ "num_stages": 1,
4720
+ "indexing": [
4721
+ "pointer",
4722
+ "tensor_descriptor",
4723
+ "pointer",
4724
+ "pointer"
4725
+ ],
4726
+ "pid_type": "flat"
4727
+ },
4728
+ "intermediate_2880_numtokens_112": {
4729
+ "block_sizes": [
4730
+ 32,
4731
+ 32
4732
+ ],
4733
+ "loop_orders": [
4734
+ [
4735
+ 0,
4736
+ 1
4737
+ ]
4738
+ ],
4739
+ "flatten_loops": [
4740
+ false
4741
+ ],
4742
+ "l2_groupings": [
4743
+ 1
4744
+ ],
4745
+ "range_unroll_factors": [
4746
+ 0
4747
+ ],
4748
+ "range_warp_specializes": [],
4749
+ "range_num_stages": [
4750
+ 0
4751
+ ],
4752
+ "range_multi_buffers": [
4753
+ null
4754
+ ],
4755
+ "range_flattens": [
4756
+ null
4757
+ ],
4758
+ "load_eviction_policies": [
4759
+ "",
4760
+ "",
4761
+ ""
4762
+ ],
4763
+ "num_warps": 1,
4764
+ "num_stages": 1,
4765
+ "indexing": [
4766
+ "pointer",
4767
+ "pointer",
4768
+ "pointer",
4769
+ "pointer"
4770
+ ],
4771
+ "pid_type": "flat"
4772
+ },
4773
+ "intermediate_4096_numtokens_112": {
4774
+ "block_sizes": [
4775
+ 32,
4776
+ 128
4777
+ ],
4778
+ "loop_orders": [
4779
+ [
4780
+ 0,
4781
+ 1
4782
+ ]
4783
+ ],
4784
+ "flatten_loops": [
4785
+ true
4786
+ ],
4787
+ "l2_groupings": [
4788
+ 1
4789
+ ],
4790
+ "range_unroll_factors": [
4791
+ 0
4792
+ ],
4793
+ "range_warp_specializes": [],
4794
+ "range_num_stages": [
4795
+ 0
4796
+ ],
4797
+ "range_multi_buffers": [
4798
+ null
4799
+ ],
4800
+ "range_flattens": [
4801
+ null
4802
+ ],
4803
+ "load_eviction_policies": [
4804
+ "last",
4805
+ "",
4806
+ ""
4807
+ ],
4808
+ "num_warps": 16,
4809
+ "num_stages": 1,
4810
+ "indexing": [
4811
+ "pointer",
4812
+ "pointer",
4813
+ "pointer",
4814
+ "pointer"
4815
+ ],
4816
+ "pid_type": "flat"
4817
+ },
4818
+ "intermediate_8192_numtokens_112": {
4819
+ "block_sizes": [
4820
+ 32,
4821
+ 128
4822
+ ],
4823
+ "loop_orders": [
4824
+ [
4825
+ 0,
4826
+ 1
4827
+ ]
4828
+ ],
4829
+ "flatten_loops": [
4830
+ true
4831
+ ],
4832
+ "l2_groupings": [
4833
+ 1
4834
+ ],
4835
+ "range_unroll_factors": [
4836
+ 0
4837
+ ],
4838
+ "range_warp_specializes": [],
4839
+ "range_num_stages": [
4840
+ 0
4841
+ ],
4842
+ "range_multi_buffers": [
4843
+ null
4844
+ ],
4845
+ "range_flattens": [
4846
+ null
4847
+ ],
4848
+ "load_eviction_policies": [
4849
+ "",
4850
+ "",
4851
+ ""
4852
+ ],
4853
+ "num_warps": 8,
4854
+ "num_stages": 1,
4855
+ "indexing": [
4856
+ "pointer",
4857
+ "pointer",
4858
+ "pointer",
4859
+ "tensor_descriptor"
4860
+ ],
4861
+ "pid_type": "flat"
4862
+ },
4863
+ "intermediate_11008_numtokens_112": {
4864
+ "block_sizes": [
4865
+ 16,
4866
+ 64
4867
+ ],
4868
+ "loop_orders": [
4869
+ [
4870
+ 0,
4871
+ 1
4872
+ ]
4873
+ ],
4874
+ "flatten_loops": [
4875
+ true
4876
+ ],
4877
+ "l2_groupings": [
4878
+ 1
4879
+ ],
4880
+ "range_unroll_factors": [
4881
+ 0
4882
+ ],
4883
+ "range_warp_specializes": [],
4884
+ "range_num_stages": [
4885
+ 0
4886
+ ],
4887
+ "range_multi_buffers": [
4888
+ null
4889
+ ],
4890
+ "range_flattens": [
4891
+ null
4892
+ ],
4893
+ "load_eviction_policies": [
4894
+ "",
4895
+ "",
4896
+ ""
4897
+ ],
4898
+ "num_warps": 8,
4899
+ "num_stages": 1,
4900
+ "indexing": [
4901
+ "pointer",
4902
+ "pointer",
4903
+ "pointer",
4904
+ "pointer"
4905
+ ],
4906
+ "pid_type": "flat"
4907
+ },
4908
+ "intermediate_14336_numtokens_112": {
4909
+ "block_sizes": [
4910
+ 32,
4911
+ 8
4912
+ ],
4913
+ "loop_orders": [
4914
+ [
4915
+ 0,
4916
+ 1
4917
+ ]
4918
+ ],
4919
+ "flatten_loops": [
4920
+ true
4921
+ ],
4922
+ "l2_groupings": [
4923
+ 2
4924
+ ],
4925
+ "range_unroll_factors": [
4926
+ 0
4927
+ ],
This file is too large to show in full.
vllm/kernels/helion/ops/silu_mul_fp8.py
+64
-29
@@ -3,6 +3,7 @@
3
4
from typing import Any
5
6
+import regex as re
7
import torch
8
9
from vllm.logger import init_logger
@@ -53,44 +54,78 @@ def silu_mul_fp8(input: torch.Tensor, scale: torch.Tensor) -> torch.Tensor:
54
return out.view(output_shape)
55
56
57
+@silu_mul_fp8.register_input_generator # type: ignore[misc]
58
+def generate_silu_mul_fp8_inputs() -> dict[str, tuple[Any, ...]]:
59
+ intermediate_sizes = [2048, 2880, 4096, 8192, 11008, 14336]
60
+
61
+ # Use the same num_tokens values as vLLM's default cudagraph capture sizes.
62
+ # See vllm/config/vllm.py _set_cudagraph_sizes() for the canonical formula.
63
+ num_tokens_list = [1, 2, 4] + list(range(8, 256, 8)) + list(range(256, 513, 16))
64
+
65
+ inputs = {}
66
+ for num_tokens in num_tokens_list:
67
+ for intermediate_size in intermediate_sizes:
68
+ # Input tensor has shape (num_tokens, 2 * intermediate_size)
69
+ # because silu_mul splits it into two halves
70
+ input_tensor = torch.randn(
71
+ num_tokens,
72
+ 2 * intermediate_size,
73
+ device="cuda",
74
+ dtype=torch.bfloat16,
75
+ )
76
+ scale = torch.tensor([1.0], device="cuda", dtype=torch.float32)
77
+
78
+ config_key = f"intermediate_{intermediate_size}_numtokens_{num_tokens}"
79
+ inputs[config_key] = (input_tensor, scale)
80
+
81
+ return inputs
82
+
83
+
84
@silu_mul_fp8.register_config_picker # type: ignore[misc]
85
def pick_silu_mul_fp8_config(
86
args: tuple[Any, ...], config_keys: list[str]
87
) -> str | None:
88
+ """Pick the best pre-tuned config for the given input shape.
89
+
90
+ Selection strategy:
91
+ 1. Find the closest intermediate_size among available configs
92
+ (exact match preferred).
93
+ 2. Among the num_tokens values tuned for that intermediate_size, pick
94
+ the smallest num_tokens >= the input's num_tokens. If the input is
95
+ larger than all available num_tokens, fall back to the largest.
96
+
97
+ Config keys must be "default" or follow the format
98
+ "intermediate_{int}_numtokens_{int}".
99
+ """
100
if not config_keys:
101
return None
102
63
- input_tensor, scale = args
103
+ input_tensor, _scale = args
104
intermediate_size = input_tensor.shape[-1] // 2
65
-
66
- # TODO(gmagosfm): Rerun autotuning to capture config for
67
- # other batch sizes.
68
- target_key = f"intermediate_{intermediate_size}_batchsize_256"
69
- if target_key in config_keys:
70
- return target_key
71
-
72
- intermediate_sizes = []
105
+ num_tokens = input_tensor.view(-1, input_tensor.shape[-1]).shape[0]
106
+ configs: dict[int, list[int]] = {}
107
for key in config_keys:
74
- if key.startswith("intermediate_") and "_batchsize_256" in key:
75
- try:
76
- size_str = key.split("_")[1]
77
- size = int(size_str)
78
- intermediate_sizes.append((abs(size - intermediate_size), key))
79
- except (ValueError, IndexError):
80
- continue
81
-
82
- if intermediate_sizes:
83
- _, best_key = min(intermediate_sizes)
84
- logger.debug(
85
- "No exact config for intermediate_size=%d, using closest match: %s",
86
- intermediate_size,
87
- best_key,
88
- )
89
- return best_key
90
- if "default" in config_keys:
91
- return "default"
92
-
93
- return None
108
+ if key == "default":
109
+ continue
110
+ match = re.fullmatch(r"intermediate_(\d+)_numtokens_(\d+)", key)
111
+ if not match:
112
+ raise ValueError(
113
+ f"Malformed config key '{key}', "
114
+ f"expected format 'intermediate_{{int}}_numtokens_{{int}}'"
115
+ )
116
+ isize_str, ntokens_str = match.groups()
117
+ configs.setdefault(int(isize_str), []).append(int(ntokens_str))
118
+
119
+ if not configs:
120
+ return "default" if "default" in config_keys else None
121
+
122
+ best_isize = min(configs, key=lambda s: abs(s - intermediate_size))
123
+ available_ntokens = sorted(configs[best_isize])
124
+ best_ntokens = next(
125
+ (n for n in available_ntokens if n >= num_tokens), available_ntokens[-1]
126
+ )
127
+
128
+ return f"intermediate_{best_isize}_numtokens_{best_ntokens}"
129
130
131
def silu_mul_fp8_baseline(input: torch.Tensor, scale: torch.Tensor) -> torch.Tensor: