A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[fix] lora benchmarks pass no_lora_flag_cpu (#23774)
Signed-off-by: Dylan Maloy <34420038+dolpm@users.noreply.github.com> Co-authored-by: Jee Jee Li <pandaleefree@gmail.com>
dolpm committed
Sep 17, 2025 at 06:22 UTC
1b962e24577dddc0d7441ae0b06392e1f9262a51
1 file changed
+7
-1
benchmarks/kernels/benchmark_lora.py
+7
-1
@@ -464,7 +464,11 @@ class BenchmarkTensors:
464
for field_name in LoRAKernelMeta.__dataclass_fields__:
465
field = getattr(self.lora_kernel_meta, field_name)
466
assert isinstance(field, torch.Tensor)
467
- setattr(self.lora_kernel_meta, field_name, to_device(field))
467
+ setattr(
468
+ self.lora_kernel_meta,
469
+ field_name,
470
+ to_device(field) if field_name != "no_lora_flag_cpu" else field,
471
+ )
472
473
def metadata(self) -> tuple[int, int, int]:
474
"""
@@ -512,6 +516,7 @@ class BenchmarkTensors:
516
"lora_token_start_loc": self.lora_kernel_meta.lora_token_start_loc,
517
"lora_ids": self.lora_kernel_meta.active_lora_ids,
518
"scaling": 1.0,
519
+ "no_lora_flag_cpu": self.lora_kernel_meta.no_lora_flag_cpu,
520
}
521
522
def as_lora_expand_kwargs(self, add_inputs: bool) -> dict[str, Any]:
@@ -552,6 +557,7 @@ class BenchmarkTensors:
557
"lora_ids": self.lora_kernel_meta.active_lora_ids,
558
"offset_start": 0,
559
"add_inputs": add_inputs,
560
+ "no_lora_flag_cpu": self.lora_kernel_meta.no_lora_flag_cpu,
561
}
562
563
def bench_fn_kwargs(