A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 17:15 UTC
[Misc] update the comments (#15780)
Signed-off-by: chengyang liu <lcy4869@gmail.com> Co-authored-by: chengyang liu <lcy4869@gmail.com>
Chengyang LIU committed
Mar 30, 2025 at 19:39 UTC
18ed3132d2bfe1df9a74729457b69243955221e8
1 file changed
+1
-1
vllm/v1/worker/gpu_model_runner.py
+1
-1
@@ -673,7 +673,7 @@ class GPUModelRunner(LoRAModelRunnerMixin):
673
# use two kernels for cascade attention. Let's imagine:
674
# Request 3's input query: [D]
675
# Request 3's kv cache: [A, B, C, D]
676
- # Request 3's num_computed_tokens: 4 (i.e., [A, B, C, D])
676
+ # Request 3's num_computed_tokens: 3 (i.e., [A, B, C])
677
# If we use [A, B, C, D] as the common prefix for Request 1-3,
678
# then Request 3 will be processed only by the first kernel,
679
# and the second kernel will get an empty input. While this is not