A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[ci] fix broken tests (#10956)
Signed-off-by: youkaichao <youkaichao@gmail.com>
youkaichao committed
Dec 6, 2024 at 11:25 UTC
dcdc3fafe535178037ef0a58f53607b2fb3e4190
1 file changed
+6
-6
vllm/worker/model_runner.py
+6
-6
@@ -1782,6 +1782,9 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1782
kv_caches: vLLM's paged memory
1783
"""
1784
1785
+ if self.vllm_config.kv_transfer_config is None:
1786
+ return False
1787
+
1788
prefill_meta = model_input.attn_metadata.prefill_metadata
1789
1790
# check if the current run is profiling
@@ -1789,9 +1792,6 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1792
# check if the current run is prefill
1793
is_prefill_run = prefill_meta is not None
1794
1792
- if self.vllm_config.kv_transfer_config is None:
1793
- return False
1794
-
1795
return self.vllm_config.kv_transfer_config.is_kv_consumer and (
1796
not is_profile_run) and is_prefill_run
1797
@@ -1807,6 +1807,9 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1807
kv_caches: vLLM's paged memory
1808
"""
1809
1810
+ if self.vllm_config.kv_transfer_config is None:
1811
+ return False
1812
+
1813
prefill_meta = model_input.attn_metadata.prefill_metadata
1814
1815
# check if the current run is profiling
@@ -1814,9 +1817,6 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1817
# check if the current run is prefill
1818
is_prefill_run = prefill_meta is not None
1819
1817
- if self.vllm_config.kv_transfer_config is None:
1818
- return False
1819
-
1820
return self.vllm_config.kv_transfer_config.is_kv_producer and (
1821
not is_profile_run) and is_prefill_run
1822