@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / dcdc3fafe5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[ci] fix broken tests (#10956)

Signed-off-by: youkaichao <youkaichao@gmail.com>

youkaichao committed Dec 6, 2024 at 11:25 UTC dcdc3fafe535178037ef0a58f53607b2fb3e4190
1 file changed +6 -6
vllm/worker/model_runner.py
+6 -6
@@ -1782,6 +1782,9 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1782 kv_caches: vLLM's paged memory
1783 """
1784
1785 + if self.vllm_config.kv_transfer_config is None:
1786 + return False
1787 +
1788 prefill_meta = model_input.attn_metadata.prefill_metadata
1789
1790 # check if the current run is profiling
@@ -1789,9 +1792,6 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1792 # check if the current run is prefill
1793 is_prefill_run = prefill_meta is not None
1794
1792 - if self.vllm_config.kv_transfer_config is None:
1793 - return False
1794 -
1795 return self.vllm_config.kv_transfer_config.is_kv_consumer and (
1796 not is_profile_run) and is_prefill_run
1797
@@ -1807,6 +1807,9 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1807 kv_caches: vLLM's paged memory
1808 """
1809
1810 + if self.vllm_config.kv_transfer_config is None:
1811 + return False
1812 +
1813 prefill_meta = model_input.attn_metadata.prefill_metadata
1814
1815 # check if the current run is profiling
@@ -1814,9 +1817,6 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1817 # check if the current run is prefill
1818 is_prefill_run = prefill_meta is not None
1819
1817 - if self.vllm_config.kv_transfer_config is None:
1818 - return False
1819 -
1820 return self.vllm_config.kv_transfer_config.is_kv_producer and (
1821 not is_profile_run) and is_prefill_run
1822