A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Misc] Fix ImportError causing by triton (#9493)
Mengqing Cao committed
Nov 8, 2024 at 13:08 UTC
7371749d54db40999d896c4a7f8935bc6984c093
1 file changed
+5
-2
vllm/executor/multiproc_gpu_executor.py
+5
-2
@@ -13,12 +13,15 @@ from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
13
from vllm.logger import init_logger
14
from vllm.model_executor.layers.sampler import SamplerOutput
15
from vllm.sequence import ExecuteModelRequest
16
-from vllm.triton_utils import maybe_set_triton_cache_manager
16
+from vllm.triton_utils.importing import HAS_TRITON
17
from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,
18
cuda_is_initialized, get_distributed_init_method,
19
get_open_port, get_vllm_instance_id, make_async,
20
update_environment_variables)
21
22
+if HAS_TRITON:
23
+ from vllm.triton_utils import maybe_set_triton_cache_manager
24
+
25
logger = init_logger(__name__)
26
27
@@ -59,7 +62,7 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
62
torch.set_num_threads(default_omp_num_threads)
63
64
# workaround for https://github.com/vllm-project/vllm/issues/6103
62
- if world_size > 1:
65
+ if HAS_TRITON and world_size > 1:
66
maybe_set_triton_cache_manager()
67
68
# Multiprocessing-based executor does not support multi-node setting.