@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7371749d54
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Misc] Fix ImportError causing by triton (#9493)

Mengqing Cao committed Nov 8, 2024 at 13:08 UTC 7371749d54db40999d896c4a7f8935bc6984c093
1 file changed +5 -2
vllm/executor/multiproc_gpu_executor.py
+5 -2
@@ -13,12 +13,15 @@ from vllm.executor.multiproc_worker_utils import (ProcessWorkerWrapper,
13 from vllm.logger import init_logger
14 from vllm.model_executor.layers.sampler import SamplerOutput
15 from vllm.sequence import ExecuteModelRequest
16 -from vllm.triton_utils import maybe_set_triton_cache_manager
16 +from vllm.triton_utils.importing import HAS_TRITON
17 from vllm.utils import (_run_task_with_lock, cuda_device_count_stateless,
18 cuda_is_initialized, get_distributed_init_method,
19 get_open_port, get_vllm_instance_id, make_async,
20 update_environment_variables)
21
22 +if HAS_TRITON:
23 + from vllm.triton_utils import maybe_set_triton_cache_manager
24 +
25 logger = init_logger(__name__)
26
27
@@ -59,7 +62,7 @@ class MultiprocessingGPUExecutor(DistributedGPUExecutor):
62 torch.set_num_threads(default_omp_num_threads)
63
64 # workaround for https://github.com/vllm-project/vllm/issues/6103
62 - if world_size > 1:
65 + if HAS_TRITON and world_size > 1:
66 maybe_set_triton_cache_manager()
67
68 # Multiprocessing-based executor does not support multi-node setting.