A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Core] Move ray_utils.py from `engine` to `executor` package (#4347)
Nick Hill committed
Apr 24, 2024 at 23:52 UTC
479d69fad0538f04cb22bf13e76ff91cfeb8a4e5
7 files changed
+11
-9
vllm/__init__.py
+1
-1
@@ -3,8 +3,8 @@
3
from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
4
from vllm.engine.async_llm_engine import AsyncLLMEngine
5
from vllm.engine.llm_engine import LLMEngine
6
-from vllm.engine.ray_utils import initialize_ray_cluster
6
from vllm.entrypoints.llm import LLM
7
+from vllm.executor.ray_utils import initialize_ray_cluster
8
from vllm.model_executor.models import ModelRegistry
9
from vllm.outputs import CompletionOutput, RequestOutput
10
from vllm.sampling_params import SamplingParams
vllm/engine/async_llm_engine.py
+1
-1
@@ -10,7 +10,7 @@ from transformers import PreTrainedTokenizer
10
from vllm.config import ModelConfig
11
from vllm.engine.arg_utils import AsyncEngineArgs
12
from vllm.engine.llm_engine import LLMEngine
13
-from vllm.engine.ray_utils import initialize_ray_cluster, ray
13
+from vllm.executor.ray_utils import initialize_ray_cluster, ray
14
from vllm.logger import init_logger
15
from vllm.lora.request import LoRARequest
16
from vllm.outputs import RequestOutput
vllm/engine/llm_engine.py
+1
-1
@@ -15,8 +15,8 @@ from vllm.engine.output_processor.interfaces import (
15
SequenceGroupOutputProcessor)
16
from vllm.engine.output_processor.stop_checker import StopChecker
17
from vllm.engine.output_processor.util import create_output_by_sequence_group
18
-from vllm.engine.ray_utils import initialize_ray_cluster
18
from vllm.executor.executor_base import ExecutorBase
19
+from vllm.executor.ray_utils import initialize_ray_cluster
20
from vllm.logger import init_logger
21
from vllm.lora.request import LoRARequest
22
from vllm.outputs import RequestOutput
vllm/executor/ray_gpu_executor.py
+6
-4
@@ -5,8 +5,8 @@ from collections import defaultdict
5
from itertools import islice, repeat
6
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple
7
8
-from vllm.engine.ray_utils import RayWorkerWrapper, ray
8
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
9
+from vllm.executor.ray_utils import RayWorkerWrapper, ray
10
from vllm.logger import init_logger
11
from vllm.lora.request import LoRARequest
12
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -74,7 +74,7 @@ class RayGPUExecutor(ExecutorBase):
74
75
# The driver dummy worker does not actually use any resources.
76
# It holds the resource for the driver worker.
77
- self.driver_dummy_worker: RayWorkerWrapper = None
77
+ self.driver_dummy_worker: Optional[RayWorkerWrapper] = None
78
# The remaining workers are the actual ray actors.
79
self.workers: List[RayWorkerWrapper] = []
80
@@ -318,6 +318,7 @@ class RayGPUExecutor(ExecutorBase):
318
driver_worker_output = self.driver_worker.execute_method(
319
method, *driver_args, **driver_kwargs)
320
else:
321
+ assert self.driver_dummy_worker is not None
322
driver_worker_output = ray.get(
323
self.driver_dummy_worker.execute_method.remote(
324
method, *driver_args, **driver_kwargs))
@@ -353,8 +354,9 @@ class RayGPUExecutor(ExecutorBase):
354
# a dummy value for now. It will be fixed soon.
355
with InputNode() as input_data:
356
forward_dag = MultiOutputNode([
356
- worker.execute_model_compiled_dag_remote.bind(input_data)
357
- for worker in self.workers
357
+ worker.execute_model_compiled_dag_remote.
358
+ bind( # type: ignore[attr-defined]
359
+ input_data) for worker in self.workers
360
])
361
return forward_dag.experimental_compile()
362
vllm/executor/ray_utils.py
renamed
vllm/transformers_utils/tokenizer_group/__init__.py
+1
-1
@@ -1,7 +1,7 @@
1
from typing import Optional
2
3
from vllm.config import TokenizerPoolConfig
4
-from vllm.engine.ray_utils import ray
4
+from vllm.executor.ray_utils import ray
5
from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
6
BaseTokenizerGroup)
7
from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+1
-1
@@ -6,7 +6,7 @@ from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
6
from transformers import PreTrainedTokenizer
7
8
from vllm.config import TokenizerPoolConfig
9
-from vllm.engine.ray_utils import ray
9
+from vllm.executor.ray_utils import ray
10
from vllm.lora.request import LoRARequest
11
from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
12
BaseTokenizerGroup)