@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 479d69fad0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Core] Move ray_utils.py from `engine` to `executor` package (#4347)

Nick Hill committed Apr 24, 2024 at 23:52 UTC 479d69fad0538f04cb22bf13e76ff91cfeb8a4e5
7 files changed +11 -9
vllm/__init__.py
+1 -1
@@ -3,8 +3,8 @@
3 from vllm.engine.arg_utils import AsyncEngineArgs, EngineArgs
4 from vllm.engine.async_llm_engine import AsyncLLMEngine
5 from vllm.engine.llm_engine import LLMEngine
6 -from vllm.engine.ray_utils import initialize_ray_cluster
6 from vllm.entrypoints.llm import LLM
7 +from vllm.executor.ray_utils import initialize_ray_cluster
8 from vllm.model_executor.models import ModelRegistry
9 from vllm.outputs import CompletionOutput, RequestOutput
10 from vllm.sampling_params import SamplingParams
vllm/engine/async_llm_engine.py
+1 -1
@@ -10,7 +10,7 @@ from transformers import PreTrainedTokenizer
10 from vllm.config import ModelConfig
11 from vllm.engine.arg_utils import AsyncEngineArgs
12 from vllm.engine.llm_engine import LLMEngine
13 -from vllm.engine.ray_utils import initialize_ray_cluster, ray
13 +from vllm.executor.ray_utils import initialize_ray_cluster, ray
14 from vllm.logger import init_logger
15 from vllm.lora.request import LoRARequest
16 from vllm.outputs import RequestOutput
vllm/engine/llm_engine.py
+1 -1
@@ -15,8 +15,8 @@ from vllm.engine.output_processor.interfaces import (
15 SequenceGroupOutputProcessor)
16 from vllm.engine.output_processor.stop_checker import StopChecker
17 from vllm.engine.output_processor.util import create_output_by_sequence_group
18 -from vllm.engine.ray_utils import initialize_ray_cluster
18 from vllm.executor.executor_base import ExecutorBase
19 +from vllm.executor.ray_utils import initialize_ray_cluster
20 from vllm.logger import init_logger
21 from vllm.lora.request import LoRARequest
22 from vllm.outputs import RequestOutput
vllm/executor/ray_gpu_executor.py
+6 -4
@@ -5,8 +5,8 @@ from collections import defaultdict
5 from itertools import islice, repeat
6 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple
7
8 -from vllm.engine.ray_utils import RayWorkerWrapper, ray
8 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
9 +from vllm.executor.ray_utils import RayWorkerWrapper, ray
10 from vllm.logger import init_logger
11 from vllm.lora.request import LoRARequest
12 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
@@ -74,7 +74,7 @@ class RayGPUExecutor(ExecutorBase):
74
75 # The driver dummy worker does not actually use any resources.
76 # It holds the resource for the driver worker.
77 - self.driver_dummy_worker: RayWorkerWrapper = None
77 + self.driver_dummy_worker: Optional[RayWorkerWrapper] = None
78 # The remaining workers are the actual ray actors.
79 self.workers: List[RayWorkerWrapper] = []
80
@@ -318,6 +318,7 @@ class RayGPUExecutor(ExecutorBase):
318 driver_worker_output = self.driver_worker.execute_method(
319 method, *driver_args, **driver_kwargs)
320 else:
321 + assert self.driver_dummy_worker is not None
322 driver_worker_output = ray.get(
323 self.driver_dummy_worker.execute_method.remote(
324 method, *driver_args, **driver_kwargs))
@@ -353,8 +354,9 @@ class RayGPUExecutor(ExecutorBase):
354 # a dummy value for now. It will be fixed soon.
355 with InputNode() as input_data:
356 forward_dag = MultiOutputNode([
356 - worker.execute_model_compiled_dag_remote.bind(input_data)
357 - for worker in self.workers
357 + worker.execute_model_compiled_dag_remote.
358 + bind( # type: ignore[attr-defined]
359 + input_data) for worker in self.workers
360 ])
361 return forward_dag.experimental_compile()
362
vllm/executor/ray_utils.py renamed
vllm/transformers_utils/tokenizer_group/__init__.py
+1 -1
@@ -1,7 +1,7 @@
1 from typing import Optional
2
3 from vllm.config import TokenizerPoolConfig
4 -from vllm.engine.ray_utils import ray
4 +from vllm.executor.ray_utils import ray
5 from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
6 BaseTokenizerGroup)
7 from vllm.transformers_utils.tokenizer_group.tokenizer_group import (
vllm/transformers_utils/tokenizer_group/ray_tokenizer_group.py
+1 -1
@@ -6,7 +6,7 @@ from ray.util.scheduling_strategies import NodeAffinitySchedulingStrategy
6 from transformers import PreTrainedTokenizer
7
8 from vllm.config import TokenizerPoolConfig
9 -from vllm.engine.ray_utils import ray
9 +from vllm.executor.ray_utils import ray
10 from vllm.lora.request import LoRARequest
11 from vllm.transformers_utils.tokenizer_group.base_tokenizer_group import (
12 BaseTokenizerGroup)