A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[misc] remove engine_use_ray (#8126)
youkaichao committed
Sep 11, 2024 at 18:23 UTC
f842a7aff143a4a1ddc59e1fb57109cb377f5475
8 files changed
+32
-197
tests/async_engine/test_api_server.py
+4
-14
@@ -1,4 +1,3 @@
1
-import os
1
import subprocess
2
import sys
3
import time
@@ -26,8 +25,7 @@ def _query_server_long(prompt: str) -> dict:
25
26
27
@pytest.fixture
29
-def api_server(tokenizer_pool_size: int, engine_use_ray: bool,
30
- worker_use_ray: bool):
28
+def api_server(tokenizer_pool_size: int, worker_use_ray: bool):
29
script_path = Path(__file__).parent.joinpath(
30
"api_server_async_engine.py").absolute()
31
commands = [
@@ -37,25 +35,17 @@ def api_server(tokenizer_pool_size: int, engine_use_ray: bool,
35
str(tokenizer_pool_size)
36
]
37
40
- # Copy the environment variables and append `VLLM_ALLOW_ENGINE_USE_RAY=1`
41
- # to prevent `--engine-use-ray` raises an exception due to it deprecation
42
- env_vars = os.environ.copy()
43
- env_vars["VLLM_ALLOW_ENGINE_USE_RAY"] = "1"
44
-
45
- if engine_use_ray:
46
- commands.append("--engine-use-ray")
38
if worker_use_ray:
39
commands.append("--worker-use-ray")
49
- uvicorn_process = subprocess.Popen(commands, env=env_vars)
40
+ uvicorn_process = subprocess.Popen(commands)
41
yield
42
uvicorn_process.terminate()
43
44
45
@pytest.mark.parametrize("tokenizer_pool_size", [0, 2])
46
@pytest.mark.parametrize("worker_use_ray", [False, True])
56
-@pytest.mark.parametrize("engine_use_ray", [False, True])
57
-def test_api_server(api_server, tokenizer_pool_size: int, worker_use_ray: bool,
58
- engine_use_ray: bool):
47
+def test_api_server(api_server, tokenizer_pool_size: int,
48
+ worker_use_ray: bool):
49
"""
50
Run the API server and test it.
51
tests/async_engine/test_async_llm_engine.py
+3
-11
@@ -1,5 +1,4 @@
1
import asyncio
2
-import os
2
from asyncio import CancelledError
3
from dataclasses import dataclass
4
from typing import Optional
@@ -72,14 +71,12 @@ class MockEngine:
71
72
73
class MockAsyncLLMEngine(AsyncLLMEngine):
75
-
76
- def _init_engine(self, *args, **kwargs):
77
- return MockEngine()
74
+ _engine_class = MockEngine
75
76
77
@pytest.mark.asyncio
78
async def test_new_requests_event():
82
- engine = MockAsyncLLMEngine(worker_use_ray=False, engine_use_ray=False)
79
+ engine = MockAsyncLLMEngine(worker_use_ray=False)
80
engine.start_background_loop()
81
await asyncio.sleep(0.01)
82
assert engine.engine.step_calls == 0
@@ -112,16 +109,11 @@ async def test_new_requests_event():
109
assert engine.engine.add_request_calls == 3
110
assert engine.engine.step_calls == old_step_calls + 1
111
115
- # Allow deprecated engine_use_ray to not raise exception
116
- os.environ["VLLM_ALLOW_ENGINE_USE_RAY"] = "1"
117
-
118
- engine = MockAsyncLLMEngine(worker_use_ray=True, engine_use_ray=True)
112
+ engine = MockAsyncLLMEngine(worker_use_ray=True)
113
assert engine.get_model_config() is not None
114
assert engine.get_tokenizer() is not None
115
assert engine.get_decoding_config() is not None
116
123
- os.environ.pop("VLLM_ALLOW_ENGINE_USE_RAY")
124
-
117
118
def start_engine():
119
wait_for_gpu_memory_to_clear(
tests/async_engine/test_openapi_server.py
renamed
+1
-6
@@ -19,16 +19,11 @@ def server():
19
"--max-model-len",
20
"2048",
21
"--enforce-eager",
22
- "--engine-use-ray",
22
"--chat-template",
23
str(chatml_jinja_path),
24
]
25
27
- # Allow `--engine-use-ray`, otherwise the launch of the server throw
28
- # an error due to try to use a deprecated feature
29
- env_dict = {"VLLM_ALLOW_ENGINE_USE_RAY": "1"}
30
- with RemoteOpenAIServer(MODEL_NAME, args,
31
- env_dict=env_dict) as remote_server:
26
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
27
yield remote_server
28
29
vllm/engine/arg_utils.py
-11
@@ -1035,7 +1035,6 @@ class EngineArgs:
1035
@dataclass
1036
class AsyncEngineArgs(EngineArgs):
1037
"""Arguments for asynchronous vLLM engine."""
1038
- engine_use_ray: bool = False
1038
disable_log_requests: bool = False
1039
1040
@staticmethod
@@ -1043,16 +1042,6 @@ class AsyncEngineArgs(EngineArgs):
1042
async_args_only: bool = False) -> FlexibleArgumentParser:
1043
if not async_args_only:
1044
parser = EngineArgs.add_cli_args(parser)
1046
- parser.add_argument('--engine-use-ray',
1047
- action='store_true',
1048
- help='Use Ray to start the LLM engine in a '
1049
- 'separate process as the server process.'
1050
- '(DEPRECATED. This argument is deprecated '
1051
- 'and will be removed in a future update. '
1052
- 'Set `VLLM_ALLOW_ENGINE_USE_RAY=1` to force '
1053
- 'use it. See '
1054
- 'https://github.com/vllm-project/vllm/issues/7045.'
1055
- ')')
1045
parser.add_argument('--disable-log-requests',
1046
action='store_true',
1047
help='Disable logging requests.')
vllm/engine/async_llm_engine.py
+21
-142
@@ -16,7 +16,7 @@ from vllm.engine.llm_engine import (DecoderPromptComponents, LLMEngine,
16
PromptComponents, SchedulerOutputState)
17
from vllm.engine.metrics_types import StatLoggerBase
18
from vllm.executor.executor_base import ExecutorAsyncBase
19
-from vllm.executor.ray_utils import initialize_ray_cluster, ray
19
+from vllm.executor.ray_utils import initialize_ray_cluster
20
from vllm.inputs import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
21
SingletonPromptInputs)
22
from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
@@ -30,7 +30,6 @@ from vllm.sampling_params import SamplingParams
30
from vllm.sequence import ExecuteModelRequest
31
from vllm.transformers_utils.tokenizer import AnyTokenizer
32
from vllm.usage.usage_lib import UsageContext
33
-from vllm.utils import print_warning_once
33
34
logger = init_logger(__name__)
35
ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -590,9 +589,6 @@ class AsyncLLMEngine:
589
worker_use_ray: Whether to use Ray for model workers. Required for
590
distributed execution. Should be the same as
591
`parallel_config.worker_use_ray`.
593
- engine_use_ray: Whether to make LLMEngine a Ray actor. If so, the
594
- async frontend will be executed in a separate process as the
595
- model workers.
592
log_requests: Whether to log the requests.
593
start_engine_loop: If True, the background task to run the engine
594
will be automatically started in the generate call.
@@ -604,41 +600,23 @@ class AsyncLLMEngine:
600
601
def __init__(self,
602
worker_use_ray: bool,
607
- engine_use_ray: bool,
603
*args,
604
log_requests: bool = True,
605
start_engine_loop: bool = True,
606
**kwargs) -> None:
607
self.worker_use_ray = worker_use_ray
613
- self.engine_use_ray = engine_use_ray
608
self.log_requests = log_requests
615
- self.engine = self._init_engine(*args, **kwargs)
609
+ self.engine = self._engine_class(*args, **kwargs)
610
611
# This ensures quick processing of request outputs
612
# so the append to asyncio queues is not delayed,
613
# especially for multi-step.
614
#
621
- # TODO: Currently, disabled for engine_use_ray, ask
622
- # Cody/Will/Woosuk about this case.
623
- self.use_process_request_outputs_callback = not self.engine_use_ray
615
+ self.use_process_request_outputs_callback = True
616
if self.use_process_request_outputs_callback:
617
self.engine.process_request_outputs_callback = \
618
self.process_request_outputs
619
628
- if self.engine_use_ray:
629
- print_warning_once(
630
- "DEPRECATED. `--engine-use-ray` is deprecated and will "
631
- "be removed in a future update. "
632
- "See https://github.com/vllm-project/vllm/issues/7045.")
633
-
634
- if envs.VLLM_ALLOW_ENGINE_USE_RAY:
635
- print_warning_once(
636
- "VLLM_ALLOW_ENGINE_USE_RAY is set, force engine use Ray")
637
- else:
638
- raise ValueError("`--engine-use-ray` is deprecated. "
639
- "Set `VLLM_ALLOW_ENGINE_USE_RAY=1` to "
640
- "force use it")
641
-
620
self.background_loop: Optional[asyncio.Future] = None
621
# We need to keep a reference to unshielded
622
# task as well to prevent it from being garbage
@@ -725,16 +703,11 @@ class AsyncLLMEngine:
703
# Create the engine configs.
704
engine_config = engine_args.create_engine_config()
705
728
- if engine_args.engine_use_ray:
729
- from vllm.executor import ray_utils
730
- ray_utils.assert_ray_available()
731
-
706
executor_class = cls._get_executor_cls(engine_config)
707
708
# Create the async LLM engine.
709
engine = cls(
710
executor_class.uses_ray,
737
- engine_args.engine_use_ray,
711
**engine_config.to_dict(),
712
executor_class=executor_class,
713
log_requests=not engine_args.disable_log_requests,
@@ -777,10 +750,6 @@ class AsyncLLMEngine:
750
self,
751
lora_request: Optional[LoRARequest] = None,
752
) -> AnyTokenizer:
780
- if self.engine_use_ray:
781
- return await self.engine.get_tokenizer.remote( # type: ignore
782
- lora_request)
783
-
753
return await (self.engine.get_tokenizer_group().
754
get_lora_tokenizer_async(lora_request))
755
@@ -814,26 +783,6 @@ class AsyncLLMEngine:
783
self._background_loop_unshielded = None
784
self.background_loop = None
785
817
- def _init_engine(self, *args,
818
- **kwargs) -> Union[_AsyncLLMEngine, "ray.ObjectRef"]:
819
- if not self.engine_use_ray:
820
- engine_class = self._engine_class
821
- elif self.worker_use_ray:
822
- engine_class = ray.remote(num_cpus=0)(self._engine_class).remote
823
- else:
824
- # FIXME(woosuk): This is a bit hacky. Be careful when changing the
825
- # order of the arguments.
826
- cache_config = kwargs["cache_config"]
827
- parallel_config = kwargs["parallel_config"]
828
- if (parallel_config.tensor_parallel_size == 1
829
- and parallel_config.pipeline_parallel_size == 1):
830
- num_gpus = cache_config.gpu_memory_utilization
831
- else:
832
- num_gpus = 1
833
- engine_class = ray.remote(num_gpus=num_gpus)(
834
- self._engine_class).remote
835
- return engine_class(*args, **kwargs)
836
-
786
async def engine_step(self, virtual_engine: int) -> bool:
787
"""Kick the engine to process the waiting requests.
788
@@ -844,13 +793,8 @@ class AsyncLLMEngine:
793
794
for new_request in new_requests:
795
# Add the request into the vLLM engine's waiting queue.
847
- # TODO: Maybe add add_request_batch to reduce Ray overhead
796
try:
849
- if self.engine_use_ray:
850
- await self.engine.add_request.remote( # type: ignore
851
- **new_request)
852
- else:
853
- await self.engine.add_request_async(**new_request)
797
+ await self.engine.add_request_async(**new_request)
798
except ValueError as e:
799
# TODO: use a vLLM specific error for failed validation
800
self._request_tracker.process_exception(
@@ -862,10 +806,7 @@ class AsyncLLMEngine:
806
if aborted_requests:
807
await self._engine_abort(aborted_requests)
808
865
- if self.engine_use_ray:
866
- request_outputs = await self.engine.step.remote() # type: ignore
867
- else:
868
- request_outputs = await self.engine.step_async(virtual_engine)
809
+ request_outputs = await self.engine.step_async(virtual_engine)
810
811
# Put the outputs into the corresponding streams.
812
# If used as a callback, then already invoked inside
@@ -891,16 +832,10 @@ class AsyncLLMEngine:
832
return all_finished
833
834
async def _engine_abort(self, request_ids: Iterable[str]):
894
- if self.engine_use_ray:
895
- await self.engine.abort_request.remote(request_ids) # type: ignore
896
- else:
897
- self.engine.abort_request(request_ids)
835
+ self.engine.abort_request(request_ids)
836
837
async def run_engine_loop(self):
900
- if self.engine_use_ray:
901
- pipeline_parallel_size = 1 # type: ignore
902
- else:
903
- pipeline_parallel_size = \
838
+ pipeline_parallel_size = \
839
self.engine.parallel_config.pipeline_parallel_size
840
has_requests_in_progress = [False] * pipeline_parallel_size
841
while True:
@@ -912,12 +847,7 @@ class AsyncLLMEngine:
847
# timeout, and unblocks the RPC thread in the workers so that
848
# they can process any other queued control plane messages,
849
# such as add/remove lora adapters.
915
- if self.engine_use_ray:
916
- await (self.engine.stop_remote_worker_execution_loop.
917
- remote() # type: ignore
918
- )
919
- else:
920
- await self.engine.stop_remote_worker_execution_loop_async()
850
+ await self.engine.stop_remote_worker_execution_loop_async()
851
await self._request_tracker.wait_for_new_requests()
852
logger.debug("Got new requests!")
853
requests_in_progress = [
@@ -938,17 +868,9 @@ class AsyncLLMEngine:
868
for task in done:
869
result = task.result()
870
virtual_engine = requests_in_progress.index(task)
941
- if self.engine_use_ray:
942
- has_unfinished_requests = (
943
- await (self.engine.
944
- has_unfinished_requests_for_virtual_engine.
945
- remote( # type: ignore
946
- virtual_engine)))
947
- else:
948
- has_unfinished_requests = (
949
- self.engine.
950
- has_unfinished_requests_for_virtual_engine(
951
- virtual_engine))
871
+ has_unfinished_requests = (
872
+ self.engine.has_unfinished_requests_for_virtual_engine(
873
+ virtual_engine))
874
if result or has_unfinished_requests:
875
requests_in_progress[virtual_engine] = (
876
asyncio.create_task(
@@ -1190,52 +1112,29 @@ class AsyncLLMEngine:
1112
1113
async def get_model_config(self) -> ModelConfig:
1114
"""Get the model configuration of the vLLM engine."""
1193
- if self.engine_use_ray:
1194
- return await self.engine.get_model_config.remote() # type: ignore
1195
- else:
1196
- return self.engine.get_model_config()
1115
+ return self.engine.get_model_config()
1116
1117
async def get_parallel_config(self) -> ParallelConfig:
1118
"""Get the parallel configuration of the vLLM engine."""
1200
- if self.engine_use_ray:
1201
- return await self.engine.get_parallel_config.remote( # type: ignore
1202
- )
1203
- else:
1204
- return self.engine.get_parallel_config()
1119
+ return self.engine.get_parallel_config()
1120
1121
async def get_decoding_config(self) -> DecodingConfig:
1122
"""Get the decoding configuration of the vLLM engine."""
1208
- if self.engine_use_ray:
1209
- return await self.engine.get_decoding_config.remote( # type: ignore
1210
- )
1211
- else:
1212
- return self.engine.get_decoding_config()
1123
+ return self.engine.get_decoding_config()
1124
1125
async def get_scheduler_config(self) -> SchedulerConfig:
1126
"""Get the scheduling configuration of the vLLM engine."""
1216
- if self.engine_use_ray:
1217
- return await self.engine.get_scheduler_config.remote( # type: ignore
1218
- )
1219
- else:
1220
- return self.engine.get_scheduler_config()
1127
+ return self.engine.get_scheduler_config()
1128
1129
async def get_lora_config(self) -> LoRAConfig:
1130
"""Get the lora configuration of the vLLM engine."""
1224
- if self.engine_use_ray:
1225
- return await self.engine.get_lora_config.remote( # type: ignore
1226
- )
1227
- else:
1228
- return self.engine.get_lora_config()
1131
+ return self.engine.get_lora_config()
1132
1133
async def do_log_stats(
1134
self,
1135
scheduler_outputs: Optional[SchedulerOutputs] = None,
1136
model_output: Optional[List[SamplerOutput]] = None) -> None:
1234
- if self.engine_use_ray:
1235
- await self.engine.do_log_stats.remote( # type: ignore
1236
- scheduler_outputs, model_output)
1237
- else:
1238
- self.engine.do_log_stats()
1137
+ self.engine.do_log_stats()
1138
1139
async def check_health(self) -> None:
1140
"""Raises an error if engine is unhealthy."""
@@ -1244,37 +1143,17 @@ class AsyncLLMEngine:
1143
if self.is_stopped:
1144
raise AsyncEngineDeadError("Background loop is stopped.")
1145
1247
- if self.engine_use_ray:
1248
- try:
1249
- await self.engine.check_health.remote() # type: ignore
1250
- except ray.exceptions.RayActorError as e:
1251
- raise RuntimeError("Engine is dead.") from e
1252
- else:
1253
- await self.engine.check_health_async()
1146
+ await self.engine.check_health_async()
1147
logger.debug("Health check took %fs", time.perf_counter() - t)
1148
1149
async def is_tracing_enabled(self) -> bool:
1257
- if self.engine_use_ray:
1258
- return await self.engine.is_tracing_enabled.remote( # type: ignore
1259
- )
1260
- else:
1261
- return self.engine.is_tracing_enabled()
1150
+ return self.engine.is_tracing_enabled()
1151
1152
def add_logger(self, logger_name: str, logger: StatLoggerBase) -> None:
1264
- if self.engine_use_ray:
1265
- ray.get(
1266
- self.engine.add_logger.remote( # type: ignore
1267
- logger_name=logger_name, logger=logger))
1268
- else:
1269
- self.engine.add_logger(logger_name=logger_name, logger=logger)
1153
+ self.engine.add_logger(logger_name=logger_name, logger=logger)
1154
1155
def remove_logger(self, logger_name: str) -> None:
1272
- if self.engine_use_ray:
1273
- ray.get(
1274
- self.engine.remove_logger.remote( # type: ignore
1275
- logger_name=logger_name))
1276
- else:
1277
- self.engine.remove_logger(logger_name=logger_name)
1156
+ self.engine.remove_logger(logger_name=logger_name)
1157
1158
async def start_profile(self) -> None:
1159
self.engine.model_executor._run_workers("start_profile")
vllm/engine/llm_engine.py
+3
-3
@@ -3,8 +3,8 @@ import time
3
from collections import deque
4
from contextlib import contextmanager
5
from dataclasses import dataclass
6
-from typing import (TYPE_CHECKING, Any, ClassVar, Deque, Dict, Iterable, List,
7
- Mapping, NamedTuple, Optional)
6
+from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
7
+ Iterable, List, Mapping, NamedTuple, Optional)
8
from typing import Sequence as GenericSequence
9
from typing import Set, Tuple, Type, Union
10
@@ -397,7 +397,7 @@ class LLMEngine:
397
398
# Currently used by AsyncLLMEngine to ensure quick append
399
# of request outputs to asyncio queues
400
- self.process_request_outputs_callback = None
400
+ self.process_request_outputs_callback: Optional[Callable] = None
401
402
# Create the scheduler.
403
# NOTE: the cache_config here have been updated with the numbers of
vllm/entrypoints/openai/run_batch.py
-1
@@ -195,7 +195,6 @@ async def main(args):
195
engine = AsyncLLMEngine.from_engine_args(
196
engine_args, usage_context=UsageContext.OPENAI_BATCH_RUNNER)
197
198
- # When using single vLLM without engine_use_ray
198
model_config = await engine.get_model_config()
199
200
if args.disable_log_requests:
vllm/envs.py
-9
@@ -58,7 +58,6 @@ if TYPE_CHECKING:
58
VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
59
VLLM_TEST_FORCE_FP8_MARLIN: bool = False
60
VLLM_RPC_GET_DATA_TIMEOUT_MS: int = 5000
61
- VLLM_ALLOW_ENGINE_USE_RAY: bool = False
61
VLLM_PLUGINS: Optional[List[str]] = None
62
VLLM_TORCH_PROFILER_DIR: Optional[str] = None
63
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
@@ -391,14 +390,6 @@ environment_variables: Dict[str, Callable[[], Any]] = {
390
"VLLM_RPC_GET_DATA_TIMEOUT_MS":
391
lambda: int(os.getenv("VLLM_RPC_GET_DATA_TIMEOUT_MS", "5000")),
392
394
- # If set, allow running the engine as a separate ray actor,
395
- # which is a deprecated feature soon to be removed.
396
- # See https://github.com/vllm-project/vllm/issues/7045
397
- "VLLM_ALLOW_ENGINE_USE_RAY":
398
- lambda:
399
- (os.environ.get("VLLM_ALLOW_ENGINE_USE_RAY", "0").strip().lower() in
400
- ("1", "true")),
401
-
393
# a list of plugin names to load, separated by commas.
394
# if this is not set, it means all plugins will be loaded
395
# if this is set to an empty string, no plugins will be loaded