@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f842a7aff1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[misc] remove engine_use_ray (#8126)

youkaichao committed Sep 11, 2024 at 18:23 UTC f842a7aff143a4a1ddc59e1fb57109cb377f5475
8 files changed +32 -197
tests/async_engine/test_api_server.py
+4 -14
@@ -1,4 +1,3 @@
1 -import os
1 import subprocess
2 import sys
3 import time
@@ -26,8 +25,7 @@ def _query_server_long(prompt: str) -> dict:
25
26
27 @pytest.fixture
29 -def api_server(tokenizer_pool_size: int, engine_use_ray: bool,
30 - worker_use_ray: bool):
28 +def api_server(tokenizer_pool_size: int, worker_use_ray: bool):
29 script_path = Path(__file__).parent.joinpath(
30 "api_server_async_engine.py").absolute()
31 commands = [
@@ -37,25 +35,17 @@ def api_server(tokenizer_pool_size: int, engine_use_ray: bool,
35 str(tokenizer_pool_size)
36 ]
37
40 - # Copy the environment variables and append `VLLM_ALLOW_ENGINE_USE_RAY=1`
41 - # to prevent `--engine-use-ray` raises an exception due to it deprecation
42 - env_vars = os.environ.copy()
43 - env_vars["VLLM_ALLOW_ENGINE_USE_RAY"] = "1"
44 -
45 - if engine_use_ray:
46 - commands.append("--engine-use-ray")
38 if worker_use_ray:
39 commands.append("--worker-use-ray")
49 - uvicorn_process = subprocess.Popen(commands, env=env_vars)
40 + uvicorn_process = subprocess.Popen(commands)
41 yield
42 uvicorn_process.terminate()
43
44
45 @pytest.mark.parametrize("tokenizer_pool_size", [0, 2])
46 @pytest.mark.parametrize("worker_use_ray", [False, True])
56 -@pytest.mark.parametrize("engine_use_ray", [False, True])
57 -def test_api_server(api_server, tokenizer_pool_size: int, worker_use_ray: bool,
58 - engine_use_ray: bool):
47 +def test_api_server(api_server, tokenizer_pool_size: int,
48 + worker_use_ray: bool):
49 """
50 Run the API server and test it.
51
tests/async_engine/test_async_llm_engine.py
+3 -11
@@ -1,5 +1,4 @@
1 import asyncio
2 -import os
2 from asyncio import CancelledError
3 from dataclasses import dataclass
4 from typing import Optional
@@ -72,14 +71,12 @@ class MockEngine:
71
72
73 class MockAsyncLLMEngine(AsyncLLMEngine):
75 -
76 - def _init_engine(self, *args, **kwargs):
77 - return MockEngine()
74 + _engine_class = MockEngine
75
76
77 @pytest.mark.asyncio
78 async def test_new_requests_event():
82 - engine = MockAsyncLLMEngine(worker_use_ray=False, engine_use_ray=False)
79 + engine = MockAsyncLLMEngine(worker_use_ray=False)
80 engine.start_background_loop()
81 await asyncio.sleep(0.01)
82 assert engine.engine.step_calls == 0
@@ -112,16 +109,11 @@ async def test_new_requests_event():
109 assert engine.engine.add_request_calls == 3
110 assert engine.engine.step_calls == old_step_calls + 1
111
115 - # Allow deprecated engine_use_ray to not raise exception
116 - os.environ["VLLM_ALLOW_ENGINE_USE_RAY"] = "1"
117 -
118 - engine = MockAsyncLLMEngine(worker_use_ray=True, engine_use_ray=True)
112 + engine = MockAsyncLLMEngine(worker_use_ray=True)
113 assert engine.get_model_config() is not None
114 assert engine.get_tokenizer() is not None
115 assert engine.get_decoding_config() is not None
116
123 - os.environ.pop("VLLM_ALLOW_ENGINE_USE_RAY")
124 -
117
118 def start_engine():
119 wait_for_gpu_memory_to_clear(
tests/async_engine/test_openapi_server.py renamed
+1 -6
@@ -19,16 +19,11 @@ def server():
19 "--max-model-len",
20 "2048",
21 "--enforce-eager",
22 - "--engine-use-ray",
22 "--chat-template",
23 str(chatml_jinja_path),
24 ]
25
27 - # Allow `--engine-use-ray`, otherwise the launch of the server throw
28 - # an error due to try to use a deprecated feature
29 - env_dict = {"VLLM_ALLOW_ENGINE_USE_RAY": "1"}
30 - with RemoteOpenAIServer(MODEL_NAME, args,
31 - env_dict=env_dict) as remote_server:
26 + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
27 yield remote_server
28
29
vllm/engine/arg_utils.py
-11
@@ -1035,7 +1035,6 @@ class EngineArgs:
1035 @dataclass
1036 class AsyncEngineArgs(EngineArgs):
1037 """Arguments for asynchronous vLLM engine."""
1038 - engine_use_ray: bool = False
1038 disable_log_requests: bool = False
1039
1040 @staticmethod
@@ -1043,16 +1042,6 @@ class AsyncEngineArgs(EngineArgs):
1042 async_args_only: bool = False) -> FlexibleArgumentParser:
1043 if not async_args_only:
1044 parser = EngineArgs.add_cli_args(parser)
1046 - parser.add_argument('--engine-use-ray',
1047 - action='store_true',
1048 - help='Use Ray to start the LLM engine in a '
1049 - 'separate process as the server process.'
1050 - '(DEPRECATED. This argument is deprecated '
1051 - 'and will be removed in a future update. '
1052 - 'Set `VLLM_ALLOW_ENGINE_USE_RAY=1` to force '
1053 - 'use it. See '
1054 - 'https://github.com/vllm-project/vllm/issues/7045.'
1055 - ')')
1045 parser.add_argument('--disable-log-requests',
1046 action='store_true',
1047 help='Disable logging requests.')
vllm/engine/async_llm_engine.py
+21 -142
@@ -16,7 +16,7 @@ from vllm.engine.llm_engine import (DecoderPromptComponents, LLMEngine,
16 PromptComponents, SchedulerOutputState)
17 from vllm.engine.metrics_types import StatLoggerBase
18 from vllm.executor.executor_base import ExecutorAsyncBase
19 -from vllm.executor.ray_utils import initialize_ray_cluster, ray
19 +from vllm.executor.ray_utils import initialize_ray_cluster
20 from vllm.inputs import (EncoderDecoderLLMInputs, LLMInputs, PromptInputs,
21 SingletonPromptInputs)
22 from vllm.inputs.parse import is_explicit_encoder_decoder_prompt
@@ -30,7 +30,6 @@ from vllm.sampling_params import SamplingParams
30 from vllm.sequence import ExecuteModelRequest
31 from vllm.transformers_utils.tokenizer import AnyTokenizer
32 from vllm.usage.usage_lib import UsageContext
33 -from vllm.utils import print_warning_once
33
34 logger = init_logger(__name__)
35 ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -590,9 +589,6 @@ class AsyncLLMEngine:
589 worker_use_ray: Whether to use Ray for model workers. Required for
590 distributed execution. Should be the same as
591 `parallel_config.worker_use_ray`.
593 - engine_use_ray: Whether to make LLMEngine a Ray actor. If so, the
594 - async frontend will be executed in a separate process as the
595 - model workers.
592 log_requests: Whether to log the requests.
593 start_engine_loop: If True, the background task to run the engine
594 will be automatically started in the generate call.
@@ -604,41 +600,23 @@ class AsyncLLMEngine:
600
601 def __init__(self,
602 worker_use_ray: bool,
607 - engine_use_ray: bool,
603 *args,
604 log_requests: bool = True,
605 start_engine_loop: bool = True,
606 **kwargs) -> None:
607 self.worker_use_ray = worker_use_ray
613 - self.engine_use_ray = engine_use_ray
608 self.log_requests = log_requests
615 - self.engine = self._init_engine(*args, **kwargs)
609 + self.engine = self._engine_class(*args, **kwargs)
610
611 # This ensures quick processing of request outputs
612 # so the append to asyncio queues is not delayed,
613 # especially for multi-step.
614 #
621 - # TODO: Currently, disabled for engine_use_ray, ask
622 - # Cody/Will/Woosuk about this case.
623 - self.use_process_request_outputs_callback = not self.engine_use_ray
615 + self.use_process_request_outputs_callback = True
616 if self.use_process_request_outputs_callback:
617 self.engine.process_request_outputs_callback = \
618 self.process_request_outputs
619
628 - if self.engine_use_ray:
629 - print_warning_once(
630 - "DEPRECATED. `--engine-use-ray` is deprecated and will "
631 - "be removed in a future update. "
632 - "See https://github.com/vllm-project/vllm/issues/7045.")
633 -
634 - if envs.VLLM_ALLOW_ENGINE_USE_RAY:
635 - print_warning_once(
636 - "VLLM_ALLOW_ENGINE_USE_RAY is set, force engine use Ray")
637 - else:
638 - raise ValueError("`--engine-use-ray` is deprecated. "
639 - "Set `VLLM_ALLOW_ENGINE_USE_RAY=1` to "
640 - "force use it")
641 -
620 self.background_loop: Optional[asyncio.Future] = None
621 # We need to keep a reference to unshielded
622 # task as well to prevent it from being garbage
@@ -725,16 +703,11 @@ class AsyncLLMEngine:
703 # Create the engine configs.
704 engine_config = engine_args.create_engine_config()
705
728 - if engine_args.engine_use_ray:
729 - from vllm.executor import ray_utils
730 - ray_utils.assert_ray_available()
731 -
706 executor_class = cls._get_executor_cls(engine_config)
707
708 # Create the async LLM engine.
709 engine = cls(
710 executor_class.uses_ray,
737 - engine_args.engine_use_ray,
711 **engine_config.to_dict(),
712 executor_class=executor_class,
713 log_requests=not engine_args.disable_log_requests,
@@ -777,10 +750,6 @@ class AsyncLLMEngine:
750 self,
751 lora_request: Optional[LoRARequest] = None,
752 ) -> AnyTokenizer:
780 - if self.engine_use_ray:
781 - return await self.engine.get_tokenizer.remote( # type: ignore
782 - lora_request)
783 -
753 return await (self.engine.get_tokenizer_group().
754 get_lora_tokenizer_async(lora_request))
755
@@ -814,26 +783,6 @@ class AsyncLLMEngine:
783 self._background_loop_unshielded = None
784 self.background_loop = None
785
817 - def _init_engine(self, *args,
818 - **kwargs) -> Union[_AsyncLLMEngine, "ray.ObjectRef"]:
819 - if not self.engine_use_ray:
820 - engine_class = self._engine_class
821 - elif self.worker_use_ray:
822 - engine_class = ray.remote(num_cpus=0)(self._engine_class).remote
823 - else:
824 - # FIXME(woosuk): This is a bit hacky. Be careful when changing the
825 - # order of the arguments.
826 - cache_config = kwargs["cache_config"]
827 - parallel_config = kwargs["parallel_config"]
828 - if (parallel_config.tensor_parallel_size == 1
829 - and parallel_config.pipeline_parallel_size == 1):
830 - num_gpus = cache_config.gpu_memory_utilization
831 - else:
832 - num_gpus = 1
833 - engine_class = ray.remote(num_gpus=num_gpus)(
834 - self._engine_class).remote
835 - return engine_class(*args, **kwargs)
836 -
786 async def engine_step(self, virtual_engine: int) -> bool:
787 """Kick the engine to process the waiting requests.
788
@@ -844,13 +793,8 @@ class AsyncLLMEngine:
793
794 for new_request in new_requests:
795 # Add the request into the vLLM engine's waiting queue.
847 - # TODO: Maybe add add_request_batch to reduce Ray overhead
796 try:
849 - if self.engine_use_ray:
850 - await self.engine.add_request.remote( # type: ignore
851 - **new_request)
852 - else:
853 - await self.engine.add_request_async(**new_request)
797 + await self.engine.add_request_async(**new_request)
798 except ValueError as e:
799 # TODO: use a vLLM specific error for failed validation
800 self._request_tracker.process_exception(
@@ -862,10 +806,7 @@ class AsyncLLMEngine:
806 if aborted_requests:
807 await self._engine_abort(aborted_requests)
808
865 - if self.engine_use_ray:
866 - request_outputs = await self.engine.step.remote() # type: ignore
867 - else:
868 - request_outputs = await self.engine.step_async(virtual_engine)
809 + request_outputs = await self.engine.step_async(virtual_engine)
810
811 # Put the outputs into the corresponding streams.
812 # If used as a callback, then already invoked inside
@@ -891,16 +832,10 @@ class AsyncLLMEngine:
832 return all_finished
833
834 async def _engine_abort(self, request_ids: Iterable[str]):
894 - if self.engine_use_ray:
895 - await self.engine.abort_request.remote(request_ids) # type: ignore
896 - else:
897 - self.engine.abort_request(request_ids)
835 + self.engine.abort_request(request_ids)
836
837 async def run_engine_loop(self):
900 - if self.engine_use_ray:
901 - pipeline_parallel_size = 1 # type: ignore
902 - else:
903 - pipeline_parallel_size = \
838 + pipeline_parallel_size = \
839 self.engine.parallel_config.pipeline_parallel_size
840 has_requests_in_progress = [False] * pipeline_parallel_size
841 while True:
@@ -912,12 +847,7 @@ class AsyncLLMEngine:
847 # timeout, and unblocks the RPC thread in the workers so that
848 # they can process any other queued control plane messages,
849 # such as add/remove lora adapters.
915 - if self.engine_use_ray:
916 - await (self.engine.stop_remote_worker_execution_loop.
917 - remote() # type: ignore
918 - )
919 - else:
920 - await self.engine.stop_remote_worker_execution_loop_async()
850 + await self.engine.stop_remote_worker_execution_loop_async()
851 await self._request_tracker.wait_for_new_requests()
852 logger.debug("Got new requests!")
853 requests_in_progress = [
@@ -938,17 +868,9 @@ class AsyncLLMEngine:
868 for task in done:
869 result = task.result()
870 virtual_engine = requests_in_progress.index(task)
941 - if self.engine_use_ray:
942 - has_unfinished_requests = (
943 - await (self.engine.
944 - has_unfinished_requests_for_virtual_engine.
945 - remote( # type: ignore
946 - virtual_engine)))
947 - else:
948 - has_unfinished_requests = (
949 - self.engine.
950 - has_unfinished_requests_for_virtual_engine(
951 - virtual_engine))
871 + has_unfinished_requests = (
872 + self.engine.has_unfinished_requests_for_virtual_engine(
873 + virtual_engine))
874 if result or has_unfinished_requests:
875 requests_in_progress[virtual_engine] = (
876 asyncio.create_task(
@@ -1190,52 +1112,29 @@ class AsyncLLMEngine:
1112
1113 async def get_model_config(self) -> ModelConfig:
1114 """Get the model configuration of the vLLM engine."""
1193 - if self.engine_use_ray:
1194 - return await self.engine.get_model_config.remote() # type: ignore
1195 - else:
1196 - return self.engine.get_model_config()
1115 + return self.engine.get_model_config()
1116
1117 async def get_parallel_config(self) -> ParallelConfig:
1118 """Get the parallel configuration of the vLLM engine."""
1200 - if self.engine_use_ray:
1201 - return await self.engine.get_parallel_config.remote( # type: ignore
1202 - )
1203 - else:
1204 - return self.engine.get_parallel_config()
1119 + return self.engine.get_parallel_config()
1120
1121 async def get_decoding_config(self) -> DecodingConfig:
1122 """Get the decoding configuration of the vLLM engine."""
1208 - if self.engine_use_ray:
1209 - return await self.engine.get_decoding_config.remote( # type: ignore
1210 - )
1211 - else:
1212 - return self.engine.get_decoding_config()
1123 + return self.engine.get_decoding_config()
1124
1125 async def get_scheduler_config(self) -> SchedulerConfig:
1126 """Get the scheduling configuration of the vLLM engine."""
1216 - if self.engine_use_ray:
1217 - return await self.engine.get_scheduler_config.remote( # type: ignore
1218 - )
1219 - else:
1220 - return self.engine.get_scheduler_config()
1127 + return self.engine.get_scheduler_config()
1128
1129 async def get_lora_config(self) -> LoRAConfig:
1130 """Get the lora configuration of the vLLM engine."""
1224 - if self.engine_use_ray:
1225 - return await self.engine.get_lora_config.remote( # type: ignore
1226 - )
1227 - else:
1228 - return self.engine.get_lora_config()
1131 + return self.engine.get_lora_config()
1132
1133 async def do_log_stats(
1134 self,
1135 scheduler_outputs: Optional[SchedulerOutputs] = None,
1136 model_output: Optional[List[SamplerOutput]] = None) -> None:
1234 - if self.engine_use_ray:
1235 - await self.engine.do_log_stats.remote( # type: ignore
1236 - scheduler_outputs, model_output)
1237 - else:
1238 - self.engine.do_log_stats()
1137 + self.engine.do_log_stats()
1138
1139 async def check_health(self) -> None:
1140 """Raises an error if engine is unhealthy."""
@@ -1244,37 +1143,17 @@ class AsyncLLMEngine:
1143 if self.is_stopped:
1144 raise AsyncEngineDeadError("Background loop is stopped.")
1145
1247 - if self.engine_use_ray:
1248 - try:
1249 - await self.engine.check_health.remote() # type: ignore
1250 - except ray.exceptions.RayActorError as e:
1251 - raise RuntimeError("Engine is dead.") from e
1252 - else:
1253 - await self.engine.check_health_async()
1146 + await self.engine.check_health_async()
1147 logger.debug("Health check took %fs", time.perf_counter() - t)
1148
1149 async def is_tracing_enabled(self) -> bool:
1257 - if self.engine_use_ray:
1258 - return await self.engine.is_tracing_enabled.remote( # type: ignore
1259 - )
1260 - else:
1261 - return self.engine.is_tracing_enabled()
1150 + return self.engine.is_tracing_enabled()
1151
1152 def add_logger(self, logger_name: str, logger: StatLoggerBase) -> None:
1264 - if self.engine_use_ray:
1265 - ray.get(
1266 - self.engine.add_logger.remote( # type: ignore
1267 - logger_name=logger_name, logger=logger))
1268 - else:
1269 - self.engine.add_logger(logger_name=logger_name, logger=logger)
1153 + self.engine.add_logger(logger_name=logger_name, logger=logger)
1154
1155 def remove_logger(self, logger_name: str) -> None:
1272 - if self.engine_use_ray:
1273 - ray.get(
1274 - self.engine.remove_logger.remote( # type: ignore
1275 - logger_name=logger_name))
1276 - else:
1277 - self.engine.remove_logger(logger_name=logger_name)
1156 + self.engine.remove_logger(logger_name=logger_name)
1157
1158 async def start_profile(self) -> None:
1159 self.engine.model_executor._run_workers("start_profile")
vllm/engine/llm_engine.py
+3 -3
@@ -3,8 +3,8 @@ import time
3 from collections import deque
4 from contextlib import contextmanager
5 from dataclasses import dataclass
6 -from typing import (TYPE_CHECKING, Any, ClassVar, Deque, Dict, Iterable, List,
7 - Mapping, NamedTuple, Optional)
6 +from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
7 + Iterable, List, Mapping, NamedTuple, Optional)
8 from typing import Sequence as GenericSequence
9 from typing import Set, Tuple, Type, Union
10
@@ -397,7 +397,7 @@ class LLMEngine:
397
398 # Currently used by AsyncLLMEngine to ensure quick append
399 # of request outputs to asyncio queues
400 - self.process_request_outputs_callback = None
400 + self.process_request_outputs_callback: Optional[Callable] = None
401
402 # Create the scheduler.
403 # NOTE: the cache_config here have been updated with the numbers of
vllm/entrypoints/openai/run_batch.py
-1
@@ -195,7 +195,6 @@ async def main(args):
195 engine = AsyncLLMEngine.from_engine_args(
196 engine_args, usage_context=UsageContext.OPENAI_BATCH_RUNNER)
197
198 - # When using single vLLM without engine_use_ray
198 model_config = await engine.get_model_config()
199
200 if args.disable_log_requests:
vllm/envs.py
-9
@@ -58,7 +58,6 @@ if TYPE_CHECKING:
58 VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
59 VLLM_TEST_FORCE_FP8_MARLIN: bool = False
60 VLLM_RPC_GET_DATA_TIMEOUT_MS: int = 5000
61 - VLLM_ALLOW_ENGINE_USE_RAY: bool = False
61 VLLM_PLUGINS: Optional[List[str]] = None
62 VLLM_TORCH_PROFILER_DIR: Optional[str] = None
63 VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
@@ -391,14 +390,6 @@ environment_variables: Dict[str, Callable[[], Any]] = {
390 "VLLM_RPC_GET_DATA_TIMEOUT_MS":
391 lambda: int(os.getenv("VLLM_RPC_GET_DATA_TIMEOUT_MS", "5000")),
392
394 - # If set, allow running the engine as a separate ray actor,
395 - # which is a deprecated feature soon to be removed.
396 - # See https://github.com/vllm-project/vllm/issues/7045
397 - "VLLM_ALLOW_ENGINE_USE_RAY":
398 - lambda:
399 - (os.environ.get("VLLM_ALLOW_ENGINE_USE_RAY", "0").strip().lower() in
400 - ("1", "true")),
401 -
393 # a list of plugin names to load, separated by commas.
394 # if this is not set, it means all plugins will be loaded
395 # if this is set to an empty string, no plugins will be loaded