@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 70b746efcf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Misc] Deprecation Warning when setting --engine-use-ray (#7424)

Signed-off-by: Wallas Santos <wallashss@ibm.com> Co-authored-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com> Co-authored-by: youkaichao <youkaichao@126.com>

Wallas Henrique committed Aug 14, 2024 at 13:44 UTC 70b746efcf9a3ddcbe5afd34d8efa23671c1569e
7 files changed +56 -3
tests/async_engine/test_api_server.py
+8 -1
@@ -1,3 +1,4 @@
1 +import os
2 import subprocess
3 import sys
4 import time
@@ -35,11 +36,17 @@ def api_server(tokenizer_pool_size: int, engine_use_ray: bool,
36 "127.0.0.1", "--tokenizer-pool-size",
37 str(tokenizer_pool_size)
38 ]
39 +
40 + # Copy the environment variables and append `VLLM_ALLOW_ENGINE_USE_RAY=1`
41 + # to prevent `--engine-use-ray` raises an exception due to it deprecation
42 + env_vars = os.environ.copy()
43 + env_vars["VLLM_ALLOW_ENGINE_USE_RAY"] = "1"
44 +
45 if engine_use_ray:
46 commands.append("--engine-use-ray")
47 if worker_use_ray:
48 commands.append("--worker-use-ray")
42 - uvicorn_process = subprocess.Popen(commands)
49 + uvicorn_process = subprocess.Popen(commands, env=env_vars)
50 yield
51 uvicorn_process.terminate()
52
tests/async_engine/test_async_llm_engine.py
+6
@@ -1,4 +1,5 @@
1 import asyncio
2 +import os
3 from dataclasses import dataclass
4
5 import pytest
@@ -106,11 +107,16 @@ async def test_new_requests_event():
107 assert engine.engine.add_request_calls == 3
108 assert engine.engine.step_calls == old_step_calls + 1
109
110 + # Allow deprecated engine_use_ray to not raise exception
111 + os.environ["VLLM_ALLOW_ENGINE_USE_RAY"] = "1"
112 +
113 engine = MockAsyncLLMEngine(worker_use_ray=True, engine_use_ray=True)
114 assert engine.get_model_config() is not None
115 assert engine.get_tokenizer() is not None
116 assert engine.get_decoding_config() is not None
117
118 + os.environ.pop("VLLM_ALLOW_ENGINE_USE_RAY")
119 +
120
121 def test_asyncio_run():
122 wait_for_gpu_memory_to_clear(
tests/async_engine/test_openapi_server_ray.py
+5 -1
@@ -23,7 +23,11 @@ def server():
23 str(chatml_jinja_path),
24 ]
25
26 - with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
26 + # Allow `--engine-use-ray`, otherwise the launch of the server throw
27 + # an error due to try to use a deprecated feature
28 + env_dict = {"VLLM_ALLOW_ENGINE_USE_RAY": "1"}
29 + with RemoteOpenAIServer(MODEL_NAME, args,
30 + env_dict=env_dict) as remote_server:
31 yield remote_server
32
33
tests/spec_decode/e2e/conftest.py
+6
@@ -1,4 +1,5 @@
1 import asyncio
2 +import os
3 from itertools import cycle
4 from typing import Dict, List, Optional, Sequence, Tuple, Union
5
@@ -56,6 +57,11 @@ class AsyncLLM:
57 ) -> None:
58 if "disable_log_stats" not in kwargs:
59 kwargs["disable_log_stats"] = True
60 +
61 + # Needed to engine_use_ray works as a deprecated feature,
62 + # otherwise the following constructor will raise an exception
63 + os.environ["VLLM_ALLOW_ENGINE_USE_RAY"] = "1"
64 +
65 engine_args = AsyncEngineArgs(
66 model=model,
67 tokenizer=tokenizer,
vllm/engine/arg_utils.py
+7 -1
@@ -923,7 +923,13 @@ class AsyncEngineArgs(EngineArgs):
923 parser.add_argument('--engine-use-ray',
924 action='store_true',
925 help='Use Ray to start the LLM engine in a '
926 - 'separate process as the server process.')
926 + 'separate process as the server process.'
927 + '(DEPRECATED. This argument is deprecated '
928 + 'and will be removed in a future update. '
929 + 'Set `VLLM_ALLOW_ENGINE_USE_RAY=1` to force '
930 + 'use it. See '
931 + 'https://github.com/vllm-project/vllm/issues/7045.'
932 + ')')
933 parser.add_argument('--disable-log-requests',
934 action='store_true',
935 help='Disable logging requests.')
vllm/engine/async_llm_engine.py
+15
@@ -29,6 +29,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
29 from vllm.sampling_params import SamplingParams
30 from vllm.sequence import ExecuteModelRequest, SamplerOutput
31 from vllm.usage.usage_lib import UsageContext
32 +from vllm.utils import print_warning_once
33
34 logger = init_logger(__name__)
35 ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -510,6 +511,20 @@ class AsyncLLMEngine:
511 self.log_requests = log_requests
512 self.engine = self._init_engine(*args, **kwargs)
513
514 + if self.engine_use_ray:
515 + print_warning_once(
516 + "DEPRECATED. `--engine-use-ray` is deprecated and will "
517 + "be removed in a future update. "
518 + "See https://github.com/vllm-project/vllm/issues/7045.")
519 +
520 + if envs.VLLM_ALLOW_ENGINE_USE_RAY:
521 + print_warning_once(
522 + "VLLM_ALLOW_ENGINE_USE_RAY is set, force engine use Ray")
523 + else:
524 + raise ValueError("`--engine-use-ray` is deprecated. "
525 + "Set `VLLM_ALLOW_ENGINE_USE_RAY=1` to "
526 + "force use it")
527 +
528 self.background_loop: Optional[asyncio.Future] = None
529 # We need to keep a reference to unshielded
530 # task as well to prevent it from being garbage
vllm/envs.py
+9
@@ -55,6 +55,7 @@ if TYPE_CHECKING:
55 VERBOSE: bool = False
56 VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
57 VLLM_TEST_FORCE_FP8_MARLIN: bool = False
58 + VLLM_ALLOW_ENGINE_USE_RAY: bool = False
59 VLLM_PLUGINS: Optional[List[str]] = None
60
61
@@ -364,6 +365,14 @@ environment_variables: Dict[str, Callable[[], Any]] = {
365 (os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower() in
366 ("1", "true")),
367
368 + # If set, allow running the engine as a separate ray actor,
369 + # which is a deprecated feature soon to be removed.
370 + # See https://github.com/vllm-project/vllm/issues/7045
371 + "VLLM_ALLOW_ENGINE_USE_RAY":
372 + lambda:
373 + (os.environ.get("VLLM_ALLOW_ENGINE_USE_RAY", "0").strip().lower() in
374 + ("1", "true")),
375 +
376 # a list of plugin names to load, separated by commas.
377 # if this is not set, it means all plugins will be loaded
378 # if this is set to an empty string, no plugins will be loaded