A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Misc] Deprecation Warning when setting --engine-use-ray (#7424)
Signed-off-by: Wallas Santos <wallashss@ibm.com> Co-authored-by: youkaichao <youkaichao@gmail.com> Co-authored-by: Nick Hill <nickhill@us.ibm.com> Co-authored-by: youkaichao <youkaichao@126.com>
Wallas Henrique committed
Aug 14, 2024 at 13:44 UTC
70b746efcf9a3ddcbe5afd34d8efa23671c1569e
7 files changed
+56
-3
tests/async_engine/test_api_server.py
+8
-1
@@ -1,3 +1,4 @@
1
+import os
2
import subprocess
3
import sys
4
import time
@@ -35,11 +36,17 @@ def api_server(tokenizer_pool_size: int, engine_use_ray: bool,
36
"127.0.0.1", "--tokenizer-pool-size",
37
str(tokenizer_pool_size)
38
]
39
+
40
+ # Copy the environment variables and append `VLLM_ALLOW_ENGINE_USE_RAY=1`
41
+ # to prevent `--engine-use-ray` raises an exception due to it deprecation
42
+ env_vars = os.environ.copy()
43
+ env_vars["VLLM_ALLOW_ENGINE_USE_RAY"] = "1"
44
+
45
if engine_use_ray:
46
commands.append("--engine-use-ray")
47
if worker_use_ray:
48
commands.append("--worker-use-ray")
42
- uvicorn_process = subprocess.Popen(commands)
49
+ uvicorn_process = subprocess.Popen(commands, env=env_vars)
50
yield
51
uvicorn_process.terminate()
52
tests/async_engine/test_async_llm_engine.py
+6
@@ -1,4 +1,5 @@
1
import asyncio
2
+import os
3
from dataclasses import dataclass
4
5
import pytest
@@ -106,11 +107,16 @@ async def test_new_requests_event():
107
assert engine.engine.add_request_calls == 3
108
assert engine.engine.step_calls == old_step_calls + 1
109
110
+ # Allow deprecated engine_use_ray to not raise exception
111
+ os.environ["VLLM_ALLOW_ENGINE_USE_RAY"] = "1"
112
+
113
engine = MockAsyncLLMEngine(worker_use_ray=True, engine_use_ray=True)
114
assert engine.get_model_config() is not None
115
assert engine.get_tokenizer() is not None
116
assert engine.get_decoding_config() is not None
117
118
+ os.environ.pop("VLLM_ALLOW_ENGINE_USE_RAY")
119
+
120
121
def test_asyncio_run():
122
wait_for_gpu_memory_to_clear(
tests/async_engine/test_openapi_server_ray.py
+5
-1
@@ -23,7 +23,11 @@ def server():
23
str(chatml_jinja_path),
24
]
25
26
- with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
26
+ # Allow `--engine-use-ray`, otherwise the launch of the server throw
27
+ # an error due to try to use a deprecated feature
28
+ env_dict = {"VLLM_ALLOW_ENGINE_USE_RAY": "1"}
29
+ with RemoteOpenAIServer(MODEL_NAME, args,
30
+ env_dict=env_dict) as remote_server:
31
yield remote_server
32
33
tests/spec_decode/e2e/conftest.py
+6
@@ -1,4 +1,5 @@
1
import asyncio
2
+import os
3
from itertools import cycle
4
from typing import Dict, List, Optional, Sequence, Tuple, Union
5
@@ -56,6 +57,11 @@ class AsyncLLM:
57
) -> None:
58
if "disable_log_stats" not in kwargs:
59
kwargs["disable_log_stats"] = True
60
+
61
+ # Needed to engine_use_ray works as a deprecated feature,
62
+ # otherwise the following constructor will raise an exception
63
+ os.environ["VLLM_ALLOW_ENGINE_USE_RAY"] = "1"
64
+
65
engine_args = AsyncEngineArgs(
66
model=model,
67
tokenizer=tokenizer,
vllm/engine/arg_utils.py
+7
-1
@@ -923,7 +923,13 @@ class AsyncEngineArgs(EngineArgs):
923
parser.add_argument('--engine-use-ray',
924
action='store_true',
925
help='Use Ray to start the LLM engine in a '
926
- 'separate process as the server process.')
926
+ 'separate process as the server process.'
927
+ '(DEPRECATED. This argument is deprecated '
928
+ 'and will be removed in a future update. '
929
+ 'Set `VLLM_ALLOW_ENGINE_USE_RAY=1` to force '
930
+ 'use it. See '
931
+ 'https://github.com/vllm-project/vllm/issues/7045.'
932
+ ')')
933
parser.add_argument('--disable-log-requests',
934
action='store_true',
935
help='Disable logging requests.')
vllm/engine/async_llm_engine.py
+15
@@ -29,6 +29,7 @@ from vllm.prompt_adapter.request import PromptAdapterRequest
29
from vllm.sampling_params import SamplingParams
30
from vllm.sequence import ExecuteModelRequest, SamplerOutput
31
from vllm.usage.usage_lib import UsageContext
32
+from vllm.utils import print_warning_once
33
34
logger = init_logger(__name__)
35
ENGINE_ITERATION_TIMEOUT_S = envs.VLLM_ENGINE_ITERATION_TIMEOUT_S
@@ -510,6 +511,20 @@ class AsyncLLMEngine:
511
self.log_requests = log_requests
512
self.engine = self._init_engine(*args, **kwargs)
513
514
+ if self.engine_use_ray:
515
+ print_warning_once(
516
+ "DEPRECATED. `--engine-use-ray` is deprecated and will "
517
+ "be removed in a future update. "
518
+ "See https://github.com/vllm-project/vllm/issues/7045.")
519
+
520
+ if envs.VLLM_ALLOW_ENGINE_USE_RAY:
521
+ print_warning_once(
522
+ "VLLM_ALLOW_ENGINE_USE_RAY is set, force engine use Ray")
523
+ else:
524
+ raise ValueError("`--engine-use-ray` is deprecated. "
525
+ "Set `VLLM_ALLOW_ENGINE_USE_RAY=1` to "
526
+ "force use it")
527
+
528
self.background_loop: Optional[asyncio.Future] = None
529
# We need to keep a reference to unshielded
530
# task as well to prevent it from being garbage
vllm/envs.py
+9
@@ -55,6 +55,7 @@ if TYPE_CHECKING:
55
VERBOSE: bool = False
56
VLLM_ALLOW_LONG_MAX_MODEL_LEN: bool = False
57
VLLM_TEST_FORCE_FP8_MARLIN: bool = False
58
+ VLLM_ALLOW_ENGINE_USE_RAY: bool = False
59
VLLM_PLUGINS: Optional[List[str]] = None
60
61
@@ -364,6 +365,14 @@ environment_variables: Dict[str, Callable[[], Any]] = {
365
(os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower() in
366
("1", "true")),
367
368
+ # If set, allow running the engine as a separate ray actor,
369
+ # which is a deprecated feature soon to be removed.
370
+ # See https://github.com/vllm-project/vllm/issues/7045
371
+ "VLLM_ALLOW_ENGINE_USE_RAY":
372
+ lambda:
373
+ (os.environ.get("VLLM_ALLOW_ENGINE_USE_RAY", "0").strip().lower() in
374
+ ("1", "true")),
375
+
376
# a list of plugin names to load, separated by commas.
377
# if this is not set, it means all plugins will be loaded
378
# if this is set to an empty string, no plugins will be loaded