A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[Misc] Remove deprecated code (#12383)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 25, 2025 at 03:45 UTC
df5dafaa5ba611f7179720958ba63e49615c927f
6 files changed
+25
-78
tests/async_engine/test_api_server.py
+14
-9
@@ -25,27 +25,32 @@ def _query_server_long(prompt: str) -> dict:
25
26
27
@pytest.fixture
28
-def api_server(tokenizer_pool_size: int, worker_use_ray: bool):
28
+def api_server(tokenizer_pool_size: int, distributed_executor_backend: str):
29
script_path = Path(__file__).parent.joinpath(
30
"api_server_async_engine.py").absolute()
31
commands = [
32
- sys.executable, "-u",
33
- str(script_path), "--model", "facebook/opt-125m", "--host",
34
- "127.0.0.1", "--tokenizer-pool-size",
35
- str(tokenizer_pool_size)
32
+ sys.executable,
33
+ "-u",
34
+ str(script_path),
35
+ "--model",
36
+ "facebook/opt-125m",
37
+ "--host",
38
+ "127.0.0.1",
39
+ "--tokenizer-pool-size",
40
+ str(tokenizer_pool_size),
41
+ "--distributed-executor-backend",
42
+ distributed_executor_backend,
43
]
44
38
- if worker_use_ray:
39
- commands.append("--worker-use-ray")
45
uvicorn_process = subprocess.Popen(commands)
46
yield
47
uvicorn_process.terminate()
48
49
50
@pytest.mark.parametrize("tokenizer_pool_size", [0, 2])
46
-@pytest.mark.parametrize("worker_use_ray", [False, True])
51
+@pytest.mark.parametrize("distributed_executor_backend", ["mp", "ray"])
52
def test_api_server(api_server, tokenizer_pool_size: int,
48
- worker_use_ray: bool):
53
+ distributed_executor_backend: str):
54
"""
55
Run the API server and test it.
56
tests/basic_correctness/test_preemption.py
+9
-9
@@ -29,10 +29,10 @@ def check_settings():
29
30
31
@pytest.fixture
32
-def worker_use_ray() -> bool:
33
- # When SPMD worker is used, use ray_use_worker=True
32
+def distributed_executor_backend() -> str:
33
+ # When SPMD worker is used, use distributed_executor_backend="ray"
34
# to test delta input optimization works with preemption.
35
- return envs.VLLM_USE_RAY_SPMD_WORKER
35
+ return "ray" if envs.VLLM_USE_RAY_SPMD_WORKER else "mp"
36
37
38
@pytest.mark.parametrize("model", MODELS)
@@ -47,7 +47,7 @@ def test_chunked_prefill_recompute(
47
dtype: str,
48
max_tokens: int,
49
chunked_prefill_token_size: int,
50
- worker_use_ray: bool,
50
+ distributed_executor_backend: str,
51
) -> None:
52
"""Ensure that chunked prefill works with preemption."""
53
max_num_seqs = min(chunked_prefill_token_size, 256)
@@ -66,7 +66,7 @@ def test_chunked_prefill_recompute(
66
max_num_batched_tokens=max_num_batched_tokens,
67
enable_chunked_prefill=enable_chunked_prefill,
68
max_num_seqs=max_num_seqs,
69
- worker_use_ray=worker_use_ray,
69
+ distributed_executor_backend=distributed_executor_backend,
70
disable_log_stats=False,
71
) as vllm_model:
72
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
@@ -93,7 +93,7 @@ def test_preemption(
93
model: str,
94
dtype: str,
95
max_tokens: int,
96
- worker_use_ray: bool,
96
+ distributed_executor_backend: str,
97
) -> None:
98
"""By default, recompute preemption is enabled"""
99
@@ -104,7 +104,7 @@ def test_preemption(
104
model,
105
dtype=dtype,
106
disable_log_stats=False,
107
- worker_use_ray=worker_use_ray,
107
+ distributed_executor_backend=distributed_executor_backend,
108
) as vllm_model:
109
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
110
assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
@@ -144,7 +144,7 @@ def test_preemption_infeasible(
144
model: str,
145
dtype: str,
146
max_tokens: int,
147
- worker_use_ray: bool,
147
+ distributed_executor_backend: str,
148
) -> None:
149
"""Verify infeasible preemption request will be ignored."""
150
BLOCK_SIZE = 16
@@ -159,7 +159,7 @@ def test_preemption_infeasible(
159
# ignored instead of hanging forever.
160
num_gpu_blocks_override=prefill_blocks + decode_blocks // 2,
161
max_model_len=((prefill_blocks + decode_blocks // 2) * BLOCK_SIZE),
162
- worker_use_ray=worker_use_ray,
162
+ distributed_executor_backend=distributed_executor_backend,
163
) as vllm_model:
164
sampling_params = SamplingParams(max_tokens=max_tokens,
165
ignore_eos=True)
tests/multi_step/test_correctness_async_llm.py
+2
-1
@@ -16,7 +16,8 @@ NUM_SCHEDULER_STEPS = [8] # Multi-step decoding steps
16
NUM_PROMPTS = [10]
17
18
DEFAULT_SERVER_ARGS: List[str] = [
19
- "--worker-use-ray",
19
+ "--distributed-executor-backend",
20
+ "ray",
21
"--gpu-memory-utilization",
22
"0.85",
23
"--swap-space",
vllm/config.py
-10
@@ -1227,9 +1227,6 @@ class ParallelConfig:
1227
pipeline_parallel_size: int = 1 # Number of pipeline parallel groups.
1228
tensor_parallel_size: int = 1 # Number of tensor parallel groups.
1229
1230
- # Deprecated, use distributed_executor_backend instead.
1231
- worker_use_ray: Optional[bool] = None
1232
-
1230
# Maximum number of multiple batches
1231
# when load model sequentially. To avoid RAM OOM when using tensor
1232
# parallel and large models.
@@ -1283,13 +1280,6 @@ class ParallelConfig:
1280
self.world_size = self.pipeline_parallel_size * \
1281
self.tensor_parallel_size
1282
1286
- if self.worker_use_ray:
1287
- if self.distributed_executor_backend is None:
1288
- self.distributed_executor_backend = "ray"
1289
- elif not self.use_ray:
1290
- raise ValueError(f"worker-use-ray can't be used with "
1291
- f"distributed executor backend "
1292
- f"'{self.distributed_executor_backend}'.")
1283
ray_only_devices = ["tpu"]
1284
from vllm.platforms import current_platform
1285
if (current_platform.device_type in ray_only_devices
vllm/engine/arg_utils.py
-6
@@ -100,7 +100,6 @@ class EngineArgs:
100
kv_cache_dtype: str = 'auto'
101
seed: int = 0
102
max_model_len: Optional[int] = None
103
- worker_use_ray: bool = False
103
# Note: Specifying a custom executor backend by passing a class
104
# is intended for expert use only. The API may change without
105
# notice.
@@ -389,10 +388,6 @@ class EngineArgs:
388
'to "ray" if Ray is installed and fail otherwise. Note that tpu '
389
'only supports Ray for distributed inference.')
390
392
- parser.add_argument(
393
- '--worker-use-ray',
394
- action='store_true',
395
- help='Deprecated, use ``--distributed-executor-backend=ray``.')
391
parser.add_argument('--pipeline-parallel-size',
392
'-pp',
393
type=int,
@@ -1071,7 +1066,6 @@ class EngineArgs:
1066
parallel_config = ParallelConfig(
1067
pipeline_parallel_size=self.pipeline_parallel_size,
1068
tensor_parallel_size=self.tensor_parallel_size,
1074
- worker_use_ray=self.worker_use_ray,
1069
max_parallel_loading_workers=self.max_parallel_loading_workers,
1070
disable_custom_all_reduce=self.disable_custom_all_reduce,
1071
tokenizer_pool_config=TokenizerPoolConfig.create_config(
vllm/engine/metrics.py
-43
@@ -259,21 +259,6 @@ class Metrics:
259
documentation="Number of emitted tokens.",
260
labelnames=labelnames))
261
262
- # Deprecated in favor of vllm:prompt_tokens_total
263
- self.gauge_avg_prompt_throughput = self._gauge_cls(
264
- name="vllm:avg_prompt_throughput_toks_per_s",
265
- documentation="Average prefill throughput in tokens/s.",
266
- labelnames=labelnames,
267
- multiprocess_mode="sum",
268
- )
269
- # Deprecated in favor of vllm:generation_tokens_total
270
- self.gauge_avg_generation_throughput = self._gauge_cls(
271
- name="vllm:avg_generation_throughput_toks_per_s",
272
- documentation="Average generation throughput in tokens/s.",
273
- labelnames=labelnames,
274
- multiprocess_mode="sum",
275
- )
276
-
262
263
# end-metrics-definitions
264
@@ -635,20 +620,6 @@ class PrometheusStatLogger(StatLoggerBase):
620
self._log_histogram(self.metrics.histogram_max_tokens_request,
621
stats.max_tokens_requests)
622
638
- def _log_prometheus_interval(self, prompt_throughput: float,
639
- generation_throughput: float) -> None:
640
- # Logs metrics to prometheus that are computed every logging_interval.
641
- # Support legacy gauge metrics that make throughput calculations on
642
- # the vLLM side. Moving forward, we should use counters like
643
- # counter_prompt_tokens, counter_generation_tokens
644
- # Which log raw data and calculate summaries using rate() on the
645
- # grafana/prometheus side. See
646
- # https://github.com/vllm-project/vllm/pull/2316#discussion_r1464204666
647
- self.metrics.gauge_avg_prompt_throughput.labels(
648
- **self.labels).set(prompt_throughput)
649
- self.metrics.gauge_avg_generation_throughput.labels(
650
- **self.labels).set(generation_throughput)
651
-
623
def log(self, stats: Stats):
624
"""Logs to prometheus and tracked stats every iteration."""
625
# Log to prometheus.
@@ -664,20 +635,6 @@ class PrometheusStatLogger(StatLoggerBase):
635
# Log locally every local_interval seconds.
636
if local_interval_elapsed(stats.now, self.last_local_log,
637
self.local_interval):
667
- # Compute summary metrics for tracked stats (and log them
668
- # to promethus if applicable).
669
- prompt_throughput = get_throughput(self.num_prompt_tokens,
670
- now=stats.now,
671
- last_log=self.last_local_log)
672
- generation_throughput = get_throughput(
673
- self.num_generation_tokens,
674
- now=stats.now,
675
- last_log=self.last_local_log)
676
-
677
- self._log_prometheus_interval(
678
- prompt_throughput=prompt_throughput,
679
- generation_throughput=generation_throughput)
680
-
638
if self.spec_decode_metrics is not None:
639
self._log_gauge(
640
self.metrics.gauge_spec_decode_draft_acceptance_rate,