@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / df5dafaa5b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[Misc] Remove deprecated code (#12383)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 25, 2025 at 03:45 UTC df5dafaa5ba611f7179720958ba63e49615c927f
6 files changed +25 -78
tests/async_engine/test_api_server.py
+14 -9
@@ -25,27 +25,32 @@ def _query_server_long(prompt: str) -> dict:
25
26
27 @pytest.fixture
28 -def api_server(tokenizer_pool_size: int, worker_use_ray: bool):
28 +def api_server(tokenizer_pool_size: int, distributed_executor_backend: str):
29 script_path = Path(__file__).parent.joinpath(
30 "api_server_async_engine.py").absolute()
31 commands = [
32 - sys.executable, "-u",
33 - str(script_path), "--model", "facebook/opt-125m", "--host",
34 - "127.0.0.1", "--tokenizer-pool-size",
35 - str(tokenizer_pool_size)
32 + sys.executable,
33 + "-u",
34 + str(script_path),
35 + "--model",
36 + "facebook/opt-125m",
37 + "--host",
38 + "127.0.0.1",
39 + "--tokenizer-pool-size",
40 + str(tokenizer_pool_size),
41 + "--distributed-executor-backend",
42 + distributed_executor_backend,
43 ]
44
38 - if worker_use_ray:
39 - commands.append("--worker-use-ray")
45 uvicorn_process = subprocess.Popen(commands)
46 yield
47 uvicorn_process.terminate()
48
49
50 @pytest.mark.parametrize("tokenizer_pool_size", [0, 2])
46 -@pytest.mark.parametrize("worker_use_ray", [False, True])
51 +@pytest.mark.parametrize("distributed_executor_backend", ["mp", "ray"])
52 def test_api_server(api_server, tokenizer_pool_size: int,
48 - worker_use_ray: bool):
53 + distributed_executor_backend: str):
54 """
55 Run the API server and test it.
56
tests/basic_correctness/test_preemption.py
+9 -9
@@ -29,10 +29,10 @@ def check_settings():
29
30
31 @pytest.fixture
32 -def worker_use_ray() -> bool:
33 - # When SPMD worker is used, use ray_use_worker=True
32 +def distributed_executor_backend() -> str:
33 + # When SPMD worker is used, use distributed_executor_backend="ray"
34 # to test delta input optimization works with preemption.
35 - return envs.VLLM_USE_RAY_SPMD_WORKER
35 + return "ray" if envs.VLLM_USE_RAY_SPMD_WORKER else "mp"
36
37
38 @pytest.mark.parametrize("model", MODELS)
@@ -47,7 +47,7 @@ def test_chunked_prefill_recompute(
47 dtype: str,
48 max_tokens: int,
49 chunked_prefill_token_size: int,
50 - worker_use_ray: bool,
50 + distributed_executor_backend: str,
51 ) -> None:
52 """Ensure that chunked prefill works with preemption."""
53 max_num_seqs = min(chunked_prefill_token_size, 256)
@@ -66,7 +66,7 @@ def test_chunked_prefill_recompute(
66 max_num_batched_tokens=max_num_batched_tokens,
67 enable_chunked_prefill=enable_chunked_prefill,
68 max_num_seqs=max_num_seqs,
69 - worker_use_ray=worker_use_ray,
69 + distributed_executor_backend=distributed_executor_backend,
70 disable_log_stats=False,
71 ) as vllm_model:
72 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
@@ -93,7 +93,7 @@ def test_preemption(
93 model: str,
94 dtype: str,
95 max_tokens: int,
96 - worker_use_ray: bool,
96 + distributed_executor_backend: str,
97 ) -> None:
98 """By default, recompute preemption is enabled"""
99
@@ -104,7 +104,7 @@ def test_preemption(
104 model,
105 dtype=dtype,
106 disable_log_stats=False,
107 - worker_use_ray=worker_use_ray,
107 + distributed_executor_backend=distributed_executor_backend,
108 ) as vllm_model:
109 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
110 assert (vllm_model.model.llm_engine.scheduler[0].artificial_preempt_cnt
@@ -144,7 +144,7 @@ def test_preemption_infeasible(
144 model: str,
145 dtype: str,
146 max_tokens: int,
147 - worker_use_ray: bool,
147 + distributed_executor_backend: str,
148 ) -> None:
149 """Verify infeasible preemption request will be ignored."""
150 BLOCK_SIZE = 16
@@ -159,7 +159,7 @@ def test_preemption_infeasible(
159 # ignored instead of hanging forever.
160 num_gpu_blocks_override=prefill_blocks + decode_blocks // 2,
161 max_model_len=((prefill_blocks + decode_blocks // 2) * BLOCK_SIZE),
162 - worker_use_ray=worker_use_ray,
162 + distributed_executor_backend=distributed_executor_backend,
163 ) as vllm_model:
164 sampling_params = SamplingParams(max_tokens=max_tokens,
165 ignore_eos=True)
tests/multi_step/test_correctness_async_llm.py
+2 -1
@@ -16,7 +16,8 @@ NUM_SCHEDULER_STEPS = [8] # Multi-step decoding steps
16 NUM_PROMPTS = [10]
17
18 DEFAULT_SERVER_ARGS: List[str] = [
19 - "--worker-use-ray",
19 + "--distributed-executor-backend",
20 + "ray",
21 "--gpu-memory-utilization",
22 "0.85",
23 "--swap-space",
vllm/config.py
-10
@@ -1227,9 +1227,6 @@ class ParallelConfig:
1227 pipeline_parallel_size: int = 1 # Number of pipeline parallel groups.
1228 tensor_parallel_size: int = 1 # Number of tensor parallel groups.
1229
1230 - # Deprecated, use distributed_executor_backend instead.
1231 - worker_use_ray: Optional[bool] = None
1232 -
1230 # Maximum number of multiple batches
1231 # when load model sequentially. To avoid RAM OOM when using tensor
1232 # parallel and large models.
@@ -1283,13 +1280,6 @@ class ParallelConfig:
1280 self.world_size = self.pipeline_parallel_size * \
1281 self.tensor_parallel_size
1282
1286 - if self.worker_use_ray:
1287 - if self.distributed_executor_backend is None:
1288 - self.distributed_executor_backend = "ray"
1289 - elif not self.use_ray:
1290 - raise ValueError(f"worker-use-ray can't be used with "
1291 - f"distributed executor backend "
1292 - f"'{self.distributed_executor_backend}'.")
1283 ray_only_devices = ["tpu"]
1284 from vllm.platforms import current_platform
1285 if (current_platform.device_type in ray_only_devices
vllm/engine/arg_utils.py
-6
@@ -100,7 +100,6 @@ class EngineArgs:
100 kv_cache_dtype: str = 'auto'
101 seed: int = 0
102 max_model_len: Optional[int] = None
103 - worker_use_ray: bool = False
103 # Note: Specifying a custom executor backend by passing a class
104 # is intended for expert use only. The API may change without
105 # notice.
@@ -389,10 +388,6 @@ class EngineArgs:
388 'to "ray" if Ray is installed and fail otherwise. Note that tpu '
389 'only supports Ray for distributed inference.')
390
392 - parser.add_argument(
393 - '--worker-use-ray',
394 - action='store_true',
395 - help='Deprecated, use ``--distributed-executor-backend=ray``.')
391 parser.add_argument('--pipeline-parallel-size',
392 '-pp',
393 type=int,
@@ -1071,7 +1066,6 @@ class EngineArgs:
1066 parallel_config = ParallelConfig(
1067 pipeline_parallel_size=self.pipeline_parallel_size,
1068 tensor_parallel_size=self.tensor_parallel_size,
1074 - worker_use_ray=self.worker_use_ray,
1069 max_parallel_loading_workers=self.max_parallel_loading_workers,
1070 disable_custom_all_reduce=self.disable_custom_all_reduce,
1071 tokenizer_pool_config=TokenizerPoolConfig.create_config(
vllm/engine/metrics.py
-43
@@ -259,21 +259,6 @@ class Metrics:
259 documentation="Number of emitted tokens.",
260 labelnames=labelnames))
261
262 - # Deprecated in favor of vllm:prompt_tokens_total
263 - self.gauge_avg_prompt_throughput = self._gauge_cls(
264 - name="vllm:avg_prompt_throughput_toks_per_s",
265 - documentation="Average prefill throughput in tokens/s.",
266 - labelnames=labelnames,
267 - multiprocess_mode="sum",
268 - )
269 - # Deprecated in favor of vllm:generation_tokens_total
270 - self.gauge_avg_generation_throughput = self._gauge_cls(
271 - name="vllm:avg_generation_throughput_toks_per_s",
272 - documentation="Average generation throughput in tokens/s.",
273 - labelnames=labelnames,
274 - multiprocess_mode="sum",
275 - )
276 -
262
263 # end-metrics-definitions
264
@@ -635,20 +620,6 @@ class PrometheusStatLogger(StatLoggerBase):
620 self._log_histogram(self.metrics.histogram_max_tokens_request,
621 stats.max_tokens_requests)
622
638 - def _log_prometheus_interval(self, prompt_throughput: float,
639 - generation_throughput: float) -> None:
640 - # Logs metrics to prometheus that are computed every logging_interval.
641 - # Support legacy gauge metrics that make throughput calculations on
642 - # the vLLM side. Moving forward, we should use counters like
643 - # counter_prompt_tokens, counter_generation_tokens
644 - # Which log raw data and calculate summaries using rate() on the
645 - # grafana/prometheus side. See
646 - # https://github.com/vllm-project/vllm/pull/2316#discussion_r1464204666
647 - self.metrics.gauge_avg_prompt_throughput.labels(
648 - **self.labels).set(prompt_throughput)
649 - self.metrics.gauge_avg_generation_throughput.labels(
650 - **self.labels).set(generation_throughput)
651 -
623 def log(self, stats: Stats):
624 """Logs to prometheus and tracked stats every iteration."""
625 # Log to prometheus.
@@ -664,20 +635,6 @@ class PrometheusStatLogger(StatLoggerBase):
635 # Log locally every local_interval seconds.
636 if local_interval_elapsed(stats.now, self.last_local_log,
637 self.local_interval):
667 - # Compute summary metrics for tracked stats (and log them
668 - # to promethus if applicable).
669 - prompt_throughput = get_throughput(self.num_prompt_tokens,
670 - now=stats.now,
671 - last_log=self.last_local_log)
672 - generation_throughput = get_throughput(
673 - self.num_generation_tokens,
674 - now=stats.now,
675 - last_log=self.last_local_log)
676 -
677 - self._log_prometheus_interval(
678 - prompt_throughput=prompt_throughput,
679 - generation_throughput=generation_throughput)
680 -
638 if self.spec_decode_metrics is not None:
639 self._log_gauge(
640 self.metrics.gauge_spec_decode_draft_acceptance_rate,