@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5757d90e26
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 03:00 UTC

[Speculative decoding] Adding configuration object for speculative decoding (#3706)

Co-authored-by: Lily Liu <lilyliupku@gmail.com>

Cade Daniel committed Apr 2, 2024 at 17:40 UTC 5757d90e26464d4582e36b55a2a0f34aec408e7f
12 files changed +394 -61
tests/spec_decode/e2e/conftest.py new
+41
@@ -0,0 +1,41 @@
1 +import pytest
2 +
3 +from tests.conftest import cleanup
4 +from vllm import LLM
5 +from vllm.model_executor.utils import set_random_seed
6 +
7 +
8 +@pytest.fixture
9 +def baseline_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
10 + baseline_llm_kwargs, seed):
11 + return create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
12 + baseline_llm_kwargs, seed)
13 +
14 +
15 +@pytest.fixture
16 +def test_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
17 + test_llm_kwargs, seed):
18 + return create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
19 + test_llm_kwargs, seed)
20 +
21 +
22 +def create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
23 + distinct_llm_kwargs, seed):
24 + kwargs = {
25 + **common_llm_kwargs,
26 + **per_test_common_llm_kwargs,
27 + **distinct_llm_kwargs,
28 + }
29 +
30 + def generator_inner():
31 + llm = LLM(**kwargs)
32 +
33 + set_random_seed(seed)
34 +
35 + yield llm
36 + del llm
37 + cleanup()
38 +
39 + for llm in generator_inner():
40 + yield llm
41 + del llm
tests/spec_decode/e2e/test_correctness.py new
+50
@@ -0,0 +1,50 @@
1 +import pytest
2 +
3 +from vllm import SamplingParams
4 +
5 +
6 +@pytest.mark.parametrize(
7 + "common_llm_kwargs",
8 + [{
9 + # Use a small model for a fast test.
10 + "model": "facebook/opt-125m",
11 + "speculative_model": "facebook/opt-125m",
12 + "num_speculative_tokens": 5,
13 +
14 + # Required for spec decode.
15 + "use_v2_block_manager": True
16 + }])
17 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
18 +@pytest.mark.parametrize("test_llm_kwargs", [{}])
19 +@pytest.mark.parametrize("seed", [1])
20 +def test_spec_decode_config(test_llm_generator):
21 + output_len = 1024
22 + temperature = 0.0
23 +
24 + prompts = [
25 + "Hello, my name is",
26 + "The president of the United States is",
27 + "The capital of France is",
28 + "The future of AI is",
29 + ]
30 +
31 + sampling_params = SamplingParams(
32 + max_tokens=output_len,
33 + ignore_eos=True,
34 + temperature=temperature,
35 + )
36 +
37 + with pytest.raises(
38 + AssertionError,
39 + match="Speculative decoding not yet supported for GPU backend"):
40 + get_token_ids_from_llm_generator(test_llm_generator, prompts,
41 + sampling_params)
42 +
43 +
44 +def get_token_ids_from_llm_generator(llm_generator, prompts, sampling_params):
45 + for llm in llm_generator:
46 + outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
47 + token_ids = [output.outputs[0].token_ids for output in outputs]
48 + del llm
49 +
50 + return token_ids
tests/spec_decode/utils.py
+8 -10
@@ -107,18 +107,16 @@ def create_worker(cls: type,
107 block_size=block_size,
108 enforce_eager=enforce_eager,
109 )
110 -
111 - (model_config, cache_config, parallel_config, scheduler_config,
112 - device_config, _, _) = engine_args.create_engine_configs()
110 + engine_config = engine_args.create_engine_config()
111
112 distributed_init_method = get_distributed_init_method(
113 get_ip(), get_open_port())
114
115 worker = cls(
118 - model_config=model_config,
119 - parallel_config=parallel_config,
120 - scheduler_config=scheduler_config,
121 - device_config=device_config,
116 + model_config=engine_config.model_config,
117 + parallel_config=engine_config.parallel_config,
118 + scheduler_config=engine_config.scheduler_config,
119 + device_config=engine_config.device_config,
120 local_rank=0,
121 rank=0,
122 distributed_init_method=distributed_init_method,
@@ -128,9 +126,9 @@ def create_worker(cls: type,
126 worker.init_device()
127 worker.load_model()
128
131 - cache_config.num_gpu_blocks = num_gpu_blocks
132 - cache_config.num_cpu_blocks = 0
133 - worker.init_cache_engine(cache_config)
129 + engine_config.cache_config.num_gpu_blocks = num_gpu_blocks
130 + engine_config.cache_config.num_cpu_blocks = 0
131 + worker.init_cache_engine(engine_config.cache_config)
132 worker.warm_up_model()
133
134 return worker
tests/worker/test_swap.py
+8 -9
@@ -10,19 +10,18 @@ def test_swap() -> None:
10 engine_args = EngineArgs(model="facebook/opt-125m",
11 dtype="half",
12 load_format="dummy")
13 - (model_config, cache_config, parallel_config, scheduler_config,
14 - device_config, _, _) = engine_args.create_engine_configs()
15 - cache_config.num_gpu_blocks = 100
16 - cache_config.num_cpu_blocks = 100
13 + engine_config = engine_args.create_engine_config()
14 + engine_config.cache_config.num_gpu_blocks = 100
15 + engine_config.cache_config.num_cpu_blocks = 100
16
17 # Create the worker.
18 distributed_init_method = get_distributed_init_method(
19 get_ip(), get_open_port())
20 worker = Worker(
22 - model_config=model_config,
23 - parallel_config=parallel_config,
24 - scheduler_config=scheduler_config,
25 - device_config=device_config,
21 + model_config=engine_config.model_config,
22 + parallel_config=engine_config.parallel_config,
23 + scheduler_config=engine_config.scheduler_config,
24 + device_config=engine_config.device_config,
25 local_rank=0,
26 rank=0,
27 distributed_init_method=distributed_init_method,
@@ -32,7 +31,7 @@ def test_swap() -> None:
31 # Initialize the worker.
32 worker.init_device()
33 worker.load_model()
35 - worker.init_cache_engine(cache_config)
34 + worker.init_cache_engine(engine_config.cache_config)
35 worker.warm_up_model()
36
37 # Randomly initialize the cache.
vllm/config.py
+187 -1
@@ -1,7 +1,7 @@
1 import enum
2 import json
3 import os
4 -from dataclasses import dataclass
4 +from dataclasses import dataclass, fields
5 from typing import TYPE_CHECKING, ClassVar, Optional, Union
6
7 import torch
@@ -617,6 +617,159 @@ class DeviceConfig:
617 self.device = torch.device(self.device_type)
618
619
620 +class SpeculativeConfig:
621 + """Configuration for speculative decoding.
622 +
623 + The configuration is currently specialized to draft-model speculative
624 + decoding with top-1 proposals.
625 + """
626 +
627 + @staticmethod
628 + def maybe_create_spec_config(
629 + target_model_config: ModelConfig,
630 + target_parallel_config: ParallelConfig,
631 + target_dtype: str,
632 + speculative_model: Optional[str],
633 + num_speculative_tokens: Optional[int],
634 + ) -> Optional["SpeculativeConfig"]:
635 + """Create a SpeculativeConfig if possible, else return None.
636 +
637 + This function attempts to create a SpeculativeConfig object based on the
638 + provided parameters. If the necessary conditions are met, it returns an
639 + instance of SpeculativeConfig. Otherwise, it returns None.
640 +
641 + Args:
642 + target_model_config (ModelConfig): The configuration of the target
643 + model.
644 + target_parallel_config (ParallelConfig): The parallel configuration
645 + for the target model.
646 + target_dtype (str): The data type used for the target model.
647 + speculative_model (Optional[str]): The name of the speculative
648 + model, if provided.
649 + num_speculative_tokens (Optional[int]): The number of speculative
650 + tokens, if provided.
651 +
652 + Returns:
653 + Optional["SpeculativeConfig"]: An instance of SpeculativeConfig if
654 + the necessary conditions are met, else None.
655 + """
656 +
657 + if (speculative_model is None and num_speculative_tokens is None):
658 + return None
659 +
660 + if speculative_model is not None and num_speculative_tokens is None:
661 + raise ValueError(
662 + "Expected both speculative_model and "
663 + "num_speculative_tokens to be provided, but found "
664 + f"{speculative_model=} and {num_speculative_tokens=}.")
665 +
666 + # TODO: The user should be able to specify revision/quantization/max
667 + # model len for the draft model. It is not currently supported.
668 + draft_revision = None
669 + draft_code_revision = None
670 + draft_quantization = None
671 + draft_max_model_len = None
672 +
673 + draft_model_config = ModelConfig(
674 + model=speculative_model,
675 + tokenizer=target_model_config.tokenizer,
676 + tokenizer_mode=target_model_config.tokenizer_mode,
677 + trust_remote_code=target_model_config.trust_remote_code,
678 + download_dir=target_model_config.download_dir,
679 + load_format=target_model_config.load_format,
680 + dtype=target_model_config.dtype,
681 + seed=target_model_config.seed,
682 + revision=draft_revision,
683 + code_revision=draft_code_revision,
684 + tokenizer_revision=target_model_config.tokenizer_revision,
685 + max_model_len=draft_max_model_len,
686 + quantization=draft_quantization,
687 + enforce_eager=target_model_config.enforce_eager,
688 + max_context_len_to_capture=target_model_config.
689 + max_context_len_to_capture,
690 + max_logprobs=target_model_config.max_logprobs,
691 + )
692 +
693 + draft_parallel_config = (
694 + SpeculativeConfig.create_draft_parallel_config(
695 + target_parallel_config))
696 +
697 + return SpeculativeConfig(
698 + draft_model_config,
699 + draft_parallel_config,
700 + num_speculative_tokens,
701 + )
702 +
703 + @staticmethod
704 + def create_draft_parallel_config(
705 + target_parallel_config: ParallelConfig) -> ParallelConfig:
706 + """Create a parallel config for use by the draft worker.
707 +
708 + This is mostly a copy of the target parallel config. In the future the
709 + draft worker can have a different parallel strategy, e.g. TP=1.
710 + """
711 + draft_parallel_config = ParallelConfig(
712 + pipeline_parallel_size=target_parallel_config.
713 + pipeline_parallel_size,
714 + tensor_parallel_size=target_parallel_config.tensor_parallel_size,
715 + worker_use_ray=target_parallel_config.worker_use_ray,
716 + max_parallel_loading_workers=target_parallel_config.
717 + max_parallel_loading_workers,
718 + disable_custom_all_reduce=target_parallel_config.
719 + disable_custom_all_reduce,
720 + tokenizer_pool_config=target_parallel_config.tokenizer_pool_config,
721 + ray_workers_use_nsight=target_parallel_config.
722 + ray_workers_use_nsight,
723 + placement_group=target_parallel_config.placement_group,
724 + )
725 +
726 + return draft_parallel_config
727 +
728 + def __init__(
729 + self,
730 + draft_model_config: ModelConfig,
731 + draft_parallel_config: ParallelConfig,
732 + num_speculative_tokens: int,
733 + ):
734 + """Create a SpeculativeConfig object.
735 +
736 + Args:
737 + draft_model_config: ModelConfig for the draft model.
738 + draft_parallel_config: ParallelConfig for the draft model.
739 + num_speculative_tokens: The number of tokens to sample from the
740 + draft model before scoring with the target model.
741 + """
742 + self.draft_model_config = draft_model_config
743 + self.draft_parallel_config = draft_parallel_config
744 + self.num_speculative_tokens = num_speculative_tokens
745 +
746 + self._verify_args()
747 +
748 + def _verify_args(self) -> None:
749 + if self.num_speculative_tokens <= 0:
750 + raise ValueError("Expected num_speculative_tokens to be greater "
751 + f"than zero ({self.num_speculative_tokens}).")
752 +
753 + if self.draft_model_config:
754 + self.draft_model_config.verify_with_parallel_config(
755 + self.draft_parallel_config)
756 +
757 + @property
758 + def num_lookahead_slots(self) -> int:
759 + """The number of additional slots the scheduler should allocate per
760 + step, in addition to the slots allocated for each known token.
761 +
762 + This is equal to the number of speculative tokens, as each speculative
763 + token must be scored.
764 + """
765 + return self.num_speculative_tokens
766 +
767 + def __repr__(self) -> str:
768 + draft_model = self.draft_model_config.model
769 + num_spec_tokens = self.num_speculative_tokens
770 + return f"SpeculativeConfig({draft_model=}, {num_spec_tokens=})"
771 +
772 +
773 @dataclass
774 class LoRAConfig:
775 max_lora_rank: int
@@ -838,3 +991,36 @@ def _get_and_verify_max_len(
991 "to incorrect model outputs or CUDA errors. Make sure the "
992 "value is correct and within the model context size.")
993 return int(max_model_len)
994 +
995 +
996 +@dataclass(frozen=True)
997 +class EngineConfig:
998 + """Dataclass which contains all engine-related configuration. This
999 + simplifies passing around the distinct configurations in the codebase.
1000 + """
1001 +
1002 + model_config: ModelConfig
1003 + cache_config: CacheConfig
1004 + parallel_config: ParallelConfig
1005 + scheduler_config: SchedulerConfig
1006 + device_config: DeviceConfig
1007 + lora_config: Optional[LoRAConfig]
1008 + vision_language_config: Optional[VisionLanguageConfig]
1009 + speculative_config: Optional[SpeculativeConfig]
1010 +
1011 + def __post_init__(self):
1012 + """Verify configs are valid & consistent with each other.
1013 + """
1014 + self.model_config.verify_with_parallel_config(self.parallel_config)
1015 + self.cache_config.verify_with_parallel_config(self.parallel_config)
1016 +
1017 + if self.lora_config:
1018 + self.lora_config.verify_with_model_config(self.model_config)
1019 + self.lora_config.verify_with_scheduler_config(
1020 + self.scheduler_config)
1021 +
1022 + def to_dict(self):
1023 + """Return the configs as a dictionary, for use in **kwargs.
1024 + """
1025 + return dict(
1026 + (field.name, getattr(self, field.name)) for field in fields(self))
vllm/engine/arg_utils.py
+44 -11
@@ -1,10 +1,11 @@
1 import argparse
2 import dataclasses
3 from dataclasses import dataclass
4 -from typing import Optional, Tuple
4 +from typing import Optional
5
6 -from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
7 - ParallelConfig, SchedulerConfig, TokenizerPoolConfig,
6 +from vllm.config import (CacheConfig, DeviceConfig, EngineConfig, LoRAConfig,
7 + ModelConfig, ParallelConfig, SchedulerConfig,
8 + SpeculativeConfig, TokenizerPoolConfig,
9 VisionLanguageConfig)
10 from vllm.utils import str_to_int_tuple
11
@@ -61,9 +62,14 @@ class EngineArgs:
62 image_token_id: Optional[int] = None
63 image_input_shape: Optional[str] = None
64 image_feature_size: Optional[int] = None
65 +
66 scheduler_delay_factor: float = 0.0
67 enable_chunked_prefill: bool = False
68
69 + # Speculative decoding configuration.
70 + speculative_model: Optional[str] = None
71 + num_speculative_tokens: Optional[int] = None
72 +
73 def __post_init__(self):
74 if self.tokenizer is None:
75 self.tokenizer = self.model
@@ -371,6 +377,20 @@ class EngineArgs:
377 default=False,
378 help='If True, the prefill requests can be chunked based on the '
379 'max_num_batched_tokens')
380 +
381 + parser.add_argument(
382 + '--speculative-model',
383 + type=str,
384 + default=None,
385 + help=
386 + 'The name of the draft model to be used in speculative decoding.')
387 +
388 + parser.add_argument(
389 + '--num-speculative-tokens',
390 + type=int,
391 + default=None,
392 + help='The number of speculative tokens to sample from '
393 + 'the draft model in speculative decoding')
394 return parser
395
396 @classmethod
@@ -381,11 +401,7 @@ class EngineArgs:
401 engine_args = cls(**{attr: getattr(args, attr) for attr in attrs})
402 return engine_args
403
384 - def create_engine_configs(
385 - self,
386 - ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig,
387 - DeviceConfig, Optional[LoRAConfig],
388 - Optional[VisionLanguageConfig]]:
404 + def create_engine_config(self, ) -> EngineConfig:
405 device_config = DeviceConfig(self.device)
406 model_config = ModelConfig(
407 self.model, self.tokenizer, self.tokenizer_mode,
@@ -409,12 +425,23 @@ class EngineArgs:
425 self.tokenizer_pool_type,
426 self.tokenizer_pool_extra_config,
427 ), self.ray_workers_use_nsight)
428 +
429 + speculative_config = SpeculativeConfig.maybe_create_spec_config(
430 + target_model_config=model_config,
431 + target_parallel_config=parallel_config,
432 + target_dtype=self.dtype,
433 + speculative_model=self.speculative_model,
434 + num_speculative_tokens=self.num_speculative_tokens,
435 + )
436 +
437 scheduler_config = SchedulerConfig(
438 self.max_num_batched_tokens,
439 self.max_num_seqs,
440 model_config.max_model_len,
441 self.use_v2_block_manager,
417 - num_lookahead_slots=self.num_lookahead_slots,
442 + num_lookahead_slots=(self.num_lookahead_slots
443 + if speculative_config is None else
444 + speculative_config.num_lookahead_slots),
445 delay_factor=self.scheduler_delay_factor,
446 enable_chunked_prefill=self.enable_chunked_prefill,
447 )
@@ -442,8 +469,14 @@ class EngineArgs:
469 else:
470 vision_language_config = None
471
445 - return (model_config, cache_config, parallel_config, scheduler_config,
446 - device_config, lora_config, vision_language_config)
472 + return EngineConfig(model_config=model_config,
473 + cache_config=cache_config,
474 + parallel_config=parallel_config,
475 + scheduler_config=scheduler_config,
476 + device_config=device_config,
477 + lora_config=lora_config,
478 + vision_language_config=vision_language_config,
479 + speculative_config=speculative_config)
480
481
482 @dataclass
vllm/engine/async_llm_engine.py
+9 -10
@@ -328,28 +328,27 @@ class AsyncLLMEngine:
328 ) -> "AsyncLLMEngine":
329 """Creates an async LLM engine from the engine arguments."""
330 # Create the engine configs.
331 - engine_configs = engine_args.create_engine_configs()
332 - parallel_config = engine_configs[2]
333 - device_config = engine_configs[4]
331 + engine_config = engine_args.create_engine_config()
332
335 - if device_config.device_type == "neuron":
333 + if engine_config.device_config.device_type == "neuron":
334 raise NotImplementedError("Neuron is not supported for "
335 "async engine yet.")
338 - elif parallel_config.worker_use_ray or engine_args.engine_use_ray:
339 - initialize_ray_cluster(parallel_config)
336 + elif (engine_config.parallel_config.worker_use_ray
337 + or engine_args.engine_use_ray):
338 + initialize_ray_cluster(engine_config.parallel_config)
339 from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
340 executor_class = RayGPUExecutorAsync
341 else:
343 - assert parallel_config.world_size == 1, (
342 + assert engine_config.parallel_config.world_size == 1, (
343 "Ray is required if parallel_config.world_size > 1.")
344 from vllm.executor.gpu_executor import GPUExecutorAsync
345 executor_class = GPUExecutorAsync
346 # Create the async LLM engine.
347 engine = cls(
349 - parallel_config.worker_use_ray,
348 + engine_config.parallel_config.worker_use_ray,
349 engine_args.engine_use_ray,
351 - *engine_configs,
352 - executor_class,
350 + **engine_config.to_dict(),
351 + executor_class=executor_class,
352 log_requests=not engine_args.disable_log_requests,
353 log_stats=not engine_args.disable_log_stats,
354 max_log_len=engine_args.max_log_len,
vllm/engine/llm_engine.py
+28 -16
@@ -5,7 +5,8 @@ from transformers import PreTrainedTokenizer
5
6 import vllm
7 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
8 - ParallelConfig, SchedulerConfig, VisionLanguageConfig)
8 + ParallelConfig, SchedulerConfig, SpeculativeConfig,
9 + VisionLanguageConfig)
10 from vllm.core.scheduler import Scheduler, SchedulerOutputs
11 from vllm.engine.arg_utils import EngineArgs
12 from vllm.engine.metrics import StatLogger, Stats
@@ -52,6 +53,11 @@ class LLMEngine:
53 parallel_config: The configuration related to distributed execution.
54 scheduler_config: The configuration related to the request scheduler.
55 device_config: The configuration related to the device.
56 + lora_config (Optional): The configuration related to serving multi-LoRA.
57 + vision_language_config (Optional): The configuration related to vision
58 + language models.
59 + speculative_config (Optional): The configuration related to speculative
60 + decoding.
61 executor_class: The model executor class for managing distributed
62 execution.
63 log_stats: Whether to log statistics.
@@ -66,7 +72,8 @@ class LLMEngine:
72 scheduler_config: SchedulerConfig,
73 device_config: DeviceConfig,
74 lora_config: Optional[LoRAConfig],
69 - vision_language_config: Optional["VisionLanguageConfig"],
75 + vision_language_config: Optional[VisionLanguageConfig],
76 + speculative_config: Optional[SpeculativeConfig],
77 executor_class: Type[ExecutorBase],
78 log_stats: bool,
79 usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -74,6 +81,7 @@ class LLMEngine:
81 logger.info(
82 f"Initializing an LLM engine (v{vllm.__version__}) with config: "
83 f"model={model_config.model!r}, "
84 + f"speculative_config={speculative_config!r}, "
85 f"tokenizer={model_config.tokenizer!r}, "
86 f"tokenizer_mode={model_config.tokenizer_mode}, "
87 f"revision={model_config.revision}, "
@@ -100,17 +108,23 @@ class LLMEngine:
108 self.parallel_config = parallel_config
109 self.scheduler_config = scheduler_config
110 self.device_config = device_config
111 + self.speculative_config = speculative_config
112 self.log_stats = log_stats
104 - self._verify_args()
113
114 self._init_tokenizer()
115 self.detokenizer = Detokenizer(self.tokenizer)
116 self.seq_counter = Counter()
117
110 - self.model_executor = executor_class(model_config, cache_config,
111 - parallel_config, scheduler_config,
112 - device_config, lora_config,
113 - vision_language_config)
118 + self.model_executor = executor_class(
119 + model_config=model_config,
120 + cache_config=cache_config,
121 + parallel_config=parallel_config,
122 + scheduler_config=scheduler_config,
123 + device_config=device_config,
124 + lora_config=lora_config,
125 + vision_language_config=vision_language_config,
126 + speculative_config=speculative_config,
127 + )
128
129 # If usage stat is enabled, collect relevant info.
130 if is_usage_stats_enabled():
@@ -171,30 +185,28 @@ class LLMEngine:
185 ) -> "LLMEngine":
186 """Creates an LLM engine from the engine arguments."""
187 # Create the engine configs.
174 - engine_configs = engine_args.create_engine_configs()
175 - parallel_config = engine_configs[2]
176 - device_config = engine_configs[4]
188 + engine_config = engine_args.create_engine_config()
189
190 # Initialize the cluster and specify the executor class.
179 - if device_config.device_type == "neuron":
191 + if engine_config.device_config.device_type == "neuron":
192 from vllm.executor.neuron_executor import NeuronExecutor
193 executor_class = NeuronExecutor
182 - elif device_config.device_type == "cpu":
194 + elif engine_config.device_config.device_type == "cpu":
195 from vllm.executor.cpu_executor import CPUExecutor
196 executor_class = CPUExecutor
185 - elif parallel_config.worker_use_ray:
186 - initialize_ray_cluster(parallel_config)
197 + elif engine_config.parallel_config.worker_use_ray:
198 + initialize_ray_cluster(engine_config.parallel_config)
199 from vllm.executor.ray_gpu_executor import RayGPUExecutor
200 executor_class = RayGPUExecutor
201 else:
190 - assert parallel_config.world_size == 1, (
202 + assert engine_config.parallel_config.world_size == 1, (
203 "Ray is required if parallel_config.world_size > 1.")
204 from vllm.executor.gpu_executor import GPUExecutor
205 executor_class = GPUExecutor
206
207 # Create the LLM engine.
208 engine = cls(
197 - *engine_configs,
209 + **engine_config.to_dict(),
210 executor_class=executor_class,
211 log_stats=not engine_args.disable_log_stats,
212 usage_context=usage_context,
vllm/executor/executor_base.py
+3 -1
@@ -2,7 +2,8 @@ from abc import ABC, abstractmethod
2 from typing import Dict, List, Optional
3
4 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
5 - ParallelConfig, SchedulerConfig, VisionLanguageConfig)
5 + ParallelConfig, SchedulerConfig, SpeculativeConfig,
6 + VisionLanguageConfig)
7 from vllm.lora.request import LoRARequest
8 from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9
@@ -25,6 +26,7 @@ class ExecutorBase(ABC):
26 device_config: DeviceConfig,
27 lora_config: Optional[LoRAConfig],
28 vision_language_config: Optional[VisionLanguageConfig],
29 + speculative_config: Optional[SpeculativeConfig],
30 ) -> None:
31 raise NotImplementedError
32
vllm/executor/gpu_executor.py
+6 -1
@@ -1,7 +1,8 @@
1 from typing import Dict, List, Optional
2
3 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4 - ParallelConfig, SchedulerConfig, VisionLanguageConfig)
4 + ParallelConfig, SchedulerConfig, SpeculativeConfig,
5 + VisionLanguageConfig)
6 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
7 from vllm.executor.utils import check_block_size_valid
8 from vllm.logger import init_logger
@@ -24,6 +25,7 @@ class GPUExecutor(ExecutorBase):
25 device_config: DeviceConfig,
26 lora_config: Optional[LoRAConfig],
27 vision_language_config: Optional[VisionLanguageConfig],
28 + speculative_config: Optional[SpeculativeConfig],
29 ) -> None:
30 self.model_config = model_config
31 self.cache_config = cache_config
@@ -33,6 +35,9 @@ class GPUExecutor(ExecutorBase):
35 self.device_config = device_config
36 self.vision_language_config = vision_language_config
37
38 + assert (not speculative_config
39 + ), "Speculative decoding not yet supported for GPU backend"
40 +
41 # Instantiate the worker and load the model to GPU.
42 self._init_worker()
43
vllm/executor/neuron_executor.py
+5 -1
@@ -1,7 +1,8 @@
1 from typing import Dict, List, Optional
2
3 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4 - ParallelConfig, SchedulerConfig, VisionLanguageConfig)
4 + ParallelConfig, SchedulerConfig, SpeculativeConfig,
5 + VisionLanguageConfig)
6 from vllm.executor.executor_base import ExecutorBase
7 from vllm.logger import init_logger
8 from vllm.lora.request import LoRARequest
@@ -21,6 +22,7 @@ class NeuronExecutor(ExecutorBase):
22 device_config: DeviceConfig,
23 lora_config: Optional[LoRAConfig],
24 vision_language_config: Optional[VisionLanguageConfig],
25 + speculative_config: Optional[SpeculativeConfig],
26 ) -> None:
27 self.model_config = model_config
28 self.cache_config = cache_config
@@ -28,6 +30,8 @@ class NeuronExecutor(ExecutorBase):
30 self.parallel_config = parallel_config
31 self.scheduler_config = scheduler_config
32 self.device_config = device_config
33 + assert (not speculative_config
34 + ), "Speculative decoding not yet supported for Neuron backend."
35
36 # Set the number of GPU blocks to be the same as the maximum number of
37 # sequences that can be processed in a single batch. This is equivalent
vllm/executor/ray_gpu_executor.py
+5 -1
@@ -6,7 +6,8 @@ from collections import defaultdict
6 from typing import TYPE_CHECKING, Any, Dict, List, Optional
7
8 from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9 - ParallelConfig, SchedulerConfig, VisionLanguageConfig)
9 + ParallelConfig, SchedulerConfig, SpeculativeConfig,
10 + VisionLanguageConfig)
11 from vllm.engine.ray_utils import RayWorkerVllm, ray
12 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
13 from vllm.executor.utils import check_block_size_valid
@@ -41,6 +42,7 @@ class RayGPUExecutor(ExecutorBase):
42 device_config: DeviceConfig,
43 lora_config: Optional[LoRAConfig],
44 vision_language_config: Optional[VisionLanguageConfig],
45 + speculative_config: Optional[SpeculativeConfig],
46 ) -> None:
47 self.model_config = model_config
48 self.cache_config = cache_config
@@ -49,6 +51,8 @@ class RayGPUExecutor(ExecutorBase):
51 self.scheduler_config = scheduler_config
52 self.device_config = device_config
53 self.vision_language_config = vision_language_config
54 + assert (not speculative_config
55 + ), "Speculative decoding not yet supported for RayGPU backend."
56
57 assert self.parallel_config.worker_use_ray
58 placement_group = self.parallel_config.placement_group