A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 03:00 UTC
[Speculative decoding] Adding configuration object for speculative decoding (#3706)
Co-authored-by: Lily Liu <lilyliupku@gmail.com>
Cade Daniel committed
Apr 2, 2024 at 17:40 UTC
5757d90e26464d4582e36b55a2a0f34aec408e7f
12 files changed
+394
-61
tests/spec_decode/e2e/conftest.py
new
+41
@@ -0,0 +1,41 @@
1
+import pytest
2
+
3
+from tests.conftest import cleanup
4
+from vllm import LLM
5
+from vllm.model_executor.utils import set_random_seed
6
+
7
+
8
+@pytest.fixture
9
+def baseline_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
10
+ baseline_llm_kwargs, seed):
11
+ return create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
12
+ baseline_llm_kwargs, seed)
13
+
14
+
15
+@pytest.fixture
16
+def test_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
17
+ test_llm_kwargs, seed):
18
+ return create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
19
+ test_llm_kwargs, seed)
20
+
21
+
22
+def create_llm_generator(common_llm_kwargs, per_test_common_llm_kwargs,
23
+ distinct_llm_kwargs, seed):
24
+ kwargs = {
25
+ **common_llm_kwargs,
26
+ **per_test_common_llm_kwargs,
27
+ **distinct_llm_kwargs,
28
+ }
29
+
30
+ def generator_inner():
31
+ llm = LLM(**kwargs)
32
+
33
+ set_random_seed(seed)
34
+
35
+ yield llm
36
+ del llm
37
+ cleanup()
38
+
39
+ for llm in generator_inner():
40
+ yield llm
41
+ del llm
tests/spec_decode/e2e/test_correctness.py
new
+50
@@ -0,0 +1,50 @@
1
+import pytest
2
+
3
+from vllm import SamplingParams
4
+
5
+
6
+@pytest.mark.parametrize(
7
+ "common_llm_kwargs",
8
+ [{
9
+ # Use a small model for a fast test.
10
+ "model": "facebook/opt-125m",
11
+ "speculative_model": "facebook/opt-125m",
12
+ "num_speculative_tokens": 5,
13
+
14
+ # Required for spec decode.
15
+ "use_v2_block_manager": True
16
+ }])
17
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
18
+@pytest.mark.parametrize("test_llm_kwargs", [{}])
19
+@pytest.mark.parametrize("seed", [1])
20
+def test_spec_decode_config(test_llm_generator):
21
+ output_len = 1024
22
+ temperature = 0.0
23
+
24
+ prompts = [
25
+ "Hello, my name is",
26
+ "The president of the United States is",
27
+ "The capital of France is",
28
+ "The future of AI is",
29
+ ]
30
+
31
+ sampling_params = SamplingParams(
32
+ max_tokens=output_len,
33
+ ignore_eos=True,
34
+ temperature=temperature,
35
+ )
36
+
37
+ with pytest.raises(
38
+ AssertionError,
39
+ match="Speculative decoding not yet supported for GPU backend"):
40
+ get_token_ids_from_llm_generator(test_llm_generator, prompts,
41
+ sampling_params)
42
+
43
+
44
+def get_token_ids_from_llm_generator(llm_generator, prompts, sampling_params):
45
+ for llm in llm_generator:
46
+ outputs = llm.generate(prompts, sampling_params, use_tqdm=True)
47
+ token_ids = [output.outputs[0].token_ids for output in outputs]
48
+ del llm
49
+
50
+ return token_ids
tests/spec_decode/utils.py
+8
-10
@@ -107,18 +107,16 @@ def create_worker(cls: type,
107
block_size=block_size,
108
enforce_eager=enforce_eager,
109
)
110
-
111
- (model_config, cache_config, parallel_config, scheduler_config,
112
- device_config, _, _) = engine_args.create_engine_configs()
110
+ engine_config = engine_args.create_engine_config()
111
112
distributed_init_method = get_distributed_init_method(
113
get_ip(), get_open_port())
114
115
worker = cls(
118
- model_config=model_config,
119
- parallel_config=parallel_config,
120
- scheduler_config=scheduler_config,
121
- device_config=device_config,
116
+ model_config=engine_config.model_config,
117
+ parallel_config=engine_config.parallel_config,
118
+ scheduler_config=engine_config.scheduler_config,
119
+ device_config=engine_config.device_config,
120
local_rank=0,
121
rank=0,
122
distributed_init_method=distributed_init_method,
@@ -128,9 +126,9 @@ def create_worker(cls: type,
126
worker.init_device()
127
worker.load_model()
128
131
- cache_config.num_gpu_blocks = num_gpu_blocks
132
- cache_config.num_cpu_blocks = 0
133
- worker.init_cache_engine(cache_config)
129
+ engine_config.cache_config.num_gpu_blocks = num_gpu_blocks
130
+ engine_config.cache_config.num_cpu_blocks = 0
131
+ worker.init_cache_engine(engine_config.cache_config)
132
worker.warm_up_model()
133
134
return worker
tests/worker/test_swap.py
+8
-9
@@ -10,19 +10,18 @@ def test_swap() -> None:
10
engine_args = EngineArgs(model="facebook/opt-125m",
11
dtype="half",
12
load_format="dummy")
13
- (model_config, cache_config, parallel_config, scheduler_config,
14
- device_config, _, _) = engine_args.create_engine_configs()
15
- cache_config.num_gpu_blocks = 100
16
- cache_config.num_cpu_blocks = 100
13
+ engine_config = engine_args.create_engine_config()
14
+ engine_config.cache_config.num_gpu_blocks = 100
15
+ engine_config.cache_config.num_cpu_blocks = 100
16
17
# Create the worker.
18
distributed_init_method = get_distributed_init_method(
19
get_ip(), get_open_port())
20
worker = Worker(
22
- model_config=model_config,
23
- parallel_config=parallel_config,
24
- scheduler_config=scheduler_config,
25
- device_config=device_config,
21
+ model_config=engine_config.model_config,
22
+ parallel_config=engine_config.parallel_config,
23
+ scheduler_config=engine_config.scheduler_config,
24
+ device_config=engine_config.device_config,
25
local_rank=0,
26
rank=0,
27
distributed_init_method=distributed_init_method,
@@ -32,7 +31,7 @@ def test_swap() -> None:
31
# Initialize the worker.
32
worker.init_device()
33
worker.load_model()
35
- worker.init_cache_engine(cache_config)
34
+ worker.init_cache_engine(engine_config.cache_config)
35
worker.warm_up_model()
36
37
# Randomly initialize the cache.
vllm/config.py
+187
-1
@@ -1,7 +1,7 @@
1
import enum
2
import json
3
import os
4
-from dataclasses import dataclass
4
+from dataclasses import dataclass, fields
5
from typing import TYPE_CHECKING, ClassVar, Optional, Union
6
7
import torch
@@ -617,6 +617,159 @@ class DeviceConfig:
617
self.device = torch.device(self.device_type)
618
619
620
+class SpeculativeConfig:
621
+ """Configuration for speculative decoding.
622
+
623
+ The configuration is currently specialized to draft-model speculative
624
+ decoding with top-1 proposals.
625
+ """
626
+
627
+ @staticmethod
628
+ def maybe_create_spec_config(
629
+ target_model_config: ModelConfig,
630
+ target_parallel_config: ParallelConfig,
631
+ target_dtype: str,
632
+ speculative_model: Optional[str],
633
+ num_speculative_tokens: Optional[int],
634
+ ) -> Optional["SpeculativeConfig"]:
635
+ """Create a SpeculativeConfig if possible, else return None.
636
+
637
+ This function attempts to create a SpeculativeConfig object based on the
638
+ provided parameters. If the necessary conditions are met, it returns an
639
+ instance of SpeculativeConfig. Otherwise, it returns None.
640
+
641
+ Args:
642
+ target_model_config (ModelConfig): The configuration of the target
643
+ model.
644
+ target_parallel_config (ParallelConfig): The parallel configuration
645
+ for the target model.
646
+ target_dtype (str): The data type used for the target model.
647
+ speculative_model (Optional[str]): The name of the speculative
648
+ model, if provided.
649
+ num_speculative_tokens (Optional[int]): The number of speculative
650
+ tokens, if provided.
651
+
652
+ Returns:
653
+ Optional["SpeculativeConfig"]: An instance of SpeculativeConfig if
654
+ the necessary conditions are met, else None.
655
+ """
656
+
657
+ if (speculative_model is None and num_speculative_tokens is None):
658
+ return None
659
+
660
+ if speculative_model is not None and num_speculative_tokens is None:
661
+ raise ValueError(
662
+ "Expected both speculative_model and "
663
+ "num_speculative_tokens to be provided, but found "
664
+ f"{speculative_model=} and {num_speculative_tokens=}.")
665
+
666
+ # TODO: The user should be able to specify revision/quantization/max
667
+ # model len for the draft model. It is not currently supported.
668
+ draft_revision = None
669
+ draft_code_revision = None
670
+ draft_quantization = None
671
+ draft_max_model_len = None
672
+
673
+ draft_model_config = ModelConfig(
674
+ model=speculative_model,
675
+ tokenizer=target_model_config.tokenizer,
676
+ tokenizer_mode=target_model_config.tokenizer_mode,
677
+ trust_remote_code=target_model_config.trust_remote_code,
678
+ download_dir=target_model_config.download_dir,
679
+ load_format=target_model_config.load_format,
680
+ dtype=target_model_config.dtype,
681
+ seed=target_model_config.seed,
682
+ revision=draft_revision,
683
+ code_revision=draft_code_revision,
684
+ tokenizer_revision=target_model_config.tokenizer_revision,
685
+ max_model_len=draft_max_model_len,
686
+ quantization=draft_quantization,
687
+ enforce_eager=target_model_config.enforce_eager,
688
+ max_context_len_to_capture=target_model_config.
689
+ max_context_len_to_capture,
690
+ max_logprobs=target_model_config.max_logprobs,
691
+ )
692
+
693
+ draft_parallel_config = (
694
+ SpeculativeConfig.create_draft_parallel_config(
695
+ target_parallel_config))
696
+
697
+ return SpeculativeConfig(
698
+ draft_model_config,
699
+ draft_parallel_config,
700
+ num_speculative_tokens,
701
+ )
702
+
703
+ @staticmethod
704
+ def create_draft_parallel_config(
705
+ target_parallel_config: ParallelConfig) -> ParallelConfig:
706
+ """Create a parallel config for use by the draft worker.
707
+
708
+ This is mostly a copy of the target parallel config. In the future the
709
+ draft worker can have a different parallel strategy, e.g. TP=1.
710
+ """
711
+ draft_parallel_config = ParallelConfig(
712
+ pipeline_parallel_size=target_parallel_config.
713
+ pipeline_parallel_size,
714
+ tensor_parallel_size=target_parallel_config.tensor_parallel_size,
715
+ worker_use_ray=target_parallel_config.worker_use_ray,
716
+ max_parallel_loading_workers=target_parallel_config.
717
+ max_parallel_loading_workers,
718
+ disable_custom_all_reduce=target_parallel_config.
719
+ disable_custom_all_reduce,
720
+ tokenizer_pool_config=target_parallel_config.tokenizer_pool_config,
721
+ ray_workers_use_nsight=target_parallel_config.
722
+ ray_workers_use_nsight,
723
+ placement_group=target_parallel_config.placement_group,
724
+ )
725
+
726
+ return draft_parallel_config
727
+
728
+ def __init__(
729
+ self,
730
+ draft_model_config: ModelConfig,
731
+ draft_parallel_config: ParallelConfig,
732
+ num_speculative_tokens: int,
733
+ ):
734
+ """Create a SpeculativeConfig object.
735
+
736
+ Args:
737
+ draft_model_config: ModelConfig for the draft model.
738
+ draft_parallel_config: ParallelConfig for the draft model.
739
+ num_speculative_tokens: The number of tokens to sample from the
740
+ draft model before scoring with the target model.
741
+ """
742
+ self.draft_model_config = draft_model_config
743
+ self.draft_parallel_config = draft_parallel_config
744
+ self.num_speculative_tokens = num_speculative_tokens
745
+
746
+ self._verify_args()
747
+
748
+ def _verify_args(self) -> None:
749
+ if self.num_speculative_tokens <= 0:
750
+ raise ValueError("Expected num_speculative_tokens to be greater "
751
+ f"than zero ({self.num_speculative_tokens}).")
752
+
753
+ if self.draft_model_config:
754
+ self.draft_model_config.verify_with_parallel_config(
755
+ self.draft_parallel_config)
756
+
757
+ @property
758
+ def num_lookahead_slots(self) -> int:
759
+ """The number of additional slots the scheduler should allocate per
760
+ step, in addition to the slots allocated for each known token.
761
+
762
+ This is equal to the number of speculative tokens, as each speculative
763
+ token must be scored.
764
+ """
765
+ return self.num_speculative_tokens
766
+
767
+ def __repr__(self) -> str:
768
+ draft_model = self.draft_model_config.model
769
+ num_spec_tokens = self.num_speculative_tokens
770
+ return f"SpeculativeConfig({draft_model=}, {num_spec_tokens=})"
771
+
772
+
773
@dataclass
774
class LoRAConfig:
775
max_lora_rank: int
@@ -838,3 +991,36 @@ def _get_and_verify_max_len(
991
"to incorrect model outputs or CUDA errors. Make sure the "
992
"value is correct and within the model context size.")
993
return int(max_model_len)
994
+
995
+
996
+@dataclass(frozen=True)
997
+class EngineConfig:
998
+ """Dataclass which contains all engine-related configuration. This
999
+ simplifies passing around the distinct configurations in the codebase.
1000
+ """
1001
+
1002
+ model_config: ModelConfig
1003
+ cache_config: CacheConfig
1004
+ parallel_config: ParallelConfig
1005
+ scheduler_config: SchedulerConfig
1006
+ device_config: DeviceConfig
1007
+ lora_config: Optional[LoRAConfig]
1008
+ vision_language_config: Optional[VisionLanguageConfig]
1009
+ speculative_config: Optional[SpeculativeConfig]
1010
+
1011
+ def __post_init__(self):
1012
+ """Verify configs are valid & consistent with each other.
1013
+ """
1014
+ self.model_config.verify_with_parallel_config(self.parallel_config)
1015
+ self.cache_config.verify_with_parallel_config(self.parallel_config)
1016
+
1017
+ if self.lora_config:
1018
+ self.lora_config.verify_with_model_config(self.model_config)
1019
+ self.lora_config.verify_with_scheduler_config(
1020
+ self.scheduler_config)
1021
+
1022
+ def to_dict(self):
1023
+ """Return the configs as a dictionary, for use in **kwargs.
1024
+ """
1025
+ return dict(
1026
+ (field.name, getattr(self, field.name)) for field in fields(self))
vllm/engine/arg_utils.py
+44
-11
@@ -1,10 +1,11 @@
1
import argparse
2
import dataclasses
3
from dataclasses import dataclass
4
-from typing import Optional, Tuple
4
+from typing import Optional
5
6
-from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
7
- ParallelConfig, SchedulerConfig, TokenizerPoolConfig,
6
+from vllm.config import (CacheConfig, DeviceConfig, EngineConfig, LoRAConfig,
7
+ ModelConfig, ParallelConfig, SchedulerConfig,
8
+ SpeculativeConfig, TokenizerPoolConfig,
9
VisionLanguageConfig)
10
from vllm.utils import str_to_int_tuple
11
@@ -61,9 +62,14 @@ class EngineArgs:
62
image_token_id: Optional[int] = None
63
image_input_shape: Optional[str] = None
64
image_feature_size: Optional[int] = None
65
+
66
scheduler_delay_factor: float = 0.0
67
enable_chunked_prefill: bool = False
68
69
+ # Speculative decoding configuration.
70
+ speculative_model: Optional[str] = None
71
+ num_speculative_tokens: Optional[int] = None
72
+
73
def __post_init__(self):
74
if self.tokenizer is None:
75
self.tokenizer = self.model
@@ -371,6 +377,20 @@ class EngineArgs:
377
default=False,
378
help='If True, the prefill requests can be chunked based on the '
379
'max_num_batched_tokens')
380
+
381
+ parser.add_argument(
382
+ '--speculative-model',
383
+ type=str,
384
+ default=None,
385
+ help=
386
+ 'The name of the draft model to be used in speculative decoding.')
387
+
388
+ parser.add_argument(
389
+ '--num-speculative-tokens',
390
+ type=int,
391
+ default=None,
392
+ help='The number of speculative tokens to sample from '
393
+ 'the draft model in speculative decoding')
394
return parser
395
396
@classmethod
@@ -381,11 +401,7 @@ class EngineArgs:
401
engine_args = cls(**{attr: getattr(args, attr) for attr in attrs})
402
return engine_args
403
384
- def create_engine_configs(
385
- self,
386
- ) -> Tuple[ModelConfig, CacheConfig, ParallelConfig, SchedulerConfig,
387
- DeviceConfig, Optional[LoRAConfig],
388
- Optional[VisionLanguageConfig]]:
404
+ def create_engine_config(self, ) -> EngineConfig:
405
device_config = DeviceConfig(self.device)
406
model_config = ModelConfig(
407
self.model, self.tokenizer, self.tokenizer_mode,
@@ -409,12 +425,23 @@ class EngineArgs:
425
self.tokenizer_pool_type,
426
self.tokenizer_pool_extra_config,
427
), self.ray_workers_use_nsight)
428
+
429
+ speculative_config = SpeculativeConfig.maybe_create_spec_config(
430
+ target_model_config=model_config,
431
+ target_parallel_config=parallel_config,
432
+ target_dtype=self.dtype,
433
+ speculative_model=self.speculative_model,
434
+ num_speculative_tokens=self.num_speculative_tokens,
435
+ )
436
+
437
scheduler_config = SchedulerConfig(
438
self.max_num_batched_tokens,
439
self.max_num_seqs,
440
model_config.max_model_len,
441
self.use_v2_block_manager,
417
- num_lookahead_slots=self.num_lookahead_slots,
442
+ num_lookahead_slots=(self.num_lookahead_slots
443
+ if speculative_config is None else
444
+ speculative_config.num_lookahead_slots),
445
delay_factor=self.scheduler_delay_factor,
446
enable_chunked_prefill=self.enable_chunked_prefill,
447
)
@@ -442,8 +469,14 @@ class EngineArgs:
469
else:
470
vision_language_config = None
471
445
- return (model_config, cache_config, parallel_config, scheduler_config,
446
- device_config, lora_config, vision_language_config)
472
+ return EngineConfig(model_config=model_config,
473
+ cache_config=cache_config,
474
+ parallel_config=parallel_config,
475
+ scheduler_config=scheduler_config,
476
+ device_config=device_config,
477
+ lora_config=lora_config,
478
+ vision_language_config=vision_language_config,
479
+ speculative_config=speculative_config)
480
481
482
@dataclass
vllm/engine/async_llm_engine.py
+9
-10
@@ -328,28 +328,27 @@ class AsyncLLMEngine:
328
) -> "AsyncLLMEngine":
329
"""Creates an async LLM engine from the engine arguments."""
330
# Create the engine configs.
331
- engine_configs = engine_args.create_engine_configs()
332
- parallel_config = engine_configs[2]
333
- device_config = engine_configs[4]
331
+ engine_config = engine_args.create_engine_config()
332
335
- if device_config.device_type == "neuron":
333
+ if engine_config.device_config.device_type == "neuron":
334
raise NotImplementedError("Neuron is not supported for "
335
"async engine yet.")
338
- elif parallel_config.worker_use_ray or engine_args.engine_use_ray:
339
- initialize_ray_cluster(parallel_config)
336
+ elif (engine_config.parallel_config.worker_use_ray
337
+ or engine_args.engine_use_ray):
338
+ initialize_ray_cluster(engine_config.parallel_config)
339
from vllm.executor.ray_gpu_executor import RayGPUExecutorAsync
340
executor_class = RayGPUExecutorAsync
341
else:
343
- assert parallel_config.world_size == 1, (
342
+ assert engine_config.parallel_config.world_size == 1, (
343
"Ray is required if parallel_config.world_size > 1.")
344
from vllm.executor.gpu_executor import GPUExecutorAsync
345
executor_class = GPUExecutorAsync
346
# Create the async LLM engine.
347
engine = cls(
349
- parallel_config.worker_use_ray,
348
+ engine_config.parallel_config.worker_use_ray,
349
engine_args.engine_use_ray,
351
- *engine_configs,
352
- executor_class,
350
+ **engine_config.to_dict(),
351
+ executor_class=executor_class,
352
log_requests=not engine_args.disable_log_requests,
353
log_stats=not engine_args.disable_log_stats,
354
max_log_len=engine_args.max_log_len,
vllm/engine/llm_engine.py
+28
-16
@@ -5,7 +5,8 @@ from transformers import PreTrainedTokenizer
5
6
import vllm
7
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
8
- ParallelConfig, SchedulerConfig, VisionLanguageConfig)
8
+ ParallelConfig, SchedulerConfig, SpeculativeConfig,
9
+ VisionLanguageConfig)
10
from vllm.core.scheduler import Scheduler, SchedulerOutputs
11
from vllm.engine.arg_utils import EngineArgs
12
from vllm.engine.metrics import StatLogger, Stats
@@ -52,6 +53,11 @@ class LLMEngine:
53
parallel_config: The configuration related to distributed execution.
54
scheduler_config: The configuration related to the request scheduler.
55
device_config: The configuration related to the device.
56
+ lora_config (Optional): The configuration related to serving multi-LoRA.
57
+ vision_language_config (Optional): The configuration related to vision
58
+ language models.
59
+ speculative_config (Optional): The configuration related to speculative
60
+ decoding.
61
executor_class: The model executor class for managing distributed
62
execution.
63
log_stats: Whether to log statistics.
@@ -66,7 +72,8 @@ class LLMEngine:
72
scheduler_config: SchedulerConfig,
73
device_config: DeviceConfig,
74
lora_config: Optional[LoRAConfig],
69
- vision_language_config: Optional["VisionLanguageConfig"],
75
+ vision_language_config: Optional[VisionLanguageConfig],
76
+ speculative_config: Optional[SpeculativeConfig],
77
executor_class: Type[ExecutorBase],
78
log_stats: bool,
79
usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
@@ -74,6 +81,7 @@ class LLMEngine:
81
logger.info(
82
f"Initializing an LLM engine (v{vllm.__version__}) with config: "
83
f"model={model_config.model!r}, "
84
+ f"speculative_config={speculative_config!r}, "
85
f"tokenizer={model_config.tokenizer!r}, "
86
f"tokenizer_mode={model_config.tokenizer_mode}, "
87
f"revision={model_config.revision}, "
@@ -100,17 +108,23 @@ class LLMEngine:
108
self.parallel_config = parallel_config
109
self.scheduler_config = scheduler_config
110
self.device_config = device_config
111
+ self.speculative_config = speculative_config
112
self.log_stats = log_stats
104
- self._verify_args()
113
114
self._init_tokenizer()
115
self.detokenizer = Detokenizer(self.tokenizer)
116
self.seq_counter = Counter()
117
110
- self.model_executor = executor_class(model_config, cache_config,
111
- parallel_config, scheduler_config,
112
- device_config, lora_config,
113
- vision_language_config)
118
+ self.model_executor = executor_class(
119
+ model_config=model_config,
120
+ cache_config=cache_config,
121
+ parallel_config=parallel_config,
122
+ scheduler_config=scheduler_config,
123
+ device_config=device_config,
124
+ lora_config=lora_config,
125
+ vision_language_config=vision_language_config,
126
+ speculative_config=speculative_config,
127
+ )
128
129
# If usage stat is enabled, collect relevant info.
130
if is_usage_stats_enabled():
@@ -171,30 +185,28 @@ class LLMEngine:
185
) -> "LLMEngine":
186
"""Creates an LLM engine from the engine arguments."""
187
# Create the engine configs.
174
- engine_configs = engine_args.create_engine_configs()
175
- parallel_config = engine_configs[2]
176
- device_config = engine_configs[4]
188
+ engine_config = engine_args.create_engine_config()
189
190
# Initialize the cluster and specify the executor class.
179
- if device_config.device_type == "neuron":
191
+ if engine_config.device_config.device_type == "neuron":
192
from vllm.executor.neuron_executor import NeuronExecutor
193
executor_class = NeuronExecutor
182
- elif device_config.device_type == "cpu":
194
+ elif engine_config.device_config.device_type == "cpu":
195
from vllm.executor.cpu_executor import CPUExecutor
196
executor_class = CPUExecutor
185
- elif parallel_config.worker_use_ray:
186
- initialize_ray_cluster(parallel_config)
197
+ elif engine_config.parallel_config.worker_use_ray:
198
+ initialize_ray_cluster(engine_config.parallel_config)
199
from vllm.executor.ray_gpu_executor import RayGPUExecutor
200
executor_class = RayGPUExecutor
201
else:
190
- assert parallel_config.world_size == 1, (
202
+ assert engine_config.parallel_config.world_size == 1, (
203
"Ray is required if parallel_config.world_size > 1.")
204
from vllm.executor.gpu_executor import GPUExecutor
205
executor_class = GPUExecutor
206
207
# Create the LLM engine.
208
engine = cls(
197
- *engine_configs,
209
+ **engine_config.to_dict(),
210
executor_class=executor_class,
211
log_stats=not engine_args.disable_log_stats,
212
usage_context=usage_context,
vllm/executor/executor_base.py
+3
-1
@@ -2,7 +2,8 @@ from abc import ABC, abstractmethod
2
from typing import Dict, List, Optional
3
4
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
5
- ParallelConfig, SchedulerConfig, VisionLanguageConfig)
5
+ ParallelConfig, SchedulerConfig, SpeculativeConfig,
6
+ VisionLanguageConfig)
7
from vllm.lora.request import LoRARequest
8
from vllm.sequence import SamplerOutput, SequenceGroupMetadata
9
@@ -25,6 +26,7 @@ class ExecutorBase(ABC):
26
device_config: DeviceConfig,
27
lora_config: Optional[LoRAConfig],
28
vision_language_config: Optional[VisionLanguageConfig],
29
+ speculative_config: Optional[SpeculativeConfig],
30
) -> None:
31
raise NotImplementedError
32
vllm/executor/gpu_executor.py
+6
-1
@@ -1,7 +1,8 @@
1
from typing import Dict, List, Optional
2
3
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4
- ParallelConfig, SchedulerConfig, VisionLanguageConfig)
4
+ ParallelConfig, SchedulerConfig, SpeculativeConfig,
5
+ VisionLanguageConfig)
6
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
7
from vllm.executor.utils import check_block_size_valid
8
from vllm.logger import init_logger
@@ -24,6 +25,7 @@ class GPUExecutor(ExecutorBase):
25
device_config: DeviceConfig,
26
lora_config: Optional[LoRAConfig],
27
vision_language_config: Optional[VisionLanguageConfig],
28
+ speculative_config: Optional[SpeculativeConfig],
29
) -> None:
30
self.model_config = model_config
31
self.cache_config = cache_config
@@ -33,6 +35,9 @@ class GPUExecutor(ExecutorBase):
35
self.device_config = device_config
36
self.vision_language_config = vision_language_config
37
38
+ assert (not speculative_config
39
+ ), "Speculative decoding not yet supported for GPU backend"
40
+
41
# Instantiate the worker and load the model to GPU.
42
self._init_worker()
43
vllm/executor/neuron_executor.py
+5
-1
@@ -1,7 +1,8 @@
1
from typing import Dict, List, Optional
2
3
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
4
- ParallelConfig, SchedulerConfig, VisionLanguageConfig)
4
+ ParallelConfig, SchedulerConfig, SpeculativeConfig,
5
+ VisionLanguageConfig)
6
from vllm.executor.executor_base import ExecutorBase
7
from vllm.logger import init_logger
8
from vllm.lora.request import LoRARequest
@@ -21,6 +22,7 @@ class NeuronExecutor(ExecutorBase):
22
device_config: DeviceConfig,
23
lora_config: Optional[LoRAConfig],
24
vision_language_config: Optional[VisionLanguageConfig],
25
+ speculative_config: Optional[SpeculativeConfig],
26
) -> None:
27
self.model_config = model_config
28
self.cache_config = cache_config
@@ -28,6 +30,8 @@ class NeuronExecutor(ExecutorBase):
30
self.parallel_config = parallel_config
31
self.scheduler_config = scheduler_config
32
self.device_config = device_config
33
+ assert (not speculative_config
34
+ ), "Speculative decoding not yet supported for Neuron backend."
35
36
# Set the number of GPU blocks to be the same as the maximum number of
37
# sequences that can be processed in a single batch. This is equivalent
vllm/executor/ray_gpu_executor.py
+5
-1
@@ -6,7 +6,8 @@ from collections import defaultdict
6
from typing import TYPE_CHECKING, Any, Dict, List, Optional
7
8
from vllm.config import (CacheConfig, DeviceConfig, LoRAConfig, ModelConfig,
9
- ParallelConfig, SchedulerConfig, VisionLanguageConfig)
9
+ ParallelConfig, SchedulerConfig, SpeculativeConfig,
10
+ VisionLanguageConfig)
11
from vllm.engine.ray_utils import RayWorkerVllm, ray
12
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
13
from vllm.executor.utils import check_block_size_valid
@@ -41,6 +42,7 @@ class RayGPUExecutor(ExecutorBase):
42
device_config: DeviceConfig,
43
lora_config: Optional[LoRAConfig],
44
vision_language_config: Optional[VisionLanguageConfig],
45
+ speculative_config: Optional[SpeculativeConfig],
46
) -> None:
47
self.model_config = model_config
48
self.cache_config = cache_config
@@ -49,6 +51,8 @@ class RayGPUExecutor(ExecutorBase):
51
self.scheduler_config = scheduler_config
52
self.device_config = device_config
53
self.vision_language_config = vision_language_config
54
+ assert (not speculative_config
55
+ ), "Speculative decoding not yet supported for RayGPU backend."
56
57
assert self.parallel_config.worker_use_ray
58
placement_group = self.parallel_config.placement_group