A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Doc] Explicitly state that PP isn't compatible with speculative decoding yet (#10975)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Dec 8, 2024 at 01:20 UTC
c889d5888bf6bbfbe3f4ea55bf27ce84a239c3d0
8 files changed
+32
-9
docs/source/usage/spec_decode.rst
+3
@@ -8,6 +8,9 @@ Speculative decoding
8
not usually yield inter-token latency reductions for all prompt datasets or sampling parameters. The work
9
to optimize it is ongoing and can be followed in `this issue. <https://github.com/vllm-project/vllm/issues/4630>`_
10
11
+.. warning::
12
+ Currently, speculative decoding in vLLM is not compatible with pipeline parallelism.
13
+
14
This document shows how to use `Speculative Decoding <https://x.com/karpathy/status/1697318534555336961>`_ with vLLM.
15
Speculative decoding is a technique which improves inter-token latency in memory-bound LLM inference.
16
tests/distributed/test_pipeline_parallel.py
+13
-3
@@ -247,9 +247,19 @@ def _compare_tp(
247
*,
248
method: Literal["generate", "encode"],
249
):
250
- tp_size, pp_size, eager_mode, chunked_prefill = parallel_setup
251
- multi_node_only, trust_remote_code, tokenizer_mode, \
252
- load_format, hf_overrides = test_options
250
+ (
251
+ tp_size,
252
+ pp_size,
253
+ eager_mode,
254
+ chunked_prefill,
255
+ ) = parallel_setup
256
+ (
257
+ multi_node_only,
258
+ trust_remote_code,
259
+ tokenizer_mode,
260
+ load_format,
261
+ hf_overrides,
262
+ ) = test_options
263
264
if num_gpus_available < tp_size * pp_size:
265
pytest.skip(f"Need at least {tp_size} x {pp_size} GPUs")
vllm/model_executor/models/exaone.py
+2
-1
@@ -473,10 +473,11 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
473
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
474
config.vocab_size,
475
logit_scale)
476
- self.sampler = get_sampler()
476
else:
477
self.lm_head = PPMissingLayer()
478
479
+ self.sampler = get_sampler()
480
+
481
self.make_empty_intermediate_tensors = (
482
self.transformer.make_empty_intermediate_tensors)
483
vllm/model_executor/models/granite.py
+3
-2
@@ -400,16 +400,17 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
400
self.lm_head.weight = self.model.embed_tokens.weight
401
402
logit_scale = getattr(config, "logit_scale", 1.0)
403
-
403
if hasattr(config, "logits_scaling"):
404
logit_scale /= config.logits_scaling
405
+
406
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
407
config.vocab_size,
408
scale=logit_scale)
409
- self.sampler = get_sampler()
409
else:
410
self.lm_head = PPMissingLayer()
411
412
+ self.sampler = get_sampler()
413
+
414
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
415
return self.model.get_input_embeddings(input_ids)
416
vllm/model_executor/models/llama.py
+2
-1
@@ -540,10 +540,11 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
540
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
541
config.vocab_size,
542
logit_scale)
543
- self.sampler = get_sampler()
543
else:
544
self.lm_head = PPMissingLayer()
545
546
+ self.sampler = get_sampler()
547
+
548
self.make_empty_intermediate_tensors = (
549
self.model.make_empty_intermediate_tensors)
550
vllm/model_executor/models/nemotron.py
+3
-1
@@ -435,9 +435,11 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
435
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
436
config.vocab_size,
437
logit_scale)
438
- self.sampler = get_sampler()
438
else:
439
self.lm_head = PPMissingLayer()
440
+
441
+ self.sampler = get_sampler()
442
+
443
self.make_empty_intermediate_tensors = (
444
self.model.make_empty_intermediate_tensors)
445
vllm/model_executor/models/solar.py
+2
-1
@@ -443,10 +443,11 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
443
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
444
config.vocab_size,
445
logit_scale)
446
- self.sampler = get_sampler()
446
else:
447
self.lm_head = PPMissingLayer()
448
449
+ self.sampler = get_sampler()
450
+
451
self.make_empty_intermediate_tensors = (
452
self.model.make_empty_intermediate_tensors)
453
vllm/spec_decode/spec_decode_worker.py
+4
@@ -54,6 +54,10 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
54
speculative_config: SpeculativeConfig = vllm_config.speculative_config
55
assert speculative_config is not None
56
57
+ if vllm_config.parallel_config.pipeline_parallel_size > 1:
58
+ raise NotImplementedError("Speculative decoding is currently "
59
+ "incompatible with pipeline parallelism")
60
+
61
draft_worker_kwargs = kwargs.copy()
62
63
kwargs["model_runner_cls"] = TargetModelRunner