@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c889d5888b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Doc] Explicitly state that PP isn't compatible with speculative decoding yet (#10975)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Dec 8, 2024 at 01:20 UTC c889d5888bf6bbfbe3f4ea55bf27ce84a239c3d0
8 files changed +32 -9
docs/source/usage/spec_decode.rst
+3
@@ -8,6 +8,9 @@ Speculative decoding
8 not usually yield inter-token latency reductions for all prompt datasets or sampling parameters. The work
9 to optimize it is ongoing and can be followed in `this issue. <https://github.com/vllm-project/vllm/issues/4630>`_
10
11 +.. warning::
12 + Currently, speculative decoding in vLLM is not compatible with pipeline parallelism.
13 +
14 This document shows how to use `Speculative Decoding <https://x.com/karpathy/status/1697318534555336961>`_ with vLLM.
15 Speculative decoding is a technique which improves inter-token latency in memory-bound LLM inference.
16
tests/distributed/test_pipeline_parallel.py
+13 -3
@@ -247,9 +247,19 @@ def _compare_tp(
247 *,
248 method: Literal["generate", "encode"],
249 ):
250 - tp_size, pp_size, eager_mode, chunked_prefill = parallel_setup
251 - multi_node_only, trust_remote_code, tokenizer_mode, \
252 - load_format, hf_overrides = test_options
250 + (
251 + tp_size,
252 + pp_size,
253 + eager_mode,
254 + chunked_prefill,
255 + ) = parallel_setup
256 + (
257 + multi_node_only,
258 + trust_remote_code,
259 + tokenizer_mode,
260 + load_format,
261 + hf_overrides,
262 + ) = test_options
263
264 if num_gpus_available < tp_size * pp_size:
265 pytest.skip(f"Need at least {tp_size} x {pp_size} GPUs")
vllm/model_executor/models/exaone.py
+2 -1
@@ -473,10 +473,11 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
473 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
474 config.vocab_size,
475 logit_scale)
476 - self.sampler = get_sampler()
476 else:
477 self.lm_head = PPMissingLayer()
478
479 + self.sampler = get_sampler()
480 +
481 self.make_empty_intermediate_tensors = (
482 self.transformer.make_empty_intermediate_tensors)
483
vllm/model_executor/models/granite.py
+3 -2
@@ -400,16 +400,17 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
400 self.lm_head.weight = self.model.embed_tokens.weight
401
402 logit_scale = getattr(config, "logit_scale", 1.0)
403 -
403 if hasattr(config, "logits_scaling"):
404 logit_scale /= config.logits_scaling
405 +
406 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
407 config.vocab_size,
408 scale=logit_scale)
409 - self.sampler = get_sampler()
409 else:
410 self.lm_head = PPMissingLayer()
411
412 + self.sampler = get_sampler()
413 +
414 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
415 return self.model.get_input_embeddings(input_ids)
416
vllm/model_executor/models/llama.py
+2 -1
@@ -540,10 +540,11 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
540 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
541 config.vocab_size,
542 logit_scale)
543 - self.sampler = get_sampler()
543 else:
544 self.lm_head = PPMissingLayer()
545
546 + self.sampler = get_sampler()
547 +
548 self.make_empty_intermediate_tensors = (
549 self.model.make_empty_intermediate_tensors)
550
vllm/model_executor/models/nemotron.py
+3 -1
@@ -435,9 +435,11 @@ class NemotronForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
435 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
436 config.vocab_size,
437 logit_scale)
438 - self.sampler = get_sampler()
438 else:
439 self.lm_head = PPMissingLayer()
440 +
441 + self.sampler = get_sampler()
442 +
443 self.make_empty_intermediate_tensors = (
444 self.model.make_empty_intermediate_tensors)
445
vllm/model_executor/models/solar.py
+2 -1
@@ -443,10 +443,11 @@ class SolarForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
443 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
444 config.vocab_size,
445 logit_scale)
446 - self.sampler = get_sampler()
446 else:
447 self.lm_head = PPMissingLayer()
448
449 + self.sampler = get_sampler()
450 +
451 self.make_empty_intermediate_tensors = (
452 self.model.make_empty_intermediate_tensors)
453
vllm/spec_decode/spec_decode_worker.py
+4
@@ -54,6 +54,10 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
54 speculative_config: SpeculativeConfig = vllm_config.speculative_config
55 assert speculative_config is not None
56
57 + if vllm_config.parallel_config.pipeline_parallel_size > 1:
58 + raise NotImplementedError("Speculative decoding is currently "
59 + "incompatible with pipeline parallelism")
60 +
61 draft_worker_kwargs = kwargs.copy()
62
63 kwargs["model_runner_cls"] = TargetModelRunner