@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 76a5e13270
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[core] move parallel sampling out from vllm core (#9302)

youkaichao committed Oct 21, 2024 at 17:31 UTC 76a5e13270f32216bb28cfe185bada5e88e407d7
4 files changed +222 -29
tests/entrypoints/openai/test_completion.py
+34
@@ -340,6 +340,40 @@ async def test_completion_streaming(client: openai.AsyncOpenAI,
340 assert "".join(chunks) == single_output
341
342
343 +@pytest.mark.asyncio
344 +@pytest.mark.parametrize(
345 + "model_name",
346 + [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
347 +)
348 +async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):
349 + """Streaming for parallel sampling.
350 + The tokens from multiple samples, are flattened into a single stream,
351 + with an index to indicate which sample the token belongs to.
352 + """
353 +
354 + prompt = "What is an LLM?"
355 + n = 3
356 + max_tokens = 5
357 +
358 + stream = await client.completions.create(model=model_name,
359 + prompt=prompt,
360 + max_tokens=max_tokens,
361 + n=n,
362 + stream=True)
363 + chunks: List[List[str]] = [[] for i in range(n)]
364 + finish_reason_count = 0
365 + async for chunk in stream:
366 + index = chunk.choices[0].index
367 + text = chunk.choices[0].text
368 + chunks[index].append(text)
369 + if chunk.choices[0].finish_reason is not None:
370 + finish_reason_count += 1
371 + assert finish_reason_count == n
372 + for chunk in chunks:
373 + assert len(chunk) == max_tokens
374 + print("".join(chunk))
375 +
376 +
377 @pytest.mark.asyncio
378 @pytest.mark.parametrize(
379 "model_name",
vllm/engine/llm_engine.py
+42 -10
@@ -44,8 +44,10 @@ from vllm.pooling_params import PoolingParams
44 from vllm.prompt_adapter.request import PromptAdapterRequest
45 from vllm.sampling_params import RequestOutputKind, SamplingParams
46 from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
47 - Sequence, SequenceGroup, SequenceGroupMetadata,
48 - SequenceGroupOutput, SequenceStatus)
47 + ParallelSampleSequenceGroup, Sequence,
48 + SequenceGroup, SequenceGroupBase,
49 + SequenceGroupMetadata, SequenceGroupOutput,
50 + SequenceStatus)
51 from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
52 init_tracer)
53 from vllm.transformers_utils.config import try_get_generation_config
@@ -474,6 +476,8 @@ class LLMEngine:
476 ),
477 ))
478
479 + self.seq_id_to_seq_group: Dict[str, SequenceGroupBase] = {}
480 +
481 def _initialize_kv_caches(self) -> None:
482 """Initialize the KV cache in the worker(s).
483
@@ -642,7 +646,10 @@ class LLMEngine:
646 prompt_adapter_request: Optional[PromptAdapterRequest],
647 trace_headers: Optional[Mapping[str, str]] = None,
648 priority: int = 0,
645 - ) -> None:
649 + ) -> SequenceGroup:
650 + """Add a processed request to the engine's request pool.
651 + return the created sequence group.
652 + """
653 self._validate_model_inputs(processed_inputs)
654 # Create the sequences.
655 block_size = self.cache_config.block_size
@@ -696,6 +703,8 @@ class LLMEngine:
703 min_cost_scheduler = self.scheduler[costs.index(min(costs))]
704 min_cost_scheduler.add_seq_group(seq_group)
705
706 + return seq_group
707 +
708 def stop_remote_worker_execution_loop(self) -> None:
709 self.model_executor.stop_remote_worker_execution_loop()
710
@@ -711,7 +720,7 @@ class LLMEngine:
720 trace_headers: Optional[Mapping[str, str]] = None,
721 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
722 priority: int = 0,
714 - ) -> None:
723 + ) -> Optional[SequenceGroup]:
724 ...
725
726 @overload
@@ -725,7 +734,7 @@ class LLMEngine:
734 trace_headers: Optional[Mapping[str, str]] = None,
735 prompt_adapter_request: Optional[PromptAdapterRequest] = None,
736 priority: int = 0,
728 - ) -> None:
737 + ) -> Optional[SequenceGroup]:
738 ...
739
740 @deprecate_kwargs(
@@ -744,7 +753,7 @@ class LLMEngine:
753 priority: int = 0,
754 *,
755 inputs: Optional[PromptType] = None, # DEPRECATED
747 - ) -> None:
756 + ) -> Optional[SequenceGroup]:
757 """Add a request to the engine's request pool.
758
759 The request is added to the request pool and will be processed by the
@@ -788,6 +797,22 @@ class LLMEngine:
797 >>> # continue the request processing
798 >>> ...
799 """
800 +
801 + if isinstance(params, SamplingParams) and params.n > 1:
802 + ParallelSampleSequenceGroup.add_request(
803 + request_id,
804 + self,
805 + params,
806 + prompt=prompt,
807 + arrival_time=arrival_time,
808 + lora_request=lora_request,
809 + trace_headers=trace_headers,
810 + prompt_adapter_request=prompt_adapter_request,
811 + priority=priority,
812 + inputs=inputs,
813 + )
814 + return None
815 +
816 if inputs is not None:
817 prompt = inputs
818 assert prompt is not None and params is not None
@@ -818,7 +843,7 @@ class LLMEngine:
843 processed_inputs["mm_processor_kwargs"] = preprocessed_inputs.get(
844 "mm_processor_kwargs")
845
821 - self._add_processed_request(
846 + return self._add_processed_request(
847 request_id=request_id,
848 processed_inputs=processed_inputs,
849 params=params,
@@ -1135,7 +1160,9 @@ class LLMEngine:
1160 seq_group = scheduled_seq_group.seq_group
1161 seq_group.maybe_set_first_token_time(now)
1162 request_output = RequestOutputFactory.create(
1138 - seq_group, use_cache=self.use_cached_outputs)
1163 + seq_group,
1164 + self.seq_id_to_seq_group,
1165 + use_cache=self.use_cached_outputs)
1166 if request_output:
1167 ctx.request_outputs.append(request_output)
1168
@@ -1175,7 +1202,9 @@ class LLMEngine:
1202 seq_group = scheduled_seq_group.seq_group
1203 seq_group.maybe_set_first_token_time(now)
1204 request_output = RequestOutputFactory.create(
1178 - seq_group, use_cache=self.use_cached_outputs)
1205 + seq_group,
1206 + self.seq_id_to_seq_group,
1207 + use_cache=self.use_cached_outputs)
1208 if request_output:
1209 ctx.request_outputs.append(request_output)
1210
@@ -1194,7 +1223,10 @@ class LLMEngine:
1223 continue
1224
1225 request_output = RequestOutputFactory.create(
1197 - seq_group, use_cache=self.use_cached_outputs)
1226 + seq_group,
1227 + self.seq_id_to_seq_group,
1228 + use_cache=self.use_cached_outputs,
1229 + )
1230 if request_output:
1231 ctx.request_outputs.append(request_output)
1232
vllm/outputs.py
+26 -17
@@ -1,13 +1,13 @@
1 import time
2 from dataclasses import dataclass
3 -from typing import List, Optional
3 +from typing import Dict, List, Optional
4 from typing import Sequence as GenericSequence
5 from typing import Union
6
7 from vllm.lora.request import LoRARequest
8 from vllm.sampling_params import RequestOutputKind
9 from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
10 - SequenceGroup, SequenceStatus)
10 + SequenceGroup, SequenceGroupBase, SequenceStatus)
11
12
13 @dataclass
@@ -114,14 +114,28 @@ class RequestOutput:
114 self.encoder_prompt_token_ids = encoder_prompt_token_ids
115
116 @classmethod
117 - def from_seq_group(cls, seq_group: SequenceGroup,
118 - use_cache: bool) -> Optional["RequestOutput"]:
117 + def from_seq_group(
118 + cls, seq_group: SequenceGroup, use_cache: bool,
119 + seq_id_to_seq_group: Dict[str, SequenceGroupBase]
120 + ) -> Optional["RequestOutput"]:
121 + finished = seq_group.is_finished()
122 +
123 + if seq_group.request_id in seq_id_to_seq_group:
124 + group: SequenceGroupBase = seq_id_to_seq_group[
125 + seq_group.request_id]
126 + if finished:
127 + group.finish_seq(seq_group)
128 + assembled_seq_group = group.maybe_assemble_group(seq_group)
129 + if assembled_seq_group is None:
130 + return None
131 + return cls.from_seq_group(assembled_seq_group, use_cache,
132 + seq_id_to_seq_group)
133 +
134 sampling_params = seq_group.sampling_params
135 if sampling_params is None:
136 raise ValueError(
137 "Sampling parameters are missing for a CompletionRequest.")
138
124 - finished = seq_group.is_finished()
139 if sampling_params.output_kind == RequestOutputKind.FINAL_ONLY and (
140 not finished):
141 return None
@@ -136,15 +150,7 @@ class RequestOutput:
150 outputs=[],
151 finished=False)
152
139 - seqs = seq_group.get_seqs()
140 - if len(seqs) == 1:
141 - top_n_seqs = seqs
142 - else:
143 - # Get the top-n sequences.
144 - n = sampling_params._real_n or sampling_params.n
145 - sorting_key = lambda seq: seq.get_cumulative_logprob()
146 - sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
147 - top_n_seqs = sorted_seqs[:n]
153 + top_n_seqs = seq_group.get_seqs()
154
155 # Create the outputs.
156 # NOTE: We need omit logprobs here explicitly because the sequence
@@ -208,7 +214,7 @@ class RequestOutput:
214
215 else:
216 output = CompletionOutput(
211 - seqs.index(seq), output_text, [output_token_ids]
217 + top_n_seqs.index(seq), output_text, [output_token_ids]
218 if isinstance(output_token_ids, int) else output_token_ids,
219 seq.get_cumulative_logprob() if include_logprobs else None,
220 output_logprobs,
@@ -309,10 +315,13 @@ class EmbeddingRequestOutput:
315 class RequestOutputFactory:
316
317 @staticmethod
312 - def create(seq_group: SequenceGroup, use_cache: bool = False):
318 + def create(seq_group: SequenceGroup,
319 + seq_id_to_seq_group: Dict[str, SequenceGroupBase],
320 + use_cache: bool = False):
321 # Determine the type based on a condition, for example:
322 if hasattr(seq_group,
323 'embeddings') and seq_group.embeddings is not None:
324 return EmbeddingRequestOutput.from_seq_group(seq_group)
325 else:
318 - return RequestOutput.from_seq_group(seq_group, use_cache)
326 + return RequestOutput.from_seq_group(seq_group, use_cache,
327 + seq_id_to_seq_group)
vllm/sequence.py
+120 -2
@@ -4,7 +4,7 @@ import enum
4 from abc import ABC, abstractmethod
5 from array import array
6 from collections import defaultdict
7 -from dataclasses import dataclass
7 +from dataclasses import dataclass, field
8 from functools import cached_property, reduce
9 from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional
10 from typing import Sequence as GenericSequence
@@ -17,7 +17,7 @@ from vllm.inputs.parse import is_encoder_decoder_inputs
17 from vllm.lora.request import LoRARequest
18 from vllm.pooling_params import PoolingParams
19 from vllm.prompt_adapter.request import PromptAdapterRequest
20 -from vllm.sampling_params import SamplingParams
20 +from vllm.sampling_params import RequestOutputKind, SamplingParams
21 from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
22
23 if TYPE_CHECKING:
@@ -1401,3 +1401,121 @@ class ExecuteModelRequest(
1401 last_sampled_token_ids=self.last_sampled_token_ids.clone()
1402 if self.last_sampled_token_ids is not None else None,
1403 async_callback=self.async_callback)
1404 +
1405 +
1406 +@dataclass
1407 +class SequenceGroupBase:
1408 + group_id: str # the original request id before splitting
1409 +
1410 + assembled_seq_group: Optional[SequenceGroup] = None
1411 +
1412 + # seq id to a unique index inside this group
1413 + seq_id_to_index: Dict[str, int] = field(default_factory=dict)
1414 +
1415 + # seq ids to be finished
1416 + to_be_finished: Dict[str, SequenceGroup] = field(default_factory=dict)
1417 +
1418 + # seq id to finished sequences
1419 + finished_reqs: Dict[str, SequenceGroup] = field(default_factory=dict)
1420 +
1421 + streaming: bool = False
1422 +
1423 + output_produced: bool = False
1424 +
1425 + @staticmethod
1426 + def add_request(request_id: str, engine, params, *args, **kwargs):
1427 + """When we are ready to add a request with request_id and params
1428 + into the engine, we can split the request into multiple requests.
1429 + """
1430 + raise NotImplementedError
1431 +
1432 + def finish_seq(self, seq: SequenceGroup):
1433 + """The sequence `seq` finishes, we should record the information.
1434 + """
1435 + del self.to_be_finished[seq.request_id]
1436 + self.finished_reqs[seq.request_id] = seq
1437 +
1438 + def maybe_assemble_group(
1439 + self, seq_group: SequenceGroup) -> Optional[SequenceGroup]:
1440 + """Assemble the sequence group, for producing the final
1441 + output, or adding request in the engine again.
1442 + """
1443 + raise NotImplementedError
1444 +
1445 +
1446 +class ParallelSampleSequenceGroup(SequenceGroupBase):
1447 +
1448 + @staticmethod
1449 + def add_request(request_id: str, engine, params, **kwargs):
1450 + original_params = params
1451 + params = copy.deepcopy(original_params)
1452 + params.n = 1
1453 + group = ParallelSampleSequenceGroup(request_id)
1454 + seqs = []
1455 + for i in range(original_params.n):
1456 + request_id_i = f"{request_id}_parallel_sample_{i}"
1457 + group.seq_id_to_index[request_id_i] = i
1458 + seq_group = engine.add_request(
1459 + request_id_i,
1460 + params=params,
1461 + **kwargs,
1462 + ) # type: ignore
1463 + assert seq_group is not None
1464 + engine.seq_id_to_seq_group[request_id_i] = group
1465 + group.to_be_finished[request_id_i] = seq_group
1466 + seqs.append(seq_group.seqs[0])
1467 +
1468 + # for parallel sampling, the `assembled_seq_group` is always
1469 + # available, since we have all the sequences ready, and they
1470 + # will not change.
1471 + group.assembled_seq_group = SequenceGroup(
1472 + request_id=request_id,
1473 + seqs=seqs,
1474 + arrival_time=seq_group.arrival_time,
1475 + sampling_params=original_params,
1476 + lora_request=seq_group.lora_request,
1477 + embeddings=seq_group.embeddings,
1478 + pooling_params=seq_group.pooling_params,
1479 + encoder_seq=seq_group.encoder_seq,
1480 + trace_headers=seq_group.trace_headers,
1481 + prompt_adapter_request=seq_group.prompt_adapter_request,
1482 + priority=seq_group.priority,
1483 + )
1484 +
1485 + group.streaming = params.output_kind == RequestOutputKind.DELTA
1486 + group.output_produced = False
1487 +
1488 + def maybe_assemble_group(
1489 + self, seq_group: SequenceGroup) -> Optional[SequenceGroup]:
1490 +
1491 + # in the streaming mode, we will return the assembled sequence
1492 + # for the first sequence, and then return None for the rest of
1493 + # sequences
1494 + if self.streaming:
1495 + if self.seq_id_to_index[seq_group.request_id] == 0:
1496 + return self.assembled_seq_group
1497 + return None
1498 +
1499 + # in the non-streaming mode, we will return the assembled sequence
1500 + # once after all sequences finish, and then return None for the
1501 + # rest of the time
1502 +
1503 + if len(self.to_be_finished) > 0:
1504 + return None
1505 +
1506 + assert self.assembled_seq_group is not None
1507 + params = self.assembled_seq_group.sampling_params
1508 + assert isinstance(params, SamplingParams)
1509 + if not self.output_produced:
1510 + self.output_produced = True
1511 + if params._real_n is not None:
1512 + # Get the top-n sequences.
1513 + n = params._real_n or params.n
1514 + seqs = self.assembled_seq_group.seqs
1515 + sorting_key = lambda seq: seq.get_cumulative_logprob()
1516 + sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
1517 + top_n_seqs = sorted_seqs[:n]
1518 + self.assembled_seq_group.seqs = top_n_seqs
1519 + return self.assembled_seq_group
1520 + if self.output_produced:
1521 + return None