A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[core] move parallel sampling out from vllm core (#9302)
youkaichao committed
Oct 21, 2024 at 17:31 UTC
76a5e13270f32216bb28cfe185bada5e88e407d7
4 files changed
+222
-29
tests/entrypoints/openai/test_completion.py
+34
@@ -340,6 +340,40 @@ async def test_completion_streaming(client: openai.AsyncOpenAI,
340
assert "".join(chunks) == single_output
341
342
343
+@pytest.mark.asyncio
344
+@pytest.mark.parametrize(
345
+ "model_name",
346
+ [MODEL_NAME, "zephyr-lora", "zephyr-pa"],
347
+)
348
+async def test_parallel_streaming(client: openai.AsyncOpenAI, model_name: str):
349
+ """Streaming for parallel sampling.
350
+ The tokens from multiple samples, are flattened into a single stream,
351
+ with an index to indicate which sample the token belongs to.
352
+ """
353
+
354
+ prompt = "What is an LLM?"
355
+ n = 3
356
+ max_tokens = 5
357
+
358
+ stream = await client.completions.create(model=model_name,
359
+ prompt=prompt,
360
+ max_tokens=max_tokens,
361
+ n=n,
362
+ stream=True)
363
+ chunks: List[List[str]] = [[] for i in range(n)]
364
+ finish_reason_count = 0
365
+ async for chunk in stream:
366
+ index = chunk.choices[0].index
367
+ text = chunk.choices[0].text
368
+ chunks[index].append(text)
369
+ if chunk.choices[0].finish_reason is not None:
370
+ finish_reason_count += 1
371
+ assert finish_reason_count == n
372
+ for chunk in chunks:
373
+ assert len(chunk) == max_tokens
374
+ print("".join(chunk))
375
+
376
+
377
@pytest.mark.asyncio
378
@pytest.mark.parametrize(
379
"model_name",
vllm/engine/llm_engine.py
+42
-10
@@ -44,8 +44,10 @@ from vllm.pooling_params import PoolingParams
44
from vllm.prompt_adapter.request import PromptAdapterRequest
45
from vllm.sampling_params import RequestOutputKind, SamplingParams
46
from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
47
- Sequence, SequenceGroup, SequenceGroupMetadata,
48
- SequenceGroupOutput, SequenceStatus)
47
+ ParallelSampleSequenceGroup, Sequence,
48
+ SequenceGroup, SequenceGroupBase,
49
+ SequenceGroupMetadata, SequenceGroupOutput,
50
+ SequenceStatus)
51
from vllm.tracing import (SpanAttributes, SpanKind, extract_trace_context,
52
init_tracer)
53
from vllm.transformers_utils.config import try_get_generation_config
@@ -474,6 +476,8 @@ class LLMEngine:
476
),
477
))
478
479
+ self.seq_id_to_seq_group: Dict[str, SequenceGroupBase] = {}
480
+
481
def _initialize_kv_caches(self) -> None:
482
"""Initialize the KV cache in the worker(s).
483
@@ -642,7 +646,10 @@ class LLMEngine:
646
prompt_adapter_request: Optional[PromptAdapterRequest],
647
trace_headers: Optional[Mapping[str, str]] = None,
648
priority: int = 0,
645
- ) -> None:
649
+ ) -> SequenceGroup:
650
+ """Add a processed request to the engine's request pool.
651
+ return the created sequence group.
652
+ """
653
self._validate_model_inputs(processed_inputs)
654
# Create the sequences.
655
block_size = self.cache_config.block_size
@@ -696,6 +703,8 @@ class LLMEngine:
703
min_cost_scheduler = self.scheduler[costs.index(min(costs))]
704
min_cost_scheduler.add_seq_group(seq_group)
705
706
+ return seq_group
707
+
708
def stop_remote_worker_execution_loop(self) -> None:
709
self.model_executor.stop_remote_worker_execution_loop()
710
@@ -711,7 +720,7 @@ class LLMEngine:
720
trace_headers: Optional[Mapping[str, str]] = None,
721
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
722
priority: int = 0,
714
- ) -> None:
723
+ ) -> Optional[SequenceGroup]:
724
...
725
726
@overload
@@ -725,7 +734,7 @@ class LLMEngine:
734
trace_headers: Optional[Mapping[str, str]] = None,
735
prompt_adapter_request: Optional[PromptAdapterRequest] = None,
736
priority: int = 0,
728
- ) -> None:
737
+ ) -> Optional[SequenceGroup]:
738
...
739
740
@deprecate_kwargs(
@@ -744,7 +753,7 @@ class LLMEngine:
753
priority: int = 0,
754
*,
755
inputs: Optional[PromptType] = None, # DEPRECATED
747
- ) -> None:
756
+ ) -> Optional[SequenceGroup]:
757
"""Add a request to the engine's request pool.
758
759
The request is added to the request pool and will be processed by the
@@ -788,6 +797,22 @@ class LLMEngine:
797
>>> # continue the request processing
798
>>> ...
799
"""
800
+
801
+ if isinstance(params, SamplingParams) and params.n > 1:
802
+ ParallelSampleSequenceGroup.add_request(
803
+ request_id,
804
+ self,
805
+ params,
806
+ prompt=prompt,
807
+ arrival_time=arrival_time,
808
+ lora_request=lora_request,
809
+ trace_headers=trace_headers,
810
+ prompt_adapter_request=prompt_adapter_request,
811
+ priority=priority,
812
+ inputs=inputs,
813
+ )
814
+ return None
815
+
816
if inputs is not None:
817
prompt = inputs
818
assert prompt is not None and params is not None
@@ -818,7 +843,7 @@ class LLMEngine:
843
processed_inputs["mm_processor_kwargs"] = preprocessed_inputs.get(
844
"mm_processor_kwargs")
845
821
- self._add_processed_request(
846
+ return self._add_processed_request(
847
request_id=request_id,
848
processed_inputs=processed_inputs,
849
params=params,
@@ -1135,7 +1160,9 @@ class LLMEngine:
1160
seq_group = scheduled_seq_group.seq_group
1161
seq_group.maybe_set_first_token_time(now)
1162
request_output = RequestOutputFactory.create(
1138
- seq_group, use_cache=self.use_cached_outputs)
1163
+ seq_group,
1164
+ self.seq_id_to_seq_group,
1165
+ use_cache=self.use_cached_outputs)
1166
if request_output:
1167
ctx.request_outputs.append(request_output)
1168
@@ -1175,7 +1202,9 @@ class LLMEngine:
1202
seq_group = scheduled_seq_group.seq_group
1203
seq_group.maybe_set_first_token_time(now)
1204
request_output = RequestOutputFactory.create(
1178
- seq_group, use_cache=self.use_cached_outputs)
1205
+ seq_group,
1206
+ self.seq_id_to_seq_group,
1207
+ use_cache=self.use_cached_outputs)
1208
if request_output:
1209
ctx.request_outputs.append(request_output)
1210
@@ -1194,7 +1223,10 @@ class LLMEngine:
1223
continue
1224
1225
request_output = RequestOutputFactory.create(
1197
- seq_group, use_cache=self.use_cached_outputs)
1226
+ seq_group,
1227
+ self.seq_id_to_seq_group,
1228
+ use_cache=self.use_cached_outputs,
1229
+ )
1230
if request_output:
1231
ctx.request_outputs.append(request_output)
1232
vllm/outputs.py
+26
-17
@@ -1,13 +1,13 @@
1
import time
2
from dataclasses import dataclass
3
-from typing import List, Optional
3
+from typing import Dict, List, Optional
4
from typing import Sequence as GenericSequence
5
from typing import Union
6
7
from vllm.lora.request import LoRARequest
8
from vllm.sampling_params import RequestOutputKind
9
from vllm.sequence import (PromptLogprobs, RequestMetrics, SampleLogprobs,
10
- SequenceGroup, SequenceStatus)
10
+ SequenceGroup, SequenceGroupBase, SequenceStatus)
11
12
13
@dataclass
@@ -114,14 +114,28 @@ class RequestOutput:
114
self.encoder_prompt_token_ids = encoder_prompt_token_ids
115
116
@classmethod
117
- def from_seq_group(cls, seq_group: SequenceGroup,
118
- use_cache: bool) -> Optional["RequestOutput"]:
117
+ def from_seq_group(
118
+ cls, seq_group: SequenceGroup, use_cache: bool,
119
+ seq_id_to_seq_group: Dict[str, SequenceGroupBase]
120
+ ) -> Optional["RequestOutput"]:
121
+ finished = seq_group.is_finished()
122
+
123
+ if seq_group.request_id in seq_id_to_seq_group:
124
+ group: SequenceGroupBase = seq_id_to_seq_group[
125
+ seq_group.request_id]
126
+ if finished:
127
+ group.finish_seq(seq_group)
128
+ assembled_seq_group = group.maybe_assemble_group(seq_group)
129
+ if assembled_seq_group is None:
130
+ return None
131
+ return cls.from_seq_group(assembled_seq_group, use_cache,
132
+ seq_id_to_seq_group)
133
+
134
sampling_params = seq_group.sampling_params
135
if sampling_params is None:
136
raise ValueError(
137
"Sampling parameters are missing for a CompletionRequest.")
138
124
- finished = seq_group.is_finished()
139
if sampling_params.output_kind == RequestOutputKind.FINAL_ONLY and (
140
not finished):
141
return None
@@ -136,15 +150,7 @@ class RequestOutput:
150
outputs=[],
151
finished=False)
152
139
- seqs = seq_group.get_seqs()
140
- if len(seqs) == 1:
141
- top_n_seqs = seqs
142
- else:
143
- # Get the top-n sequences.
144
- n = sampling_params._real_n or sampling_params.n
145
- sorting_key = lambda seq: seq.get_cumulative_logprob()
146
- sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
147
- top_n_seqs = sorted_seqs[:n]
153
+ top_n_seqs = seq_group.get_seqs()
154
155
# Create the outputs.
156
# NOTE: We need omit logprobs here explicitly because the sequence
@@ -208,7 +214,7 @@ class RequestOutput:
214
215
else:
216
output = CompletionOutput(
211
- seqs.index(seq), output_text, [output_token_ids]
217
+ top_n_seqs.index(seq), output_text, [output_token_ids]
218
if isinstance(output_token_ids, int) else output_token_ids,
219
seq.get_cumulative_logprob() if include_logprobs else None,
220
output_logprobs,
@@ -309,10 +315,13 @@ class EmbeddingRequestOutput:
315
class RequestOutputFactory:
316
317
@staticmethod
312
- def create(seq_group: SequenceGroup, use_cache: bool = False):
318
+ def create(seq_group: SequenceGroup,
319
+ seq_id_to_seq_group: Dict[str, SequenceGroupBase],
320
+ use_cache: bool = False):
321
# Determine the type based on a condition, for example:
322
if hasattr(seq_group,
323
'embeddings') and seq_group.embeddings is not None:
324
return EmbeddingRequestOutput.from_seq_group(seq_group)
325
else:
318
- return RequestOutput.from_seq_group(seq_group, use_cache)
326
+ return RequestOutput.from_seq_group(seq_group, use_cache,
327
+ seq_id_to_seq_group)
vllm/sequence.py
+120
-2
@@ -4,7 +4,7 @@ import enum
4
from abc import ABC, abstractmethod
5
from array import array
6
from collections import defaultdict
7
-from dataclasses import dataclass
7
+from dataclasses import dataclass, field
8
from functools import cached_property, reduce
9
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional
10
from typing import Sequence as GenericSequence
@@ -17,7 +17,7 @@ from vllm.inputs.parse import is_encoder_decoder_inputs
17
from vllm.lora.request import LoRARequest
18
from vllm.pooling_params import PoolingParams
19
from vllm.prompt_adapter.request import PromptAdapterRequest
20
-from vllm.sampling_params import SamplingParams
20
+from vllm.sampling_params import RequestOutputKind, SamplingParams
21
from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics
22
23
if TYPE_CHECKING:
@@ -1401,3 +1401,121 @@ class ExecuteModelRequest(
1401
last_sampled_token_ids=self.last_sampled_token_ids.clone()
1402
if self.last_sampled_token_ids is not None else None,
1403
async_callback=self.async_callback)
1404
+
1405
+
1406
+@dataclass
1407
+class SequenceGroupBase:
1408
+ group_id: str # the original request id before splitting
1409
+
1410
+ assembled_seq_group: Optional[SequenceGroup] = None
1411
+
1412
+ # seq id to a unique index inside this group
1413
+ seq_id_to_index: Dict[str, int] = field(default_factory=dict)
1414
+
1415
+ # seq ids to be finished
1416
+ to_be_finished: Dict[str, SequenceGroup] = field(default_factory=dict)
1417
+
1418
+ # seq id to finished sequences
1419
+ finished_reqs: Dict[str, SequenceGroup] = field(default_factory=dict)
1420
+
1421
+ streaming: bool = False
1422
+
1423
+ output_produced: bool = False
1424
+
1425
+ @staticmethod
1426
+ def add_request(request_id: str, engine, params, *args, **kwargs):
1427
+ """When we are ready to add a request with request_id and params
1428
+ into the engine, we can split the request into multiple requests.
1429
+ """
1430
+ raise NotImplementedError
1431
+
1432
+ def finish_seq(self, seq: SequenceGroup):
1433
+ """The sequence `seq` finishes, we should record the information.
1434
+ """
1435
+ del self.to_be_finished[seq.request_id]
1436
+ self.finished_reqs[seq.request_id] = seq
1437
+
1438
+ def maybe_assemble_group(
1439
+ self, seq_group: SequenceGroup) -> Optional[SequenceGroup]:
1440
+ """Assemble the sequence group, for producing the final
1441
+ output, or adding request in the engine again.
1442
+ """
1443
+ raise NotImplementedError
1444
+
1445
+
1446
+class ParallelSampleSequenceGroup(SequenceGroupBase):
1447
+
1448
+ @staticmethod
1449
+ def add_request(request_id: str, engine, params, **kwargs):
1450
+ original_params = params
1451
+ params = copy.deepcopy(original_params)
1452
+ params.n = 1
1453
+ group = ParallelSampleSequenceGroup(request_id)
1454
+ seqs = []
1455
+ for i in range(original_params.n):
1456
+ request_id_i = f"{request_id}_parallel_sample_{i}"
1457
+ group.seq_id_to_index[request_id_i] = i
1458
+ seq_group = engine.add_request(
1459
+ request_id_i,
1460
+ params=params,
1461
+ **kwargs,
1462
+ ) # type: ignore
1463
+ assert seq_group is not None
1464
+ engine.seq_id_to_seq_group[request_id_i] = group
1465
+ group.to_be_finished[request_id_i] = seq_group
1466
+ seqs.append(seq_group.seqs[0])
1467
+
1468
+ # for parallel sampling, the `assembled_seq_group` is always
1469
+ # available, since we have all the sequences ready, and they
1470
+ # will not change.
1471
+ group.assembled_seq_group = SequenceGroup(
1472
+ request_id=request_id,
1473
+ seqs=seqs,
1474
+ arrival_time=seq_group.arrival_time,
1475
+ sampling_params=original_params,
1476
+ lora_request=seq_group.lora_request,
1477
+ embeddings=seq_group.embeddings,
1478
+ pooling_params=seq_group.pooling_params,
1479
+ encoder_seq=seq_group.encoder_seq,
1480
+ trace_headers=seq_group.trace_headers,
1481
+ prompt_adapter_request=seq_group.prompt_adapter_request,
1482
+ priority=seq_group.priority,
1483
+ )
1484
+
1485
+ group.streaming = params.output_kind == RequestOutputKind.DELTA
1486
+ group.output_produced = False
1487
+
1488
+ def maybe_assemble_group(
1489
+ self, seq_group: SequenceGroup) -> Optional[SequenceGroup]:
1490
+
1491
+ # in the streaming mode, we will return the assembled sequence
1492
+ # for the first sequence, and then return None for the rest of
1493
+ # sequences
1494
+ if self.streaming:
1495
+ if self.seq_id_to_index[seq_group.request_id] == 0:
1496
+ return self.assembled_seq_group
1497
+ return None
1498
+
1499
+ # in the non-streaming mode, we will return the assembled sequence
1500
+ # once after all sequences finish, and then return None for the
1501
+ # rest of the time
1502
+
1503
+ if len(self.to_be_finished) > 0:
1504
+ return None
1505
+
1506
+ assert self.assembled_seq_group is not None
1507
+ params = self.assembled_seq_group.sampling_params
1508
+ assert isinstance(params, SamplingParams)
1509
+ if not self.output_produced:
1510
+ self.output_produced = True
1511
+ if params._real_n is not None:
1512
+ # Get the top-n sequences.
1513
+ n = params._real_n or params.n
1514
+ seqs = self.assembled_seq_group.seqs
1515
+ sorting_key = lambda seq: seq.get_cumulative_logprob()
1516
+ sorted_seqs = sorted(seqs, key=sorting_key, reverse=True)
1517
+ top_n_seqs = sorted_seqs[:n]
1518
+ self.assembled_seq_group.seqs = top_n_seqs
1519
+ return self.assembled_seq_group
1520
+ if self.output_produced:
1521
+ return None