A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 15:45 UTC
[Frontend] Use MQLLMEngine for embeddings models too (#8584)
Nick Hill committed
Sep 19, 2024 at 17:51 UTC
76515f303b44cb3ffc6de63c49148d5081a77119
3 files changed
+90
-46
vllm/engine/multiprocessing/__init__.py
+4
-3
@@ -2,6 +2,7 @@ from dataclasses import dataclass
2
from enum import Enum
3
from typing import List, Mapping, Optional, Union
4
5
+from vllm import PoolingParams
6
from vllm.inputs import PromptInputs
7
from vllm.lora.request import LoRARequest
8
from vllm.outputs import RequestOutput
@@ -21,9 +22,9 @@ class MQEngineDeadError(RuntimeError):
22
23
24
@dataclass
24
-class RPCGenerateRequest:
25
+class RPCProcessRequest:
26
inputs: PromptInputs
26
- sampling_params: SamplingParams
27
+ params: Union[SamplingParams, PoolingParams]
28
request_id: str
29
lora_request: Optional[LoRARequest] = None
30
trace_headers: Optional[Mapping[str, str]] = None
@@ -55,7 +56,7 @@ class RPCStartupResponse:
56
tracing_enabled: bool
57
58
58
-RPC_REQUEST_T = Union[RPCGenerateRequest, RPCAbortRequest, RPCHealthRequest,
59
+RPC_REQUEST_T = Union[RPCProcessRequest, RPCAbortRequest, RPCHealthRequest,
60
RPCStartupRequest]
61
62
REQUEST_OUTPUTS_T = Union[List[RequestOutput], RPCError]
vllm/engine/multiprocessing/client.py
+74
-32
@@ -11,6 +11,7 @@ import zmq.asyncio
11
from zmq import Frame # type: ignore[attr-defined]
12
from zmq.asyncio import Socket
13
14
+from vllm import PoolingParams
15
from vllm.config import DecodingConfig, EngineConfig, ModelConfig
16
from vllm.engine.arg_utils import AsyncEngineArgs
17
# yapf conflicts with isort for this block
@@ -19,8 +20,8 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
20
IPC_HEALTH_EXT, IPC_INPUT_EXT,
21
IPC_OUTPUT_EXT, RPC_REQUEST_T,
22
VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
22
- RPCError, RPCGenerateRequest,
23
- RPCHealthRequest, RPCStartupRequest,
23
+ RPCError, RPCHealthRequest,
24
+ RPCProcessRequest, RPCStartupRequest,
25
RPCStartupResponse)
26
# yapf: enable
27
from vllm.envs import VLLM_RPC_TIMEOUT
@@ -111,20 +112,8 @@ class MQLLMEngineClient:
112
113
@staticmethod
114
def is_unsupported_config(engine_args: AsyncEngineArgs):
114
- if engine_args.pipeline_parallel_size > 1:
115
- return True
116
-
117
- is_embedding = ModelConfig(
118
- model=engine_args.model,
119
- revision=engine_args.revision,
120
- tokenizer=engine_args.model,
121
- tokenizer_mode="auto",
122
- trust_remote_code=engine_args.trust_remote_code,
123
- quantization=engine_args.quantization,
124
- seed=0,
125
- dtype="auto").embedding_mode
126
-
127
- return is_embedding
115
+ # Pipeline parallel not yet supported
116
+ return engine_args.pipeline_parallel_size > 1
117
118
@contextmanager
119
def get_data_socket(self) -> Iterator[Socket]:
@@ -382,12 +371,9 @@ class MQLLMEngineClient:
371
372
@property
373
def dead_error(self) -> BaseException:
385
- if self._errored_with is not None:
386
- return ENGINE_DEAD_ERROR(self._errored_with)
387
- else:
388
- return ENGINE_DEAD_ERROR()
374
+ return ENGINE_DEAD_ERROR(self._errored_with)
375
390
- async def generate(
376
+ def generate(
377
self,
378
inputs: PromptInputs,
379
sampling_params: SamplingParams,
@@ -396,6 +382,67 @@ class MQLLMEngineClient:
382
trace_headers: Optional[Mapping[str, str]] = None,
383
prompt_adapter_request: Optional[PromptAdapterRequest] = None
384
) -> AsyncGenerator[RequestOutput, None]:
385
+ """Generate outputs for a request.
386
+
387
+ Generate outputs for a request. This method is a coroutine. It adds the
388
+ request into the waiting queue of the LLMEngine and streams the outputs
389
+ from the LLMEngine to the caller.
390
+
391
+ Args:
392
+ inputs: The inputs to the LLM. See
393
+ :class:`~vllm.inputs.PromptInputs`
394
+ for more details about the format of each input.
395
+ sampling_params: The sampling parameters of the request.
396
+ request_id: The unique id of the request.
397
+ lora_request: LoRA request to use for generation, if any.
398
+ trace_headers: OpenTelemetry trace headers.
399
+ prompt_adapter_request: Prompt Adapter request to use
400
+ for generation, if any.
401
+ """
402
+ return self._process_request(inputs, sampling_params, request_id,
403
+ lora_request, trace_headers,
404
+ prompt_adapter_request)
405
+
406
+ def encode(
407
+ self,
408
+ inputs: PromptInputs,
409
+ pooling_params: PoolingParams,
410
+ request_id: str,
411
+ lora_request: Optional[LoRARequest] = None,
412
+ trace_headers: Optional[Mapping[str, str]] = None,
413
+ ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
414
+ """Generate outputs for a request from an embedding model.
415
+
416
+ Generate outputs for a request. This method is a coroutine. It adds the
417
+ request into the waiting queue of the LLMEngine and streams the outputs
418
+ from the LLMEngine to the caller.
419
+
420
+ Args:
421
+ inputs: The inputs to the LLM. See
422
+ :class:`~vllm.inputs.PromptInputs`
423
+ for more details about the format of each input.
424
+ pooling_params: The pooling parameters of the request.
425
+ request_id: The unique id of the request.
426
+ lora_request: LoRA request to use for generation, if any.
427
+ trace_headers: OpenTelemetry trace headers.
428
+
429
+ Yields:
430
+ The output `EmbeddingRequestOutput` objects from the LLMEngine
431
+ for the request.
432
+ """
433
+ return self._process_request(inputs, pooling_params, request_id,
434
+ lora_request, trace_headers)
435
+
436
+ async def _process_request(
437
+ self,
438
+ inputs: PromptInputs,
439
+ params: Union[SamplingParams, PoolingParams],
440
+ request_id: str,
441
+ lora_request: Optional[LoRARequest] = None,
442
+ trace_headers: Optional[Mapping[str, str]] = None,
443
+ prompt_adapter_request: Optional[PromptAdapterRequest] = None
444
+ ) -> Union[AsyncGenerator[RequestOutput, None], AsyncGenerator[
445
+ EmbeddingRequestOutput, None]]:
446
"""Send an RPCGenerateRequest to the RPCServer and stream responses."""
447
448
# If already dead, error out.
@@ -410,19 +457,19 @@ class MQLLMEngineClient:
457
try:
458
# 2) Detach logits processors so that they can be pickled
459
# separately (may require cloudpickle which is slower)
413
- if sampling_params.logits_processors:
460
+ if isinstance(params, SamplingParams) and params.logits_processors:
461
# Defensive shallow copy
415
- sampling_params = copy.copy(sampling_params)
416
- logits_processors = sampling_params.logits_processors
417
- sampling_params.logits_processors = None
462
+ params = copy.copy(params)
463
+ logits_processors = params.logits_processors
464
+ params.logits_processors = None
465
lp_bytes = cloudpickle.dumps(logits_processors)
466
else:
467
lp_bytes = None
468
469
request_bytes = pickle.dumps(
423
- RPCGenerateRequest(
470
+ RPCProcessRequest(
471
inputs=inputs,
425
- sampling_params=sampling_params,
472
+ params=params,
473
request_id=request_id,
474
lora_request=lora_request,
475
trace_headers=trace_headers,
@@ -452,8 +499,3 @@ class MQLLMEngineClient:
499
await self.abort(request_id)
500
finally:
501
self.output_queues.pop(request_id)
455
-
456
- async def encode(self, *args,
457
- **kwargs) -> AsyncGenerator[EmbeddingRequestOutput, None]:
458
- raise NotImplementedError(
459
- "Embeddings not supported with multiprocessing backend")
vllm/engine/multiprocessing/engine.py
+12
-11
@@ -6,7 +6,7 @@ from typing import Iterator, List, Optional, Union
6
import cloudpickle
7
import zmq
8
9
-from vllm import AsyncEngineArgs, LLMEngine
9
+from vllm import AsyncEngineArgs, LLMEngine, SamplingParams
10
from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
11
ParallelConfig, SchedulerConfig)
12
# yapf conflicts with isort for this block
@@ -15,8 +15,8 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
15
IPC_HEALTH_EXT, IPC_INPUT_EXT,
16
IPC_OUTPUT_EXT, REQUEST_OUTPUTS_T,
17
VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
18
- RPCError, RPCGenerateRequest,
19
- RPCHealthRequest, RPCStartupRequest,
18
+ RPCError, RPCHealthRequest,
19
+ RPCProcessRequest, RPCStartupRequest,
20
RPCStartupResponse)
21
# yapf: enable
22
from vllm.logger import init_logger
@@ -39,8 +39,8 @@ class MQLLMEngine:
39
in concurrnet manner. It runs a background loop and uses zeromq to
40
receive new requests and stream outputs incrementally via ipc.
41
42
- The :class:`LLMEngine.generate` is kicked off when a new
43
- RPCGenerateRequest is received by the input_socket.
42
+ The :class:`LLMEngine` generate or encode process is kicked off when a new
43
+ RPCProcessRequest is received by the input_socket.
44
45
The self.engine_loop checks the input_socket for new requests,
46
adds them to the LLMEngine if there are any, calls the internal
@@ -213,12 +213,13 @@ class MQLLMEngine:
213
frames = self.input_socket.recv_multipart(copy=False)
214
request = pickle.loads(frames[0].buffer)
215
216
- if isinstance(request, RPCGenerateRequest):
216
+ if isinstance(request, RPCProcessRequest):
217
if len(frames) > 1:
218
# Use cloudpickle for logits processors
219
+ assert isinstance(request.params, SamplingParams)
220
lprocs = cloudpickle.loads(frames[1].buffer)
220
- request.sampling_params.logits_processors = lprocs
221
- self._handle_generate_request(request)
221
+ request.params.logits_processors = lprocs
222
+ self._handle_process_request(request)
223
elif isinstance(request, RPCAbortRequest):
224
self._handle_abort_request(request)
225
elif isinstance(request, RPCHealthRequest):
@@ -231,8 +232,8 @@ class MQLLMEngine:
232
self._send_unhealthy(e)
233
raise e
234
234
- def _handle_generate_request(self, request: RPCGenerateRequest):
235
- """Handle RPCGenerateRequest by adding it to the LLMEngine."""
235
+ def _handle_process_request(self, request: RPCProcessRequest):
236
+ """Handle RPCProcessRequest by adding it to the LLMEngine."""
237
request_id = request.request_id
238
239
if self._errored_with is not None:
@@ -245,7 +246,7 @@ class MQLLMEngine:
246
self.engine.add_request(
247
request_id=request_id,
248
inputs=request.inputs,
248
- params=request.sampling_params,
249
+ params=request.params,
250
lora_request=request.lora_request,
251
trace_headers=request.trace_headers,
252
prompt_adapter_request=request.prompt_adapter_request)