@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 76515f303b
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 15:45 UTC

[Frontend] Use MQLLMEngine for embeddings models too (#8584)

Nick Hill committed Sep 19, 2024 at 17:51 UTC 76515f303b44cb3ffc6de63c49148d5081a77119
3 files changed +90 -46
vllm/engine/multiprocessing/__init__.py
+4 -3
@@ -2,6 +2,7 @@ from dataclasses import dataclass
2 from enum import Enum
3 from typing import List, Mapping, Optional, Union
4
5 +from vllm import PoolingParams
6 from vllm.inputs import PromptInputs
7 from vllm.lora.request import LoRARequest
8 from vllm.outputs import RequestOutput
@@ -21,9 +22,9 @@ class MQEngineDeadError(RuntimeError):
22
23
24 @dataclass
24 -class RPCGenerateRequest:
25 +class RPCProcessRequest:
26 inputs: PromptInputs
26 - sampling_params: SamplingParams
27 + params: Union[SamplingParams, PoolingParams]
28 request_id: str
29 lora_request: Optional[LoRARequest] = None
30 trace_headers: Optional[Mapping[str, str]] = None
@@ -55,7 +56,7 @@ class RPCStartupResponse:
56 tracing_enabled: bool
57
58
58 -RPC_REQUEST_T = Union[RPCGenerateRequest, RPCAbortRequest, RPCHealthRequest,
59 +RPC_REQUEST_T = Union[RPCProcessRequest, RPCAbortRequest, RPCHealthRequest,
60 RPCStartupRequest]
61
62 REQUEST_OUTPUTS_T = Union[List[RequestOutput], RPCError]
vllm/engine/multiprocessing/client.py
+74 -32
@@ -11,6 +11,7 @@ import zmq.asyncio
11 from zmq import Frame # type: ignore[attr-defined]
12 from zmq.asyncio import Socket
13
14 +from vllm import PoolingParams
15 from vllm.config import DecodingConfig, EngineConfig, ModelConfig
16 from vllm.engine.arg_utils import AsyncEngineArgs
17 # yapf conflicts with isort for this block
@@ -19,8 +20,8 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
20 IPC_HEALTH_EXT, IPC_INPUT_EXT,
21 IPC_OUTPUT_EXT, RPC_REQUEST_T,
22 VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
22 - RPCError, RPCGenerateRequest,
23 - RPCHealthRequest, RPCStartupRequest,
23 + RPCError, RPCHealthRequest,
24 + RPCProcessRequest, RPCStartupRequest,
25 RPCStartupResponse)
26 # yapf: enable
27 from vllm.envs import VLLM_RPC_TIMEOUT
@@ -111,20 +112,8 @@ class MQLLMEngineClient:
112
113 @staticmethod
114 def is_unsupported_config(engine_args: AsyncEngineArgs):
114 - if engine_args.pipeline_parallel_size > 1:
115 - return True
116 -
117 - is_embedding = ModelConfig(
118 - model=engine_args.model,
119 - revision=engine_args.revision,
120 - tokenizer=engine_args.model,
121 - tokenizer_mode="auto",
122 - trust_remote_code=engine_args.trust_remote_code,
123 - quantization=engine_args.quantization,
124 - seed=0,
125 - dtype="auto").embedding_mode
126 -
127 - return is_embedding
115 + # Pipeline parallel not yet supported
116 + return engine_args.pipeline_parallel_size > 1
117
118 @contextmanager
119 def get_data_socket(self) -> Iterator[Socket]:
@@ -382,12 +371,9 @@ class MQLLMEngineClient:
371
372 @property
373 def dead_error(self) -> BaseException:
385 - if self._errored_with is not None:
386 - return ENGINE_DEAD_ERROR(self._errored_with)
387 - else:
388 - return ENGINE_DEAD_ERROR()
374 + return ENGINE_DEAD_ERROR(self._errored_with)
375
390 - async def generate(
376 + def generate(
377 self,
378 inputs: PromptInputs,
379 sampling_params: SamplingParams,
@@ -396,6 +382,67 @@ class MQLLMEngineClient:
382 trace_headers: Optional[Mapping[str, str]] = None,
383 prompt_adapter_request: Optional[PromptAdapterRequest] = None
384 ) -> AsyncGenerator[RequestOutput, None]:
385 + """Generate outputs for a request.
386 +
387 + Generate outputs for a request. This method is a coroutine. It adds the
388 + request into the waiting queue of the LLMEngine and streams the outputs
389 + from the LLMEngine to the caller.
390 +
391 + Args:
392 + inputs: The inputs to the LLM. See
393 + :class:`~vllm.inputs.PromptInputs`
394 + for more details about the format of each input.
395 + sampling_params: The sampling parameters of the request.
396 + request_id: The unique id of the request.
397 + lora_request: LoRA request to use for generation, if any.
398 + trace_headers: OpenTelemetry trace headers.
399 + prompt_adapter_request: Prompt Adapter request to use
400 + for generation, if any.
401 + """
402 + return self._process_request(inputs, sampling_params, request_id,
403 + lora_request, trace_headers,
404 + prompt_adapter_request)
405 +
406 + def encode(
407 + self,
408 + inputs: PromptInputs,
409 + pooling_params: PoolingParams,
410 + request_id: str,
411 + lora_request: Optional[LoRARequest] = None,
412 + trace_headers: Optional[Mapping[str, str]] = None,
413 + ) -> AsyncGenerator[EmbeddingRequestOutput, None]:
414 + """Generate outputs for a request from an embedding model.
415 +
416 + Generate outputs for a request. This method is a coroutine. It adds the
417 + request into the waiting queue of the LLMEngine and streams the outputs
418 + from the LLMEngine to the caller.
419 +
420 + Args:
421 + inputs: The inputs to the LLM. See
422 + :class:`~vllm.inputs.PromptInputs`
423 + for more details about the format of each input.
424 + pooling_params: The pooling parameters of the request.
425 + request_id: The unique id of the request.
426 + lora_request: LoRA request to use for generation, if any.
427 + trace_headers: OpenTelemetry trace headers.
428 +
429 + Yields:
430 + The output `EmbeddingRequestOutput` objects from the LLMEngine
431 + for the request.
432 + """
433 + return self._process_request(inputs, pooling_params, request_id,
434 + lora_request, trace_headers)
435 +
436 + async def _process_request(
437 + self,
438 + inputs: PromptInputs,
439 + params: Union[SamplingParams, PoolingParams],
440 + request_id: str,
441 + lora_request: Optional[LoRARequest] = None,
442 + trace_headers: Optional[Mapping[str, str]] = None,
443 + prompt_adapter_request: Optional[PromptAdapterRequest] = None
444 + ) -> Union[AsyncGenerator[RequestOutput, None], AsyncGenerator[
445 + EmbeddingRequestOutput, None]]:
446 """Send an RPCGenerateRequest to the RPCServer and stream responses."""
447
448 # If already dead, error out.
@@ -410,19 +457,19 @@ class MQLLMEngineClient:
457 try:
458 # 2) Detach logits processors so that they can be pickled
459 # separately (may require cloudpickle which is slower)
413 - if sampling_params.logits_processors:
460 + if isinstance(params, SamplingParams) and params.logits_processors:
461 # Defensive shallow copy
415 - sampling_params = copy.copy(sampling_params)
416 - logits_processors = sampling_params.logits_processors
417 - sampling_params.logits_processors = None
462 + params = copy.copy(params)
463 + logits_processors = params.logits_processors
464 + params.logits_processors = None
465 lp_bytes = cloudpickle.dumps(logits_processors)
466 else:
467 lp_bytes = None
468
469 request_bytes = pickle.dumps(
423 - RPCGenerateRequest(
470 + RPCProcessRequest(
471 inputs=inputs,
425 - sampling_params=sampling_params,
472 + params=params,
473 request_id=request_id,
474 lora_request=lora_request,
475 trace_headers=trace_headers,
@@ -452,8 +499,3 @@ class MQLLMEngineClient:
499 await self.abort(request_id)
500 finally:
501 self.output_queues.pop(request_id)
455 -
456 - async def encode(self, *args,
457 - **kwargs) -> AsyncGenerator[EmbeddingRequestOutput, None]:
458 - raise NotImplementedError(
459 - "Embeddings not supported with multiprocessing backend")
vllm/engine/multiprocessing/engine.py
+12 -11
@@ -6,7 +6,7 @@ from typing import Iterator, List, Optional, Union
6 import cloudpickle
7 import zmq
8
9 -from vllm import AsyncEngineArgs, LLMEngine
9 +from vllm import AsyncEngineArgs, LLMEngine, SamplingParams
10 from vllm.config import (DecodingConfig, LoRAConfig, ModelConfig,
11 ParallelConfig, SchedulerConfig)
12 # yapf conflicts with isort for this block
@@ -15,8 +15,8 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
15 IPC_HEALTH_EXT, IPC_INPUT_EXT,
16 IPC_OUTPUT_EXT, REQUEST_OUTPUTS_T,
17 VLLM_RPC_SUCCESS_STR, RPCAbortRequest,
18 - RPCError, RPCGenerateRequest,
19 - RPCHealthRequest, RPCStartupRequest,
18 + RPCError, RPCHealthRequest,
19 + RPCProcessRequest, RPCStartupRequest,
20 RPCStartupResponse)
21 # yapf: enable
22 from vllm.logger import init_logger
@@ -39,8 +39,8 @@ class MQLLMEngine:
39 in concurrnet manner. It runs a background loop and uses zeromq to
40 receive new requests and stream outputs incrementally via ipc.
41
42 - The :class:`LLMEngine.generate` is kicked off when a new
43 - RPCGenerateRequest is received by the input_socket.
42 + The :class:`LLMEngine` generate or encode process is kicked off when a new
43 + RPCProcessRequest is received by the input_socket.
44
45 The self.engine_loop checks the input_socket for new requests,
46 adds them to the LLMEngine if there are any, calls the internal
@@ -213,12 +213,13 @@ class MQLLMEngine:
213 frames = self.input_socket.recv_multipart(copy=False)
214 request = pickle.loads(frames[0].buffer)
215
216 - if isinstance(request, RPCGenerateRequest):
216 + if isinstance(request, RPCProcessRequest):
217 if len(frames) > 1:
218 # Use cloudpickle for logits processors
219 + assert isinstance(request.params, SamplingParams)
220 lprocs = cloudpickle.loads(frames[1].buffer)
220 - request.sampling_params.logits_processors = lprocs
221 - self._handle_generate_request(request)
221 + request.params.logits_processors = lprocs
222 + self._handle_process_request(request)
223 elif isinstance(request, RPCAbortRequest):
224 self._handle_abort_request(request)
225 elif isinstance(request, RPCHealthRequest):
@@ -231,8 +232,8 @@ class MQLLMEngine:
232 self._send_unhealthy(e)
233 raise e
234
234 - def _handle_generate_request(self, request: RPCGenerateRequest):
235 - """Handle RPCGenerateRequest by adding it to the LLMEngine."""
235 + def _handle_process_request(self, request: RPCProcessRequest):
236 + """Handle RPCProcessRequest by adding it to the LLMEngine."""
237 request_id = request.request_id
238
239 if self._errored_with is not None:
@@ -245,7 +246,7 @@ class MQLLMEngine:
246 self.engine.add_request(
247 request_id=request_id,
248 inputs=request.inputs,
248 - params=request.sampling_params,
249 + params=request.params,
250 lora_request=request.lora_request,
251 trace_headers=request.trace_headers,
252 prompt_adapter_request=request.prompt_adapter_request)