@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e254497b66
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Model][Misc] Add e5-mistral-7b-instruct and Embedding API (#3734)

Chang Su committed May 11, 2024 at 11:30 UTC e254497b66dcd87038969b0ad34d34425edfc5fe
38 files changed +1627 -160
examples/offline_inference_embedding.py new
+17
@@ -0,0 +1,17 @@
1 +from vllm import LLM
2 +
3 +# Sample prompts.
4 +prompts = [
5 + "Hello, my name is",
6 + "The president of the United States is",
7 + "The capital of France is",
8 + "The future of AI is",
9 +]
10 +
11 +# Create an LLM.
12 +model = LLM(model="intfloat/e5-mistral-7b-instruct", enforce_eager=True)
13 +# Generate embedding. The output is a list of EmbeddingRequestOutputs.
14 +outputs = model.encode(prompts)
15 +# Print the outputs.
16 +for output in outputs:
17 + print(output.outputs.embedding) # list of 4096 floats
examples/openai_embedding_client.py new
+23
@@ -0,0 +1,23 @@
1 +from openai import OpenAI
2 +
3 +# Modify OpenAI's API key and API base to use vLLM's API server.
4 +openai_api_key = "EMPTY"
5 +openai_api_base = "http://localhost:8000/v1"
6 +
7 +client = OpenAI(
8 + # defaults to os.environ.get("OPENAI_API_KEY")
9 + api_key=openai_api_key,
10 + base_url=openai_api_base,
11 +)
12 +
13 +models = client.models.list()
14 +model = models.data[0].id
15 +
16 +responses = client.embeddings.create(input=[
17 + "Hello my name is",
18 + "The best thing about vLLM is that it supports many different models"
19 +],
20 + model=model)
21 +
22 +for data in responses.data:
23 + print(data.embedding) # list of float of len 4096
requirements-dev.txt
+6 -3
@@ -19,12 +19,15 @@ pytest-forked
19 pytest-asyncio
20 pytest-rerunfailures
21 pytest-shard
22 -httpx
22 +
23 +# testing utils
24 +awscli
25 einops # required for MPT
26 +httpx
27 +peft
28 requests
29 ray
26 -peft
27 -awscli
30 +sentence-transformers # required for embedding
31
32 # Benchmarking
33 aiohttp
tests/conftest.py
+30 -8
@@ -133,6 +133,10 @@ _VISION_LANGUAGE_MODELS = {
133 "llava-hf/llava-1.5-7b-hf": LlavaForConditionalGeneration,
134 }
135
136 +_EMBEDDING_MODELS = [
137 + "intfloat/e5-mistral-7b-instruct",
138 +]
139 +
140
141 class HfRunner:
142
@@ -145,14 +149,7 @@ class HfRunner:
149 assert dtype in _STR_DTYPE_TO_TORCH_DTYPE
150 torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
151 self.model_name = model_name
148 - if model_name not in _VISION_LANGUAGE_MODELS:
149 - self.model = AutoModelForCausalLM.from_pretrained(
150 - model_name,
151 - torch_dtype=torch_dtype,
152 - trust_remote_code=True,
153 - ).cuda()
154 - self.processor = None
155 - else:
152 + if model_name in _VISION_LANGUAGE_MODELS:
153 self.model = _VISION_LANGUAGE_MODELS[model_name].from_pretrained(
154 model_name,
155 torch_dtype=torch_dtype,
@@ -162,6 +159,20 @@ class HfRunner:
159 model_name,
160 torch_dtype=torch_dtype,
161 )
162 + elif model_name in _EMBEDDING_MODELS:
163 + # Lazy init required for AMD CI
164 + from sentence_transformers import SentenceTransformer
165 + self.model = SentenceTransformer(
166 + model_name,
167 + device="cpu",
168 + ).to(dtype=torch_dtype).cuda()
169 + else:
170 + self.model = AutoModelForCausalLM.from_pretrained(
171 + model_name,
172 + torch_dtype=torch_dtype,
173 + trust_remote_code=True,
174 + ).cuda()
175 + self.processor = None
176 if tokenizer_name is None:
177 tokenizer_name = model_name
178 self.tokenizer = get_tokenizer(tokenizer_name, trust_remote_code=True)
@@ -334,6 +345,9 @@ class HfRunner:
345 return [(output_ids, output_str, output_logprobs)
346 for output_ids, output_str, output_logprobs in outputs]
347
348 + def encode(self, prompts: List[str]) -> List[List[torch.Tensor]]:
349 + return self.model.encode(prompts)
350 +
351 def __del__(self):
352 del self.model
353 cleanup()
@@ -459,6 +473,14 @@ class VllmRunner:
473 outputs = self.generate(prompts, beam_search_params)
474 return outputs
475
476 + def encode(self, prompts: List[str]) -> List[List[float]]:
477 + req_outputs = self.model.encode(prompts)
478 + outputs = []
479 + for req_output in req_outputs:
480 + embedding = req_output.outputs.embedding
481 + outputs.append(embedding)
482 + return outputs
483 +
484 def __del__(self):
485 del self.model
486 cleanup()
tests/engine/output_processor/test_multi_step.py
+6 -6
@@ -9,8 +9,8 @@ from vllm.core.scheduler import Scheduler
9 from vllm.engine.output_processor.multi_step import MultiStepOutputProcessor
10 from vllm.engine.output_processor.stop_checker import StopChecker
11 from vllm.sampling_params import SamplingParams
12 -from vllm.sequence import (Logprob, SequenceGroupOutput, SequenceOutput,
13 - SequenceStatus)
12 +from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
13 + SequenceOutput, SequenceStatus)
14 from vllm.transformers_utils.detokenizer import Detokenizer
15 from vllm.utils import Counter
16
@@ -51,7 +51,7 @@ def test_appends_token_ids(num_new_tokens: int, seq_output_len: int):
51 new_token_ids = list(range(num_new_tokens))
52
53 outputs = [
54 - SequenceGroupOutput(
54 + CompletionSequenceGroupOutput(
55 samples=[
56 SequenceOutput(
57 parent_seq_id=seq.seq_id,
@@ -103,7 +103,7 @@ def test_respects_max_tokens(num_new_tokens: int, seq_prompt_len: int,
103 new_token_ids = list(range(num_new_tokens))
104
105 outputs = [
106 - SequenceGroupOutput(
106 + CompletionSequenceGroupOutput(
107 samples=[
108 SequenceOutput(
109 parent_seq_id=seq.seq_id,
@@ -170,7 +170,7 @@ def test_respects_eos_token_id(num_new_tokens: int, seq_prompt_len: int,
170 new_token_ids[eos_index] = eos_token_id
171
172 outputs = [
173 - SequenceGroupOutput(
173 + CompletionSequenceGroupOutput(
174 samples=[
175 SequenceOutput(
176 parent_seq_id=seq.seq_id,
@@ -239,7 +239,7 @@ def test_ignores_eos_token_id(num_new_tokens: int, seq_prompt_len: int,
239 new_token_ids[eos_index] = eos_token_id
240
241 outputs = [
242 - SequenceGroupOutput(
242 + CompletionSequenceGroupOutput(
243 samples=[
244 SequenceOutput(
245 parent_seq_id=seq.seq_id,
tests/entrypoints/openai/test_serving_chat.py
+1
@@ -14,6 +14,7 @@ class MockModelConfig:
14 tokenizer_mode = "auto"
15 max_model_len = 100
16 tokenizer_revision = None
17 + embedding_mode = False
18
19
20 @dataclass
tests/entrypoints/test_openai_server.py
+95 -1
@@ -23,6 +23,7 @@ from vllm.transformers_utils.tokenizer import get_tokenizer
23 MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
24 # any model with a chat template should work here
25 MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
26 +EMBEDDING_MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
27 # technically this needs Mistral-7B-v0.1 as base, but we're not testing
28 # generation quality here
29 LORA_NAME = "typeof/zephyr-7b-beta-lora"
@@ -121,7 +122,7 @@ def zephyr_lora_files():
122 return snapshot_download(repo_id=LORA_NAME)
123
124
124 -@pytest.fixture(scope="session")
125 +@pytest.fixture(scope="module")
126 def server(zephyr_lora_files):
127 ray.init()
128 server_runner = ServerRunner.remote([
@@ -150,6 +151,25 @@ def server(zephyr_lora_files):
151 ray.shutdown()
152
153
154 +@pytest.fixture(scope="module")
155 +def embedding_server(zephyr_lora_files):
156 + ray.shutdown()
157 + ray.init()
158 + server_runner = ServerRunner.remote([
159 + "--model",
160 + EMBEDDING_MODEL_NAME,
161 + # use half precision for speed and memory savings in CI environment
162 + "--dtype",
163 + "bfloat16",
164 + "--max-model-len",
165 + "8192",
166 + "--enforce-eager",
167 + ])
168 + ray.get(server_runner.ready.remote())
169 + yield server_runner
170 + ray.shutdown()
171 +
172 +
173 @pytest.fixture(scope="module")
174 def client():
175 client = openai.AsyncOpenAI(
@@ -890,5 +910,79 @@ async def test_long_seed(server, client: openai.AsyncOpenAI):
910 or "less_than_equal" in exc_info.value.message)
911
912
913 +@pytest.mark.parametrize(
914 + "model_name",
915 + [EMBEDDING_MODEL_NAME],
916 +)
917 +async def test_single_embedding(embedding_server, client: openai.AsyncOpenAI,
918 + model_name: str):
919 + input = [
920 + "The chef prepared a delicious meal.",
921 + ]
922 +
923 + # test single embedding
924 + embeddings = await client.embeddings.create(
925 + model=model_name,
926 + input=input,
927 + encoding_format="float",
928 + )
929 + assert embeddings.id is not None
930 + assert embeddings.data is not None and len(embeddings.data) == 1
931 + assert len(embeddings.data[0].embedding) == 4096
932 + assert embeddings.usage.completion_tokens == 0
933 + assert embeddings.usage.prompt_tokens == 9
934 + assert embeddings.usage.total_tokens == 9
935 +
936 + # test using token IDs
937 + input = [1, 1, 1, 1, 1]
938 + embeddings = await client.embeddings.create(
939 + model=model_name,
940 + input=input,
941 + encoding_format="float",
942 + )
943 + assert embeddings.id is not None
944 + assert embeddings.data is not None and len(embeddings.data) == 1
945 + assert len(embeddings.data[0].embedding) == 4096
946 + assert embeddings.usage.completion_tokens == 0
947 + assert embeddings.usage.prompt_tokens == 5
948 + assert embeddings.usage.total_tokens == 5
949 +
950 +
951 +@pytest.mark.parametrize(
952 + "model_name",
953 + [EMBEDDING_MODEL_NAME],
954 +)
955 +async def test_batch_embedding(embedding_server, client: openai.AsyncOpenAI,
956 + model_name: str):
957 + # test List[str]
958 + inputs = [
959 + "The cat sat on the mat.", "A feline was resting on a rug.",
960 + "Stars twinkle brightly in the night sky."
961 + ]
962 + embeddings = await client.embeddings.create(
963 + model=model_name,
964 + input=inputs,
965 + encoding_format="float",
966 + )
967 + assert embeddings.id is not None
968 + assert embeddings.data is not None and len(embeddings.data) == 3
969 + assert len(embeddings.data[0].embedding) == 4096
970 +
971 + # test List[List[int]]
972 + inputs = [[4, 5, 7, 9, 20], [15, 29, 499], [24, 24, 24, 24, 24],
973 + [25, 32, 64, 77]]
974 + embeddings = await client.embeddings.create(
975 + model=model_name,
976 + input=inputs,
977 + encoding_format="float",
978 + )
979 + assert embeddings.id is not None
980 + assert embeddings.data is not None and len(embeddings.data) == 4
981 + assert len(embeddings.data[0].embedding) == 4096
982 + assert embeddings.usage.completion_tokens == 0
983 + assert embeddings.usage.prompt_tokens == 17
984 + assert embeddings.usage.total_tokens == 17
985 +
986 +
987 if __name__ == "__main__":
988 pytest.main([__file__])
tests/models/test_embedding.py new
+44
@@ -0,0 +1,44 @@
1 +"""Compare the outputs of HF and vLLM for Mistral models using greedy sampling.
2 +
3 +Run `pytest tests/models/test_llama_embedding.py`.
4 +"""
5 +import pytest
6 +import torch
7 +import torch.nn.functional as F
8 +
9 +MODELS = [
10 + "intfloat/e5-mistral-7b-instruct",
11 +]
12 +
13 +
14 +def compare_embeddings(embeddings1, embeddings2):
15 + similarities = [
16 + F.cosine_similarity(torch.tensor(e1), torch.tensor(e2), dim=0)
17 + for e1, e2 in zip(embeddings1, embeddings2)
18 + ]
19 + return similarities
20 +
21 +
22 +@pytest.mark.parametrize("model", MODELS)
23 +@pytest.mark.parametrize("dtype", ["half"])
24 +def test_models(
25 + hf_runner,
26 + vllm_runner,
27 + example_prompts,
28 + model: str,
29 + dtype: str,
30 +) -> None:
31 + hf_model = hf_runner(model, dtype=dtype)
32 + hf_outputs = hf_model.encode(example_prompts)
33 + del hf_model
34 +
35 + vllm_model = vllm_runner(model, dtype=dtype)
36 + vllm_outputs = vllm_model.encode(example_prompts)
37 + del vllm_model
38 +
39 + similarities = compare_embeddings(hf_outputs, vllm_outputs)
40 + all_similarities = torch.stack(similarities)
41 + tolerance = 1e-2
42 + assert torch.all((all_similarities <= 1.0 + tolerance)
43 + & (all_similarities >= 1.0 - tolerance)
44 + ), f"Not all values are within {tolerance} of 1.0"
tests/samplers/test_logits_processor.py
+3 -3
@@ -36,14 +36,14 @@ def test_logits_processor_force_generate(
36 # test logits_processors when prompt_logprobs is not None
37 vllm_model.model._add_request(
38 prompt=example_prompts[0],
39 - sampling_params=params_with_logprobs,
39 + params=params_with_logprobs,
40 prompt_token_ids=None,
41 )
42
43 # test prompt_logprobs is not None
44 vllm_model.model._add_request(
45 prompt=example_prompts[1],
46 - sampling_params=SamplingParams(
46 + params=SamplingParams(
47 prompt_logprobs=3,
48 max_tokens=max_tokens,
49 ),
@@ -53,7 +53,7 @@ def test_logits_processor_force_generate(
53 # test grouped requests
54 vllm_model.model._add_request(
55 prompt=example_prompts[2],
56 - sampling_params=SamplingParams(max_tokens=max_tokens),
56 + params=SamplingParams(max_tokens=max_tokens),
57 prompt_token_ids=None,
58 )
59
tests/samplers/test_seeded_generate.py
+1 -1
@@ -60,7 +60,7 @@ def test_random_sample_with_seed(
60 llm._add_request(
61 prompt=prompt,
62 prompt_token_ids=None,
63 - sampling_params=params,
63 + params=params,
64 )
65
66 results = llm._run_engine(use_tqdm=False)
tests/spec_decode/utils.py
+3 -3
@@ -7,8 +7,8 @@ import torch
7 from vllm.engine.arg_utils import EngineArgs
8 from vllm.model_executor.utils import set_random_seed
9 from vllm.sampling_params import SamplingParams
10 -from vllm.sequence import (Logprob, SamplerOutput, SequenceData,
11 - SequenceGroupMetadata, SequenceGroupOutput,
10 +from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
11 + SamplerOutput, SequenceData, SequenceGroupMetadata,
12 SequenceOutput)
13 from vllm.utils import get_distributed_init_method, get_ip, get_open_port
14 from vllm.worker.cache_engine import CacheEngine
@@ -170,7 +170,7 @@ def create_sampler_output_list(
170
171 return [
172 SamplerOutput(outputs=[
173 - SequenceGroupOutput(
173 + CompletionSequenceGroupOutput(
174 samples=[
175 SequenceOutput(
176 output_token=token_id,
tests/test_sequence.py
+6 -6
@@ -1,17 +1,17 @@
1 import pytest
2
3 from tests.core.utils import create_dummy_prompt
4 -from vllm.sequence import (SamplerOutput, SequenceData, SequenceGroupOutput,
5 - SequenceOutput)
4 +from vllm.sequence import (CompletionSequenceGroupOutput, SamplerOutput,
5 + SequenceData, SequenceOutput)
6
7
8 @pytest.fixture
9 def sample_outputs():
10 return [
11 - SequenceGroupOutput(samples=[
11 + CompletionSequenceGroupOutput(samples=[
12 SequenceOutput(parent_seq_id=0, output_token=i, logprobs={})
13 ],
14 - prompt_logprobs=None) for i in range(5)
14 + prompt_logprobs=None) for i in range(5)
15 ]
16
17
@@ -32,10 +32,10 @@ def test_sampler_output_getitem(sampler_output, sample_outputs):
32
33
34 def test_sampler_output_setitem(sampler_output):
35 - new_output = SequenceGroupOutput(samples=[
35 + new_output = CompletionSequenceGroupOutput(samples=[
36 SequenceOutput(parent_seq_id=0, output_token=99, logprobs={})
37 ],
38 - prompt_logprobs=None)
38 + prompt_logprobs=None)
39 sampler_output[2] = new_output
40 assert sampler_output[2] == new_output
41
vllm/__init__.py
+6 -1
@@ -6,7 +6,9 @@ from vllm.engine.llm_engine import LLMEngine
6 from vllm.entrypoints.llm import LLM
7 from vllm.executor.ray_utils import initialize_ray_cluster
8 from vllm.model_executor.models import ModelRegistry
9 -from vllm.outputs import CompletionOutput, RequestOutput
9 +from vllm.outputs import (CompletionOutput, EmbeddingOutput,
10 + EmbeddingRequestOutput, RequestOutput)
11 +from vllm.pooling_params import PoolingParams
12 from vllm.sampling_params import SamplingParams
13
14 __version__ = "0.4.2"
@@ -17,9 +19,12 @@ __all__ = [
19 "SamplingParams",
20 "RequestOutput",
21 "CompletionOutput",
22 + "EmbeddingOutput",
23 + "EmbeddingRequestOutput",
24 "LLMEngine",
25 "EngineArgs",
26 "AsyncLLMEngine",
27 "AsyncEngineArgs",
28 "initialize_ray_cluster",
29 + "PoolingParams",
30 ]
vllm/config.py
+15
@@ -9,6 +9,7 @@ from transformers import PretrainedConfig
9 from vllm.logger import init_logger
10 from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
11 get_quantization_config)
12 +from vllm.model_executor.models import ModelRegistry
13 from vllm.transformers_utils.config import get_config, get_hf_text_config
14 from vllm.utils import get_cpu_memory, is_cpu, is_hip, is_neuron
15
@@ -22,6 +23,7 @@ if TYPE_CHECKING:
23 logger = init_logger(__name__)
24
25 _GB = 1 << 30
26 +_EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
27
28
29 class ModelConfig:
@@ -126,6 +128,7 @@ class ModelConfig:
128 served_model_name)
129 if not self.skip_tokenizer_init:
130 self._verify_tokenizer_mode()
131 + self._verify_embedding_mode()
132 self._verify_quantization()
133 self._verify_cuda_graph()
134
@@ -137,6 +140,11 @@ class ModelConfig:
140 "either 'auto' or 'slow'.")
141 self.tokenizer_mode = tokenizer_mode
142
143 + def _verify_embedding_mode(self) -> None:
144 + architectures = getattr(self.hf_config, "architectures", [])
145 + self.embedding_mode = any(
146 + ModelRegistry.is_embedding_model(arch) for arch in architectures)
147 +
148 def _verify_quantization(self) -> None:
149 supported_quantization = [*QUANTIZATION_METHODS]
150 rocm_supported_quantization = ["gptq", "squeezellm"]
@@ -591,6 +599,7 @@ class SchedulerConfig:
599 prompt latency) before scheduling next prompt.
600 enable_chunked_prefill: If True, prefill requests can be chunked based
601 on the remaining max_num_batched_tokens.
602 + embedding_mode: Whether the running model is for embedding.
603 """
604
605 def __init__(
@@ -602,6 +611,7 @@ class SchedulerConfig:
611 num_lookahead_slots: int = 0,
612 delay_factor: float = 0.0,
613 enable_chunked_prefill: bool = False,
614 + embedding_mode: Optional[bool] = False,
615 ) -> None:
616 if max_num_batched_tokens is not None:
617 self.max_num_batched_tokens = max_num_batched_tokens
@@ -610,6 +620,10 @@ class SchedulerConfig:
620 # It is the values that have the best balance between ITL
621 # and TTFT on A100. Note it is not optimized for throughput.
622 self.max_num_batched_tokens = 512
623 + elif embedding_mode:
624 + # For embedding, choose specific value for higher throughput
625 + self.max_num_batched_tokens = max(
626 + max_model_len, _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS)
627 else:
628 # If max_model_len is too short, use 2048 as the default value
629 # for higher throughput.
@@ -623,6 +637,7 @@ class SchedulerConfig:
637 self.num_lookahead_slots = num_lookahead_slots
638 self.delay_factor = delay_factor
639 self.chunked_prefill_enabled = enable_chunked_prefill
640 + self.embedding_mode = embedding_mode
641
642 self._verify_args()
643
vllm/core/embedding_model_block_manager.py new
+84
@@ -0,0 +1,84 @@
1 +from typing import List, Tuple
2 +
3 +from vllm.core.interfaces import AllocStatus, BlockSpaceManager
4 +from vllm.sequence import Sequence, SequenceGroup
5 +
6 +
7 +class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
8 + """An embedding version of BlockSpaceManager for use in environments
9 + with embedding models where block management is not required.
10 +
11 + This class provides the same interface as BlockSpaceManager, but its
12 + methods perform no actions or return simple values like True in specific
13 + actions. It's designed to be used in scenarios where the overhead of
14 + block management is unnecessary, such as in an embedding environment.
15 + """
16 +
17 + def __init__(
18 + self,
19 + **kwargs,
20 + ) -> None:
21 + pass
22 +
23 + def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
24 + # Always return OK for dummy purposes
25 + return AllocStatus.OK
26 +
27 + def allocate(self, seq_group: SequenceGroup) -> None:
28 + # No actual allocation logic needed
29 + pass
30 +
31 + def can_append_slots(self, seq_group: SequenceGroup,
32 + num_lookahead_slots: int) -> bool:
33 + return True
34 +
35 + def append_slots(
36 + self,
37 + seq: Sequence,
38 + num_lookahead_slots: int,
39 + ) -> List[Tuple[int, int]]:
40 + return None # type: ignore
41 +
42 + def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
43 + pass
44 +
45 + def can_swap_in(self, seq_group: SequenceGroup,
46 + num_lookahead_slots: int) -> AllocStatus:
47 + return AllocStatus.OK
48 +
49 + def swap_in(self, seq_group: SequenceGroup,
50 + num_lookahead_slots: int) -> List[Tuple[int, int]]:
51 + return None # type: ignore
52 +
53 + def can_swap_out(self, seq_group: SequenceGroup) -> bool:
54 + return True
55 +
56 + def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
57 + return None # type: ignore
58 +
59 + def free(self, seq: Sequence) -> None:
60 + # No operation on free
61 + return
62 +
63 + def get_block_table(self, seq: Sequence) -> List[int]:
64 + return None # type: ignore
65 +
66 + def get_num_free_gpu_blocks(self) -> int:
67 + return 1
68 +
69 + def get_num_free_cpu_blocks(self) -> int:
70 + return 1
71 +
72 + def access_all_blocks_in_seq(
73 + self,
74 + seq: Sequence,
75 + access_time: float,
76 + ) -> None:
77 + pass
78 +
79 + def get_common_computed_block_ids(self,
80 + seq_group: SequenceGroup) -> List[int]:
81 + return None # type: ignore
82 +
83 + def mark_blocks_as_computed(self, seq_group: SequenceGroup):
84 + pass
vllm/core/interfaces.py
+5
@@ -35,6 +35,11 @@ class BlockSpaceManager(ABC):
35 from vllm.core.block_manager_v2 import BlockSpaceManagerV2
36 return BlockSpaceManagerV2
37
38 + if version == "embedding":
39 + from vllm.core.embedding_model_block_manager import (
40 + EmbeddingModelBlockSpaceManager)
41 + return EmbeddingModelBlockSpaceManager
42 +
43 raise ValueError(f"Unknown version {version=}")
44
45 @abstractmethod
vllm/core/scheduler.py
+8 -2
@@ -270,9 +270,14 @@ class Scheduler:
270 self.scheduler_config.max_model_len,
271 self.scheduler_config.max_num_batched_tokens)
272
273 + version = "v1"
274 + if self.scheduler_config.use_v2_block_manager:
275 + version = "v2"
276 + if self.scheduler_config.embedding_mode:
277 + version = "embedding"
278 +
279 BlockSpaceManagerImpl = BlockSpaceManager.get_block_space_manager_class(
274 - version="v2" if self.scheduler_config.
275 - use_v2_block_manager else "v1")
280 + version)
281
282 # Create the block space manager.
283 self.block_manager = BlockSpaceManagerImpl(
@@ -968,6 +973,7 @@ class Scheduler:
973 sampling_params=seq_group.sampling_params,
974 block_tables=block_tables,
975 do_sample=do_sample,
976 + pooling_params=seq_group.pooling_params,
977 token_chunk_size=token_chunk_size,
978 lora_request=seq_group.lora_request,
979 computed_block_nums=common_computed_block_nums,
vllm/engine/arg_utils.py
+1
@@ -574,6 +574,7 @@ class EngineArgs:
574 speculative_config.num_lookahead_slots),
575 delay_factor=self.scheduler_delay_factor,
576 enable_chunked_prefill=self.enable_chunked_prefill,
577 + embedding_mode=model_config.embedding_mode,
578 )
579 lora_config = LoRAConfig(
580 max_lora_rank=self.max_lora_rank,
vllm/engine/async_llm_engine.py
+130 -28
@@ -14,7 +14,8 @@ from vllm.engine.llm_engine import LLMEngine
14 from vllm.executor.ray_utils import initialize_ray_cluster, ray
15 from vllm.logger import init_logger
16 from vllm.lora.request import LoRARequest
17 -from vllm.outputs import RequestOutput
17 +from vllm.outputs import EmbeddingRequestOutput, RequestOutput
18 +from vllm.pooling_params import PoolingParams
19 from vllm.sampling_params import SamplingParams
20 from vllm.sequence import ExecuteModelRequest, MultiModalData, SamplerOutput
21 from vllm.usage.usage_lib import UsageContext
@@ -47,15 +48,16 @@ def _raise_exception_on_finish(
48
49
50 class AsyncStream:
50 - """A stream of RequestOutputs for a request that can be
51 - iterated over asynchronously."""
51 + """A stream of RequestOutputs or EmbeddingRequestOutputs for a request
52 + that can be iterated over asynchronously."""
53
54 def __init__(self, request_id: str) -> None:
55 self.request_id = request_id
56 self._queue: asyncio.Queue = asyncio.Queue()
57 self._finished = False
58
58 - def put(self, item: Union[RequestOutput, Exception]) -> None:
59 + def put(self, item: Union[RequestOutput, EmbeddingRequestOutput,
60 + Exception]) -> None:
61 if self._finished:
62 return
63 self._queue.put_nowait(item)
@@ -71,7 +73,7 @@ class AsyncStream:
73 def __aiter__(self):
74 return self
75
74 - async def __anext__(self) -> RequestOutput:
76 + async def __anext__(self) -> Union[RequestOutput, EmbeddingRequestOutput]:
77 result = await self._queue.get()
78 if isinstance(result, Exception):
79 raise result
@@ -108,7 +110,8 @@ class RequestTracker:
110 self.abort_request(rid)
111
112 def process_request_output(self,
111 - request_output: RequestOutput,
113 + request_output: Union[RequestOutput,
114 + EmbeddingRequestOutput],
115 *,
116 verbose: bool = False) -> None:
117 """Process a request output from the engine."""
@@ -196,7 +199,8 @@ class RequestTracker:
199 class _AsyncLLMEngine(LLMEngine):
200 """Extension of LLMEngine to add async methods."""
201
199 - async def step_async(self) -> List[RequestOutput]:
202 + async def step_async(
203 + self) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
204 """Performs one decoding iteration and returns newly generated results.
205 The workers are ran asynchronously if possible.
206
@@ -251,7 +255,7 @@ class _AsyncLLMEngine(LLMEngine):
255 self,
256 request_id: str,
257 prompt: Optional[str],
254 - sampling_params: SamplingParams,
258 + params: Union[SamplingParams, PoolingParams],
259 prompt_token_ids: Optional[List[int]] = None,
260 arrival_time: Optional[float] = None,
261 lora_request: Optional[LoRARequest] = None,
@@ -270,8 +274,8 @@ class _AsyncLLMEngine(LLMEngine):
274
275 return self.add_request(request_id,
276 prompt=prompt,
277 + params=params,
278 prompt_token_ids=prompt_token_ids,
274 - sampling_params=sampling_params,
279 arrival_time=arrival_time,
280 lora_request=lora_request,
281 multi_modal_data=multi_modal_data)
@@ -511,7 +515,7 @@ class AsyncLLMEngine:
515 self,
516 request_id: str,
517 prompt: Optional[str],
514 - sampling_params: SamplingParams,
518 + params: Union[SamplingParams, PoolingParams],
519 prompt_token_ids: Optional[List[int]] = None,
520 arrival_time: Optional[float] = None,
521 lora_request: Optional[LoRARequest] = None,
@@ -528,9 +532,9 @@ class AsyncLLMEngine:
532 max_log_len]
533 logger.info(
534 "Received request %s: prompt: %r, "
531 - "sampling_params: %s, prompt_token_ids: %s, "
532 - "lora_request: %s.", request_id, shortened_prompt,
533 - sampling_params, shortened_token_ids, lora_request)
535 + "params: %s, prompt_token_ids: %s, "
536 + "lora_request: %s.", request_id, shortened_prompt, params,
537 + shortened_token_ids, lora_request)
538
539 if not self.is_running:
540 if self.start_engine_loop:
@@ -562,7 +566,7 @@ class AsyncLLMEngine:
566 stream = self._request_tracker.add_request(
567 request_id,
568 prompt=prompt,
565 - sampling_params=sampling_params,
569 + params=params,
570 prompt_token_ids=prompt_token_ids,
571 arrival_time=arrival_time,
572 lora_request=lora_request,
@@ -597,8 +601,8 @@ class AsyncLLMEngine:
601 multi_modal_data: Multi modal data per request.
602
603 Yields:
600 - The output `RequestOutput` objects from the LLMEngine for the
601 - request.
604 + The output `RequestOutput` objects from the LLMEngine
605 + for the request.
606
607 Details:
608 - If the engine is not running, start the background loop,
@@ -643,25 +647,123 @@ class AsyncLLMEngine:
647 >>> # Process and return the final output
648 >>> ...
649 """
646 - # Preprocess the request.
647 - arrival_time = time.time()
648 -
649 - try:
650 - stream = await self.add_request(
650 + async for output in self.process_request(
651 request_id,
652 prompt,
653 sampling_params,
654 - prompt_token_ids=prompt_token_ids,
655 - arrival_time=arrival_time,
656 - lora_request=lora_request,
657 - multi_modal_data=multi_modal_data,
658 - )
654 + prompt_token_ids,
655 + lora_request,
656 + multi_modal_data,
657 + ):
658 + yield output
659 +
660 + async def encode(
661 + self,
662 + prompt: Optional[str],
663 + pooling_params: PoolingParams,
664 + request_id: str,
665 + prompt_token_ids: Optional[List[int]] = None,
666 + lora_request: Optional[LoRARequest] = None,
667 + multi_modal_data: Optional[MultiModalData] = None
668 + ) -> AsyncIterator[EmbeddingRequestOutput]:
669 + """Generate outputs for a request from an embedding model.
670 +
671 + Generate outputs for a request. This method is a coroutine. It adds the
672 + request into the waiting queue of the LLMEngine and streams the outputs
673 + from the LLMEngine to the caller.
674 +
675 + Args:
676 + prompt: The prompt string. Can be None if prompt_token_ids is
677 + provided.
678 + pooling_params: The pooling parameters of the request.
679 + request_id: The unique id of the request.
680 + prompt_token_ids: The token IDs of the prompt. If None, we
681 + use the tokenizer to convert the prompts to token IDs.
682 + lora_request: LoRA request to use for generation, if any.
683 + multi_modal_data: Multi modal data per request.
684 +
685 + Yields:
686 + The output `EmbeddingRequestOutput` objects from the LLMEngine
687 + for the request.
688 +
689 + Details:
690 + - If the engine is not running, start the background loop,
691 + which iteratively invokes
692 + :meth:`~vllm.engine.async_llm_engine.AsyncLLMEngine.engine_step`
693 + to process the waiting requests.
694 + - Add the request to the engine's `RequestTracker`.
695 + On the next background loop, this request will be sent to
696 + the underlying engine.
697 + Also, a corresponding `AsyncStream` will be created.
698 + - Wait for the request outputs from `AsyncStream` and yield them.
699 +
700 + Example:
701 + >>> # Please refer to entrypoints/api_server.py for
702 + >>> # the complete example.
703 + >>>
704 + >>> # initialize the engine and the example input
705 + >>> engine = AsyncLLMEngine.from_engine_args(engine_args)
706 + >>> example_input = {
707 + >>> "input": "What is LLM?",
708 + >>> "request_id": 0,
709 + >>> }
710 + >>>
711 + >>> # start the generation
712 + >>> results_generator = engine.encode(
713 + >>> example_input["input"],
714 + >>> PoolingParams(),
715 + >>> example_input["request_id"])
716 + >>>
717 + >>> # get the results
718 + >>> final_output = None
719 + >>> async for request_output in results_generator:
720 + >>> if await request.is_disconnected():
721 + >>> # Abort the request if the client disconnects.
722 + >>> await engine.abort(request_id)
723 + >>> # Return or raise an error
724 + >>> ...
725 + >>> final_output = request_output
726 + >>>
727 + >>> # Process and return the final output
728 + >>> ...
729 + """
730 + async for output in self.process_request(
731 + request_id,
732 + prompt,
733 + pooling_params,
734 + prompt_token_ids,
735 + lora_request,
736 + multi_modal_data,
737 + ):
738 + yield output
739 +
740 + async def process_request(
741 + self,
742 + request_id: str,
743 + prompt: Optional[str],
744 + params: Union[SamplingParams, PoolingParams],
745 + prompt_token_ids: Optional[List[int]] = None,
746 + lora_request: Optional[LoRARequest] = None,
747 + multi_modal_data: Optional[MultiModalData] = None,
748 + ) -> AsyncIterator[Union[RequestOutput, EmbeddingRequestOutput]]:
749 + """Common logic to process requests with SamplingParams or
750 + PoolingParams."""
751 + arrival_time = time.time()
752 +
753 + stream = await self.add_request(
754 + request_id,
755 + prompt,
756 + params,
757 + prompt_token_ids=prompt_token_ids,
758 + arrival_time=arrival_time,
759 + lora_request=lora_request,
760 + multi_modal_data=multi_modal_data,
761 + )
762
763 + try:
764 async for request_output in stream:
765 yield request_output
766 except (Exception, asyncio.CancelledError) as e:
663 - # If there is an exception or coroutine is cancelled, abort the
664 - # request.
767 self._abort(request_id)
768 raise e
769
vllm/engine/llm_engine.py
+115 -28
@@ -20,9 +20,12 @@ from vllm.executor.executor_base import ExecutorBase
20 from vllm.executor.ray_utils import initialize_ray_cluster
21 from vllm.logger import init_logger
22 from vllm.lora.request import LoRARequest
23 -from vllm.outputs import RequestOutput
23 +from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
24 + RequestOutputFactory)
25 +from vllm.pooling_params import PoolingParams
26 from vllm.sampling_params import SamplingParams
25 -from vllm.sequence import (ExecuteModelRequest, MultiModalData, SamplerOutput,
27 +from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
28 + MultiModalData, PoolerOutput, SamplerOutput,
29 Sequence, SequenceGroup, SequenceGroupMetadata,
30 SequenceStatus)
31 from vllm.transformers_utils.detokenizer import Detokenizer
@@ -169,7 +172,8 @@ class LLMEngine:
172 load_config=load_config,
173 )
174
172 - self._initialize_kv_caches()
175 + if not self.model_config.embedding_mode:
176 + self._initialize_kv_caches()
177
178 # If usage stat is enabled, collect relevant info.
179 if is_usage_stats_enabled():
@@ -354,7 +358,7 @@ class LLMEngine:
358 self,
359 request_id: str,
360 prompt: Optional[str],
357 - sampling_params: SamplingParams,
361 + params: Union[SamplingParams, PoolingParams],
362 prompt_token_ids: Optional[List[int]] = None,
363 arrival_time: Optional[float] = None,
364 lora_request: Optional[LoRARequest] = None,
@@ -370,7 +374,8 @@ class LLMEngine:
374 request_id: The unique ID of the request.
375 prompt: The prompt string. Can be None if prompt_token_ids is
376 provided.
373 - sampling_params: The sampling parameters for text generation.
377 + params: Parameters for sampling or pooling. SamplingParams
378 + for text generation. PoolingParams for pooling.
379 prompt_token_ids: The token IDs of the prompt. If None, we
380 use the tokenizer to convert the prompts to token IDs.
381 arrival_time: The arrival time of the request. If None, we use
@@ -404,13 +409,6 @@ class LLMEngine:
409 if lora_request is not None and not self.lora_config:
410 raise ValueError(f"Got lora_request {lora_request} but LoRA is "
411 "not enabled!")
407 - max_logprobs = self.get_model_config().max_logprobs
408 - if (sampling_params.logprobs
409 - and sampling_params.logprobs > max_logprobs) or (
410 - sampling_params.prompt_logprobs
411 - and sampling_params.prompt_logprobs > max_logprobs):
412 - raise ValueError(f"Cannot request more than "
413 - f"{max_logprobs} logprobs.")
412 if arrival_time is None:
413 arrival_time = time.time()
414 prompt_token_ids = self.encode_request(
@@ -432,6 +430,50 @@ class LLMEngine:
430 seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
431 eos_token_id, lora_request)
432
433 + # Create a SequenceGroup based on SamplingParams or PoolingParams
434 + if isinstance(params, SamplingParams):
435 + seq_group = self._create_sequence_group_with_sampling(
436 + request_id,
437 + seq,
438 + params,
439 + arrival_time,
440 + lora_request,
441 + multi_modal_data,
442 + )
443 + elif isinstance(params, PoolingParams):
444 + seq_group = self._create_sequence_group_with_pooling(
445 + request_id,
446 + seq,
447 + params,
448 + arrival_time,
449 + lora_request,
450 + multi_modal_data,
451 + )
452 + else:
453 + raise ValueError(
454 + "Either SamplingParams or PoolingParams must be provided.")
455 +
456 + # Add the sequence group to the scheduler.
457 + self.scheduler.add_seq_group(seq_group)
458 +
459 + def _create_sequence_group_with_sampling(
460 + self,
461 + request_id: str,
462 + seq: Sequence,
463 + sampling_params: SamplingParams,
464 + arrival_time: Optional[float] = None,
465 + lora_request: Optional[LoRARequest] = None,
466 + multi_modal_data: Optional[MultiModalData] = None,
467 + ) -> SequenceGroup:
468 + """Creates a SequenceGroup with SamplingParams."""
469 + max_logprobs = self.get_model_config().max_logprobs
470 + if (sampling_params.logprobs
471 + and sampling_params.logprobs > max_logprobs) or (
472 + sampling_params.prompt_logprobs
473 + and sampling_params.prompt_logprobs > max_logprobs):
474 + raise ValueError(f"Cannot request more than "
475 + f"{max_logprobs} logprobs.")
476 +
477 # Defensive copy of SamplingParams, which are used by the sampler,
478 # this doesn't deep-copy LogitsProcessor objects
479 sampling_params = sampling_params.clone()
@@ -443,11 +485,35 @@ class LLMEngine:
485 self.generation_config_fields)
486
487 # Create the sequence group.
446 - seq_group = SequenceGroup(request_id, [seq], sampling_params,
447 - arrival_time, lora_request, multi_modal_data)
488 + seq_group = SequenceGroup(request_id=request_id,
489 + seqs=[seq],
490 + arrival_time=arrival_time,
491 + sampling_params=sampling_params,
492 + lora_request=lora_request,
493 + multi_modal_data=multi_modal_data)
494
449 - # Add the sequence group to the scheduler.
450 - self.scheduler.add_seq_group(seq_group)
495 + return seq_group
496 +
497 + def _create_sequence_group_with_pooling(
498 + self,
499 + request_id: str,
500 + seq: Sequence,
501 + pooling_params: PoolingParams,
502 + arrival_time: Optional[float] = None,
503 + lora_request: Optional[LoRARequest] = None,
504 + multi_modal_data: Optional[MultiModalData] = None,
505 + ) -> SequenceGroup:
506 + """Creates a SequenceGroup with PoolingParams."""
507 + # Defensive copy of PoolingParams, which are used by the pooler
508 + pooling_params = pooling_params.clone()
509 + # Create the sequence group.
510 + seq_group = SequenceGroup(request_id=request_id,
511 + seqs=[seq],
512 + arrival_time=arrival_time,
513 + lora_request=lora_request,
514 + multi_modal_data=multi_modal_data,
515 + pooling_params=pooling_params)
516 + return seq_group
517
518 def abort_request(self, request_id: Union[str, Iterable[str]]) -> None:
519 """Aborts a request(s) with the given ID.
@@ -484,13 +550,25 @@ class LLMEngine:
550 """Returns True if there are unfinished requests."""
551 return self.scheduler.has_unfinished_seqs()
552
553 + def _process_sequence_group_outputs(
554 + self,
555 + seq_group: SequenceGroup,
556 + outputs: List[EmbeddingSequenceGroupOutput],
557 + ) -> None:
558 + seq_group.embeddings = outputs[0].embeddings
559 +
560 + for seq in seq_group.get_seqs():
561 + seq.status = SequenceStatus.FINISHED_STOPPED
562 +
563 + return
564 +
565 def _process_model_outputs(
566 self,
489 - output: List[SamplerOutput],
567 + output: List[Union[SamplerOutput, PoolerOutput]],
568 scheduled_seq_groups: List[ScheduledSequenceGroup],
569 ignored_seq_groups: List[SequenceGroup],
570 seq_group_metadata_list: List[SequenceGroupMetadata],
493 - ) -> List[RequestOutput]:
571 + ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
572 """Apply the model output to the sequences in the scheduled seq groups.
573
574 Returns RequestOutputs that can be returned to the client.
@@ -510,6 +588,9 @@ class LLMEngine:
588 seq_group = scheduled_seq_group.seq_group
589 seq_group.update_num_computed_tokens(
590 scheduled_seq_group.token_chunk_size)
591 + if self.model_config.embedding_mode:
592 + self._process_sequence_group_outputs(seq_group, outputs)
593 + continue
594
595 self.output_processor.process_prompt_logprob(seq_group, outputs)
596 if seq_group_meta.do_sample:
@@ -519,18 +600,19 @@ class LLMEngine:
600 self.scheduler.free_finished_seq_groups()
601
602 # Create the outputs.
522 - request_outputs: List[RequestOutput] = []
603 + request_outputs: List[Union[RequestOutput,
604 + EmbeddingRequestOutput]] = []
605 for scheduled_seq_group in scheduled_seq_groups:
606 seq_group = scheduled_seq_group.seq_group
607 seq_group.maybe_set_first_token_time(now)
526 - request_output = RequestOutput.from_seq_group(seq_group)
608 + request_output = RequestOutputFactory.create(seq_group)
609 request_outputs.append(request_output)
610 for seq_group in ignored_seq_groups:
529 - request_output = RequestOutput.from_seq_group(seq_group)
611 + request_output = RequestOutputFactory.create(seq_group)
612 request_outputs.append(request_output)
613 return request_outputs
614
533 - def step(self) -> List[RequestOutput]:
615 + def step(self) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
616 """Performs one decoding iteration and returns newly generated results.
617
618 .. figure:: https://i.imgur.com/sv2HssD.png
@@ -570,7 +652,7 @@ class LLMEngine:
652 >>> while True:
653 >>> if example_inputs:
654 >>> req_id, prompt, sampling_params = example_inputs.pop(0)
573 - >>> engine.add_request(str(req_id), prompt, sampling_params)
655 + >>> engine.add_request(str(req_id),prompt,sampling_params)
656 >>>
657 >>> # continue the request processing
658 >>> request_outputs = engine.step()
@@ -637,12 +719,15 @@ class LLMEngine:
719
720 # KV Cache Usage in %
721 num_total_gpu = self.cache_config.num_gpu_blocks
640 - num_free_gpu = self.scheduler.block_manager.get_num_free_gpu_blocks()
641 - gpu_cache_usage_sys = 1.0 - (num_free_gpu / num_total_gpu)
722 + gpu_cache_usage_sys = 0.
723 + if num_total_gpu is not None:
724 + num_free_gpu = self.scheduler.block_manager.get_num_free_gpu_blocks(
725 + )
726 + gpu_cache_usage_sys = 1.0 - (num_free_gpu / num_total_gpu)
727
728 num_total_cpu = self.cache_config.num_cpu_blocks
729 cpu_cache_usage_sys = 0.
645 - if num_total_cpu > 0:
730 + if num_total_cpu is not None and num_total_cpu > 0:
731 num_free_cpu = self.scheduler.block_manager.get_num_free_cpu_blocks(
732 )
733 cpu_cache_usage_sys = 1.0 - (num_free_cpu / num_total_cpu)
@@ -716,8 +801,10 @@ class LLMEngine:
801 seq.get_output_len()
802 for seq in seq_group.get_finished_seqs()
803 ])
719 - best_of_requests.append(seq_group.sampling_params.best_of)
720 - n_requests.append(seq_group.sampling_params.n)
804 + if seq_group.sampling_params is not None:
805 + best_of_requests.append(
806 + seq_group.sampling_params.best_of)
807 + n_requests.append(seq_group.sampling_params.n)
808 finished_reason_requests.extend([
809 SequenceStatus.get_finished_reason(seq.status)
810 for seq in seq_group.get_finished_seqs()
vllm/entrypoints/llm.py
+121 -29
@@ -6,13 +6,17 @@ from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
6
7 from vllm.engine.arg_utils import EngineArgs
8 from vllm.engine.llm_engine import LLMEngine
9 +from vllm.logger import init_logger
10 from vllm.lora.request import LoRARequest
10 -from vllm.outputs import RequestOutput
11 +from vllm.outputs import EmbeddingRequestOutput, RequestOutput
12 +from vllm.pooling_params import PoolingParams
13 from vllm.sampling_params import SamplingParams
14 from vllm.sequence import MultiModalData
15 from vllm.usage.usage_lib import UsageContext
16 from vllm.utils import Counter
17
18 +logger = init_logger(__name__)
19 +
20
21 class LLM:
22 """An LLM for generating texts from given prompts and sampling parameters.
@@ -164,8 +168,89 @@ class LLM:
168 multi_modal_data: Multi modal data.
169
170 Returns:
167 - A list of `RequestOutput` objects containing the generated
168 - completions in the same order as the input prompts.
171 + A list of `RequestOutput` objects containing the
172 + generated completions in the same order as the input prompts.
173 + """
174 + if sampling_params is None:
175 + # Use default sampling params.
176 + sampling_params = SamplingParams()
177 +
178 + requests_data = self._validate_and_prepare_requests(
179 + prompts,
180 + sampling_params,
181 + prompt_token_ids,
182 + lora_request,
183 + multi_modal_data,
184 + )
185 +
186 + # Add requests to the engine and run the engine
187 + for request_data in requests_data:
188 + self._add_request(**request_data)
189 +
190 + return self._run_engine(use_tqdm)
191 +
192 + def encode(
193 + self,
194 + prompts: Optional[Union[str, List[str]]] = None,
195 + pooling_params: Optional[Union[PoolingParams,
196 + List[PoolingParams]]] = None,
197 + prompt_token_ids: Optional[List[List[int]]] = None,
198 + use_tqdm: bool = True,
199 + lora_request: Optional[LoRARequest] = None,
200 + multi_modal_data: Optional[MultiModalData] = None,
201 + ) -> List[EmbeddingRequestOutput]:
202 + """Generates the completions for the input prompts.
203 +
204 + NOTE: This class automatically batches the given prompts, considering
205 + the memory constraint. For the best performance, put all of your prompts
206 + into a single list and pass it to this method.
207 +
208 + Args:
209 + prompts: A list of prompts to generate completions for.
210 + pooling_params: The pooling parameters for pooling. If None, we
211 + use the default pooling parameters.
212 + prompt_token_ids: A list of token IDs for the prompts. If None, we
213 + use the tokenizer to convert the prompts to token IDs.
214 + use_tqdm: Whether to use tqdm to display the progress bar.
215 + lora_request: LoRA request to use for generation, if any.
216 + multi_modal_data: Multi modal data.
217 +
218 + Returns:
219 + A list of `EmbeddingRequestOutput` objects containing the
220 + generated embeddings in the same order as the input prompts.
221 + """
222 + if pooling_params is None:
223 + # Use default pooling params.
224 + pooling_params = PoolingParams()
225 +
226 + requests_data = self._validate_and_prepare_requests(
227 + prompts,
228 + pooling_params,
229 + prompt_token_ids,
230 + lora_request,
231 + multi_modal_data,
232 + )
233 +
234 + # Add requests to the engine and run the engine
235 + for request_data in requests_data:
236 + self._add_request(**request_data)
237 +
238 + return self._run_engine(use_tqdm)
239 +
240 + def _validate_and_prepare_requests(
241 + self,
242 + prompts: Optional[Union[str, List[str]]],
243 + params: Union[Union[SamplingParams, PoolingParams],
244 + List[Union[SamplingParams,
245 + PoolingParams]]], # Unified parameter
246 + prompt_token_ids: Optional[List[List[int]]] = None,
247 + lora_request: Optional[LoRARequest] = None,
248 + multi_modal_data: Optional[MultiModalData] = None,
249 + ) -> List[dict]:
250 + """Validates and prepares request data for adding to the engine.
251 +
252 + Ensures prompts and token IDs are consistent, and returns a list of
253 + dictionaries with request data for further processing.
254 """
255 if prompts is None and prompt_token_ids is None:
256 raise ValueError("Either prompts or prompt_token_ids must be "
@@ -188,40 +273,43 @@ class LLM:
273 assert prompt_token_ids is not None
274 num_requests = len(prompt_token_ids)
275
191 - if sampling_params is None:
192 - # Use default sampling params.
193 - sampling_params = SamplingParams()
194 -
195 - elif isinstance(sampling_params,
196 - list) and len(sampling_params) != num_requests:
197 - raise ValueError("The lengths of prompts and sampling_params "
276 + if isinstance(params, list) and len(params) != num_requests:
277 + raise ValueError("The lengths of prompts and params "
278 "must be the same.")
279 if multi_modal_data:
280 multi_modal_data.data = multi_modal_data.data.to(torch.float16)
281
282 # Add requests to the engine.
283 + requests_data = []
284 for i in range(num_requests):
285 prompt = prompts[i] if prompts is not None else None
286 token_ids = None if prompt_token_ids is None else prompt_token_ids[
287 i]
207 - self._add_request(
288 +
289 + multi_modal_item = MultiModalData(
290 + type=multi_modal_data.type,
291 + data=multi_modal_data.data[i].unsqueeze(0),
292 + ) if multi_modal_data else None
293 +
294 + requests_data.append({
295 + "prompt":
296 prompt,
209 - sampling_params[i]
210 - if isinstance(sampling_params, list) else sampling_params,
297 + "params":
298 + params[i] if isinstance(params, list) else params,
299 + "prompt_token_ids":
300 token_ids,
212 - lora_request=lora_request,
213 - # Get ith image while maintaining the batch dim.
214 - multi_modal_data=MultiModalData(
215 - type=multi_modal_data.type,
216 - data=multi_modal_data.data[i].unsqueeze(0))
217 - if multi_modal_data else None,
218 - )
219 - return self._run_engine(use_tqdm)
301 + "lora_request":
302 + lora_request,
303 + "multi_modal_data":
304 + multi_modal_item,
305 + })
306 +
307 + return requests_data
308
309 def _add_request(
310 self,
311 prompt: Optional[str],
224 - sampling_params: SamplingParams,
312 + params: Union[SamplingParams, PoolingParams],
313 prompt_token_ids: Optional[List[int]],
314 lora_request: Optional[LoRARequest] = None,
315 multi_modal_data: Optional[MultiModalData] = None,
@@ -229,12 +317,14 @@ class LLM:
317 request_id = str(next(self.request_counter))
318 self.llm_engine.add_request(request_id,
319 prompt,
232 - sampling_params,
320 + params,
321 prompt_token_ids,
322 lora_request=lora_request,
323 multi_modal_data=multi_modal_data)
324
237 - def _run_engine(self, use_tqdm: bool) -> List[RequestOutput]:
325 + def _run_engine(
326 + self, use_tqdm: bool
327 + ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
328 # Initialize tqdm.
329 if use_tqdm:
330 num_requests = self.llm_engine.get_num_unfinished_requests()
@@ -245,7 +335,7 @@ class LLM:
335 postfix=f"Generation Speed: {0:.2f} toks/s",
336 )
337 # Run the engine.
248 - outputs: List[RequestOutput] = []
338 + outputs: List[Union[RequestOutput, EmbeddingRequestOutput]] = []
339 total_toks = 0
340 while self.llm_engine.has_unfinished_requests():
341 step_outputs = self.llm_engine.step()
@@ -253,10 +343,12 @@ class LLM:
343 if output.finished:
344 outputs.append(output)
345 if use_tqdm:
256 - total_toks += (sum(
257 - len(stp.token_ids) for stp in output.outputs))
258 - spd = total_toks / pbar.format_dict["elapsed"]
259 - pbar.postfix = f"Generation Speed: {spd:.2f} toks/s"
346 + if isinstance(output, RequestOutput):
347 + # Calculate tokens only for RequestOutput
348 + total_toks += sum(
349 + len(stp.token_ids) for stp in output.outputs)
350 + spd = total_toks / pbar.format_dict["elapsed"]
351 + pbar.postfix = f"Generation Speed: {spd:.2f} toks/s"
352 pbar.update(1)
353 if use_tqdm:
354 pbar.close()
vllm/entrypoints/openai/api_server.py
+18 -2
@@ -22,9 +22,11 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
22 from vllm.entrypoints.openai.cli_args import make_arg_parser
23 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
24 ChatCompletionResponse,
25 - CompletionRequest, ErrorResponse)
25 + CompletionRequest,
26 + EmbeddingRequest, ErrorResponse)
27 from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
28 from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
29 +from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
30 from vllm.logger import init_logger
31 from vllm.usage.usage_lib import UsageContext
32
@@ -32,6 +34,8 @@ TIMEOUT_KEEP_ALIVE = 5 # seconds
34
35 openai_serving_chat: OpenAIServingChat
36 openai_serving_completion: OpenAIServingCompletion
37 +openai_serving_embedding: OpenAIServingEmbedding
38 +
39 logger = init_logger(__name__)
40
41 _running_tasks: Set[asyncio.Task] = set()
@@ -123,6 +127,17 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
127 return JSONResponse(content=generator.model_dump())
128
129
130 +@app.post("/v1/embeddings")
131 +async def create_embedding(request: EmbeddingRequest, raw_request: Request):
132 + generator = await openai_serving_embedding.create_embedding(
133 + request, raw_request)
134 + if isinstance(generator, ErrorResponse):
135 + return JSONResponse(content=generator.model_dump(),
136 + status_code=generator.code)
137 + else:
138 + return JSONResponse(content=generator.model_dump())
139 +
140 +
141 if __name__ == "__main__":
142 args = parse_args()
143
@@ -190,7 +205,8 @@ if __name__ == "__main__":
205 args.chat_template)
206 openai_serving_completion = OpenAIServingCompletion(
207 engine, model_config, served_model_names, args.lora_modules)
193 -
208 + openai_serving_embedding = OpenAIServingEmbedding(engine, model_config,
209 + served_model_names)
210 app.root_path = args.root_path
211 uvicorn.run(app,
212 host=args.host,
vllm/entrypoints/openai/protocol.py
+35 -1
@@ -1,13 +1,14 @@
1 # Adapted from
2 # https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
3 import time
4 -from typing import Dict, List, Literal, Optional, Union
4 +from typing import Any, Dict, List, Literal, Optional, Union
5
6 import torch
7 from openai.types.chat import ChatCompletionMessageParam
8 from pydantic import BaseModel, ConfigDict, Field, model_validator
9 from typing_extensions import Annotated
10
11 +from vllm.pooling_params import PoolingParams
12 from vllm.sampling_params import SamplingParams
13 from vllm.utils import random_uuid
14
@@ -363,6 +364,24 @@ class CompletionRequest(OpenAIBaseModel):
364 return data
365
366
367 +class EmbeddingRequest(BaseModel):
368 + # Ordered by official OpenAI API documentation
369 + # https://platform.openai.com/docs/api-reference/embeddings
370 + model: str
371 + input: Union[List[int], List[List[int]], str, List[str]]
372 + encoding_format: Optional[str] = Field('float', pattern='^(float|base64)$')
373 + dimensions: Optional[int] = None
374 + user: Optional[str] = None
375 +
376 + # doc: begin-embedding-pooling-params
377 + additional_data: Optional[Any] = None
378 +
379 + # doc: end-embedding-pooling-params
380 +
381 + def to_pooling_params(self):
382 + return PoolingParams(additional_data=self.additional_data)
383 +
384 +
385 class LogProbs(OpenAIBaseModel):
386 text_offset: List[int] = Field(default_factory=list)
387 token_logprobs: List[Optional[float]] = Field(default_factory=list)
@@ -416,6 +435,21 @@ class CompletionStreamResponse(OpenAIBaseModel):
435 usage: Optional[UsageInfo] = Field(default=None)
436
437
438 +class EmbeddingResponseData(BaseModel):
439 + index: int
440 + object: str = "embedding"
441 + embedding: List[float]
442 +
443 +
444 +class EmbeddingResponse(BaseModel):
445 + id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
446 + object: str = "list"
447 + created: int = Field(default_factory=lambda: int(time.time()))
448 + model: str
449 + data: List[EmbeddingResponseData]
450 + usage: UsageInfo
451 +
452 +
453 class ChatMessage(OpenAIBaseModel):
454 role: str
455 content: str
vllm/entrypoints/openai/serving_embedding.py new
+134
@@ -0,0 +1,134 @@
1 +import time
2 +from typing import AsyncIterator, List, Tuple
3 +
4 +from fastapi import Request
5 +
6 +from vllm.config import ModelConfig
7 +from vllm.engine.async_llm_engine import AsyncLLMEngine
8 +from vllm.entrypoints.openai.protocol import (EmbeddingRequest,
9 + EmbeddingResponse,
10 + EmbeddingResponseData, UsageInfo)
11 +from vllm.entrypoints.openai.serving_completion import parse_prompt_format
12 +from vllm.entrypoints.openai.serving_engine import OpenAIServing
13 +from vllm.logger import init_logger
14 +from vllm.outputs import EmbeddingRequestOutput
15 +from vllm.utils import merge_async_iterators, random_uuid
16 +
17 +logger = init_logger(__name__)
18 +
19 +TypeTokenIDs = List[int]
20 +
21 +
22 +def request_output_to_embedding_response(
23 + final_res_batch: List[EmbeddingRequestOutput],
24 + request_id: str,
25 + created_time: int,
26 + model_name: str,
27 +) -> EmbeddingResponse:
28 + data = []
29 + num_prompt_tokens = 0
30 + for idx, final_res in enumerate(final_res_batch):
31 + assert final_res is not None
32 + prompt_token_ids = final_res.prompt_token_ids
33 +
34 + embedding_data = EmbeddingResponseData(
35 + index=idx, embedding=final_res.outputs.embedding)
36 + data.append(embedding_data)
37 +
38 + num_prompt_tokens += len(prompt_token_ids)
39 +
40 + usage = UsageInfo(
41 + prompt_tokens=num_prompt_tokens,
42 + total_tokens=num_prompt_tokens,
43 + )
44 +
45 + return EmbeddingResponse(
46 + id=request_id,
47 + created=created_time,
48 + model=model_name,
49 + data=data,
50 + usage=usage,
51 + )
52 +
53 +
54 +class OpenAIServingEmbedding(OpenAIServing):
55 +
56 + def __init__(self, engine: AsyncLLMEngine, model_config: ModelConfig,
57 + served_model_names: List[str]):
58 + super().__init__(engine=engine,
59 + model_config=model_config,
60 + served_model_names=served_model_names,
61 + lora_modules=None)
62 + self._check_embedding_mode(model_config.embedding_mode)
63 +
64 + async def create_embedding(self, request: EmbeddingRequest,
65 + raw_request: Request):
66 + """Completion API similar to OpenAI's API.
67 +
68 + See https://platform.openai.com/docs/api-reference/embeddings/create
69 + for the API specification. This API mimics the OpenAI Embedding API.
70 + """
71 + error_check_ret = await self._check_model(request)
72 + if error_check_ret is not None:
73 + return error_check_ret
74 +
75 + # Return error for unsupported features.
76 + if request.encoding_format == "base64":
77 + return self.create_error_response(
78 + "base64 encoding is not currently supported")
79 + if request.dimensions is not None:
80 + return self.create_error_response(
81 + "dimensions is currently not supported")
82 +
83 + model_name = request.model
84 + request_id = f"cmpl-{random_uuid()}"
85 + created_time = int(time.monotonic())
86 +
87 + # Schedule the request and get the result generator.
88 + generators = []
89 + try:
90 + prompt_is_tokens, prompts = parse_prompt_format(request.input)
91 + pooling_params = request.to_pooling_params()
92 +
93 + for i, prompt in enumerate(prompts):
94 + if prompt_is_tokens:
95 + prompt_formats = self._validate_prompt_and_tokenize(
96 + request, prompt_ids=prompt)
97 + else:
98 + prompt_formats = self._validate_prompt_and_tokenize(
99 + request, prompt=prompt)
100 +
101 + prompt_ids, prompt_text = prompt_formats
102 +
103 + generators.append(
104 + self.engine.generate(prompt_text,
105 + pooling_params,
106 + f"{request_id}-{i}",
107 + prompt_token_ids=prompt_ids))
108 + except ValueError as e:
109 + # TODO: Use a vllm-specific Validation Error
110 + return self.create_error_response(str(e))
111 +
112 + result_generator: AsyncIterator[Tuple[
113 + int, EmbeddingRequestOutput]] = merge_async_iterators(*generators)
114 +
115 + # Non-streaming response
116 + final_res_batch: EmbeddingRequestOutput = [None] * len(prompts)
117 + async for i, res in result_generator:
118 + if await raw_request.is_disconnected():
119 + # Abort the request if the client disconnects.
120 + await self.engine.abort(f"{request_id}-{i}")
121 + # TODO: Use a vllm-specific Validation Error
122 + return self.create_error_response("Client disconnected")
123 + final_res_batch[i] = res
124 + response = request_output_to_embedding_response(
125 + final_res_batch, request_id, created_time, model_name)
126 +
127 + return response
128 +
129 + def _check_embedding_mode(self, embedding_mode: bool):
130 + if not embedding_mode:
131 + logger.warning(
132 + "embedding_mode is False. Embedding API will not work.")
133 + else:
134 + logger.info("Activating the server engine with embedding enabled.")
vllm/entrypoints/openai/serving_engine.py
+14 -2
@@ -9,7 +9,8 @@ from typing_extensions import Annotated
9 from vllm.config import ModelConfig
10 from vllm.engine.async_llm_engine import AsyncLLMEngine
11 from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12 - CompletionRequest, ErrorResponse,
12 + CompletionRequest,
13 + EmbeddingRequest, ErrorResponse,
14 LogProbs, ModelCard, ModelList,
15 ModelPermission)
16 from vllm.logger import init_logger
@@ -165,7 +166,8 @@ class OpenAIServing:
166
167 def _validate_prompt_and_tokenize(
168 self,
168 - request: Union[ChatCompletionRequest, CompletionRequest],
169 + request: Union[ChatCompletionRequest, CompletionRequest,
170 + EmbeddingRequest],
171 prompt: Optional[str] = None,
172 prompt_ids: Optional[List[int]] = None,
173 truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
@@ -191,6 +193,16 @@ class OpenAIServing:
193 prompt_ids)
194 token_num = len(input_ids)
195
196 + # Note: EmbeddingRequest doesn't have max_tokens
197 + if isinstance(request, EmbeddingRequest):
198 + if token_num > self.max_model_len:
199 + raise ValueError(
200 + f"This model's maximum context length is "
201 + f"{self.max_model_len} tokens. However, you requested "
202 + f"{token_num} tokens in the input for embedding "
203 + f"generation. Please reduce the length of the input.", )
204 + return input_ids, input_text
205 +
206 if request.max_tokens is None:
207 if token_num >= self.max_model_len:
208 raise ValueError(
vllm/executor/gpu_executor.py
+5 -5
@@ -1,9 +1,9 @@
1 -from typing import Any, Dict, List, Optional, Set, Tuple
1 +from typing import Any, Dict, List, Optional, Set, Tuple, Union
2
3 from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4 from vllm.logger import init_logger
5 from vllm.lora.request import LoRARequest
6 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
6 +from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
7 from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
8 make_async)
9 from vllm.worker.worker_base import WorkerWrapperBase
@@ -123,8 +123,8 @@ class GPUExecutor(ExecutorBase):
123 self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
124
125 def execute_model(
126 - self,
127 - execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
126 + self, execute_model_req: ExecuteModelRequest
127 + ) -> List[Union[SamplerOutput, PoolerOutput]]:
128 output = self.driver_worker.execute_model(execute_model_req)
129 return output
130
@@ -150,7 +150,7 @@ class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
150 async def execute_model_async(
151 self,
152 execute_model_req: ExecuteModelRequest,
153 - ) -> List[SamplerOutput]:
153 + ) -> List[Union[SamplerOutput, PoolerOutput]]:
154 output = await make_async(self.driver_worker.execute_model
155 )(execute_model_req=execute_model_req, )
156 return output
vllm/model_executor/layers/pooler.py new
+56
@@ -0,0 +1,56 @@
1 +from enum import IntEnum
2 +
3 +import torch
4 +import torch.nn as nn
5 +
6 +from vllm.model_executor.pooling_metadata import (PoolingMetadata,
7 + PoolingTensors)
8 +from vllm.sequence import EmbeddingSequenceGroupOutput, PoolerOutput
9 +
10 +
11 +class PoolingType(IntEnum):
12 + """Enumeration for different types of pooling methods."""
13 + LAST = 0
14 +
15 +
16 +class Pooler(nn.Module):
17 + """A layer that pools specific information from hidden states.
18 +
19 + This layer does the following:
20 + 1. Extracts specific tokens or aggregates data based on pooling method.
21 + 2. Normalizes output if specified.
22 + 3. Returns structured results as `PoolerOutput`.
23 +
24 + Attributes:
25 + pooling_type: The type of pooling to use (LAST, AVERAGE, MAX).
26 + normalize: Whether to normalize the pooled data.
27 + """
28 +
29 + def __init__(self, pooling_type: PoolingType, normalize: bool):
30 + super().__init__()
31 + self.pooling_type = pooling_type
32 + self.normalize = normalize
33 +
34 + def forward(
35 + self,
36 + hidden_states: torch.Tensor,
37 + pooling_metadata: PoolingMetadata,
38 + ) -> PoolerOutput:
39 + """Pools specific information from hidden states based on metadata."""
40 + prompt_lens = PoolingTensors.from_pooling_metadata(
41 + pooling_metadata, hidden_states.device).prompt_lens
42 +
43 + if self.pooling_type == PoolingType.LAST:
44 + last_token_flat_indices = torch.cumsum(prompt_lens, dim=0) - 1
45 + pooled_data = hidden_states[last_token_flat_indices]
46 + else:
47 + raise ValueError(f"Invalid pooling type: {self.pooling_type}")
48 +
49 + if self.normalize:
50 + pooled_data = nn.functional.normalize(pooled_data, p=2, dim=1)
51 +
52 + pooled_outputs = [
53 + EmbeddingSequenceGroupOutput(data.tolist()) for data in pooled_data
54 + ]
55 +
56 + return PoolerOutput(outputs=pooled_outputs)
vllm/model_executor/layers/sampler.py
+4 -3
@@ -10,8 +10,9 @@ from vllm.model_executor.sampling_metadata import (SamplingMetadata,
10 SamplingTensors,
11 SequenceGroupToSample)
12 from vllm.sampling_params import SamplingType
13 -from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
14 - SamplerOutput, SequenceGroupOutput, SequenceOutput)
13 +from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
14 + PromptLogprobs, SampleLogprobs, SamplerOutput,
15 + SequenceOutput)
16
17 # (num_token_ids, num_parent_ids) per sequence group.
18 SampleResultType = List[Tuple[List[int], List[int]]]
@@ -1019,7 +1020,7 @@ def _build_sampler_output(
1020 seq_outputs.append(
1021 SequenceOutput(seq_ids[parent_id], next_token_id, logprobs))
1022 sampler_output.append(
1022 - SequenceGroupOutput(seq_outputs, group_prompt_logprobs))
1023 + CompletionSequenceGroupOutput(seq_outputs, group_prompt_logprobs))
1024
1025 # If not specified, store None values in SamplerOutput.
1026 if on_device_tensors is not None:
vllm/model_executor/models/__init__.py
+11 -1
@@ -9,7 +9,7 @@ from vllm.utils import is_hip
9 logger = init_logger(__name__)
10
11 # Architecture -> (module, class).
12 -_MODELS = {
12 +_GENERATION_MODELS = {
13 "AquilaModel": ("llama", "LlamaForCausalLM"),
14 "AquilaForCausalLM": ("llama", "LlamaForCausalLM"), # AquilaChat2
15 "BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
@@ -58,6 +58,12 @@ _MODELS = {
58 "XverseForCausalLM": ("xverse", "XverseForCausalLM"),
59 }
60
61 +_EMBEDDING_MODELS = {
62 + "MistralModel": ("llama_embedding", "LlamaEmbeddingModel"),
63 +}
64 +
65 +_MODELS = {**_GENERATION_MODELS, **_EMBEDDING_MODELS}
66 +
67 # Architecture -> type.
68 # out of tree models
69 _OOT_MODELS: Dict[str, Type[nn.Module]] = {}
@@ -114,6 +120,10 @@ class ModelRegistry:
120 global _OOT_MODELS
121 _OOT_MODELS[model_arch] = model_cls
122
123 + @staticmethod
124 + def is_embedding_model(model_arch: str) -> bool:
125 + return model_arch in _EMBEDDING_MODELS
126 +
127
128 __all__ = [
129 "ModelRegistry",
vllm/model_executor/models/llama_embedding.py new
+87
@@ -0,0 +1,87 @@
1 +from typing import Iterable, List, Optional, Tuple
2 +
3 +import torch
4 +from torch import nn
5 +
6 +from vllm.attention import AttentionMetadata
7 +from vllm.model_executor.layers.pooler import Pooler, PoolingType
8 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
9 +from vllm.model_executor.models.llama import LlamaModel
10 +from vllm.model_executor.pooling_metadata import PoolingMetadata
11 +from vllm.sequence import PoolerOutput
12 +
13 +
14 +class LlamaEmbeddingModel(nn.Module):
15 + """A model that uses Llama with additional embedding functionalities.
16 +
17 + This class encapsulates the LlamaModel and provides an interface for
18 + embedding operations and customized pooling functions.
19 +
20 + Attributes:
21 + model: An instance of LlamaModel used for forward operations.
22 + _pooler: An instance of Pooler used for pooling operations.
23 + """
24 +
25 + def __init__(
26 + self,
27 + **kwargs,
28 + ) -> None:
29 + super().__init__()
30 + self.model = LlamaModel(**kwargs)
31 + self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
32 +
33 + def forward(
34 + self,
35 + input_ids: Optional[torch.Tensor],
36 + positions: torch.Tensor,
37 + kv_caches: List[torch.Tensor],
38 + attn_metadata: AttentionMetadata,
39 + inputs_embeds: Optional[torch.Tensor] = None,
40 + ) -> torch.Tensor:
41 + return self.model.forward(input_ids, positions, kv_caches,
42 + attn_metadata, inputs_embeds)
43 +
44 + def pooler(
45 + self,
46 + hidden_states: torch.Tensor,
47 + pooling_metadata: PoolingMetadata,
48 + ) -> Optional[PoolerOutput]:
49 + return self._pooler(hidden_states, pooling_metadata)
50 +
51 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
52 + stacked_params_mapping = [
53 + # (param_name, shard_name, shard_id)
54 + ("qkv_proj", "q_proj", "q"),
55 + ("qkv_proj", "k_proj", "k"),
56 + ("qkv_proj", "v_proj", "v"),
57 + ("gate_up_proj", "gate_proj", 0),
58 + ("gate_up_proj", "up_proj", 1),
59 + ]
60 + params_dict = dict(self.model.named_parameters())
61 + for name, loaded_weight in weights:
62 + if "rotary_emb.inv_freq" in name:
63 + continue
64 + if ("rotary_emb.cos_cached" in name
65 + or "rotary_emb.sin_cached" in name):
66 + # Models trained using ColossalAI may include these tensors in
67 + # the checkpoint. Skip them.
68 + continue
69 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
70 + if weight_name not in name:
71 + continue
72 + name = name.replace(weight_name, param_name)
73 + # Skip loading extra bias for GPTQ models.
74 + if name.endswith(".bias") and name not in params_dict:
75 + continue
76 + param = params_dict[name]
77 + weight_loader = param.weight_loader
78 + weight_loader(param, loaded_weight, shard_id)
79 + break
80 + else:
81 + # Skip loading extra bias for GPTQ models.
82 + if name.endswith(".bias") and name not in params_dict:
83 + continue
84 + param = params_dict[name]
85 + weight_loader = getattr(param, "weight_loader",
86 + default_weight_loader)
87 + weight_loader(param, loaded_weight)
vllm/model_executor/pooling_metadata.py new
+69
@@ -0,0 +1,69 @@
1 +from dataclasses import dataclass
2 +from typing import Any, Dict, List, Tuple
3 +
4 +import torch
5 +
6 +from vllm.pooling_params import PoolingParams
7 +from vllm.utils import is_pin_memory_available
8 +
9 +
10 +class PoolingMetadata:
11 + """Metadata for pooling operations in the Pooler layer.
12 +
13 + This class holds the necessary information for pooling operations,
14 + providing context for how to perform pooling and other related operations.
15 +
16 + Attributes:
17 + seq_groups: List of (seq_ids, pooling_params).
18 + seq_data: A mapping of sequence ID to additional sequence data.
19 + prompt_lens: List of the lengths of each prompt.
20 + """
21 +
22 + def __init__(
23 + self,
24 + seq_groups: List[Tuple[List[int], PoolingParams]],
25 + seq_data: Dict[int, Any], # Specific data related to sequences
26 + prompt_lens: List[int],
27 + ) -> None:
28 + self.seq_groups = seq_groups
29 + self.seq_data = seq_data
30 + self.prompt_lens = prompt_lens
31 +
32 + def __repr__(self) -> str:
33 + return ("PoolingMetadata("
34 + f"seq_groups={self.seq_groups}, "
35 + f"seq_data={self.seq_data}, "
36 + f"prompt_lens={self.prompt_lens})")
37 +
38 +
39 +@dataclass
40 +class PoolingTensors:
41 + """Tensors for pooling."""
42 +
43 + prompt_lens: torch.Tensor
44 +
45 + @classmethod
46 + def from_pooling_metadata(
47 + cls,
48 + pooling_metadata: "PoolingMetadata",
49 + device: torch.device,
50 + ) -> "PoolingTensors":
51 + """
52 + Create PoolingTensors from PoolingMetadata.
53 +
54 + Args:
55 + pooling_metadata: PoolingMetadata instance to convert.
56 + device: Device to store the tensors.
57 + """
58 + # Convert prompt lengths to tensor
59 + pin_memory = is_pin_memory_available()
60 +
61 + prompt_lens_t = torch.tensor(
62 + pooling_metadata.prompt_lens,
63 + device="cpu",
64 + dtype=torch.long,
65 + pin_memory=pin_memory,
66 + )
67 +
68 + return cls(prompt_lens=prompt_lens_t.to(device=device,
69 + non_blocking=True), )
vllm/outputs.py
+81 -1
@@ -57,8 +57,27 @@ class CompletionOutput:
57 f"stop_reason={self.stop_reason})")
58
59
60 +class EmbeddingOutput:
61 + """The output data of one completion output of a request.
62 +
63 + Args:
64 + embedding: The embedding vector, which is a list of floats. The
65 + length of vector depends on the model as listed in the embedding guide.
66 + """
67 +
68 + def __init__(
69 + self,
70 + embedding: List[float],
71 + ) -> None:
72 + self.embedding = embedding
73 +
74 + def __repr__(self) -> str:
75 + return (f"EmbeddingOutput("
76 + f"embedding={len(self.embedding)}")
77 +
78 +
79 class RequestOutput:
61 - """The output data of a request to the LLM.
80 + """The output data of a completion request to the LLM.
81
82 Args:
83 request_id: The unique ID of the request.
@@ -93,6 +112,9 @@ class RequestOutput:
112
113 @classmethod
114 def from_seq_group(cls, seq_group: SequenceGroup) -> "RequestOutput":
115 + if seq_group.sampling_params is None:
116 + raise ValueError(
117 + "Sampling parameters are missing for a CompletionRequest.")
118 seqs = seq_group.get_seqs()
119 if len(seqs) == 1:
120 top_n_seqs = seqs
@@ -148,3 +170,61 @@ class RequestOutput:
170 f"finished={self.finished}, "
171 f"metrics={self.metrics}, "
172 f"lora_request={self.lora_request})")
173 +
174 +
175 +class EmbeddingRequestOutput:
176 + """
177 + The output data of an embedding request to the LLM.
178 +
179 + Args:
180 + request_id (str): A unique identifier for the embedding request.
181 + outputs (EmbeddingOutput): The embedding results for the given input.
182 + prompt_token_ids (List[int]): A list of token IDs used in the prompt.
183 + finished (bool): A flag indicating whether the embedding is completed.
184 + """
185 +
186 + def __init__(self, request_id: str, outputs: 'EmbeddingOutput',
187 + prompt_token_ids: List[int], finished: bool):
188 + self.request_id = request_id
189 + self.prompt_token_ids = prompt_token_ids
190 + self.finished = finished
191 + self.outputs = outputs
192 +
193 + @classmethod
194 + def from_seq_group(cls,
195 + seq_group: 'SequenceGroup') -> "EmbeddingRequestOutput":
196 + if seq_group.embeddings is None:
197 + raise ValueError(
198 + "Embeddings are missing in seq_group for EmbeddingRequest.")
199 + output = EmbeddingOutput(seq_group.embeddings)
200 + prompt_token_ids = seq_group.prompt_token_ids
201 + finished = seq_group.is_finished()
202 +
203 + return cls(seq_group.request_id, output, prompt_token_ids, finished)
204 +
205 + def __repr__(self):
206 + """
207 + Returns a string representation of an EmbeddingRequestOutput instance.
208 +
209 + The representation includes the request_id and the number of outputs,
210 + providing a quick overview of the embedding request's results.
211 +
212 + Returns:
213 + str: A string representation of the EmbeddingRequestOutput instance.
214 + """
215 + return (f"EmbeddingRequestOutput(request_id='{self.request_id}', "
216 + f"outputs={repr(self.outputs)}, "
217 + f"prompt_token_ids={self.prompt_token_ids}, "
218 + f"finished={self.finished})")
219 +
220 +
221 +class RequestOutputFactory:
222 +
223 + @staticmethod
224 + def create(seq_group):
225 + # Determine the type based on a condition, for example:
226 + if hasattr(seq_group,
227 + 'embeddings') and seq_group.embeddings is not None:
228 + return EmbeddingRequestOutput.from_seq_group(seq_group)
229 + else:
230 + return RequestOutput.from_seq_group(seq_group)
vllm/pooling_params.py new
+20
@@ -0,0 +1,20 @@
1 +from typing import Any, Optional
2 +
3 +
4 +class PoolingParams:
5 + """Pooling parameters for pooling.
6 +
7 + Attributes:
8 + additional_data: Any additional data needed for pooling.
9 + """
10 +
11 + def __init__(self, additional_data: Optional[Any] = None):
12 + self.additional_data = additional_data
13 +
14 + def clone(self) -> "PoolingParams":
15 + """Returns a deep copy of the PoolingParams instance."""
16 + return PoolingParams(additional_data=self.additional_data, )
17 +
18 + def __repr__(self) -> str:
19 + return (f"PoolingParams("
20 + f"additional_metadata={self.additional_data})")
vllm/sequence.py
+77 -12
@@ -1,11 +1,13 @@
1 """Sequence and its related classes."""
2 import copy
3 import enum
4 +from abc import ABC, abstractmethod
5 from dataclasses import dataclass, field
6 from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
7
8 from vllm.block import LogicalTokenBlock
9 from vllm.lora.request import LoRARequest
10 +from vllm.pooling_params import PoolingParams
11 from vllm.sampling_params import SamplingParams
12
13 if TYPE_CHECKING:
@@ -375,12 +377,12 @@ class SequenceGroupState:
377
378 class MultiModalData:
379 """Multi modal request.
378 -
380 +
381 Args:
382 type: The data type.
383 data: The actual data.
384 The required shape and semantic meaning of it depends on the vision
383 - language config of the hosted model.
385 + language config of the hosted model.
386 See `VisionLanguageConfig` in `config.py`.
387 """
388
@@ -402,16 +404,22 @@ class SequenceGroup:
404 arrival_time: The arrival time of the request.
405 lora_request: LoRA request.
406 multi_modal_data: Multi modal data associated with the request.
407 + embeddings: The embeddings vectors of the prompt of the sequence group
408 + for an embedding model.
409 + pooling_params: The pooling parameters used to generate the pooling
410 + for an embedding model.
411 """
412
413 def __init__(
414 self,
415 request_id: str,
416 seqs: List[Sequence],
411 - sampling_params: SamplingParams,
417 arrival_time: float,
418 + sampling_params: Optional[SamplingParams] = None,
419 lora_request: Optional[LoRARequest] = None,
420 multi_modal_data: Optional[MultiModalData] = None,
421 + embeddings: Optional[List[float]] = None,
422 + pooling_params: Optional[PoolingParams] = None,
423 ) -> None:
424 self.request_id = request_id
425 self.seqs_dict = {seq.seq_id: seq for seq in seqs}
@@ -425,6 +433,8 @@ class SequenceGroup:
433 self.prompt_logprobs: Optional[PromptLogprobs] = None
434 self.state = SequenceGroupState()
435 self.multi_modal_data = multi_modal_data
436 + self.embeddings = embeddings
437 + self.pooling_params = pooling_params
438
439 @property
440 def prompt(self) -> str:
@@ -479,12 +489,13 @@ class SequenceGroup:
489 def get_max_num_running_seqs(self) -> int:
490 """The maximum number of sequences running in parallel in the remaining
491 lifetime of the request."""
482 - if self.sampling_params.use_beam_search:
492 + if self.sampling_params and self.sampling_params.use_beam_search:
493 # For beam search, maximally there will always be `best_of` beam
494 # candidates running in the future.
495 return self.sampling_params.best_of
496 else:
487 - if self.sampling_params.best_of > self.num_seqs():
497 + if (self.sampling_params
498 + and self.sampling_params.best_of > self.num_seqs()):
499 # At prompt stage, the sequence group is not yet filled up
500 # and only have one sequence running. However, in the
501 # generation stage, we will have `best_of` sequences running.
@@ -555,7 +566,7 @@ class SequenceGroup:
566 return all(seq.is_finished() for seq in self.get_seqs())
567
568 def is_prefill(self) -> bool:
558 - # Every sequences should be in the same stage.
569 + # Every sequence should be in the same stage.
570 return self.get_seqs()[0].is_prefill()
571
572 def __repr__(self) -> str:
@@ -594,6 +605,7 @@ class SequenceGroupMetadata:
605 sampling_params: SamplingParams,
606 block_tables: Dict[int, List[int]],
607 do_sample: bool = True,
608 + pooling_params: Optional[PoolingParams] = None,
609 token_chunk_size: Optional[int] = None,
610 lora_request: Optional[LoRARequest] = None,
611 computed_block_nums: Optional[List[int]] = None,
@@ -605,6 +617,7 @@ class SequenceGroupMetadata:
617 self.seq_data = seq_data
618 self.sampling_params = sampling_params
619 self.block_tables = block_tables
620 + self.pooling_params = pooling_params
621 self.lora_request = lora_request
622 self.computed_block_nums = computed_block_nums
623 self.multi_modal_data = multi_modal_data
@@ -669,8 +682,20 @@ class SequenceOutput:
682 return equal and log_probs_equal
683
684
672 -class SequenceGroupOutput:
673 - """The model output associated with a sequence group."""
685 +class SequenceGroupOutput(ABC):
686 + """The base class for model outputs associated with a sequence group."""
687 +
688 + @abstractmethod
689 + def __repr__(self) -> str:
690 + pass
691 +
692 + @abstractmethod
693 + def __eq__(self, other: object) -> bool:
694 + pass
695 +
696 +
697 +class CompletionSequenceGroupOutput(SequenceGroupOutput):
698 + """The model output associated with a completion sequence group."""
699
700 def __init__(
701 self,
@@ -682,26 +707,45 @@ class SequenceGroupOutput:
707 self.prompt_logprobs = prompt_logprobs
708
709 def __repr__(self) -> str:
685 - return (f"SequenceGroupOutput(samples={self.samples}, "
710 + return (f"CompletionSequenceGroupOutput(samples={self.samples}, "
711 f"prompt_logprobs={self.prompt_logprobs})")
712
713 def __eq__(self, other: object) -> bool:
689 - if not isinstance(other, SequenceGroupOutput):
714 + if not isinstance(other, CompletionSequenceGroupOutput):
715 raise NotImplementedError()
716 return (self.samples == other.samples
717 and self.prompt_logprobs == other.prompt_logprobs)
718
719
720 +class EmbeddingSequenceGroupOutput(SequenceGroupOutput):
721 + """The model output associated with an embedding sequence group."""
722 +
723 + def __init__(
724 + self,
725 + embeddings: List[float],
726 + ) -> None:
727 + self.embeddings = embeddings
728 +
729 + def __repr__(self) -> str:
730 + return (f"EmbeddingSequenceGroupOutput("
731 + f"embeddings_shape={len(self.embeddings)})")
732 +
733 + def __eq__(self, other: object) -> bool:
734 + if not isinstance(other, EmbeddingSequenceGroupOutput):
735 + raise NotImplementedError()
736 + return self.embeddings == other.embeddings
737 +
738 +
739 @dataclass
740 class SamplerOutput:
741 """For each sequence group, we generate a list of SequenceOutput object,
742 each of which contains one possible candidate for the next token.
743
700 - This datastructure implements methods so it can be used like a list, but
744 + This data structure implements methods, so it can be used like a list, but
745 also has optional fields for device tensors.
746 """
747
704 - outputs: List[SequenceGroupOutput]
748 + outputs: List[CompletionSequenceGroupOutput]
749
750 # On-device tensor containing probabilities of each token.
751 sampled_token_probs: Optional["torch.Tensor"] = None
@@ -742,6 +786,27 @@ class SamplerOutput:
786 f"spec_decode_worker_metrics={self.spec_decode_worker_metrics})")
787
788
789 +@dataclass
790 +class PoolerOutput:
791 + """The output from a pooling operation in the embedding model."""
792 + outputs: List[EmbeddingSequenceGroupOutput]
793 +
794 + spec_decode_worker_metrics: Optional["SpecDecodeWorkerMetrics"] = None
795 +
796 + def __getitem__(self, idx: int):
797 + return self.outputs[idx]
798 +
799 + def __setitem__(self, idx: int, value):
800 + self.outputs[idx] = value
801 +
802 + def __len__(self):
803 + return len(self.outputs)
804 +
805 + def __eq__(self, other: object):
806 + return isinstance(other,
807 + self.__class__) and self.outputs == other.outputs
808 +
809 +
810 @dataclass
811 class ExecuteModelRequest:
812 """The model execution request."""
vllm/spec_decode/util.py
+3 -2
@@ -4,7 +4,8 @@ from typing import Dict, List, Tuple
4
5 import torch
6
7 -from vllm.sequence import (Logprob, SamplerOutput, SequenceGroupMetadata,
7 +from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
8 + SamplerOutput, SequenceGroupMetadata,
9 SequenceGroupOutput, SequenceOutput)
10
11 SeqId = int
@@ -94,7 +95,7 @@ def create_sequence_group_output(
95 for topk_logprob_index, _ in enumerate(topk_token_ids)
96 })
97
97 - return SequenceGroupOutput(
98 + return CompletionSequenceGroupOutput(
99 samples=[
100 SequenceOutput(parent_seq_id=seq_id,
101 output_token=token_id,
vllm/worker/embedding_model_runner.py new
+266
@@ -0,0 +1,266 @@
1 +from typing import Dict, List, Optional, Set, Tuple
2 +
3 +import torch
4 +
5 +from vllm.attention import AttentionMetadata
6 +from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7 + ModelConfig, ParallelConfig, SchedulerConfig,
8 + VisionLanguageConfig)
9 +from vllm.distributed import broadcast_tensor_dict
10 +from vllm.logger import init_logger
11 +from vllm.lora.layers import LoRAMapping
12 +from vllm.lora.request import LoRARequest
13 +from vllm.model_executor.pooling_metadata import PoolingMetadata
14 +from vllm.pooling_params import PoolingParams
15 +from vllm.sequence import PoolerOutput, SequenceData, SequenceGroupMetadata
16 +from vllm.worker.model_runner import BatchType, ModelRunner
17 +
18 +logger = init_logger(__name__)
19 +
20 +
21 +class EmbeddingModelRunner(ModelRunner):
22 +
23 + def __init__(
24 + self,
25 + model_config: ModelConfig,
26 + parallel_config: ParallelConfig,
27 + scheduler_config: SchedulerConfig,
28 + device_config: DeviceConfig,
29 + cache_config: CacheConfig,
30 + load_config: LoadConfig,
31 + lora_config: Optional[LoRAConfig],
32 + kv_cache_dtype: Optional[str] = "auto",
33 + is_driver_worker: bool = False,
34 + vision_language_config: Optional[VisionLanguageConfig] = None,
35 + ):
36 + super().__init__(model_config,
37 + parallel_config,
38 + scheduler_config,
39 + device_config,
40 + cache_config,
41 + load_config,
42 + lora_config=lora_config,
43 + kv_cache_dtype=kv_cache_dtype,
44 + is_driver_worker=is_driver_worker,
45 + vision_language_config=vision_language_config)
46 +
47 + @torch.inference_mode()
48 + def execute_model(
49 + self,
50 + seq_group_metadata_list: List[SequenceGroupMetadata],
51 + kv_caches: List[torch.Tensor],
52 + ) -> Optional[PoolerOutput]:
53 + (input_tokens, input_positions, attn_metadata, pooling_metadata,
54 + lora_requests, lora_mapping, multi_modal_input
55 + ) = self.prepare_input_tensors(seq_group_metadata_list)
56 +
57 + if self.lora_config:
58 + self.set_active_loras(lora_requests, lora_mapping)
59 +
60 + # Currently cuda graph is only supported by the decode phase.
61 + prefill_meta = attn_metadata.prefill_metadata
62 + decode_meta = attn_metadata.decode_metadata
63 + if prefill_meta is None and decode_meta.use_cuda_graph:
64 + graph_batch_size = input_tokens.shape[0]
65 + model_executable = self.graph_runners[graph_batch_size]
66 + else:
67 + model_executable = self.model
68 +
69 + num_layers = self.model_config.get_num_layers(self.parallel_config)
70 + kv_caches = [None] * num_layers
71 +
72 + execute_model_kwargs = {
73 + "input_ids": input_tokens,
74 + "positions": input_positions,
75 + "kv_caches": kv_caches,
76 + "attn_metadata": attn_metadata,
77 + }
78 + if self.vision_language_config:
79 + execute_model_kwargs.update({"image_input": multi_modal_input})
80 + hidden_states = model_executable(**execute_model_kwargs)
81 +
82 + return self.model.pooler(hidden_states=hidden_states,
83 + pooling_metadata=pooling_metadata)
84 +
85 + def prepare_input_tensors(
86 + self,
87 + seq_group_metadata_list: List[SequenceGroupMetadata],
88 + ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, PoolingMetadata,
89 + Set[LoRARequest], LoRAMapping, torch.Tensor]:
90 + if self.is_driver_worker:
91 + prefill_reqs = []
92 + decode_reqs = []
93 + for seq_group_meta in seq_group_metadata_list:
94 + if seq_group_meta.is_prompt:
95 + prefill_reqs.append(seq_group_meta)
96 + else:
97 + decode_reqs.append(seq_group_meta)
98 +
99 + # Prepare input tensors.
100 + (
101 + input_tokens,
102 + input_positions,
103 + prefill_attn_metadata,
104 + prompt_lens,
105 + subquery_lens,
106 + lora_index_mapping,
107 + lora_prompt_mapping,
108 + lora_requests,
109 + multi_modal_input,
110 + slot_mapping,
111 + ) = self._prepare_prompt(prefill_reqs)
112 + (
113 + decode_input_tokens,
114 + decode_input_positions,
115 + decode_attn_metadata,
116 + decode_lora_index_mapping,
117 + decode_lora_prompt_mapping,
118 + decode_lora_requests,
119 + decode_slot_mapping,
120 + ) = self._prepare_decode(decode_reqs)
121 +
122 + # Prepare PoolingMetadata
123 + pooling_metadata = self._prepare_pooling(seq_group_metadata_list,
124 + prompt_lens)
125 +
126 + if not self.scheduler_config.chunked_prefill_enabled:
127 + assert (len(prefill_reqs) and len(decode_reqs)) == 0
128 +
129 + num_prefills = len(prompt_lens)
130 + num_prefill_tokens = len(input_tokens)
131 + num_decode_tokens = len(decode_input_tokens)
132 +
133 + # Coalesce tensors. Note that attn_metadata is currently not
134 + # coalesced for simplicity.
135 + input_tokens.extend(decode_input_tokens)
136 + input_positions.extend(decode_input_positions)
137 + slot_mapping.extend(decode_slot_mapping)
138 + lora_index_mapping.extend(decode_lora_index_mapping)
139 + lora_prompt_mapping.extend(decode_lora_prompt_mapping)
140 + lora_requests.update(decode_lora_requests)
141 +
142 + input_tokens = torch.tensor(input_tokens,
143 + dtype=torch.long,
144 + device=self.device)
145 + input_positions = torch.tensor(input_positions,
146 + dtype=torch.long,
147 + device=self.device)
148 + slot_mapping = torch.tensor(slot_mapping,
149 + dtype=torch.long,
150 + device=self.device)
151 +
152 + if self.lora_config:
153 + lora_mapping = LoRAMapping(
154 + lora_index_mapping,
155 + lora_prompt_mapping,
156 + )
157 + else:
158 + lora_mapping = None
159 +
160 + # Broadcast the metadata.
161 + # If batch contains both prefill and decode, it sends 2 broadcasts.
162 + # If it only contains 1 type, it triggers a single broadcast.
163 + if (prefill_attn_metadata is not None
164 + and decode_attn_metadata is not None):
165 + batch_type = BatchType.MIXED
166 + elif prefill_attn_metadata is not None:
167 + batch_type = BatchType.PREFILL
168 + else:
169 + batch_type = BatchType.DECODE
170 +
171 + metadata_dict = {
172 + "input_tokens": input_tokens,
173 + "input_positions": input_positions,
174 + "lora_requests": lora_requests,
175 + "lora_mapping": lora_mapping,
176 + "multi_modal_input": multi_modal_input,
177 + "num_prefill_tokens": num_prefill_tokens,
178 + "num_decode_tokens": num_decode_tokens,
179 + "slot_mapping": slot_mapping,
180 + "num_prefills": num_prefills,
181 + "batch_type": batch_type,
182 + }
183 + if prefill_attn_metadata is not None:
184 + metadata_dict.update(prefill_attn_metadata.asdict_zerocopy())
185 + else:
186 + assert decode_attn_metadata is not None
187 + metadata_dict.update(decode_attn_metadata.asdict_zerocopy())
188 + broadcast_tensor_dict(metadata_dict, src=0)
189 +
190 + # Broadcast decode attn metadata for mixed batch type.
191 + # The additional broadcast costs 300us overhead on 4 A10 GPUs.
192 + # We can potentially reduce the overhead by coelescing tensors.
193 + if batch_type == BatchType.MIXED:
194 + assert decode_attn_metadata is not None
195 + metadata_dict = decode_attn_metadata.asdict_zerocopy()
196 + broadcast_tensor_dict(metadata_dict, src=0)
197 + else:
198 + metadata_dict = broadcast_tensor_dict(src=0)
199 + input_tokens = metadata_dict.pop("input_tokens")
200 + input_positions = metadata_dict.pop("input_positions")
201 + slot_mapping = metadata_dict.pop("slot_mapping")
202 + num_prefills = metadata_dict.pop("num_prefills")
203 + lora_mapping = metadata_dict.pop("lora_mapping")
204 + lora_requests = metadata_dict.pop("lora_requests")
205 + multi_modal_input = metadata_dict.pop("multi_modal_input")
206 + num_prefill_tokens = metadata_dict.pop("num_prefill_tokens")
207 + num_decode_tokens = metadata_dict.pop("num_decode_tokens")
208 + batch_type = metadata_dict.pop("batch_type")
209 +
210 + # Create an attention metadata.
211 + prefill_attn_metadata = None
212 + decode_attn_metadata = None
213 + if batch_type == BatchType.PREFILL or batch_type == BatchType.MIXED:
214 + prefill_attn_metadata = self.attn_backend.make_metadata(
215 + **metadata_dict)
216 + else:
217 + decode_attn_metadata = self.attn_backend.make_metadata(
218 + **metadata_dict)
219 +
220 + pooling_metadata = PoolingMetadata(seq_groups=None,
221 + seq_data=None,
222 + prompt_lens=None)
223 +
224 + # if it is a mixed batch, decode attn_metadata is broadcasted
225 + # separately.
226 + if batch_type == BatchType.MIXED:
227 + metadata_dict = broadcast_tensor_dict(src=0)
228 + decode_attn_metadata = self.attn_backend.make_metadata(
229 + **metadata_dict)
230 +
231 + attn_metadata = AttentionMetadata(
232 + num_prefills=num_prefills,
233 + slot_mapping=slot_mapping,
234 + num_prefill_tokens=num_prefill_tokens,
235 + num_decode_tokens=num_decode_tokens,
236 + prefill_metadata=prefill_attn_metadata,
237 + decode_metadata=decode_attn_metadata,
238 + kv_cache_dtype=self.kv_cache_dtype,
239 + )
240 +
241 + return (input_tokens, input_positions, attn_metadata, pooling_metadata,
242 + lora_requests, lora_mapping, multi_modal_input)
243 +
244 + def _prepare_pooling(
245 + self,
246 + seq_group_metadata_list: List[SequenceGroupMetadata],
247 + prompt_lens: List[int],
248 + ) -> PoolingMetadata:
249 + """Prepare PoolingMetadata for the sequence group metadata list."""
250 + seq_groups: List[Tuple[List[int], PoolingParams]] = []
251 + for i, seq_group_metadata in enumerate(seq_group_metadata_list):
252 + seq_ids = list(seq_group_metadata.seq_data.keys())
253 + pooling_params = seq_group_metadata.pooling_params
254 + seq_groups.append((seq_ids, pooling_params))
255 +
256 + seq_data: Dict[int, SequenceData] = {}
257 + for seq_group_metadata in seq_group_metadata_list:
258 + seq_data.update(seq_group_metadata.seq_data)
259 +
260 + pooling_metadata = PoolingMetadata(
261 + seq_groups=seq_groups,
262 + seq_data=seq_data,
263 + prompt_lens=prompt_lens,
264 + )
265 +
266 + return pooling_metadata
vllm/worker/model_runner.py
+18 -7
@@ -1,6 +1,6 @@
1 import time
2 from enum import IntEnum
3 -from typing import Dict, List, NamedTuple, Optional, Set, Tuple
3 +from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union
4
5 import numpy as np
6 import torch
@@ -287,18 +287,18 @@ class ModelRunner:
287 lora_requests.add(seq_group_metadata.lora_request)
288
289 lora_index_mapping += [lora_id] * (seq_len - context_len)
290 - lora_prompt_mapping.extend(
291 - [lora_id] *
292 - (seq_len - context_len
293 - if seq_group_metadata.sampling_params.prompt_logprobs else 1))
290 + lora_prompt_mapping.extend([lora_id] * (
291 + seq_len - context_len if seq_group_metadata.sampling_params
292 + and seq_group_metadata.sampling_params.prompt_logprobs else 1))
293
294 if seq_group_metadata.multi_modal_data:
295 multi_modal_input_list.append(
296 seq_group_metadata.multi_modal_data.data)
297
299 - if seq_group_metadata.block_tables is None:
298 + if _is_block_tables_empty(seq_group_metadata.block_tables):
299 # During memory profiling, the block tables are not initialized
300 # yet. In this case, we just use a dummy slot mapping.
301 + # In embeddings, the block tables are {seq_id: None}.
302 slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
303 continue
304
@@ -813,7 +813,6 @@ class ModelRunner:
813 sampling_params = SamplingParams(top_p=0.99, top_k=self.vocab_size - 1)
814 max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
815 max_num_seqs = self.scheduler_config.max_num_seqs
816 -
816 # This represents the maximum number of different requests
817 # that will have unique loras, an therefore the max amount of memory
818 # consumption create dummy lora request copies from the lora request
@@ -1139,3 +1138,15 @@ def _prepare_fake_inputs(
1138 prompt_tokens = [0] * seq_len
1139 fake_image_input = None
1140 return SequenceData(prompt_tokens), fake_image_input
1141 +
1142 +
1143 +def _is_block_tables_empty(block_tables: Union[None, Dict]):
1144 + """
1145 + Check if block_tables is None or a dictionary with all None values.
1146 + """
1147 + if block_tables is None:
1148 + return True
1149 + if isinstance(block_tables, dict) and all(
1150 + value is None for value in block_tables.values()):
1151 + return True
1152 + return False
vllm/worker/worker.py
+9 -5
@@ -1,7 +1,7 @@
1 """A GPU worker class."""
2 import gc
3 import os
4 -from typing import Any, Dict, List, Optional, Set, Tuple
4 +from typing import Any, Dict, List, Optional, Set, Tuple, Union
5
6 import torch
7 import torch.distributed
@@ -16,8 +16,9 @@ from vllm.distributed.device_communicators.custom_all_reduce import (
16 init_custom_ar)
17 from vllm.lora.request import LoRARequest
18 from vllm.model_executor import set_random_seed
19 -from vllm.sequence import ExecuteModelRequest, SamplerOutput
19 +from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
20 from vllm.worker.cache_engine import CacheEngine
21 +from vllm.worker.embedding_model_runner import EmbeddingModelRunner
22 from vllm.worker.model_runner import ModelRunner
23 from vllm.worker.worker_base import WorkerBase
24
@@ -68,7 +69,9 @@ class Worker(WorkerBase):
69 assert not self.lora_config, (
70 "To be tested: vision language model with LoRA settings.")
71
71 - self.model_runner = ModelRunner(
72 + ModelRunnerClass = (EmbeddingModelRunner if
73 + self.model_config.embedding_mode else ModelRunner)
74 + self.model_runner = ModelRunnerClass(
75 model_config,
76 parallel_config,
77 scheduler_config,
@@ -83,7 +86,8 @@ class Worker(WorkerBase):
86 # Uninitialized cache engine. Will be initialized by
87 # initialize_cache.
88 self.cache_engine: CacheEngine
86 - self.gpu_cache: List[torch.Tensor]
89 + # Initialize gpu_cache as embedding models don't initialize kv_caches
90 + self.gpu_cache: Optional[List[torch.tensor]] = None
91
92 def init_device(self) -> None:
93 if self.device_config.device.type == "cuda":
@@ -209,7 +213,7 @@ class Worker(WorkerBase):
213 def execute_model(
214 self,
215 execute_model_req: Optional[ExecuteModelRequest] = None
212 - ) -> List[SamplerOutput]:
216 + ) -> List[Union[SamplerOutput, PoolerOutput]]:
217
218 if execute_model_req is None:
219 seq_group_metadata_list = None