A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Model][Misc] Add e5-mistral-7b-instruct and Embedding API (#3734)
Chang Su committed
May 11, 2024 at 11:30 UTC
e254497b66dcd87038969b0ad34d34425edfc5fe
38 files changed
+1627
-160
examples/offline_inference_embedding.py
new
+17
@@ -0,0 +1,17 @@
1
+from vllm import LLM
2
+
3
+# Sample prompts.
4
+prompts = [
5
+ "Hello, my name is",
6
+ "The president of the United States is",
7
+ "The capital of France is",
8
+ "The future of AI is",
9
+]
10
+
11
+# Create an LLM.
12
+model = LLM(model="intfloat/e5-mistral-7b-instruct", enforce_eager=True)
13
+# Generate embedding. The output is a list of EmbeddingRequestOutputs.
14
+outputs = model.encode(prompts)
15
+# Print the outputs.
16
+for output in outputs:
17
+ print(output.outputs.embedding) # list of 4096 floats
examples/openai_embedding_client.py
new
+23
@@ -0,0 +1,23 @@
1
+from openai import OpenAI
2
+
3
+# Modify OpenAI's API key and API base to use vLLM's API server.
4
+openai_api_key = "EMPTY"
5
+openai_api_base = "http://localhost:8000/v1"
6
+
7
+client = OpenAI(
8
+ # defaults to os.environ.get("OPENAI_API_KEY")
9
+ api_key=openai_api_key,
10
+ base_url=openai_api_base,
11
+)
12
+
13
+models = client.models.list()
14
+model = models.data[0].id
15
+
16
+responses = client.embeddings.create(input=[
17
+ "Hello my name is",
18
+ "The best thing about vLLM is that it supports many different models"
19
+],
20
+ model=model)
21
+
22
+for data in responses.data:
23
+ print(data.embedding) # list of float of len 4096
requirements-dev.txt
+6
-3
@@ -19,12 +19,15 @@ pytest-forked
19
pytest-asyncio
20
pytest-rerunfailures
21
pytest-shard
22
-httpx
22
+
23
+# testing utils
24
+awscli
25
einops # required for MPT
26
+httpx
27
+peft
28
requests
29
ray
26
-peft
27
-awscli
30
+sentence-transformers # required for embedding
31
32
# Benchmarking
33
aiohttp
tests/conftest.py
+30
-8
@@ -133,6 +133,10 @@ _VISION_LANGUAGE_MODELS = {
133
"llava-hf/llava-1.5-7b-hf": LlavaForConditionalGeneration,
134
}
135
136
+_EMBEDDING_MODELS = [
137
+ "intfloat/e5-mistral-7b-instruct",
138
+]
139
+
140
141
class HfRunner:
142
@@ -145,14 +149,7 @@ class HfRunner:
149
assert dtype in _STR_DTYPE_TO_TORCH_DTYPE
150
torch_dtype = _STR_DTYPE_TO_TORCH_DTYPE[dtype]
151
self.model_name = model_name
148
- if model_name not in _VISION_LANGUAGE_MODELS:
149
- self.model = AutoModelForCausalLM.from_pretrained(
150
- model_name,
151
- torch_dtype=torch_dtype,
152
- trust_remote_code=True,
153
- ).cuda()
154
- self.processor = None
155
- else:
152
+ if model_name in _VISION_LANGUAGE_MODELS:
153
self.model = _VISION_LANGUAGE_MODELS[model_name].from_pretrained(
154
model_name,
155
torch_dtype=torch_dtype,
@@ -162,6 +159,20 @@ class HfRunner:
159
model_name,
160
torch_dtype=torch_dtype,
161
)
162
+ elif model_name in _EMBEDDING_MODELS:
163
+ # Lazy init required for AMD CI
164
+ from sentence_transformers import SentenceTransformer
165
+ self.model = SentenceTransformer(
166
+ model_name,
167
+ device="cpu",
168
+ ).to(dtype=torch_dtype).cuda()
169
+ else:
170
+ self.model = AutoModelForCausalLM.from_pretrained(
171
+ model_name,
172
+ torch_dtype=torch_dtype,
173
+ trust_remote_code=True,
174
+ ).cuda()
175
+ self.processor = None
176
if tokenizer_name is None:
177
tokenizer_name = model_name
178
self.tokenizer = get_tokenizer(tokenizer_name, trust_remote_code=True)
@@ -334,6 +345,9 @@ class HfRunner:
345
return [(output_ids, output_str, output_logprobs)
346
for output_ids, output_str, output_logprobs in outputs]
347
348
+ def encode(self, prompts: List[str]) -> List[List[torch.Tensor]]:
349
+ return self.model.encode(prompts)
350
+
351
def __del__(self):
352
del self.model
353
cleanup()
@@ -459,6 +473,14 @@ class VllmRunner:
473
outputs = self.generate(prompts, beam_search_params)
474
return outputs
475
476
+ def encode(self, prompts: List[str]) -> List[List[float]]:
477
+ req_outputs = self.model.encode(prompts)
478
+ outputs = []
479
+ for req_output in req_outputs:
480
+ embedding = req_output.outputs.embedding
481
+ outputs.append(embedding)
482
+ return outputs
483
+
484
def __del__(self):
485
del self.model
486
cleanup()
tests/engine/output_processor/test_multi_step.py
+6
-6
@@ -9,8 +9,8 @@ from vllm.core.scheduler import Scheduler
9
from vllm.engine.output_processor.multi_step import MultiStepOutputProcessor
10
from vllm.engine.output_processor.stop_checker import StopChecker
11
from vllm.sampling_params import SamplingParams
12
-from vllm.sequence import (Logprob, SequenceGroupOutput, SequenceOutput,
13
- SequenceStatus)
12
+from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
13
+ SequenceOutput, SequenceStatus)
14
from vllm.transformers_utils.detokenizer import Detokenizer
15
from vllm.utils import Counter
16
@@ -51,7 +51,7 @@ def test_appends_token_ids(num_new_tokens: int, seq_output_len: int):
51
new_token_ids = list(range(num_new_tokens))
52
53
outputs = [
54
- SequenceGroupOutput(
54
+ CompletionSequenceGroupOutput(
55
samples=[
56
SequenceOutput(
57
parent_seq_id=seq.seq_id,
@@ -103,7 +103,7 @@ def test_respects_max_tokens(num_new_tokens: int, seq_prompt_len: int,
103
new_token_ids = list(range(num_new_tokens))
104
105
outputs = [
106
- SequenceGroupOutput(
106
+ CompletionSequenceGroupOutput(
107
samples=[
108
SequenceOutput(
109
parent_seq_id=seq.seq_id,
@@ -170,7 +170,7 @@ def test_respects_eos_token_id(num_new_tokens: int, seq_prompt_len: int,
170
new_token_ids[eos_index] = eos_token_id
171
172
outputs = [
173
- SequenceGroupOutput(
173
+ CompletionSequenceGroupOutput(
174
samples=[
175
SequenceOutput(
176
parent_seq_id=seq.seq_id,
@@ -239,7 +239,7 @@ def test_ignores_eos_token_id(num_new_tokens: int, seq_prompt_len: int,
239
new_token_ids[eos_index] = eos_token_id
240
241
outputs = [
242
- SequenceGroupOutput(
242
+ CompletionSequenceGroupOutput(
243
samples=[
244
SequenceOutput(
245
parent_seq_id=seq.seq_id,
tests/entrypoints/openai/test_serving_chat.py
+1
@@ -14,6 +14,7 @@ class MockModelConfig:
14
tokenizer_mode = "auto"
15
max_model_len = 100
16
tokenizer_revision = None
17
+ embedding_mode = False
18
19
20
@dataclass
tests/entrypoints/test_openai_server.py
+95
-1
@@ -23,6 +23,7 @@ from vllm.transformers_utils.tokenizer import get_tokenizer
23
MAX_SERVER_START_WAIT_S = 600 # wait for server to start for 60 seconds
24
# any model with a chat template should work here
25
MODEL_NAME = "HuggingFaceH4/zephyr-7b-beta"
26
+EMBEDDING_MODEL_NAME = "intfloat/e5-mistral-7b-instruct"
27
# technically this needs Mistral-7B-v0.1 as base, but we're not testing
28
# generation quality here
29
LORA_NAME = "typeof/zephyr-7b-beta-lora"
@@ -121,7 +122,7 @@ def zephyr_lora_files():
122
return snapshot_download(repo_id=LORA_NAME)
123
124
124
-@pytest.fixture(scope="session")
125
+@pytest.fixture(scope="module")
126
def server(zephyr_lora_files):
127
ray.init()
128
server_runner = ServerRunner.remote([
@@ -150,6 +151,25 @@ def server(zephyr_lora_files):
151
ray.shutdown()
152
153
154
+@pytest.fixture(scope="module")
155
+def embedding_server(zephyr_lora_files):
156
+ ray.shutdown()
157
+ ray.init()
158
+ server_runner = ServerRunner.remote([
159
+ "--model",
160
+ EMBEDDING_MODEL_NAME,
161
+ # use half precision for speed and memory savings in CI environment
162
+ "--dtype",
163
+ "bfloat16",
164
+ "--max-model-len",
165
+ "8192",
166
+ "--enforce-eager",
167
+ ])
168
+ ray.get(server_runner.ready.remote())
169
+ yield server_runner
170
+ ray.shutdown()
171
+
172
+
173
@pytest.fixture(scope="module")
174
def client():
175
client = openai.AsyncOpenAI(
@@ -890,5 +910,79 @@ async def test_long_seed(server, client: openai.AsyncOpenAI):
910
or "less_than_equal" in exc_info.value.message)
911
912
913
+@pytest.mark.parametrize(
914
+ "model_name",
915
+ [EMBEDDING_MODEL_NAME],
916
+)
917
+async def test_single_embedding(embedding_server, client: openai.AsyncOpenAI,
918
+ model_name: str):
919
+ input = [
920
+ "The chef prepared a delicious meal.",
921
+ ]
922
+
923
+ # test single embedding
924
+ embeddings = await client.embeddings.create(
925
+ model=model_name,
926
+ input=input,
927
+ encoding_format="float",
928
+ )
929
+ assert embeddings.id is not None
930
+ assert embeddings.data is not None and len(embeddings.data) == 1
931
+ assert len(embeddings.data[0].embedding) == 4096
932
+ assert embeddings.usage.completion_tokens == 0
933
+ assert embeddings.usage.prompt_tokens == 9
934
+ assert embeddings.usage.total_tokens == 9
935
+
936
+ # test using token IDs
937
+ input = [1, 1, 1, 1, 1]
938
+ embeddings = await client.embeddings.create(
939
+ model=model_name,
940
+ input=input,
941
+ encoding_format="float",
942
+ )
943
+ assert embeddings.id is not None
944
+ assert embeddings.data is not None and len(embeddings.data) == 1
945
+ assert len(embeddings.data[0].embedding) == 4096
946
+ assert embeddings.usage.completion_tokens == 0
947
+ assert embeddings.usage.prompt_tokens == 5
948
+ assert embeddings.usage.total_tokens == 5
949
+
950
+
951
+@pytest.mark.parametrize(
952
+ "model_name",
953
+ [EMBEDDING_MODEL_NAME],
954
+)
955
+async def test_batch_embedding(embedding_server, client: openai.AsyncOpenAI,
956
+ model_name: str):
957
+ # test List[str]
958
+ inputs = [
959
+ "The cat sat on the mat.", "A feline was resting on a rug.",
960
+ "Stars twinkle brightly in the night sky."
961
+ ]
962
+ embeddings = await client.embeddings.create(
963
+ model=model_name,
964
+ input=inputs,
965
+ encoding_format="float",
966
+ )
967
+ assert embeddings.id is not None
968
+ assert embeddings.data is not None and len(embeddings.data) == 3
969
+ assert len(embeddings.data[0].embedding) == 4096
970
+
971
+ # test List[List[int]]
972
+ inputs = [[4, 5, 7, 9, 20], [15, 29, 499], [24, 24, 24, 24, 24],
973
+ [25, 32, 64, 77]]
974
+ embeddings = await client.embeddings.create(
975
+ model=model_name,
976
+ input=inputs,
977
+ encoding_format="float",
978
+ )
979
+ assert embeddings.id is not None
980
+ assert embeddings.data is not None and len(embeddings.data) == 4
981
+ assert len(embeddings.data[0].embedding) == 4096
982
+ assert embeddings.usage.completion_tokens == 0
983
+ assert embeddings.usage.prompt_tokens == 17
984
+ assert embeddings.usage.total_tokens == 17
985
+
986
+
987
if __name__ == "__main__":
988
pytest.main([__file__])
tests/models/test_embedding.py
new
+44
@@ -0,0 +1,44 @@
1
+"""Compare the outputs of HF and vLLM for Mistral models using greedy sampling.
2
+
3
+Run `pytest tests/models/test_llama_embedding.py`.
4
+"""
5
+import pytest
6
+import torch
7
+import torch.nn.functional as F
8
+
9
+MODELS = [
10
+ "intfloat/e5-mistral-7b-instruct",
11
+]
12
+
13
+
14
+def compare_embeddings(embeddings1, embeddings2):
15
+ similarities = [
16
+ F.cosine_similarity(torch.tensor(e1), torch.tensor(e2), dim=0)
17
+ for e1, e2 in zip(embeddings1, embeddings2)
18
+ ]
19
+ return similarities
20
+
21
+
22
+@pytest.mark.parametrize("model", MODELS)
23
+@pytest.mark.parametrize("dtype", ["half"])
24
+def test_models(
25
+ hf_runner,
26
+ vllm_runner,
27
+ example_prompts,
28
+ model: str,
29
+ dtype: str,
30
+) -> None:
31
+ hf_model = hf_runner(model, dtype=dtype)
32
+ hf_outputs = hf_model.encode(example_prompts)
33
+ del hf_model
34
+
35
+ vllm_model = vllm_runner(model, dtype=dtype)
36
+ vllm_outputs = vllm_model.encode(example_prompts)
37
+ del vllm_model
38
+
39
+ similarities = compare_embeddings(hf_outputs, vllm_outputs)
40
+ all_similarities = torch.stack(similarities)
41
+ tolerance = 1e-2
42
+ assert torch.all((all_similarities <= 1.0 + tolerance)
43
+ & (all_similarities >= 1.0 - tolerance)
44
+ ), f"Not all values are within {tolerance} of 1.0"
tests/samplers/test_logits_processor.py
+3
-3
@@ -36,14 +36,14 @@ def test_logits_processor_force_generate(
36
# test logits_processors when prompt_logprobs is not None
37
vllm_model.model._add_request(
38
prompt=example_prompts[0],
39
- sampling_params=params_with_logprobs,
39
+ params=params_with_logprobs,
40
prompt_token_ids=None,
41
)
42
43
# test prompt_logprobs is not None
44
vllm_model.model._add_request(
45
prompt=example_prompts[1],
46
- sampling_params=SamplingParams(
46
+ params=SamplingParams(
47
prompt_logprobs=3,
48
max_tokens=max_tokens,
49
),
@@ -53,7 +53,7 @@ def test_logits_processor_force_generate(
53
# test grouped requests
54
vllm_model.model._add_request(
55
prompt=example_prompts[2],
56
- sampling_params=SamplingParams(max_tokens=max_tokens),
56
+ params=SamplingParams(max_tokens=max_tokens),
57
prompt_token_ids=None,
58
)
59
tests/samplers/test_seeded_generate.py
+1
-1
@@ -60,7 +60,7 @@ def test_random_sample_with_seed(
60
llm._add_request(
61
prompt=prompt,
62
prompt_token_ids=None,
63
- sampling_params=params,
63
+ params=params,
64
)
65
66
results = llm._run_engine(use_tqdm=False)
tests/spec_decode/utils.py
+3
-3
@@ -7,8 +7,8 @@ import torch
7
from vllm.engine.arg_utils import EngineArgs
8
from vllm.model_executor.utils import set_random_seed
9
from vllm.sampling_params import SamplingParams
10
-from vllm.sequence import (Logprob, SamplerOutput, SequenceData,
11
- SequenceGroupMetadata, SequenceGroupOutput,
10
+from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
11
+ SamplerOutput, SequenceData, SequenceGroupMetadata,
12
SequenceOutput)
13
from vllm.utils import get_distributed_init_method, get_ip, get_open_port
14
from vllm.worker.cache_engine import CacheEngine
@@ -170,7 +170,7 @@ def create_sampler_output_list(
170
171
return [
172
SamplerOutput(outputs=[
173
- SequenceGroupOutput(
173
+ CompletionSequenceGroupOutput(
174
samples=[
175
SequenceOutput(
176
output_token=token_id,
tests/test_sequence.py
+6
-6
@@ -1,17 +1,17 @@
1
import pytest
2
3
from tests.core.utils import create_dummy_prompt
4
-from vllm.sequence import (SamplerOutput, SequenceData, SequenceGroupOutput,
5
- SequenceOutput)
4
+from vllm.sequence import (CompletionSequenceGroupOutput, SamplerOutput,
5
+ SequenceData, SequenceOutput)
6
7
8
@pytest.fixture
9
def sample_outputs():
10
return [
11
- SequenceGroupOutput(samples=[
11
+ CompletionSequenceGroupOutput(samples=[
12
SequenceOutput(parent_seq_id=0, output_token=i, logprobs={})
13
],
14
- prompt_logprobs=None) for i in range(5)
14
+ prompt_logprobs=None) for i in range(5)
15
]
16
17
@@ -32,10 +32,10 @@ def test_sampler_output_getitem(sampler_output, sample_outputs):
32
33
34
def test_sampler_output_setitem(sampler_output):
35
- new_output = SequenceGroupOutput(samples=[
35
+ new_output = CompletionSequenceGroupOutput(samples=[
36
SequenceOutput(parent_seq_id=0, output_token=99, logprobs={})
37
],
38
- prompt_logprobs=None)
38
+ prompt_logprobs=None)
39
sampler_output[2] = new_output
40
assert sampler_output[2] == new_output
41
vllm/__init__.py
+6
-1
@@ -6,7 +6,9 @@ from vllm.engine.llm_engine import LLMEngine
6
from vllm.entrypoints.llm import LLM
7
from vllm.executor.ray_utils import initialize_ray_cluster
8
from vllm.model_executor.models import ModelRegistry
9
-from vllm.outputs import CompletionOutput, RequestOutput
9
+from vllm.outputs import (CompletionOutput, EmbeddingOutput,
10
+ EmbeddingRequestOutput, RequestOutput)
11
+from vllm.pooling_params import PoolingParams
12
from vllm.sampling_params import SamplingParams
13
14
__version__ = "0.4.2"
@@ -17,9 +19,12 @@ __all__ = [
19
"SamplingParams",
20
"RequestOutput",
21
"CompletionOutput",
22
+ "EmbeddingOutput",
23
+ "EmbeddingRequestOutput",
24
"LLMEngine",
25
"EngineArgs",
26
"AsyncLLMEngine",
27
"AsyncEngineArgs",
28
"initialize_ray_cluster",
29
+ "PoolingParams",
30
]
vllm/config.py
+15
@@ -9,6 +9,7 @@ from transformers import PretrainedConfig
9
from vllm.logger import init_logger
10
from vllm.model_executor.layers.quantization import (QUANTIZATION_METHODS,
11
get_quantization_config)
12
+from vllm.model_executor.models import ModelRegistry
13
from vllm.transformers_utils.config import get_config, get_hf_text_config
14
from vllm.utils import get_cpu_memory, is_cpu, is_hip, is_neuron
15
@@ -22,6 +23,7 @@ if TYPE_CHECKING:
23
logger = init_logger(__name__)
24
25
_GB = 1 << 30
26
+_EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS = 32768
27
28
29
class ModelConfig:
@@ -126,6 +128,7 @@ class ModelConfig:
128
served_model_name)
129
if not self.skip_tokenizer_init:
130
self._verify_tokenizer_mode()
131
+ self._verify_embedding_mode()
132
self._verify_quantization()
133
self._verify_cuda_graph()
134
@@ -137,6 +140,11 @@ class ModelConfig:
140
"either 'auto' or 'slow'.")
141
self.tokenizer_mode = tokenizer_mode
142
143
+ def _verify_embedding_mode(self) -> None:
144
+ architectures = getattr(self.hf_config, "architectures", [])
145
+ self.embedding_mode = any(
146
+ ModelRegistry.is_embedding_model(arch) for arch in architectures)
147
+
148
def _verify_quantization(self) -> None:
149
supported_quantization = [*QUANTIZATION_METHODS]
150
rocm_supported_quantization = ["gptq", "squeezellm"]
@@ -591,6 +599,7 @@ class SchedulerConfig:
599
prompt latency) before scheduling next prompt.
600
enable_chunked_prefill: If True, prefill requests can be chunked based
601
on the remaining max_num_batched_tokens.
602
+ embedding_mode: Whether the running model is for embedding.
603
"""
604
605
def __init__(
@@ -602,6 +611,7 @@ class SchedulerConfig:
611
num_lookahead_slots: int = 0,
612
delay_factor: float = 0.0,
613
enable_chunked_prefill: bool = False,
614
+ embedding_mode: Optional[bool] = False,
615
) -> None:
616
if max_num_batched_tokens is not None:
617
self.max_num_batched_tokens = max_num_batched_tokens
@@ -610,6 +620,10 @@ class SchedulerConfig:
620
# It is the values that have the best balance between ITL
621
# and TTFT on A100. Note it is not optimized for throughput.
622
self.max_num_batched_tokens = 512
623
+ elif embedding_mode:
624
+ # For embedding, choose specific value for higher throughput
625
+ self.max_num_batched_tokens = max(
626
+ max_model_len, _EMBEDDING_MODEL_MAX_NUM_BATCHED_TOKENS)
627
else:
628
# If max_model_len is too short, use 2048 as the default value
629
# for higher throughput.
@@ -623,6 +637,7 @@ class SchedulerConfig:
637
self.num_lookahead_slots = num_lookahead_slots
638
self.delay_factor = delay_factor
639
self.chunked_prefill_enabled = enable_chunked_prefill
640
+ self.embedding_mode = embedding_mode
641
642
self._verify_args()
643
vllm/core/embedding_model_block_manager.py
new
+84
@@ -0,0 +1,84 @@
1
+from typing import List, Tuple
2
+
3
+from vllm.core.interfaces import AllocStatus, BlockSpaceManager
4
+from vllm.sequence import Sequence, SequenceGroup
5
+
6
+
7
+class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
8
+ """An embedding version of BlockSpaceManager for use in environments
9
+ with embedding models where block management is not required.
10
+
11
+ This class provides the same interface as BlockSpaceManager, but its
12
+ methods perform no actions or return simple values like True in specific
13
+ actions. It's designed to be used in scenarios where the overhead of
14
+ block management is unnecessary, such as in an embedding environment.
15
+ """
16
+
17
+ def __init__(
18
+ self,
19
+ **kwargs,
20
+ ) -> None:
21
+ pass
22
+
23
+ def can_allocate(self, seq_group: SequenceGroup) -> AllocStatus:
24
+ # Always return OK for dummy purposes
25
+ return AllocStatus.OK
26
+
27
+ def allocate(self, seq_group: SequenceGroup) -> None:
28
+ # No actual allocation logic needed
29
+ pass
30
+
31
+ def can_append_slots(self, seq_group: SequenceGroup,
32
+ num_lookahead_slots: int) -> bool:
33
+ return True
34
+
35
+ def append_slots(
36
+ self,
37
+ seq: Sequence,
38
+ num_lookahead_slots: int,
39
+ ) -> List[Tuple[int, int]]:
40
+ return None # type: ignore
41
+
42
+ def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
43
+ pass
44
+
45
+ def can_swap_in(self, seq_group: SequenceGroup,
46
+ num_lookahead_slots: int) -> AllocStatus:
47
+ return AllocStatus.OK
48
+
49
+ def swap_in(self, seq_group: SequenceGroup,
50
+ num_lookahead_slots: int) -> List[Tuple[int, int]]:
51
+ return None # type: ignore
52
+
53
+ def can_swap_out(self, seq_group: SequenceGroup) -> bool:
54
+ return True
55
+
56
+ def swap_out(self, seq_group: SequenceGroup) -> List[Tuple[int, int]]:
57
+ return None # type: ignore
58
+
59
+ def free(self, seq: Sequence) -> None:
60
+ # No operation on free
61
+ return
62
+
63
+ def get_block_table(self, seq: Sequence) -> List[int]:
64
+ return None # type: ignore
65
+
66
+ def get_num_free_gpu_blocks(self) -> int:
67
+ return 1
68
+
69
+ def get_num_free_cpu_blocks(self) -> int:
70
+ return 1
71
+
72
+ def access_all_blocks_in_seq(
73
+ self,
74
+ seq: Sequence,
75
+ access_time: float,
76
+ ) -> None:
77
+ pass
78
+
79
+ def get_common_computed_block_ids(self,
80
+ seq_group: SequenceGroup) -> List[int]:
81
+ return None # type: ignore
82
+
83
+ def mark_blocks_as_computed(self, seq_group: SequenceGroup):
84
+ pass
vllm/core/interfaces.py
+5
@@ -35,6 +35,11 @@ class BlockSpaceManager(ABC):
35
from vllm.core.block_manager_v2 import BlockSpaceManagerV2
36
return BlockSpaceManagerV2
37
38
+ if version == "embedding":
39
+ from vllm.core.embedding_model_block_manager import (
40
+ EmbeddingModelBlockSpaceManager)
41
+ return EmbeddingModelBlockSpaceManager
42
+
43
raise ValueError(f"Unknown version {version=}")
44
45
@abstractmethod
vllm/core/scheduler.py
+8
-2
@@ -270,9 +270,14 @@ class Scheduler:
270
self.scheduler_config.max_model_len,
271
self.scheduler_config.max_num_batched_tokens)
272
273
+ version = "v1"
274
+ if self.scheduler_config.use_v2_block_manager:
275
+ version = "v2"
276
+ if self.scheduler_config.embedding_mode:
277
+ version = "embedding"
278
+
279
BlockSpaceManagerImpl = BlockSpaceManager.get_block_space_manager_class(
274
- version="v2" if self.scheduler_config.
275
- use_v2_block_manager else "v1")
280
+ version)
281
282
# Create the block space manager.
283
self.block_manager = BlockSpaceManagerImpl(
@@ -968,6 +973,7 @@ class Scheduler:
973
sampling_params=seq_group.sampling_params,
974
block_tables=block_tables,
975
do_sample=do_sample,
976
+ pooling_params=seq_group.pooling_params,
977
token_chunk_size=token_chunk_size,
978
lora_request=seq_group.lora_request,
979
computed_block_nums=common_computed_block_nums,
vllm/engine/arg_utils.py
+1
@@ -574,6 +574,7 @@ class EngineArgs:
574
speculative_config.num_lookahead_slots),
575
delay_factor=self.scheduler_delay_factor,
576
enable_chunked_prefill=self.enable_chunked_prefill,
577
+ embedding_mode=model_config.embedding_mode,
578
)
579
lora_config = LoRAConfig(
580
max_lora_rank=self.max_lora_rank,
vllm/engine/async_llm_engine.py
+130
-28
@@ -14,7 +14,8 @@ from vllm.engine.llm_engine import LLMEngine
14
from vllm.executor.ray_utils import initialize_ray_cluster, ray
15
from vllm.logger import init_logger
16
from vllm.lora.request import LoRARequest
17
-from vllm.outputs import RequestOutput
17
+from vllm.outputs import EmbeddingRequestOutput, RequestOutput
18
+from vllm.pooling_params import PoolingParams
19
from vllm.sampling_params import SamplingParams
20
from vllm.sequence import ExecuteModelRequest, MultiModalData, SamplerOutput
21
from vllm.usage.usage_lib import UsageContext
@@ -47,15 +48,16 @@ def _raise_exception_on_finish(
48
49
50
class AsyncStream:
50
- """A stream of RequestOutputs for a request that can be
51
- iterated over asynchronously."""
51
+ """A stream of RequestOutputs or EmbeddingRequestOutputs for a request
52
+ that can be iterated over asynchronously."""
53
54
def __init__(self, request_id: str) -> None:
55
self.request_id = request_id
56
self._queue: asyncio.Queue = asyncio.Queue()
57
self._finished = False
58
58
- def put(self, item: Union[RequestOutput, Exception]) -> None:
59
+ def put(self, item: Union[RequestOutput, EmbeddingRequestOutput,
60
+ Exception]) -> None:
61
if self._finished:
62
return
63
self._queue.put_nowait(item)
@@ -71,7 +73,7 @@ class AsyncStream:
73
def __aiter__(self):
74
return self
75
74
- async def __anext__(self) -> RequestOutput:
76
+ async def __anext__(self) -> Union[RequestOutput, EmbeddingRequestOutput]:
77
result = await self._queue.get()
78
if isinstance(result, Exception):
79
raise result
@@ -108,7 +110,8 @@ class RequestTracker:
110
self.abort_request(rid)
111
112
def process_request_output(self,
111
- request_output: RequestOutput,
113
+ request_output: Union[RequestOutput,
114
+ EmbeddingRequestOutput],
115
*,
116
verbose: bool = False) -> None:
117
"""Process a request output from the engine."""
@@ -196,7 +199,8 @@ class RequestTracker:
199
class _AsyncLLMEngine(LLMEngine):
200
"""Extension of LLMEngine to add async methods."""
201
199
- async def step_async(self) -> List[RequestOutput]:
202
+ async def step_async(
203
+ self) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
204
"""Performs one decoding iteration and returns newly generated results.
205
The workers are ran asynchronously if possible.
206
@@ -251,7 +255,7 @@ class _AsyncLLMEngine(LLMEngine):
255
self,
256
request_id: str,
257
prompt: Optional[str],
254
- sampling_params: SamplingParams,
258
+ params: Union[SamplingParams, PoolingParams],
259
prompt_token_ids: Optional[List[int]] = None,
260
arrival_time: Optional[float] = None,
261
lora_request: Optional[LoRARequest] = None,
@@ -270,8 +274,8 @@ class _AsyncLLMEngine(LLMEngine):
274
275
return self.add_request(request_id,
276
prompt=prompt,
277
+ params=params,
278
prompt_token_ids=prompt_token_ids,
274
- sampling_params=sampling_params,
279
arrival_time=arrival_time,
280
lora_request=lora_request,
281
multi_modal_data=multi_modal_data)
@@ -511,7 +515,7 @@ class AsyncLLMEngine:
515
self,
516
request_id: str,
517
prompt: Optional[str],
514
- sampling_params: SamplingParams,
518
+ params: Union[SamplingParams, PoolingParams],
519
prompt_token_ids: Optional[List[int]] = None,
520
arrival_time: Optional[float] = None,
521
lora_request: Optional[LoRARequest] = None,
@@ -528,9 +532,9 @@ class AsyncLLMEngine:
532
max_log_len]
533
logger.info(
534
"Received request %s: prompt: %r, "
531
- "sampling_params: %s, prompt_token_ids: %s, "
532
- "lora_request: %s.", request_id, shortened_prompt,
533
- sampling_params, shortened_token_ids, lora_request)
535
+ "params: %s, prompt_token_ids: %s, "
536
+ "lora_request: %s.", request_id, shortened_prompt, params,
537
+ shortened_token_ids, lora_request)
538
539
if not self.is_running:
540
if self.start_engine_loop:
@@ -562,7 +566,7 @@ class AsyncLLMEngine:
566
stream = self._request_tracker.add_request(
567
request_id,
568
prompt=prompt,
565
- sampling_params=sampling_params,
569
+ params=params,
570
prompt_token_ids=prompt_token_ids,
571
arrival_time=arrival_time,
572
lora_request=lora_request,
@@ -597,8 +601,8 @@ class AsyncLLMEngine:
601
multi_modal_data: Multi modal data per request.
602
603
Yields:
600
- The output `RequestOutput` objects from the LLMEngine for the
601
- request.
604
+ The output `RequestOutput` objects from the LLMEngine
605
+ for the request.
606
607
Details:
608
- If the engine is not running, start the background loop,
@@ -643,25 +647,123 @@ class AsyncLLMEngine:
647
>>> # Process and return the final output
648
>>> ...
649
"""
646
- # Preprocess the request.
647
- arrival_time = time.time()
648
-
649
- try:
650
- stream = await self.add_request(
650
+ async for output in self.process_request(
651
request_id,
652
prompt,
653
sampling_params,
654
- prompt_token_ids=prompt_token_ids,
655
- arrival_time=arrival_time,
656
- lora_request=lora_request,
657
- multi_modal_data=multi_modal_data,
658
- )
654
+ prompt_token_ids,
655
+ lora_request,
656
+ multi_modal_data,
657
+ ):
658
+ yield output
659
+
660
+ async def encode(
661
+ self,
662
+ prompt: Optional[str],
663
+ pooling_params: PoolingParams,
664
+ request_id: str,
665
+ prompt_token_ids: Optional[List[int]] = None,
666
+ lora_request: Optional[LoRARequest] = None,
667
+ multi_modal_data: Optional[MultiModalData] = None
668
+ ) -> AsyncIterator[EmbeddingRequestOutput]:
669
+ """Generate outputs for a request from an embedding model.
670
+
671
+ Generate outputs for a request. This method is a coroutine. It adds the
672
+ request into the waiting queue of the LLMEngine and streams the outputs
673
+ from the LLMEngine to the caller.
674
+
675
+ Args:
676
+ prompt: The prompt string. Can be None if prompt_token_ids is
677
+ provided.
678
+ pooling_params: The pooling parameters of the request.
679
+ request_id: The unique id of the request.
680
+ prompt_token_ids: The token IDs of the prompt. If None, we
681
+ use the tokenizer to convert the prompts to token IDs.
682
+ lora_request: LoRA request to use for generation, if any.
683
+ multi_modal_data: Multi modal data per request.
684
+
685
+ Yields:
686
+ The output `EmbeddingRequestOutput` objects from the LLMEngine
687
+ for the request.
688
+
689
+ Details:
690
+ - If the engine is not running, start the background loop,
691
+ which iteratively invokes
692
+ :meth:`~vllm.engine.async_llm_engine.AsyncLLMEngine.engine_step`
693
+ to process the waiting requests.
694
+ - Add the request to the engine's `RequestTracker`.
695
+ On the next background loop, this request will be sent to
696
+ the underlying engine.
697
+ Also, a corresponding `AsyncStream` will be created.
698
+ - Wait for the request outputs from `AsyncStream` and yield them.
699
+
700
+ Example:
701
+ >>> # Please refer to entrypoints/api_server.py for
702
+ >>> # the complete example.
703
+ >>>
704
+ >>> # initialize the engine and the example input
705
+ >>> engine = AsyncLLMEngine.from_engine_args(engine_args)
706
+ >>> example_input = {
707
+ >>> "input": "What is LLM?",
708
+ >>> "request_id": 0,
709
+ >>> }
710
+ >>>
711
+ >>> # start the generation
712
+ >>> results_generator = engine.encode(
713
+ >>> example_input["input"],
714
+ >>> PoolingParams(),
715
+ >>> example_input["request_id"])
716
+ >>>
717
+ >>> # get the results
718
+ >>> final_output = None
719
+ >>> async for request_output in results_generator:
720
+ >>> if await request.is_disconnected():
721
+ >>> # Abort the request if the client disconnects.
722
+ >>> await engine.abort(request_id)
723
+ >>> # Return or raise an error
724
+ >>> ...
725
+ >>> final_output = request_output
726
+ >>>
727
+ >>> # Process and return the final output
728
+ >>> ...
729
+ """
730
+ async for output in self.process_request(
731
+ request_id,
732
+ prompt,
733
+ pooling_params,
734
+ prompt_token_ids,
735
+ lora_request,
736
+ multi_modal_data,
737
+ ):
738
+ yield output
739
+
740
+ async def process_request(
741
+ self,
742
+ request_id: str,
743
+ prompt: Optional[str],
744
+ params: Union[SamplingParams, PoolingParams],
745
+ prompt_token_ids: Optional[List[int]] = None,
746
+ lora_request: Optional[LoRARequest] = None,
747
+ multi_modal_data: Optional[MultiModalData] = None,
748
+ ) -> AsyncIterator[Union[RequestOutput, EmbeddingRequestOutput]]:
749
+ """Common logic to process requests with SamplingParams or
750
+ PoolingParams."""
751
+ arrival_time = time.time()
752
+
753
+ stream = await self.add_request(
754
+ request_id,
755
+ prompt,
756
+ params,
757
+ prompt_token_ids=prompt_token_ids,
758
+ arrival_time=arrival_time,
759
+ lora_request=lora_request,
760
+ multi_modal_data=multi_modal_data,
761
+ )
762
763
+ try:
764
async for request_output in stream:
765
yield request_output
766
except (Exception, asyncio.CancelledError) as e:
663
- # If there is an exception or coroutine is cancelled, abort the
664
- # request.
767
self._abort(request_id)
768
raise e
769
vllm/engine/llm_engine.py
+115
-28
@@ -20,9 +20,12 @@ from vllm.executor.executor_base import ExecutorBase
20
from vllm.executor.ray_utils import initialize_ray_cluster
21
from vllm.logger import init_logger
22
from vllm.lora.request import LoRARequest
23
-from vllm.outputs import RequestOutput
23
+from vllm.outputs import (EmbeddingRequestOutput, RequestOutput,
24
+ RequestOutputFactory)
25
+from vllm.pooling_params import PoolingParams
26
from vllm.sampling_params import SamplingParams
25
-from vllm.sequence import (ExecuteModelRequest, MultiModalData, SamplerOutput,
27
+from vllm.sequence import (EmbeddingSequenceGroupOutput, ExecuteModelRequest,
28
+ MultiModalData, PoolerOutput, SamplerOutput,
29
Sequence, SequenceGroup, SequenceGroupMetadata,
30
SequenceStatus)
31
from vllm.transformers_utils.detokenizer import Detokenizer
@@ -169,7 +172,8 @@ class LLMEngine:
172
load_config=load_config,
173
)
174
172
- self._initialize_kv_caches()
175
+ if not self.model_config.embedding_mode:
176
+ self._initialize_kv_caches()
177
178
# If usage stat is enabled, collect relevant info.
179
if is_usage_stats_enabled():
@@ -354,7 +358,7 @@ class LLMEngine:
358
self,
359
request_id: str,
360
prompt: Optional[str],
357
- sampling_params: SamplingParams,
361
+ params: Union[SamplingParams, PoolingParams],
362
prompt_token_ids: Optional[List[int]] = None,
363
arrival_time: Optional[float] = None,
364
lora_request: Optional[LoRARequest] = None,
@@ -370,7 +374,8 @@ class LLMEngine:
374
request_id: The unique ID of the request.
375
prompt: The prompt string. Can be None if prompt_token_ids is
376
provided.
373
- sampling_params: The sampling parameters for text generation.
377
+ params: Parameters for sampling or pooling. SamplingParams
378
+ for text generation. PoolingParams for pooling.
379
prompt_token_ids: The token IDs of the prompt. If None, we
380
use the tokenizer to convert the prompts to token IDs.
381
arrival_time: The arrival time of the request. If None, we use
@@ -404,13 +409,6 @@ class LLMEngine:
409
if lora_request is not None and not self.lora_config:
410
raise ValueError(f"Got lora_request {lora_request} but LoRA is "
411
"not enabled!")
407
- max_logprobs = self.get_model_config().max_logprobs
408
- if (sampling_params.logprobs
409
- and sampling_params.logprobs > max_logprobs) or (
410
- sampling_params.prompt_logprobs
411
- and sampling_params.prompt_logprobs > max_logprobs):
412
- raise ValueError(f"Cannot request more than "
413
- f"{max_logprobs} logprobs.")
412
if arrival_time is None:
413
arrival_time = time.time()
414
prompt_token_ids = self.encode_request(
@@ -432,6 +430,50 @@ class LLMEngine:
430
seq = Sequence(seq_id, prompt, prompt_token_ids, block_size,
431
eos_token_id, lora_request)
432
433
+ # Create a SequenceGroup based on SamplingParams or PoolingParams
434
+ if isinstance(params, SamplingParams):
435
+ seq_group = self._create_sequence_group_with_sampling(
436
+ request_id,
437
+ seq,
438
+ params,
439
+ arrival_time,
440
+ lora_request,
441
+ multi_modal_data,
442
+ )
443
+ elif isinstance(params, PoolingParams):
444
+ seq_group = self._create_sequence_group_with_pooling(
445
+ request_id,
446
+ seq,
447
+ params,
448
+ arrival_time,
449
+ lora_request,
450
+ multi_modal_data,
451
+ )
452
+ else:
453
+ raise ValueError(
454
+ "Either SamplingParams or PoolingParams must be provided.")
455
+
456
+ # Add the sequence group to the scheduler.
457
+ self.scheduler.add_seq_group(seq_group)
458
+
459
+ def _create_sequence_group_with_sampling(
460
+ self,
461
+ request_id: str,
462
+ seq: Sequence,
463
+ sampling_params: SamplingParams,
464
+ arrival_time: Optional[float] = None,
465
+ lora_request: Optional[LoRARequest] = None,
466
+ multi_modal_data: Optional[MultiModalData] = None,
467
+ ) -> SequenceGroup:
468
+ """Creates a SequenceGroup with SamplingParams."""
469
+ max_logprobs = self.get_model_config().max_logprobs
470
+ if (sampling_params.logprobs
471
+ and sampling_params.logprobs > max_logprobs) or (
472
+ sampling_params.prompt_logprobs
473
+ and sampling_params.prompt_logprobs > max_logprobs):
474
+ raise ValueError(f"Cannot request more than "
475
+ f"{max_logprobs} logprobs.")
476
+
477
# Defensive copy of SamplingParams, which are used by the sampler,
478
# this doesn't deep-copy LogitsProcessor objects
479
sampling_params = sampling_params.clone()
@@ -443,11 +485,35 @@ class LLMEngine:
485
self.generation_config_fields)
486
487
# Create the sequence group.
446
- seq_group = SequenceGroup(request_id, [seq], sampling_params,
447
- arrival_time, lora_request, multi_modal_data)
488
+ seq_group = SequenceGroup(request_id=request_id,
489
+ seqs=[seq],
490
+ arrival_time=arrival_time,
491
+ sampling_params=sampling_params,
492
+ lora_request=lora_request,
493
+ multi_modal_data=multi_modal_data)
494
449
- # Add the sequence group to the scheduler.
450
- self.scheduler.add_seq_group(seq_group)
495
+ return seq_group
496
+
497
+ def _create_sequence_group_with_pooling(
498
+ self,
499
+ request_id: str,
500
+ seq: Sequence,
501
+ pooling_params: PoolingParams,
502
+ arrival_time: Optional[float] = None,
503
+ lora_request: Optional[LoRARequest] = None,
504
+ multi_modal_data: Optional[MultiModalData] = None,
505
+ ) -> SequenceGroup:
506
+ """Creates a SequenceGroup with PoolingParams."""
507
+ # Defensive copy of PoolingParams, which are used by the pooler
508
+ pooling_params = pooling_params.clone()
509
+ # Create the sequence group.
510
+ seq_group = SequenceGroup(request_id=request_id,
511
+ seqs=[seq],
512
+ arrival_time=arrival_time,
513
+ lora_request=lora_request,
514
+ multi_modal_data=multi_modal_data,
515
+ pooling_params=pooling_params)
516
+ return seq_group
517
518
def abort_request(self, request_id: Union[str, Iterable[str]]) -> None:
519
"""Aborts a request(s) with the given ID.
@@ -484,13 +550,25 @@ class LLMEngine:
550
"""Returns True if there are unfinished requests."""
551
return self.scheduler.has_unfinished_seqs()
552
553
+ def _process_sequence_group_outputs(
554
+ self,
555
+ seq_group: SequenceGroup,
556
+ outputs: List[EmbeddingSequenceGroupOutput],
557
+ ) -> None:
558
+ seq_group.embeddings = outputs[0].embeddings
559
+
560
+ for seq in seq_group.get_seqs():
561
+ seq.status = SequenceStatus.FINISHED_STOPPED
562
+
563
+ return
564
+
565
def _process_model_outputs(
566
self,
489
- output: List[SamplerOutput],
567
+ output: List[Union[SamplerOutput, PoolerOutput]],
568
scheduled_seq_groups: List[ScheduledSequenceGroup],
569
ignored_seq_groups: List[SequenceGroup],
570
seq_group_metadata_list: List[SequenceGroupMetadata],
493
- ) -> List[RequestOutput]:
571
+ ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
572
"""Apply the model output to the sequences in the scheduled seq groups.
573
574
Returns RequestOutputs that can be returned to the client.
@@ -510,6 +588,9 @@ class LLMEngine:
588
seq_group = scheduled_seq_group.seq_group
589
seq_group.update_num_computed_tokens(
590
scheduled_seq_group.token_chunk_size)
591
+ if self.model_config.embedding_mode:
592
+ self._process_sequence_group_outputs(seq_group, outputs)
593
+ continue
594
595
self.output_processor.process_prompt_logprob(seq_group, outputs)
596
if seq_group_meta.do_sample:
@@ -519,18 +600,19 @@ class LLMEngine:
600
self.scheduler.free_finished_seq_groups()
601
602
# Create the outputs.
522
- request_outputs: List[RequestOutput] = []
603
+ request_outputs: List[Union[RequestOutput,
604
+ EmbeddingRequestOutput]] = []
605
for scheduled_seq_group in scheduled_seq_groups:
606
seq_group = scheduled_seq_group.seq_group
607
seq_group.maybe_set_first_token_time(now)
526
- request_output = RequestOutput.from_seq_group(seq_group)
608
+ request_output = RequestOutputFactory.create(seq_group)
609
request_outputs.append(request_output)
610
for seq_group in ignored_seq_groups:
529
- request_output = RequestOutput.from_seq_group(seq_group)
611
+ request_output = RequestOutputFactory.create(seq_group)
612
request_outputs.append(request_output)
613
return request_outputs
614
533
- def step(self) -> List[RequestOutput]:
615
+ def step(self) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
616
"""Performs one decoding iteration and returns newly generated results.
617
618
.. figure:: https://i.imgur.com/sv2HssD.png
@@ -570,7 +652,7 @@ class LLMEngine:
652
>>> while True:
653
>>> if example_inputs:
654
>>> req_id, prompt, sampling_params = example_inputs.pop(0)
573
- >>> engine.add_request(str(req_id), prompt, sampling_params)
655
+ >>> engine.add_request(str(req_id),prompt,sampling_params)
656
>>>
657
>>> # continue the request processing
658
>>> request_outputs = engine.step()
@@ -637,12 +719,15 @@ class LLMEngine:
719
720
# KV Cache Usage in %
721
num_total_gpu = self.cache_config.num_gpu_blocks
640
- num_free_gpu = self.scheduler.block_manager.get_num_free_gpu_blocks()
641
- gpu_cache_usage_sys = 1.0 - (num_free_gpu / num_total_gpu)
722
+ gpu_cache_usage_sys = 0.
723
+ if num_total_gpu is not None:
724
+ num_free_gpu = self.scheduler.block_manager.get_num_free_gpu_blocks(
725
+ )
726
+ gpu_cache_usage_sys = 1.0 - (num_free_gpu / num_total_gpu)
727
728
num_total_cpu = self.cache_config.num_cpu_blocks
729
cpu_cache_usage_sys = 0.
645
- if num_total_cpu > 0:
730
+ if num_total_cpu is not None and num_total_cpu > 0:
731
num_free_cpu = self.scheduler.block_manager.get_num_free_cpu_blocks(
732
)
733
cpu_cache_usage_sys = 1.0 - (num_free_cpu / num_total_cpu)
@@ -716,8 +801,10 @@ class LLMEngine:
801
seq.get_output_len()
802
for seq in seq_group.get_finished_seqs()
803
])
719
- best_of_requests.append(seq_group.sampling_params.best_of)
720
- n_requests.append(seq_group.sampling_params.n)
804
+ if seq_group.sampling_params is not None:
805
+ best_of_requests.append(
806
+ seq_group.sampling_params.best_of)
807
+ n_requests.append(seq_group.sampling_params.n)
808
finished_reason_requests.extend([
809
SequenceStatus.get_finished_reason(seq.status)
810
for seq in seq_group.get_finished_seqs()
vllm/entrypoints/llm.py
+121
-29
@@ -6,13 +6,17 @@ from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
6
7
from vllm.engine.arg_utils import EngineArgs
8
from vllm.engine.llm_engine import LLMEngine
9
+from vllm.logger import init_logger
10
from vllm.lora.request import LoRARequest
10
-from vllm.outputs import RequestOutput
11
+from vllm.outputs import EmbeddingRequestOutput, RequestOutput
12
+from vllm.pooling_params import PoolingParams
13
from vllm.sampling_params import SamplingParams
14
from vllm.sequence import MultiModalData
15
from vllm.usage.usage_lib import UsageContext
16
from vllm.utils import Counter
17
18
+logger = init_logger(__name__)
19
+
20
21
class LLM:
22
"""An LLM for generating texts from given prompts and sampling parameters.
@@ -164,8 +168,89 @@ class LLM:
168
multi_modal_data: Multi modal data.
169
170
Returns:
167
- A list of `RequestOutput` objects containing the generated
168
- completions in the same order as the input prompts.
171
+ A list of `RequestOutput` objects containing the
172
+ generated completions in the same order as the input prompts.
173
+ """
174
+ if sampling_params is None:
175
+ # Use default sampling params.
176
+ sampling_params = SamplingParams()
177
+
178
+ requests_data = self._validate_and_prepare_requests(
179
+ prompts,
180
+ sampling_params,
181
+ prompt_token_ids,
182
+ lora_request,
183
+ multi_modal_data,
184
+ )
185
+
186
+ # Add requests to the engine and run the engine
187
+ for request_data in requests_data:
188
+ self._add_request(**request_data)
189
+
190
+ return self._run_engine(use_tqdm)
191
+
192
+ def encode(
193
+ self,
194
+ prompts: Optional[Union[str, List[str]]] = None,
195
+ pooling_params: Optional[Union[PoolingParams,
196
+ List[PoolingParams]]] = None,
197
+ prompt_token_ids: Optional[List[List[int]]] = None,
198
+ use_tqdm: bool = True,
199
+ lora_request: Optional[LoRARequest] = None,
200
+ multi_modal_data: Optional[MultiModalData] = None,
201
+ ) -> List[EmbeddingRequestOutput]:
202
+ """Generates the completions for the input prompts.
203
+
204
+ NOTE: This class automatically batches the given prompts, considering
205
+ the memory constraint. For the best performance, put all of your prompts
206
+ into a single list and pass it to this method.
207
+
208
+ Args:
209
+ prompts: A list of prompts to generate completions for.
210
+ pooling_params: The pooling parameters for pooling. If None, we
211
+ use the default pooling parameters.
212
+ prompt_token_ids: A list of token IDs for the prompts. If None, we
213
+ use the tokenizer to convert the prompts to token IDs.
214
+ use_tqdm: Whether to use tqdm to display the progress bar.
215
+ lora_request: LoRA request to use for generation, if any.
216
+ multi_modal_data: Multi modal data.
217
+
218
+ Returns:
219
+ A list of `EmbeddingRequestOutput` objects containing the
220
+ generated embeddings in the same order as the input prompts.
221
+ """
222
+ if pooling_params is None:
223
+ # Use default pooling params.
224
+ pooling_params = PoolingParams()
225
+
226
+ requests_data = self._validate_and_prepare_requests(
227
+ prompts,
228
+ pooling_params,
229
+ prompt_token_ids,
230
+ lora_request,
231
+ multi_modal_data,
232
+ )
233
+
234
+ # Add requests to the engine and run the engine
235
+ for request_data in requests_data:
236
+ self._add_request(**request_data)
237
+
238
+ return self._run_engine(use_tqdm)
239
+
240
+ def _validate_and_prepare_requests(
241
+ self,
242
+ prompts: Optional[Union[str, List[str]]],
243
+ params: Union[Union[SamplingParams, PoolingParams],
244
+ List[Union[SamplingParams,
245
+ PoolingParams]]], # Unified parameter
246
+ prompt_token_ids: Optional[List[List[int]]] = None,
247
+ lora_request: Optional[LoRARequest] = None,
248
+ multi_modal_data: Optional[MultiModalData] = None,
249
+ ) -> List[dict]:
250
+ """Validates and prepares request data for adding to the engine.
251
+
252
+ Ensures prompts and token IDs are consistent, and returns a list of
253
+ dictionaries with request data for further processing.
254
"""
255
if prompts is None and prompt_token_ids is None:
256
raise ValueError("Either prompts or prompt_token_ids must be "
@@ -188,40 +273,43 @@ class LLM:
273
assert prompt_token_ids is not None
274
num_requests = len(prompt_token_ids)
275
191
- if sampling_params is None:
192
- # Use default sampling params.
193
- sampling_params = SamplingParams()
194
-
195
- elif isinstance(sampling_params,
196
- list) and len(sampling_params) != num_requests:
197
- raise ValueError("The lengths of prompts and sampling_params "
276
+ if isinstance(params, list) and len(params) != num_requests:
277
+ raise ValueError("The lengths of prompts and params "
278
"must be the same.")
279
if multi_modal_data:
280
multi_modal_data.data = multi_modal_data.data.to(torch.float16)
281
282
# Add requests to the engine.
283
+ requests_data = []
284
for i in range(num_requests):
285
prompt = prompts[i] if prompts is not None else None
286
token_ids = None if prompt_token_ids is None else prompt_token_ids[
287
i]
207
- self._add_request(
288
+
289
+ multi_modal_item = MultiModalData(
290
+ type=multi_modal_data.type,
291
+ data=multi_modal_data.data[i].unsqueeze(0),
292
+ ) if multi_modal_data else None
293
+
294
+ requests_data.append({
295
+ "prompt":
296
prompt,
209
- sampling_params[i]
210
- if isinstance(sampling_params, list) else sampling_params,
297
+ "params":
298
+ params[i] if isinstance(params, list) else params,
299
+ "prompt_token_ids":
300
token_ids,
212
- lora_request=lora_request,
213
- # Get ith image while maintaining the batch dim.
214
- multi_modal_data=MultiModalData(
215
- type=multi_modal_data.type,
216
- data=multi_modal_data.data[i].unsqueeze(0))
217
- if multi_modal_data else None,
218
- )
219
- return self._run_engine(use_tqdm)
301
+ "lora_request":
302
+ lora_request,
303
+ "multi_modal_data":
304
+ multi_modal_item,
305
+ })
306
+
307
+ return requests_data
308
309
def _add_request(
310
self,
311
prompt: Optional[str],
224
- sampling_params: SamplingParams,
312
+ params: Union[SamplingParams, PoolingParams],
313
prompt_token_ids: Optional[List[int]],
314
lora_request: Optional[LoRARequest] = None,
315
multi_modal_data: Optional[MultiModalData] = None,
@@ -229,12 +317,14 @@ class LLM:
317
request_id = str(next(self.request_counter))
318
self.llm_engine.add_request(request_id,
319
prompt,
232
- sampling_params,
320
+ params,
321
prompt_token_ids,
322
lora_request=lora_request,
323
multi_modal_data=multi_modal_data)
324
237
- def _run_engine(self, use_tqdm: bool) -> List[RequestOutput]:
325
+ def _run_engine(
326
+ self, use_tqdm: bool
327
+ ) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
328
# Initialize tqdm.
329
if use_tqdm:
330
num_requests = self.llm_engine.get_num_unfinished_requests()
@@ -245,7 +335,7 @@ class LLM:
335
postfix=f"Generation Speed: {0:.2f} toks/s",
336
)
337
# Run the engine.
248
- outputs: List[RequestOutput] = []
338
+ outputs: List[Union[RequestOutput, EmbeddingRequestOutput]] = []
339
total_toks = 0
340
while self.llm_engine.has_unfinished_requests():
341
step_outputs = self.llm_engine.step()
@@ -253,10 +343,12 @@ class LLM:
343
if output.finished:
344
outputs.append(output)
345
if use_tqdm:
256
- total_toks += (sum(
257
- len(stp.token_ids) for stp in output.outputs))
258
- spd = total_toks / pbar.format_dict["elapsed"]
259
- pbar.postfix = f"Generation Speed: {spd:.2f} toks/s"
346
+ if isinstance(output, RequestOutput):
347
+ # Calculate tokens only for RequestOutput
348
+ total_toks += sum(
349
+ len(stp.token_ids) for stp in output.outputs)
350
+ spd = total_toks / pbar.format_dict["elapsed"]
351
+ pbar.postfix = f"Generation Speed: {spd:.2f} toks/s"
352
pbar.update(1)
353
if use_tqdm:
354
pbar.close()
vllm/entrypoints/openai/api_server.py
+18
-2
@@ -22,9 +22,11 @@ from vllm.engine.async_llm_engine import AsyncLLMEngine
22
from vllm.entrypoints.openai.cli_args import make_arg_parser
23
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
24
ChatCompletionResponse,
25
- CompletionRequest, ErrorResponse)
25
+ CompletionRequest,
26
+ EmbeddingRequest, ErrorResponse)
27
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
28
from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
29
+from vllm.entrypoints.openai.serving_embedding import OpenAIServingEmbedding
30
from vllm.logger import init_logger
31
from vllm.usage.usage_lib import UsageContext
32
@@ -32,6 +34,8 @@ TIMEOUT_KEEP_ALIVE = 5 # seconds
34
35
openai_serving_chat: OpenAIServingChat
36
openai_serving_completion: OpenAIServingCompletion
37
+openai_serving_embedding: OpenAIServingEmbedding
38
+
39
logger = init_logger(__name__)
40
41
_running_tasks: Set[asyncio.Task] = set()
@@ -123,6 +127,17 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
127
return JSONResponse(content=generator.model_dump())
128
129
130
+@app.post("/v1/embeddings")
131
+async def create_embedding(request: EmbeddingRequest, raw_request: Request):
132
+ generator = await openai_serving_embedding.create_embedding(
133
+ request, raw_request)
134
+ if isinstance(generator, ErrorResponse):
135
+ return JSONResponse(content=generator.model_dump(),
136
+ status_code=generator.code)
137
+ else:
138
+ return JSONResponse(content=generator.model_dump())
139
+
140
+
141
if __name__ == "__main__":
142
args = parse_args()
143
@@ -190,7 +205,8 @@ if __name__ == "__main__":
205
args.chat_template)
206
openai_serving_completion = OpenAIServingCompletion(
207
engine, model_config, served_model_names, args.lora_modules)
193
-
208
+ openai_serving_embedding = OpenAIServingEmbedding(engine, model_config,
209
+ served_model_names)
210
app.root_path = args.root_path
211
uvicorn.run(app,
212
host=args.host,
vllm/entrypoints/openai/protocol.py
+35
-1
@@ -1,13 +1,14 @@
1
# Adapted from
2
# https://github.com/lm-sys/FastChat/blob/168ccc29d3f7edc50823016105c024fe2282732a/fastchat/protocol/openai_api_protocol.py
3
import time
4
-from typing import Dict, List, Literal, Optional, Union
4
+from typing import Any, Dict, List, Literal, Optional, Union
5
6
import torch
7
from openai.types.chat import ChatCompletionMessageParam
8
from pydantic import BaseModel, ConfigDict, Field, model_validator
9
from typing_extensions import Annotated
10
11
+from vllm.pooling_params import PoolingParams
12
from vllm.sampling_params import SamplingParams
13
from vllm.utils import random_uuid
14
@@ -363,6 +364,24 @@ class CompletionRequest(OpenAIBaseModel):
364
return data
365
366
367
+class EmbeddingRequest(BaseModel):
368
+ # Ordered by official OpenAI API documentation
369
+ # https://platform.openai.com/docs/api-reference/embeddings
370
+ model: str
371
+ input: Union[List[int], List[List[int]], str, List[str]]
372
+ encoding_format: Optional[str] = Field('float', pattern='^(float|base64)$')
373
+ dimensions: Optional[int] = None
374
+ user: Optional[str] = None
375
+
376
+ # doc: begin-embedding-pooling-params
377
+ additional_data: Optional[Any] = None
378
+
379
+ # doc: end-embedding-pooling-params
380
+
381
+ def to_pooling_params(self):
382
+ return PoolingParams(additional_data=self.additional_data)
383
+
384
+
385
class LogProbs(OpenAIBaseModel):
386
text_offset: List[int] = Field(default_factory=list)
387
token_logprobs: List[Optional[float]] = Field(default_factory=list)
@@ -416,6 +435,21 @@ class CompletionStreamResponse(OpenAIBaseModel):
435
usage: Optional[UsageInfo] = Field(default=None)
436
437
438
+class EmbeddingResponseData(BaseModel):
439
+ index: int
440
+ object: str = "embedding"
441
+ embedding: List[float]
442
+
443
+
444
+class EmbeddingResponse(BaseModel):
445
+ id: str = Field(default_factory=lambda: f"cmpl-{random_uuid()}")
446
+ object: str = "list"
447
+ created: int = Field(default_factory=lambda: int(time.time()))
448
+ model: str
449
+ data: List[EmbeddingResponseData]
450
+ usage: UsageInfo
451
+
452
+
453
class ChatMessage(OpenAIBaseModel):
454
role: str
455
content: str
vllm/entrypoints/openai/serving_embedding.py
new
+134
@@ -0,0 +1,134 @@
1
+import time
2
+from typing import AsyncIterator, List, Tuple
3
+
4
+from fastapi import Request
5
+
6
+from vllm.config import ModelConfig
7
+from vllm.engine.async_llm_engine import AsyncLLMEngine
8
+from vllm.entrypoints.openai.protocol import (EmbeddingRequest,
9
+ EmbeddingResponse,
10
+ EmbeddingResponseData, UsageInfo)
11
+from vllm.entrypoints.openai.serving_completion import parse_prompt_format
12
+from vllm.entrypoints.openai.serving_engine import OpenAIServing
13
+from vllm.logger import init_logger
14
+from vllm.outputs import EmbeddingRequestOutput
15
+from vllm.utils import merge_async_iterators, random_uuid
16
+
17
+logger = init_logger(__name__)
18
+
19
+TypeTokenIDs = List[int]
20
+
21
+
22
+def request_output_to_embedding_response(
23
+ final_res_batch: List[EmbeddingRequestOutput],
24
+ request_id: str,
25
+ created_time: int,
26
+ model_name: str,
27
+) -> EmbeddingResponse:
28
+ data = []
29
+ num_prompt_tokens = 0
30
+ for idx, final_res in enumerate(final_res_batch):
31
+ assert final_res is not None
32
+ prompt_token_ids = final_res.prompt_token_ids
33
+
34
+ embedding_data = EmbeddingResponseData(
35
+ index=idx, embedding=final_res.outputs.embedding)
36
+ data.append(embedding_data)
37
+
38
+ num_prompt_tokens += len(prompt_token_ids)
39
+
40
+ usage = UsageInfo(
41
+ prompt_tokens=num_prompt_tokens,
42
+ total_tokens=num_prompt_tokens,
43
+ )
44
+
45
+ return EmbeddingResponse(
46
+ id=request_id,
47
+ created=created_time,
48
+ model=model_name,
49
+ data=data,
50
+ usage=usage,
51
+ )
52
+
53
+
54
+class OpenAIServingEmbedding(OpenAIServing):
55
+
56
+ def __init__(self, engine: AsyncLLMEngine, model_config: ModelConfig,
57
+ served_model_names: List[str]):
58
+ super().__init__(engine=engine,
59
+ model_config=model_config,
60
+ served_model_names=served_model_names,
61
+ lora_modules=None)
62
+ self._check_embedding_mode(model_config.embedding_mode)
63
+
64
+ async def create_embedding(self, request: EmbeddingRequest,
65
+ raw_request: Request):
66
+ """Completion API similar to OpenAI's API.
67
+
68
+ See https://platform.openai.com/docs/api-reference/embeddings/create
69
+ for the API specification. This API mimics the OpenAI Embedding API.
70
+ """
71
+ error_check_ret = await self._check_model(request)
72
+ if error_check_ret is not None:
73
+ return error_check_ret
74
+
75
+ # Return error for unsupported features.
76
+ if request.encoding_format == "base64":
77
+ return self.create_error_response(
78
+ "base64 encoding is not currently supported")
79
+ if request.dimensions is not None:
80
+ return self.create_error_response(
81
+ "dimensions is currently not supported")
82
+
83
+ model_name = request.model
84
+ request_id = f"cmpl-{random_uuid()}"
85
+ created_time = int(time.monotonic())
86
+
87
+ # Schedule the request and get the result generator.
88
+ generators = []
89
+ try:
90
+ prompt_is_tokens, prompts = parse_prompt_format(request.input)
91
+ pooling_params = request.to_pooling_params()
92
+
93
+ for i, prompt in enumerate(prompts):
94
+ if prompt_is_tokens:
95
+ prompt_formats = self._validate_prompt_and_tokenize(
96
+ request, prompt_ids=prompt)
97
+ else:
98
+ prompt_formats = self._validate_prompt_and_tokenize(
99
+ request, prompt=prompt)
100
+
101
+ prompt_ids, prompt_text = prompt_formats
102
+
103
+ generators.append(
104
+ self.engine.generate(prompt_text,
105
+ pooling_params,
106
+ f"{request_id}-{i}",
107
+ prompt_token_ids=prompt_ids))
108
+ except ValueError as e:
109
+ # TODO: Use a vllm-specific Validation Error
110
+ return self.create_error_response(str(e))
111
+
112
+ result_generator: AsyncIterator[Tuple[
113
+ int, EmbeddingRequestOutput]] = merge_async_iterators(*generators)
114
+
115
+ # Non-streaming response
116
+ final_res_batch: EmbeddingRequestOutput = [None] * len(prompts)
117
+ async for i, res in result_generator:
118
+ if await raw_request.is_disconnected():
119
+ # Abort the request if the client disconnects.
120
+ await self.engine.abort(f"{request_id}-{i}")
121
+ # TODO: Use a vllm-specific Validation Error
122
+ return self.create_error_response("Client disconnected")
123
+ final_res_batch[i] = res
124
+ response = request_output_to_embedding_response(
125
+ final_res_batch, request_id, created_time, model_name)
126
+
127
+ return response
128
+
129
+ def _check_embedding_mode(self, embedding_mode: bool):
130
+ if not embedding_mode:
131
+ logger.warning(
132
+ "embedding_mode is False. Embedding API will not work.")
133
+ else:
134
+ logger.info("Activating the server engine with embedding enabled.")
vllm/entrypoints/openai/serving_engine.py
+14
-2
@@ -9,7 +9,8 @@ from typing_extensions import Annotated
9
from vllm.config import ModelConfig
10
from vllm.engine.async_llm_engine import AsyncLLMEngine
11
from vllm.entrypoints.openai.protocol import (ChatCompletionRequest,
12
- CompletionRequest, ErrorResponse,
12
+ CompletionRequest,
13
+ EmbeddingRequest, ErrorResponse,
14
LogProbs, ModelCard, ModelList,
15
ModelPermission)
16
from vllm.logger import init_logger
@@ -165,7 +166,8 @@ class OpenAIServing:
166
167
def _validate_prompt_and_tokenize(
168
self,
168
- request: Union[ChatCompletionRequest, CompletionRequest],
169
+ request: Union[ChatCompletionRequest, CompletionRequest,
170
+ EmbeddingRequest],
171
prompt: Optional[str] = None,
172
prompt_ids: Optional[List[int]] = None,
173
truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
@@ -191,6 +193,16 @@ class OpenAIServing:
193
prompt_ids)
194
token_num = len(input_ids)
195
196
+ # Note: EmbeddingRequest doesn't have max_tokens
197
+ if isinstance(request, EmbeddingRequest):
198
+ if token_num > self.max_model_len:
199
+ raise ValueError(
200
+ f"This model's maximum context length is "
201
+ f"{self.max_model_len} tokens. However, you requested "
202
+ f"{token_num} tokens in the input for embedding "
203
+ f"generation. Please reduce the length of the input.", )
204
+ return input_ids, input_text
205
+
206
if request.max_tokens is None:
207
if token_num >= self.max_model_len:
208
raise ValueError(
vllm/executor/gpu_executor.py
+5
-5
@@ -1,9 +1,9 @@
1
-from typing import Any, Dict, List, Optional, Set, Tuple
1
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
2
3
from vllm.executor.executor_base import ExecutorAsyncBase, ExecutorBase
4
from vllm.logger import init_logger
5
from vllm.lora.request import LoRARequest
6
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
6
+from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
7
from vllm.utils import (get_distributed_init_method, get_ip, get_open_port,
8
make_async)
9
from vllm.worker.worker_base import WorkerWrapperBase
@@ -123,8 +123,8 @@ class GPUExecutor(ExecutorBase):
123
self.driver_worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
124
125
def execute_model(
126
- self,
127
- execute_model_req: ExecuteModelRequest) -> List[SamplerOutput]:
126
+ self, execute_model_req: ExecuteModelRequest
127
+ ) -> List[Union[SamplerOutput, PoolerOutput]]:
128
output = self.driver_worker.execute_model(execute_model_req)
129
return output
130
@@ -150,7 +150,7 @@ class GPUExecutorAsync(GPUExecutor, ExecutorAsyncBase):
150
async def execute_model_async(
151
self,
152
execute_model_req: ExecuteModelRequest,
153
- ) -> List[SamplerOutput]:
153
+ ) -> List[Union[SamplerOutput, PoolerOutput]]:
154
output = await make_async(self.driver_worker.execute_model
155
)(execute_model_req=execute_model_req, )
156
return output
vllm/model_executor/layers/pooler.py
new
+56
@@ -0,0 +1,56 @@
1
+from enum import IntEnum
2
+
3
+import torch
4
+import torch.nn as nn
5
+
6
+from vllm.model_executor.pooling_metadata import (PoolingMetadata,
7
+ PoolingTensors)
8
+from vllm.sequence import EmbeddingSequenceGroupOutput, PoolerOutput
9
+
10
+
11
+class PoolingType(IntEnum):
12
+ """Enumeration for different types of pooling methods."""
13
+ LAST = 0
14
+
15
+
16
+class Pooler(nn.Module):
17
+ """A layer that pools specific information from hidden states.
18
+
19
+ This layer does the following:
20
+ 1. Extracts specific tokens or aggregates data based on pooling method.
21
+ 2. Normalizes output if specified.
22
+ 3. Returns structured results as `PoolerOutput`.
23
+
24
+ Attributes:
25
+ pooling_type: The type of pooling to use (LAST, AVERAGE, MAX).
26
+ normalize: Whether to normalize the pooled data.
27
+ """
28
+
29
+ def __init__(self, pooling_type: PoolingType, normalize: bool):
30
+ super().__init__()
31
+ self.pooling_type = pooling_type
32
+ self.normalize = normalize
33
+
34
+ def forward(
35
+ self,
36
+ hidden_states: torch.Tensor,
37
+ pooling_metadata: PoolingMetadata,
38
+ ) -> PoolerOutput:
39
+ """Pools specific information from hidden states based on metadata."""
40
+ prompt_lens = PoolingTensors.from_pooling_metadata(
41
+ pooling_metadata, hidden_states.device).prompt_lens
42
+
43
+ if self.pooling_type == PoolingType.LAST:
44
+ last_token_flat_indices = torch.cumsum(prompt_lens, dim=0) - 1
45
+ pooled_data = hidden_states[last_token_flat_indices]
46
+ else:
47
+ raise ValueError(f"Invalid pooling type: {self.pooling_type}")
48
+
49
+ if self.normalize:
50
+ pooled_data = nn.functional.normalize(pooled_data, p=2, dim=1)
51
+
52
+ pooled_outputs = [
53
+ EmbeddingSequenceGroupOutput(data.tolist()) for data in pooled_data
54
+ ]
55
+
56
+ return PoolerOutput(outputs=pooled_outputs)
vllm/model_executor/layers/sampler.py
+4
-3
@@ -10,8 +10,9 @@ from vllm.model_executor.sampling_metadata import (SamplingMetadata,
10
SamplingTensors,
11
SequenceGroupToSample)
12
from vllm.sampling_params import SamplingType
13
-from vllm.sequence import (Logprob, PromptLogprobs, SampleLogprobs,
14
- SamplerOutput, SequenceGroupOutput, SequenceOutput)
13
+from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
14
+ PromptLogprobs, SampleLogprobs, SamplerOutput,
15
+ SequenceOutput)
16
17
# (num_token_ids, num_parent_ids) per sequence group.
18
SampleResultType = List[Tuple[List[int], List[int]]]
@@ -1019,7 +1020,7 @@ def _build_sampler_output(
1020
seq_outputs.append(
1021
SequenceOutput(seq_ids[parent_id], next_token_id, logprobs))
1022
sampler_output.append(
1022
- SequenceGroupOutput(seq_outputs, group_prompt_logprobs))
1023
+ CompletionSequenceGroupOutput(seq_outputs, group_prompt_logprobs))
1024
1025
# If not specified, store None values in SamplerOutput.
1026
if on_device_tensors is not None:
vllm/model_executor/models/__init__.py
+11
-1
@@ -9,7 +9,7 @@ from vllm.utils import is_hip
9
logger = init_logger(__name__)
10
11
# Architecture -> (module, class).
12
-_MODELS = {
12
+_GENERATION_MODELS = {
13
"AquilaModel": ("llama", "LlamaForCausalLM"),
14
"AquilaForCausalLM": ("llama", "LlamaForCausalLM"), # AquilaChat2
15
"BaiChuanForCausalLM": ("baichuan", "BaiChuanForCausalLM"), # baichuan-7b
@@ -58,6 +58,12 @@ _MODELS = {
58
"XverseForCausalLM": ("xverse", "XverseForCausalLM"),
59
}
60
61
+_EMBEDDING_MODELS = {
62
+ "MistralModel": ("llama_embedding", "LlamaEmbeddingModel"),
63
+}
64
+
65
+_MODELS = {**_GENERATION_MODELS, **_EMBEDDING_MODELS}
66
+
67
# Architecture -> type.
68
# out of tree models
69
_OOT_MODELS: Dict[str, Type[nn.Module]] = {}
@@ -114,6 +120,10 @@ class ModelRegistry:
120
global _OOT_MODELS
121
_OOT_MODELS[model_arch] = model_cls
122
123
+ @staticmethod
124
+ def is_embedding_model(model_arch: str) -> bool:
125
+ return model_arch in _EMBEDDING_MODELS
126
+
127
128
__all__ = [
129
"ModelRegistry",
vllm/model_executor/models/llama_embedding.py
new
+87
@@ -0,0 +1,87 @@
1
+from typing import Iterable, List, Optional, Tuple
2
+
3
+import torch
4
+from torch import nn
5
+
6
+from vllm.attention import AttentionMetadata
7
+from vllm.model_executor.layers.pooler import Pooler, PoolingType
8
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
9
+from vllm.model_executor.models.llama import LlamaModel
10
+from vllm.model_executor.pooling_metadata import PoolingMetadata
11
+from vllm.sequence import PoolerOutput
12
+
13
+
14
+class LlamaEmbeddingModel(nn.Module):
15
+ """A model that uses Llama with additional embedding functionalities.
16
+
17
+ This class encapsulates the LlamaModel and provides an interface for
18
+ embedding operations and customized pooling functions.
19
+
20
+ Attributes:
21
+ model: An instance of LlamaModel used for forward operations.
22
+ _pooler: An instance of Pooler used for pooling operations.
23
+ """
24
+
25
+ def __init__(
26
+ self,
27
+ **kwargs,
28
+ ) -> None:
29
+ super().__init__()
30
+ self.model = LlamaModel(**kwargs)
31
+ self._pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
32
+
33
+ def forward(
34
+ self,
35
+ input_ids: Optional[torch.Tensor],
36
+ positions: torch.Tensor,
37
+ kv_caches: List[torch.Tensor],
38
+ attn_metadata: AttentionMetadata,
39
+ inputs_embeds: Optional[torch.Tensor] = None,
40
+ ) -> torch.Tensor:
41
+ return self.model.forward(input_ids, positions, kv_caches,
42
+ attn_metadata, inputs_embeds)
43
+
44
+ def pooler(
45
+ self,
46
+ hidden_states: torch.Tensor,
47
+ pooling_metadata: PoolingMetadata,
48
+ ) -> Optional[PoolerOutput]:
49
+ return self._pooler(hidden_states, pooling_metadata)
50
+
51
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
52
+ stacked_params_mapping = [
53
+ # (param_name, shard_name, shard_id)
54
+ ("qkv_proj", "q_proj", "q"),
55
+ ("qkv_proj", "k_proj", "k"),
56
+ ("qkv_proj", "v_proj", "v"),
57
+ ("gate_up_proj", "gate_proj", 0),
58
+ ("gate_up_proj", "up_proj", 1),
59
+ ]
60
+ params_dict = dict(self.model.named_parameters())
61
+ for name, loaded_weight in weights:
62
+ if "rotary_emb.inv_freq" in name:
63
+ continue
64
+ if ("rotary_emb.cos_cached" in name
65
+ or "rotary_emb.sin_cached" in name):
66
+ # Models trained using ColossalAI may include these tensors in
67
+ # the checkpoint. Skip them.
68
+ continue
69
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
70
+ if weight_name not in name:
71
+ continue
72
+ name = name.replace(weight_name, param_name)
73
+ # Skip loading extra bias for GPTQ models.
74
+ if name.endswith(".bias") and name not in params_dict:
75
+ continue
76
+ param = params_dict[name]
77
+ weight_loader = param.weight_loader
78
+ weight_loader(param, loaded_weight, shard_id)
79
+ break
80
+ else:
81
+ # Skip loading extra bias for GPTQ models.
82
+ if name.endswith(".bias") and name not in params_dict:
83
+ continue
84
+ param = params_dict[name]
85
+ weight_loader = getattr(param, "weight_loader",
86
+ default_weight_loader)
87
+ weight_loader(param, loaded_weight)
vllm/model_executor/pooling_metadata.py
new
+69
@@ -0,0 +1,69 @@
1
+from dataclasses import dataclass
2
+from typing import Any, Dict, List, Tuple
3
+
4
+import torch
5
+
6
+from vllm.pooling_params import PoolingParams
7
+from vllm.utils import is_pin_memory_available
8
+
9
+
10
+class PoolingMetadata:
11
+ """Metadata for pooling operations in the Pooler layer.
12
+
13
+ This class holds the necessary information for pooling operations,
14
+ providing context for how to perform pooling and other related operations.
15
+
16
+ Attributes:
17
+ seq_groups: List of (seq_ids, pooling_params).
18
+ seq_data: A mapping of sequence ID to additional sequence data.
19
+ prompt_lens: List of the lengths of each prompt.
20
+ """
21
+
22
+ def __init__(
23
+ self,
24
+ seq_groups: List[Tuple[List[int], PoolingParams]],
25
+ seq_data: Dict[int, Any], # Specific data related to sequences
26
+ prompt_lens: List[int],
27
+ ) -> None:
28
+ self.seq_groups = seq_groups
29
+ self.seq_data = seq_data
30
+ self.prompt_lens = prompt_lens
31
+
32
+ def __repr__(self) -> str:
33
+ return ("PoolingMetadata("
34
+ f"seq_groups={self.seq_groups}, "
35
+ f"seq_data={self.seq_data}, "
36
+ f"prompt_lens={self.prompt_lens})")
37
+
38
+
39
+@dataclass
40
+class PoolingTensors:
41
+ """Tensors for pooling."""
42
+
43
+ prompt_lens: torch.Tensor
44
+
45
+ @classmethod
46
+ def from_pooling_metadata(
47
+ cls,
48
+ pooling_metadata: "PoolingMetadata",
49
+ device: torch.device,
50
+ ) -> "PoolingTensors":
51
+ """
52
+ Create PoolingTensors from PoolingMetadata.
53
+
54
+ Args:
55
+ pooling_metadata: PoolingMetadata instance to convert.
56
+ device: Device to store the tensors.
57
+ """
58
+ # Convert prompt lengths to tensor
59
+ pin_memory = is_pin_memory_available()
60
+
61
+ prompt_lens_t = torch.tensor(
62
+ pooling_metadata.prompt_lens,
63
+ device="cpu",
64
+ dtype=torch.long,
65
+ pin_memory=pin_memory,
66
+ )
67
+
68
+ return cls(prompt_lens=prompt_lens_t.to(device=device,
69
+ non_blocking=True), )
vllm/outputs.py
+81
-1
@@ -57,8 +57,27 @@ class CompletionOutput:
57
f"stop_reason={self.stop_reason})")
58
59
60
+class EmbeddingOutput:
61
+ """The output data of one completion output of a request.
62
+
63
+ Args:
64
+ embedding: The embedding vector, which is a list of floats. The
65
+ length of vector depends on the model as listed in the embedding guide.
66
+ """
67
+
68
+ def __init__(
69
+ self,
70
+ embedding: List[float],
71
+ ) -> None:
72
+ self.embedding = embedding
73
+
74
+ def __repr__(self) -> str:
75
+ return (f"EmbeddingOutput("
76
+ f"embedding={len(self.embedding)}")
77
+
78
+
79
class RequestOutput:
61
- """The output data of a request to the LLM.
80
+ """The output data of a completion request to the LLM.
81
82
Args:
83
request_id: The unique ID of the request.
@@ -93,6 +112,9 @@ class RequestOutput:
112
113
@classmethod
114
def from_seq_group(cls, seq_group: SequenceGroup) -> "RequestOutput":
115
+ if seq_group.sampling_params is None:
116
+ raise ValueError(
117
+ "Sampling parameters are missing for a CompletionRequest.")
118
seqs = seq_group.get_seqs()
119
if len(seqs) == 1:
120
top_n_seqs = seqs
@@ -148,3 +170,61 @@ class RequestOutput:
170
f"finished={self.finished}, "
171
f"metrics={self.metrics}, "
172
f"lora_request={self.lora_request})")
173
+
174
+
175
+class EmbeddingRequestOutput:
176
+ """
177
+ The output data of an embedding request to the LLM.
178
+
179
+ Args:
180
+ request_id (str): A unique identifier for the embedding request.
181
+ outputs (EmbeddingOutput): The embedding results for the given input.
182
+ prompt_token_ids (List[int]): A list of token IDs used in the prompt.
183
+ finished (bool): A flag indicating whether the embedding is completed.
184
+ """
185
+
186
+ def __init__(self, request_id: str, outputs: 'EmbeddingOutput',
187
+ prompt_token_ids: List[int], finished: bool):
188
+ self.request_id = request_id
189
+ self.prompt_token_ids = prompt_token_ids
190
+ self.finished = finished
191
+ self.outputs = outputs
192
+
193
+ @classmethod
194
+ def from_seq_group(cls,
195
+ seq_group: 'SequenceGroup') -> "EmbeddingRequestOutput":
196
+ if seq_group.embeddings is None:
197
+ raise ValueError(
198
+ "Embeddings are missing in seq_group for EmbeddingRequest.")
199
+ output = EmbeddingOutput(seq_group.embeddings)
200
+ prompt_token_ids = seq_group.prompt_token_ids
201
+ finished = seq_group.is_finished()
202
+
203
+ return cls(seq_group.request_id, output, prompt_token_ids, finished)
204
+
205
+ def __repr__(self):
206
+ """
207
+ Returns a string representation of an EmbeddingRequestOutput instance.
208
+
209
+ The representation includes the request_id and the number of outputs,
210
+ providing a quick overview of the embedding request's results.
211
+
212
+ Returns:
213
+ str: A string representation of the EmbeddingRequestOutput instance.
214
+ """
215
+ return (f"EmbeddingRequestOutput(request_id='{self.request_id}', "
216
+ f"outputs={repr(self.outputs)}, "
217
+ f"prompt_token_ids={self.prompt_token_ids}, "
218
+ f"finished={self.finished})")
219
+
220
+
221
+class RequestOutputFactory:
222
+
223
+ @staticmethod
224
+ def create(seq_group):
225
+ # Determine the type based on a condition, for example:
226
+ if hasattr(seq_group,
227
+ 'embeddings') and seq_group.embeddings is not None:
228
+ return EmbeddingRequestOutput.from_seq_group(seq_group)
229
+ else:
230
+ return RequestOutput.from_seq_group(seq_group)
vllm/pooling_params.py
new
+20
@@ -0,0 +1,20 @@
1
+from typing import Any, Optional
2
+
3
+
4
+class PoolingParams:
5
+ """Pooling parameters for pooling.
6
+
7
+ Attributes:
8
+ additional_data: Any additional data needed for pooling.
9
+ """
10
+
11
+ def __init__(self, additional_data: Optional[Any] = None):
12
+ self.additional_data = additional_data
13
+
14
+ def clone(self) -> "PoolingParams":
15
+ """Returns a deep copy of the PoolingParams instance."""
16
+ return PoolingParams(additional_data=self.additional_data, )
17
+
18
+ def __repr__(self) -> str:
19
+ return (f"PoolingParams("
20
+ f"additional_metadata={self.additional_data})")
vllm/sequence.py
+77
-12
@@ -1,11 +1,13 @@
1
"""Sequence and its related classes."""
2
import copy
3
import enum
4
+from abc import ABC, abstractmethod
5
from dataclasses import dataclass, field
6
from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
7
8
from vllm.block import LogicalTokenBlock
9
from vllm.lora.request import LoRARequest
10
+from vllm.pooling_params import PoolingParams
11
from vllm.sampling_params import SamplingParams
12
13
if TYPE_CHECKING:
@@ -375,12 +377,12 @@ class SequenceGroupState:
377
378
class MultiModalData:
379
"""Multi modal request.
378
-
380
+
381
Args:
382
type: The data type.
383
data: The actual data.
384
The required shape and semantic meaning of it depends on the vision
383
- language config of the hosted model.
385
+ language config of the hosted model.
386
See `VisionLanguageConfig` in `config.py`.
387
"""
388
@@ -402,16 +404,22 @@ class SequenceGroup:
404
arrival_time: The arrival time of the request.
405
lora_request: LoRA request.
406
multi_modal_data: Multi modal data associated with the request.
407
+ embeddings: The embeddings vectors of the prompt of the sequence group
408
+ for an embedding model.
409
+ pooling_params: The pooling parameters used to generate the pooling
410
+ for an embedding model.
411
"""
412
413
def __init__(
414
self,
415
request_id: str,
416
seqs: List[Sequence],
411
- sampling_params: SamplingParams,
417
arrival_time: float,
418
+ sampling_params: Optional[SamplingParams] = None,
419
lora_request: Optional[LoRARequest] = None,
420
multi_modal_data: Optional[MultiModalData] = None,
421
+ embeddings: Optional[List[float]] = None,
422
+ pooling_params: Optional[PoolingParams] = None,
423
) -> None:
424
self.request_id = request_id
425
self.seqs_dict = {seq.seq_id: seq for seq in seqs}
@@ -425,6 +433,8 @@ class SequenceGroup:
433
self.prompt_logprobs: Optional[PromptLogprobs] = None
434
self.state = SequenceGroupState()
435
self.multi_modal_data = multi_modal_data
436
+ self.embeddings = embeddings
437
+ self.pooling_params = pooling_params
438
439
@property
440
def prompt(self) -> str:
@@ -479,12 +489,13 @@ class SequenceGroup:
489
def get_max_num_running_seqs(self) -> int:
490
"""The maximum number of sequences running in parallel in the remaining
491
lifetime of the request."""
482
- if self.sampling_params.use_beam_search:
492
+ if self.sampling_params and self.sampling_params.use_beam_search:
493
# For beam search, maximally there will always be `best_of` beam
494
# candidates running in the future.
495
return self.sampling_params.best_of
496
else:
487
- if self.sampling_params.best_of > self.num_seqs():
497
+ if (self.sampling_params
498
+ and self.sampling_params.best_of > self.num_seqs()):
499
# At prompt stage, the sequence group is not yet filled up
500
# and only have one sequence running. However, in the
501
# generation stage, we will have `best_of` sequences running.
@@ -555,7 +566,7 @@ class SequenceGroup:
566
return all(seq.is_finished() for seq in self.get_seqs())
567
568
def is_prefill(self) -> bool:
558
- # Every sequences should be in the same stage.
569
+ # Every sequence should be in the same stage.
570
return self.get_seqs()[0].is_prefill()
571
572
def __repr__(self) -> str:
@@ -594,6 +605,7 @@ class SequenceGroupMetadata:
605
sampling_params: SamplingParams,
606
block_tables: Dict[int, List[int]],
607
do_sample: bool = True,
608
+ pooling_params: Optional[PoolingParams] = None,
609
token_chunk_size: Optional[int] = None,
610
lora_request: Optional[LoRARequest] = None,
611
computed_block_nums: Optional[List[int]] = None,
@@ -605,6 +617,7 @@ class SequenceGroupMetadata:
617
self.seq_data = seq_data
618
self.sampling_params = sampling_params
619
self.block_tables = block_tables
620
+ self.pooling_params = pooling_params
621
self.lora_request = lora_request
622
self.computed_block_nums = computed_block_nums
623
self.multi_modal_data = multi_modal_data
@@ -669,8 +682,20 @@ class SequenceOutput:
682
return equal and log_probs_equal
683
684
672
-class SequenceGroupOutput:
673
- """The model output associated with a sequence group."""
685
+class SequenceGroupOutput(ABC):
686
+ """The base class for model outputs associated with a sequence group."""
687
+
688
+ @abstractmethod
689
+ def __repr__(self) -> str:
690
+ pass
691
+
692
+ @abstractmethod
693
+ def __eq__(self, other: object) -> bool:
694
+ pass
695
+
696
+
697
+class CompletionSequenceGroupOutput(SequenceGroupOutput):
698
+ """The model output associated with a completion sequence group."""
699
700
def __init__(
701
self,
@@ -682,26 +707,45 @@ class SequenceGroupOutput:
707
self.prompt_logprobs = prompt_logprobs
708
709
def __repr__(self) -> str:
685
- return (f"SequenceGroupOutput(samples={self.samples}, "
710
+ return (f"CompletionSequenceGroupOutput(samples={self.samples}, "
711
f"prompt_logprobs={self.prompt_logprobs})")
712
713
def __eq__(self, other: object) -> bool:
689
- if not isinstance(other, SequenceGroupOutput):
714
+ if not isinstance(other, CompletionSequenceGroupOutput):
715
raise NotImplementedError()
716
return (self.samples == other.samples
717
and self.prompt_logprobs == other.prompt_logprobs)
718
719
720
+class EmbeddingSequenceGroupOutput(SequenceGroupOutput):
721
+ """The model output associated with an embedding sequence group."""
722
+
723
+ def __init__(
724
+ self,
725
+ embeddings: List[float],
726
+ ) -> None:
727
+ self.embeddings = embeddings
728
+
729
+ def __repr__(self) -> str:
730
+ return (f"EmbeddingSequenceGroupOutput("
731
+ f"embeddings_shape={len(self.embeddings)})")
732
+
733
+ def __eq__(self, other: object) -> bool:
734
+ if not isinstance(other, EmbeddingSequenceGroupOutput):
735
+ raise NotImplementedError()
736
+ return self.embeddings == other.embeddings
737
+
738
+
739
@dataclass
740
class SamplerOutput:
741
"""For each sequence group, we generate a list of SequenceOutput object,
742
each of which contains one possible candidate for the next token.
743
700
- This datastructure implements methods so it can be used like a list, but
744
+ This data structure implements methods, so it can be used like a list, but
745
also has optional fields for device tensors.
746
"""
747
704
- outputs: List[SequenceGroupOutput]
748
+ outputs: List[CompletionSequenceGroupOutput]
749
750
# On-device tensor containing probabilities of each token.
751
sampled_token_probs: Optional["torch.Tensor"] = None
@@ -742,6 +786,27 @@ class SamplerOutput:
786
f"spec_decode_worker_metrics={self.spec_decode_worker_metrics})")
787
788
789
+@dataclass
790
+class PoolerOutput:
791
+ """The output from a pooling operation in the embedding model."""
792
+ outputs: List[EmbeddingSequenceGroupOutput]
793
+
794
+ spec_decode_worker_metrics: Optional["SpecDecodeWorkerMetrics"] = None
795
+
796
+ def __getitem__(self, idx: int):
797
+ return self.outputs[idx]
798
+
799
+ def __setitem__(self, idx: int, value):
800
+ self.outputs[idx] = value
801
+
802
+ def __len__(self):
803
+ return len(self.outputs)
804
+
805
+ def __eq__(self, other: object):
806
+ return isinstance(other,
807
+ self.__class__) and self.outputs == other.outputs
808
+
809
+
810
@dataclass
811
class ExecuteModelRequest:
812
"""The model execution request."""
vllm/spec_decode/util.py
+3
-2
@@ -4,7 +4,8 @@ from typing import Dict, List, Tuple
4
5
import torch
6
7
-from vllm.sequence import (Logprob, SamplerOutput, SequenceGroupMetadata,
7
+from vllm.sequence import (CompletionSequenceGroupOutput, Logprob,
8
+ SamplerOutput, SequenceGroupMetadata,
9
SequenceGroupOutput, SequenceOutput)
10
11
SeqId = int
@@ -94,7 +95,7 @@ def create_sequence_group_output(
95
for topk_logprob_index, _ in enumerate(topk_token_ids)
96
})
97
97
- return SequenceGroupOutput(
98
+ return CompletionSequenceGroupOutput(
99
samples=[
100
SequenceOutput(parent_seq_id=seq_id,
101
output_token=token_id,
vllm/worker/embedding_model_runner.py
new
+266
@@ -0,0 +1,266 @@
1
+from typing import Dict, List, Optional, Set, Tuple
2
+
3
+import torch
4
+
5
+from vllm.attention import AttentionMetadata
6
+from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
7
+ ModelConfig, ParallelConfig, SchedulerConfig,
8
+ VisionLanguageConfig)
9
+from vllm.distributed import broadcast_tensor_dict
10
+from vllm.logger import init_logger
11
+from vllm.lora.layers import LoRAMapping
12
+from vllm.lora.request import LoRARequest
13
+from vllm.model_executor.pooling_metadata import PoolingMetadata
14
+from vllm.pooling_params import PoolingParams
15
+from vllm.sequence import PoolerOutput, SequenceData, SequenceGroupMetadata
16
+from vllm.worker.model_runner import BatchType, ModelRunner
17
+
18
+logger = init_logger(__name__)
19
+
20
+
21
+class EmbeddingModelRunner(ModelRunner):
22
+
23
+ def __init__(
24
+ self,
25
+ model_config: ModelConfig,
26
+ parallel_config: ParallelConfig,
27
+ scheduler_config: SchedulerConfig,
28
+ device_config: DeviceConfig,
29
+ cache_config: CacheConfig,
30
+ load_config: LoadConfig,
31
+ lora_config: Optional[LoRAConfig],
32
+ kv_cache_dtype: Optional[str] = "auto",
33
+ is_driver_worker: bool = False,
34
+ vision_language_config: Optional[VisionLanguageConfig] = None,
35
+ ):
36
+ super().__init__(model_config,
37
+ parallel_config,
38
+ scheduler_config,
39
+ device_config,
40
+ cache_config,
41
+ load_config,
42
+ lora_config=lora_config,
43
+ kv_cache_dtype=kv_cache_dtype,
44
+ is_driver_worker=is_driver_worker,
45
+ vision_language_config=vision_language_config)
46
+
47
+ @torch.inference_mode()
48
+ def execute_model(
49
+ self,
50
+ seq_group_metadata_list: List[SequenceGroupMetadata],
51
+ kv_caches: List[torch.Tensor],
52
+ ) -> Optional[PoolerOutput]:
53
+ (input_tokens, input_positions, attn_metadata, pooling_metadata,
54
+ lora_requests, lora_mapping, multi_modal_input
55
+ ) = self.prepare_input_tensors(seq_group_metadata_list)
56
+
57
+ if self.lora_config:
58
+ self.set_active_loras(lora_requests, lora_mapping)
59
+
60
+ # Currently cuda graph is only supported by the decode phase.
61
+ prefill_meta = attn_metadata.prefill_metadata
62
+ decode_meta = attn_metadata.decode_metadata
63
+ if prefill_meta is None and decode_meta.use_cuda_graph:
64
+ graph_batch_size = input_tokens.shape[0]
65
+ model_executable = self.graph_runners[graph_batch_size]
66
+ else:
67
+ model_executable = self.model
68
+
69
+ num_layers = self.model_config.get_num_layers(self.parallel_config)
70
+ kv_caches = [None] * num_layers
71
+
72
+ execute_model_kwargs = {
73
+ "input_ids": input_tokens,
74
+ "positions": input_positions,
75
+ "kv_caches": kv_caches,
76
+ "attn_metadata": attn_metadata,
77
+ }
78
+ if self.vision_language_config:
79
+ execute_model_kwargs.update({"image_input": multi_modal_input})
80
+ hidden_states = model_executable(**execute_model_kwargs)
81
+
82
+ return self.model.pooler(hidden_states=hidden_states,
83
+ pooling_metadata=pooling_metadata)
84
+
85
+ def prepare_input_tensors(
86
+ self,
87
+ seq_group_metadata_list: List[SequenceGroupMetadata],
88
+ ) -> Tuple[torch.Tensor, torch.Tensor, AttentionMetadata, PoolingMetadata,
89
+ Set[LoRARequest], LoRAMapping, torch.Tensor]:
90
+ if self.is_driver_worker:
91
+ prefill_reqs = []
92
+ decode_reqs = []
93
+ for seq_group_meta in seq_group_metadata_list:
94
+ if seq_group_meta.is_prompt:
95
+ prefill_reqs.append(seq_group_meta)
96
+ else:
97
+ decode_reqs.append(seq_group_meta)
98
+
99
+ # Prepare input tensors.
100
+ (
101
+ input_tokens,
102
+ input_positions,
103
+ prefill_attn_metadata,
104
+ prompt_lens,
105
+ subquery_lens,
106
+ lora_index_mapping,
107
+ lora_prompt_mapping,
108
+ lora_requests,
109
+ multi_modal_input,
110
+ slot_mapping,
111
+ ) = self._prepare_prompt(prefill_reqs)
112
+ (
113
+ decode_input_tokens,
114
+ decode_input_positions,
115
+ decode_attn_metadata,
116
+ decode_lora_index_mapping,
117
+ decode_lora_prompt_mapping,
118
+ decode_lora_requests,
119
+ decode_slot_mapping,
120
+ ) = self._prepare_decode(decode_reqs)
121
+
122
+ # Prepare PoolingMetadata
123
+ pooling_metadata = self._prepare_pooling(seq_group_metadata_list,
124
+ prompt_lens)
125
+
126
+ if not self.scheduler_config.chunked_prefill_enabled:
127
+ assert (len(prefill_reqs) and len(decode_reqs)) == 0
128
+
129
+ num_prefills = len(prompt_lens)
130
+ num_prefill_tokens = len(input_tokens)
131
+ num_decode_tokens = len(decode_input_tokens)
132
+
133
+ # Coalesce tensors. Note that attn_metadata is currently not
134
+ # coalesced for simplicity.
135
+ input_tokens.extend(decode_input_tokens)
136
+ input_positions.extend(decode_input_positions)
137
+ slot_mapping.extend(decode_slot_mapping)
138
+ lora_index_mapping.extend(decode_lora_index_mapping)
139
+ lora_prompt_mapping.extend(decode_lora_prompt_mapping)
140
+ lora_requests.update(decode_lora_requests)
141
+
142
+ input_tokens = torch.tensor(input_tokens,
143
+ dtype=torch.long,
144
+ device=self.device)
145
+ input_positions = torch.tensor(input_positions,
146
+ dtype=torch.long,
147
+ device=self.device)
148
+ slot_mapping = torch.tensor(slot_mapping,
149
+ dtype=torch.long,
150
+ device=self.device)
151
+
152
+ if self.lora_config:
153
+ lora_mapping = LoRAMapping(
154
+ lora_index_mapping,
155
+ lora_prompt_mapping,
156
+ )
157
+ else:
158
+ lora_mapping = None
159
+
160
+ # Broadcast the metadata.
161
+ # If batch contains both prefill and decode, it sends 2 broadcasts.
162
+ # If it only contains 1 type, it triggers a single broadcast.
163
+ if (prefill_attn_metadata is not None
164
+ and decode_attn_metadata is not None):
165
+ batch_type = BatchType.MIXED
166
+ elif prefill_attn_metadata is not None:
167
+ batch_type = BatchType.PREFILL
168
+ else:
169
+ batch_type = BatchType.DECODE
170
+
171
+ metadata_dict = {
172
+ "input_tokens": input_tokens,
173
+ "input_positions": input_positions,
174
+ "lora_requests": lora_requests,
175
+ "lora_mapping": lora_mapping,
176
+ "multi_modal_input": multi_modal_input,
177
+ "num_prefill_tokens": num_prefill_tokens,
178
+ "num_decode_tokens": num_decode_tokens,
179
+ "slot_mapping": slot_mapping,
180
+ "num_prefills": num_prefills,
181
+ "batch_type": batch_type,
182
+ }
183
+ if prefill_attn_metadata is not None:
184
+ metadata_dict.update(prefill_attn_metadata.asdict_zerocopy())
185
+ else:
186
+ assert decode_attn_metadata is not None
187
+ metadata_dict.update(decode_attn_metadata.asdict_zerocopy())
188
+ broadcast_tensor_dict(metadata_dict, src=0)
189
+
190
+ # Broadcast decode attn metadata for mixed batch type.
191
+ # The additional broadcast costs 300us overhead on 4 A10 GPUs.
192
+ # We can potentially reduce the overhead by coelescing tensors.
193
+ if batch_type == BatchType.MIXED:
194
+ assert decode_attn_metadata is not None
195
+ metadata_dict = decode_attn_metadata.asdict_zerocopy()
196
+ broadcast_tensor_dict(metadata_dict, src=0)
197
+ else:
198
+ metadata_dict = broadcast_tensor_dict(src=0)
199
+ input_tokens = metadata_dict.pop("input_tokens")
200
+ input_positions = metadata_dict.pop("input_positions")
201
+ slot_mapping = metadata_dict.pop("slot_mapping")
202
+ num_prefills = metadata_dict.pop("num_prefills")
203
+ lora_mapping = metadata_dict.pop("lora_mapping")
204
+ lora_requests = metadata_dict.pop("lora_requests")
205
+ multi_modal_input = metadata_dict.pop("multi_modal_input")
206
+ num_prefill_tokens = metadata_dict.pop("num_prefill_tokens")
207
+ num_decode_tokens = metadata_dict.pop("num_decode_tokens")
208
+ batch_type = metadata_dict.pop("batch_type")
209
+
210
+ # Create an attention metadata.
211
+ prefill_attn_metadata = None
212
+ decode_attn_metadata = None
213
+ if batch_type == BatchType.PREFILL or batch_type == BatchType.MIXED:
214
+ prefill_attn_metadata = self.attn_backend.make_metadata(
215
+ **metadata_dict)
216
+ else:
217
+ decode_attn_metadata = self.attn_backend.make_metadata(
218
+ **metadata_dict)
219
+
220
+ pooling_metadata = PoolingMetadata(seq_groups=None,
221
+ seq_data=None,
222
+ prompt_lens=None)
223
+
224
+ # if it is a mixed batch, decode attn_metadata is broadcasted
225
+ # separately.
226
+ if batch_type == BatchType.MIXED:
227
+ metadata_dict = broadcast_tensor_dict(src=0)
228
+ decode_attn_metadata = self.attn_backend.make_metadata(
229
+ **metadata_dict)
230
+
231
+ attn_metadata = AttentionMetadata(
232
+ num_prefills=num_prefills,
233
+ slot_mapping=slot_mapping,
234
+ num_prefill_tokens=num_prefill_tokens,
235
+ num_decode_tokens=num_decode_tokens,
236
+ prefill_metadata=prefill_attn_metadata,
237
+ decode_metadata=decode_attn_metadata,
238
+ kv_cache_dtype=self.kv_cache_dtype,
239
+ )
240
+
241
+ return (input_tokens, input_positions, attn_metadata, pooling_metadata,
242
+ lora_requests, lora_mapping, multi_modal_input)
243
+
244
+ def _prepare_pooling(
245
+ self,
246
+ seq_group_metadata_list: List[SequenceGroupMetadata],
247
+ prompt_lens: List[int],
248
+ ) -> PoolingMetadata:
249
+ """Prepare PoolingMetadata for the sequence group metadata list."""
250
+ seq_groups: List[Tuple[List[int], PoolingParams]] = []
251
+ for i, seq_group_metadata in enumerate(seq_group_metadata_list):
252
+ seq_ids = list(seq_group_metadata.seq_data.keys())
253
+ pooling_params = seq_group_metadata.pooling_params
254
+ seq_groups.append((seq_ids, pooling_params))
255
+
256
+ seq_data: Dict[int, SequenceData] = {}
257
+ for seq_group_metadata in seq_group_metadata_list:
258
+ seq_data.update(seq_group_metadata.seq_data)
259
+
260
+ pooling_metadata = PoolingMetadata(
261
+ seq_groups=seq_groups,
262
+ seq_data=seq_data,
263
+ prompt_lens=prompt_lens,
264
+ )
265
+
266
+ return pooling_metadata
vllm/worker/model_runner.py
+18
-7
@@ -1,6 +1,6 @@
1
import time
2
from enum import IntEnum
3
-from typing import Dict, List, NamedTuple, Optional, Set, Tuple
3
+from typing import Dict, List, NamedTuple, Optional, Set, Tuple, Union
4
5
import numpy as np
6
import torch
@@ -287,18 +287,18 @@ class ModelRunner:
287
lora_requests.add(seq_group_metadata.lora_request)
288
289
lora_index_mapping += [lora_id] * (seq_len - context_len)
290
- lora_prompt_mapping.extend(
291
- [lora_id] *
292
- (seq_len - context_len
293
- if seq_group_metadata.sampling_params.prompt_logprobs else 1))
290
+ lora_prompt_mapping.extend([lora_id] * (
291
+ seq_len - context_len if seq_group_metadata.sampling_params
292
+ and seq_group_metadata.sampling_params.prompt_logprobs else 1))
293
294
if seq_group_metadata.multi_modal_data:
295
multi_modal_input_list.append(
296
seq_group_metadata.multi_modal_data.data)
297
299
- if seq_group_metadata.block_tables is None:
298
+ if _is_block_tables_empty(seq_group_metadata.block_tables):
299
# During memory profiling, the block tables are not initialized
300
# yet. In this case, we just use a dummy slot mapping.
301
+ # In embeddings, the block tables are {seq_id: None}.
302
slot_mapping.extend([_PAD_SLOT_ID] * seq_len)
303
continue
304
@@ -813,7 +813,6 @@ class ModelRunner:
813
sampling_params = SamplingParams(top_p=0.99, top_k=self.vocab_size - 1)
814
max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
815
max_num_seqs = self.scheduler_config.max_num_seqs
816
-
816
# This represents the maximum number of different requests
817
# that will have unique loras, an therefore the max amount of memory
818
# consumption create dummy lora request copies from the lora request
@@ -1139,3 +1138,15 @@ def _prepare_fake_inputs(
1138
prompt_tokens = [0] * seq_len
1139
fake_image_input = None
1140
return SequenceData(prompt_tokens), fake_image_input
1141
+
1142
+
1143
+def _is_block_tables_empty(block_tables: Union[None, Dict]):
1144
+ """
1145
+ Check if block_tables is None or a dictionary with all None values.
1146
+ """
1147
+ if block_tables is None:
1148
+ return True
1149
+ if isinstance(block_tables, dict) and all(
1150
+ value is None for value in block_tables.values()):
1151
+ return True
1152
+ return False
vllm/worker/worker.py
+9
-5
@@ -1,7 +1,7 @@
1
"""A GPU worker class."""
2
import gc
3
import os
4
-from typing import Any, Dict, List, Optional, Set, Tuple
4
+from typing import Any, Dict, List, Optional, Set, Tuple, Union
5
6
import torch
7
import torch.distributed
@@ -16,8 +16,9 @@ from vllm.distributed.device_communicators.custom_all_reduce import (
16
init_custom_ar)
17
from vllm.lora.request import LoRARequest
18
from vllm.model_executor import set_random_seed
19
-from vllm.sequence import ExecuteModelRequest, SamplerOutput
19
+from vllm.sequence import ExecuteModelRequest, PoolerOutput, SamplerOutput
20
from vllm.worker.cache_engine import CacheEngine
21
+from vllm.worker.embedding_model_runner import EmbeddingModelRunner
22
from vllm.worker.model_runner import ModelRunner
23
from vllm.worker.worker_base import WorkerBase
24
@@ -68,7 +69,9 @@ class Worker(WorkerBase):
69
assert not self.lora_config, (
70
"To be tested: vision language model with LoRA settings.")
71
71
- self.model_runner = ModelRunner(
72
+ ModelRunnerClass = (EmbeddingModelRunner if
73
+ self.model_config.embedding_mode else ModelRunner)
74
+ self.model_runner = ModelRunnerClass(
75
model_config,
76
parallel_config,
77
scheduler_config,
@@ -83,7 +86,8 @@ class Worker(WorkerBase):
86
# Uninitialized cache engine. Will be initialized by
87
# initialize_cache.
88
self.cache_engine: CacheEngine
86
- self.gpu_cache: List[torch.Tensor]
89
+ # Initialize gpu_cache as embedding models don't initialize kv_caches
90
+ self.gpu_cache: Optional[List[torch.tensor]] = None
91
92
def init_device(self) -> None:
93
if self.device_config.device.type == "cuda":
@@ -209,7 +213,7 @@ class Worker(WorkerBase):
213
def execute_model(
214
self,
215
execute_model_req: Optional[ExecuteModelRequest] = None
212
- ) -> List[SamplerOutput]:
216
+ ) -> List[Union[SamplerOutput, PoolerOutput]]:
217
218
if execute_model_req is None:
219
seq_group_metadata_list = None