A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Core] Interface for accessing model from `VllmRunner` (#10353)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Cyrus Leung committed
Jan 20, 2025 at 15:00 UTC
59a0192fb9bef026086d0a2ed32705d870a9466a
35 files changed
+474
-307
tests/conftest.py
+5
@@ -244,6 +244,7 @@ def video_assets() -> _VideoAssets:
244
245
246
_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature, dict)
247
+_R = TypeVar("_R")
248
249
250
class HfRunner:
@@ -930,6 +931,10 @@ class VllmRunner:
931
req_outputs = self.model.score(text_1, text_2)
932
return [req_output.outputs.score for req_output in req_outputs]
933
934
+ def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
935
+ executor = self.model.llm_engine.model_executor
936
+ return executor.apply_model(func)
937
+
938
def __enter__(self):
939
return self
940
tests/engine/test_custom_executor.py
+3
-1
@@ -51,7 +51,9 @@ def test_custom_executor(model, tmp_path):
51
assert not os.path.exists(".marker")
52
53
engine_args = EngineArgs(
54
- model=model, distributed_executor_backend=CustomUniExecutor)
54
+ model=model,
55
+ distributed_executor_backend=CustomUniExecutor,
56
+ )
57
engine = LLMEngine.from_engine_args(engine_args)
58
sampling_params = SamplingParams(max_tokens=1)
59
tests/model_executor/test_model_load_with_params.py
+33
-31
@@ -25,13 +25,12 @@ def test_model_loading_with_params(vllm_runner):
25
with vllm_runner(model_name=MODEL_NAME,
26
revision=REVISION,
27
dtype="float16",
28
- max_model_len=MAX_MODEL_LEN) as model:
29
- output = model.encode("Write a short story about a robot that"
30
- " dreams for the first time.\n")
28
+ max_model_len=MAX_MODEL_LEN) as vllm_model:
29
+ output = vllm_model.encode("Write a short story about a robot that"
30
+ " dreams for the first time.\n")
31
32
- model_config = model.model.llm_engine.model_config
33
-
34
- model_tokenizer = model.model.llm_engine.tokenizer
32
+ model_config = vllm_model.model.llm_engine.model_config
33
+ model_tokenizer = vllm_model.model.llm_engine.tokenizer
34
35
# asserts on the bert model config file
36
assert model_config.encoder_config["max_seq_length"] == 512
@@ -46,11 +45,13 @@ def test_model_loading_with_params(vllm_runner):
45
assert model_tokenizer.tokenizer_config["do_lower_case"]
46
assert model_tokenizer.tokenizer.model_max_length == 512
47
49
- model = model.model.llm_engine.model_executor\
50
- .driver_worker.model_runner.model
51
- assert isinstance(model, BertEmbeddingModel)
52
- assert model._pooler.pooling_type == PoolingType.CLS
53
- assert model._pooler.normalize
48
+ def check_model(model):
49
+ assert isinstance(model, BertEmbeddingModel)
50
+ assert model._pooler.pooling_type == PoolingType.CLS
51
+ assert model._pooler.normalize
52
+
53
+ vllm_model.apply_model(check_model)
54
+
55
# assert output
56
assert output
57
@@ -64,13 +65,12 @@ def test_roberta_model_loading_with_params(vllm_runner):
65
with vllm_runner(model_name=MODEL_NAME_ROBERTA,
66
revision=REVISION_ROBERTA,
67
dtype="float16",
67
- max_model_len=MAX_MODEL_LEN) as model:
68
- output = model.encode("Write a short story about a robot that"
69
- " dreams for the first time.\n")
68
+ max_model_len=MAX_MODEL_LEN) as vllm_model:
69
+ output = vllm_model.encode("Write a short story about a robot that"
70
+ " dreams for the first time.\n")
71
71
- model_config = model.model.llm_engine.model_config
72
-
73
- model_tokenizer = model.model.llm_engine.tokenizer
72
+ model_config = vllm_model.model.llm_engine.model_config
73
+ model_tokenizer = vllm_model.model.llm_engine.tokenizer
74
75
# asserts on the bert model config file
76
assert model_config.encoder_config["max_seq_length"] == 512
@@ -84,11 +84,12 @@ def test_roberta_model_loading_with_params(vllm_runner):
84
assert model_tokenizer.tokenizer_id == "intfloat/multilingual-e5-large"
85
assert not model_tokenizer.tokenizer_config["do_lower_case"]
86
87
- model = model.model.llm_engine.model_executor\
88
- .driver_worker.model_runner.model
89
- assert isinstance(model, RobertaEmbeddingModel)
90
- assert model._pooler.pooling_type == PoolingType.MEAN
91
- assert model._pooler.normalize
87
+ def check_model(model):
88
+ assert isinstance(model, RobertaEmbeddingModel)
89
+ assert model._pooler.pooling_type == PoolingType.MEAN
90
+ assert model._pooler.normalize
91
+
92
+ vllm_model.apply_model(check_model)
93
94
# assert output
95
assert output
@@ -103,17 +104,18 @@ def test_facebook_roberta_model_loading_with_params(vllm_runner):
104
model_name = "FacebookAI/roberta-base"
105
with vllm_runner(model_name=model_name,
106
dtype="float16",
106
- max_model_len=MAX_MODEL_LEN) as model:
107
- output = model.encode("Write a short story about a robot that"
108
- " dreams for the first time.\n")
107
+ max_model_len=MAX_MODEL_LEN) as vllm_model:
108
+ output = vllm_model.encode("Write a short story about a robot that"
109
+ " dreams for the first time.\n")
110
110
- model_tokenizer = model.model.llm_engine.tokenizer
111
+ model_tokenizer = vllm_model.model.llm_engine.tokenizer
112
assert model_tokenizer.tokenizer_id == model_name
113
113
- model = model.model.llm_engine.model_executor\
114
- .driver_worker.model_runner.model
115
- assert not hasattr(model, "lm_head")
116
- assert isinstance(model, RobertaEmbeddingModel)
117
- assert isinstance(model._pooler, CLSPool)
114
+ def check_model(model):
115
+ assert isinstance(model, RobertaEmbeddingModel)
116
+ assert not hasattr(model, "lm_head")
117
+ assert isinstance(model._pooler, CLSPool)
118
+
119
+ vllm_model.apply_model(check_model)
120
121
assert output
tests/models/decoder_only/language/test_jamba.py
+5
-2
@@ -33,10 +33,13 @@ def test_models(
33
34
with vllm_runner(model, dtype=dtype) as vllm_model:
35
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
36
+
37
# This test is for verifying whether the model's extra_repr
38
# can be printed correctly.
38
- print(vllm_model.model.llm_engine.model_executor.driver_worker.
39
- model_runner.model)
39
+ def print_model(model):
40
+ print(model)
41
+
42
+ vllm_model.apply_model(print_model)
43
44
for i in range(len(example_prompts)):
45
hf_output_ids, hf_output_str = hf_outputs[i]
tests/models/decoder_only/language/test_mamba.py
+5
-2
@@ -51,10 +51,13 @@ def test_models(
51
52
with vllm_runner(model, dtype=dtype) as vllm_model:
53
vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
54
+
55
# This test is for verifying whether the model's extra_repr
56
# can be printed correctly.
56
- print(vllm_model.model.llm_engine.model_executor.driver_worker.
57
- model_runner.model)
57
+ def print_model(model):
58
+ print(model)
59
+
60
+ vllm_model.apply_model(print_model)
61
62
for i in range(len(example_prompts)):
63
hf_output_ids, hf_output_str = hf_outputs[i]
tests/models/decoder_only/language/test_models.py
+5
-2
@@ -73,10 +73,13 @@ def test_models(
73
with vllm_runner(model, dtype=dtype) as vllm_model:
74
vllm_outputs = vllm_model.generate_greedy_logprobs(
75
example_prompts, max_tokens, num_logprobs)
76
+
77
# This test is for verifying whether the model's extra_repr
78
# can be printed correctly.
78
- print(vllm_model.model.llm_engine.model_executor.driver_worker.
79
- model_runner.model)
79
+ def print_model(model):
80
+ print(model)
81
+
82
+ vllm_model.apply_model(print_model)
83
84
check_logprobs_close(
85
outputs_0_lst=hf_outputs,
tests/models/decoder_only/vision_language/test_qwen2_vl.py
+26
-23
@@ -5,7 +5,6 @@ import pytest
5
import torch
6
from PIL import Image
7
8
-from vllm.entrypoints.llm import LLM
8
from vllm.multimodal.image import rescale_image_size
9
from vllm.multimodal.video import rescale_video_size, sample_frames_from_video
10
@@ -69,7 +68,7 @@ class Qwen2VLPromptVideoEmbeddingInput(TypedDict):
68
69
def batch_make_image_embeddings(
70
image_batches: List[Union[Image.Image, List[Image.Image]]], processor,
72
- llm: LLM) -> List[Qwen2VLPromptImageEmbeddingInput]:
71
+ llm: VllmRunner) -> List[Qwen2VLPromptImageEmbeddingInput]:
72
"""batched image embeddings for Qwen2-VL
73
74
This will infer all images' embeddings in a single batch,
@@ -106,16 +105,18 @@ def batch_make_image_embeddings(
105
image_grid_thw = preprocess_result["image_grid_thw"]
106
107
# pixel values to embeddings & grid_thws
109
- with torch.no_grad():
110
- visual = llm.llm_engine.model_executor.driver_worker. \
111
- model_runner.model.visual
108
+ def get_image_embeds(model):
109
+ with torch.no_grad():
110
+ visual = model.visual
111
113
- pixel_values_on_device = pixel_values.to(visual.device,
114
- dtype=visual.dtype)
115
- image_grid_thw_on_device = image_grid_thw.to(visual.device,
116
- dtype=torch.int64)
117
- image_embeds = visual(pixel_values_on_device,
118
- grid_thw=image_grid_thw_on_device)
112
+ pixel_values_on_device = pixel_values.to(visual.device,
113
+ dtype=visual.dtype)
114
+ image_grid_thw_on_device = image_grid_thw.to(visual.device,
115
+ dtype=torch.int64)
116
+ return visual(pixel_values_on_device,
117
+ grid_thw=image_grid_thw_on_device)
118
+
119
+ image_embeds = torch.concat(llm.apply_model(get_image_embeds))
120
121
# split into original batches
122
result: List[Qwen2VLPromptImageEmbeddingInput] = []
@@ -150,7 +151,7 @@ def batch_make_image_embeddings(
151
152
def batch_make_video_embeddings(
153
video_batches: PromptVideoInput, processor,
153
- llm: LLM) -> List[Qwen2VLPromptVideoEmbeddingInput]:
154
+ llm: VllmRunner) -> List[Qwen2VLPromptVideoEmbeddingInput]:
155
"""batched video embeddings for Qwen2-VL
156
157
A NDArray represents a single video's all frames.
@@ -187,16 +188,18 @@ def batch_make_video_embeddings(
188
video_grid_thw = preprocess_result["video_grid_thw"]
189
190
# pixel values to embeddings & grid_thws
190
- with torch.no_grad():
191
- visual = llm.llm_engine.model_executor.driver_worker.\
192
- model_runner.model.visual
191
+ def get_image_embeds(model):
192
+ with torch.no_grad():
193
+ visual = model.visual
194
+
195
+ pixel_values_on_device = pixel_values.to(visual.device,
196
+ dtype=visual.dtype)
197
+ video_grid_thw_on_device = video_grid_thw.to(visual.device,
198
+ dtype=torch.int64)
199
+ return visual(pixel_values_on_device,
200
+ grid_thw=video_grid_thw_on_device)
201
194
- pixel_values_on_device = pixel_values.to(visual.device,
195
- dtype=visual.dtype)
196
- video_grid_thw_on_device = video_grid_thw.to(visual.device,
197
- dtype=torch.int64)
198
- video_embeds = visual(pixel_values_on_device,
199
- grid_thw=video_grid_thw_on_device)
202
+ video_embeds = torch.concat(llm.apply_model(get_image_embeds))
203
204
# split into original batches
205
result: List[Qwen2VLPromptVideoEmbeddingInput] = []
@@ -278,9 +281,9 @@ def run_embedding_input_test(
281
max_tokens,
282
num_logprobs=num_logprobs,
283
images=batch_make_image_embeddings(
281
- images, processor, vllm_model.model) if images else None,
284
+ images, processor, vllm_model) if images else None,
285
videos=batch_make_video_embeddings(
283
- videos, processor, vllm_model.model) if videos else None)
286
+ videos, processor, vllm_model) if videos else None)
287
for prompts, images, videos in inputs
288
]
289
tests/models/embedding/language/test_cls_models.py
+5
-2
@@ -24,10 +24,13 @@ def test_classification_models(
24
) -> None:
25
with vllm_runner(model, dtype=dtype) as vllm_model:
26
vllm_outputs = vllm_model.classify(example_prompts)
27
+
28
# This test is for verifying whether the model's extra_repr
29
# can be printed correctly.
29
- print(vllm_model.model.llm_engine.model_executor.driver_worker.
30
- model_runner.model)
30
+ def print_model(model):
31
+ print(model)
32
+
33
+ vllm_model.apply_model(print_model)
34
35
with hf_runner(model,
36
dtype=dtype,
tests/models/embedding/language/test_embedding.py
+5
-2
@@ -62,10 +62,13 @@ def test_models(
62
max_model_len=None,
63
**vllm_extra_kwargs) as vllm_model:
64
vllm_outputs = vllm_model.encode(example_prompts)
65
+
66
# This test is for verifying whether the model's extra_repr
67
# can be printed correctly.
67
- print(vllm_model.model.llm_engine.model_executor.driver_worker.
68
- model_runner.model)
68
+ def print_model(model):
69
+ print(model)
70
+
71
+ vllm_model.apply_model(print_model)
72
73
check_embeddings_close(
74
embeddings_0_lst=hf_outputs,
tests/quantization/test_compressed_tensors.py
+137
-105
@@ -30,50 +30,55 @@ from vllm.platforms import current_platform
30
def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
31
model_path, strategy, quant_type, shape_0, is_symmetric = model_args
32
with vllm_runner(model_path, enforce_eager=True) as llm:
33
- model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
34
- layer = model.model.layers[0]
35
-
36
- qkv_proj = layer.self_attn.qkv_proj
37
- o_proj = layer.self_attn.o_proj
38
- gate_up_proj = layer.mlp.gate_up_proj
39
- down_proj = layer.mlp.down_proj
40
-
41
- # assert zp for symmetric and asymmetric cases
42
- def zp_valid(zp: Optional[torch.Tensor]):
43
- if is_symmetric:
44
- return zp is None
45
-
46
- return zp is not None and zp.dtype is torch.int32
47
-
48
- assert zp_valid(qkv_proj.input_zero_point)
49
- assert zp_valid(o_proj.input_zero_point)
50
- assert zp_valid(gate_up_proj.input_zero_point)
51
- assert zp_valid(down_proj.input_zero_point)
52
-
53
- assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
54
- assert isinstance(o_proj.quant_method, CompressedTensorsLinearMethod)
55
- assert isinstance(gate_up_proj.quant_method,
56
- CompressedTensorsLinearMethod)
57
- assert isinstance(down_proj.quant_method,
58
- CompressedTensorsLinearMethod)
59
- assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Int8)
60
-
61
- assert qkv_proj.scheme.strategy == strategy
62
- assert qkv_proj.scheme.is_static_input_scheme
63
- expected_type = torch.int8
64
-
65
- assert qkv_proj.weight.dtype is expected_type
66
- assert o_proj.weight.dtype is expected_type
67
- assert gate_up_proj.weight.dtype is expected_type
68
-
69
- if qkv_proj.scheme.strategy == "tensor":
70
- # Make sure it is a channelwise buffer
71
- # After running process_weights_after_loading
72
- assert len(qkv_proj.weight_scale.shape) == 2
73
- assert qkv_proj.weight_scale.shape[0] == shape_0
74
- assert qkv_proj.weight_scale.shape[1] == 1
75
- assert qkv_proj.weight_scale.dtype is torch.float32
76
- assert qkv_proj.input_scale.dtype is torch.float32
33
+
34
+ def check_model(model):
35
+ layer = model.model.layers[0]
36
+
37
+ qkv_proj = layer.self_attn.qkv_proj
38
+ o_proj = layer.self_attn.o_proj
39
+ gate_up_proj = layer.mlp.gate_up_proj
40
+ down_proj = layer.mlp.down_proj
41
+
42
+ # assert zp for symmetric and asymmetric cases
43
+ def zp_valid(zp: Optional[torch.Tensor]):
44
+ if is_symmetric:
45
+ return zp is None
46
+
47
+ return zp is not None and zp.dtype is torch.int32
48
+
49
+ assert zp_valid(qkv_proj.input_zero_point)
50
+ assert zp_valid(o_proj.input_zero_point)
51
+ assert zp_valid(gate_up_proj.input_zero_point)
52
+ assert zp_valid(down_proj.input_zero_point)
53
+
54
+ assert isinstance(qkv_proj.quant_method,
55
+ CompressedTensorsLinearMethod)
56
+ assert isinstance(o_proj.quant_method,
57
+ CompressedTensorsLinearMethod)
58
+ assert isinstance(gate_up_proj.quant_method,
59
+ CompressedTensorsLinearMethod)
60
+ assert isinstance(down_proj.quant_method,
61
+ CompressedTensorsLinearMethod)
62
+ assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Int8)
63
+
64
+ assert qkv_proj.scheme.strategy == strategy
65
+ assert qkv_proj.scheme.is_static_input_scheme
66
+ expected_type = torch.int8
67
+
68
+ assert qkv_proj.weight.dtype is expected_type
69
+ assert o_proj.weight.dtype is expected_type
70
+ assert gate_up_proj.weight.dtype is expected_type
71
+
72
+ if qkv_proj.scheme.strategy == "tensor":
73
+ # Make sure it is a channelwise buffer
74
+ # After running process_weights_after_loading
75
+ assert len(qkv_proj.weight_scale.shape) == 2
76
+ assert qkv_proj.weight_scale.shape[0] == shape_0
77
+ assert qkv_proj.weight_scale.shape[1] == 1
78
+ assert qkv_proj.weight_scale.dtype is torch.float32
79
+ assert qkv_proj.input_scale.dtype is torch.float32
80
+
81
+ llm.apply_model(check_model)
82
83
output = llm.generate_greedy(["Hello my name is"], max_tokens=20)
84
assert output
@@ -129,16 +134,20 @@ def test_compressed_tensors_no_enforce_eager(vllm_runner):
134
def test_compressed_tensors_w8a8_dynamic_per_token(vllm_runner, model_args):
135
model_path, strategy = model_args
136
with vllm_runner(model_path, dtype=torch.float16) as llm:
132
- model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
133
- layer = model.model.layers[0]
137
135
- qkv_proj = layer.self_attn.qkv_proj
138
+ def check_model(model):
139
+ layer = model.model.layers[0]
140
+
141
+ qkv_proj = layer.self_attn.qkv_proj
142
+
143
+ assert isinstance(qkv_proj.quant_method,
144
+ CompressedTensorsLinearMethod)
145
+ assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Int8)
146
+ assert not qkv_proj.scheme.is_static_input_scheme
147
+ assert qkv_proj.scheme.strategy == strategy
148
+ assert qkv_proj.weight.dtype is torch.int8
149
137
- assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
138
- assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Int8)
139
- assert not qkv_proj.scheme.is_static_input_scheme
140
- assert qkv_proj.scheme.strategy == strategy
141
- assert qkv_proj.weight.dtype is torch.int8
150
+ llm.apply_model(check_model)
151
152
output = llm.generate_greedy(["Hello my name is"], max_tokens=20)
153
assert output
@@ -152,19 +161,24 @@ def test_compressed_tensors_w8a8_dynamic_per_token(vllm_runner, model_args):
161
def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
162
model, strategy, group, pack_factor = wNa16_args
163
with vllm_runner(model) as llm:
155
- model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
156
- layer = model.model.layers[0]
164
158
- qkv_proj = layer.self_attn.qkv_proj
159
- assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
160
- assert isinstance(qkv_proj.scheme, CompressedTensorsWNA16)
165
+ def check_model(model):
166
+ layer = model.model.layers[0]
167
162
- assert qkv_proj.scheme.strategy == strategy
163
- assert qkv_proj.scheme.group_size == (-1 if group is None else group)
168
+ qkv_proj = layer.self_attn.qkv_proj
169
+ assert isinstance(qkv_proj.quant_method,
170
+ CompressedTensorsLinearMethod)
171
+ assert isinstance(qkv_proj.scheme, CompressedTensorsWNA16)
172
165
- assert qkv_proj.weight_packed.dtype is torch.int32
166
- assert qkv_proj.weight_scale.dtype is torch.float16
167
- assert qkv_proj.scheme.pack_factor == pack_factor
173
+ assert qkv_proj.scheme.strategy == strategy
174
+ assert qkv_proj.scheme.group_size == (-1
175
+ if group is None else group)
176
+
177
+ assert qkv_proj.weight_packed.dtype is torch.int32
178
+ assert qkv_proj.weight_scale.dtype is torch.float16
179
+ assert qkv_proj.scheme.pack_factor == pack_factor
180
+
181
+ llm.apply_model(check_model)
182
183
output = llm.generate_greedy("Hello my name is", max_tokens=20)
184
assert output
@@ -173,14 +187,18 @@ def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
187
def test_compressed_tensors_w4a16_marlin24(vllm_runner):
188
model_path = "nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t"
189
with vllm_runner(model_path) as llm:
176
- model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
177
- layer = model.model.layers[0]
190
179
- qkv_proj = layer.self_attn.qkv_proj
191
+ def check_model(model):
192
+ layer = model.model.layers[0]
193
+
194
+ qkv_proj = layer.self_attn.qkv_proj
195
181
- assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
182
- assert isinstance(qkv_proj.scheme, CompressedTensorsW4A16Sparse24)
183
- assert qkv_proj.weight_packed.dtype is torch.int32
196
+ assert isinstance(qkv_proj.quant_method,
197
+ CompressedTensorsLinearMethod)
198
+ assert isinstance(qkv_proj.scheme, CompressedTensorsW4A16Sparse24)
199
+ assert qkv_proj.weight_packed.dtype is torch.int32
200
+
201
+ llm.apply_model(check_model)
202
203
output = llm.generate_greedy("Hello my name is", max_tokens=20)
204
assert output
@@ -189,23 +207,27 @@ def test_compressed_tensors_w4a16_marlin24(vllm_runner):
207
def test_compressed_tensors_fp8(vllm_runner):
208
model_path = "nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test"
209
with vllm_runner(model_path) as llm:
192
- model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
193
- layer = model.model.layers[0]
210
195
- qkv_proj = layer.self_attn.qkv_proj
211
+ def check_model(model):
212
+ layer = model.model.layers[0]
213
197
- assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
198
- assert isinstance(
199
- qkv_proj.scheme,
200
- (CompressedTensorsW8A8Fp8, CompressedTensorsW8A16Fp8))
214
+ qkv_proj = layer.self_attn.qkv_proj
215
202
- assert qkv_proj.input_scale.dtype is torch.float32
216
+ assert isinstance(qkv_proj.quant_method,
217
+ CompressedTensorsLinearMethod)
218
+ assert isinstance(
219
+ qkv_proj.scheme,
220
+ (CompressedTensorsW8A8Fp8, CompressedTensorsW8A16Fp8))
221
204
- if isinstance(qkv_proj.scheme, CompressedTensorsW8A8Fp8):
205
- assert len(qkv_proj.input_scale.shape) == 0
206
- assert qkv_proj.weight.dtype is torch.float8_e4m3fn
207
- assert qkv_proj.weight_scale.dtype is torch.float32
208
- assert len(qkv_proj.weight_scale.shape) == 0
222
+ assert qkv_proj.input_scale.dtype is torch.float32
223
+
224
+ if isinstance(qkv_proj.scheme, CompressedTensorsW8A8Fp8):
225
+ assert len(qkv_proj.input_scale.shape) == 0
226
+ assert qkv_proj.weight.dtype is torch.float8_e4m3fn
227
+ assert qkv_proj.weight_scale.dtype is torch.float32
228
+ assert len(qkv_proj.weight_scale.shape) == 0
229
+
230
+ llm.apply_model(check_model)
231
232
output = llm.generate_greedy("Hello my name is", max_tokens=20)
233
assert output
@@ -248,12 +270,15 @@ def _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy):
270
def test_compressed_tensors_2of4_quant_fp8(vllm_runner, args_2of4):
271
model, weight_strategy, input_strategy = args_2of4
272
with vllm_runner(model) as llm:
251
- model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
252
- layer = model.model.layers[0]
273
254
- qkv_proj = layer.self_attn.qkv_proj
255
- assert qkv_proj.scheme.weights_dtype == torch.float8_e4m3fn
256
- _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy)
274
+ def check_model(model):
275
+ layer = model.model.layers[0]
276
+
277
+ qkv_proj = layer.self_attn.qkv_proj
278
+ assert qkv_proj.scheme.weights_dtype == torch.float8_e4m3fn
279
+ _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy)
280
+
281
+ llm.apply_model(check_model)
282
283
output = llm.generate_greedy("Hello my name is", max_tokens=20)
284
print(output)
@@ -273,12 +298,15 @@ def test_compressed_tensors_2of4_quant_fp8(vllm_runner, args_2of4):
298
def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):
299
model, weight_strategy, input_strategy = args_2of4
300
with vllm_runner(model) as llm:
276
- model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
277
- layer = model.model.layers[0]
301
279
- qkv_proj = layer.self_attn.qkv_proj
280
- assert qkv_proj.scheme.weights_dtype == torch.int8
281
- _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy)
302
+ def check_model(model):
303
+ layer = model.model.layers[0]
304
+
305
+ qkv_proj = layer.self_attn.qkv_proj
306
+ assert qkv_proj.scheme.weights_dtype == torch.int8
307
+ _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy)
308
+
309
+ llm.apply_model(check_model)
310
311
output = llm.generate_greedy("Hello my name is", max_tokens=20)
312
print(output)
@@ -293,20 +321,24 @@ def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):
321
def test_compressed_tensors_2of4_sparse(vllm_runner, args_2of4):
322
model = args_2of4
323
with vllm_runner(model) as llm:
296
- model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
297
- layer = model.model.layers[0]
298
-
299
- qkv_proj = layer.self_attn.qkv_proj
300
- assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
301
- assert isinstance(qkv_proj.scheme, CompressedTensors24)
302
-
303
- assert qkv_proj.scheme.weight_quant is None
304
- assert qkv_proj.scheme.input_quant is None
305
- assert not qkv_proj.scheme.quantized
306
- assert qkv_proj.quant_method.quantization_config.sparsity_scheme_map
307
- sparsity_map = qkv_proj.quant_method.quantization_config.sparsity_scheme_map # noqa: E501
308
- assert sparsity_map.get("Linear").format == "dense"
309
- assert sparsity_map.get("Linear").sparsity_structure == "2:4"
324
+
325
+ def check_model(model):
326
+ layer = model.model.layers[0]
327
+
328
+ qkv_proj = layer.self_attn.qkv_proj
329
+ assert isinstance(qkv_proj.quant_method,
330
+ CompressedTensorsLinearMethod)
331
+ assert isinstance(qkv_proj.scheme, CompressedTensors24)
332
+
333
+ assert qkv_proj.scheme.weight_quant is None
334
+ assert qkv_proj.scheme.input_quant is None
335
+ assert not qkv_proj.scheme.quantized
336
+ assert qkv_proj.quant_method.quantization_config.sparsity_scheme_map
337
+ sparsity_map = qkv_proj.quant_method.quantization_config.sparsity_scheme_map # noqa: E501
338
+ assert sparsity_map.get("Linear").format == "dense"
339
+ assert sparsity_map.get("Linear").sparsity_structure == "2:4"
340
+
341
+ llm.apply_model(check_model)
342
343
output = llm.generate_greedy("Hello my name is", max_tokens=20)
344
print(output)
tests/quantization/test_fp8.py
+29
-23
@@ -49,13 +49,17 @@ KV_CACHE_MODELS = [
49
def test_kv_cache_model_load_and_run(vllm_runner, model_id: str):
50
with vllm_runner(model_id, kv_cache_dtype="fp8") as llm:
51
52
- model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
53
- attn = model.model.layers[0].self_attn.attn
54
- assert isinstance(attn.quant_method, Fp8KVCacheMethod)
55
- # NOTE: it is valid for scales to be 1.0 (default value), but we know
56
- # these checkpoints have scales < 1.0
57
- assert 0.0 < attn._k_scale < 1.0
58
- assert 0.0 < attn._v_scale < 1.0
52
+ def check_model(model):
53
+ attn = model.model.layers[0].self_attn.attn
54
+
55
+ assert isinstance(attn.quant_method, Fp8KVCacheMethod)
56
+
57
+ # NOTE: it is valid for scales to be 1.0 (default value), but
58
+ # we know these checkpoints have scales < 1.0
59
+ assert 0.0 < attn._k_scale < 1.0
60
+ assert 0.0 < attn._v_scale < 1.0
61
+
62
+ llm.apply_model(check_model)
63
64
# note: this does not test accuracy, just that we can run through
65
# see lm-eval tests for accuracy
@@ -77,22 +81,24 @@ def test_load_fp16_model(vllm_runner, kv_cache_dtype: str, force_marlin: bool,
81
quantization="fp8",
82
kv_cache_dtype=kv_cache_dtype) as llm:
83
80
- model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
81
- fc1 = model.model.decoder.layers[0].fc1
82
- assert isinstance(fc1.quant_method, Fp8LinearMethod)
83
- if kv_cache_dtype == "fp8":
84
- attn = model.model.decoder.layers[0].self_attn.attn
85
- assert isinstance(attn.quant_method, Fp8KVCacheMethod)
86
- assert attn._k_scale == 1.0
87
- assert attn._v_scale == 1.0
88
-
89
- if current_platform.has_device_capability(89) and not force_marlin:
90
- # For GPUs with hardware support, we keep weights in fp8
91
- assert fc1.weight.dtype == torch.float8_e4m3fn
92
- else:
93
- # For GPUs without hardware support, we pack the fp8 weights
94
- # for weight-only quantization using Marlin kernels
95
- assert fc1.weight.dtype == torch.int32
84
+ def check_model(model):
85
+ fc1 = model.model.decoder.layers[0].fc1
86
+ assert isinstance(fc1.quant_method, Fp8LinearMethod)
87
+ if kv_cache_dtype == "fp8":
88
+ attn = model.model.decoder.layers[0].self_attn.attn
89
+ assert isinstance(attn.quant_method, Fp8KVCacheMethod)
90
+ assert attn._k_scale == 1.0
91
+ assert attn._v_scale == 1.0
92
+
93
+ if current_platform.has_device_capability(89) and not force_marlin:
94
+ # For GPUs with hardware support, we keep weights in fp8
95
+ assert fc1.weight.dtype == torch.float8_e4m3fn
96
+ else:
97
+ # For GPUs without hardware support, we pack the fp8 weights
98
+ # for weight-only quantization using Marlin kernels
99
+ assert fc1.weight.dtype == torch.int32
100
+
101
+ llm.apply_model(check_model)
102
103
104
@pytest.mark.skipif(not is_quant_method_supported("fp8"),
tests/quantization/test_lm_head.py
+20
-17
@@ -28,20 +28,23 @@ def test_lm_head(
28
model_lm_head_quant: Tuple[str, bool],
29
) -> None:
30
model, lm_head_quantized = model_lm_head_quant
31
- vllm_model = vllm_runner(model, dtype=torch.float16, max_model_len=2048)
32
-
33
- lm_head_layer = (vllm_model.model.llm_engine.model_executor.driver_worker.
34
- model_runner.model.lm_head)
35
-
36
- if lm_head_quantized:
37
- assert isinstance(
38
- lm_head_layer.linear_method,
39
- (GPTQLinearMethod, GPTQMarlinLinearMethod, MarlinLinearMethod))
40
- else:
41
- assert isinstance(lm_head_layer.linear_method,
42
- UnquantizedEmbeddingMethod)
43
-
44
- print(
45
- vllm_model.generate_greedy(prompts=["Hello my name is"],
46
- max_tokens=10)[0][1])
47
- del vllm_model
31
+
32
+ with vllm_runner(model, dtype=torch.float16,
33
+ max_model_len=2048) as vllm_model:
34
+
35
+ def check_model(model):
36
+ lm_head_layer = model.lm_head
37
+
38
+ if lm_head_quantized:
39
+ assert isinstance(lm_head_layer.linear_method,
40
+ (GPTQLinearMethod, GPTQMarlinLinearMethod,
41
+ MarlinLinearMethod))
42
+ else:
43
+ assert isinstance(lm_head_layer.linear_method,
44
+ UnquantizedEmbeddingMethod)
45
+
46
+ vllm_model.apply_model(check_model)
47
+
48
+ print(
49
+ vllm_model.generate_greedy(prompts=["Hello my name is"],
50
+ max_tokens=10)[0][1])
tests/quantization/test_quark.py
+13
-10
@@ -12,19 +12,22 @@ from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
12
def test_quark_fp8(vllm_runner):
13
model_path = "amd/Llama-3.1-8B-Instruct-FP8-KV-Quark-test"
14
with vllm_runner(model_path) as llm:
15
- model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
16
- layer = model.model.layers[0]
15
18
- qkv_proj = layer.self_attn.qkv_proj
16
+ def check_model(model):
17
+ layer = model.model.layers[0]
18
20
- assert isinstance(qkv_proj.quant_method, QuarkLinearMethod)
21
- assert isinstance(qkv_proj.scheme, QuarkW8A8Fp8)
19
+ qkv_proj = layer.self_attn.qkv_proj
20
23
- if isinstance(qkv_proj.scheme, QuarkW8A8Fp8):
24
- assert len(qkv_proj.input_scale.shape) == 0
25
- assert qkv_proj.weight.dtype is torch.float8_e4m3fn
26
- #assert qkv_proj.weight.dtype is torch.float8_e4m3fnuz
27
- assert len(qkv_proj.weight_scale.shape) == 0
21
+ assert isinstance(qkv_proj.quant_method, QuarkLinearMethod)
22
+ assert isinstance(qkv_proj.scheme, QuarkW8A8Fp8)
23
+
24
+ if isinstance(qkv_proj.scheme, QuarkW8A8Fp8):
25
+ assert len(qkv_proj.input_scale.shape) == 0
26
+ assert qkv_proj.weight.dtype is torch.float8_e4m3fn
27
+ #assert qkv_proj.weight.dtype is torch.float8_e4m3fnuz
28
+ assert len(qkv_proj.weight_scale.shape) == 0
29
+
30
+ llm.apply_model(check_model)
31
32
output = llm.generate_greedy("Hello my name is", max_tokens=20)
33
assert output
tests/tensorizer_loader/test_tensorizer.py
+16
-18
@@ -3,6 +3,7 @@ import json
3
import os
4
import pathlib
5
import subprocess
6
+from functools import partial
7
from unittest.mock import MagicMock, patch
8
9
import openai
@@ -24,7 +25,6 @@ from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
25
# yapf: enable
26
from vllm.utils import PlaceholderModule, import_from_path
27
27
-from ..conftest import VllmRunner
28
from ..utils import VLLM_PATH, RemoteOpenAIServer
29
from .conftest import retry_until_skip
30
@@ -58,16 +58,6 @@ def is_curl_installed():
58
return False
59
60
61
-def get_torch_model(vllm_runner: VllmRunner):
62
- return vllm_runner \
63
- .model \
64
- .llm_engine \
65
- .model_executor \
66
- .driver_worker \
67
- .model_runner \
68
- .model
69
-
70
-
61
def write_keyfile(keyfile_path: str):
62
encryption_params = EncryptionParams.random()
63
pathlib.Path(keyfile_path).parent.mkdir(parents=True, exist_ok=True)
@@ -121,8 +111,10 @@ def test_deserialized_encrypted_vllm_model_has_same_outputs(
111
112
config_for_serializing = TensorizerConfig(tensorizer_uri=model_path,
113
encryption_keyfile=key_path)
124
- serialize_vllm_model(get_torch_model(vllm_model),
125
- config_for_serializing)
114
+
115
+ vllm_model.apply_model(
116
+ partial(serialize_vllm_model,
117
+ tensorizer_config=config_for_serializing))
118
119
config_for_deserializing = TensorizerConfig(tensorizer_uri=model_path,
120
encryption_keyfile=key_path)
@@ -175,8 +167,10 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
167
with vllm_runner(model_ref, ) as vllm_model:
168
model_path = tmp_path / (model_ref + ".tensors")
169
178
- serialize_vllm_model(get_torch_model(vllm_model),
179
- TensorizerConfig(tensorizer_uri=model_path))
170
+ vllm_model.apply_model(
171
+ partial(
172
+ serialize_vllm_model,
173
+ tensorizer_config=TensorizerConfig(tensorizer_uri=model_path)))
174
175
with vllm_runner(
176
model_ref,
@@ -215,8 +209,10 @@ def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
209
with vllm_runner(model_ref, ) as vllm_model:
210
model_path = tmp_path / (model_ref + ".tensors")
211
218
- serialize_vllm_model(get_torch_model(vllm_model),
219
- TensorizerConfig(tensorizer_uri=model_path))
212
+ vllm_model.apply_model(
213
+ partial(
214
+ serialize_vllm_model,
215
+ tensorizer_config=TensorizerConfig(tensorizer_uri=model_path)))
216
217
model_loader_extra_config = {
218
"tensorizer_uri": str(model_path),
@@ -337,7 +333,9 @@ def test_vllm_tensorized_model_has_same_outputs(vllm_runner, tmp_path):
333
334
with vllm_runner(model_ref) as vllm_model:
335
outputs = vllm_model.generate(prompts, sampling_params)
340
- serialize_vllm_model(get_torch_model(vllm_model), config)
336
+
337
+ vllm_model.apply_model(
338
+ partial(serialize_vllm_model, tensorizer_config=config))
339
340
assert is_vllm_tensorized(config)
341
vllm/engine/llm_engine.py
+3
-14
@@ -5,10 +5,10 @@ from collections import deque
5
from contextlib import contextmanager
6
from dataclasses import dataclass
7
from functools import partial
8
-from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
9
- Iterable, List, Mapping, NamedTuple, Optional)
8
+from typing import (TYPE_CHECKING, Callable, ClassVar, Deque, Dict, Iterable,
9
+ List, Mapping, NamedTuple, Optional)
10
from typing import Sequence as GenericSequence
11
-from typing import Set, Tuple, Type, Union, cast, overload
11
+from typing import Set, Type, Union, cast, overload
12
13
import torch
14
from typing_extensions import TypeVar, deprecated
@@ -1818,17 +1818,6 @@ class LLMEngine:
1818
def stop_profile(self) -> None:
1819
self.model_executor.stop_profile()
1820
1821
- def collective_rpc(self,
1822
- method: Union[str, Callable],
1823
- timeout: Optional[float] = None,
1824
- args: Tuple = (),
1825
- kwargs: Optional[Dict] = None) -> List[Any]:
1826
- """
1827
- See LLM.collective_rpc for more details.
1828
- """
1829
- return self.model_executor.collective_rpc(method, timeout, args,
1830
- kwargs)
1831
-
1821
def check_health(self) -> None:
1822
if self.tokenizer:
1823
self.tokenizer.check_health()
vllm/entrypoints/llm.py
+36
-16
@@ -5,8 +5,9 @@ from typing import (Any, Callable, ClassVar, Dict, List, Optional, Sequence,
5
Tuple, Type, Union, cast, overload)
6
7
import cloudpickle
8
+import torch.nn as nn
9
from tqdm import tqdm
9
-from typing_extensions import deprecated
10
+from typing_extensions import TypeVar, deprecated
11
12
from vllm import envs
13
from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
@@ -42,6 +43,8 @@ from vllm.utils import Counter, deprecate_args, deprecate_kwargs, is_list_of
43
44
logger = init_logger(__name__)
45
46
+_R = TypeVar("_R", default=Any)
47
+
48
49
class LLM:
50
"""An LLM for generating texts from given prompts and sampling parameters.
@@ -464,25 +467,42 @@ class LLM:
467
return self.engine_class.validate_outputs(outputs, RequestOutput)
468
469
def collective_rpc(self,
467
- method: Union[str, Callable],
470
+ method: Union[str, Callable[..., _R]],
471
timeout: Optional[float] = None,
472
args: Tuple = (),
470
- kwargs: Optional[Dict] = None) -> List[Any]:
473
+ kwargs: Optional[Dict[str, Any]] = None) -> List[_R]:
474
+ """
475
+ Execute an RPC call on all workers.
476
+
477
+ Args:
478
+ method: Name of the worker method to execute, or a callable that
479
+ is serialized and sent to all workers to execute.
480
+
481
+ If the method is a callable, it should accept an additional
482
+ `self` argument, in addition to the arguments passed in `args`
483
+ and `kwargs`. The `self` argument will be the worker object.
484
+ timeout: Maximum time in seconds to wait for execution. Raises a
485
+ :exc:`TimeoutError` on timeout. `None` means wait indefinitely.
486
+ args: Positional arguments to pass to the worker method.
487
+ kwargs: Keyword arguments to pass to the worker method.
488
+
489
+ Returns:
490
+ A list containing the results from each worker.
491
+
492
+ Note:
493
+ It is recommended to use this API to only pass control messages,
494
+ and set up data-plane communication to pass data.
495
+ """
496
+ executor = self.llm_engine.model_executor
497
+ return executor.collective_rpc(method, timeout, args, kwargs)
498
+
499
+ def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
500
"""
472
- Run a method on all workers, with homogeneous arguments.
473
- The main extension point for the LLM entrypoint.
474
- Users can provide custom worker class through `worker_cls`
475
- argument, and implement new methods in the worker class.
476
- Then, users can call the new methods through this API.
477
- It is recommended to use this API to only pass control messages,
478
- and set up data-plane communication to pass data.
479
- The method can also be a callable, which will be serialized
480
- and sent to all workers to execute.
481
- If the method is a callable, it should accept an additional
482
- `self` argument, in addition to the arguments passed in `args`
483
- and `kwargs`. The `self` argument will be the worker object.
501
+ Run a function directly on the model inside each worker,
502
+ returning the result for each of them.
503
"""
485
- return self.llm_engine.collective_rpc(method, timeout, args, kwargs)
504
+ executor = self.llm_engine.model_executor
505
+ return executor.apply_model(func)
506
507
def beam_search(
508
self,
vllm/executor/executor_base.py
+41
-9
@@ -3,6 +3,9 @@ from abc import ABC, abstractmethod
3
from typing import (Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple,
4
Union)
5
6
+import torch.nn as nn
7
+from typing_extensions import TypeVar
8
+
9
from vllm.config import VllmConfig
10
from vllm.logger import init_logger
11
from vllm.lora.request import LoRARequest
@@ -11,9 +14,12 @@ from vllm.platforms import current_platform
14
from vllm.prompt_adapter.request import PromptAdapterRequest
15
from vllm.sequence import ExecuteModelRequest, PoolerOutput
16
from vllm.utils import make_async
17
+from vllm.worker.worker_base import WorkerBase
18
19
logger = init_logger(__name__)
20
21
+_R = TypeVar("_R", default=Any)
22
+
23
24
class ExecutorBase(ABC):
25
"""Base class for all executors.
@@ -44,22 +50,37 @@ class ExecutorBase(ABC):
50
51
@abstractmethod
52
def _init_executor(self) -> None:
47
- pass
53
+ raise NotImplementedError
54
55
@abstractmethod
56
def collective_rpc(self,
51
- method: Union[str, Callable],
57
+ method: Union[str, Callable[..., _R]],
58
timeout: Optional[float] = None,
59
args: Tuple = (),
54
- kwargs: Optional[Dict] = None) -> List[Any]:
60
+ kwargs: Optional[Dict[str, Any]] = None) -> List[_R]:
61
"""
56
- The main interface of the executor to run a method on all workers,
57
- with homogeneous arguments.
58
- If the args are heterogeneous, then we can pack them into a list,
59
- and unpack them in the method of every worker, because every worker
60
- knows their own rank.
62
+ Execute an RPC call on all workers.
63
+
64
+ Args:
65
+ method: Name of the worker method to execute, or a callable that
66
+ is serialized and sent to all workers to execute.
67
+
68
+ If the method is a callable, it should accept an additional
69
+ `self` argument, in addition to the arguments passed in `args`
70
+ and `kwargs`. The `self` argument will be the worker object.
71
+ timeout: Maximum time in seconds to wait for execution. Raises a
72
+ :exc:`TimeoutError` on timeout. `None` means wait indefinitely.
73
+ args: Positional arguments to pass to the worker method.
74
+ kwargs: Keyword arguments to pass to the worker method.
75
+
76
+ Returns:
77
+ A list containing the results from each worker.
78
+
79
+ Note:
80
+ It is recommended to use this API to only pass control messages,
81
+ and set up data-plane communication to pass data.
82
"""
62
- pass
83
+ raise NotImplementedError
84
85
def determine_num_available_blocks(self) -> Tuple[int, int]:
86
"""Determine the number of available blocks for the GPU KV cache and
@@ -97,6 +118,17 @@ class ExecutorBase(ABC):
118
self.collective_rpc("initialize_cache",
119
args=(num_gpu_blocks, num_cpu_blocks))
120
121
+ def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
122
+ """
123
+ Run a function directly on the model inside each worker,
124
+ returning the result for each of them.
125
+ """
126
+
127
+ def rpc_func(worker: WorkerBase) -> _R:
128
+ return func(worker.get_model())
129
+
130
+ return self.collective_rpc(rpc_func)
131
+
132
def execute_model(
133
self, execute_model_req: ExecuteModelRequest
134
) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:
vllm/executor/mp_distributed_executor.py
+1
-1
@@ -148,7 +148,7 @@ class MultiprocessingDistributedExecutor(DistributedExecutorBase):
148
async_run_tensor_parallel_workers_only: bool = False,
149
max_concurrent_workers: Optional[int] = None,
150
**kwargs,
151
- ) -> Any:
151
+ ) -> List[Any]:
152
"""Runs the given method on all workers.
153
154
Args:
vllm/model_executor/model_loader/tensorizer.py
+4
-13
@@ -459,16 +459,7 @@ def tensorize_vllm_model(engine_args: EngineArgs,
459
stream.write(encryption_params.key)
460
461
engine = LLMEngine.from_engine_args(engine_args)
462
- if tensorizer_config._is_sharded:
463
- # if the engine is a distributed engine (for tensor parallel) then each
464
- # worker shard needs to serialize its part of the model.
465
- engine.model_executor._run_workers(
466
- "save_tensorized_model",
467
- tensorizer_config=tensorizer_config,
468
- )
469
- else:
470
- # with a single worker, we can get to the underlying model directly
471
- serialize_vllm_model(
472
- engine.model_executor.driver_worker.model_runner.model,
473
- tensorizer_config,
474
- )
462
+ engine.model_executor.collective_rpc(
463
+ "save_tensorized_model",
464
+ kwargs=dict(tensorizer_config=tensorizer_config),
465
+ )
vllm/spec_decode/ngram_worker.py
+11
-1
@@ -2,6 +2,7 @@ import weakref
2
from typing import List, Optional, Set, Tuple
3
4
import torch
5
+import torch.nn as nn
6
7
from vllm.model_executor.layers.sampler import SamplerOutput
8
from vllm.sequence import ExecuteModelRequest
@@ -10,6 +11,10 @@ from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
11
from vllm.spec_decode.top1_proposer import Top1Proposer
12
13
14
+class _DummyModel(nn.Module):
15
+ pass
16
+
17
+
18
class NGramWorker(NonLLMProposerWorkerBase):
19
"""NGramWorker provides a light drafter without need for model.
20
@@ -36,7 +41,6 @@ class NGramWorker(NonLLMProposerWorkerBase):
41
42
def init_device(self):
43
self.device = torch.device(f"{self.device_type}:{self.local_rank}")
39
- self.load_model = lambda *args, **kwargs: None
44
45
# Current NGramWorker only supports Top1Proposer
46
self._proposer = Top1Proposer(
@@ -45,6 +49,12 @@ class NGramWorker(NonLLMProposerWorkerBase):
49
vocab_size=self.vocab_size,
50
)
51
52
+ def load_model(self) -> None:
53
+ pass # Dummy
54
+
55
+ def get_model(self) -> nn.Module:
56
+ return _DummyModel()
57
+
58
def sampler_output(
59
self,
60
execute_model_req: ExecuteModelRequest,
vllm/spec_decode/smaller_tp_proposer_worker.py
+12
@@ -1,6 +1,7 @@
1
from typing import List, Optional, Set, Tuple
2
3
import torch
4
+import torch.nn as nn
5
6
from vllm.distributed.parallel_state import (get_tp_group,
7
init_model_parallel_group,
@@ -15,6 +16,10 @@ from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
16
logger = init_logger(__name__)
17
18
19
+class _DummyModel(nn.Module):
20
+ pass
21
+
22
+
23
class SmallerTpProposerWorker(ProposerWorkerBase):
24
"""Class which allows a speculative draft model to run with smaller tensor
25
parallel degree than target model.
@@ -139,6 +144,13 @@ class SmallerTpProposerWorker(ProposerWorkerBase):
144
return self._worker.get_spec_proposals(
145
execute_model_req, seq_ids_with_bonus_token_in_last_step)
146
147
+ def get_model(self) -> nn.Module:
148
+ if self._is_dummy:
149
+ return _DummyModel()
150
+
151
+ with self._patch_tensor_parallel_group():
152
+ return self._worker.get_model()
153
+
154
def execute_model(
155
self,
156
execute_model_req: Optional[ExecuteModelRequest] = None
vllm/spec_decode/spec_decode_worker.py
+4
@@ -4,6 +4,7 @@ from functools import cached_property
4
from typing import Any, Dict, List, Optional, Set, Tuple, Type
5
6
import torch
7
+import torch.nn as nn
8
9
from vllm.config import ParallelConfig, SpeculativeConfig, VllmConfig
10
from vllm.distributed.communication_op import broadcast_tensor_dict
@@ -403,6 +404,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
404
self.proposer_worker.initialize_cache(num_gpu_blocks=num_gpu_blocks,
405
num_cpu_blocks=num_cpu_blocks)
406
407
+ def get_model(self) -> nn.Module:
408
+ return self.scorer_worker.get_model()
409
+
410
@torch.inference_mode()
411
def execute_model(
412
self,
vllm/v1/executor/multiproc_executor.py
+3
-13
@@ -94,22 +94,12 @@ class MultiprocExecutor(Executor):
94
timeout: Optional[float] = None,
95
args: Tuple = (),
96
kwargs: Optional[Dict] = None) -> List[Any]:
97
- """
98
- Execute an RPC call on workers.
99
-
100
- Args:
101
- method: Name of the worker method to execute
102
- timeout: Maximum time in seconds to wait for execution. Rases a
103
- TimeoutError on timeout. None means wait indefinitely.
104
- args: Positional arguments to pass to the worker method
105
- kwargs: Keyword arguments to pass to the worker method
106
-
107
- Returns:
108
- List of results from each worker
109
- """
97
start_time = time.monotonic()
98
kwargs = kwargs or {}
99
100
+ # NOTE: If the args are heterogeneous, then we pack them into a list,
101
+ # and unpack them in the method of every worker, because every worker
102
+ # knows their own rank.
103
try:
104
if isinstance(method, str):
105
send_method = method
vllm/v1/worker/gpu_model_runner.py
+3
@@ -689,6 +689,9 @@ class GPUModelRunner:
689
encoder_outputs.append(encoder_output[start_idx:end_idx])
690
return encoder_outputs
691
692
+ def get_model(self) -> nn.Module:
693
+ return self.model
694
+
695
@torch.inference_mode()
696
def execute_model(
697
self,
vllm/v1/worker/gpu_worker.py
+4
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Optional
5
6
import torch
7
import torch.distributed
8
+import torch.nn as nn
9
10
import vllm.envs as envs
11
from vllm.config import CacheConfig, ModelConfig, ParallelConfig, VllmConfig
@@ -176,6 +177,9 @@ class Worker:
177
# the model initialization and profiling.
178
set_random_seed(self.model_config.seed)
179
180
+ def get_model(self) -> nn.Module:
181
+ return self.model_runner.get_model()
182
+
183
@torch.inference_mode()
184
def execute_model(
185
self,
vllm/worker/cpu_model_runner.py
+3
@@ -509,6 +509,9 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
509
)
510
self.model = self.lora_manager.create_lora_manager(self.model)
511
512
+ def get_model(self) -> nn.Module:
513
+ return self.model
514
+
515
def _prepare_model_input_tensors(
516
self,
517
seq_group_metadata_list: List[SequenceGroupMetadata],
vllm/worker/hpu_model_runner.py
+4
@@ -21,6 +21,7 @@ from typing import (TYPE_CHECKING, Any, Callable, Dict, List, NamedTuple,
21
import habana_frameworks.torch as htorch
22
import habana_frameworks.torch.internal.bridge_config as bc
23
import torch
24
+import torch.nn as nn
25
from vllm_hpu_extension.ops import LoraMask as LoraMask
26
from vllm_hpu_extension.profiler import (HabanaHighLevelProfiler,
27
HabanaMemoryProfiler, format_bytes)
@@ -676,6 +677,9 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
677
msg = f"Loading model weights took in total {m.get_summary_string()}"
678
logger.info(msg)
679
680
+ def get_model(self) -> nn.Module:
681
+ return self.model
682
+
683
def _use_graphs(self, batch_size, seq_len, is_prompt):
684
if self.enforce_eager:
685
return False
vllm/worker/model_runner.py
+3
@@ -1176,6 +1176,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1176
fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,
1177
backend=backend)
1178
1179
+ def get_model(self) -> nn.Module:
1180
+ return self.model
1181
+
1182
def save_sharded_state(
1183
self,
1184
path: str,
vllm/worker/model_runner_base.py
+7
-2
@@ -7,6 +7,7 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List,
7
Optional, Type, TypeVar)
8
9
import torch
10
+import torch.nn as nn
11
from torch import is_tensor
12
13
from vllm.config import VllmConfig
@@ -264,6 +265,10 @@ class ModelRunnerBase(ABC, Generic[T]):
265
"""
266
raise NotImplementedError
267
268
+ @abstractmethod
269
+ def get_model(self) -> nn.Module:
270
+ raise NotImplementedError
271
+
272
def execute_model(
273
self,
274
model_input: T,
@@ -297,9 +302,9 @@ class ModelRunnerWrapperBase:
302
303
def __init__(
304
self,
300
- moderl_runner: ModelRunnerBase,
305
+ model_runner: ModelRunnerBase,
306
) -> None:
302
- self.model_runner: ModelRunnerBase = moderl_runner
307
+ self.model_runner: ModelRunnerBase = model_runner
308
309
def __getattr__(self, attr):
310
return getattr(self.model_runner, attr)
vllm/worker/neuron_model_runner.py
+3
@@ -113,6 +113,9 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
113
raise NotImplementedError(
114
"Supports only Transformer-NeuronX based models.")
115
116
+ def get_model(self) -> nn.Module:
117
+ return self.model
118
+
119
def _prepare_prompt(
120
self,
121
seq_group_metadata_list: List[SequenceGroupMetadata],
vllm/worker/openvino_model_runner.py
+3
@@ -84,6 +84,9 @@ class OpenVINOModelRunner(ModelRunnerBase):
84
kv_cache_dtype=self.kv_cache_dtype,
85
ov_core=self.ov_core)
86
87
+ def get_model(self) -> nn.Module:
88
+ return self.model
89
+
90
def _prepare_model_input(
91
self,
92
seq_group_metadata_list: List[SequenceGroupMetadata],
vllm/worker/openvino_worker.py
+4
@@ -4,6 +4,7 @@ from typing import Any, Dict, List, Optional, Tuple
4
import openvino as ov
5
import torch
6
import torch.distributed
7
+import torch.nn as nn
8
9
import vllm.envs as envs
10
from vllm.attention import get_attn_backend
@@ -362,6 +363,9 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
363
) -> None:
364
self.cache_engine.copy(blocks_to_copy) # type: ignore
365
366
+ def get_model(self) -> nn.Module:
367
+ return self.model_runner.get_model()
368
+
369
@torch.inference_mode()
370
def execute_model(
371
self,
vllm/worker/tpu_model_runner.py
+3
@@ -158,6 +158,9 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
158
fullgraph=True,
159
dynamic=False)
160
161
+ def get_model(self) -> nn.Module:
162
+ return self.model.model
163
+
164
def _dummy_run(
165
self,
166
batch_size: int,
vllm/worker/worker_base.py
+12
@@ -6,6 +6,7 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
6
7
import cloudpickle
8
import torch
9
+import torch.nn as nn
10
11
from vllm.config import ObservabilityConfig, VllmConfig
12
from vllm.distributed import broadcast_tensor_dict, get_pp_group, get_tp_group
@@ -90,6 +91,11 @@ class WorkerBase(ABC):
91
if output is None:
92
return None
93
94
+ @abstractmethod
95
+ def get_model(self) -> nn.Module:
96
+ raise NotImplementedError
97
+
98
+ @abstractmethod
99
def execute_model(
100
self,
101
execute_model_req: Optional[ExecuteModelRequest] = None
@@ -147,6 +153,9 @@ class DelegateWorkerBase(WorkerBase):
153
num_cpu_blocks: int) -> None:
154
self.worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
155
156
+ def get_model(self) -> nn.Module:
157
+ return self.worker.get_model()
158
+
159
def execute_model(
160
self,
161
execute_model_req: Optional[ExecuteModelRequest] = None
@@ -363,6 +372,9 @@ class LocalOrDistributedWorkerBase(WorkerBase):
372
else:
373
return self._get_worker_input_from_broadcast()
374
375
+ def get_model(self) -> nn.Module:
376
+ return self.model_runner.get_model()
377
+
378
def execute_model(
379
self,
380
execute_model_req: Optional[ExecuteModelRequest] = None,
vllm/worker/xpu_model_runner.py
+3
@@ -416,6 +416,9 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
416
logger.info("Loading model weights took %.4f GB",
417
self.model_memory_usage / float(2**30))
418
419
+ def get_model(self) -> nn.Module:
420
+ return self.model
421
+
422
@property
423
def vocab_size(self) -> int:
424
return self.model_config.get_vocab_size()