@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 59a0192fb9
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Core] Interface for accessing model from `VllmRunner` (#10353)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

Cyrus Leung committed Jan 20, 2025 at 15:00 UTC 59a0192fb9bef026086d0a2ed32705d870a9466a
35 files changed +474 -307
tests/conftest.py
+5
@@ -244,6 +244,7 @@ def video_assets() -> _VideoAssets:
244
245
246 _T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature, dict)
247 +_R = TypeVar("_R")
248
249
250 class HfRunner:
@@ -930,6 +931,10 @@ class VllmRunner:
931 req_outputs = self.model.score(text_1, text_2)
932 return [req_output.outputs.score for req_output in req_outputs]
933
934 + def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
935 + executor = self.model.llm_engine.model_executor
936 + return executor.apply_model(func)
937 +
938 def __enter__(self):
939 return self
940
tests/engine/test_custom_executor.py
+3 -1
@@ -51,7 +51,9 @@ def test_custom_executor(model, tmp_path):
51 assert not os.path.exists(".marker")
52
53 engine_args = EngineArgs(
54 - model=model, distributed_executor_backend=CustomUniExecutor)
54 + model=model,
55 + distributed_executor_backend=CustomUniExecutor,
56 + )
57 engine = LLMEngine.from_engine_args(engine_args)
58 sampling_params = SamplingParams(max_tokens=1)
59
tests/model_executor/test_model_load_with_params.py
+33 -31
@@ -25,13 +25,12 @@ def test_model_loading_with_params(vllm_runner):
25 with vllm_runner(model_name=MODEL_NAME,
26 revision=REVISION,
27 dtype="float16",
28 - max_model_len=MAX_MODEL_LEN) as model:
29 - output = model.encode("Write a short story about a robot that"
30 - " dreams for the first time.\n")
28 + max_model_len=MAX_MODEL_LEN) as vllm_model:
29 + output = vllm_model.encode("Write a short story about a robot that"
30 + " dreams for the first time.\n")
31
32 - model_config = model.model.llm_engine.model_config
33 -
34 - model_tokenizer = model.model.llm_engine.tokenizer
32 + model_config = vllm_model.model.llm_engine.model_config
33 + model_tokenizer = vllm_model.model.llm_engine.tokenizer
34
35 # asserts on the bert model config file
36 assert model_config.encoder_config["max_seq_length"] == 512
@@ -46,11 +45,13 @@ def test_model_loading_with_params(vllm_runner):
45 assert model_tokenizer.tokenizer_config["do_lower_case"]
46 assert model_tokenizer.tokenizer.model_max_length == 512
47
49 - model = model.model.llm_engine.model_executor\
50 - .driver_worker.model_runner.model
51 - assert isinstance(model, BertEmbeddingModel)
52 - assert model._pooler.pooling_type == PoolingType.CLS
53 - assert model._pooler.normalize
48 + def check_model(model):
49 + assert isinstance(model, BertEmbeddingModel)
50 + assert model._pooler.pooling_type == PoolingType.CLS
51 + assert model._pooler.normalize
52 +
53 + vllm_model.apply_model(check_model)
54 +
55 # assert output
56 assert output
57
@@ -64,13 +65,12 @@ def test_roberta_model_loading_with_params(vllm_runner):
65 with vllm_runner(model_name=MODEL_NAME_ROBERTA,
66 revision=REVISION_ROBERTA,
67 dtype="float16",
67 - max_model_len=MAX_MODEL_LEN) as model:
68 - output = model.encode("Write a short story about a robot that"
69 - " dreams for the first time.\n")
68 + max_model_len=MAX_MODEL_LEN) as vllm_model:
69 + output = vllm_model.encode("Write a short story about a robot that"
70 + " dreams for the first time.\n")
71
71 - model_config = model.model.llm_engine.model_config
72 -
73 - model_tokenizer = model.model.llm_engine.tokenizer
72 + model_config = vllm_model.model.llm_engine.model_config
73 + model_tokenizer = vllm_model.model.llm_engine.tokenizer
74
75 # asserts on the bert model config file
76 assert model_config.encoder_config["max_seq_length"] == 512
@@ -84,11 +84,12 @@ def test_roberta_model_loading_with_params(vllm_runner):
84 assert model_tokenizer.tokenizer_id == "intfloat/multilingual-e5-large"
85 assert not model_tokenizer.tokenizer_config["do_lower_case"]
86
87 - model = model.model.llm_engine.model_executor\
88 - .driver_worker.model_runner.model
89 - assert isinstance(model, RobertaEmbeddingModel)
90 - assert model._pooler.pooling_type == PoolingType.MEAN
91 - assert model._pooler.normalize
87 + def check_model(model):
88 + assert isinstance(model, RobertaEmbeddingModel)
89 + assert model._pooler.pooling_type == PoolingType.MEAN
90 + assert model._pooler.normalize
91 +
92 + vllm_model.apply_model(check_model)
93
94 # assert output
95 assert output
@@ -103,17 +104,18 @@ def test_facebook_roberta_model_loading_with_params(vllm_runner):
104 model_name = "FacebookAI/roberta-base"
105 with vllm_runner(model_name=model_name,
106 dtype="float16",
106 - max_model_len=MAX_MODEL_LEN) as model:
107 - output = model.encode("Write a short story about a robot that"
108 - " dreams for the first time.\n")
107 + max_model_len=MAX_MODEL_LEN) as vllm_model:
108 + output = vllm_model.encode("Write a short story about a robot that"
109 + " dreams for the first time.\n")
110
110 - model_tokenizer = model.model.llm_engine.tokenizer
111 + model_tokenizer = vllm_model.model.llm_engine.tokenizer
112 assert model_tokenizer.tokenizer_id == model_name
113
113 - model = model.model.llm_engine.model_executor\
114 - .driver_worker.model_runner.model
115 - assert not hasattr(model, "lm_head")
116 - assert isinstance(model, RobertaEmbeddingModel)
117 - assert isinstance(model._pooler, CLSPool)
114 + def check_model(model):
115 + assert isinstance(model, RobertaEmbeddingModel)
116 + assert not hasattr(model, "lm_head")
117 + assert isinstance(model._pooler, CLSPool)
118 +
119 + vllm_model.apply_model(check_model)
120
121 assert output
tests/models/decoder_only/language/test_jamba.py
+5 -2
@@ -33,10 +33,13 @@ def test_models(
33
34 with vllm_runner(model, dtype=dtype) as vllm_model:
35 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
36 +
37 # This test is for verifying whether the model's extra_repr
38 # can be printed correctly.
38 - print(vllm_model.model.llm_engine.model_executor.driver_worker.
39 - model_runner.model)
39 + def print_model(model):
40 + print(model)
41 +
42 + vllm_model.apply_model(print_model)
43
44 for i in range(len(example_prompts)):
45 hf_output_ids, hf_output_str = hf_outputs[i]
tests/models/decoder_only/language/test_mamba.py
+5 -2
@@ -51,10 +51,13 @@ def test_models(
51
52 with vllm_runner(model, dtype=dtype) as vllm_model:
53 vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
54 +
55 # This test is for verifying whether the model's extra_repr
56 # can be printed correctly.
56 - print(vllm_model.model.llm_engine.model_executor.driver_worker.
57 - model_runner.model)
57 + def print_model(model):
58 + print(model)
59 +
60 + vllm_model.apply_model(print_model)
61
62 for i in range(len(example_prompts)):
63 hf_output_ids, hf_output_str = hf_outputs[i]
tests/models/decoder_only/language/test_models.py
+5 -2
@@ -73,10 +73,13 @@ def test_models(
73 with vllm_runner(model, dtype=dtype) as vllm_model:
74 vllm_outputs = vllm_model.generate_greedy_logprobs(
75 example_prompts, max_tokens, num_logprobs)
76 +
77 # This test is for verifying whether the model's extra_repr
78 # can be printed correctly.
78 - print(vllm_model.model.llm_engine.model_executor.driver_worker.
79 - model_runner.model)
79 + def print_model(model):
80 + print(model)
81 +
82 + vllm_model.apply_model(print_model)
83
84 check_logprobs_close(
85 outputs_0_lst=hf_outputs,
tests/models/decoder_only/vision_language/test_qwen2_vl.py
+26 -23
@@ -5,7 +5,6 @@ import pytest
5 import torch
6 from PIL import Image
7
8 -from vllm.entrypoints.llm import LLM
8 from vllm.multimodal.image import rescale_image_size
9 from vllm.multimodal.video import rescale_video_size, sample_frames_from_video
10
@@ -69,7 +68,7 @@ class Qwen2VLPromptVideoEmbeddingInput(TypedDict):
68
69 def batch_make_image_embeddings(
70 image_batches: List[Union[Image.Image, List[Image.Image]]], processor,
72 - llm: LLM) -> List[Qwen2VLPromptImageEmbeddingInput]:
71 + llm: VllmRunner) -> List[Qwen2VLPromptImageEmbeddingInput]:
72 """batched image embeddings for Qwen2-VL
73
74 This will infer all images' embeddings in a single batch,
@@ -106,16 +105,18 @@ def batch_make_image_embeddings(
105 image_grid_thw = preprocess_result["image_grid_thw"]
106
107 # pixel values to embeddings & grid_thws
109 - with torch.no_grad():
110 - visual = llm.llm_engine.model_executor.driver_worker. \
111 - model_runner.model.visual
108 + def get_image_embeds(model):
109 + with torch.no_grad():
110 + visual = model.visual
111
113 - pixel_values_on_device = pixel_values.to(visual.device,
114 - dtype=visual.dtype)
115 - image_grid_thw_on_device = image_grid_thw.to(visual.device,
116 - dtype=torch.int64)
117 - image_embeds = visual(pixel_values_on_device,
118 - grid_thw=image_grid_thw_on_device)
112 + pixel_values_on_device = pixel_values.to(visual.device,
113 + dtype=visual.dtype)
114 + image_grid_thw_on_device = image_grid_thw.to(visual.device,
115 + dtype=torch.int64)
116 + return visual(pixel_values_on_device,
117 + grid_thw=image_grid_thw_on_device)
118 +
119 + image_embeds = torch.concat(llm.apply_model(get_image_embeds))
120
121 # split into original batches
122 result: List[Qwen2VLPromptImageEmbeddingInput] = []
@@ -150,7 +151,7 @@ def batch_make_image_embeddings(
151
152 def batch_make_video_embeddings(
153 video_batches: PromptVideoInput, processor,
153 - llm: LLM) -> List[Qwen2VLPromptVideoEmbeddingInput]:
154 + llm: VllmRunner) -> List[Qwen2VLPromptVideoEmbeddingInput]:
155 """batched video embeddings for Qwen2-VL
156
157 A NDArray represents a single video's all frames.
@@ -187,16 +188,18 @@ def batch_make_video_embeddings(
188 video_grid_thw = preprocess_result["video_grid_thw"]
189
190 # pixel values to embeddings & grid_thws
190 - with torch.no_grad():
191 - visual = llm.llm_engine.model_executor.driver_worker.\
192 - model_runner.model.visual
191 + def get_image_embeds(model):
192 + with torch.no_grad():
193 + visual = model.visual
194 +
195 + pixel_values_on_device = pixel_values.to(visual.device,
196 + dtype=visual.dtype)
197 + video_grid_thw_on_device = video_grid_thw.to(visual.device,
198 + dtype=torch.int64)
199 + return visual(pixel_values_on_device,
200 + grid_thw=video_grid_thw_on_device)
201
194 - pixel_values_on_device = pixel_values.to(visual.device,
195 - dtype=visual.dtype)
196 - video_grid_thw_on_device = video_grid_thw.to(visual.device,
197 - dtype=torch.int64)
198 - video_embeds = visual(pixel_values_on_device,
199 - grid_thw=video_grid_thw_on_device)
202 + video_embeds = torch.concat(llm.apply_model(get_image_embeds))
203
204 # split into original batches
205 result: List[Qwen2VLPromptVideoEmbeddingInput] = []
@@ -278,9 +281,9 @@ def run_embedding_input_test(
281 max_tokens,
282 num_logprobs=num_logprobs,
283 images=batch_make_image_embeddings(
281 - images, processor, vllm_model.model) if images else None,
284 + images, processor, vllm_model) if images else None,
285 videos=batch_make_video_embeddings(
283 - videos, processor, vllm_model.model) if videos else None)
286 + videos, processor, vllm_model) if videos else None)
287 for prompts, images, videos in inputs
288 ]
289
tests/models/embedding/language/test_cls_models.py
+5 -2
@@ -24,10 +24,13 @@ def test_classification_models(
24 ) -> None:
25 with vllm_runner(model, dtype=dtype) as vllm_model:
26 vllm_outputs = vllm_model.classify(example_prompts)
27 +
28 # This test is for verifying whether the model's extra_repr
29 # can be printed correctly.
29 - print(vllm_model.model.llm_engine.model_executor.driver_worker.
30 - model_runner.model)
30 + def print_model(model):
31 + print(model)
32 +
33 + vllm_model.apply_model(print_model)
34
35 with hf_runner(model,
36 dtype=dtype,
tests/models/embedding/language/test_embedding.py
+5 -2
@@ -62,10 +62,13 @@ def test_models(
62 max_model_len=None,
63 **vllm_extra_kwargs) as vllm_model:
64 vllm_outputs = vllm_model.encode(example_prompts)
65 +
66 # This test is for verifying whether the model's extra_repr
67 # can be printed correctly.
67 - print(vllm_model.model.llm_engine.model_executor.driver_worker.
68 - model_runner.model)
68 + def print_model(model):
69 + print(model)
70 +
71 + vllm_model.apply_model(print_model)
72
73 check_embeddings_close(
74 embeddings_0_lst=hf_outputs,
tests/quantization/test_compressed_tensors.py
+137 -105
@@ -30,50 +30,55 @@ from vllm.platforms import current_platform
30 def test_compressed_tensors_w8a8_static_setup(vllm_runner, model_args):
31 model_path, strategy, quant_type, shape_0, is_symmetric = model_args
32 with vllm_runner(model_path, enforce_eager=True) as llm:
33 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
34 - layer = model.model.layers[0]
35 -
36 - qkv_proj = layer.self_attn.qkv_proj
37 - o_proj = layer.self_attn.o_proj
38 - gate_up_proj = layer.mlp.gate_up_proj
39 - down_proj = layer.mlp.down_proj
40 -
41 - # assert zp for symmetric and asymmetric cases
42 - def zp_valid(zp: Optional[torch.Tensor]):
43 - if is_symmetric:
44 - return zp is None
45 -
46 - return zp is not None and zp.dtype is torch.int32
47 -
48 - assert zp_valid(qkv_proj.input_zero_point)
49 - assert zp_valid(o_proj.input_zero_point)
50 - assert zp_valid(gate_up_proj.input_zero_point)
51 - assert zp_valid(down_proj.input_zero_point)
52 -
53 - assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
54 - assert isinstance(o_proj.quant_method, CompressedTensorsLinearMethod)
55 - assert isinstance(gate_up_proj.quant_method,
56 - CompressedTensorsLinearMethod)
57 - assert isinstance(down_proj.quant_method,
58 - CompressedTensorsLinearMethod)
59 - assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Int8)
60 -
61 - assert qkv_proj.scheme.strategy == strategy
62 - assert qkv_proj.scheme.is_static_input_scheme
63 - expected_type = torch.int8
64 -
65 - assert qkv_proj.weight.dtype is expected_type
66 - assert o_proj.weight.dtype is expected_type
67 - assert gate_up_proj.weight.dtype is expected_type
68 -
69 - if qkv_proj.scheme.strategy == "tensor":
70 - # Make sure it is a channelwise buffer
71 - # After running process_weights_after_loading
72 - assert len(qkv_proj.weight_scale.shape) == 2
73 - assert qkv_proj.weight_scale.shape[0] == shape_0
74 - assert qkv_proj.weight_scale.shape[1] == 1
75 - assert qkv_proj.weight_scale.dtype is torch.float32
76 - assert qkv_proj.input_scale.dtype is torch.float32
33 +
34 + def check_model(model):
35 + layer = model.model.layers[0]
36 +
37 + qkv_proj = layer.self_attn.qkv_proj
38 + o_proj = layer.self_attn.o_proj
39 + gate_up_proj = layer.mlp.gate_up_proj
40 + down_proj = layer.mlp.down_proj
41 +
42 + # assert zp for symmetric and asymmetric cases
43 + def zp_valid(zp: Optional[torch.Tensor]):
44 + if is_symmetric:
45 + return zp is None
46 +
47 + return zp is not None and zp.dtype is torch.int32
48 +
49 + assert zp_valid(qkv_proj.input_zero_point)
50 + assert zp_valid(o_proj.input_zero_point)
51 + assert zp_valid(gate_up_proj.input_zero_point)
52 + assert zp_valid(down_proj.input_zero_point)
53 +
54 + assert isinstance(qkv_proj.quant_method,
55 + CompressedTensorsLinearMethod)
56 + assert isinstance(o_proj.quant_method,
57 + CompressedTensorsLinearMethod)
58 + assert isinstance(gate_up_proj.quant_method,
59 + CompressedTensorsLinearMethod)
60 + assert isinstance(down_proj.quant_method,
61 + CompressedTensorsLinearMethod)
62 + assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Int8)
63 +
64 + assert qkv_proj.scheme.strategy == strategy
65 + assert qkv_proj.scheme.is_static_input_scheme
66 + expected_type = torch.int8
67 +
68 + assert qkv_proj.weight.dtype is expected_type
69 + assert o_proj.weight.dtype is expected_type
70 + assert gate_up_proj.weight.dtype is expected_type
71 +
72 + if qkv_proj.scheme.strategy == "tensor":
73 + # Make sure it is a channelwise buffer
74 + # After running process_weights_after_loading
75 + assert len(qkv_proj.weight_scale.shape) == 2
76 + assert qkv_proj.weight_scale.shape[0] == shape_0
77 + assert qkv_proj.weight_scale.shape[1] == 1
78 + assert qkv_proj.weight_scale.dtype is torch.float32
79 + assert qkv_proj.input_scale.dtype is torch.float32
80 +
81 + llm.apply_model(check_model)
82
83 output = llm.generate_greedy(["Hello my name is"], max_tokens=20)
84 assert output
@@ -129,16 +134,20 @@ def test_compressed_tensors_no_enforce_eager(vllm_runner):
134 def test_compressed_tensors_w8a8_dynamic_per_token(vllm_runner, model_args):
135 model_path, strategy = model_args
136 with vllm_runner(model_path, dtype=torch.float16) as llm:
132 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
133 - layer = model.model.layers[0]
137
135 - qkv_proj = layer.self_attn.qkv_proj
138 + def check_model(model):
139 + layer = model.model.layers[0]
140 +
141 + qkv_proj = layer.self_attn.qkv_proj
142 +
143 + assert isinstance(qkv_proj.quant_method,
144 + CompressedTensorsLinearMethod)
145 + assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Int8)
146 + assert not qkv_proj.scheme.is_static_input_scheme
147 + assert qkv_proj.scheme.strategy == strategy
148 + assert qkv_proj.weight.dtype is torch.int8
149
137 - assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
138 - assert isinstance(qkv_proj.scheme, CompressedTensorsW8A8Int8)
139 - assert not qkv_proj.scheme.is_static_input_scheme
140 - assert qkv_proj.scheme.strategy == strategy
141 - assert qkv_proj.weight.dtype is torch.int8
150 + llm.apply_model(check_model)
151
152 output = llm.generate_greedy(["Hello my name is"], max_tokens=20)
153 assert output
@@ -152,19 +161,24 @@ def test_compressed_tensors_w8a8_dynamic_per_token(vllm_runner, model_args):
161 def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
162 model, strategy, group, pack_factor = wNa16_args
163 with vllm_runner(model) as llm:
155 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
156 - layer = model.model.layers[0]
164
158 - qkv_proj = layer.self_attn.qkv_proj
159 - assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
160 - assert isinstance(qkv_proj.scheme, CompressedTensorsWNA16)
165 + def check_model(model):
166 + layer = model.model.layers[0]
167
162 - assert qkv_proj.scheme.strategy == strategy
163 - assert qkv_proj.scheme.group_size == (-1 if group is None else group)
168 + qkv_proj = layer.self_attn.qkv_proj
169 + assert isinstance(qkv_proj.quant_method,
170 + CompressedTensorsLinearMethod)
171 + assert isinstance(qkv_proj.scheme, CompressedTensorsWNA16)
172
165 - assert qkv_proj.weight_packed.dtype is torch.int32
166 - assert qkv_proj.weight_scale.dtype is torch.float16
167 - assert qkv_proj.scheme.pack_factor == pack_factor
173 + assert qkv_proj.scheme.strategy == strategy
174 + assert qkv_proj.scheme.group_size == (-1
175 + if group is None else group)
176 +
177 + assert qkv_proj.weight_packed.dtype is torch.int32
178 + assert qkv_proj.weight_scale.dtype is torch.float16
179 + assert qkv_proj.scheme.pack_factor == pack_factor
180 +
181 + llm.apply_model(check_model)
182
183 output = llm.generate_greedy("Hello my name is", max_tokens=20)
184 assert output
@@ -173,14 +187,18 @@ def test_compressed_tensors_wNa16(vllm_runner, wNa16_args):
187 def test_compressed_tensors_w4a16_marlin24(vllm_runner):
188 model_path = "nm-testing/llama7b-one-shot-2_4-w4a16-marlin24-t"
189 with vllm_runner(model_path) as llm:
176 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
177 - layer = model.model.layers[0]
190
179 - qkv_proj = layer.self_attn.qkv_proj
191 + def check_model(model):
192 + layer = model.model.layers[0]
193 +
194 + qkv_proj = layer.self_attn.qkv_proj
195
181 - assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
182 - assert isinstance(qkv_proj.scheme, CompressedTensorsW4A16Sparse24)
183 - assert qkv_proj.weight_packed.dtype is torch.int32
196 + assert isinstance(qkv_proj.quant_method,
197 + CompressedTensorsLinearMethod)
198 + assert isinstance(qkv_proj.scheme, CompressedTensorsW4A16Sparse24)
199 + assert qkv_proj.weight_packed.dtype is torch.int32
200 +
201 + llm.apply_model(check_model)
202
203 output = llm.generate_greedy("Hello my name is", max_tokens=20)
204 assert output
@@ -189,23 +207,27 @@ def test_compressed_tensors_w4a16_marlin24(vllm_runner):
207 def test_compressed_tensors_fp8(vllm_runner):
208 model_path = "nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test"
209 with vllm_runner(model_path) as llm:
192 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
193 - layer = model.model.layers[0]
210
195 - qkv_proj = layer.self_attn.qkv_proj
211 + def check_model(model):
212 + layer = model.model.layers[0]
213
197 - assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
198 - assert isinstance(
199 - qkv_proj.scheme,
200 - (CompressedTensorsW8A8Fp8, CompressedTensorsW8A16Fp8))
214 + qkv_proj = layer.self_attn.qkv_proj
215
202 - assert qkv_proj.input_scale.dtype is torch.float32
216 + assert isinstance(qkv_proj.quant_method,
217 + CompressedTensorsLinearMethod)
218 + assert isinstance(
219 + qkv_proj.scheme,
220 + (CompressedTensorsW8A8Fp8, CompressedTensorsW8A16Fp8))
221
204 - if isinstance(qkv_proj.scheme, CompressedTensorsW8A8Fp8):
205 - assert len(qkv_proj.input_scale.shape) == 0
206 - assert qkv_proj.weight.dtype is torch.float8_e4m3fn
207 - assert qkv_proj.weight_scale.dtype is torch.float32
208 - assert len(qkv_proj.weight_scale.shape) == 0
222 + assert qkv_proj.input_scale.dtype is torch.float32
223 +
224 + if isinstance(qkv_proj.scheme, CompressedTensorsW8A8Fp8):
225 + assert len(qkv_proj.input_scale.shape) == 0
226 + assert qkv_proj.weight.dtype is torch.float8_e4m3fn
227 + assert qkv_proj.weight_scale.dtype is torch.float32
228 + assert len(qkv_proj.weight_scale.shape) == 0
229 +
230 + llm.apply_model(check_model)
231
232 output = llm.generate_greedy("Hello my name is", max_tokens=20)
233 assert output
@@ -248,12 +270,15 @@ def _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy):
270 def test_compressed_tensors_2of4_quant_fp8(vllm_runner, args_2of4):
271 model, weight_strategy, input_strategy = args_2of4
272 with vllm_runner(model) as llm:
251 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
252 - layer = model.model.layers[0]
273
254 - qkv_proj = layer.self_attn.qkv_proj
255 - assert qkv_proj.scheme.weights_dtype == torch.float8_e4m3fn
256 - _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy)
274 + def check_model(model):
275 + layer = model.model.layers[0]
276 +
277 + qkv_proj = layer.self_attn.qkv_proj
278 + assert qkv_proj.scheme.weights_dtype == torch.float8_e4m3fn
279 + _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy)
280 +
281 + llm.apply_model(check_model)
282
283 output = llm.generate_greedy("Hello my name is", max_tokens=20)
284 print(output)
@@ -273,12 +298,15 @@ def test_compressed_tensors_2of4_quant_fp8(vllm_runner, args_2of4):
298 def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):
299 model, weight_strategy, input_strategy = args_2of4
300 with vllm_runner(model) as llm:
276 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
277 - layer = model.model.layers[0]
301
279 - qkv_proj = layer.self_attn.qkv_proj
280 - assert qkv_proj.scheme.weights_dtype == torch.int8
281 - _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy)
302 + def check_model(model):
303 + layer = model.model.layers[0]
304 +
305 + qkv_proj = layer.self_attn.qkv_proj
306 + assert qkv_proj.scheme.weights_dtype == torch.int8
307 + _test_2of4_quant_models(qkv_proj, weight_strategy, input_strategy)
308 +
309 + llm.apply_model(check_model)
310
311 output = llm.generate_greedy("Hello my name is", max_tokens=20)
312 print(output)
@@ -293,20 +321,24 @@ def test_compressed_tensors_2of4_quant_int8(vllm_runner, args_2of4):
321 def test_compressed_tensors_2of4_sparse(vllm_runner, args_2of4):
322 model = args_2of4
323 with vllm_runner(model) as llm:
296 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
297 - layer = model.model.layers[0]
298 -
299 - qkv_proj = layer.self_attn.qkv_proj
300 - assert isinstance(qkv_proj.quant_method, CompressedTensorsLinearMethod)
301 - assert isinstance(qkv_proj.scheme, CompressedTensors24)
302 -
303 - assert qkv_proj.scheme.weight_quant is None
304 - assert qkv_proj.scheme.input_quant is None
305 - assert not qkv_proj.scheme.quantized
306 - assert qkv_proj.quant_method.quantization_config.sparsity_scheme_map
307 - sparsity_map = qkv_proj.quant_method.quantization_config.sparsity_scheme_map # noqa: E501
308 - assert sparsity_map.get("Linear").format == "dense"
309 - assert sparsity_map.get("Linear").sparsity_structure == "2:4"
324 +
325 + def check_model(model):
326 + layer = model.model.layers[0]
327 +
328 + qkv_proj = layer.self_attn.qkv_proj
329 + assert isinstance(qkv_proj.quant_method,
330 + CompressedTensorsLinearMethod)
331 + assert isinstance(qkv_proj.scheme, CompressedTensors24)
332 +
333 + assert qkv_proj.scheme.weight_quant is None
334 + assert qkv_proj.scheme.input_quant is None
335 + assert not qkv_proj.scheme.quantized
336 + assert qkv_proj.quant_method.quantization_config.sparsity_scheme_map
337 + sparsity_map = qkv_proj.quant_method.quantization_config.sparsity_scheme_map # noqa: E501
338 + assert sparsity_map.get("Linear").format == "dense"
339 + assert sparsity_map.get("Linear").sparsity_structure == "2:4"
340 +
341 + llm.apply_model(check_model)
342
343 output = llm.generate_greedy("Hello my name is", max_tokens=20)
344 print(output)
tests/quantization/test_fp8.py
+29 -23
@@ -49,13 +49,17 @@ KV_CACHE_MODELS = [
49 def test_kv_cache_model_load_and_run(vllm_runner, model_id: str):
50 with vllm_runner(model_id, kv_cache_dtype="fp8") as llm:
51
52 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
53 - attn = model.model.layers[0].self_attn.attn
54 - assert isinstance(attn.quant_method, Fp8KVCacheMethod)
55 - # NOTE: it is valid for scales to be 1.0 (default value), but we know
56 - # these checkpoints have scales < 1.0
57 - assert 0.0 < attn._k_scale < 1.0
58 - assert 0.0 < attn._v_scale < 1.0
52 + def check_model(model):
53 + attn = model.model.layers[0].self_attn.attn
54 +
55 + assert isinstance(attn.quant_method, Fp8KVCacheMethod)
56 +
57 + # NOTE: it is valid for scales to be 1.0 (default value), but
58 + # we know these checkpoints have scales < 1.0
59 + assert 0.0 < attn._k_scale < 1.0
60 + assert 0.0 < attn._v_scale < 1.0
61 +
62 + llm.apply_model(check_model)
63
64 # note: this does not test accuracy, just that we can run through
65 # see lm-eval tests for accuracy
@@ -77,22 +81,24 @@ def test_load_fp16_model(vllm_runner, kv_cache_dtype: str, force_marlin: bool,
81 quantization="fp8",
82 kv_cache_dtype=kv_cache_dtype) as llm:
83
80 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
81 - fc1 = model.model.decoder.layers[0].fc1
82 - assert isinstance(fc1.quant_method, Fp8LinearMethod)
83 - if kv_cache_dtype == "fp8":
84 - attn = model.model.decoder.layers[0].self_attn.attn
85 - assert isinstance(attn.quant_method, Fp8KVCacheMethod)
86 - assert attn._k_scale == 1.0
87 - assert attn._v_scale == 1.0
88 -
89 - if current_platform.has_device_capability(89) and not force_marlin:
90 - # For GPUs with hardware support, we keep weights in fp8
91 - assert fc1.weight.dtype == torch.float8_e4m3fn
92 - else:
93 - # For GPUs without hardware support, we pack the fp8 weights
94 - # for weight-only quantization using Marlin kernels
95 - assert fc1.weight.dtype == torch.int32
84 + def check_model(model):
85 + fc1 = model.model.decoder.layers[0].fc1
86 + assert isinstance(fc1.quant_method, Fp8LinearMethod)
87 + if kv_cache_dtype == "fp8":
88 + attn = model.model.decoder.layers[0].self_attn.attn
89 + assert isinstance(attn.quant_method, Fp8KVCacheMethod)
90 + assert attn._k_scale == 1.0
91 + assert attn._v_scale == 1.0
92 +
93 + if current_platform.has_device_capability(89) and not force_marlin:
94 + # For GPUs with hardware support, we keep weights in fp8
95 + assert fc1.weight.dtype == torch.float8_e4m3fn
96 + else:
97 + # For GPUs without hardware support, we pack the fp8 weights
98 + # for weight-only quantization using Marlin kernels
99 + assert fc1.weight.dtype == torch.int32
100 +
101 + llm.apply_model(check_model)
102
103
104 @pytest.mark.skipif(not is_quant_method_supported("fp8"),
tests/quantization/test_lm_head.py
+20 -17
@@ -28,20 +28,23 @@ def test_lm_head(
28 model_lm_head_quant: Tuple[str, bool],
29 ) -> None:
30 model, lm_head_quantized = model_lm_head_quant
31 - vllm_model = vllm_runner(model, dtype=torch.float16, max_model_len=2048)
32 -
33 - lm_head_layer = (vllm_model.model.llm_engine.model_executor.driver_worker.
34 - model_runner.model.lm_head)
35 -
36 - if lm_head_quantized:
37 - assert isinstance(
38 - lm_head_layer.linear_method,
39 - (GPTQLinearMethod, GPTQMarlinLinearMethod, MarlinLinearMethod))
40 - else:
41 - assert isinstance(lm_head_layer.linear_method,
42 - UnquantizedEmbeddingMethod)
43 -
44 - print(
45 - vllm_model.generate_greedy(prompts=["Hello my name is"],
46 - max_tokens=10)[0][1])
47 - del vllm_model
31 +
32 + with vllm_runner(model, dtype=torch.float16,
33 + max_model_len=2048) as vllm_model:
34 +
35 + def check_model(model):
36 + lm_head_layer = model.lm_head
37 +
38 + if lm_head_quantized:
39 + assert isinstance(lm_head_layer.linear_method,
40 + (GPTQLinearMethod, GPTQMarlinLinearMethod,
41 + MarlinLinearMethod))
42 + else:
43 + assert isinstance(lm_head_layer.linear_method,
44 + UnquantizedEmbeddingMethod)
45 +
46 + vllm_model.apply_model(check_model)
47 +
48 + print(
49 + vllm_model.generate_greedy(prompts=["Hello my name is"],
50 + max_tokens=10)[0][1])
tests/quantization/test_quark.py
+13 -10
@@ -12,19 +12,22 @@ from vllm.model_executor.layers.quantization.quark.quark import ( # noqa: E501
12 def test_quark_fp8(vllm_runner):
13 model_path = "amd/Llama-3.1-8B-Instruct-FP8-KV-Quark-test"
14 with vllm_runner(model_path) as llm:
15 - model = llm.model.llm_engine.model_executor.driver_worker.model_runner.model # noqa: E501
16 - layer = model.model.layers[0]
15
18 - qkv_proj = layer.self_attn.qkv_proj
16 + def check_model(model):
17 + layer = model.model.layers[0]
18
20 - assert isinstance(qkv_proj.quant_method, QuarkLinearMethod)
21 - assert isinstance(qkv_proj.scheme, QuarkW8A8Fp8)
19 + qkv_proj = layer.self_attn.qkv_proj
20
23 - if isinstance(qkv_proj.scheme, QuarkW8A8Fp8):
24 - assert len(qkv_proj.input_scale.shape) == 0
25 - assert qkv_proj.weight.dtype is torch.float8_e4m3fn
26 - #assert qkv_proj.weight.dtype is torch.float8_e4m3fnuz
27 - assert len(qkv_proj.weight_scale.shape) == 0
21 + assert isinstance(qkv_proj.quant_method, QuarkLinearMethod)
22 + assert isinstance(qkv_proj.scheme, QuarkW8A8Fp8)
23 +
24 + if isinstance(qkv_proj.scheme, QuarkW8A8Fp8):
25 + assert len(qkv_proj.input_scale.shape) == 0
26 + assert qkv_proj.weight.dtype is torch.float8_e4m3fn
27 + #assert qkv_proj.weight.dtype is torch.float8_e4m3fnuz
28 + assert len(qkv_proj.weight_scale.shape) == 0
29 +
30 + llm.apply_model(check_model)
31
32 output = llm.generate_greedy("Hello my name is", max_tokens=20)
33 assert output
tests/tensorizer_loader/test_tensorizer.py
+16 -18
@@ -3,6 +3,7 @@ import json
3 import os
4 import pathlib
5 import subprocess
6 +from functools import partial
7 from unittest.mock import MagicMock, patch
8
9 import openai
@@ -24,7 +25,6 @@ from vllm.model_executor.model_loader.tensorizer import (TensorizerConfig,
25 # yapf: enable
26 from vllm.utils import PlaceholderModule, import_from_path
27
27 -from ..conftest import VllmRunner
28 from ..utils import VLLM_PATH, RemoteOpenAIServer
29 from .conftest import retry_until_skip
30
@@ -58,16 +58,6 @@ def is_curl_installed():
58 return False
59
60
61 -def get_torch_model(vllm_runner: VllmRunner):
62 - return vllm_runner \
63 - .model \
64 - .llm_engine \
65 - .model_executor \
66 - .driver_worker \
67 - .model_runner \
68 - .model
69 -
70 -
61 def write_keyfile(keyfile_path: str):
62 encryption_params = EncryptionParams.random()
63 pathlib.Path(keyfile_path).parent.mkdir(parents=True, exist_ok=True)
@@ -121,8 +111,10 @@ def test_deserialized_encrypted_vllm_model_has_same_outputs(
111
112 config_for_serializing = TensorizerConfig(tensorizer_uri=model_path,
113 encryption_keyfile=key_path)
124 - serialize_vllm_model(get_torch_model(vllm_model),
125 - config_for_serializing)
114 +
115 + vllm_model.apply_model(
116 + partial(serialize_vllm_model,
117 + tensorizer_config=config_for_serializing))
118
119 config_for_deserializing = TensorizerConfig(tensorizer_uri=model_path,
120 encryption_keyfile=key_path)
@@ -175,8 +167,10 @@ def test_vllm_model_can_load_with_lora(vllm_runner, tmp_path):
167 with vllm_runner(model_ref, ) as vllm_model:
168 model_path = tmp_path / (model_ref + ".tensors")
169
178 - serialize_vllm_model(get_torch_model(vllm_model),
179 - TensorizerConfig(tensorizer_uri=model_path))
170 + vllm_model.apply_model(
171 + partial(
172 + serialize_vllm_model,
173 + tensorizer_config=TensorizerConfig(tensorizer_uri=model_path)))
174
175 with vllm_runner(
176 model_ref,
@@ -215,8 +209,10 @@ def test_openai_apiserver_with_tensorizer(vllm_runner, tmp_path):
209 with vllm_runner(model_ref, ) as vllm_model:
210 model_path = tmp_path / (model_ref + ".tensors")
211
218 - serialize_vllm_model(get_torch_model(vllm_model),
219 - TensorizerConfig(tensorizer_uri=model_path))
212 + vllm_model.apply_model(
213 + partial(
214 + serialize_vllm_model,
215 + tensorizer_config=TensorizerConfig(tensorizer_uri=model_path)))
216
217 model_loader_extra_config = {
218 "tensorizer_uri": str(model_path),
@@ -337,7 +333,9 @@ def test_vllm_tensorized_model_has_same_outputs(vllm_runner, tmp_path):
333
334 with vllm_runner(model_ref) as vllm_model:
335 outputs = vllm_model.generate(prompts, sampling_params)
340 - serialize_vllm_model(get_torch_model(vllm_model), config)
336 +
337 + vllm_model.apply_model(
338 + partial(serialize_vllm_model, tensorizer_config=config))
339
340 assert is_vllm_tensorized(config)
341
vllm/engine/llm_engine.py
+3 -14
@@ -5,10 +5,10 @@ from collections import deque
5 from contextlib import contextmanager
6 from dataclasses import dataclass
7 from functools import partial
8 -from typing import (TYPE_CHECKING, Any, Callable, ClassVar, Deque, Dict,
9 - Iterable, List, Mapping, NamedTuple, Optional)
8 +from typing import (TYPE_CHECKING, Callable, ClassVar, Deque, Dict, Iterable,
9 + List, Mapping, NamedTuple, Optional)
10 from typing import Sequence as GenericSequence
11 -from typing import Set, Tuple, Type, Union, cast, overload
11 +from typing import Set, Type, Union, cast, overload
12
13 import torch
14 from typing_extensions import TypeVar, deprecated
@@ -1818,17 +1818,6 @@ class LLMEngine:
1818 def stop_profile(self) -> None:
1819 self.model_executor.stop_profile()
1820
1821 - def collective_rpc(self,
1822 - method: Union[str, Callable],
1823 - timeout: Optional[float] = None,
1824 - args: Tuple = (),
1825 - kwargs: Optional[Dict] = None) -> List[Any]:
1826 - """
1827 - See LLM.collective_rpc for more details.
1828 - """
1829 - return self.model_executor.collective_rpc(method, timeout, args,
1830 - kwargs)
1831 -
1821 def check_health(self) -> None:
1822 if self.tokenizer:
1823 self.tokenizer.check_health()
vllm/entrypoints/llm.py
+36 -16
@@ -5,8 +5,9 @@ from typing import (Any, Callable, ClassVar, Dict, List, Optional, Sequence,
5 Tuple, Type, Union, cast, overload)
6
7 import cloudpickle
8 +import torch.nn as nn
9 from tqdm import tqdm
9 -from typing_extensions import deprecated
10 +from typing_extensions import TypeVar, deprecated
11
12 from vllm import envs
13 from vllm.beam_search import (BeamSearchInstance, BeamSearchOutput,
@@ -42,6 +43,8 @@ from vllm.utils import Counter, deprecate_args, deprecate_kwargs, is_list_of
43
44 logger = init_logger(__name__)
45
46 +_R = TypeVar("_R", default=Any)
47 +
48
49 class LLM:
50 """An LLM for generating texts from given prompts and sampling parameters.
@@ -464,25 +467,42 @@ class LLM:
467 return self.engine_class.validate_outputs(outputs, RequestOutput)
468
469 def collective_rpc(self,
467 - method: Union[str, Callable],
470 + method: Union[str, Callable[..., _R]],
471 timeout: Optional[float] = None,
472 args: Tuple = (),
470 - kwargs: Optional[Dict] = None) -> List[Any]:
473 + kwargs: Optional[Dict[str, Any]] = None) -> List[_R]:
474 + """
475 + Execute an RPC call on all workers.
476 +
477 + Args:
478 + method: Name of the worker method to execute, or a callable that
479 + is serialized and sent to all workers to execute.
480 +
481 + If the method is a callable, it should accept an additional
482 + `self` argument, in addition to the arguments passed in `args`
483 + and `kwargs`. The `self` argument will be the worker object.
484 + timeout: Maximum time in seconds to wait for execution. Raises a
485 + :exc:`TimeoutError` on timeout. `None` means wait indefinitely.
486 + args: Positional arguments to pass to the worker method.
487 + kwargs: Keyword arguments to pass to the worker method.
488 +
489 + Returns:
490 + A list containing the results from each worker.
491 +
492 + Note:
493 + It is recommended to use this API to only pass control messages,
494 + and set up data-plane communication to pass data.
495 + """
496 + executor = self.llm_engine.model_executor
497 + return executor.collective_rpc(method, timeout, args, kwargs)
498 +
499 + def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
500 """
472 - Run a method on all workers, with homogeneous arguments.
473 - The main extension point for the LLM entrypoint.
474 - Users can provide custom worker class through `worker_cls`
475 - argument, and implement new methods in the worker class.
476 - Then, users can call the new methods through this API.
477 - It is recommended to use this API to only pass control messages,
478 - and set up data-plane communication to pass data.
479 - The method can also be a callable, which will be serialized
480 - and sent to all workers to execute.
481 - If the method is a callable, it should accept an additional
482 - `self` argument, in addition to the arguments passed in `args`
483 - and `kwargs`. The `self` argument will be the worker object.
501 + Run a function directly on the model inside each worker,
502 + returning the result for each of them.
503 """
485 - return self.llm_engine.collective_rpc(method, timeout, args, kwargs)
504 + executor = self.llm_engine.model_executor
505 + return executor.apply_model(func)
506
507 def beam_search(
508 self,
vllm/executor/executor_base.py
+41 -9
@@ -3,6 +3,9 @@ from abc import ABC, abstractmethod
3 from typing import (Any, Awaitable, Callable, Dict, List, Optional, Set, Tuple,
4 Union)
5
6 +import torch.nn as nn
7 +from typing_extensions import TypeVar
8 +
9 from vllm.config import VllmConfig
10 from vllm.logger import init_logger
11 from vllm.lora.request import LoRARequest
@@ -11,9 +14,12 @@ from vllm.platforms import current_platform
14 from vllm.prompt_adapter.request import PromptAdapterRequest
15 from vllm.sequence import ExecuteModelRequest, PoolerOutput
16 from vllm.utils import make_async
17 +from vllm.worker.worker_base import WorkerBase
18
19 logger = init_logger(__name__)
20
21 +_R = TypeVar("_R", default=Any)
22 +
23
24 class ExecutorBase(ABC):
25 """Base class for all executors.
@@ -44,22 +50,37 @@ class ExecutorBase(ABC):
50
51 @abstractmethod
52 def _init_executor(self) -> None:
47 - pass
53 + raise NotImplementedError
54
55 @abstractmethod
56 def collective_rpc(self,
51 - method: Union[str, Callable],
57 + method: Union[str, Callable[..., _R]],
58 timeout: Optional[float] = None,
59 args: Tuple = (),
54 - kwargs: Optional[Dict] = None) -> List[Any]:
60 + kwargs: Optional[Dict[str, Any]] = None) -> List[_R]:
61 """
56 - The main interface of the executor to run a method on all workers,
57 - with homogeneous arguments.
58 - If the args are heterogeneous, then we can pack them into a list,
59 - and unpack them in the method of every worker, because every worker
60 - knows their own rank.
62 + Execute an RPC call on all workers.
63 +
64 + Args:
65 + method: Name of the worker method to execute, or a callable that
66 + is serialized and sent to all workers to execute.
67 +
68 + If the method is a callable, it should accept an additional
69 + `self` argument, in addition to the arguments passed in `args`
70 + and `kwargs`. The `self` argument will be the worker object.
71 + timeout: Maximum time in seconds to wait for execution. Raises a
72 + :exc:`TimeoutError` on timeout. `None` means wait indefinitely.
73 + args: Positional arguments to pass to the worker method.
74 + kwargs: Keyword arguments to pass to the worker method.
75 +
76 + Returns:
77 + A list containing the results from each worker.
78 +
79 + Note:
80 + It is recommended to use this API to only pass control messages,
81 + and set up data-plane communication to pass data.
82 """
62 - pass
83 + raise NotImplementedError
84
85 def determine_num_available_blocks(self) -> Tuple[int, int]:
86 """Determine the number of available blocks for the GPU KV cache and
@@ -97,6 +118,17 @@ class ExecutorBase(ABC):
118 self.collective_rpc("initialize_cache",
119 args=(num_gpu_blocks, num_cpu_blocks))
120
121 + def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
122 + """
123 + Run a function directly on the model inside each worker,
124 + returning the result for each of them.
125 + """
126 +
127 + def rpc_func(worker: WorkerBase) -> _R:
128 + return func(worker.get_model())
129 +
130 + return self.collective_rpc(rpc_func)
131 +
132 def execute_model(
133 self, execute_model_req: ExecuteModelRequest
134 ) -> Optional[List[Union[SamplerOutput, PoolerOutput]]]:
vllm/executor/mp_distributed_executor.py
+1 -1
@@ -148,7 +148,7 @@ class MultiprocessingDistributedExecutor(DistributedExecutorBase):
148 async_run_tensor_parallel_workers_only: bool = False,
149 max_concurrent_workers: Optional[int] = None,
150 **kwargs,
151 - ) -> Any:
151 + ) -> List[Any]:
152 """Runs the given method on all workers.
153
154 Args:
vllm/model_executor/model_loader/tensorizer.py
+4 -13
@@ -459,16 +459,7 @@ def tensorize_vllm_model(engine_args: EngineArgs,
459 stream.write(encryption_params.key)
460
461 engine = LLMEngine.from_engine_args(engine_args)
462 - if tensorizer_config._is_sharded:
463 - # if the engine is a distributed engine (for tensor parallel) then each
464 - # worker shard needs to serialize its part of the model.
465 - engine.model_executor._run_workers(
466 - "save_tensorized_model",
467 - tensorizer_config=tensorizer_config,
468 - )
469 - else:
470 - # with a single worker, we can get to the underlying model directly
471 - serialize_vllm_model(
472 - engine.model_executor.driver_worker.model_runner.model,
473 - tensorizer_config,
474 - )
462 + engine.model_executor.collective_rpc(
463 + "save_tensorized_model",
464 + kwargs=dict(tensorizer_config=tensorizer_config),
465 + )
vllm/spec_decode/ngram_worker.py
+11 -1
@@ -2,6 +2,7 @@ import weakref
2 from typing import List, Optional, Set, Tuple
3
4 import torch
5 +import torch.nn as nn
6
7 from vllm.model_executor.layers.sampler import SamplerOutput
8 from vllm.sequence import ExecuteModelRequest
@@ -10,6 +11,10 @@ from vllm.spec_decode.proposer_worker_base import NonLLMProposerWorkerBase
11 from vllm.spec_decode.top1_proposer import Top1Proposer
12
13
14 +class _DummyModel(nn.Module):
15 + pass
16 +
17 +
18 class NGramWorker(NonLLMProposerWorkerBase):
19 """NGramWorker provides a light drafter without need for model.
20
@@ -36,7 +41,6 @@ class NGramWorker(NonLLMProposerWorkerBase):
41
42 def init_device(self):
43 self.device = torch.device(f"{self.device_type}:{self.local_rank}")
39 - self.load_model = lambda *args, **kwargs: None
44
45 # Current NGramWorker only supports Top1Proposer
46 self._proposer = Top1Proposer(
@@ -45,6 +49,12 @@ class NGramWorker(NonLLMProposerWorkerBase):
49 vocab_size=self.vocab_size,
50 )
51
52 + def load_model(self) -> None:
53 + pass # Dummy
54 +
55 + def get_model(self) -> nn.Module:
56 + return _DummyModel()
57 +
58 def sampler_output(
59 self,
60 execute_model_req: ExecuteModelRequest,
vllm/spec_decode/smaller_tp_proposer_worker.py
+12
@@ -1,6 +1,7 @@
1 from typing import List, Optional, Set, Tuple
2
3 import torch
4 +import torch.nn as nn
5
6 from vllm.distributed.parallel_state import (get_tp_group,
7 init_model_parallel_group,
@@ -15,6 +16,10 @@ from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
16 logger = init_logger(__name__)
17
18
19 +class _DummyModel(nn.Module):
20 + pass
21 +
22 +
23 class SmallerTpProposerWorker(ProposerWorkerBase):
24 """Class which allows a speculative draft model to run with smaller tensor
25 parallel degree than target model.
@@ -139,6 +144,13 @@ class SmallerTpProposerWorker(ProposerWorkerBase):
144 return self._worker.get_spec_proposals(
145 execute_model_req, seq_ids_with_bonus_token_in_last_step)
146
147 + def get_model(self) -> nn.Module:
148 + if self._is_dummy:
149 + return _DummyModel()
150 +
151 + with self._patch_tensor_parallel_group():
152 + return self._worker.get_model()
153 +
154 def execute_model(
155 self,
156 execute_model_req: Optional[ExecuteModelRequest] = None
vllm/spec_decode/spec_decode_worker.py
+4
@@ -4,6 +4,7 @@ from functools import cached_property
4 from typing import Any, Dict, List, Optional, Set, Tuple, Type
5
6 import torch
7 +import torch.nn as nn
8
9 from vllm.config import ParallelConfig, SpeculativeConfig, VllmConfig
10 from vllm.distributed.communication_op import broadcast_tensor_dict
@@ -403,6 +404,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
404 self.proposer_worker.initialize_cache(num_gpu_blocks=num_gpu_blocks,
405 num_cpu_blocks=num_cpu_blocks)
406
407 + def get_model(self) -> nn.Module:
408 + return self.scorer_worker.get_model()
409 +
410 @torch.inference_mode()
411 def execute_model(
412 self,
vllm/v1/executor/multiproc_executor.py
+3 -13
@@ -94,22 +94,12 @@ class MultiprocExecutor(Executor):
94 timeout: Optional[float] = None,
95 args: Tuple = (),
96 kwargs: Optional[Dict] = None) -> List[Any]:
97 - """
98 - Execute an RPC call on workers.
99 -
100 - Args:
101 - method: Name of the worker method to execute
102 - timeout: Maximum time in seconds to wait for execution. Rases a
103 - TimeoutError on timeout. None means wait indefinitely.
104 - args: Positional arguments to pass to the worker method
105 - kwargs: Keyword arguments to pass to the worker method
106 -
107 - Returns:
108 - List of results from each worker
109 - """
97 start_time = time.monotonic()
98 kwargs = kwargs or {}
99
100 + # NOTE: If the args are heterogeneous, then we pack them into a list,
101 + # and unpack them in the method of every worker, because every worker
102 + # knows their own rank.
103 try:
104 if isinstance(method, str):
105 send_method = method
vllm/v1/worker/gpu_model_runner.py
+3
@@ -689,6 +689,9 @@ class GPUModelRunner:
689 encoder_outputs.append(encoder_output[start_idx:end_idx])
690 return encoder_outputs
691
692 + def get_model(self) -> nn.Module:
693 + return self.model
694 +
695 @torch.inference_mode()
696 def execute_model(
697 self,
vllm/v1/worker/gpu_worker.py
+4
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Optional
5
6 import torch
7 import torch.distributed
8 +import torch.nn as nn
9
10 import vllm.envs as envs
11 from vllm.config import CacheConfig, ModelConfig, ParallelConfig, VllmConfig
@@ -176,6 +177,9 @@ class Worker:
177 # the model initialization and profiling.
178 set_random_seed(self.model_config.seed)
179
180 + def get_model(self) -> nn.Module:
181 + return self.model_runner.get_model()
182 +
183 @torch.inference_mode()
184 def execute_model(
185 self,
vllm/worker/cpu_model_runner.py
+3
@@ -509,6 +509,9 @@ class CPUModelRunnerBase(ModelRunnerBase[TModelInputForCPU]):
509 )
510 self.model = self.lora_manager.create_lora_manager(self.model)
511
512 + def get_model(self) -> nn.Module:
513 + return self.model
514 +
515 def _prepare_model_input_tensors(
516 self,
517 seq_group_metadata_list: List[SequenceGroupMetadata],
vllm/worker/hpu_model_runner.py
+4
@@ -21,6 +21,7 @@ from typing import (TYPE_CHECKING, Any, Callable, Dict, List, NamedTuple,
21 import habana_frameworks.torch as htorch
22 import habana_frameworks.torch.internal.bridge_config as bc
23 import torch
24 +import torch.nn as nn
25 from vllm_hpu_extension.ops import LoraMask as LoraMask
26 from vllm_hpu_extension.profiler import (HabanaHighLevelProfiler,
27 HabanaMemoryProfiler, format_bytes)
@@ -676,6 +677,9 @@ class HPUModelRunnerBase(ModelRunnerBase[TModelInputForHPU]):
677 msg = f"Loading model weights took in total {m.get_summary_string()}"
678 logger.info(msg)
679
680 + def get_model(self) -> nn.Module:
681 + return self.model
682 +
683 def _use_graphs(self, batch_size, seq_len, is_prompt):
684 if self.enforce_eager:
685 return False
vllm/worker/model_runner.py
+3
@@ -1176,6 +1176,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1176 fullgraph=envs.VLLM_TEST_DYNAMO_FULLGRAPH_CAPTURE,
1177 backend=backend)
1178
1179 + def get_model(self) -> nn.Module:
1180 + return self.model
1181 +
1182 def save_sharded_state(
1183 self,
1184 path: str,
vllm/worker/model_runner_base.py
+7 -2
@@ -7,6 +7,7 @@ from typing import (TYPE_CHECKING, Any, Dict, Generic, Iterable, List,
7 Optional, Type, TypeVar)
8
9 import torch
10 +import torch.nn as nn
11 from torch import is_tensor
12
13 from vllm.config import VllmConfig
@@ -264,6 +265,10 @@ class ModelRunnerBase(ABC, Generic[T]):
265 """
266 raise NotImplementedError
267
268 + @abstractmethod
269 + def get_model(self) -> nn.Module:
270 + raise NotImplementedError
271 +
272 def execute_model(
273 self,
274 model_input: T,
@@ -297,9 +302,9 @@ class ModelRunnerWrapperBase:
302
303 def __init__(
304 self,
300 - moderl_runner: ModelRunnerBase,
305 + model_runner: ModelRunnerBase,
306 ) -> None:
302 - self.model_runner: ModelRunnerBase = moderl_runner
307 + self.model_runner: ModelRunnerBase = model_runner
308
309 def __getattr__(self, attr):
310 return getattr(self.model_runner, attr)
vllm/worker/neuron_model_runner.py
+3
@@ -113,6 +113,9 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
113 raise NotImplementedError(
114 "Supports only Transformer-NeuronX based models.")
115
116 + def get_model(self) -> nn.Module:
117 + return self.model
118 +
119 def _prepare_prompt(
120 self,
121 seq_group_metadata_list: List[SequenceGroupMetadata],
vllm/worker/openvino_model_runner.py
+3
@@ -84,6 +84,9 @@ class OpenVINOModelRunner(ModelRunnerBase):
84 kv_cache_dtype=self.kv_cache_dtype,
85 ov_core=self.ov_core)
86
87 + def get_model(self) -> nn.Module:
88 + return self.model
89 +
90 def _prepare_model_input(
91 self,
92 seq_group_metadata_list: List[SequenceGroupMetadata],
vllm/worker/openvino_worker.py
+4
@@ -4,6 +4,7 @@ from typing import Any, Dict, List, Optional, Tuple
4 import openvino as ov
5 import torch
6 import torch.distributed
7 +import torch.nn as nn
8
9 import vllm.envs as envs
10 from vllm.attention import get_attn_backend
@@ -362,6 +363,9 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
363 ) -> None:
364 self.cache_engine.copy(blocks_to_copy) # type: ignore
365
366 + def get_model(self) -> nn.Module:
367 + return self.model_runner.get_model()
368 +
369 @torch.inference_mode()
370 def execute_model(
371 self,
vllm/worker/tpu_model_runner.py
+3
@@ -158,6 +158,9 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
158 fullgraph=True,
159 dynamic=False)
160
161 + def get_model(self) -> nn.Module:
162 + return self.model.model
163 +
164 def _dummy_run(
165 self,
166 batch_size: int,
vllm/worker/worker_base.py
+12
@@ -6,6 +6,7 @@ from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
6
7 import cloudpickle
8 import torch
9 +import torch.nn as nn
10
11 from vllm.config import ObservabilityConfig, VllmConfig
12 from vllm.distributed import broadcast_tensor_dict, get_pp_group, get_tp_group
@@ -90,6 +91,11 @@ class WorkerBase(ABC):
91 if output is None:
92 return None
93
94 + @abstractmethod
95 + def get_model(self) -> nn.Module:
96 + raise NotImplementedError
97 +
98 + @abstractmethod
99 def execute_model(
100 self,
101 execute_model_req: Optional[ExecuteModelRequest] = None
@@ -147,6 +153,9 @@ class DelegateWorkerBase(WorkerBase):
153 num_cpu_blocks: int) -> None:
154 self.worker.initialize_cache(num_gpu_blocks, num_cpu_blocks)
155
156 + def get_model(self) -> nn.Module:
157 + return self.worker.get_model()
158 +
159 def execute_model(
160 self,
161 execute_model_req: Optional[ExecuteModelRequest] = None
@@ -363,6 +372,9 @@ class LocalOrDistributedWorkerBase(WorkerBase):
372 else:
373 return self._get_worker_input_from_broadcast()
374
375 + def get_model(self) -> nn.Module:
376 + return self.model_runner.get_model()
377 +
378 def execute_model(
379 self,
380 execute_model_req: Optional[ExecuteModelRequest] = None,
vllm/worker/xpu_model_runner.py
+3
@@ -416,6 +416,9 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
416 logger.info("Loading model weights took %.4f GB",
417 self.model_memory_usage / float(2**30))
418
419 + def get_model(self) -> nn.Module:
420 + return self.model
421 +
422 @property
423 def vocab_size(self) -> int:
424 return self.model_config.get_vocab_size()