@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 15bb8330aa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Bugfix] Fix tensor parallel for qwen2 classification model (#10297)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Nov 14, 2024 at 10:54 UTC 15bb8330aa50ca6ec86f827a0fe79134b1dbac8c
2 files changed +9 -4
tests/models/embedding/language/test_cls_models.py
+3 -3
@@ -21,14 +21,14 @@ def test_classification_models(
21 model: str,
22 dtype: str,
23 ) -> None:
24 + with vllm_runner(model, dtype=dtype) as vllm_model:
25 + vllm_outputs = vllm_model.classify(example_prompts)
26 +
27 with hf_runner(model,
28 dtype=dtype,
29 auto_cls=AutoModelForSequenceClassification) as hf_model:
30 hf_outputs = hf_model.classify(example_prompts)
31
29 - with vllm_runner(model, dtype=dtype) as vllm_model:
30 - vllm_outputs = vllm_model.classify(example_prompts)
31 -
32 print(hf_outputs, vllm_outputs)
33
34 # check logits difference
vllm/model_executor/models/qwen2_cls.py
+6 -1
@@ -69,9 +69,14 @@ class Qwen2ForSequenceClassification(nn.Module):
69 self.model = Qwen2Model(vllm_config=vllm_config,
70 prefix=maybe_prefix(prefix, "model"))
71
72 + # hidden_states from Qwen2Model has been reduced,
73 + # the input of score layer is not parallelized.
74 self.score = RowParallelLinear(config.hidden_size,
75 config.num_labels,
74 - quant_config=quant_config)
76 + quant_config=quant_config,
77 + input_is_parallel=False,
78 + bias=False,
79 + prefix=maybe_prefix(prefix, "score"))
80 self._pooler = Pooler.from_config_with_defaults(
81 pooler_config,
82 pooling_type=PoolingType.LAST,