@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d6f4bd7cdd
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Misc]Add customized information for models (#4132)

Jee Li committed May 1, 2024 at 12:18 UTC d6f4bd7cddc9546c38568c92c3772d22940a09f2
9 files changed +87
tests/models/test_big_models.py
+15
@@ -43,3 +43,18 @@ def test_models(
43 f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
44 assert hf_output_ids == vllm_output_ids, (
45 f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
46 +
47 +
48 +@pytest.mark.parametrize("model", MODELS)
49 +@pytest.mark.parametrize("dtype", ["half"])
50 +def test_model_print(
51 + vllm_runner,
52 + model: str,
53 + dtype: str,
54 +) -> None:
55 + vllm_model = vllm_runner(model, dtype=dtype)
56 + # This test is for verifying whether the model's extra_repr
57 + # can be printed correctly.
58 + print(vllm_model.model.llm_engine.model_executor.driver_worker.
59 + model_runner.model)
60 + del vllm_model
tests/models/test_models.py
+15
@@ -49,3 +49,18 @@ def test_models(
49 f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
50 assert hf_output_ids == vllm_output_ids, (
51 f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
52 +
53 +
54 +@pytest.mark.parametrize("model", MODELS)
55 +@pytest.mark.parametrize("dtype", ["float"])
56 +def test_model_print(
57 + vllm_runner,
58 + model: str,
59 + dtype: str,
60 +) -> None:
61 + vllm_model = vllm_runner(model, dtype=dtype)
62 + # This test is for verifying whether the model's extra_repr
63 + # can be printed correctly.
64 + print(vllm_model.model.llm_engine.model_executor.driver_worker.
65 + model_runner.model)
66 + del vllm_model
vllm/attention/layer.py
+7
@@ -47,3 +47,10 @@ class Attention(nn.Module):
47 ) -> torch.Tensor:
48 return self.impl.forward(query, key, value, kv_cache, attn_metadata,
49 kv_scale)
50 +
51 + def extra_repr(self) -> str:
52 + s = f"head_size={self.impl.head_size}" # type: ignore
53 + s += f", num_heads={self.impl.num_heads}" # type: ignore
54 + s += f", num_kv_heads={self.impl.num_kv_heads}" # type: ignore
55 + s += f", scale={self.impl.scale}" # type: ignore
56 + return s
vllm/model_executor/layers/activation.py
+3
@@ -67,6 +67,9 @@ class GeluAndMul(nn.Module):
67 ops.gelu_tanh_and_mul(out, x)
68 return out
69
70 + def extra_repr(self) -> str:
71 + return f'approximate={repr(self.approximate)}'
72 +
73
74 class NewGELU(nn.Module):
75
vllm/model_executor/layers/layernorm.py
+5
@@ -64,3 +64,8 @@ class RMSNorm(nn.Module):
64 self.variance_epsilon,
65 )
66 return out
67 +
68 + def extra_repr(self) -> str:
69 + s = f"hidden_size={self.weight.data.size(0)}"
70 + s += f", eps={self.variance_epsilon}"
71 + return s
vllm/model_executor/layers/linear.py
+22
@@ -181,6 +181,12 @@ class ReplicatedLinear(LinearBase):
181 output_bias = self.bias if self.skip_bias_add else None
182 return output, output_bias
183
184 + def extra_repr(self) -> str:
185 + s = f"in_features={self.input_size}"
186 + s += f", output_features={self.output_size}"
187 + s += f", bias={self.bias is not None}"
188 + return s
189 +
190
191 class ColumnParallelLinear(LinearBase):
192 """Linear layer with column parallelism.
@@ -281,6 +287,14 @@ class ColumnParallelLinear(LinearBase):
287 output_bias = self.bias if self.skip_bias_add else None
288 return output, output_bias
289
290 + def extra_repr(self) -> str:
291 + s = f"in_features={self.input_size}"
292 + s += f", output_features={self.output_size_per_partition}"
293 + s += f", bias={self.bias is not None}"
294 + s += f", tp_size={get_tensor_model_parallel_world_size()}"
295 + s += f", gather_output={self.gather_output}"
296 + return s
297 +
298
299 class MergedColumnParallelLinear(ColumnParallelLinear):
300 """Packed linear layers with column parallelism.
@@ -685,3 +699,11 @@ class RowParallelLinear(LinearBase):
699 output = output_
700 output_bias = self.bias
701 return output, output_bias
702 +
703 + def extra_repr(self) -> str:
704 + s = f"input_features={self.input_size_per_partition}"
705 + s += f", output_features={self.output_size}"
706 + s += f", bias={self.bias is not None}"
707 + s += f", tp_size={self.tp_size}"
708 + s += f", reduce_results={self.reduce_results}"
709 + return s
vllm/model_executor/layers/logits_processor.py
+6
@@ -70,6 +70,12 @@ class LogitsProcessor(nn.Module):
70 logits = logits[:, :self.org_vocab_size]
71 return logits
72
73 + def extra_repr(self) -> str:
74 + s = f"vocab_size={self.vocab_size}"
75 + s += f", forg_vocab_size={self.org_vocab_size}"
76 + s += f", scale={self.scale}, logits_as_input={self.logits_as_input}"
77 + return s
78 +
79
80 def _prune_hidden_states(
81 hidden_states: torch.Tensor,
vllm/model_executor/layers/rotary_embedding.py
+6
@@ -156,6 +156,12 @@ class RotaryEmbedding(nn.Module):
156 self.cos_sin_cache, self.is_neox_style)
157 return query, key
158
159 + def extra_repr(self) -> str:
160 + s = f"head_size={self.head_size}, rotary_dim={self.rotary_dim}"
161 + s += f", max_position_embeddings={self.max_position_embeddings}"
162 + s += f", base={self.base}, is_neox_style={self.is_neox_style}"
163 + return s
164 +
165
166 class LinearScalingRotaryEmbedding(RotaryEmbedding):
167 """RotaryEmbedding extended with linear scaling.
vllm/model_executor/layers/vocab_parallel_embedding.py
+8
@@ -105,6 +105,14 @@ class VocabParallelEmbedding(torch.nn.Module):
105 output = tensor_model_parallel_all_reduce(output_parallel)
106 return output
107
108 + def extra_repr(self) -> str:
109 + s = f"num_embeddings={self.num_embeddings_per_partition}"
110 + s += f", embedding_dim={self.embedding_dim}"
111 + s += f", org_vocab_size={self.org_vocab_size}"
112 + s += f', num_embeddings_padded={self.num_embeddings_padded}'
113 + s += f', tp_size={self.tp_size}'
114 + return s
115 +
116
117 class ParallelLMHead(VocabParallelEmbedding):
118 """Parallelized LM head.