8f4b313c37 · tduncan/Nvidia-Optimization-Stack-VLLM

benchmarks/kernels/benchmark_moe.py

+1 -1

@@ -631,7 +631,7 @@ def main(args: argparse.Namespace):

631 else:

632 ensure_divisibility(intermediate_size, args.tp_size, "intermediate_size")

633 shard_intermediate_size = 2 * intermediate_size // args.tp_size

634 - dtype = torch.float16 if current_platform.is_rocm() else config.torch_dtype

634 + dtype = torch.float16 if current_platform.is_rocm() else config.dtype

635 use_fp8_w8a8 = args.dtype == "fp8_w8a8"

636 use_int8_w8a16 = args.dtype == "int8_w8a16"

637 block_quant_shape = get_weight_block_size_safety(config)

benchmarks/kernels/benchmark_moe_permute_unpermute.py

+1 -1

@@ -344,7 +344,7 @@ def main(args: argparse.Namespace):

344 topk = config.num_experts_per_tok

345

346 hidden_size = config.hidden_size

347 - dtype = torch.float16 if current_platform.is_rocm() else config.torch_dtype

347 + dtype = torch.float16 if current_platform.is_rocm() else config.dtype

348 use_fp8_w8a8 = args.dtype == "fp8_w8a8"

349 use_int8_w8a16 = args.dtype == "int8_w8a16"

350 use_customized_permute = args.use_customized_permute

docs/features/quantization/auto_round.md

+1 -1

@@ -58,7 +58,7 @@ from transformers import AutoModelForCausalLM, AutoTokenizer

58 from auto_round import AutoRound

59

60 model_name = "Qwen/Qwen3-0.6B"

61 -model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype="auto")

61 +model = AutoModelForCausalLM.from_pretrained(model_name, dtype="auto")

62 tokenizer = AutoTokenizer.from_pretrained(model_name)

63

64 bits, group_size, sym = 4, 128, True

docs/features/quantization/fp8.md

+1 -1

@@ -43,7 +43,7 @@ MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct"

43 model = AutoModelForCausalLM.from_pretrained(

44 MODEL_ID,

45 device_map="auto",

46 - torch_dtype="auto",

46 + dtype="auto",

47 )

48 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

49 ```

docs/features/quantization/int4.md

+1 -1

@@ -41,7 +41,7 @@ MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct"

41 model = AutoModelForCausalLM.from_pretrained(

42 MODEL_ID,

43 device_map="auto",

44 - torch_dtype="auto",

44 + dtype="auto",

45 )

46 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

47 ```

docs/features/quantization/int8.md

+1 -1

@@ -46,7 +46,7 @@ MODEL_ID = "meta-llama/Meta-Llama-3-8B-Instruct"

46 model = AutoModelForCausalLM.from_pretrained(

47 MODEL_ID,

48 device_map="auto",

49 - torch_dtype="auto",

49 + dtype="auto",

50 )

51 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

52 ```

docs/features/quantization/quantized_kvcache.md

+1 -1

@@ -82,7 +82,7 @@ Here's a complete example using `meta-llama/Llama-3.1-8B-Instruct` (most models

82

83 # Select model and load it

84 MODEL_ID = "meta-llama/Llama-3.1-8B-Instruct"

85 - model = AutoModelForCausalLM.from_pretrained(MODEL_ID, device_map="auto", torch_dtype="auto")

85 + model = AutoModelForCausalLM.from_pretrained(MODEL_ID, device_map="auto", dtype="auto")

86 tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)

87

88 # Select calibration dataset

docs/features/quantization/quark.md

+1 -1

@@ -50,7 +50,7 @@ to fetch model and tokenizer.

50 model = AutoModelForCausalLM.from_pretrained(

51 MODEL_ID,

52 device_map="auto",

53 - torch_dtype="auto",

53 + dtype="auto",

54 )

55 model.eval()

56

docs/features/quantization/torchao.md

+1 -1

@@ -27,7 +27,7 @@ You can quantize your own huggingface model with torchao, e.g. [transformers](ht

27 quantization_config = TorchAoConfig(Int8WeightOnlyConfig())

28 quantized_model = AutoModelForCausalLM.from_pretrained(

29 model_name,

30 - torch_dtype="auto",

30 + dtype="auto",

31 device_map="auto",

32 quantization_config=quantization_config

33 )

requirements/common.txt

+1 -1

@@ -7,7 +7,7 @@ requests >= 2.26.0

7 tqdm

8 blake3

9 py-cpuinfo

10 -transformers >= 4.55.2

10 +transformers >= 4.56.0

11 tokenizers >= 0.21.1 # Required for fast incremental detokenization.

12 protobuf # Required by LlamaTokenizer.

13 fastapi[standard] >= 0.115.0 # Required by FastAPI's form models in the OpenAI API server's audio transcriptions endpoint.

tests/conftest.py

+4 -4

@@ -334,7 +334,7 @@ class HfRunner:

334 trust_remote_code=trust_remote_code,

335 )

336 self.device = self.get_default_device()

337 - self.dtype = torch_dtype = _get_and_verify_dtype(

337 + self.dtype = dtype = _get_and_verify_dtype(

338 self.model_name,

339 self.config,

340 dtype=dtype,

@@ -342,7 +342,7 @@ class HfRunner:

342 )

343

344 model_kwargs = model_kwargs if model_kwargs is not None else {}

345 - model_kwargs.setdefault("torch_dtype", torch_dtype)

345 + model_kwargs.setdefault("dtype", dtype)

346

347 if is_sentence_transformer:

348 # Lazy init required for AMD CI

@@ -388,7 +388,7 @@ class HfRunner:

388 if not skip_tokenizer_init:

389 self.tokenizer = AutoTokenizer.from_pretrained(

390 model_name,

391 - torch_dtype=torch_dtype,

391 + dtype=dtype,

392 trust_remote_code=trust_remote_code,

393 )

394

@@ -398,7 +398,7 @@ class HfRunner:

398

399 self.processor = AutoProcessor.from_pretrained(

400 model_name,

401 - torch_dtype=torch_dtype,

401 + dtype=dtype,

402 trust_remote_code=trust_remote_code,

403 )

404 if skip_tokenizer_init:

tests/models/multimodal/pooling/test_intern_vit.py

+1 -1

@@ -38,7 +38,7 @@ def run_intern_vit_test(

38 config.norm_type = "rms_norm"

39

40 hf_model = AutoModel.from_pretrained(

41 - model, torch_dtype=torch_dtype, trust_remote_code=True

41 + model, dtype=torch_dtype, trust_remote_code=True

42 ).to("cuda")

43 hf_outputs_per_image = [

44 hf_model(pixel_value.to("cuda")).last_hidden_state

tests/models/multimodal/pooling/test_radio.py

+1 -1

@@ -45,7 +45,7 @@ def run_radio_test(

45 hf_model = AutoModel.from_pretrained(

46 model_id,

47 config=config,

48 - torch_dtype=torch_dtype,

48 + dtype=torch_dtype,

49 trust_remote_code=True,

50 ).to("cuda")

51 hf_model.eval()

vllm/benchmarks/throughput.py

+1 -1

@@ -251,7 +251,7 @@ def run_hf(

251 disable_detokenize: bool = False,

252 ) -> float:

253 llm = AutoModelForCausalLM.from_pretrained(

254 - model, torch_dtype=torch.float16, trust_remote_code=trust_remote_code

254 + model, dtype=torch.float16, trust_remote_code=trust_remote_code

255 )

256 if llm.config.model_type == "llama":

257 # To enable padding in the HF backend.

vllm/config/model.py

+7 -7

@@ -1837,18 +1837,18 @@ def _find_dtype(

1837 *,

1838 revision: str | None,

1839 ):

1840 - # NOTE: getattr(config, "torch_dtype", torch.float32) is not correct

1841 - # because config.torch_dtype can be None.

1842 - config_dtype = getattr(config, "torch_dtype", None)

1840 + # NOTE: getattr(config, "dtype", torch.float32) is not correct

1841 + # because config.dtype can be None.

1842 + config_dtype = getattr(config, "dtype", None)

1843

1844 # Fallbacks for multi-modal models if the root config

1845 - # does not define torch_dtype

1845 + # does not define dtype

1846 if config_dtype is None:

1847 - config_dtype = getattr(config.get_text_config(), "torch_dtype", None)

1847 + config_dtype = getattr(config.get_text_config(), "dtype", None)

1848 if config_dtype is None and hasattr(config, "vision_config"):

1849 - config_dtype = getattr(config.vision_config, "torch_dtype", None)

1849 + config_dtype = getattr(config.vision_config, "dtype", None)

1850 if config_dtype is None and hasattr(config, "encoder_config"):

1851 - config_dtype = getattr(config.encoder_config, "torch_dtype", None)

1851 + config_dtype = getattr(config.encoder_config, "dtype", None)

1852

1853 # Try to read the dtype of the weights if they are in safetensors format

1854 if config_dtype is None:

vllm/entrypoints/llm.py

+2 -3

@@ -117,9 +117,8 @@ class LLM:

117 execution with tensor parallelism.

118 dtype: The data type for the model weights and activations. Currently,

119 we support `float32`, `float16`, and `bfloat16`. If `auto`, we use

120 - the `torch_dtype` attribute specified in the model config file.

121 - However, if the `torch_dtype` in the config is `float32`, we will

122 - use `float16` instead.

120 + the `dtype` attribute of the Transformers model's config. However,

121 + if the `dtype` in the config is `float32`, we will use `float16` instead.

122 quantization: The method used to quantize the model weights. Currently,

123 we support "awq", "gptq", and "fp8" (experimental).

124 If None, we first check the `quantization_config` attribute in the

vllm/model_executor/model_loader/tensorizer.py

+1 -1

@@ -518,7 +518,7 @@ def init_tensorizer_model(

518 ) -> nn.Module:

519 assert tensorizer_config.hf_config is not None

520 model_args = tensorizer_config.hf_config

521 - model_args.torch_dtype = tensorizer_config.dtype

521 + model_args.dtype = tensorizer_config.dtype

522 assert tensorizer_config.model_class is not None

523 # TODO: Do we need to consider old-style model class?

524 with meta_tensor_mode(), set_current_vllm_config(vllm_config, check_compile=True):

vllm/model_executor/models/chameleon.py

+1 -1

@@ -999,7 +999,7 @@ class ChameleonForConditionalGeneration(

999 return []

1000 assert self.model.vqmodel is not None

1001 image_tokens = self.model.get_image_tokens(

1002 - image_input["data"].to(self.config.torch_dtype)

1002 + image_input["data"].to(self.config.dtype)

1003 )

1004 vision_embeddings = self.model.get_input_embeddings(image_tokens)

1005 return vision_embeddings

vllm/model_executor/models/ernie45_vl.py

+1 -1

@@ -1089,7 +1089,7 @@ class Ernie4_5VLMultiModalProcessor(BaseMultiModalProcessor[Ernie4_5_VLProcessin

1089 pixel_values = (

1090 rescale_factor * pixel_values.to(torch.float32) - image_mean_tensor

1091 ) / image_std_tensor

1092 - pixel_values = pixel_values.to(hf_config.torch_dtype)

1092 + pixel_values = pixel_values.to(hf_config.dtype)

1093 return pixel_values

1094

1095 def _call_hf_processor(

vllm/model_executor/models/glm4v.py

+1 -1

@@ -615,7 +615,7 @@ class GLM4VForCausalLM(

615 return None

616

617 def _process_image_input(self, image_input: GLMVImagePixelInputs) -> torch.Tensor:

618 - pixel_values = image_input["data"].to(dtype=self.config.torch_dtype)

618 + pixel_values = image_input["data"].to(dtype=self.config.dtype)

619

620 return self.transformer.vision(pixel_values)

621

vllm/model_executor/models/longcat_flash.py

+2 -2

@@ -114,7 +114,7 @@ class FlashConfig(PretrainedConfig):

114 attention_dropout=0.0,

115 mla_scale_q_lora=False,

116 mla_scale_kv_lora=False,

117 - torch_dtype="bfloat16",

117 + dtype="bfloat16",

118 params_dtype="bfloat16",

119 router_dtype="float32",

120 router_bias=False,

@@ -130,7 +130,7 @@ class FlashConfig(PretrainedConfig):

130 bos_token_id=bos_token_id,

131 eos_token_id=eos_token_id,

132 tie_word_embeddings=tie_word_embeddings,

133 - torch_dtype=torch_dtype,

133 + dtype=dtype,

134 params_dtype=params_dtype,

135 router_dtype=router_dtype,

136 topk_method=topk_method,

vllm/model_executor/models/nano_nemotron_vl.py

+2 -2

@@ -987,7 +987,7 @@ class NemotronH_Nano_VL_V2(

987 prefix=maybe_prefix(prefix, "language_model"),

988 )

989 self.vision_model = self.get_vit_model_from_radio_config(config).to(

990 - self.language_model.config.torch_dtype

990 + self.language_model.config.dtype

991 )

992

993 # Construct the vision projection.

@@ -1008,7 +1008,7 @@ class NemotronH_Nano_VL_V2(

1008 ReLUSquaredActivation(),

1009 nn.Linear(vision_projection_hidden_size, llm_hidden_size, bias=False),

1010 )

1011 - self.mlp1 = self.mlp1.to(self.language_model.config.torch_dtype)

1011 + self.mlp1 = self.mlp1.to(self.language_model.config.dtype)

1012

1013 self.config = config

1014 self.model_config = vllm_config.model_config

vllm/model_executor/models/qwen3_next.py

+3 -3

@@ -338,7 +338,7 @@ class Qwen3NextGatedDeltaNet(nn.Module, MambaBase):

338 group_size=None,

339 norm_before_gate=True,

340 device=current_platform.current_device(),

341 - dtype=config.torch_dtype,

341 + dtype=config.dtype,

342 )

343

344 self.out_proj = RowParallelLinear(

@@ -847,7 +847,7 @@ class Qwen3NextDecoderLayer(nn.Module):

847 1,

848 1,

849 config.hidden_size,

850 - dtype=config.torch_dtype,

850 + dtype=config.dtype,

851 ),

852 )

853 self.ffn_layer_scale = torch.nn.Parameter(

@@ -855,7 +855,7 @@ class Qwen3NextDecoderLayer(nn.Module):

855 1,

856 1,

857 config.hidden_size,

858 - dtype=config.torch_dtype,

858 + dtype=config.dtype,

859 ),

860 )

861

vllm/model_executor/models/transformers.py

+1 -1

@@ -530,7 +530,7 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):

530 with init_on_device_without_buffers("meta"):

531 self.model: PreTrainedModel = AutoModel.from_config(

532 self.config,

533 - torch_dtype=self.model_config.dtype,

533 + dtype=self.model_config.dtype,

534 trust_remote_code=self.model_config.trust_remote_code,

535 )

536

vllm/model_executor/models/transformers_pooling.py

+1 -1

@@ -157,7 +157,7 @@ class TransformersForSequenceClassification(TransformersPoolingBase):

157 with torch.device("meta"):

158 seq_cls_model = AutoModelForSequenceClassification.from_config(

159 self.config,

160 - torch_dtype=self.model_config.dtype,

160 + dtype=self.model_config.dtype,

161 trust_remote_code=self.model_config.trust_remote_code,

162 )

163

vllm/platforms/cuda.py

+2 -2

@@ -500,8 +500,8 @@ class CudaPlatformBase(Platform):

500 return supported

501

502 @classmethod

503 - def check_if_supports_dtype(cls, torch_dtype: torch.dtype):

504 - if torch_dtype == torch.bfloat16: # noqa: SIM102

503 + def check_if_supports_dtype(cls, dtype: torch.dtype):

504 + if dtype == torch.bfloat16: # noqa: SIM102

505 if not cls.has_device_capability(80):

506 capability = cls.get_device_capability()

507 gpu_name = cls.get_device_name()

vllm/platforms/interface.py

+1 -1

@@ -563,7 +563,7 @@ class Platform:

563 return False

564

565 @classmethod

566 - def check_if_supports_dtype(cls, torch_dtype: torch.dtype):

566 + def check_if_supports_dtype(cls, dtype: torch.dtype):

567 """

568 Check if the dtype is supported by the current platform.

569 """

vllm/platforms/rocm.py

+2 -2

@@ -484,8 +484,8 @@ class RocmPlatform(Platform):

484 return True

485

486 @classmethod

487 - def check_if_supports_dtype(cls, torch_dtype: torch.dtype):

488 - if torch_dtype == torch.bfloat16: # noqa: SIM102

487 + def check_if_supports_dtype(cls, dtype: torch.dtype):

488 + if dtype == torch.bfloat16: # noqa: SIM102

489 if not cls.has_device_capability(80):

490 capability = cls.get_device_capability()

491 gpu_name = cls.get_device_name()

vllm/platforms/xpu.py

+2 -2

@@ -236,8 +236,8 @@ class XPUPlatform(Platform):

236 return torch.xpu.device_count()

237

238 @classmethod

239 - def check_if_supports_dtype(cls, torch_dtype: torch.dtype):

240 - if torch_dtype == torch.bfloat16: # noqa: SIM102

239 + def check_if_supports_dtype(cls, dtype: torch.dtype):

240 + if dtype == torch.bfloat16: # noqa: SIM102

241 device_name = cls.get_device_name().lower()

242 # client gpu a770

243 if device_name.count("a770") > 0:

vllm/utils/__init__.py

+6 -8

@@ -806,7 +806,7 @@ def create_kv_caches_with_random_flash(

806

807 current_platform.seed_everything(seed)

808

809 - torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)

809 + dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)

810 generic_kv_cache_shape = (num_blocks, 2, block_size, num_heads, head_size)

811 assert cache_layout in ("NHD", "HND")

812 stride_order = (0, 1, 2, 3, 4) if cache_layout == "NHD" else (0, 1, 3, 2, 4)

@@ -819,7 +819,7 @@ def create_kv_caches_with_random_flash(

819

820 for _ in range(num_layers):

821 key_value_cache = torch.empty(

822 - size=kv_cache_allocation_shape, dtype=torch_dtype, device=device

822 + size=kv_cache_allocation_shape, dtype=dtype, device=device

823 ).permute(*stride_order)

824 if cache_dtype in ["auto", "half", "bfloat16", "float"]:

825 key_value_cache.uniform_(-scale, scale)

@@ -851,14 +851,14 @@ def create_kv_caches_with_random(

851

852 current_platform.seed_everything(seed)

853

854 - torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)

854 + dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)

855

856 scale = head_size**-0.5

857 - x = 16 // torch.tensor([], dtype=torch_dtype).element_size()

857 + x = 16 // torch.tensor([], dtype=dtype).element_size()

858 key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)

859 key_caches: list[torch.Tensor] = []

860 for _ in range(num_layers):

861 - key_cache = torch.empty(size=key_cache_shape, dtype=torch_dtype, device=device)

861 + key_cache = torch.empty(size=key_cache_shape, dtype=dtype, device=device)

862 if cache_dtype in ["auto", "half", "bfloat16", "float"]:

863 key_cache.uniform_(-scale, scale)

864 elif cache_dtype == "fp8":

@@ -870,9 +870,7 @@ def create_kv_caches_with_random(

870 value_cache_shape = (num_blocks, num_heads, head_size, block_size)

871 value_caches: list[torch.Tensor] = []

872 for _ in range(num_layers):

873 - value_cache = torch.empty(

874 - size=value_cache_shape, dtype=torch_dtype, device=device

875 - )

873 + value_cache = torch.empty(size=value_cache_shape, dtype=dtype, device=device)

874 if cache_dtype in ["auto", "half", "bfloat16", "float"]:

875 value_cache.uniform_(-scale, scale)

876 elif cache_dtype == "fp8":

[Misc] rename torch_dtype to dtype (#26695)