A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 23:15 UTC
[ModelOpt] Support Qwen3.5/3.6 VLM quantized prefix mapping (#42546)
Signed-off-by: weimingc <17592131+meenchen@users.noreply.github.com>
Wei-Ming Chen committed
May 22, 2026 at 23:23 UTC
09a219c075276f17a18088233a593479fada9345
1 file changed
+33
-10
vllm/model_executor/layers/quantization/modelopt.py
+33
-10
@@ -2330,18 +2330,22 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
2330
is not found.
2331
"""
2332
# 1. Direct lookup
2333
- if prefix in self.quantized_layers:
2334
- return self.quantized_layers[prefix]["quant_algo"].upper()
2333
+ for candidate in self._quantized_layer_prefix_candidates(prefix):
2334
+ if candidate in self.quantized_layers:
2335
+ return self.quantized_layers[candidate]["quant_algo"].upper()
2336
2337
# 2. Packed / fused layer lookup
2338
proj_name = prefix.rsplit(".", 1)[-1]
2339
if self.packed_modules_mapping and proj_name in self.packed_modules_mapping:
2340
algos: set[str] = set()
2341
base = prefix.rsplit(".", 1)[0]
2341
- for shard_name in self.packed_modules_mapping[proj_name]:
2342
- shard_prefix = f"{base}.{shard_name}"
2343
- if shard_prefix in self.quantized_layers:
2344
- algos.add(self.quantized_layers[shard_prefix]["quant_algo"].upper())
2342
+ for base_candidate in self._quantized_layer_prefix_candidates(base):
2343
+ for shard_name in self.packed_modules_mapping[proj_name]:
2344
+ shard_prefix = f"{base_candidate}.{shard_name}"
2345
+ if shard_prefix in self.quantized_layers:
2346
+ algos.add(
2347
+ self.quantized_layers[shard_prefix]["quant_algo"].upper()
2348
+ )
2349
if len(algos) == 1:
2350
return algos.pop()
2351
if len(algos) > 1:
@@ -2351,13 +2355,32 @@ class ModelOptMixedPrecisionConfig(ModelOptQuantConfigBase):
2355
)
2356
2357
# 3. Prefix-based lookup (for RoutedExperts / parent modules)
2354
- prefix_dot = prefix + "."
2355
- for key, info in self.quantized_layers.items():
2356
- if key.startswith(prefix_dot):
2357
- return info["quant_algo"].upper()
2358
+ for candidate in self._quantized_layer_prefix_candidates(prefix):
2359
+ prefix_dot = candidate + "."
2360
+ for key, info in self.quantized_layers.items():
2361
+ if key.startswith(prefix_dot):
2362
+ return info["quant_algo"].upper()
2363
2364
return None
2365
2366
+ @staticmethod
2367
+ def _quantized_layer_prefix_candidates(prefix: str) -> tuple[str, ...]:
2368
+ candidates = [prefix]
2369
+
2370
+ if prefix.endswith(".lm_head"):
2371
+ candidates.append("lm_head")
2372
+
2373
+ if prefix.startswith("language_model.model."):
2374
+ candidates.append(
2375
+ "model.language_model." + prefix[len("language_model.model.") :]
2376
+ )
2377
+ elif prefix.startswith("model.language_model."):
2378
+ candidates.append(
2379
+ "language_model.model." + prefix[len("model.language_model.") :]
2380
+ )
2381
+
2382
+ return tuple(dict.fromkeys(candidates))
2383
+
2384
def get_quant_method(
2385
self, layer: torch.nn.Module, prefix: str
2386
) -> "QuantizeMethodBase | None":