@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 73030b7dae
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[ Misc ] Enable Quantizing All Layers of DeekSeekv2 (#6423)

Robert Shaw committed Jul 14, 2024 at 17:38 UTC 73030b7dae676f730ea43652af056501584ecbfe
2 files changed +6 -1
.buildkite/lm-eval-harness/run-lm-eval-gsm-vllm-baseline.sh
+1 -1
@@ -46,6 +46,6 @@ while getopts "m:b:l:f:t:" OPT; do
46 done
47
48 lm_eval --model vllm \
49 - --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,distributed_executor_backend="ray",trust_remote_code=true \
49 + --model_args pretrained=$MODEL,tensor_parallel_size=$TP_SIZE,add_bos_token=true,distributed_executor_backend="ray",trust_remote_code=true,max_model_len=4096 \
50 --tasks gsm8k --num_fewshot $FEWSHOT --limit $LIMIT \
51 --batch_size $BATCH_SIZE
vllm/model_executor/model_loader/weight_utils.py
+5
@@ -431,6 +431,11 @@ def convert_pyslice_to_tensor(x: Any) -> torch.Tensor:
431 def default_weight_loader(param: torch.Tensor,
432 loaded_weight: torch.Tensor) -> None:
433 """Default weight loader."""
434 + # If the weight on disk does not have a shape, give it one
435 + # (such scales for AutoFp8).
436 + if len(loaded_weight.shape) == 0:
437 + loaded_weight = loaded_weight.reshape(1)
438 +
439 assert param.size() == loaded_weight.size()
440 param.data.copy_(loaded_weight)
441