A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Fix Weight Loading Multiple GPU Test - Large Models (#9213)
Michael Goin committed
Oct 10, 2024 at 02:15 UTC
ce00231a8bfb5eae85167b5a3def1b7304c723b6
2 files changed
+1
-1
tests/weight_loading/models-large.txt
-1
@@ -1,6 +1,5 @@
1
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-quantized, main
2
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W4A16-channel-quantized, main
3
compressed-tensors, nm-testing/Mixtral-8x7B-Instruct-v0.1-W8A16-quantized, main
4
-compressed-tensors, mgoin/DeepSeek-Coder-V2-Lite-Instruct-FP8, main
4
gptq_marlin, TheBloke/Mixtral-8x7B-v0.1-GPTQ, main
5
awq_marlin, casperhansen/deepseek-coder-v2-instruct-awq, main
\ No newline at end of file
tests/weight_loading/models.txt
+1
@@ -20,6 +20,7 @@ compressed-tensors, nm-testing/Meta-Llama-3-8B-FP8-compressed-tensors-test, main
20
compressed-tensors, nm-testing/Phi-3-mini-128k-instruct-FP8, main
21
compressed-tensors, neuralmagic/Phi-3-medium-128k-instruct-quantized.w4a16, main
22
compressed-tensors, nm-testing/TinyLlama-1.1B-Chat-v1.0-actorder-group, main
23
+compressed-tensors, mgoin/DeepSeek-Coder-V2-Lite-Instruct-FP8, main
24
awq, casperhansen/mixtral-instruct-awq, main
25
awq_marlin, casperhansen/mixtral-instruct-awq, main
26
fp8, neuralmagic/Meta-Llama-3-8B-Instruct-FP8-KV, main