A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix] Fix usage stats logging exception warning with OpenVINO (#6349)
Helena Kloosterman committed
Jul 12, 2024 at 04:47 UTC
adf32e0a0f58edafe0c71b5c235848a487be2a71
2 files changed
+2
-2
requirements-openvino.txt
+1
-1
@@ -4,6 +4,6 @@
4
# OpenVINO dependencies
5
torch >= 2.1.2
6
openvino ~= 2024.3.0.dev
7
-optimum-intel[openvino] >= 1.17.2
7
+optimum-intel[openvino] >= 1.18.1
8
9
triton >= 2.2.0 # FIXME(woosuk): This is a hack to avoid import error.
vllm/engine/llm_engine.py
+1
-1
@@ -284,7 +284,7 @@ class LLMEngine:
284
"quantization":
285
model_config.quantization,
286
"kv_cache_dtype":
287
- cache_config.cache_dtype,
287
+ str(cache_config.cache_dtype),
288
289
# Feature flags
290
"enable_lora":