@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / adf32e0a0f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Bugfix] Fix usage stats logging exception warning with OpenVINO (#6349)

Helena Kloosterman committed Jul 12, 2024 at 04:47 UTC adf32e0a0f58edafe0c71b5c235848a487be2a71
2 files changed +2 -2
requirements-openvino.txt
+1 -1
@@ -4,6 +4,6 @@
4 # OpenVINO dependencies
5 torch >= 2.1.2
6 openvino ~= 2024.3.0.dev
7 -optimum-intel[openvino] >= 1.17.2
7 +optimum-intel[openvino] >= 1.18.1
8
9 triton >= 2.2.0 # FIXME(woosuk): This is a hack to avoid import error.
vllm/engine/llm_engine.py
+1 -1
@@ -284,7 +284,7 @@ class LLMEngine:
284 "quantization":
285 model_config.quantization,
286 "kv_cache_dtype":
287 - cache_config.cache_dtype,
287 + str(cache_config.cache_dtype),
288
289 # Feature flags
290 "enable_lora":