A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
Fix openvino on GPU (#10793)
Jani Monoses committed
Dec 2, 2024 at 20:52 UTC
9b14d978aa8c286b738f107fab4626273f4fc088
1 file changed
+3
-3
vllm/worker/openvino_worker.py
+3
-3
@@ -489,7 +489,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
489
block_size = cache_config.block_size
490
seq_num_blocks = (seq_len + block_size - 1) // block_size
491
492
- seq_data, dummy_multi_modal_data = input_registry \
492
+ dummy_data = input_registry \
493
.dummy_data_for_profiling(model_config,
494
seq_len,
495
mm_registry)
@@ -498,11 +498,11 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
498
seq = SequenceGroupMetadata(
499
request_id=str(group_id),
500
is_prompt=True,
501
- seq_data={group_id: seq_data},
501
+ seq_data={group_id: dummy_data.seq_data},
502
sampling_params=sampling_params,
503
block_tables=block_tables,
504
lora_request=None,
505
- multi_modal_data=dummy_multi_modal_data)
505
+ multi_modal_data=dummy_data.multi_modal_data)
506
seqs.append(seq)
507
508
self.model_runner.block_size = tmp_cache_config.block_size