@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9b14d978aa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

Fix openvino on GPU (#10793)

Jani Monoses committed Dec 2, 2024 at 20:52 UTC 9b14d978aa8c286b738f107fab4626273f4fc088
1 file changed +3 -3
vllm/worker/openvino_worker.py
+3 -3
@@ -489,7 +489,7 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
489 block_size = cache_config.block_size
490 seq_num_blocks = (seq_len + block_size - 1) // block_size
491
492 - seq_data, dummy_multi_modal_data = input_registry \
492 + dummy_data = input_registry \
493 .dummy_data_for_profiling(model_config,
494 seq_len,
495 mm_registry)
@@ -498,11 +498,11 @@ class OpenVINOWorker(LoraNotSupportedWorkerBase):
498 seq = SequenceGroupMetadata(
499 request_id=str(group_id),
500 is_prompt=True,
501 - seq_data={group_id: seq_data},
501 + seq_data={group_id: dummy_data.seq_data},
502 sampling_params=sampling_params,
503 block_tables=block_tables,
504 lora_request=None,
505 - multi_modal_data=dummy_multi_modal_data)
505 + multi_modal_data=dummy_data.multi_modal_data)
506 seqs.append(seq)
507
508 self.model_runner.block_size = tmp_cache_config.block_size