@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9c1f78d5d6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 18:15 UTC

[Bugfix] update neuron for version > 0.5.0 (#7175)

Signed-off-by: omrishiv <327609+omrishiv@users.noreply.github.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

omrishiv committed Aug 15, 2024 at 09:44 UTC 9c1f78d5d66c3630ad5f5e149eca6f84b2d254a2
4 files changed +7 -4
vllm/engine/arg_utils.py
+1 -1
@@ -316,7 +316,7 @@ class EngineArgs:
316 parser.add_argument('--block-size',
317 type=int,
318 default=EngineArgs.block_size,
319 - choices=[8, 16, 32],
319 + choices=[8, 16, 32, 128, 256, 512, 1024, 2048],
320 help='Token block size for contiguous chunks of '
321 'tokens.')
322
vllm/executor/neuron_executor.py
+2 -3
@@ -100,9 +100,8 @@ class NeuronExecutorAsync(NeuronExecutor, ExecutorAsyncBase):
100 self,
101 execute_model_req: ExecuteModelRequest,
102 ) -> List[SamplerOutput]:
103 - output = await make_async(
104 - self.driver_worker.execute_model
105 - )(seq_group_metadata_list=execute_model_req.seq_group_metadata_list, )
103 + output = await make_async(self.driver_worker.execute_model
104 + )(execute_model_req=execute_model_req, )
105 return output
106
107 async def check_health_async(self) -> None:
vllm/worker/neuron_model_runner.py
+1
@@ -197,6 +197,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
197 virtual_engine: int = 0,
198 finished_requests_ids: Optional[List[str]] = None
199 ) -> ModelInputForNeuron:
200 + multi_modal_kwargs = None
201 # NOTE: We assume that all sequences in the group are all prompts or
202 # all decodes.
203 is_prompt = seq_group_metadata_list[0].is_prompt
vllm/worker/neuron_worker.py
+3
@@ -89,6 +89,9 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
89 return WorkerInput(num_seq_groups=len(
90 execute_model_req.seq_group_metadata_list), )
91
92 + def execute_worker(self, worker_input: WorkerInput) -> None:
93 + pass
94 +
95 def get_cache_block_size_bytes(self) -> int:
96 """Determine the size in bytes of a cache block.
97