A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 18:15 UTC
[Bugfix] update neuron for version > 0.5.0 (#7175)
Signed-off-by: omrishiv <327609+omrishiv@users.noreply.github.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
omrishiv committed
Aug 15, 2024 at 09:44 UTC
9c1f78d5d66c3630ad5f5e149eca6f84b2d254a2
4 files changed
+7
-4
vllm/engine/arg_utils.py
+1
-1
@@ -316,7 +316,7 @@ class EngineArgs:
316
parser.add_argument('--block-size',
317
type=int,
318
default=EngineArgs.block_size,
319
- choices=[8, 16, 32],
319
+ choices=[8, 16, 32, 128, 256, 512, 1024, 2048],
320
help='Token block size for contiguous chunks of '
321
'tokens.')
322
vllm/executor/neuron_executor.py
+2
-3
@@ -100,9 +100,8 @@ class NeuronExecutorAsync(NeuronExecutor, ExecutorAsyncBase):
100
self,
101
execute_model_req: ExecuteModelRequest,
102
) -> List[SamplerOutput]:
103
- output = await make_async(
104
- self.driver_worker.execute_model
105
- )(seq_group_metadata_list=execute_model_req.seq_group_metadata_list, )
103
+ output = await make_async(self.driver_worker.execute_model
104
+ )(execute_model_req=execute_model_req, )
105
return output
106
107
async def check_health_async(self) -> None:
vllm/worker/neuron_model_runner.py
+1
@@ -197,6 +197,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
197
virtual_engine: int = 0,
198
finished_requests_ids: Optional[List[str]] = None
199
) -> ModelInputForNeuron:
200
+ multi_modal_kwargs = None
201
# NOTE: We assume that all sequences in the group are all prompts or
202
# all decodes.
203
is_prompt = seq_group_metadata_list[0].is_prompt
vllm/worker/neuron_worker.py
+3
@@ -89,6 +89,9 @@ class NeuronWorker(LoraNotSupportedWorkerBase, LocalOrDistributedWorkerBase):
89
return WorkerInput(num_seq_groups=len(
90
execute_model_req.seq_group_metadata_list), )
91
92
+ def execute_worker(self, worker_input: WorkerInput) -> None:
93
+ pass
94
+
95
def get_cache_block_size_bytes(self) -> int:
96
"""Determine the size in bytes of a cache block.
97