A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 08:30 UTC
[Neuron] Add multi-LoRA support for Neuron. (#18284)
Signed-off-by: Satyajith Chilappagari <satchill@amazon.com>
Satyajith Chilappagari committed
May 29, 2025 at 01:41 UTC
972eddf7c9cee072bb1f92618072782a4ed0b7c0
6 files changed
+343
-26
tests/neuron/2_core/test_multi_lora.py
new
+98
@@ -0,0 +1,98 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+from huggingface_hub import snapshot_download
4
+
5
+from vllm import LLM, SamplingParams
6
+from vllm.lora.request import LoRARequest
7
+
8
+
9
+def test_llama_single_lora():
10
+ sql_lora_files = snapshot_download(
11
+ repo_id="yard1/llama-2-7b-sql-lora-test")
12
+ llm = LLM(model="meta-llama/Llama-2-7b-hf",
13
+ tensor_parallel_size=2,
14
+ max_num_seqs=4,
15
+ max_model_len=512,
16
+ use_v2_block_manager=True,
17
+ override_neuron_config={
18
+ "sequence_parallel_enabled": False,
19
+ "skip_warmup": True,
20
+ "lora_modules": [{
21
+ "name": "lora_id_1",
22
+ "path": sql_lora_files
23
+ }]
24
+ },
25
+ enable_lora=True,
26
+ max_loras=1,
27
+ max_lora_rank=256,
28
+ device="neuron")
29
+ """For multi-lora requests using NxDI as the backend, only the lora_name
30
+ needs to be specified. The lora_id and lora_path are supplied at the LLM
31
+ class/server initialization, after which the paths are handled by NxDI"""
32
+ lora_req_1 = LoRARequest("lora_id_1", 0, " ")
33
+ prompts = [
34
+ "The president of the United States is",
35
+ "The capital of France is",
36
+ ]
37
+ outputs = llm.generate(prompts,
38
+ SamplingParams(top_k=1),
39
+ lora_request=[lora_req_1, lora_req_1])
40
+
41
+ expected_outputs = [
42
+ " the head of state and head of government of the United States. "
43
+ "The president direct",
44
+ " a city of contrasts. The city is home to the Eiffel Tower"
45
+ ]
46
+
47
+ for expected_output, output in zip(expected_outputs, outputs):
48
+ generated_text = output.outputs[0].text
49
+ assert (expected_output == generated_text)
50
+
51
+
52
+def test_llama_multiple_lora():
53
+ sql_lora_files = snapshot_download(
54
+ repo_id="yard1/llama-2-7b-sql-lora-test")
55
+ llm = LLM(model="meta-llama/Llama-2-7b-hf",
56
+ tensor_parallel_size=2,
57
+ max_num_seqs=4,
58
+ max_model_len=512,
59
+ use_v2_block_manager=True,
60
+ override_neuron_config={
61
+ "sequence_parallel_enabled":
62
+ False,
63
+ "skip_warmup":
64
+ True,
65
+ "lora_modules": [{
66
+ "name": "lora_id_1",
67
+ "path": sql_lora_files
68
+ }, {
69
+ "name": "lora_id_2",
70
+ "path": sql_lora_files
71
+ }]
72
+ },
73
+ enable_lora=True,
74
+ max_loras=2,
75
+ max_lora_rank=256,
76
+ device="neuron")
77
+ """For multi-lora requests using NxDI as the backend, only the lora_name
78
+ needs to be specified. The lora_id and lora_path are supplied at the LLM
79
+ class/server initialization, after which the paths are handled by NxDI"""
80
+ lora_req_1 = LoRARequest("lora_id_1", 0, " ")
81
+ lora_req_2 = LoRARequest("lora_id_2", 1, " ")
82
+ prompts = [
83
+ "The president of the United States is",
84
+ "The capital of France is",
85
+ ]
86
+ outputs = llm.generate(prompts,
87
+ SamplingParams(top_k=1),
88
+ lora_request=[lora_req_1, lora_req_2])
89
+
90
+ expected_outputs = [
91
+ " the head of state and head of government of the United States. "
92
+ "The president direct",
93
+ " a city of contrasts. The city is home to the Eiffel Tower"
94
+ ]
95
+
96
+ for expected_output, output in zip(expected_outputs, outputs):
97
+ generated_text = output.outputs[0].text
98
+ assert (expected_output == generated_text)
vllm/model_executor/model_loader/neuronx_distributed.py
+18
-13
@@ -17,6 +17,8 @@ from neuronx_distributed_inference.models.config import (
17
FusedSpecNeuronConfig, OnDeviceSamplingConfig)
18
from neuronx_distributed_inference.models.mllama.utils import (
19
create_vision_mask)
20
+from neuronx_distributed_inference.modules.lora_serving import (
21
+ LoraServingConfig)
22
from neuronx_distributed_inference.utils.hf_adapter import (
23
load_pretrained_config)
24
from transformers import AutoModelForCausalLM, AutoTokenizer, PretrainedConfig
@@ -80,25 +82,26 @@ class NeuronCausalLM(nn.Module):
82
# Lazy initialized
83
self.model: nn.Module
84
83
- def forward(
84
- self,
85
- input_ids: torch.Tensor,
86
- positions: torch.Tensor,
87
- input_block_ids: torch.Tensor,
88
- sampling_params: torch.Tensor,
89
- ) -> torch.Tensor:
85
+ def forward(self,
86
+ input_ids: torch.Tensor,
87
+ positions: torch.Tensor,
88
+ input_block_ids: torch.Tensor,
89
+ sampling_params: torch.Tensor,
90
+ prev_hidden: Optional[torch.Tensor] = None,
91
+ adapter_ids: Optional[torch.Tensor] = None) -> torch.Tensor:
92
# sort block ids sequentially for perf/neuron support reasons
93
sorted_input_block_ids, sorted_indices = torch.sort(input_block_ids)
94
input_ids = torch.index_select(input_ids, 0, sorted_indices)
95
positions = torch.index_select(positions, 0, sorted_indices)
96
sampling_params = torch.index_select(sampling_params, 0,
97
sorted_indices)
96
-
98
output = self.model(input_ids,
99
attention_mask=None,
100
position_ids=positions,
101
seq_ids=sorted_input_block_ids,
101
- sampling_params=sampling_params)
102
+ sampling_params=sampling_params,
103
+ prev_hidden=prev_hidden,
104
+ adapter_ids=adapter_ids)
105
# on-device sampling
106
if self.config.neuron_config.on_device_sampling_config:
107
output = output.hidden_states
@@ -522,7 +525,8 @@ def _get_model_architecture(config: PretrainedConfig) -> str:
525
526
def _get_default_neuron_config(model_config: ModelConfig,
527
parallel_config: ParallelConfig,
525
- scheduler_config: SchedulerConfig):
528
+ scheduler_config: SchedulerConfig,
529
+ lora_serving_config: LoraServingConfig):
530
"""Generate a neuron config based on vllm config args."""
531
on_device_sampling_config = OnDeviceSamplingConfig(dynamic=True,
532
deterministic=False)
@@ -541,7 +545,7 @@ def _get_default_neuron_config(model_config: ModelConfig,
545
padding_side="right",
546
on_device_sampling_config=on_device_sampling_config,
547
sequence_parallel_enabled=True,
544
- )
548
+ lora_serving_config=lora_serving_config)
549
return neuron_config
550
551
@@ -581,7 +585,8 @@ def _get_neuron_config_after_override(default_neuron_config,
585
586
def get_neuron_model(model_config: ModelConfig,
587
parallel_config: ParallelConfig,
584
- scheduler_config: SchedulerConfig) -> nn.Module:
588
+ scheduler_config: SchedulerConfig,
589
+ lora_serving_config: LoraServingConfig) -> nn.Module:
590
"""Initializes a neuron-optimized model for inference."""
591
model_arch = _get_model_architecture(model_config.hf_config)
592
if model_arch == "MllamaForConditionalGeneration":
@@ -589,7 +594,7 @@ def get_neuron_model(model_config: ModelConfig,
594
else:
595
model = NeuronCausalLM(model_config.hf_config)
596
default_neuron_config_args = _get_default_neuron_config(
592
- model_config, parallel_config, scheduler_config)
597
+ model_config, parallel_config, scheduler_config, lora_serving_config)
598
neuron_config = _get_neuron_config_after_override(
599
default_neuron_config_args, model_config.override_neuron_config)
600
vllm/platforms/neuron.py
-3
@@ -49,9 +49,6 @@ class NeuronPlatform(Platform):
49
if parallel_config.world_size > 1:
50
parallel_config.distributed_executor_backend = "uni"
51
52
- assert (vllm_config.lora_config
53
- is None), "LoRA is not supported for Neuron backend."
54
-
52
if vllm_config.cache_config and vllm_config.model_config:
53
# neuron needs block_size = max_model_len
54
vllm_config.cache_config.block_size = \
vllm/worker/neuron_model_runner.py
+31
-1
@@ -2,13 +2,15 @@
2
3
import os
4
from dataclasses import dataclass
5
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
5
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple, Union
6
7
import torch
8
from torch import nn
9
10
from vllm.config import DeviceConfig, VllmConfig
11
from vllm.logger import init_logger
12
+from vllm.lora.layers import LoRAMapping
13
+from vllm.lora.request import LoRARequest
14
from vllm.model_executor import SamplingMetadata
15
from vllm.model_executor.layers.sampler import SamplerOutput
16
from vllm.model_executor.model_loader.neuron import get_neuron_model
@@ -36,6 +38,7 @@ class ModelInputForNeuron(ModelRunnerInputBase):
38
input_block_ids: Optional[torch.Tensor] = None
39
sampling_metadata: SamplingMetadata = None
40
multi_modal_kwargs: BatchedTensorInputs = None
41
+ adapter_ids: Optional[str] = None
42
43
def as_broadcastable_tensor_dict(
44
self) -> Dict[str, Union[int, torch.Tensor]]:
@@ -80,6 +83,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
83
"The model will run without sliding window.")
84
self.device_config = (self.device_config if self.device_config
85
is not None else DeviceConfig())
86
+ self.lora_config = vllm_config.lora_config
87
self.device = self.device_config.device
88
self.pin_memory = is_pin_memory_available()
89
@@ -378,6 +382,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
382
positions=model_input.input_positions,
383
input_block_ids=model_input.input_block_ids,
384
sampling_params=sampling_params,
385
+ adapter_ids=model_input.adapter_ids,
386
**MultiModalKwargs.as_kwargs(
387
model_input.multi_modal_kwargs or {},
388
dtype=self.model_config.dtype,
@@ -416,3 +421,28 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
421
@property
422
def vocab_size(self) -> int:
423
return self.model_config.get_vocab_size()
424
+
425
+ def remove_all_loras(self):
426
+ raise NotImplementedError(
427
+ "LoRAs are not supported for Transformers NeuronX framework")
428
+
429
+ def set_active_loras(self, lora_requests: Set[LoRARequest],
430
+ lora_mapping: LoRAMapping) -> None:
431
+ raise NotImplementedError(
432
+ "LoRAs are not supported for Transformers NeuronX framework")
433
+
434
+ def add_lora(self, lora_request: LoRARequest):
435
+ raise NotImplementedError(
436
+ "LoRAs are not supported for Transformers NeuronX framework")
437
+
438
+ def remove_lora(self, lora_id: int) -> bool:
439
+ raise NotImplementedError(
440
+ "LoRAs are not supported for Transformers NeuronX framework")
441
+
442
+ def pin_lora(self, lora_id: int) -> bool:
443
+ raise NotImplementedError(
444
+ "LoRAs are not supported for Transformers NeuronX framework")
445
+
446
+ def list_loras(self) -> Set[int]:
447
+ raise NotImplementedError(
448
+ "LoRAs are not supported for Transformers NeuronX framework")
vllm/worker/neuron_worker.py
+38
-4
@@ -1,7 +1,7 @@
1
# SPDX-License-Identifier: Apache-2.0
2
"""A Neuron worker class."""
3
import os
4
-from typing import List, Optional, Tuple
4
+from typing import List, Optional, Set, Tuple
5
6
import torch.distributed
7
@@ -9,19 +9,19 @@ from vllm.config import VllmConfig
9
from vllm.distributed import (ensure_model_parallel_initialized,
10
init_distributed_environment)
11
from vllm.logger import init_logger
12
+from vllm.lora.request import LoRARequest
13
from vllm.model_executor import set_random_seed
14
from vllm.platforms import current_platform
15
from vllm.platforms.neuron import NeuronFramework
16
from vllm.sequence import ExecuteModelRequest
17
from vllm.worker.neuron_model_runner import NeuronModelRunner
17
-from vllm.worker.worker_base import (LocalOrDistributedWorkerBase,
18
- LoRANotSupportedWorkerBase, WorkerBase,
18
+from vllm.worker.worker_base import (LocalOrDistributedWorkerBase, WorkerBase,
19
WorkerInput)
20
21
logger = init_logger(__name__)
22
23
24
-class NeuronWorker(LoRANotSupportedWorkerBase, LocalOrDistributedWorkerBase):
24
+class NeuronWorker(LocalOrDistributedWorkerBase):
25
"""A worker class that executes the model on a group of neuron cores.
26
"""
27
@@ -38,6 +38,7 @@ class NeuronWorker(LoRANotSupportedWorkerBase, LocalOrDistributedWorkerBase):
38
self.rank = rank
39
self.distributed_init_method = distributed_init_method
40
self.is_driver_worker = is_driver_worker
41
+ self.lora_config = vllm_config.lora_config
42
43
if self.model_config.trust_remote_code:
44
# note: lazy import to avoid importing torch before initializing
@@ -59,6 +60,9 @@ class NeuronWorker(LoRANotSupportedWorkerBase, LocalOrDistributedWorkerBase):
60
"[transformers-neuronx, neuronx-distributed-inference]")
61
62
def get_tnx_model_runner(self, vllm_config):
63
+ assert (self.lora_config
64
+ is None), ("LoRA is not supported for TransformersNeuronX "
65
+ "framework.")
66
from vllm.worker.multi_step_neuron_model_runner import (
67
MultiStepNeuronModelRunner)
68
if self.speculative_config is not None:
@@ -72,6 +76,8 @@ class NeuronWorker(LoRANotSupportedWorkerBase, LocalOrDistributedWorkerBase):
76
from vllm.worker.neuronx_distributed_model_runner import (
77
NeuronxDistributedModelRunner)
78
if self.speculative_config is not None:
79
+ assert (self.lora_config
80
+ is None), "LoRA is not supported for Speculative Decoding"
81
return MultiStepNeuronxDistributedModelRunner(
82
vllm_config=vllm_config)
83
else:
@@ -156,3 +162,31 @@ class NeuronWorker(LoRANotSupportedWorkerBase, LocalOrDistributedWorkerBase):
162
1,
163
1,
164
)
165
+
166
+ def add_lora(self, lora_request: LoRARequest) -> bool:
167
+ if current_platform.use_transformers_neuronx():
168
+ raise NotImplementedError(
169
+ f"{type(self)} does not support LoRA with Neuron Framework "
170
+ f"Transformers NeuronX")
171
+ return self.model_runner.add_lora(lora_request)
172
+
173
+ def remove_lora(self, lora_id: int) -> bool:
174
+ if current_platform.use_transformers_neuronx():
175
+ raise NotImplementedError(
176
+ f"{type(self)} does not support LoRA with Neuron Framework "
177
+ f"Transformers NeuronX")
178
+ return self.model_runner.remove_lora(lora_id)
179
+
180
+ def pin_lora(self, lora_id: int) -> bool:
181
+ if current_platform.use_transformers_neuronx():
182
+ raise NotImplementedError(
183
+ f"{type(self)} does not support LoRA with Neuron Framework "
184
+ f"Transformers NeuronX")
185
+ return self.model_runner.pin_lora(lora_id)
186
+
187
+ def list_loras(self) -> Set[int]:
188
+ if current_platform.use_transformers_neuronx():
189
+ raise NotImplementedError(
190
+ f"{type(self)} does not support LoRA with Neuron Framework "
191
+ f"Transformers NeuronX")
192
+ return self.model_runner.list_loras()
vllm/worker/neuronx_distributed_model_runner.py
+158
-5
@@ -1,17 +1,24 @@
1
# SPDX-License-Identifier: Apache-2.0
2
3
-from typing import List, Optional
3
+from typing import List, Optional, Set
4
5
import torch
6
from neuronx_distributed_inference.modules.generation.sampling import (
7
prepare_sampling_params)
8
+from neuronx_distributed_inference.modules.lora_serving import (
9
+ LoraCheckpoint, LoraServingConfig)
10
11
from vllm.config import VllmConfig
12
+from vllm.entrypoints.openai.serving_models import LoRAModulePath
13
from vllm.logger import init_logger
14
+from vllm.lora.layers import LoRAMapping
15
+from vllm.lora.request import LoRARequest
16
+from vllm.model_executor import SamplingMetadata
17
from vllm.model_executor.layers.sampler import SamplerOutput
18
from vllm.model_executor.model_loader.neuronx_distributed import (
19
_get_model_architecture, get_neuron_model)
14
-from vllm.sequence import IntermediateTensors
20
+from vllm.platforms import current_platform
21
+from vllm.sequence import IntermediateTensors, SequenceGroupMetadata
22
from vllm.worker.neuron_model_runner import (ModelInputForNeuron,
23
NeuronModelRunner)
24
@@ -25,11 +32,44 @@ class NeuronxDistributedModelRunner(NeuronModelRunner):
32
vllm_config: VllmConfig,
33
):
34
super().__init__(vllm_config)
35
+ self.lora_checkpoint = None
36
+ self.model = None
37
+ self.lora_serving_config = None
38
+
39
+ @staticmethod
40
+ def _get_lora_paths_strings(lora_modules: List[LoRAModulePath]):
41
+ if not lora_modules:
42
+ return None
43
+ return {_.get("name"): _.get("path") for _ in lora_modules}
44
+
45
+ def _get_nxdi_lora_config(self):
46
+ override_neuron_config = self.model_config.override_neuron_config
47
+ lora_modules = override_neuron_config.pop("lora_modules", None)
48
+ target_modules = override_neuron_config.pop("target_modules", None)
49
+ lora_ckpt_paths = self._get_lora_paths_strings(lora_modules)
50
+ if self.lora_config.max_loras < len(lora_ckpt_paths):
51
+ raise ValueError(
52
+ "Number of LoRAs (%s) exceeds maximum "
53
+ "allowed (%s)", len(lora_ckpt_paths),
54
+ self.lora_config.max_loras)
55
+
56
+ return LoraServingConfig(
57
+ max_loras=self.lora_config.max_loras,
58
+ max_lora_rank=self.lora_config.max_lora_rank,
59
+ target_modules=target_modules,
60
+ lora_ckpt_paths=lora_ckpt_paths,
61
+ )
62
63
def load_model(self) -> None:
30
- self.model = get_neuron_model(self.model_config,
31
- parallel_config=self.parallel_config,
32
- scheduler_config=self.scheduler_config)
64
+ # Update LoRA config
65
+ if self.lora_config is not None:
66
+ self.lora_serving_config = self._get_nxdi_lora_config()
67
+ self.lora_checkpoint = LoraCheckpoint(self.lora_serving_config)
68
+ self.model = get_neuron_model(
69
+ self.model_config,
70
+ parallel_config=self.parallel_config,
71
+ scheduler_config=self.scheduler_config,
72
+ lora_serving_config=self.lora_serving_config)
73
74
def get_nxd_sampling_params(self, sampling_metadata):
75
if self.model.config.neuron_config.on_device_sampling_config:
@@ -134,3 +174,116 @@ class NeuronxDistributedModelRunner(NeuronModelRunner):
174
)
175
176
return [output]
177
+
178
+ def _get_lora_adapter_ids(self, seq_group_metadata_list):
179
+ # set LoRA adapter IDs for multi-lora serving
180
+ batch_size = len(seq_group_metadata_list)
181
+ if self.lora_checkpoint is not None:
182
+ # "0" indicates NxDI to use the base model for inference
183
+ adapter_ids = ["0"] * batch_size
184
+ for idx, seq_group_metadata in enumerate(seq_group_metadata_list):
185
+ if seq_group_metadata.lora_request is not None:
186
+ adapter_ids[
187
+ idx] = seq_group_metadata.lora_request.lora_name
188
+
189
+ # convert adapter_ids from strings to integers
190
+ adapter_ids = self.lora_checkpoint.convert_adapter_ids_to_indices(
191
+ adapter_ids, batch_size)
192
+ else:
193
+ adapter_ids = torch.zeros((batch_size), dtype=torch.int32)
194
+
195
+ return adapter_ids
196
+
197
+ def prepare_model_input(
198
+ self,
199
+ seq_group_metadata_list: List[SequenceGroupMetadata],
200
+ virtual_engine: int = 0,
201
+ finished_requests_ids: Optional[List[str]] = None
202
+ ) -> ModelInputForNeuron:
203
+ multi_modal_kwargs = None
204
+ # NOTE: We assume that all sequences in the group are all prompts or
205
+ # all decodes.
206
+ is_prompt = seq_group_metadata_list[0].is_prompt
207
+ # Prepare input tensors.
208
+ if is_prompt:
209
+ (input_tokens, input_positions, input_block_ids, seq_lens,
210
+ multi_modal_kwargs
211
+ ) = self._prepare_prompt(seq_group_metadata_list)
212
+ else:
213
+ (input_tokens, input_positions,
214
+ input_block_ids) = self._prepare_decode(seq_group_metadata_list)
215
+ seq_lens = None
216
+
217
+ if not self._on_device_sampling_disabled:
218
+ for seq_group_metadata in seq_group_metadata_list:
219
+ sampling_params = seq_group_metadata.sampling_params
220
+ top_k, top_p, temperature = (
221
+ self._convert_to_neuron_sampling_params(sampling_params))
222
+ sampling_params.top_k = top_k
223
+ sampling_params.top_p = top_p
224
+ sampling_params.temperature = temperature
225
+
226
+ lora_adapter_ids = self._get_lora_adapter_ids(seq_group_metadata_list)
227
+
228
+ sampling_metadata = SamplingMetadata.prepare(
229
+ seq_group_metadata_list,
230
+ seq_lens,
231
+ # query_lens is not needed if chunked prefill is not
232
+ # supported. Since neuron worker doesn't support chunked prefill
233
+ # just use seq_lens instead.
234
+ seq_lens,
235
+ self.device,
236
+ self.pin_memory,
237
+ generators=self.get_generators(finished_requests_ids))
238
+
239
+ if current_platform.use_transformers_neuronx(
240
+ ) and not self._on_device_sampling_disabled:
241
+ # Once the request IDs are changed in current iteration, we will
242
+ # update the on-device sampling parameters.
243
+ current_batch_request_ids = [
244
+ seq_group_meta_data.request_id
245
+ for seq_group_meta_data in seq_group_metadata_list
246
+ ]
247
+ if current_batch_request_ids != self._previous_batch_request_ids:
248
+ self._update_neuron_sampling_params(seq_group_metadata_list)
249
+ self._previous_batch_request_ids = current_batch_request_ids
250
+
251
+ return ModelInputForNeuron(input_tokens=input_tokens,
252
+ input_positions=input_positions,
253
+ input_block_ids=input_block_ids,
254
+ sampling_metadata=sampling_metadata,
255
+ multi_modal_kwargs=multi_modal_kwargs,
256
+ adapter_ids=lora_adapter_ids)
257
+
258
+ def remove_all_loras(self):
259
+ raise NotImplementedError(
260
+ "Managing LoRAs is only supported through the "
261
+ "lora_modules parameter in override_neuron_config")
262
+
263
+ def set_active_loras(self, lora_requests: Set[LoRARequest],
264
+ lora_mapping: LoRAMapping) -> None:
265
+ raise NotImplementedError(
266
+ "Managing LoRAs is only supported through the "
267
+ "lora_modules parameter in override_neuron_config")
268
+
269
+ def add_lora(self, lora_request: LoRARequest):
270
+ logger.warning(
271
+ "Adding LoRAs is only supported through the "
272
+ "lora_modules parameter in override_neuron_config. If you supplied "
273
+ "the parameter, you can ignore this warning. Ignoring"
274
+ "lora request: ", lora_request)
275
+
276
+ def remove_lora(self, lora_id: int) -> bool:
277
+ raise NotImplementedError(
278
+ "Managing LoRAs is only supported through the "
279
+ "lora_modules parameter in override_neuron_config")
280
+
281
+ def pin_lora(self, lora_id: int) -> bool:
282
+ raise NotImplementedError(
283
+ "Managing LoRAs is only supported through the "
284
+ "lora_modules parameter in override_neuron_config")
285
+
286
+ def list_loras(self) -> Set[int]:
287
+ raise NotImplementedError(
288
+ "Managing LoRAs is only supported through the "
289
+ "lora_modules parameter in override_neuron_config")