A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[BugFix] Fix DeepSeek remote code (#7178)
Dipika Sikka committed
Aug 6, 2024 at 11:16 UTC
a3bbbfa1d8c2f30581d37c6f30429d648bbbf87c
2 files changed
+5
-1
.buildkite/lm-eval-harness/configs/DeepSeek-V2-Lite-Chat.yaml
+1
@@ -9,3 +9,4 @@ tasks:
9
value: 0.664
10
limit: 1000
11
num_fewshot: 5
12
+trust_remote_code: True
\ No newline at end of file
.buildkite/lm-eval-harness/test_lm_eval_correctness.py
+4
-1
@@ -23,9 +23,12 @@ TP_SIZE = os.environ.get("LM_EVAL_TP_SIZE", 1)
23
24
25
def launch_lm_eval(eval_config):
26
+ trust_remote_code = eval_config.get('trust_remote_code', False)
27
+
28
model_args = f"pretrained={eval_config['model_name']}," \
29
f"tensor_parallel_size={TP_SIZE}," \
28
- f"add_bos_token=true"
30
+ f"add_bos_token=true," \
31
+ f"trust_remote_code={trust_remote_code}"
32
33
results = lm_eval.simple_evaluate(
34
model="vllm",