@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / a3bbbfa1d8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[BugFix] Fix DeepSeek remote code (#7178)

Dipika Sikka committed Aug 6, 2024 at 11:16 UTC a3bbbfa1d8c2f30581d37c6f30429d648bbbf87c
2 files changed +5 -1
.buildkite/lm-eval-harness/configs/DeepSeek-V2-Lite-Chat.yaml
+1
@@ -9,3 +9,4 @@ tasks:
9 value: 0.664
10 limit: 1000
11 num_fewshot: 5
12 +trust_remote_code: True
\ No newline at end of file
.buildkite/lm-eval-harness/test_lm_eval_correctness.py
+4 -1
@@ -23,9 +23,12 @@ TP_SIZE = os.environ.get("LM_EVAL_TP_SIZE", 1)
23
24
25 def launch_lm_eval(eval_config):
26 + trust_remote_code = eval_config.get('trust_remote_code', False)
27 +
28 model_args = f"pretrained={eval_config['model_name']}," \
29 f"tensor_parallel_size={TP_SIZE}," \
28 - f"add_bos_token=true"
30 + f"add_bos_token=true," \
31 + f"trust_remote_code={trust_remote_code}"
32
33 results = lm_eval.simple_evaluate(
34 model="vllm",