@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3ee5c4bca5
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 23:15 UTC

[ci] Add A100 queue into AWS CI template (#5648)

Signed-off-by: kevin <kevin@anyscale.com>

Kevin H. Luu committed Jun 19, 2024 at 07:42 UTC 3ee5c4bca514ee95592a018fae95e050fd6763c0
3 files changed +52
.buildkite/nightly-benchmarks/benchmark-pipeline.yaml
+1
@@ -17,6 +17,7 @@ steps:
17 plugins:
18 - kubernetes:
19 podSpec:
20 + priorityClassName: perf-benchmark
21 containers:
22 - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
23 command:
.buildkite/test-pipeline.yaml
+5
@@ -181,3 +181,8 @@ steps:
181 commands:
182 - pip install -r requirements-docs.txt
183 - SPHINXOPTS=\"-W\" make html
184 +
185 +- label: A100 status
186 + gpu: a100
187 + commands:
188 + - nvidia-smi
.buildkite/test-template-aws.j2
+46
@@ -49,6 +49,51 @@ steps:
49 command: bash .buildkite/run-cpu-test.sh
50
51 {% for step in steps %}
52 + {% if step.gpu == "a100" %}
53 + - label: "{{ step.label }}"
54 + agents:
55 + queue: a100-queue
56 + soft_fail: {{ step.soft_fail or false }}
57 + {% if step.parallelism %}
58 + parallelism: {{ step.parallelism }}
59 + {% endif %}
60 + retry:
61 + automatic:
62 + - exit_status: -1 # Agent was lost
63 + limit: 5
64 + - exit_status: -10 # Agent was lost
65 + limit: 5
66 + plugins:
67 + - kubernetes:
68 + podSpec:
69 + priorityClassName: ci
70 + containers:
71 + - image: {{ docker_image }}
72 + command: ["bash"]
73 + args:
74 + - '-c'
75 + - "'cd {{ (step.working_dir or default_working_dir) | safe }} && {{ step.command or (step.commands | join(' && ')) | safe }}'"
76 + resources:
77 + limits:
78 + nvidia.com/gpu: 8
79 + volumeMounts:
80 + - name: devshm
81 + mountPath: /dev/shm
82 + env:
83 + - name: VLLM_USAGE_SOURCE
84 + value: ci-test
85 + - name: HF_TOKEN
86 + valueFrom:
87 + secretKeyRef:
88 + name: hf-token-secret
89 + key: token
90 + nodeSelector:
91 + nvidia.com/gpu.product: NVIDIA-A100-SXM4-80GB
92 + volumes:
93 + - name: devshm
94 + emptyDir:
95 + medium: Memory
96 + {% else %}
97 - label: "{{ step.label }}"
98 agents:
99 {% if step.label == "Documentation Build" %}
@@ -90,4 +135,5 @@ steps:
135 {% endif %}
136 volumes:
137 - /dev/shm:/dev/shm
138 + {% endif %}
139 {% endfor %}