A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 23:15 UTC
[ci] Add A100 queue into AWS CI template (#5648)
Signed-off-by: kevin <kevin@anyscale.com>
Kevin H. Luu committed
Jun 19, 2024 at 07:42 UTC
3ee5c4bca514ee95592a018fae95e050fd6763c0
3 files changed
+52
.buildkite/nightly-benchmarks/benchmark-pipeline.yaml
+1
@@ -17,6 +17,7 @@ steps:
17
plugins:
18
- kubernetes:
19
podSpec:
20
+ priorityClassName: perf-benchmark
21
containers:
22
- image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
23
command:
.buildkite/test-pipeline.yaml
+5
@@ -181,3 +181,8 @@ steps:
181
commands:
182
- pip install -r requirements-docs.txt
183
- SPHINXOPTS=\"-W\" make html
184
+
185
+- label: A100 status
186
+ gpu: a100
187
+ commands:
188
+ - nvidia-smi
.buildkite/test-template-aws.j2
+46
@@ -49,6 +49,51 @@ steps:
49
command: bash .buildkite/run-cpu-test.sh
50
51
{% for step in steps %}
52
+ {% if step.gpu == "a100" %}
53
+ - label: "{{ step.label }}"
54
+ agents:
55
+ queue: a100-queue
56
+ soft_fail: {{ step.soft_fail or false }}
57
+ {% if step.parallelism %}
58
+ parallelism: {{ step.parallelism }}
59
+ {% endif %}
60
+ retry:
61
+ automatic:
62
+ - exit_status: -1 # Agent was lost
63
+ limit: 5
64
+ - exit_status: -10 # Agent was lost
65
+ limit: 5
66
+ plugins:
67
+ - kubernetes:
68
+ podSpec:
69
+ priorityClassName: ci
70
+ containers:
71
+ - image: {{ docker_image }}
72
+ command: ["bash"]
73
+ args:
74
+ - '-c'
75
+ - "'cd {{ (step.working_dir or default_working_dir) | safe }} && {{ step.command or (step.commands | join(' && ')) | safe }}'"
76
+ resources:
77
+ limits:
78
+ nvidia.com/gpu: 8
79
+ volumeMounts:
80
+ - name: devshm
81
+ mountPath: /dev/shm
82
+ env:
83
+ - name: VLLM_USAGE_SOURCE
84
+ value: ci-test
85
+ - name: HF_TOKEN
86
+ valueFrom:
87
+ secretKeyRef:
88
+ name: hf-token-secret
89
+ key: token
90
+ nodeSelector:
91
+ nvidia.com/gpu.product: NVIDIA-A100-SXM4-80GB
92
+ volumes:
93
+ - name: devshm
94
+ emptyDir:
95
+ medium: Memory
96
+ {% else %}
97
- label: "{{ step.label }}"
98
agents:
99
{% if step.label == "Documentation Build" %}
@@ -90,4 +135,5 @@ steps:
135
{% endif %}
136
volumes:
137
- /dev/shm:/dev/shm
138
+ {% endif %}
139
{% endfor %}