@tduncan / Nvidia-Optimization-Stack-VLLM
Code Commits Issues Pulls Cloud Agent CI/CD
main
main
yaml 174 lines 5.23 KB
Raw
1 # -- Default values for chart vllm
2 # -- Declare variables to be passed into your templates.
3
4 # -- Image configuration
5 image:
6 # -- Image repository
7 repository: "vllm/vllm-openai"
8 # -- Image tag
9 tag: "latest"
10 # -- Container launch command
11 command: ["vllm", "serve", "/data/", "--served-model-name", "opt-125m", "--enforce-eager", "--dtype", "bfloat16", "--block-size", "16", "--host", "0.0.0.0", "--port", "8000"]
12
13 # -- Container port
14 containerPort: 8000
15 # -- Service name
16 serviceName:
17 # -- Service port
18 servicePort: 80
19 # -- Additional ports configuration
20 extraPorts: []
21
22 # -- Number of replicas
23 replicaCount: 1
24
25 # -- Deployment strategy configuration
26 deploymentStrategy: {}
27
28 # -- Resource configuration
29 resources:
30 requests:
31 # -- Number of CPUs
32 cpu: 4
33 # -- CPU memory configuration
34 memory: 16Gi
35 # -- Number of gpus used
36 nvidia.com/gpu: 1
37 limits:
38 # -- Number of CPUs
39 cpu: 4
40 # -- CPU memory configuration
41 memory: 16Gi
42 # -- Number of gpus used
43 nvidia.com/gpu: 1
44
45 # -- Type of gpu used
46 gpuModels:
47 - "TYPE_GPU_USED"
48
49 # -- Autoscaling configuration
50 autoscaling:
51 # -- Enable autoscaling
52 enabled: false
53 # -- Minimum replicas
54 minReplicas: 1
55 # -- Maximum replicas
56 maxReplicas: 100
57 # -- Target CPU utilization for autoscaling
58 targetCPUUtilizationPercentage: 80
59 # targetMemoryUtilizationPercentage: 80
60
61 # -- Configmap
62 configs: {}
63
64 # -- Secrets configuration
65 secrets: {}
66
67 # -- External configuration
68 externalConfigs: []
69
70 # -- Custom Objects configuration
71 customObjects: []
72
73 # -- Disruption Budget Configuration
74 maxUnavailablePodDisruptionBudget: ""
75
76 # -- Additional configuration for the init container
77 extraInit:
78 # -- Model download functionality (optional)
79 modelDownload:
80 # -- Enable model download job and wait container
81 enabled: true
82 # -- Image configuration for model download operations
83 image:
84 # -- Image repository
85 repository: "amazon/aws-cli"
86 # -- Image tag
87 tag: "2.6.4"
88 # -- Image pull policy
89 pullPolicy: "IfNotPresent"
90 # -- Wait container configuration (init container that waits for model to be ready)
91 waitContainer:
92 # -- Command to execute
93 command: ["/bin/bash"]
94 # -- Arguments for the wait container
95 args:
96 - "-eucx"
97 - "while aws --endpoint-url $S3_ENDPOINT_URL s3 sync --dryrun s3://$S3_BUCKET_NAME/$S3_PATH /data | grep -q download; do sleep 10; done"
98 # -- Environment variables (optional, overrides S3 defaults entirely if specified)
99 # env:
100 # - name: HUGGING_FACE_HUB_TOKEN
101 # value: "your-token"
102 # - name: MODEL_ID
103 # value: "meta-llama/Llama-2-7b"
104 # -- Download job configuration (job that actually downloads the model)
105 downloadJob:
106 # -- Command to execute
107 command: ["/bin/bash"]
108 # -- Arguments for the download job
109 args:
110 - "-eucx"
111 - "aws --endpoint-url $S3_ENDPOINT_URL s3 sync s3://$S3_BUCKET_NAME/$S3_PATH /data"
112 # -- Environment variables (optional, overrides S3 defaults entirely if specified)
113 # env:
114 # - name: HUGGING_FACE_HUB_TOKEN
115 # value: "your-token"
116 # - name: MODEL_ID
117 # value: "meta-llama/Llama-2-7b"
118
119 # -- Custom init containers (appended after wait-download-model if modelDownload is enabled)
120 initContainers: []
121 # Example for llm-d sidecar:
122 # initContainers:
123 # - name: llm-d-routing-proxy
124 # image: ghcr.io/llm-d/llm-d-routing-sidecar:v0.2.0
125 # imagePullPolicy: IfNotPresent
126 # ports:
127 # - containerPort: 8080
128 # name: proxy
129 # securityContext:
130 # runAsUser: 1000
131
132 # -- Path of the model on the s3 which hosts model weights and config files
133 s3modelpath: "relative_s3_model_path/opt-125m"
134 # -- Storage size for the PVC
135 pvcStorage: "1Gi"
136 # -- Disable AWS EC2 metadata service
137 awsEc2MetadataDisabled: true
138
139 # -- Additional containers configuration
140 extraContainers: []
141
142 # -- Readiness probe configuration
143 readinessProbe:
144 # -- Number of seconds after the container has started before readiness probe is initiated
145 initialDelaySeconds: 5
146 # -- How often (in seconds) to perform the readiness probe
147 periodSeconds: 5
148 # -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not ready
149 failureThreshold: 3
150 # -- Configuration of the Kubelet http request on the server
151 httpGet:
152 # -- Path to access on the HTTP server
153 path: /health
154 # -- Name or number of the port to access on the container, on which the server is listening
155 port: 8000
156
157 # -- Liveness probe configuration
158 livenessProbe:
159 # -- Number of seconds after the container has started before liveness probe is initiated
160 initialDelaySeconds: 15
161 # -- Number of times after which if a probe fails in a row, Kubernetes considers that the overall check has failed: the container is not alive
162 failureThreshold: 3
163 # -- How often (in seconds) to perform the liveness probe
164 periodSeconds: 10
165 # -- Configuration of the Kubelet http request on the server
166 httpGet:
167 # -- Path to access on the HTTP server
168 path: /health
169 # -- Name or number of the port to access on the container, on which the server is listening
170 port: 8000
171
172 labels:
173 environment: "test"
174 release: "test"