main
py 369 lines 18 KB
Raw
1 #!/usr/bin/env python3
2 """Static LOCAL/TEST/AZURE configuration and Helm invariants.
3
4 This command renders manifests only. It never connects to a Kubernetes cluster
5 or an Azure subscription.
6 """
7
8 from __future__ import annotations
9
10 import re
11 import subprocess
12 import sys
13 from pathlib import Path
14 from typing import Any
15
16 import yaml
17
18
19 ROOT = Path(__file__).resolve().parents[1]
20 CHART = ROOT / "infrastructure" / "helm" / "ai-investment-platform"
21
22
23 def fail(message: str) -> None:
24 raise AssertionError(message)
25
26
27 def load_yaml(path: Path) -> dict[str, Any]:
28 return yaml.safe_load(path.read_text(encoding="utf-8")) or {}
29
30
31 def merge(left: dict[str, Any], right: dict[str, Any]) -> dict[str, Any]:
32 result = dict(left)
33 for key, value in right.items():
34 if isinstance(value, dict) and isinstance(result.get(key), dict):
35 result[key] = merge(result[key], value)
36 else:
37 result[key] = value
38 return result
39
40
41 def render(values_file: str, *extra: str) -> tuple[str, list[dict[str, Any]]]:
42 command = [
43 "helm", "template", "aip", str(CHART),
44 "--namespace", "ai-investment", "-f", str(CHART / values_file), *extra,
45 ]
46 completed = subprocess.run(command, cwd=ROOT, check=True, capture_output=True, text=True)
47 documents = [item for item in yaml.safe_load_all(completed.stdout) if isinstance(item, dict)]
48 return completed.stdout, documents
49
50
51 def render_must_fail(values_file: str, expected: str, *extra: str) -> None:
52 command = [
53 "helm", "template", "aip", str(CHART),
54 "--namespace", "ai-investment", "-f", str(CHART / values_file), *extra,
55 ]
56 completed = subprocess.run(command, cwd=ROOT, capture_output=True, text=True)
57 assert completed.returncode != 0
58 assert expected in completed.stderr
59
60
61 def resources(documents: list[dict[str, Any]], kind: str) -> list[dict[str, Any]]:
62 return [document for document in documents if document.get("kind") == kind]
63
64
65 def deployment(documents: list[dict[str, Any]], name: str) -> dict[str, Any]:
66 for item in resources(documents, "Deployment"):
67 if item.get("metadata", {}).get("name") == name:
68 return item
69 fail(f"Deployment {name!r} did not render")
70
71
72 def env_by_name(workload: dict[str, Any]) -> dict[str, dict[str, Any]]:
73 containers = workload["spec"]["template"]["spec"]["containers"]
74 return {entry["name"]: entry for entry in containers[0].get("env", [])}
75
76
77 def value(env: dict[str, dict[str, Any]], name: str) -> str:
78 if name not in env:
79 fail(f"Environment variable {name} did not render")
80 return str(env[name].get("value", ""))
81
82
83 def assert_profile_values() -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
84 base = load_yaml(CHART / "values.yaml")
85 local = merge(base, load_yaml(CHART / "values-dev.yaml"))
86 test = merge(base, load_yaml(CHART / "values-test.yaml"))
87 azure = merge(base, load_yaml(CHART / "values-azure.yaml"))
88 assert local["global"]["environment"] == "LOCAL"
89 assert local["global"]["springProfile"] == "LOCAL"
90 assert test["global"]["environment"] == "TEST"
91 assert test["global"]["springProfile"] == "TEST"
92 assert azure["global"]["environment"] == "AZURE"
93 assert azure["global"]["springProfile"] == "AZURE"
94 assert azure["devDependencies"]["enabled"] is False
95 return local, test, azure
96
97
98 def main() -> int:
99 local_values, _, azure_values = assert_profile_values()
100 local_text, local_docs = render("values-dev.yaml")
101 azure_text, azure_docs = render("values-azure.yaml")
102 _, kafka_docs = render(
103 "values-azure.yaml", "--set", "kafka.enabled=true",
104 "--set", "kafka.sslTruststoreSecretName=aip-kafka-ca",
105 )
106 _, yahoo_mcp_docs = render(
107 "values-azure.yaml",
108 "--set", "yahooFinanceMcp.enabled=true",
109 "--set", "mcpGateway.externalProvidersEnabled=true",
110 "--set", "mcpGateway.yahooFinance.enabled=true",
111 "--set", "research.mcpAcquisition.enabled=true",
112 )
113 render_must_fail(
114 "values-dev.yaml",
115 "MCP gateway is INTERNAL_ONLY",
116 "--set", "mcpGateway.ingress.enabled=true",
117 )
118 render_must_fail(
119 "values-dev.yaml",
120 "MCP gateway must use a ClusterIP Service",
121 "--set", "mcpGateway.service.type=LoadBalancer",
122 )
123 render_must_fail(
124 "values-dev.yaml",
125 "First-party Yahoo Finance MCP is INTERNAL_ONLY",
126 "--set", "yahooFinanceMcp.ingress.enabled=true",
127 )
128 render_must_fail(
129 "values-dev.yaml",
130 "First-party Yahoo Finance MCP must use a ClusterIP Service",
131 "--set", "yahooFinanceMcp.service.type=LoadBalancer",
132 )
133 render_must_fail(
134 "values-azure.yaml",
135 "Yahoo Finance MCP requires the external provider gateway",
136 "--set", "mcpGateway.yahooFinance.enabled=true",
137 )
138
139 assert "SecretProviderClass" not in {item.get("kind") for item in local_docs}
140 assert "azure.workload.identity/use" not in local_text
141 assert "AZURE_CLIENT_SECRET" not in azure_text
142 assert "clientSecret" not in azure_text
143
144 services = resources(azure_docs, "Service")
145 assert services and all(item.get("spec", {}).get("type", "ClusterIP") == "ClusterIP" for item in services)
146 local_mcp = deployment(local_docs, "mcp-gateway")
147 azure_mcp = deployment(azure_docs, "mcp-gateway")
148 local_mcp_services = [
149 item for item in resources(local_docs, "Service")
150 if item.get("metadata", {}).get("name") == "mcp-gateway"
151 ]
152 assert len(local_mcp_services) == 1 and local_mcp_services[0]["spec"]["type"] == "ClusterIP"
153 mcp_services = [item for item in services if item.get("metadata", {}).get("name") == "mcp-gateway"]
154 assert len(mcp_services) == 1 and mcp_services[0]["spec"]["type"] == "ClusterIP"
155 ingress_names = {item["metadata"]["name"] for item in resources(azure_docs, "Ingress")}
156 assert ingress_names == {"api-gateway"}
157 assert "mcp-gateway" not in ingress_names
158
159 local_mcp_env = env_by_name(local_mcp)
160 azure_mcp_env = env_by_name(azure_mcp)
161 assert value(local_mcp_env, "AIP_MCP_AUTHENTICATION_TYPE") == "LOCAL_SERVICE"
162 assert value(local_mcp_env, "AIP_MCP_EXTERNAL_PROVIDERS_ENABLED") == "true"
163 assert value(local_mcp_env, "AIP_MCP_YAHOO_ENABLED") == "true"
164 assert "get_quote" in value(local_mcp_env, "AIP_MCP_YAHOO_CAPABILITIES_JSON")
165 assert value(local_mcp_env, "AIP_MCP_YAHOO_ENDPOINT") == "http://yahoo-finance-mcp/mcp"
166 assert value(azure_mcp_env, "AIP_MCP_AUTHENTICATION_TYPE") == "WORKLOAD_IDENTITY"
167 assert value(azure_mcp_env, "AIP_MCP_EXTERNAL_PROVIDERS_ENABLED") == "false"
168 assert value(azure_mcp_env, "AIP_MCP_YAHOO_ENABLED") == "false"
169 assert "get_quote" in value(azure_mcp_env, "AIP_MCP_YAHOO_CAPABILITIES_JSON")
170 assert "AIP_MCP_YAHOO_AUTH_TOKEN" not in azure_mcp_env
171 yahoo_gateway_env = env_by_name(deployment(yahoo_mcp_docs, "mcp-gateway"))
172 assert value(yahoo_gateway_env, "AIP_MCP_EXTERNAL_PROVIDERS_ENABLED") == "true"
173 assert value(yahoo_gateway_env, "AIP_MCP_YAHOO_ENABLED") == "true"
174 assert "get_quote" in value(yahoo_gateway_env, "AIP_MCP_YAHOO_CAPABILITIES_JSON")
175 assert value(azure_mcp_env, "AIP_MCP_RESEARCH_BASE_URL") == "http://research-engine"
176 assert value(azure_mcp_env, "AIP_FEATURE_MCP_ENABLED") == "true"
177 assert value(azure_mcp_env, "OTEL_SDK_DISABLED") == "false"
178 assert value(azure_mcp_env, "OTEL_EXPORTER_OTLP_ENDPOINT").startswith("https://")
179 assert azure_mcp["spec"]["template"]["metadata"]["labels"]["azure.workload.identity/use"] == "true"
180 mcp_volumes = azure_mcp["spec"]["template"]["spec"].get("volumes", [])
181 assert any(volume.get("name") == "key-vault-secrets" for volume in mcp_volumes)
182 mcp_mounts = azure_mcp["spec"]["template"]["spec"]["containers"][0].get("volumeMounts", [])
183 assert any(mount.get("name") == "key-vault-secrets" and mount.get("readOnly") for mount in mcp_mounts)
184 local_research_env = env_by_name(deployment(local_docs, "research-engine"))
185 azure_research_env = env_by_name(deployment(azure_docs, "research-engine"))
186 assert value(local_research_env, "AIP_RESEARCH_MCP_FIRST_ENABLED") == "true"
187 assert value(azure_research_env, "AIP_RESEARCH_MCP_FIRST_ENABLED") == "false"
188 assert value(azure_research_env, "AIP_RESEARCH_MCP_GATEWAY_BASE_URL") == "http://mcp-gateway"
189
190 local_yahoo = deployment(local_docs, "yahoo-finance-mcp")
191 azure_yahoo = deployment(yahoo_mcp_docs, "yahoo-finance-mcp")
192 local_yahoo_services = [
193 item for item in resources(local_docs, "Service")
194 if item.get("metadata", {}).get("name") == "yahoo-finance-mcp"
195 ]
196 azure_yahoo_services = [
197 item for item in resources(yahoo_mcp_docs, "Service")
198 if item.get("metadata", {}).get("name") == "yahoo-finance-mcp"
199 ]
200 assert len(local_yahoo_services) == 1 and local_yahoo_services[0]["spec"]["type"] == "ClusterIP"
201 assert len(azure_yahoo_services) == 1 and azure_yahoo_services[0]["spec"]["type"] == "ClusterIP"
202 assert "yahoo-finance-mcp" not in ingress_names
203 yahoo_env = env_by_name(azure_yahoo)
204 assert value(yahoo_env, "AIP_YAHOO_MCP_TRANSPORT") == "streamable-http"
205 assert value(yahoo_env, "OTEL_SDK_DISABLED") == "false"
206 assert not any(
207 marker in variable_name
208 for variable_name in yahoo_env
209 for marker in ("TOKEN", "SECRET", "PASSWORD", "API_KEY", "CLIENT_KEY")
210 )
211 yahoo_container = azure_yahoo["spec"]["template"]["spec"]["containers"][0]
212 assert yahoo_container["securityContext"]["allowPrivilegeEscalation"] is False
213 assert yahoo_container["securityContext"]["readOnlyRootFilesystem"] is True
214 assert yahoo_container["securityContext"]["capabilities"]["drop"] == ["ALL"]
215 assert azure_yahoo["spec"]["template"]["spec"]["securityContext"]["runAsNonRoot"] is True
216 yahoo_network_policies = {
217 item["metadata"]["name"] for item in resources(yahoo_mcp_docs, "NetworkPolicy")
218 }
219 assert "yahoo-finance-mcp-internal-only" in yahoo_network_policies
220 yahoo_hpas = {item["metadata"]["name"] for item in resources(yahoo_mcp_docs, "HorizontalPodAutoscaler")}
221 yahoo_pdbs = {item["metadata"]["name"] for item in resources(yahoo_mcp_docs, "PodDisruptionBudget")}
222 assert "yahoo-finance-mcp" in yahoo_hpas and "yahoo-finance-mcp" in yahoo_pdbs
223
224 allowed_origins = azure_values["apiGateway"]["cors"]["allowedOrigins"]
225 assert allowed_origins and "*" not in allowed_origins
226 assert all("localhost" not in origin and "127.0.0.1" not in origin for origin in allowed_origins)
227
228 auth_env = env_by_name(deployment(azure_docs, "auth-service"))
229 assert "sslmode=verify-full" in value(auth_env, "SPRING_DATASOURCE_URL")
230 assert value(auth_env, "DB_POOL_MAXIMUM_SIZE") == "6"
231 portfolio_env = env_by_name(deployment(azure_docs, "portfolio-service"))
232 assert value(portfolio_env, "SPRING_DATA_REDIS_SSL_ENABLED") == "true"
233 kafka_env = env_by_name(deployment(kafka_docs, "api-gateway"))
234 assert value(kafka_env, "AIP_KAFKA_SECURITY_PROTOCOL") == "SASL_SSL"
235 assert value(kafka_env, "AIP_KAFKA_SASL_MECHANISM") == "PLAIN"
236 assert value(kafka_env, "AIP_KAFKA_CONSUMER_GROUP_PREFIX") == "REPLACE_WITH_ENVIRONMENT_CONSUMER_GROUP_PREFIX"
237 assert value(kafka_env, "AIP_KAFKA_CLIENT_DNS_LOOKUP") == "use_all_dns_ips"
238 assert value(kafka_env, "AIP_KAFKA_REQUEST_TIMEOUT_MS") == "30000"
239 assert value(kafka_env, "AIP_KAFKA_DELIVERY_TIMEOUT_MS") == "120000"
240 assert value(kafka_env, "AIP_KAFKA_RETRIES") == "5"
241 assert value(kafka_env, "AIP_KAFKA_SSL_TRUSTSTORE_LOCATION") == "/etc/ai-investment/kafka/KAFKA_TRUSTSTORE"
242 kafka_gateway = deployment(kafka_docs, "api-gateway")
243 kafka_volumes = kafka_gateway["spec"]["template"]["spec"].get("volumes", [])
244 assert any(volume.get("secret", {}).get("secretName") == "aip-kafka-ca" for volume in kafka_volumes)
245
246 local_gateway_env = env_by_name(deployment(local_docs, "api-gateway"))
247 assert value(local_gateway_env, "OTEL_SDK_DISABLED") == "true"
248 azure_gateway_env = env_by_name(deployment(azure_docs, "api-gateway"))
249 assert value(azure_gateway_env, "GATEWAY_HTTP_CONNECT_TIMEOUT") == "3s"
250 assert value(azure_gateway_env, "GATEWAY_HTTP_READ_TIMEOUT") == "30s"
251 assert value(azure_gateway_env, "OTEL_SDK_DISABLED") == "false"
252 assert value(azure_gateway_env, "OTEL_EXPORTER_OTLP_ENDPOINT").startswith("https://")
253
254 azure_images = [
255 container["image"]
256 for item in resources(azure_docs, "Deployment")
257 for container in item["spec"]["template"]["spec"]["containers"]
258 ]
259 local_images = [
260 container["image"]
261 for item in resources(local_docs, "Deployment")
262 for container in item["spec"]["template"]["spec"]["containers"]
263 ]
264 assert azure_images and all(image.startswith("REPLACE_WITH_ACR_NAME.azurecr.io/") for image in azure_images)
265 assert all(image.endswith(":REPLACE_WITH_GIT_SHA") for image in azure_images)
266 application_local_images = [image for image in local_images if image.startswith("localhost:5001/")]
267 assert len(application_local_images) >= 15
268 assert azure_values["devDependencies"]["postgres"]["persistence"]["storageClass"] == ""
269 assert "local-path" not in azure_text
270
271 app_deployments = resources(azure_docs, "Deployment")
272 assert len(app_deployments) >= 15
273 for item in app_deployments:
274 name = item["metadata"]["name"]
275 pod_spec = item["spec"]["template"]["spec"]
276 assert pod_spec.get("automountServiceAccountToken") is False, name
277 assert pod_spec.get("securityContext", {}).get("runAsNonRoot") is True, name
278 container = pod_spec["containers"][0]
279 security = container.get("securityContext", {})
280 assert security.get("allowPrivilegeEscalation") is False, name
281 assert security.get("capabilities", {}).get("drop") == ["ALL"], name
282 assert container.get("startupProbe"), name
283 assert container.get("readinessProbe"), name
284 assert container.get("livenessProbe"), name
285 assert container.get("resources", {}).get("requests"), name
286 assert container.get("resources", {}).get("limits"), name
287
288 hpas = {item["metadata"]["name"] for item in resources(azure_docs, "HorizontalPodAutoscaler")}
289 pdbs = {item["metadata"]["name"] for item in resources(azure_docs, "PodDisruptionBudget")}
290 assert {"frontend", "api-gateway", "auth-service", "mcp-gateway"}.issubset(hpas)
291 assert {"frontend", "api-gateway", "auth-service", "mcp-gateway"}.issubset(pdbs)
292 assert resources(azure_docs, "NetworkPolicy")
293 network_policy_names = {
294 item["metadata"]["name"] for item in resources(azure_docs, "NetworkPolicy")
295 }
296 assert "mcp-gateway-internal-only" in network_policy_names
297
298 for item in resources(azure_docs, "ConfigMap"):
299 serialized = yaml.safe_dump(item).lower()
300 assert not any(word in serialized for word in ("password", "client_secret", "api_key", "bearer "))
301
302 sensitive_names = {
303 "DB_PASSWORD", "SPRING_DATA_REDIS_PASSWORD", "AIP_KAFKA_PASSWORD",
304 "AUTH_JWT_SECRET", "SMTP_USERNAME", "SMTP_PASSWORD", "AIP_EODHD_API_KEY",
305 "AIP_INTERNAL_TOKEN", "ICICI_DIRECT_APP_KEY", "ICICI_DIRECT_SECRET_KEY",
306 }
307 for item in app_deployments:
308 for entry in env_by_name(item).values():
309 if entry["name"] in sensitive_names:
310 assert "valueFrom" in entry and "value" not in entry, (item["metadata"]["name"], entry["name"])
311
312 public_env_text = "\n".join(
313 path.read_text(encoding="utf-8", errors="ignore")
314 for path in (ROOT / "frontend").rglob("*")
315 if path.is_file() and "node_modules" not in path.parts and ".next" not in path.parts
316 and path.stat().st_size < 1_000_000
317 )
318 forbidden_public = ("PASSWORD", "SECRET", "PRIVATE_KEY", "DATABASE", "SMTP", "BROKER_TOKEN", "LLM_KEY")
319 assert not any(f"NEXT_PUBLIC_{name}" in public_env_text for name in forbidden_public)
320
321 dockerfiles = [
322 path for path in ROOT.rglob("Dockerfile")
323 if "node_modules" not in path.parts and ".next" not in path.parts
324 ]
325 assert len(dockerfiles) == 17
326 assert ROOT / "ai" / "yahoo-finance-mcp" / "Dockerfile" in dockerfiles
327 for dockerfile in dockerfiles:
328 dockerfile_text = dockerfile.read_text(encoding="utf-8")
329 assert re.search(r"^USER [0-9]+(?::[0-9]+)?$", dockerfile_text, re.MULTILINE), dockerfile
330
331 azure_research = deployment(azure_docs, "research-engine")
332 research_env = env_by_name(azure_research)
333 assert value(research_env, "AIP_RESEARCH_DISTRIBUTED_LOCK_BACKEND") == "process"
334 assert azure_research["spec"]["replicas"] == 1
335 assert value(research_env, "AIP_YAHOO_SEARCH_URL").startswith("https://")
336 assert value(research_env, "AIP_NSE_ANNOUNCEMENTS_URL").startswith("https://")
337 assert value(azure_gateway_env, "AIP_FEATURE_SCHEDULED_JOBS_ENABLED") == "false"
338 assert azure_values["auth"]["email"]["publicUrl"].startswith("https://")
339 assert "callbackUrl" in azure_values["ibkr"]
340
341 service_account = resources(azure_docs, "ServiceAccount")[0]
342 annotations = service_account["metadata"].get("annotations", {})
343 assert annotations.get("azure.workload.identity/client-id")
344 spcs = resources(azure_docs, "SecretProviderClass")
345 assert len(spcs) == 1
346 spc_text = yaml.safe_dump(spcs[0]).lower()
347 assert "clientsecret" not in spc_text and "password:" not in spc_text
348
349 mcp_server_source = (ROOT / "ai" / "mcp-gateway" / "app" / "server.py").read_text(encoding="utf-8")
350 assert '"externalProviderDependency": False' in mcp_server_source
351 assert "place_order" not in mcp_server_source
352 platform_script = (ROOT / "platform.ps1").read_text(encoding="utf-8")
353 assert '"mcp-gateway"' in platform_script
354 assert 'Name = "yahoo-finance-mcp"' in platform_script
355
356 print(f"azure_readiness_static_assertions=54")
357 print(f"local_manifest_documents={len(local_docs)}")
358 print(f"azure_manifest_documents={len(azure_docs)}")
359 print(f"azure_application_deployments={len(app_deployments)}")
360 print("azure_readiness_validation=PASS")
361 return 0
362
363
364 if __name__ == "__main__":
365 try:
366 raise SystemExit(main())
367 except (AssertionError, subprocess.CalledProcessError) as error:
368 print(f"azure_readiness_validation=FAIL: {error}", file=sys.stderr)
369 raise SystemExit(1)