From 26fdf6ab2f63b7ee1281142d4dd3caa6740a8e28 Mon Sep 17 00:00:00 2001 From: Jeff McCollum <16550786+jeffmccollum@users.noreply.github.com> Date: Tue, 8 Sep 2026 10:45:55 -0500 Subject: [PATCH 1/6] update example to use role label instead of name --- braintrust/README.md | 17 +++++++++++++++-- .../examples/google-standard/values.yaml | 18 +++++++++++++----- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/braintrust/README.md b/braintrust/README.md index c22dacb..e4ae409 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -152,6 +152,8 @@ brainstore: For Standard mode clusters, create node pools with local SSDs, then deploy: +Use the stable `braintrust/node-pool` role label in each node selector. Terraform node pool replacements can use generated GKE pool names. + **Configure the Helm chart:** ```yaml cloud: "google" @@ -159,10 +161,18 @@ For Standard mode clusters, create node pools with local SSDs, then deploy: google: mode: "standard" + api: + nodeSelector: + braintrust/node-pool: "services" + + aiGateway: + nodeSelector: + braintrust/node-pool: "services" + brainstore: reader: nodeSelector: - cloud.google.com/gke-nodepool: "brainstore" # Target your node pool + braintrust/node-pool: "brainstore" resources: requests: cpu: "44" @@ -179,9 +189,12 @@ For Standard mode clusters, create node pools with local SSDs, then deploy: - brainstore-reader - brainstore-writer topologyKey: kubernetes.io/hostname + fastreader: + nodeSelector: + braintrust/node-pool: "brainstore" writer: nodeSelector: - cloud.google.com/gke-nodepool: "brainstore" + braintrust/node-pool: "brainstore" resources: requests: cpu: "44" diff --git a/braintrust/examples/google-standard/values.yaml b/braintrust/examples/google-standard/values.yaml index 887bdf9..a75cc55 100644 --- a/braintrust/examples/google-standard/values.yaml +++ b/braintrust/examples/google-standard/values.yaml @@ -43,7 +43,7 @@ api: # Native GCS authentication via Workload Identity. Set to true to use GCS natively instead of S3-compatible access. enableGcsAuth: false nodeSelector: - cloud.google.com/gke-nodepool: "api" + braintrust/node-pool: "services" resources: requests: cpu: "4" @@ -56,6 +56,10 @@ api: - name: AWS_REGION value: "us-central1" +aiGateway: + nodeSelector: + braintrust/node-pool: "services" + # Brainstore configuration (split into reader and writer) brainstore: serviceAccount: @@ -78,7 +82,7 @@ brainstore: port: 4000 portName: http nodeSelector: - cloud.google.com/gke-nodepool: "brainstore" # Target your node pool + braintrust/node-pool: "brainstore" resources: requests: cpu: "16" @@ -104,7 +108,12 @@ brainstore: volume: size: "200Gi" extraEnvVars: - + + # Brainstore Fast Reader configuration + fastreader: + nodeSelector: + braintrust/node-pool: "brainstore" + # Brainstore Writer configuration writer: name: "brainstore-writer" @@ -115,7 +124,7 @@ brainstore: port: 4000 portName: http nodeSelector: - cloud.google.com/gke-nodepool: "brainstore" + braintrust/node-pool: "brainstore" resources: requests: cpu: "32" @@ -141,4 +150,3 @@ brainstore: volume: size: "200Gi" extraEnvVars: - \ No newline at end of file From 8d2bc5e29806d98f0a1770dd6a14a9cfd19c2440 Mon Sep 17 00:00:00 2001 From: Jeff McCollum <16550786+jeffmccollum@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:05:59 -0500 Subject: [PATCH 2/6] add pdq --- AGENTS.md | 33 +++++ README.md | 10 ++ braintrust/README.md | 76 ++++++++-- .../examples/google-standard/values.yaml | 25 +++- braintrust/templates/api-pdb.yaml | 4 + braintrust/templates/brainstore-pdb.yaml | 25 ++++ braintrust/tests/brainstore-pdb_test.yaml | 138 ++++++++++++++++++ braintrust/values.yaml | 10 ++ 8 files changed, 306 insertions(+), 15 deletions(-) create mode 100644 braintrust/templates/brainstore-pdb.yaml create mode 100644 braintrust/tests/brainstore-pdb_test.yaml diff --git a/AGENTS.md b/AGENTS.md index 62689d8..9620779 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,6 +76,18 @@ Always use the namespace helper: namespace: {{ include "braintrust.namespace" . }} ``` +## GKE mode recommendation + +GKE Autopilot is the preferred solution for new Braintrust deployments. +Standard is supported when customer requirements prevent Autopilot use. +The separate Standard example does not change this preference. +Existing deployments must keep their initial cluster mode. +A mode change replaces the cluster, causes downtime, and requires Helm release redeployment. + +Recommend Autopilot first for new deployments. +Present Standard as a supported alternative for customer constraints. +Do not imply that existing Standard customers must switch modes. + ## Critical Safety Constraints These constraints exist because of real incidents and confirmed engineering guidance. Do not "simplify" or "clean up" code that implements them. @@ -100,6 +112,27 @@ The four brainstore configmaps (`brainstore-reader-configmap.yaml`, `brainstore- The optional Automation Writer pool (`brainstore.automationwriter`, default `replicas: 0`) isolates the automations writer loop via `BRAINSTORE_WRITER_LOOP_CONFIG`. That variable exists starting in brainstore image `v2.16.0`. Enabling the pool (`replicas > 0`) fails the render when `brainstore.image.tag` is a release tag (`vX.Y.Z`) older than `v2.16.0`. Commit SHAs and other custom tags must keep rendering, so `braintrust.automationWriter.validate` only calls `semverCompare` after the tag matches the release-tag regex (`semverCompare` errors on a SHA). Do not bump the chart-wide default image for this. `brainstore-automationwriter-configmap.yaml` sets `include:automations`, and `brainstore-writer-configmap.yaml` sets `exclude:automations` only when `brainstore.automationwriter.replicas > 0`. A null or missing replica count is 0: an empty `replicas` field would make Kubernetes run 1 pod while the writer still handled automations. A missing `brainstore.automationwriter` map — what `helm upgrade --reuse-values` retains from a chart that predates the pool — is disabled: the writer does not exclude automations, and the automation writer manifests are omitted. A partial map, such as `--set brainstore.automationwriter.replicas=1` on those retained values, is merged with the template defaults in `braintrust.automationWriter.defaults` before any field is read. On Azure with the Container Storage driver, `brainstore.automationwriter.volume.size` is required only when `replicas > 0`; a zero-replica pool renders with `emptyDir` so an upgrade does not demand disk for a pool that is off. These two must stay in sync: the automation writer runs automations and the regular writer must exclude them whenever the pool is enabled. An enabled pool has an internal Service and publishes `BRAINSTORE_AUTOMATION_WRITER_URL` to all API pools whenever replicas are positive. A compatible API release decides which queries use the endpoint. The API startup gate and its RBAC deployment names cover readers, writers, and fast readers only. Resolve names and Service fields through `braintrust.automationWriter.config` so partial reused values keep working. Use `brainstore.automationwriter.service.port` wherever an automation writer port is required. Coordinate API routing changes before scaling the pool to zero; old API processes retain their URL until restarted. +### Disruption budgets and GKE Standard + +Brainstore readers, fast readers, and writers each expose an optional `podDisruptionBudget`. +Budgets default to disabled on every cloud. An enabled Brainstore budget defaults to `maxUnavailable: 1`. +The API retains `minAvailable` unless an explicit `maxUnavailable` takes precedence. +The GKE Standard example enables separate budgets for the API and all Brainstore roles. +A single writer can stop briefly during eviction. With multiple writers, the budget permits one unavailable replica. +Each role has an independent budget, so different roles can lose a replica simultaneously. +Readiness probes determine healthy replicas. Deployment rollout settings and `minReadySeconds` do not control node eviction. +Stable `braintrust/node-pool` labels connect Helm selectors to Terraform pools. +The Terraform pool map key sets this label. Automatic replacement preserves the label. +GKE deletion protection must exist on the source pool before hardware replacement. Its PDB protection expires after one hour. + +- Keep PDB templates cloud-independent and disabled by default. +- Enable the budgets in the GKE Standard example. +- Preserve the single-writer interruption exception. +- Use stable workload labels instead of generated GKE pool names in selectors. +- Document the Helm deployment and source pool protection steps before hardware replacement. +- Do not describe PDBs as an unconditional zero-downtime guarantee. +- Test role isolation, API compatibility, and cross-cloud behavior after PDB changes. + ### Version Numbers Chart version numbers are semantically meaningful for the upgrade path: diff --git a/README.md b/README.md index 86d9ee6..68009db 100644 --- a/README.md +++ b/README.md @@ -24,3 +24,13 @@ helm upgrade --install \ Before installing the Braintrust Helm chart, ensure you have run the appropriate braintrust terraform module [Google](https://github.com/braintrustdata/terraform-google-braintrust-data-plane) or [Azure](https://github.com/braintrustdata/terraform-azure-braintrust-data-plane) to deploy the base infrastructure. See the [Braintrust Helm Chart](./braintrust/README.md) for more details. + +## GKE deployment modes + +GKE Autopilot is the preferred solution for new Braintrust deployments. GKE Standard is supported when customer requirements prevent Autopilot use. + +## GKE Standard node pool changes + +The [Standard example](braintrust/examples/google-standard/values.yaml) uses stable workload selectors and enables optional API and Brainstore disruption budgets. +The chart defaults remain unchanged for other deployments. +The [disruption budget guidance](braintrust/README.md#optional-disruption-budgets) explains the single-writer exception and the preparation steps for existing GKE pools. diff --git a/braintrust/README.md b/braintrust/README.md index e4ae409..61378b6 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -99,6 +99,8 @@ Kubernetes limits each Secret to 1 MiB. A normal combined CA bundle is typically ## GKE with Local SSDs +GKE Autopilot is the preferred solution for new Braintrust deployments. GKE Standard is supported when customer requirements prevent Autopilot use. + Braintrust requires local SSDs for maximum disk performance. Configuration varies depending on whether you're using GKE Autopilot or Standard mode. ### GKE Autopilot @@ -150,9 +152,13 @@ brainstore: ### GKE Standard Mode -For Standard mode clusters, create node pools with local SSDs, then deploy: +The Terraform module creates separate services and Brainstore pools. Brainstore uses a bundled Local SSD machine type without a disk count input. + +The stable `braintrust/node-pool` label connects Helm selectors to replacement pools with generated GKE names. +The Terraform map key sets the label. The `services` and `brainstore` keys match the selectors below. -Use the stable `braintrust/node-pool` role label in each node selector. Terraform node pool replacements can use generated GKE pool names. +The example enables PDBs for node eviction. Existing pools need deletion protection before a hardware replacement. +See [Optional disruption budgets](#optional-disruption-budgets) for the preparation sequence and limits. **Configure the Helm chart:** ```yaml @@ -162,6 +168,9 @@ Use the stable `braintrust/node-pool` role label in each node selector. Terrafor mode: "standard" api: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 nodeSelector: braintrust/node-pool: "services" @@ -171,6 +180,9 @@ Use the stable `braintrust/node-pool` role label in each node selector. Terrafor brainstore: reader: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 nodeSelector: braintrust/node-pool: "brainstore" resources: @@ -190,9 +202,15 @@ Use the stable `braintrust/node-pool` role label in each node selector. Terrafor - brainstore-writer topologyKey: kubernetes.io/hostname fastreader: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 nodeSelector: braintrust/node-pool: "brainstore" writer: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 nodeSelector: braintrust/node-pool: "brainstore" resources: @@ -213,10 +231,13 @@ Use the stable `braintrust/node-pool` role label in each node selector. Terrafor topologyKey: kubernetes.io/hostname ``` -**What happens:** -- Pods are scheduled on your pre-configured node pools -- Local SSDs are automatically available via emptyDir volumes -- Pod anti-affinity ensures readers and writers don't share nodes (each pod gets dedicated node access) +The configuration has these effects: + +- Pods use the stable labels to select eligible nodes. +- Brainstore uses Local SSD storage through `emptyDir` volumes. +- Anti-affinity separates readers and writers across nodes. It does not exclude other workloads from those nodes. +- Separate PDBs limit voluntary evictions for each role. +- A single writer can stop briefly during a drain. ## AWS EKS Local Storage @@ -376,10 +397,45 @@ creating replacements. This causes a complete role outage and, with the default single writer, pauses background processing until the replacement becomes Ready. -These settings pace Deployment-managed rollouts only. The chart does not -currently create PodDisruptionBudgets for Brainstore, so these controls do not -limit voluntary disruptions such as node drains or protect against involuntary -pod or node failures. +These settings pace Deployment rollouts only. Optional PodDisruptionBudgets protect voluntary evictions such as node drains. +Neither mechanism protects against node failure. + +## Optional disruption budgets + +Brainstore readers, fast readers, and writers each support an optional PDB. +PDBs default to disabled on every cloud. Each enabled role defaults to `maxUnavailable: 1`. +The budget permits a single writer to stop briefly. Replicated roles retain all but one healthy replica during permitted evictions. + +```yaml +brainstore: + reader: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 + fastreader: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 + writer: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 +api: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 +``` + +The API retains its existing `minAvailable` behavior unless `maxUnavailable` is set. An explicit maximum takes precedence over the inherited minimum. +Each role has an independent budget. Evictions can proceed simultaneously across roles. +Readiness probes determine healthy replicas for PDBs. Deployment `minReadySeconds` does not delay PDB eviction permission. +The GKE Standard example enables these budgets. Existing EKS and AKS defaults remain unchanged. +GKE requires deletion protection on the source pool before a replacement, and its PDB protection expires after one hour. + +1. Deploy the Helm budgets before a node pool replacement. +2. Enable GKE deletion protection on existing pools in a separate Terraform apply before hardware changes. +3. Verify sufficient replicas and replacement capacity. + ## Brainstore Automation Writer pool diff --git a/braintrust/examples/google-standard/values.yaml b/braintrust/examples/google-standard/values.yaml index a75cc55..eb4e002 100644 --- a/braintrust/examples/google-standard/values.yaml +++ b/braintrust/examples/google-standard/values.yaml @@ -1,10 +1,12 @@ +# GKE Autopilot is preferred for new deployments. +# Use this supported Standard example when customer requirements prevent Autopilot use. # Sample values for GKE Standard deployment # -# GKE Standard requires manual node pool configuration: -# - Create a dedicated node pool with local NVMe SSDs for Brainstore workloads -# - Recommended machine types: c4-standard-32 or higher with local SSDs -# - Configure local SSDs: Use 4x375GB local SSDs (1500GB total) or more -# - Total local SSD capacity should exceed the volume.size configured below +# Terraform creates the services and brainstore node pools. +# Brainstore requires a bundled Local SSD machine type, such as c4a-standard-48-lssd. +# GKE selects the fixed SSD count from the machine type. +# Enable PDB protection on existing pools before a hardware replacement. +# Deploy these PDBs before that replacement. # Global configs global: @@ -28,6 +30,9 @@ api: annotations: service: networking.gke.io/load-balancer-type: "Internal" + podDisruptionBudget: + enabled: true + maxUnavailable: 1 replicas: 4 # Uncomment the following section to use a different image or tag from the version in the Helm release #image: @@ -74,6 +79,9 @@ brainstore: # Brainstore Reader configuration reader: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 name: "brainstore-reader" replicas: 2 service: @@ -111,11 +119,18 @@ brainstore: # Brainstore Fast Reader configuration fastreader: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 nodeSelector: braintrust/node-pool: "brainstore" # Brainstore Writer configuration + # One writer can stop briefly during a node drain. writer: + podDisruptionBudget: + enabled: true + maxUnavailable: 1 name: "brainstore-writer" replicas: 1 service: diff --git a/braintrust/templates/api-pdb.yaml b/braintrust/templates/api-pdb.yaml index 3fe35ff..769b991 100644 --- a/braintrust/templates/api-pdb.yaml +++ b/braintrust/templates/api-pdb.yaml @@ -18,7 +18,11 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: + {{- if hasKey $api.podDisruptionBudget "maxUnavailable" }} + maxUnavailable: {{ $api.podDisruptionBudget.maxUnavailable | toYaml | trim }} + {{- else }} minAvailable: {{ $api.podDisruptionBudget.minAvailable }} + {{- end }} selector: matchLabels: app: {{ $api.name }} diff --git a/braintrust/templates/brainstore-pdb.yaml b/braintrust/templates/brainstore-pdb.yaml new file mode 100644 index 0000000..1a70a8e --- /dev/null +++ b/braintrust/templates/brainstore-pdb.yaml @@ -0,0 +1,25 @@ +{{- $root := . -}} +{{- $rendered := 0 -}} +{{- range $role := list "reader" "fastreader" "writer" -}} +{{- $config := index $root.Values.brainstore $role -}} +{{- if $config.podDisruptionBudget.enabled }} +{{- if gt $rendered 0 }} +--- +{{- end }} +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: {{ $config.name }} + namespace: {{ include "braintrust.namespace" $root }} + {{- with (merge (deepCopy $config.labels) $root.Values.global.labels) }} + labels: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + maxUnavailable: {{ $config.podDisruptionBudget.maxUnavailable | toYaml | trim }} + selector: + matchLabels: + app: {{ $config.name }} +{{- $rendered = add1 $rendered -}} +{{- end }} +{{- end }} diff --git a/braintrust/tests/brainstore-pdb_test.yaml b/braintrust/tests/brainstore-pdb_test.yaml new file mode 100644 index 0000000..ab30029 --- /dev/null +++ b/braintrust/tests/brainstore-pdb_test.yaml @@ -0,0 +1,138 @@ +suite: Brainstore disruption budgets +templates: + - brainstore-pdb.yaml + - api-pdb.yaml +tests: + - it: leaves all PDBs absent by default + template: brainstore-pdb.yaml + asserts: + - hasDocuments: + count: 0 + - it: enables only the selected role and preserves a zero budget + template: brainstore-pdb.yaml + release: + namespace: custom-namespace + set: + brainstore.writer.name: custom-writer + brainstore.writer.podDisruptionBudget.enabled: true + brainstore.writer.podDisruptionBudget.maxUnavailable: 0 + global.namespace: custom-namespace + asserts: + - hasDocuments: + count: 1 + - equal: + path: metadata.namespace + value: custom-namespace + - equal: + path: spec.selector.matchLabels.app + value: custom-writer + - equal: + path: spec.maxUnavailable + value: 0 + - notExists: + path: spec.minAvailable + - it: keeps labels isolated between roles + template: brainstore-pdb.yaml + set: + global.labels.shared: global + brainstore.reader.labels.readerOnly: reader + brainstore.writer.labels.writerOnly: writer + brainstore.reader.podDisruptionBudget.enabled: true + brainstore.writer.podDisruptionBudget.enabled: true + asserts: + - hasDocuments: + count: 2 + - equal: + path: metadata.labels.readerOnly + value: reader + documentIndex: 0 + - notExists: + path: metadata.labels.writerOnly + documentIndex: 0 + - equal: + path: metadata.labels.writerOnly + value: writer + documentIndex: 1 + - notExists: + path: metadata.labels.readerOnly + documentIndex: 1 + - equal: + path: metadata.labels.shared + value: global + documentIndex: 1 + - it: enables all roles in the GKE Standard example + template: brainstore-pdb.yaml + values: + - ../examples/google-standard/values.yaml + asserts: + - hasDocuments: + count: 3 + - equal: + path: spec.maxUnavailable + value: 1 + documentIndex: 0 + - equal: + path: spec.maxUnavailable + value: 1 + documentIndex: 1 + - equal: + path: spec.maxUnavailable + value: 1 + documentIndex: 2 + - it: uses the API maximum budget in the GKE Standard example + template: api-pdb.yaml + values: + - ../examples/google-standard/values.yaml + asserts: + - equal: + path: spec.maxUnavailable + value: 1 + - notExists: + path: spec.minAvailable + - it: preserves the existing API minimum budget + template: api-pdb.yaml + set: + api.podDisruptionBudget.enabled: true + api.podDisruptionBudget.minAvailable: 2 + asserts: + - equal: + path: spec.minAvailable + value: 2 + - notExists: + path: spec.maxUnavailable + - it: supports optional budgets on aws + template: brainstore-pdb.yaml + set: + cloud: aws + brainstore.reader.podDisruptionBudget.enabled: true + brainstore.reader.podDisruptionBudget.maxUnavailable: "25%" + asserts: + - hasDocuments: + count: 1 + - equal: + path: spec.maxUnavailable + value: "25%" + - it: supports optional budgets on azure + template: brainstore-pdb.yaml + set: + cloud: azure + brainstore.reader.podDisruptionBudget.enabled: true + brainstore.reader.podDisruptionBudget.maxUnavailable: "25%" + asserts: + - hasDocuments: + count: 1 + - equal: + path: spec.maxUnavailable + value: "25%" + - it: supports optional budgets on google + template: brainstore-pdb.yaml + set: + cloud: google + brainstore.reader.podDisruptionBudget.enabled: true + brainstore.reader.podDisruptionBudget.maxUnavailable: "25%" + asserts: + - hasDocuments: + count: 1 + - equal: + path: spec.maxUnavailable + value: "25%" diff --git a/braintrust/values.yaml b/braintrust/values.yaml index 7f19f44..d3b85fa 100644 --- a/braintrust/values.yaml +++ b/braintrust/values.yaml @@ -234,6 +234,7 @@ api: maxUnavailable: 0 podDisruptionBudget: enabled: false + # An explicit maxUnavailable overrides this minimum. minAvailable: 1 topologySpread: enabled: false @@ -424,6 +425,9 @@ brainstore: deployment: {} service: {} pod: {} + podDisruptionBudget: + enabled: false + maxUnavailable: 1 replicas: 2 # Configure rollout pacing for this Brainstore role. Defaults preserve the # rollout behavior from earlier chart versions. @@ -484,6 +488,9 @@ brainstore: deployment: {} service: {} pod: {} + podDisruptionBudget: + enabled: false + maxUnavailable: 1 replicas: 2 minReadySeconds: 0 progressDeadlineSeconds: 600 @@ -542,6 +549,9 @@ brainstore: deployment: {} service: {} pod: {} + podDisruptionBudget: + enabled: false + maxUnavailable: 1 replicas: 1 minReadySeconds: 0 progressDeadlineSeconds: 600 From 2fd06140f3101e419e7d069b2f9d78c70e7df212 Mon Sep 17 00:00:00 2001 From: Erik Weathers <1111441+erikdw@users.noreply.github.com> Date: Wed, 28 Oct 2026 13:38:43 -0700 Subject: [PATCH 3/6] Select GKE Standard pools on braintrust/workload The Terraform module sets braintrust/node-pool to each pool's map key, so a Brainstore replacement pool keyed differently never matched the brainstore selector and Brainstore pods could not move to it. braintrust/workload is shared by every pool that serves a workload, which is what the Terraform module's README tells Helm to select on. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 5 +++-- braintrust/README.md | 15 ++++++++------- braintrust/examples/google-standard/values.yaml | 10 +++++----- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9620779..2315542 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -121,8 +121,9 @@ The GKE Standard example enables separate budgets for the API and all Brainstore A single writer can stop briefly during eviction. With multiple writers, the budget permits one unavailable replica. Each role has an independent budget, so different roles can lose a replica simultaneously. Readiness probes determine healthy replicas. Deployment rollout settings and `minReadySeconds` do not control node eviction. -Stable `braintrust/node-pool` labels connect Helm selectors to Terraform pools. -The Terraform pool map key sets this label. Automatic replacement preserves the label. +Stable `braintrust/workload` labels connect Helm selectors to Terraform pools. +The Terraform pool map key sets this label unless the pool sets `workload`. Replacement pools and automatic replacement preserve it. +Do not select on `braintrust/node-pool` or `cloud.google.com/gke-nodepool`. Neither matches a replacement pool. GKE deletion protection must exist on the source pool before hardware replacement. Its PDB protection expires after one hour. - Keep PDB templates cloud-independent and disabled by default. diff --git a/braintrust/README.md b/braintrust/README.md index 61378b6..5adb943 100644 --- a/braintrust/README.md +++ b/braintrust/README.md @@ -154,8 +154,9 @@ brainstore: The Terraform module creates separate services and Brainstore pools. Brainstore uses a bundled Local SSD machine type without a disk count input. -The stable `braintrust/node-pool` label connects Helm selectors to replacement pools with generated GKE names. -The Terraform map key sets the label. The `services` and `brainstore` keys match the selectors below. +The stable `braintrust/workload` label connects Helm selectors to Terraform pools, including replacement pools with generated GKE names. +The Terraform map key sets the label unless the pool sets `workload`. The `services` and `brainstore` values match the selectors below. +A replacement pool with `workload = "brainstore"` receives Brainstore pods before the source pool is deleted. The example enables PDBs for node eviction. Existing pools need deletion protection before a hardware replacement. See [Optional disruption budgets](#optional-disruption-budgets) for the preparation sequence and limits. @@ -172,11 +173,11 @@ See [Optional disruption budgets](#optional-disruption-budgets) for the preparat enabled: true maxUnavailable: 1 nodeSelector: - braintrust/node-pool: "services" + braintrust/workload: "services" aiGateway: nodeSelector: - braintrust/node-pool: "services" + braintrust/workload: "services" brainstore: reader: @@ -184,7 +185,7 @@ See [Optional disruption budgets](#optional-disruption-budgets) for the preparat enabled: true maxUnavailable: 1 nodeSelector: - braintrust/node-pool: "brainstore" + braintrust/workload: "brainstore" resources: requests: cpu: "44" @@ -206,13 +207,13 @@ See [Optional disruption budgets](#optional-disruption-budgets) for the preparat enabled: true maxUnavailable: 1 nodeSelector: - braintrust/node-pool: "brainstore" + braintrust/workload: "brainstore" writer: podDisruptionBudget: enabled: true maxUnavailable: 1 nodeSelector: - braintrust/node-pool: "brainstore" + braintrust/workload: "brainstore" resources: requests: cpu: "44" diff --git a/braintrust/examples/google-standard/values.yaml b/braintrust/examples/google-standard/values.yaml index eb4e002..2d19f90 100644 --- a/braintrust/examples/google-standard/values.yaml +++ b/braintrust/examples/google-standard/values.yaml @@ -48,7 +48,7 @@ api: # Native GCS authentication via Workload Identity. Set to true to use GCS natively instead of S3-compatible access. enableGcsAuth: false nodeSelector: - braintrust/node-pool: "services" + braintrust/workload: "services" resources: requests: cpu: "4" @@ -63,7 +63,7 @@ api: aiGateway: nodeSelector: - braintrust/node-pool: "services" + braintrust/workload: "services" # Brainstore configuration (split into reader and writer) brainstore: @@ -90,7 +90,7 @@ brainstore: port: 4000 portName: http nodeSelector: - braintrust/node-pool: "brainstore" + braintrust/workload: "brainstore" resources: requests: cpu: "16" @@ -123,7 +123,7 @@ brainstore: enabled: true maxUnavailable: 1 nodeSelector: - braintrust/node-pool: "brainstore" + braintrust/workload: "brainstore" # Brainstore Writer configuration # One writer can stop briefly during a node drain. @@ -139,7 +139,7 @@ brainstore: port: 4000 portName: http nodeSelector: - braintrust/node-pool: "brainstore" + braintrust/workload: "brainstore" resources: requests: cpu: "32" From 8e7ea125bac33b756fc7228e5ed66da2de97cafc Mon Sep 17 00:00:00 2001 From: Erik Weathers <1111441+erikdw@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:47:30 -0700 Subject: [PATCH 4/6] Default Brainstore PDBs for reused values `helm upgrade --reuse-values` from a chart without Brainstore budgets left `podDisruptionBudget` unset, and the template failed to render on every cloud. Treat a missing or null budget map as disabled, and default a missing or null `maxUnavailable` to 1 while keeping an explicit 0. Co-Authored-By: Claude Opus 5.5 --- braintrust/templates/brainstore-pdb.yaml | 11 +++- .../pre-brainstore-pdb-values.yaml | 10 ++++ braintrust/tests/brainstore-pdb_test.yaml | 57 +++++++++++++++++++ 3 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 braintrust/tests/__fixtures__/pre-brainstore-pdb-values.yaml diff --git a/braintrust/templates/brainstore-pdb.yaml b/braintrust/templates/brainstore-pdb.yaml index 1a70a8e..567cf44 100644 --- a/braintrust/templates/brainstore-pdb.yaml +++ b/braintrust/templates/brainstore-pdb.yaml @@ -1,8 +1,13 @@ {{- $root := . -}} {{- $rendered := 0 -}} {{- range $role := list "reader" "fastreader" "writer" -}} -{{- $config := index $root.Values.brainstore $role -}} -{{- if $config.podDisruptionBudget.enabled }} +{{- $config := index $root.Values.brainstore $role | default dict -}} +{{- $pdb := $config.podDisruptionBudget | default dict -}} +{{- $maxUnavailable := 1 -}} +{{- if and (hasKey $pdb "maxUnavailable") (not (kindIs "invalid" $pdb.maxUnavailable)) -}} +{{- $maxUnavailable = $pdb.maxUnavailable -}} +{{- end -}} +{{- if $pdb.enabled }} {{- if gt $rendered 0 }} --- {{- end }} @@ -16,7 +21,7 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - maxUnavailable: {{ $config.podDisruptionBudget.maxUnavailable | toYaml | trim }} + maxUnavailable: {{ $maxUnavailable | toYaml | trim }} selector: matchLabels: app: {{ $config.name }} diff --git a/braintrust/tests/__fixtures__/pre-brainstore-pdb-values.yaml b/braintrust/tests/__fixtures__/pre-brainstore-pdb-values.yaml new file mode 100644 index 0000000..abede38 --- /dev/null +++ b/braintrust/tests/__fixtures__/pre-brainstore-pdb-values.yaml @@ -0,0 +1,10 @@ +# Simulate values retained by `helm upgrade --reuse-values` from a chart release +# that predates Brainstore disruption budgets. An explicit null removes the new +# chart default during Helm value coalescing. +brainstore: + reader: + podDisruptionBudget: null + fastreader: + podDisruptionBudget: null + writer: + podDisruptionBudget: null diff --git a/braintrust/tests/brainstore-pdb_test.yaml b/braintrust/tests/brainstore-pdb_test.yaml index ab30029..1b8d7f9 100644 --- a/braintrust/tests/brainstore-pdb_test.yaml +++ b/braintrust/tests/brainstore-pdb_test.yaml @@ -31,6 +31,63 @@ tests: value: 0 - notExists: path: spec.minAvailable + - it: omits budgets when reused values have no budget maps + template: brainstore-pdb.yaml + values: + - __fixtures__/pre-brainstore-pdb-values.yaml + asserts: + - hasDocuments: + count: 0 + - it: omits budgets for reused values on aws + template: brainstore-pdb.yaml + values: + - __fixtures__/pre-brainstore-pdb-values.yaml + set: + cloud: aws + asserts: + - hasDocuments: + count: 0 + - it: omits budgets for reused values on azure + template: brainstore-pdb.yaml + values: + - __fixtures__/pre-brainstore-pdb-values.yaml + set: + cloud: azure + asserts: + - hasDocuments: + count: 0 + - it: omits budgets for reused values on google + template: brainstore-pdb.yaml + values: + - __fixtures__/pre-brainstore-pdb-values.yaml + set: + cloud: google + asserts: + - hasDocuments: + count: 0 + - it: defaults maxUnavailable for a partial budget on reused values + template: brainstore-pdb.yaml + values: + - __fixtures__/pre-brainstore-pdb-values.yaml + set: + brainstore.reader.podDisruptionBudget.enabled: true + asserts: + - hasDocuments: + count: 1 + - equal: + path: spec.maxUnavailable + value: 1 + - it: defaults a null maxUnavailable + template: brainstore-pdb.yaml + set: + brainstore.writer.podDisruptionBudget.enabled: true + brainstore.writer.podDisruptionBudget.maxUnavailable: null + asserts: + - hasDocuments: + count: 1 + - equal: + path: spec.maxUnavailable + value: 1 - it: keeps labels isolated between roles template: brainstore-pdb.yaml set: From 959acd9311b35f0debffc65898dbf4ee128fd5bc Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Fri, 30 Oct 2026 16:36:42 -0400 Subject: [PATCH 5/6] Handle null API disruption budgets correctly --- braintrust/templates/api-pdb.yaml | 9 +-- braintrust/tests/brainstore-pdb_test.yaml | 76 +++++++++++++++++++++++ 2 files changed, 81 insertions(+), 4 deletions(-) diff --git a/braintrust/templates/api-pdb.yaml b/braintrust/templates/api-pdb.yaml index 769b991..994f390 100644 --- a/braintrust/templates/api-pdb.yaml +++ b/braintrust/templates/api-pdb.yaml @@ -3,7 +3,8 @@ {{- $rendered := 0 -}} {{- range $pool := $pools -}} {{- $api := $pool.config -}} -{{- if $api.podDisruptionBudget.enabled }} +{{- $pdb := $api.podDisruptionBudget | default dict -}} +{{- if $pdb.enabled }} {{- if gt $rendered 0 }} --- {{- end }} @@ -18,10 +19,10 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} spec: - {{- if hasKey $api.podDisruptionBudget "maxUnavailable" }} - maxUnavailable: {{ $api.podDisruptionBudget.maxUnavailable | toYaml | trim }} + {{- if ne $pdb.maxUnavailable nil }} + maxUnavailable: {{ $pdb.maxUnavailable | toYaml | trim }} {{- else }} - minAvailable: {{ $api.podDisruptionBudget.minAvailable }} + minAvailable: {{ $pdb.minAvailable }} {{- end }} selector: matchLabels: diff --git a/braintrust/tests/brainstore-pdb_test.yaml b/braintrust/tests/brainstore-pdb_test.yaml index 1b8d7f9..a9141f9 100644 --- a/braintrust/tests/brainstore-pdb_test.yaml +++ b/braintrust/tests/brainstore-pdb_test.yaml @@ -157,6 +157,82 @@ tests: value: 2 - notExists: path: spec.maxUnavailable + - it: omits the API budget when its map is absent from reused values + template: api-pdb.yaml + set: + api.podDisruptionBudget: null + asserts: + - hasDocuments: + count: 0 + - it: falls back to the API minimum when maxUnavailable is null + template: api-pdb.yaml + set: + cloud: google + api.podDisruptionBudget.enabled: true + api.podDisruptionBudget.minAvailable: 2 + api.podDisruptionBudget.maxUnavailable: null + asserts: + - equal: + path: spec.minAvailable + value: 2 + - notExists: + path: spec.maxUnavailable + - it: preserves an explicit zero API maximum over the minimum + template: api-pdb.yaml + set: + cloud: aws + api.podDisruptionBudget.enabled: true + api.podDisruptionBudget.maxUnavailable: 0 + asserts: + - equal: + path: spec.maxUnavailable + value: 0 + - notExists: + path: spec.minAvailable + - it: preserves percentage API maximums + template: api-pdb.yaml + set: + cloud: azure + api.podDisruptionBudget.enabled: true + api.podDisruptionBudget.maxUnavailable: "25%" + asserts: + - equal: + path: spec.maxUnavailable + value: "25%" + - notExists: + path: spec.minAvailable + - it: leaves an empty API maximum for Kubernetes to validate + template: api-pdb.yaml + set: + api.podDisruptionBudget.enabled: true + api.podDisruptionBudget.maxUnavailable: "" + asserts: + - equal: + path: spec.maxUnavailable + value: "" + - notExists: + path: spec.minAvailable + - it: lets an isolated pool clear an inherited API maximum + template: api-pdb.yaml + set: + api.podDisruptionBudget.maxUnavailable: 1 + api.workloadIsolation.enabled: true + api.workloadIsolation.background.podDisruptionBudget.maxUnavailable: null + api.workloadIsolation.background.podDisruptionBudget.minAvailable: 2 + asserts: + - hasDocuments: + count: 2 + - equal: + path: spec.maxUnavailable + value: 1 + documentIndex: 0 + - equal: + path: spec.minAvailable + value: 2 + documentIndex: 1 + - notExists: + path: spec.maxUnavailable + documentIndex: 1 - it: supports optional budgets on aws template: brainstore-pdb.yaml set: From 3d4da11ecbdd553ab9316eba1237a51090f08d36 Mon Sep 17 00:00:00 2001 From: Alex Rhee Date: Fri, 30 Oct 2026 16:50:22 -0400 Subject: [PATCH 6/6] Consolidate disruption budget test fixtures --- .../pre-brainstore-pdb-values.yaml | 2 + braintrust/tests/brainstore-pdb_test.yaml | 107 ++++-------------- 2 files changed, 27 insertions(+), 82 deletions(-) diff --git a/braintrust/tests/__fixtures__/pre-brainstore-pdb-values.yaml b/braintrust/tests/__fixtures__/pre-brainstore-pdb-values.yaml index abede38..daddcdd 100644 --- a/braintrust/tests/__fixtures__/pre-brainstore-pdb-values.yaml +++ b/braintrust/tests/__fixtures__/pre-brainstore-pdb-values.yaml @@ -1,6 +1,8 @@ # Simulate values retained by `helm upgrade --reuse-values` from a chart release # that predates Brainstore disruption budgets. An explicit null removes the new # chart default during Helm value coalescing. +api: + podDisruptionBudget: null brainstore: reader: podDisruptionBudget: null diff --git a/braintrust/tests/brainstore-pdb_test.yaml b/braintrust/tests/brainstore-pdb_test.yaml index a9141f9..46938e6 100644 --- a/braintrust/tests/brainstore-pdb_test.yaml +++ b/braintrust/tests/brainstore-pdb_test.yaml @@ -1,11 +1,10 @@ -suite: Brainstore disruption budgets +suite: API and Brainstore disruption budgets templates: - brainstore-pdb.yaml - api-pdb.yaml tests: - it: leaves all PDBs absent by default - template: brainstore-pdb.yaml - asserts: + asserts: &absent - hasDocuments: count: 0 - it: enables only the selected role and preserves a zero budget @@ -32,46 +31,30 @@ tests: - notExists: path: spec.minAvailable - it: omits budgets when reused values have no budget maps - template: brainstore-pdb.yaml - values: + values: &reused-values - __fixtures__/pre-brainstore-pdb-values.yaml - asserts: - - hasDocuments: - count: 0 + asserts: *absent - it: omits budgets for reused values on aws - template: brainstore-pdb.yaml - values: - - __fixtures__/pre-brainstore-pdb-values.yaml + values: *reused-values set: cloud: aws - asserts: - - hasDocuments: - count: 0 + asserts: *absent - it: omits budgets for reused values on azure - template: brainstore-pdb.yaml - values: - - __fixtures__/pre-brainstore-pdb-values.yaml + values: *reused-values set: cloud: azure - asserts: - - hasDocuments: - count: 0 + asserts: *absent - it: omits budgets for reused values on google - template: brainstore-pdb.yaml - values: - - __fixtures__/pre-brainstore-pdb-values.yaml + values: *reused-values set: cloud: google - asserts: - - hasDocuments: - count: 0 + asserts: *absent - it: defaults maxUnavailable for a partial budget on reused values template: brainstore-pdb.yaml - values: - - __fixtures__/pre-brainstore-pdb-values.yaml + values: *reused-values set: brainstore.reader.podDisruptionBudget.enabled: true - asserts: + asserts: &one-unavailable - hasDocuments: count: 1 - equal: @@ -82,12 +65,7 @@ tests: set: brainstore.writer.podDisruptionBudget.enabled: true brainstore.writer.podDisruptionBudget.maxUnavailable: null - asserts: - - hasDocuments: - count: 1 - - equal: - path: spec.maxUnavailable - value: 1 + asserts: *one-unavailable - it: keeps labels isolated between roles template: brainstore-pdb.yaml set: @@ -151,19 +129,12 @@ tests: set: api.podDisruptionBudget.enabled: true api.podDisruptionBudget.minAvailable: 2 - asserts: + asserts: &api-minimum - equal: path: spec.minAvailable value: 2 - notExists: path: spec.maxUnavailable - - it: omits the API budget when its map is absent from reused values - template: api-pdb.yaml - set: - api.podDisruptionBudget: null - asserts: - - hasDocuments: - count: 0 - it: falls back to the API minimum when maxUnavailable is null template: api-pdb.yaml set: @@ -171,12 +142,7 @@ tests: api.podDisruptionBudget.enabled: true api.podDisruptionBudget.minAvailable: 2 api.podDisruptionBudget.maxUnavailable: null - asserts: - - equal: - path: spec.minAvailable - value: 2 - - notExists: - path: spec.maxUnavailable + asserts: *api-minimum - it: preserves an explicit zero API maximum over the minimum template: api-pdb.yaml set: @@ -189,18 +155,6 @@ tests: value: 0 - notExists: path: spec.minAvailable - - it: preserves percentage API maximums - template: api-pdb.yaml - set: - cloud: azure - api.podDisruptionBudget.enabled: true - api.podDisruptionBudget.maxUnavailable: "25%" - asserts: - - equal: - path: spec.maxUnavailable - value: "25%" - - notExists: - path: spec.minAvailable - it: leaves an empty API maximum for Kubernetes to validate template: api-pdb.yaml set: @@ -234,38 +188,27 @@ tests: path: spec.maxUnavailable documentIndex: 1 - it: supports optional budgets on aws - template: brainstore-pdb.yaml - set: + set: &percentage-budgets cloud: aws brainstore.reader.podDisruptionBudget.enabled: true brainstore.reader.podDisruptionBudget.maxUnavailable: "25%" - asserts: + api.podDisruptionBudget.enabled: true + api.podDisruptionBudget.maxUnavailable: "25%" + asserts: &percentage - hasDocuments: count: 1 - equal: path: spec.maxUnavailable value: "25%" + - notExists: + path: spec.minAvailable - it: supports optional budgets on azure - template: brainstore-pdb.yaml set: + <<: *percentage-budgets cloud: azure - brainstore.reader.podDisruptionBudget.enabled: true - brainstore.reader.podDisruptionBudget.maxUnavailable: "25%" - asserts: - - hasDocuments: - count: 1 - - equal: - path: spec.maxUnavailable - value: "25%" + asserts: *percentage - it: supports optional budgets on google - template: brainstore-pdb.yaml set: + <<: *percentage-budgets cloud: google - brainstore.reader.podDisruptionBudget.enabled: true - brainstore.reader.podDisruptionBudget.maxUnavailable: "25%" - asserts: - - hasDocuments: - count: 1 - - equal: - path: spec.maxUnavailable - value: "25%" + asserts: *percentage