Skip to content

fix: add node affinity for node where PVC is mounted to workspace's backup Jobs #1677

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dkwon17 merged 1 commit into from
Jul 29, 2026
Merged

fix: add node affinity for node where PVC is mounted to workspace's backup Jobs #1677

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions controllers/backupcronjob/backupcronjob_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"github.com/devfile/devworkspace-operator/pkg/constants"
"github.com/devfile/devworkspace-operator/pkg/infrastructure"
"github.com/devfile/devworkspace-operator/pkg/library/storage"
provstorage "github.com/devfile/devworkspace-operator/pkg/provision/storage"
"github.com/devfile/devworkspace-operator/pkg/secrets"
"github.com/go-logr/logr"
"github.com/robfig/cron/v3"
Expand Down Expand Up @@ -456,6 +457,18 @@ func (r *BackupCronJobReconciler) createBackupJob(
},
},
}
// Pin backup Job to the node where the PVC is currently mounted to avoid
// Multi-Attach errors with ReadWriteOnce PVCs on multi-node clusters.
targetNode, err := provstorage.FindNodeForPVC(ctx, r.Client, workspace.Namespace, pvc.Name)
if err != nil {
log.Error(err, "Failed to find node with PVC, backup Job will not have node affinity", "pvc", pvc.Name)
} else if targetNode == "" {
log.Info("No target node for backup job, NodeAffinity will not be defined", "pvc", pvc.Name)
}
Comment thread
dkwon17 marked this conversation as resolved.
if targetNode != "" {
job.Spec.Template.Spec.Affinity = provstorage.NodeAffinityForHostname(targetNode)
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

if registryAuthSecret != nil {
job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, corev1.Volume{
Name: constants.RegistryAuthVolumeName,
Expand Down
110 changes: 110 additions & 0 deletions controllers/backupcronjob/backupcronjob_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -426,6 +426,116 @@ var _ = Describe("BackupCronJobReconciler", func() {
Expect(*jobList.Items[0].Spec.BackoffLimit).To(Equal(int32(2)))
})

It("creates a Job with node affinity when a running pod mounts the PVC", func() {
enabled := true
schedule := "* * * * *"
dwoc := &controllerv1alpha1.DevWorkspaceOperatorConfig{
ObjectMeta: metav1.ObjectMeta{Name: nameNamespace.Name, Namespace: nameNamespace.Namespace},
Config: &controllerv1alpha1.OperatorConfiguration{
Workspace: &controllerv1alpha1.WorkspaceConfig{
BackupCronJob: &controllerv1alpha1.BackupCronJobConfig{
Enable: &enabled,
Schedule: schedule,
Registry: &controllerv1alpha1.RegistryConfig{
Path: "fake-registry",
AuthSecret: "backup-auth",
},
},
},
},
}
Expect(fakeClient.Create(ctx, dwoc)).To(Succeed())
dw := createDevWorkspace("dw-affinity", "ns-affinity", false, metav1.NewTime(time.Now().Add(-10*time.Minute)))
dw.Status.Phase = dwv2.DevWorkspaceStatusStopped
dw.Status.DevWorkspaceId = "id-affinity"
Expect(fakeClient.Create(ctx, dw)).To(Succeed())

pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "claim-devworkspace", Namespace: dw.Namespace}}
Expect(fakeClient.Create(ctx, pvc)).To(Succeed())

authSecret := createAuthSecret("backup-auth", nameNamespace.Namespace, map[string][]byte{})
Expect(fakeClient.Create(ctx, authSecret)).To(Succeed())

// Create a running pod that mounts the PVC on a specific node
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "workspace-pod",
Namespace: dw.Namespace,
Labels: map[string]string{constants.DevWorkspaceIDLabel: "other-workspace-id"},
},
Spec: corev1.PodSpec{
NodeName: "worker-node-1",
Volumes: []corev1.Volume{
{
Name: "workspace-data",
VolumeSource: corev1.VolumeSource{
PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
ClaimName: "claim-devworkspace",
},
},
},
},
},
Status: corev1.PodStatus{Phase: corev1.PodRunning},
}
Expect(fakeClient.Create(ctx, pod)).To(Succeed())

Expect(reconciler.executeBackupSync(ctx, dwoc, log)).To(Succeed())

jobList := &batchv1.JobList{}
Expect(fakeClient.List(ctx, jobList, &client.ListOptions{Namespace: dw.Namespace})).To(Succeed())
Expect(jobList.Items).To(HaveLen(1))
job := jobList.Items[0]
Expect(job.Spec.Template.Spec.Affinity).ToNot(BeNil())
Expect(job.Spec.Template.Spec.Affinity.NodeAffinity).ToNot(BeNil())
nodeSelector := job.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution
Expect(nodeSelector).ToNot(BeNil())
Expect(nodeSelector.NodeSelectorTerms).To(HaveLen(1))
Expect(nodeSelector.NodeSelectorTerms[0].MatchExpressions).To(HaveLen(1))
expr := nodeSelector.NodeSelectorTerms[0].MatchExpressions[0]
Expect(expr.Key).To(Equal("kubernetes.io/hostname"))
Expect(expr.Operator).To(Equal(corev1.NodeSelectorOpIn))
Expect(expr.Values).To(Equal([]string{"worker-node-1"}))
})

It("creates a Job without node affinity when no running pod mounts the PVC", func() {
enabled := true
schedule := "* * * * *"
dwoc := &controllerv1alpha1.DevWorkspaceOperatorConfig{
ObjectMeta: metav1.ObjectMeta{Name: nameNamespace.Name, Namespace: nameNamespace.Namespace},
Config: &controllerv1alpha1.OperatorConfiguration{
Workspace: &controllerv1alpha1.WorkspaceConfig{
BackupCronJob: &controllerv1alpha1.BackupCronJobConfig{
Enable: &enabled,
Schedule: schedule,
Registry: &controllerv1alpha1.RegistryConfig{
Path: "fake-registry",
AuthSecret: "backup-auth",
},
},
},
},
}
Expect(fakeClient.Create(ctx, dwoc)).To(Succeed())
dw := createDevWorkspace("dw-no-affinity", "ns-no-affinity", false, metav1.NewTime(time.Now().Add(-10*time.Minute)))
dw.Status.Phase = dwv2.DevWorkspaceStatusStopped
dw.Status.DevWorkspaceId = "id-no-affinity"
Expect(fakeClient.Create(ctx, dw)).To(Succeed())

pvc := &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "claim-devworkspace", Namespace: dw.Namespace}}
Expect(fakeClient.Create(ctx, pvc)).To(Succeed())

authSecret := createAuthSecret("backup-auth", nameNamespace.Namespace, map[string][]byte{})
Expect(fakeClient.Create(ctx, authSecret)).To(Succeed())

Expect(reconciler.executeBackupSync(ctx, dwoc, log)).To(Succeed())

jobList := &batchv1.JobList{}
Expect(fakeClient.List(ctx, jobList, &client.ListOptions{Namespace: dw.Namespace})).To(Succeed())
Expect(jobList.Items).To(HaveLen(1))
Expect(jobList.Items[0].Spec.Template.Spec.Affinity).To(BeNil())
})

It("creates a Job with configured podSecurityContext", func() {
enabled := true
schedule := "* * * * *"
Expand Down
57 changes: 2 additions & 55 deletions pkg/provision/storage/cleanup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,7 @@ import (
k8sErrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

Expand Down Expand Up @@ -121,7 +119,7 @@ func getSpecCommonPVCCleanupJob(workspace *common.DevWorkspaceWithConfig, cluste
pvcName = workspace.Config.Workspace.PVCName
}

targetNode, err := getTargetNodeName(workspace, clusterAPI)
targetNode, err := FindNodeForPVC(clusterAPI.Ctx, clusterAPI.Client, workspace.Namespace, pvcName)
if err != nil {
clusterAPI.Logger.Error(err, "Error getting target node for cleanup job")
} else if targetNode == "" {
Expand Down Expand Up @@ -197,21 +195,7 @@ func getSpecCommonPVCCleanupJob(workspace *common.DevWorkspaceWithConfig, cluste
}

if targetNode != "" {
job.Spec.Template.Spec.Affinity.NodeAffinity = &corev1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
NodeSelectorTerms: []corev1.NodeSelectorTerm{
{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: corev1.LabelHostname,
Operator: corev1.NodeSelectorOpIn,
Values: []string{targetNode},
},
},
},
},
},
}
job.Spec.Template.Spec.Affinity = NodeAffinityForHostname(targetNode)
}

podTolerations, nodeSelector, err := nsconfig.GetNamespacePodTolerationsAndNodeSelector(workspace.Namespace, clusterAPI)
Expand Down Expand Up @@ -246,40 +230,3 @@ func commonPVCExists(workspace *common.DevWorkspaceWithConfig, clusterAPI sync.C
}
return true, nil
}

// getTargetNodeName returns the node name of the node a running devworkspace pod that already mounts the
// common PVC is running in.
// Returns an empty string if no such pod exists.
func getTargetNodeName(workspace *common.DevWorkspaceWithConfig, clusterAPI sync.ClusterAPI) (string, error) {

labelSelector, err := labels.Parse(constants.DevWorkspaceIDLabel)
if err != nil {
return "", err
}

listOptions := &client.ListOptions{
Namespace: workspace.Namespace,
LabelSelector: labelSelector,
}

found := &corev1.PodList{}
err = clusterAPI.Client.List(clusterAPI.Ctx, found, listOptions)
if err != nil {
return "", err
}

return getNodeNameWithPVC(found, workspace.Config.Workspace.PVCName), nil
}

func getNodeNameWithPVC(list *corev1.PodList, pvcName string) string {
for _, pod := range list.Items {
if pod.Status.Phase == corev1.PodRunning {
for _, volume := range pod.Spec.Volumes {
if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == pvcName {
return pod.Spec.NodeName
}
}
}
}
return ""
}
62 changes: 62 additions & 0 deletions pkg/provision/storage/cleanup_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,68 @@ func TestGetSpecCommonPVCCleanupJobUsesConfigPodSecurityContext(t *testing.T) {
assert.Equal(t, customPodSecurityContext, job.Spec.Template.Spec.SecurityContext)
}

func TestGetSpecCommonPVCCleanupJobHasNodeAffinityWhenPodMountsPVC(t *testing.T) {
infrastructure.InitializeForTesting(infrastructure.Kubernetes)

namespace := "test-ns"
pvcName := "claim-devworkspace"

pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "workspace-pod",
Namespace: namespace,
Labels: map[string]string{constants.DevWorkspaceIDLabel: "other-workspace-id"},
},
Spec: corev1.PodSpec{NodeName: "worker-node-1", Volumes: []corev1.Volume{{Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: pvcName}}}}},
Status: corev1.PodStatus{Phase: corev1.PodRunning},
}

fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(
&corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}},
pod,
).Build()

workspace := &common.DevWorkspaceWithConfig{
DevWorkspace: &dw.DevWorkspace{
ObjectMeta: metav1.ObjectMeta{
Name: "test-workspace",
Namespace: namespace,
Labels: map[string]string{
constants.DevWorkspaceCreatorLabel: "test-creator",
},
},
Status: dw.DevWorkspaceStatus{
DevWorkspaceId: "test-workspace-id",
},
},
Config: &v1alpha1.OperatorConfiguration{
Workspace: &v1alpha1.WorkspaceConfig{
PVCName: pvcName,
},
},
}

clusterAPI := sync.ClusterAPI{
Client: fakeClient,
Scheme: scheme,
Logger: zap.New(zap.UseDevMode(true)),
Ctx: context.Background(),
}

job, err := getSpecCommonPVCCleanupJob(workspace, clusterAPI)
assert.NoError(t, err)
assert.NotNil(t, job.Spec.Template.Spec.Affinity)
assert.NotNil(t, job.Spec.Template.Spec.Affinity.NodeAffinity)
nodeSelector := job.Spec.Template.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution
assert.NotNil(t, nodeSelector)
assert.Len(t, nodeSelector.NodeSelectorTerms, 1)
assert.Len(t, nodeSelector.NodeSelectorTerms[0].MatchExpressions, 1)
expr := nodeSelector.NodeSelectorTerms[0].MatchExpressions[0]
assert.Equal(t, "kubernetes.io/hostname", expr.Key)
assert.Equal(t, corev1.NodeSelectorOpIn, expr.Operator)
assert.Equal(t, []string{"worker-node-1"}, expr.Values)
}

func TestGetSpecCommonPVCCleanupJobWithNilPodSecurityContext(t *testing.T) {
infrastructure.InitializeForTesting(infrastructure.Kubernetes)

Expand Down
56 changes: 56 additions & 0 deletions pkg/provision/storage/shared.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package storage

import (
"context"
"errors"
"fmt"

Expand All @@ -27,6 +28,7 @@ import (
k8sErrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"

Expand Down Expand Up @@ -266,6 +268,60 @@ func getSharedPVCWorkspaceCount(namespace string, api sync.ClusterAPI) (total in
return total, nil
}

// FindNodeForPVC lists DevWorkspace pods in the given namespace and returns the
// node name where a running pod mounts the specified PVC. Returns an empty
// string (and nil error) when no such pod is found.
func FindNodeForPVC(ctx context.Context, k8sClient client.Client, namespace, pvcName string) (string, error) {
labelSelector, err := labels.Parse(constants.DevWorkspaceIDLabel)
if err != nil {
return "", err
}
podList := &corev1.PodList{}
if err := k8sClient.List(ctx, podList, &client.ListOptions{
Namespace: namespace,
LabelSelector: labelSelector,
}); err != nil {
Comment on lines +274 to +283

@coderabbitai coderabbitai Bot Jul 24, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🩺 Stability & Availability | 🟠 Major | ⚡ Quick win

🧩 Analysis chain

🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "Files:"
git ls-files | grep -E '(^|/)(shared\.go|backupcronjob_controller\.go|cleanup\.go)$' || true

echo
echo "Relevant occurrences:"
rg -n "FindNodeForPVC|NonCachingClient|backupcronjob_controller\.go|cleanup\.go" -S .

echo
echo "Context shared.go around FindNodeForPVC:"
file=$(git ls-files | grep 'pkg/provision/storage/shared.go' | head -n1)
if [ -n "${file:-}" ]; then
  wc -l "$file"
  sed -n '240,310p' "$file" | cat -n
fi

echo
echo "Context backup cron job around call:"
file=$(git ls-files | grep 'controllers/backupcronjob/backupcronjob_controller.go' | head -n1)
if [ -n "${file:-}" ]; then
  wc -l "$file"
  sed -n '430,485p' "$file" | cat -n
fi

echo
echo "Context cleanup around call:"
file=$(git ls-files | grep 'pkg/provision/storage/cleanup.go' |头 n1 | sed 's/ / /;s/  */ /')
if [ -n "${file:-}" ]; then
  wc -l "$file"
  sed -n '90,145p' "$file" | cat -n
fi

Repository: devfile/devworkspace-operator

Length of output: 8245

🏁 Script executed:

#!/bin/bash
set -euo pipefail

echo "Backup caller and type fields:"
sed -n '48,70p' controllers/backupcronjob/backupcronjob_controller.go | cat -n
sed -n '445,468p' controllers/backupcronjob/backupcronjob_controller.go | cat -n

echo
echo "Cleanup callers:"
rg -n "FindNodeForPVC\\(" controllers pkg/provision/storage -S
sed -n '110,130p' controllers/workspace/cleanup.go | cat -n
sed -n '90,130p' pkg/provision/storage/cleanup.go | cat -n

echo
echo "ClusterAPI fields:"
rg -n "type ClusterAPI|NonCachingClient|Client.*ClusterAPI|Ctx.*ClusterAPI" -S pkg controllers -g '*.go' | head -n 80
sed -n '1,110p' pkg/provision/sync/cluster_api.go | cat -n

Repository: devfile/devworkspace-operator

Length of output: 8337

Use NonCachingClient for PVC placement lookups.

FindNodeForPVC determines where the PVC is currently mounted, but both callers pass cached clients:

  • controllers/backupcronjob/backupcronjob_controller.go:462 uses r.Client
  • controllers/workspace/cleanup.go:122 uses clusterAPI.Client

Fresh pod data is needed to avoid pinning cleanup or backup Jobs to a stale node and causing ReadWriteOnce PV Multi-Attach failures. Pass each caller’s NonCachingClient.

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@pkg/provision/storage/shared.go` around lines 274 - 283, The callers of
FindNodeForPVC in the backup cronjob reconciliation and workspace cleanup flows
must use their respective NonCachingClient instead of cached clients. Replace
r.Client and clusterAPI.Client at those call sites while leaving
FindNodeForPVC’s lookup logic unchanged.

Source: Coding guidelines

return "", err
}
return getNodeNameWithPVC(podList, pvcName), nil
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}

// NodeAffinityForHostname returns an Affinity that pins a pod to the given node
// using the kubernetes.io/hostname label. Used by both cleanup and backup Jobs
// to avoid Multi-Attach errors with ReadWriteOnce PVCs on multi-node clusters.
func NodeAffinityForHostname(nodeName string) *corev1.Affinity {
return &corev1.Affinity{
NodeAffinity: &corev1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
NodeSelectorTerms: []corev1.NodeSelectorTerm{
{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: corev1.LabelHostname,
Operator: corev1.NodeSelectorOpIn,
Values: []string{nodeName},
},
},
},
},
},
},
}
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

func getNodeNameWithPVC(list *corev1.PodList, pvcName string) string {
for _, pod := range list.Items {
if pod.Status.Phase == corev1.PodRunning {
for _, volume := range pod.Spec.Volumes {
if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == pvcName {
return pod.Spec.NodeName
}
}
}
}
return ""
}

func checkPVCTerminating(name, namespace string, api sync.ClusterAPI) (bool, error) {
if name == "" {
// Should not happen
Expand Down
Loading
Loading