diff --git a/pkg/builder/kaniko/publisher.go b/pkg/builder/kaniko/publisher.go
index 6529dcb..535cfbb 100644
--- a/pkg/builder/kaniko/publisher.go
+++ b/pkg/builder/kaniko/publisher.go
@@ -26,7 +26,6 @@
 	"time"
 
 	"github.com/apache/camel-k/pkg/builder"
-	"github.com/apache/camel-k/pkg/platform"
 	"github.com/apache/camel-k/pkg/util/defaults"
 	"github.com/apache/camel-k/pkg/util/kubernetes"
 	"github.com/apache/camel-k/pkg/util/tar"
@@ -187,25 +186,14 @@
 		},
 	}
 
-	var labelKey string
-	var labelValue string
-	if ctx.Namespace == platform.GetOperatorNamespace() {
-		// Check if the operator is running in the same namespace
-		labelKey = "camel.apache.org/component"
-		labelValue = "operator"
-	} else {
-		labelKey = "camel.apache.org/build"
-		labelValue = ctx.Build.Meta.Name
-	}
-
-	// Co-locate with builder pod for sharing the volume
+	// Co-locate with the build pod for sharing the volume
 	pod.Spec.Affinity = &corev1.Affinity{
 		PodAffinity: &corev1.PodAffinity{
 			RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
 				{
 					LabelSelector: &metav1.LabelSelector{
 						MatchLabels: map[string]string{
-							labelKey: labelValue,
+							"camel.apache.org/build": ctx.Build.Meta.Name,
 						},
 					},
 					TopologyKey: "kubernetes.io/hostname",
diff --git a/pkg/controller/build/schedule_pod.go b/pkg/controller/build/schedule_pod.go
index b3b3dcd..affba9b 100644
--- a/pkg/controller/build/schedule_pod.go
+++ b/pkg/controller/build/schedule_pod.go
@@ -21,16 +21,18 @@
 	"context"
 	"sync"
 
-	"github.com/apache/camel-k/pkg/apis/camel/v1alpha1"
-	"github.com/apache/camel-k/pkg/platform"
-	"github.com/apache/camel-k/pkg/util/defaults"
-
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
 
 	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
+	"github.com/apache/camel-k/pkg/apis/camel/v1alpha1"
+	"github.com/apache/camel-k/pkg/platform"
+	"github.com/apache/camel-k/pkg/util/defaults"
+
 	"github.com/pkg/errors"
 )
 
@@ -96,7 +98,10 @@
 
 		// We may want to explicitly manage build priority as opposed to relying on
 		// the reconcile loop to handle the queuing
-		pod = newBuildPod(build, operatorImage)
+		pod, err = action.newBuildPod(ctx, build, operatorImage)
+		if err != nil {
+			return nil, err
+		}
 
 		// Set the Build instance as the owner and controller
 		if err := controllerutil.SetControllerReference(build, pod, action.client.GetScheme()); err != nil {
@@ -113,7 +118,7 @@
 	return build, nil
 }
 
-func newBuildPod(build *v1alpha1.Build, operatorImage string) *corev1.Pod {
+func (action *schedulePodAction) newBuildPod(ctx context.Context, build *v1alpha1.Build, operatorImage string) (*corev1.Pod, error) {
 	builderImage := operatorImage
 	if builderImage == "" {
 		builderImage = defaults.ImageName + ":" + defaults.Version
@@ -127,7 +132,8 @@
 			Namespace: build.Namespace,
 			Name:      buildPodName(build.Spec.Meta),
 			Labels: map[string]string{
-				"camel.apache.org/build": build.Name,
+				"camel.apache.org/build":     build.Name,
+				"camel.apache.org/component": "builder",
 			},
 		},
 		Spec: corev1.PodSpec{
@@ -185,21 +191,52 @@
 			},
 		})
 
-		// Use affinity only when the operator is present in the namespaced
-		if build.Namespace == platform.GetOperatorNamespace() {
-			// Co-locate with the builder pod for sharing the host path volume as the current
+		// Use affinity when Kaniko cache warming is enabled
+		if build.Spec.Platform.Build.IsKanikoCacheEnabled() {
+			// Co-locate with the Kaniko warmer pod for sharing the host path volume as the current
 			// persistent volume claim uses the default storage class which is likely relying
 			// on the host path provisioner.
+			// This has to be done manually by retrieving the Kaniko warmer pod node name and using
+			// node affinity as pod affinity only works for running pods and the Kaniko warmer pod
+			// has already completed at that stage.
+
+			// Locate the kaniko warmer pod
+			byComponentLabel, err := labels.NewRequirement("camel.apache.org/component", selection.Equals, []string{"kaniko-warmer"})
+			if err != nil {
+				return nil, err
+			}
+
+			selector := labels.NewSelector().Add(*byComponentLabel)
+
+			options := &k8sclient.ListOptions{
+				Namespace:     build.Namespace,
+				LabelSelector: selector,
+			}
+
+			pods := &corev1.PodList{}
+			err = action.client.List(ctx, options, pods)
+			if err != nil {
+				return nil, err
+			}
+
+			if len(pods.Items) != 1 {
+				return nil, errors.New("failed to locate the Kaniko cache warmer pod")
+			}
+
+			// Use node affinity with the Kaniko warmer pod node name
 			pod.Spec.Affinity = &corev1.Affinity{
-				PodAffinity: &corev1.PodAffinity{
-					RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
-						{
-							LabelSelector: &metav1.LabelSelector{
-								MatchLabels: map[string]string{
-									"camel.apache.org/component": "operator",
+				NodeAffinity: &corev1.NodeAffinity{
+					RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
+						NodeSelectorTerms: []corev1.NodeSelectorTerm{
+							{
+								MatchExpressions: []corev1.NodeSelectorRequirement{
+									{
+										Key:      "kubernetes.io/hostname",
+										Operator: "In",
+										Values:   []string{pods.Items[0].Spec.NodeName},
+									},
 								},
 							},
-							TopologyKey: "kubernetes.io/hostname",
 						},
 					},
 				},
@@ -207,5 +244,5 @@
 		}
 	}
 
-	return pod
+	return pod, nil
 }
diff --git a/pkg/controller/integrationplatform/kaniko_cache.go b/pkg/controller/integrationplatform/kaniko_cache.go
index 3470663..217c50f 100644
--- a/pkg/controller/integrationplatform/kaniko_cache.go
+++ b/pkg/controller/integrationplatform/kaniko_cache.go
@@ -48,6 +48,9 @@
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: platform.Namespace,
 			Name:      platform.Name + "-cache",
+			Labels: map[string]string{
+				"camel.apache.org/component": "kaniko-warmer",
+			},
 		},
 		Spec: corev1.PodSpec{
 			Containers: []corev1.Container{
@@ -93,23 +96,6 @@
 					},
 				},
 			},
-			// Co-locate with the builder pod for sharing the host path volume as the current
-			// persistent volume claim uses the default storage class which is likely relying
-			// on the host path provisioner.
-			Affinity: &corev1.Affinity{
-				PodAffinity: &corev1.PodAffinity{
-					RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
-						{
-							LabelSelector: &metav1.LabelSelector{
-								MatchLabels: map[string]string{
-									"camel.apache.org/component": "operator",
-								},
-							},
-							TopologyKey: "kubernetes.io/hostname",
-						},
-					},
-				},
-			},
 		},
 	}