Get correct SolrPods in SolrCloud controller (#634)
diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go
index 6bd81e8..af82887 100644
--- a/controllers/solrcloud_controller.go
+++ b/controllers/solrcloud_controller.go
@@ -437,16 +437,21 @@
// Get the SolrCloud's Pods and initialize them if necessary
var podList []corev1.Pod
var podSelector labels.Selector
- if podSelector, podList, err = r.initializePods(ctx, instance, logger); err != nil {
+ if podSelector, podList, err = r.initializePods(ctx, instance, statefulSet, logger); err != nil {
return requeueOrNot, err
}
// Make sure the SolrCloud status is up-to-date with the state of the cluster
var outOfDatePods util.OutOfDatePodSegmentation
var availableUpdatedPodCount int
- outOfDatePods, availableUpdatedPodCount, err = createCloudStatus(instance, &newStatus, statefulSet.Status, podSelector, podList)
+ var shouldRequeue bool
+ outOfDatePods, availableUpdatedPodCount, shouldRequeue, err = createCloudStatus(instance, &newStatus, statefulSet.Status, podSelector, podList)
if err != nil {
return requeueOrNot, err
+ } else if shouldRequeue {
+ // There is an issue with the status, so requeue to get a more up-to-date view of the cluster
+ updateRequeueAfter(&requeueOrNot, time.Second*1)
+ return requeueOrNot, nil
}
// We only want to do one cluster operation at a time, so we use a lock to ensure that.
@@ -620,7 +625,7 @@
}
// InitializePods Ensure that all SolrCloud Pods are initialized
-func (r *SolrCloudReconciler) initializePods(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, logger logr.Logger) (podSelector labels.Selector, podList []corev1.Pod, err error) {
+func (r *SolrCloudReconciler) initializePods(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, statefulSet *appsv1.StatefulSet, logger logr.Logger) (podSelector labels.Selector, podList []corev1.Pod, err error) {
foundPods := &corev1.PodList{}
selectorLabels := solrCloud.SharedLabels()
selectorLabels["technology"] = solrv1beta1.SolrTechnologyLabel
@@ -635,14 +640,24 @@
logger.Error(err, "Error listing pods for SolrCloud")
return
}
- podList = foundPods.Items
// Initialize the pod's notStopped readinessCondition so that they can receive traffic until they are stopped
- for i, pod := range podList {
+ for _, pod := range foundPods.Items {
+ isOwnedByCurrentStatefulSet := false
+ for _, ownerRef := range pod.ObjectMeta.OwnerReferences {
+ if ownerRef.UID == statefulSet.UID {
+ isOwnedByCurrentStatefulSet = true
+ break
+ }
+ }
+ // Do not include pods that match, but are not owned by the current statefulSet
+ if !isOwnedByCurrentStatefulSet {
+ continue
+ }
if updatedPod, podError := r.initializePod(ctx, &pod, logger); podError != nil {
err = podError
} else if updatedPod != nil {
- podList[i] = *updatedPod
+ podList = append(podList, *updatedPod)
}
}
return
@@ -676,7 +691,7 @@
// Initialize the SolrCloud.Status object
func createCloudStatus(solrCloud *solrv1beta1.SolrCloud,
newStatus *solrv1beta1.SolrCloudStatus, statefulSetStatus appsv1.StatefulSetStatus, podSelector labels.Selector,
- podList []corev1.Pod) (outOfDatePods util.OutOfDatePodSegmentation, availableUpdatedPodCount int, err error) {
+ podList []corev1.Pod) (outOfDatePods util.OutOfDatePodSegmentation, availableUpdatedPodCount int, shouldRequeue bool, err error) {
var otherVersions []string
nodeNames := make([]string, len(podList))
nodeStatusMap := map[string]solrv1beta1.SolrNodeStatus{}
@@ -798,8 +813,9 @@
extAddress := solrCloud.UrlScheme(true) + "://" + solrCloud.ExternalCommonUrl(solrCloud.Spec.SolrAddressability.External.DomainName, true)
newStatus.ExternalCommonAddress = &extAddress
}
+ shouldRequeue = newStatus.ReadyReplicas != statefulSetStatus.ReadyReplicas || newStatus.Replicas != statefulSetStatus.Replicas || newStatus.UpToDateNodes != statefulSetStatus.UpdatedReplicas
- return outOfDatePods, availableUpdatedPodCount, nil
+ return outOfDatePods, availableUpdatedPodCount, shouldRequeue, nil
}
func (r *SolrCloudReconciler) reconcileNodeService(ctx context.Context, logger logr.Logger, instance *solrv1beta1.SolrCloud, nodeName string) (err error, ip string) {
diff --git a/controllers/solrcloud_controller_basic_auth_test.go b/controllers/solrcloud_controller_basic_auth_test.go
index 7997212..7a4cee8 100644
--- a/controllers/solrcloud_controller_basic_auth_test.go
+++ b/controllers/solrcloud_controller_basic_auth_test.go
@@ -283,7 +283,7 @@
g.Expect(basicAuthSecretVolMount.MountPath).To(Equal("/etc/secrets/"+secretName), "Wrong path used to mount Basic Auth volume")
expProbeCmd := fmt.Sprintf("JAVA_TOOL_OPTIONS=\"-Dbasicauth=$(cat /etc/secrets/%s-solrcloud-basic-auth/username):$(cat /etc/secrets/%s-solrcloud-basic-auth/password) -Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory\" "+
- "solr api -get \"http://${POD_NAME}:8983%s\"",
+ "solr api -get \"http://${SOLR_HOST}:8983%s\"",
solrCloud.Name, solrCloud.Name, expProbePath)
g.Expect(mainContainer.LivenessProbe).To(Not(BeNil()), "main container should have a liveness probe defined")
g.Expect(mainContainer.LivenessProbe.Exec).To(Not(BeNil()), "liveness probe should have an exec when auth is enabled")
diff --git a/controllers/solrcloud_controller_tls_test.go b/controllers/solrcloud_controller_tls_test.go
index a585f1d..22daa96 100644
--- a/controllers/solrcloud_controller_tls_test.go
+++ b/controllers/solrcloud_controller_tls_test.go
@@ -540,7 +540,7 @@
if solrCloud.Spec.CustomSolrKubeOptions.PodOptions != nil && solrCloud.Spec.CustomSolrKubeOptions.PodOptions.LivenessProbe != nil {
path = solrCloud.Spec.CustomSolrKubeOptions.PodOptions.LivenessProbe.HTTPGet.Path
}
- g.Expect(mainContainer.LivenessProbe.Exec.Command[2]).To(ContainSubstring(fmt.Sprintf("solr api -get \"%s://%s:%d%s\"", solrCloud.UrlScheme(false), "${POD_NAME}", solrCloud.Spec.SolrAddressability.PodPort, path)), "liveness probe should invoke solr api -get to contact Solr securely")
+ g.Expect(mainContainer.LivenessProbe.Exec.Command[2]).To(ContainSubstring(fmt.Sprintf("solr api -get \"%s://${SOLR_HOST}:%d%s\"", solrCloud.UrlScheme(false), solrCloud.Spec.SolrAddressability.PodPort, path)), "liveness probe should invoke solr api -get to contact Solr securely")
g.Expect(mainContainer.ReadinessProbe).To(Not(BeNil()), "main container should have a readiness probe defined")
g.Expect(mainContainer.ReadinessProbe.Exec).To(Not(BeNil()), "readiness probe should have an exec when mTLS is enabled")
g.Expect(mainContainer.ReadinessProbe.Exec.Command).To(HaveLen(3), "readiness probe command has wrong number of args")
@@ -548,7 +548,7 @@
if solrCloud.Spec.CustomSolrKubeOptions.PodOptions != nil && solrCloud.Spec.CustomSolrKubeOptions.PodOptions.ReadinessProbe != nil {
path = solrCloud.Spec.CustomSolrKubeOptions.PodOptions.ReadinessProbe.HTTPGet.Path
}
- g.Expect(mainContainer.ReadinessProbe.Exec.Command[2]).To(ContainSubstring(fmt.Sprintf("solr api -get \"%s://%s:%d%s\"", solrCloud.UrlScheme(false), "${POD_NAME}", solrCloud.Spec.SolrAddressability.PodPort, path)), "readiness probe should invoke solr api -get to contact Solr securely")
+ g.Expect(mainContainer.ReadinessProbe.Exec.Command[2]).To(ContainSubstring(fmt.Sprintf("solr api -get \"%s://${SOLR_HOST}:%d%s\"", solrCloud.UrlScheme(false), solrCloud.Spec.SolrAddressability.PodPort, path)), "readiness probe should invoke solr api -get to contact Solr securely")
if solrCloud.Spec.CustomSolrKubeOptions.PodOptions != nil && solrCloud.Spec.CustomSolrKubeOptions.PodOptions.StartupProbe != nil {
g.Expect(mainContainer.StartupProbe).To(Not(BeNil()), "main container should have a startup probe defined")
g.Expect(mainContainer.StartupProbe.Exec).To(Not(BeNil()), "startup probe should have an exec when auth is enabled")
@@ -557,7 +557,7 @@
if solrCloud.Spec.CustomSolrKubeOptions.PodOptions != nil && solrCloud.Spec.CustomSolrKubeOptions.PodOptions.StartupProbe != nil {
path = solrCloud.Spec.CustomSolrKubeOptions.PodOptions.StartupProbe.HTTPGet.Path
}
- g.Expect(mainContainer.StartupProbe.Exec.Command[2]).To(ContainSubstring(fmt.Sprintf("solr api -get \"%s://%s:%d%s\"", solrCloud.UrlScheme(false), "${POD_NAME}", solrCloud.Spec.SolrAddressability.PodPort, path)), "startup probe should invoke solr api -get to contact Solr securely")
+ g.Expect(mainContainer.StartupProbe.Exec.Command[2]).To(ContainSubstring(fmt.Sprintf("solr api -get \"%s://${SOLR_HOST}:%d%s\"", solrCloud.UrlScheme(false), solrCloud.Spec.SolrAddressability.PodPort, path)), "startup probe should invoke solr api -get to contact Solr securely")
}
} else if solrCloud != nil {
g.Expect(mainContainer.LivenessProbe).To(Not(BeNil()), "main container should have a liveness probe defined")
diff --git a/controllers/util/solr_security_util.go b/controllers/util/solr_security_util.go
index 3f94de6..8e76dc0 100644
--- a/controllers/util/solr_security_util.go
+++ b/controllers/util/solr_security_util.go
@@ -499,7 +499,7 @@
javaToolOptionsStr = ""
}
- probeCommand := fmt.Sprintf("%ssolr api -get \"%s://%s:%d%s\"", javaToolOptionsStr, solrCloud.UrlScheme(false), "${POD_NAME}", probe.HTTPGet.Port.IntVal, probe.HTTPGet.Path)
+ probeCommand := fmt.Sprintf("%ssolr api -get \"%s://${SOLR_HOST}:%d%s\"", javaToolOptionsStr, solrCloud.UrlScheme(false), probe.HTTPGet.Port.IntVal, probe.HTTPGet.Path)
probeCommand = regexp.MustCompile(`\s+`).ReplaceAllString(strings.TrimSpace(probeCommand), " ")
// use an Exec instead of an HTTP GET
diff --git a/controllers/util/solr_util.go b/controllers/util/solr_util.go
index 5c5b614..b6867dd 100644
--- a/controllers/util/solr_util.go
+++ b/controllers/util/solr_util.go
@@ -328,6 +328,15 @@
},
},
{
+ Name: "POD_IP",
+ ValueFrom: &corev1.EnvVarSource{
+ FieldRef: &corev1.ObjectFieldSelector{
+ FieldPath: "status.podIP",
+ APIVersion: "v1",
+ },
+ },
+ },
+ {
Name: "POD_NAMESPACE",
ValueFrom: &corev1.EnvVarSource{
FieldRef: &corev1.ObjectFieldSelector{
diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go
index 7ba3e94..f27a90f 100644
--- a/tests/e2e/suite_test.go
+++ b/tests/e2e/suite_test.go
@@ -32,6 +32,7 @@
"golang.org/x/text/cases"
"golang.org/x/text/language"
"io"
+ appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
@@ -291,7 +292,7 @@
}
foundPods := &corev1.PodList{}
- Expect(k8sClient.List(ctx, foundPods, listOps)).To(Succeed(), "Could not fetch Solr Operator pod")
+ Expect(k8sClient.List(ctx, foundPods, listOps)).To(Succeed(), "Could not fetch Solr pods")
Expect(foundPods).ToNot(BeNil(), "No Solr pods could be found")
for _, pod := range foundPods.Items {
writeAllPodInfoToFiles(
@@ -300,6 +301,42 @@
&pod,
)
}
+
+ foundStatefulSets := &appsv1.StatefulSetList{}
+ Expect(k8sClient.List(ctx, foundStatefulSets, listOps)).To(Succeed(), "Could not fetch Solr statefulSets")
+ Expect(foundStatefulSets).ToNot(BeNil(), "No Solr statefulSet could be found")
+ for _, statefulSet := range foundStatefulSets.Items {
+ writeAllStatefulSetInfoToFiles(
+ directory+statefulSet.Name+".statefulSet",
+ &statefulSet,
+ )
+ }
+}
+
+// writeAllStatefulSetInfoToFiles writes the following each to a separate file with the given base name & directory.
+// - StatefulSet Spec/Status
+// - StatefulSet Events
+func writeAllStatefulSetInfoToFiles(baseFilename string, statefulSet *appsv1.StatefulSet) {
+ // Write statefulSet to a file
+ statusFile, err := os.Create(baseFilename + ".status.json")
+ defer statusFile.Close()
+ Expect(err).ToNot(HaveOccurred(), "Could not open file to save statefulSet status: %s", baseFilename+".status.json")
+ jsonBytes, marshErr := json.MarshalIndent(statefulSet, "", "\t")
+ Expect(marshErr).ToNot(HaveOccurred(), "Could not serialize statefulSet json")
+ _, writeErr := statusFile.Write(jsonBytes)
+ Expect(writeErr).ToNot(HaveOccurred(), "Could not write statefulSet json to file")
+
+ // Write events for statefulSet to a file
+ eventsFile, err := os.Create(baseFilename + ".events.json")
+ defer eventsFile.Close()
+ Expect(err).ToNot(HaveOccurred(), "Could not open file to save statefulSet events: %s", baseFilename+".events.yaml")
+
+ eventList, err := rawK8sClient.CoreV1().Events(statefulSet.Namespace).Search(scheme.Scheme, statefulSet)
+ Expect(err).ToNot(HaveOccurred(), "Could not find events for statefulSet: %s", statefulSet.Name)
+ jsonBytes, marshErr = json.MarshalIndent(eventList, "", "\t")
+ Expect(marshErr).ToNot(HaveOccurred(), "Could not serialize statefulSet events json")
+ _, writeErr = eventsFile.Write(jsonBytes)
+ Expect(writeErr).ToNot(HaveOccurred(), "Could not write statefulSet events json to file")
}
// writeAllPodInfoToFile writes the following each to a separate file with the given base name & directory.
@@ -319,24 +356,26 @@
// Write events for pod to a file
eventsFile, err := os.Create(baseFilename + ".events.json")
defer eventsFile.Close()
- Expect(err).ToNot(HaveOccurred(), "Could not open file to save status: %s", baseFilename+".events.yaml")
+ Expect(err).ToNot(HaveOccurred(), "Could not open file to save pod events: %s", baseFilename+".events.yaml")
eventList, err := rawK8sClient.CoreV1().Events(pod.Namespace).Search(scheme.Scheme, pod)
Expect(err).ToNot(HaveOccurred(), "Could not find events for pod: %s", pod.Name)
jsonBytes, marshErr = json.MarshalIndent(eventList, "", "\t")
- Expect(marshErr).ToNot(HaveOccurred(), "Could not serialize events json")
+ Expect(marshErr).ToNot(HaveOccurred(), "Could not serialize pod events json")
_, writeErr = eventsFile.Write(jsonBytes)
- Expect(writeErr).ToNot(HaveOccurred(), "Could not write events json to file")
+ Expect(writeErr).ToNot(HaveOccurred(), "Could not write pod events json to file")
// Write pod logs to a file
- writePodLogsToFile(
- ctx,
- baseFilename+".log",
- pod.Name,
- pod.Namespace,
- nil,
- "",
- )
+ if len(pod.Status.ContainerStatuses) > 0 && pod.Status.ContainerStatuses[0].Started != nil && *pod.Status.ContainerStatuses[0].Started {
+ writePodLogsToFile(
+ ctx,
+ baseFilename+".log",
+ pod.Name,
+ pod.Namespace,
+ nil,
+ "",
+ )
+ }
}
func writePodLogsToFile(ctx context.Context, filename string, podName string, podNamespace string, startTimeRaw *time.Time, filterLinesWithString string) {
