Move rolling update logic under clusterOp (#586)
* Fix integration test, and ensure locking during update test
diff --git a/controllers/solr_cluster_ops_util.go b/controllers/solr_cluster_ops_util.go
index 5d8ec3a..c642dc3 100644
--- a/controllers/solr_cluster_ops_util.go
+++ b/controllers/solr_cluster_ops_util.go
@@ -156,6 +156,79 @@
return
}
+func determineRollingUpdateClusterOpLockIfNecessary(ctx context.Context, r *SolrCloudReconciler, instance *solrv1beta1.SolrCloud, statefulSet *appsv1.StatefulSet, outOfDatePods util.OutOfDatePodSegmentation, logger logr.Logger) (clusterLockAcquired bool, retryLaterDuration time.Duration, err error) {
+ if instance.Spec.UpdateStrategy.Method == solrv1beta1.ManagedUpdate && !outOfDatePods.IsEmpty() {
+ // Managed Rolling Upgrade!
+ originalStatefulSet := statefulSet.DeepCopy()
+ statefulSet.Annotations[util.ClusterOpsLockAnnotation] = util.UpdateLock
+ // No rolling update metadata is currently required
+ statefulSet.Annotations[util.ClusterOpsMetadataAnnotation] = ""
+ if err = r.Patch(ctx, statefulSet, client.StrategicMergeFrom(originalStatefulSet)); err != nil {
+ logger.Error(err, "Error while patching StatefulSet to start clusterOp", "clusterOp", util.UpdateLock, "clusterOpMetadata", "")
+ } else {
+ clusterLockAcquired = true
+ }
+ }
+ return
+}
+
+// handleManagedCloudRollingUpdate does the logic of a managed and "locked" cloud rolling update operation.
+// This will take many reconcile loops to complete, as it is deleting pods/moving replicas.
+func handleManagedCloudRollingUpdate(ctx context.Context, r *SolrCloudReconciler, instance *solrv1beta1.SolrCloud, statefulSet *appsv1.StatefulSet, outOfDatePods util.OutOfDatePodSegmentation, hasReadyPod bool, availableUpdatedPodCount int, logger logr.Logger) (retryLaterDuration time.Duration, err error) {
+ // Manage the updating of out-of-spec pods, if the Managed UpdateStrategy has been specified.
+ updateLogger := logger.WithName("ManagedUpdateSelector")
+
+ // First check if all pods are up to date. If so the rolling update is complete
+ if outOfDatePods.IsEmpty() {
+ // Once the rolling update is complete, finish the cluster operation by deleting the statefulSet annotations
+ originalStatefulSet := statefulSet.DeepCopy()
+ delete(statefulSet.Annotations, util.ClusterOpsLockAnnotation)
+ delete(statefulSet.Annotations, util.ClusterOpsMetadataAnnotation)
+ if err = r.Patch(ctx, statefulSet, client.StrategicMergeFrom(originalStatefulSet)); err != nil {
+ logger.Error(err, "Error while patching StatefulSet to finish the managed SolrCloud rollingUpdate clusterOp")
+ }
+
+ // TODO: Create event for the CRD.
+ } else {
+ // The out of date pods that have not been started, should all be updated immediately.
+ // There is no use "safely" updating pods which have not been started yet.
+ podsToUpdate := append([]corev1.Pod{}, outOfDatePods.NotStarted...)
+ for _, pod := range outOfDatePods.NotStarted {
+ updateLogger.Info("Pod killed for update.", "pod", pod.Name, "reason", "The solr container in the pod has not yet started, thus it is safe to update.")
+ }
+
+ // Pick which pods should be deleted for an update.
+ // Don't exit on an error, which would only occur because of an HTTP Exception. Requeue later instead.
+ additionalPodsToUpdate, podsHaveReplicas, retryLater, clusterStateError :=
+ util.DeterminePodsSafeToUpdate(ctx, instance, int(*statefulSet.Spec.Replicas), outOfDatePods, hasReadyPod, availableUpdatedPodCount, updateLogger)
+ // If we do not have the clusterState, it's not safe to update pods that are running
+ if clusterStateError != nil {
+ retryLater = true
+ } else {
+ podsToUpdate = append(podsToUpdate, outOfDatePods.ScheduledForDeletion...)
+ podsToUpdate = append(podsToUpdate, additionalPodsToUpdate...)
+ }
+
+ // Only actually delete a running pod if it has been evicted, or doesn't need eviction (persistent storage)
+ for _, pod := range podsToUpdate {
+ retryLaterDurationTemp, errTemp := DeletePodForUpdate(ctx, r, instance, &pod, podsHaveReplicas[pod.Name], updateLogger)
+
+ // Use the retryLaterDuration of the pod that requires a retry the soonest (smallest duration > 0)
+ if retryLaterDurationTemp > 0 && (retryLaterDurationTemp < retryLaterDuration || retryLaterDuration == 0) {
+ retryLaterDuration = retryLaterDurationTemp
+ }
+ if errTemp != nil {
+ err = errTemp
+ }
+ }
+
+ if retryLater && retryLaterDuration == 0 {
+ retryLaterDuration = time.Second * 10
+ }
+ }
+ return
+}
+
// clearClusterOp simply removes any clusterOp for the given statefulSet.
// This should only be used as a "break-glass" scenario. Do not use this to finish off successful clusterOps.
func clearClusterOp(ctx context.Context, r *SolrCloudReconciler, statefulSet *appsv1.StatefulSet, reason string, logger logr.Logger) (err error) {
diff --git a/controllers/solrcloud_controller.go b/controllers/solrcloud_controller.go
index a4f91ec..56fdbd7 100644
--- a/controllers/solrcloud_controller.go
+++ b/controllers/solrcloud_controller.go
@@ -285,6 +285,13 @@
var security *util.SecurityConfig = nil
if instance.Spec.SolrSecurity != nil {
security, err = util.ReconcileSecurityConfig(ctx, &r.Client, instance)
+ if err == nil && security != nil {
+ // If authn enabled on Solr, we need to pass the auth header when making requests
+ ctx, err = security.AddAuthToContext(ctx)
+ if err != nil {
+ logger.Error(err, "failed to create Authorization header when reconciling")
+ }
+ }
if err != nil {
return requeueOrNot, err
}
@@ -443,12 +450,14 @@
}
// We only want to do one cluster operation at a time, so we use a lock to ensure that.
- // Update or a Scale at a time. We do not want to do both.
-
+ // Update or Scale, one-at-a-time. We do not want to do both.
+ hasReadyPod := newStatus.ReadyReplicas > 0
var retryLaterDuration time.Duration
if clusterOpLock, hasAnn := statefulSet.Annotations[util.ClusterOpsLockAnnotation]; hasAnn {
clusterOpMetadata := statefulSet.Annotations[util.ClusterOpsMetadataAnnotation]
switch clusterOpLock {
+ case util.UpdateLock:
+ retryLaterDuration, err = handleManagedCloudRollingUpdate(ctx, r, instance, statefulSet, outOfDatePods, hasReadyPod, availableUpdatedPodCount, logger)
case util.ScaleDownLock:
retryLaterDuration, err = handleManagedCloudScaleDown(ctx, r, instance, statefulSet, clusterOpMetadata, podList, logger)
case util.ScaleUpLock:
@@ -459,12 +468,22 @@
err = clearClusterOp(ctx, r, statefulSet, "clusterOp not supported", logger)
}
} else {
+ lockAcquired := false
// Start cluster operations if needed.
// The operations will be actually run in future reconcile loops, but a clusterOpLock will be acquired here.
// And that lock will tell future reconcile loops that the operation needs to be done.
// If a non-managed scale needs to take place, this method will update the StatefulSet without starting
// a "locked" cluster operation
- _, retryLaterDuration, err = determineScaleClusterOpLockIfNecessary(ctx, r, instance, statefulSet, podList, logger)
+ lockAcquired, retryLaterDuration, err = determineRollingUpdateClusterOpLockIfNecessary(ctx, r, instance, statefulSet, outOfDatePods, logger)
+ // Start cluster operations if needed.
+ // The operations will be actually run in future reconcile loops, but a clusterOpLock will be acquired here.
+ // And that lock will tell future reconcile loops that the operation needs to be done.
+ // If a non-managed scale needs to take place, this method will update the StatefulSet without starting
+ // a "locked" cluster operation
+ if !lockAcquired {
+ lockAcquired, retryLaterDuration, err = determineScaleClusterOpLockIfNecessary(ctx, r, instance, statefulSet, podList, logger)
+ }
+ // After a lock is acquired, the reconcile will be started again because the StatefulSet is being watched
}
if err != nil && retryLaterDuration == 0 {
retryLaterDuration = time.Second * 5
@@ -476,62 +495,6 @@
return requeueOrNot, err
}
- // TODO: Move this logic into the ClusterOpLock, with the "rollingUpdate" lock
- if instance.Spec.UpdateStrategy.Method == solrv1beta1.ManagedUpdate && len(outOfDatePods.NotStarted)+len(outOfDatePods.ScheduledForDeletion)+len(outOfDatePods.Running) > 0 {
- // Manage the updating of out-of-spec pods, if the Managed UpdateStrategy has been specified.
- updateLogger := logger.WithName("ManagedUpdateSelector")
-
- // The out of date pods that have not been started, should all be updated immediately.
- // There is no use "safely" updating pods which have not been started yet.
- podsToUpdate := append([]corev1.Pod{}, outOfDatePods.NotStarted...)
- for _, pod := range outOfDatePods.NotStarted {
- updateLogger.Info("Pod killed for update.", "pod", pod.Name, "reason", "The solr container in the pod has not yet started, thus it is safe to update.")
- }
-
- // If authn enabled on Solr, we need to pass the auth header
- if security != nil {
- ctx, err = security.AddAuthToContext(ctx)
- if err != nil {
- updateLogger.Error(err, "failed to create Authorization header when reconciling", "SolrCloud", instance.Name)
- return requeueOrNot, err
- }
- }
-
- // Pick which pods should be deleted for an update.
- // Don't exit on an error, which would only occur because of an HTTP Exception. Requeue later instead.
- additionalPodsToUpdate, podsHaveReplicas, retryLater, clusterStateError := util.DeterminePodsSafeToUpdate(ctx, instance, outOfDatePods, int(newStatus.ReadyReplicas), availableUpdatedPodCount, updateLogger)
- // If we do not have the clusterState, it's not safe to update pods that are running
- if clusterStateError != nil {
- retryLater = true
- } else {
- podsToUpdate = append(podsToUpdate, outOfDatePods.ScheduledForDeletion...)
- podsToUpdate = append(podsToUpdate, additionalPodsToUpdate...)
- }
-
- // Only actually delete a running pod if it has been evicted, or doesn't need eviction (persistent storage)
- for _, pod := range podsToUpdate {
- retryLaterDurationTemp, errTemp := DeletePodForUpdate(ctx, r, instance, &pod, podsHaveReplicas[pod.Name], updateLogger)
-
- // Use the retryLaterDuration of the pod that requires a retry the soonest (smallest duration > 0)
- if retryLaterDurationTemp > 0 && (retryLaterDurationTemp < retryLaterDuration || retryLaterDuration == 0) {
- retryLaterDuration = retryLaterDurationTemp
- }
- if errTemp != nil {
- err = errTemp
- }
- }
-
- if err != nil || retryLaterDuration > 0 || retryLater {
- if retryLaterDuration == 0 {
- retryLaterDuration = time.Second * 10
- }
- updateRequeueAfter(&requeueOrNot, retryLaterDuration)
- }
- if err != nil {
- return requeueOrNot, err
- }
- }
-
// Upsert or delete solrcloud-wide PodDisruptionBudget(s) based on 'Enabled' flag.
pdb := util.GeneratePodDisruptionBudget(instance, statefulSet.Spec.Selector.MatchLabels)
if instance.Spec.Availability.PodDisruptionBudget.Enabled != nil && *instance.Spec.Availability.PodDisruptionBudget.Enabled {
diff --git a/controllers/util/solr_update_util.go b/controllers/util/solr_update_util.go
index a5235f5..e0ca2f0 100644
--- a/controllers/util/solr_update_util.go
+++ b/controllers/util/solr_update_util.go
@@ -92,6 +92,10 @@
Running []corev1.Pod
}
+func (seg OutOfDatePodSegmentation) IsEmpty() bool {
+ return len(seg.NotStarted)+len(seg.ScheduledForDeletion)+len(seg.Running) == 0
+}
+
// DeterminePodsSafeToUpdate takes a list of solr Pods and returns a list of pods that are safe to upgrade now.
// This function MUST be idempotent and return the same list of pods given the same kubernetes/solr state.
//
@@ -101,9 +105,9 @@
// TODO:
// - Think about caching this for ~250 ms? Not a huge need to send these requests milliseconds apart.
// - Might be too much complexity for very little gain.
-func DeterminePodsSafeToUpdate(ctx context.Context, cloud *solr.SolrCloud, outOfDatePods OutOfDatePodSegmentation, readyPods int, availableUpdatedPodCount int, logger logr.Logger) (podsToUpdate []corev1.Pod, podsHaveReplicas map[string]bool, retryLater bool, err error) {
+func DeterminePodsSafeToUpdate(ctx context.Context, cloud *solr.SolrCloud, totalPods int, outOfDatePods OutOfDatePodSegmentation, hasReadyPod bool, availableUpdatedPodCount int, logger logr.Logger) (podsToUpdate []corev1.Pod, podsHaveReplicas map[string]bool, retryLater bool, err error) {
// Before fetching the cluster state, be sure that there is room to update at least 1 pod
- maxPodsUnavailable, unavailableUpdatedPodCount, maxPodsToUpdate := calculateMaxPodsToUpdate(cloud, len(outOfDatePods.Running), len(outOfDatePods.NotStarted)+len(outOfDatePods.ScheduledForDeletion), availableUpdatedPodCount)
+ maxPodsUnavailable, unavailableUpdatedPodCount, maxPodsToUpdate := calculateMaxPodsToUpdate(cloud, totalPods, len(outOfDatePods.Running), len(outOfDatePods.NotStarted)+len(outOfDatePods.ScheduledForDeletion), availableUpdatedPodCount)
if maxPodsToUpdate <= 0 {
logger.Info("Pod update selection canceled. The number of updated pods unavailable equals or exceeds the calculated maxPodsUnavailable.",
"unavailableUpdatedPods", unavailableUpdatedPodCount, "outOfDatePodsNotStarted", len(outOfDatePods.NotStarted), "alreadyScheduledForDeletion", len(outOfDatePods.ScheduledForDeletion), "maxPodsUnavailable", maxPodsUnavailable)
@@ -111,7 +115,7 @@
clusterResp := &solr_api.SolrClusterStatusResponse{}
overseerResp := &solr_api.SolrOverseerStatusResponse{}
- if readyPods > 0 {
+ if hasReadyPod {
queryParams := url.Values{}
queryParams.Add("action", "CLUSTERSTATUS")
err = solr_api.CallCollectionsApi(ctx, cloud, queryParams, clusterResp)
@@ -148,8 +152,7 @@
}
// calculateMaxPodsToUpdate determines the maximum number of additional pods that can be updated.
-func calculateMaxPodsToUpdate(cloud *solr.SolrCloud, outOfDatePodCount int, outOfDatePodsNotStartedCount int, availableUpdatedPodCount int) (maxPodsUnavailable int, unavailableUpdatedPodCount int, maxPodsToUpdate int) {
- totalPods := int(*cloud.Spec.Replicas)
+func calculateMaxPodsToUpdate(cloud *solr.SolrCloud, totalPods int, outOfDatePodCount int, outOfDatePodsNotStartedCount int, availableUpdatedPodCount int) (maxPodsUnavailable int, unavailableUpdatedPodCount int, maxPodsToUpdate int) {
// In order to calculate the number of updated pods that are unavailable take all pods, take the total pods and subtract those that are available and updated, and those that are not updated.
unavailableUpdatedPodCount = totalPods - availableUpdatedPodCount - outOfDatePodCount - outOfDatePodsNotStartedCount
// If the maxBatchNodeUpgradeSpec is passed as a decimal between 0 and 1, then calculate as a percentage of the number of nodes.
diff --git a/controllers/util/solr_update_util_test.go b/controllers/util/solr_update_util_test.go
index 1ff982d..3729dbf 100644
--- a/controllers/util/solr_update_util_test.go
+++ b/controllers/util/solr_update_util_test.go
@@ -524,38 +524,38 @@
},
}
- foundMaxPodsUnavailable, foundUnavailableUpdatedPodCount, foundMaxPodsToUpdate := calculateMaxPodsToUpdate(solrCloud, 4, 0, 4)
+ foundMaxPodsUnavailable, foundUnavailableUpdatedPodCount, foundMaxPodsToUpdate := calculateMaxPodsToUpdate(solrCloud, 10, 4, 0, 4)
assert.Equal(t, 2, foundMaxPodsUnavailable, "Incorrect value of maxPodsUnavailable given fromInt(2)")
assert.Equal(t, 2, foundUnavailableUpdatedPodCount, "Incorrect value of unavailableUpdatedPodCount")
assert.Equal(t, 0, foundMaxPodsToUpdate, "Incorrect value of maxPodsToUpdate")
- foundMaxPodsUnavailable, foundUnavailableUpdatedPodCount, foundMaxPodsToUpdate = calculateMaxPodsToUpdate(solrCloud, 4, 0, 3)
+ foundMaxPodsUnavailable, foundUnavailableUpdatedPodCount, foundMaxPodsToUpdate = calculateMaxPodsToUpdate(solrCloud, 10, 4, 0, 3)
assert.Equal(t, 2, foundMaxPodsUnavailable, "Incorrect value of maxPodsUnavailable given fromInt(2)")
assert.Equal(t, 3, foundUnavailableUpdatedPodCount, "Incorrect value of unavailableUpdatedPodCount")
assert.Equal(t, -1, foundMaxPodsToUpdate, "Incorrect value of maxPodsToUpdate")
- foundMaxPodsUnavailable, foundUnavailableUpdatedPodCount, foundMaxPodsToUpdate = calculateMaxPodsToUpdate(solrCloud, 3, 1, 3)
+ foundMaxPodsUnavailable, foundUnavailableUpdatedPodCount, foundMaxPodsToUpdate = calculateMaxPodsToUpdate(solrCloud, 10, 3, 1, 3)
assert.Equal(t, 2, foundMaxPodsUnavailable, "Incorrect value of maxPodsUnavailable given fromInt(2)")
assert.Equal(t, 3, foundUnavailableUpdatedPodCount, "Incorrect value of unavailableUpdatedPodCount")
assert.Equal(t, -2, foundMaxPodsToUpdate, "Incorrect value of maxPodsToUpdate")
maxPodsUnavailable = intstr.FromString("45%")
- foundMaxPodsUnavailable, foundUnavailableUpdatedPodCount, foundMaxPodsToUpdate = calculateMaxPodsToUpdate(solrCloud, 3, 0, 5)
+ foundMaxPodsUnavailable, foundUnavailableUpdatedPodCount, foundMaxPodsToUpdate = calculateMaxPodsToUpdate(solrCloud, 10, 3, 0, 5)
assert.Equal(t, 4, foundMaxPodsUnavailable, "Incorrect value of maxPodsUnavailable given fromString(\"45%\")")
assert.Equal(t, 2, foundMaxPodsToUpdate, "Incorrect value of maxPodsToUpdate")
maxPodsUnavailable = intstr.FromString("45%")
- foundMaxPodsUnavailable, foundUnavailableUpdatedPodCount, foundMaxPodsToUpdate = calculateMaxPodsToUpdate(solrCloud, 1, 2, 5)
+ foundMaxPodsUnavailable, foundUnavailableUpdatedPodCount, foundMaxPodsToUpdate = calculateMaxPodsToUpdate(solrCloud, 10, 1, 2, 5)
assert.Equal(t, 4, foundMaxPodsUnavailable, "Incorrect value of maxPodsUnavailable given fromString(\"45%\")")
assert.Equal(t, 0, foundMaxPodsToUpdate, "Incorrect value of maxPodsToUpdate")
maxPodsUnavailable = intstr.FromString("70%")
- foundMaxPodsUnavailable, foundUnavailableUpdatedPodCount, foundMaxPodsToUpdate = calculateMaxPodsToUpdate(solrCloud, 3, 0, 2)
+ foundMaxPodsUnavailable, foundUnavailableUpdatedPodCount, foundMaxPodsToUpdate = calculateMaxPodsToUpdate(solrCloud, 10, 3, 0, 2)
assert.Equal(t, 7, foundMaxPodsUnavailable, "Incorrect value of maxPodsUnavailable given fromString(\"70%\")")
assert.Equal(t, 2, foundMaxPodsToUpdate, "Incorrect value of maxPodsToUpdate")
solrCloud.Spec.UpdateStrategy.ManagedUpdateOptions.MaxPodsUnavailable = nil
- foundMaxPodsUnavailable, foundUnavailableUpdatedPodCount, foundMaxPodsToUpdate = calculateMaxPodsToUpdate(solrCloud, 3, 0, 2)
+ foundMaxPodsUnavailable, foundUnavailableUpdatedPodCount, foundMaxPodsToUpdate = calculateMaxPodsToUpdate(solrCloud, 10, 3, 0, 2)
assert.Equal(t, 2, foundMaxPodsUnavailable, "Incorrect value of maxPodsUnavailable given fromString(\"25%\")")
assert.Equal(t, -3, foundMaxPodsToUpdate, "Incorrect value of maxPodsToUpdate")
}
diff --git a/helm/solr-operator/Chart.yaml b/helm/solr-operator/Chart.yaml
index 3f94074..41c10ad 100644
--- a/helm/solr-operator/Chart.yaml
+++ b/helm/solr-operator/Chart.yaml
@@ -84,6 +84,13 @@
url: https://github.com/apache/solr-operator/issues/570
- name: Github PR
url: https://github.com/apache/solr-operator/pull/578
+ - kind: changed
+ description: Managed Rolling Updates are now computed via a Cluster Lock, like scaling operations.
+ links:
+ - name: Github Issue
+ url: https://github.com/apache/solr-operator/issues/560
+ - name: Github PR
+ url: https://github.com/apache/solr-operator/pull/586
artifacthub.io/images: |
- name: solr-operator
image: apache/solr-operator:v0.8.0-prerelease
diff --git a/tests/e2e/solrcloud_rolling_upgrade_test.go b/tests/e2e/solrcloud_rolling_upgrade_test.go
index 567e359..c3fe789 100644
--- a/tests/e2e/solrcloud_rolling_upgrade_test.go
+++ b/tests/e2e/solrcloud_rolling_upgrade_test.go
@@ -20,6 +20,7 @@
import (
"context"
solrv1beta1 "github.com/apache/solr-operator/api/v1beta1"
+ "github.com/apache/solr-operator/controllers/util"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/util/intstr"
@@ -81,12 +82,14 @@
Expect(k8sClient.Patch(ctx, patchedSolrCloud, client.MergeFrom(solrCloud))).To(Succeed(), "Could not add annotation to SolrCloud pod to initiate rolling restart")
By("waiting for the rolling restart to begin")
- expectSolrCloudWithChecks(ctx, solrCloud, func(g Gomega, cloud *solrv1beta1.SolrCloud) {
+ solrCloud = expectSolrCloudWithChecks(ctx, solrCloud, func(g Gomega, cloud *solrv1beta1.SolrCloud) {
g.Expect(cloud.Status.UpToDateNodes).To(BeZero(), "Cloud did not get to a state with zero up-to-date replicas when rolling restart began.")
for _, nodeStatus := range cloud.Status.SolrNodes {
g.Expect(nodeStatus.SpecUpToDate).To(BeFalse(), "Node not starting as out-of-date when rolling restart begins: %s", nodeStatus.Name)
}
})
+ statefulSet := expectStatefulSet(ctx, solrCloud, solrCloud.StatefulSetName())
+ Expect(statefulSet.Annotations).To(HaveKeyWithValue(util.ClusterOpsLockAnnotation, util.UpdateLock), "StatefulSet does not have a RollingUpdate lock after starting managed update.")
By("waiting for the rolling restart to complete")
// Expect the SolrCloud to be up-to-date, or in a valid restarting state
@@ -140,6 +143,9 @@
Expect(nodeStatus.Ready).To(BeTrue(), "Node not finishing as ready when rolling restart ends: %s", nodeStatus.Name)
}
+ statefulSet = expectStatefulSet(ctx, solrCloud, solrCloud.StatefulSetName())
+ Expect(statefulSet.Annotations).To(Not(HaveKey(util.ClusterOpsLockAnnotation)), "StatefulSet should not have a RollingUpdate lock after finishing a managed update.")
+
By("checking that the collections can be queried after the restart")
queryCollection(ctx, solrCloud, solrCollection1, 0)
queryCollection(ctx, solrCloud, solrCollection2, 0)
diff --git a/tests/e2e/test_utils_test.go b/tests/e2e/test_utils_test.go
index 055254b..2568ed8 100644
--- a/tests/e2e/test_utils_test.go
+++ b/tests/e2e/test_utils_test.go
@@ -308,14 +308,30 @@
fmt.Sprintf("http://localhost:%d/solr/%s/select?rows=0", solrCloud.Spec.SolrAddressability.PodPort, collection),
},
)
- g.ExpectWithOffset(resolveOffset(additionalOffset), err).ToNot(HaveOccurred(), "Error occurred while querying empty Solr Collection")
- g.ExpectWithOffset(resolveOffset(additionalOffset), response).To(ContainSubstring("\"numFound\":%d", docCount), "Error occurred while querying Solr Collection '%s'", collection)
- }, time.Second*5).WithContext(ctx).Should(Succeed(), "Could not successfully query collection")
+ innerG.Expect(err).ToNot(HaveOccurred(), "Error occurred while querying empty Solr Collection")
+ innerG.Expect(response).To(ContainSubstring("\"numFound\":%d", docCount), "Error occurred while querying Solr Collection '%s'", collection)
+ }, time.Second*5).WithContext(ctx).Should(Succeed(), "Could not successfully query collection: %v", fetchClusterStatus(ctx, solrCloud))
// Only wait 5 seconds for the collection to be query-able
}
+func fetchClusterStatus(ctx context.Context, solrCloud *solrv1beta1.SolrCloud) string {
+ response, err := runExecForContainer(
+ ctx,
+ util.SolrNodeContainer,
+ solrCloud.GetRandomSolrPodName(),
+ solrCloud.Namespace,
+ []string{
+ "curl",
+ fmt.Sprintf("http://localhost:%d/solr/admin/collections?action=CLUSTERSTATUS", solrCloud.Spec.SolrAddressability.PodPort),
+ },
+ )
+ Expect(err).ToNot(HaveOccurred(), "Could not fetch clusterStatus for cloud")
+
+ return response
+}
+
func queryCollectionWithNoReplicaAvailable(ctx context.Context, solrCloud *solrv1beta1.SolrCloud, collection string, additionalOffset ...int) {
- EventuallyWithOffset(resolveOffset(additionalOffset), func(g Gomega) {
+ EventuallyWithOffset(resolveOffset(additionalOffset), func(innerG Gomega) {
response, err := runExecForContainer(
ctx,
util.SolrNodeContainer,
@@ -326,8 +342,8 @@
fmt.Sprintf("http://localhost:%d/solr/%s/select", solrCloud.Spec.SolrAddressability.PodPort, collection),
},
)
- g.Expect(err).ToNot(HaveOccurred(), "Error occurred while querying empty Solr Collection")
- g.Expect(response).To(ContainSubstring("Error trying to proxy request for url"), "Wrong occurred while querying Solr Collection '%s', expected a proxy forwarding error", collection)
+ innerG.Expect(err).ToNot(HaveOccurred(), "Error occurred while querying empty Solr Collection")
+ innerG.Expect(response).To(ContainSubstring("Error trying to proxy request for url"), "Wrong occurred while querying Solr Collection '%s', expected a proxy forwarding error", collection)
}, time.Second*5).WithContext(ctx).Should(Succeed(), "Collection query did not fail in the correct way")
}
diff --git a/tests/scripts/manage_e2e_tests.sh b/tests/scripts/manage_e2e_tests.sh
index f85a3e1..98f23c4 100755
--- a/tests/scripts/manage_e2e_tests.sh
+++ b/tests/scripts/manage_e2e_tests.sh
@@ -104,7 +104,7 @@
kind load docker-image --name "${CLUSTER_NAME}" "${IMAGE}"
else
if [ "${PULL_IF_NOT_LOCAL}" = true ]; then
- printf "\nPulling image \"%s\" since it was not found locally.\n\n" "${IMAGE}" "${IMAGE}"
+ printf "\nPulling image \"%s\" since it was not found locally.\n\n" "${IMAGE}"
docker pull "${IMAGE}"
kind load docker-image --name "${CLUSTER_NAME}" "${IMAGE}"
else
