Ability to customize probes for the PrometheusExporter (#297)
diff --git a/controllers/controller_utils_test.go b/controllers/controller_utils_test.go
index ba48ec2..dace14f 100644
--- a/controllers/controller_utils_test.go
+++ b/controllers/controller_utils_test.go
@@ -23,7 +23,6 @@
"github.com/apache/solr-operator/controllers/util"
zkv1beta1 "github.com/pravega/zookeeper-operator/pkg/apis/zookeeper/v1beta1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- "reflect"
"strings"
"testing"
@@ -392,11 +391,11 @@
}
func testPodTolerations(t *testing.T, expectedTolerations []corev1.Toleration, foundTolerations []corev1.Toleration) {
- assert.True(t, reflect.DeepEqual(expectedTolerations, foundTolerations), "Expected tolerations and found tolerations don't match")
+ assert.EqualValues(t, expectedTolerations, foundTolerations, "Expected tolerations and found tolerations don't match")
}
-func testPodProbe(t *testing.T, expectedProbe *corev1.Probe, foundProbe *corev1.Probe) {
- assert.True(t, reflect.DeepEqual(expectedProbe, foundProbe), "Expected probe and found probe don't match")
+func testPodProbe(t *testing.T, expectedProbe *corev1.Probe, foundProbe *corev1.Probe, probeType string) {
+ assert.EqualValuesf(t, expectedProbe, foundProbe, "Incorrect default container %s probe", probeType)
}
func testMapsEqual(t *testing.T, mapName string, expected map[string]string, found map[string]string) {
diff --git a/controllers/solrcloud_controller_test.go b/controllers/solrcloud_controller_test.go
index 1e9a3e6..5a9ecdb 100644
--- a/controllers/solrcloud_controller_test.go
+++ b/controllers/solrcloud_controller_test.go
@@ -294,8 +294,9 @@
}
testMapsEqual(t, "pod annotations", util.MergeLabelsOrAnnotations(map[string]string{"solr.apache.org/solrXmlMd5": fmt.Sprintf("%x", md5.Sum([]byte(configMap.Data["solr.xml"])))}, testPodAnnotations), statefulSet.Spec.Template.Annotations)
testMapsEqual(t, "pod node selectors", testNodeSelectors, statefulSet.Spec.Template.Spec.NodeSelector)
- testPodProbe(t, testProbeLivenessNonDefaults, statefulSet.Spec.Template.Spec.Containers[0].LivenessProbe)
- testPodProbe(t, testProbeReadinessNonDefaults, statefulSet.Spec.Template.Spec.Containers[0].ReadinessProbe)
+ testPodProbe(t, testProbeLivenessNonDefaults, statefulSet.Spec.Template.Spec.Containers[0].LivenessProbe, "liveness")
+ testPodProbe(t, testProbeReadinessNonDefaults, statefulSet.Spec.Template.Spec.Containers[0].ReadinessProbe, "readiness")
+ testPodProbe(t, testProbeStartup, statefulSet.Spec.Template.Spec.Containers[0].StartupProbe, "startup")
assert.Equal(t, []string{"solr", "stop", "-p", "8983"}, statefulSet.Spec.Template.Spec.Containers[0].Lifecycle.PreStop.Exec.Command, "Incorrect pre-stop command")
testPodTolerations(t, testTolerations, statefulSet.Spec.Template.Spec.Tolerations)
assert.EqualValues(t, testPriorityClass, statefulSet.Spec.Template.Spec.PriorityClassName, "Incorrect Priority class name for Pod Spec")
diff --git a/controllers/solrprometheusexporter_controller_test.go b/controllers/solrprometheusexporter_controller_test.go
index 271f3c2..be8f96e 100644
--- a/controllers/solrprometheusexporter_controller_test.go
+++ b/controllers/solrprometheusexporter_controller_test.go
@@ -29,6 +29,7 @@
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
+ "k8s.io/apimachinery/pkg/util/intstr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/manager"
@@ -65,6 +66,8 @@
InitContainers: extraContainers1,
ImagePullSecrets: testAdditionalImagePullSecrets,
TerminationGracePeriodSeconds: &testTerminationGracePeriodSeconds,
+ LivenessProbe: testProbeLivenessNonDefaults,
+ ReadinessProbe: testProbeReadinessNonDefaults,
},
},
ExporterEntrypoint: "/test/entry-point",
@@ -132,6 +135,11 @@
assert.ElementsMatch(t, append(testAdditionalImagePullSecrets, corev1.LocalObjectReference{Name: testImagePullSecretName}), deployment.Spec.Template.Spec.ImagePullSecrets, "Incorrect imagePullSecrets")
assert.EqualValues(t, &testTerminationGracePeriodSeconds, deployment.Spec.Template.Spec.TerminationGracePeriodSeconds, "Incorrect terminationGracePeriodSeconds")
+ testPodProbe(t, testProbeLivenessNonDefaults, deployment.Spec.Template.Spec.Containers[0].LivenessProbe, "liveness")
+ testPodProbe(t, testProbeReadinessNonDefaults, deployment.Spec.Template.Spec.Containers[0].ReadinessProbe, "readiness")
+ assert.Nilf(t, deployment.Spec.Template.Spec.Containers[0].StartupProbe, "%s probe should be nil since it was not specified", "startup")
+
+ // Check the Service
service := expectService(t, g, requests, expectedMetricsRequest, metricsSKey, deployment.Spec.Template.Labels)
assert.Equal(t, "true", service.Annotations["prometheus.io/scrape"], "Metrics Service Prometheus scraping is not enabled.")
assert.EqualValues(t, "solr-metrics", service.Spec.Ports[0].Name, "Wrong port name on common Service")
@@ -151,6 +159,7 @@
Tolerations: testTolerationsPromExporter,
NodeSelector: testNodeSelectors,
PriorityClassName: testPriorityClass,
+ StartupProbe: testProbeStartup,
},
DeploymentOptions: &solr.DeploymentOptions{
Annotations: testDeploymentAnnotations,
@@ -228,6 +237,24 @@
assert.Equal(t, extraVolumes[0].Name, deployment.Spec.Template.Spec.Volumes[1].Name, "Additional Volume from podOptions not loaded into pod properly.")
assert.Equal(t, extraVolumes[0].Source, deployment.Spec.Template.Spec.Volumes[1].VolumeSource, "Additional Volume from podOptions not loaded into pod properly.")
+ testPodProbe(t, &corev1.Probe{
+ Handler: corev1.Handler{
+ HTTPGet: &corev1.HTTPGetAction{
+ Scheme: corev1.URISchemeHTTP,
+ Path: "/metrics",
+ Port: intstr.FromInt(util.SolrMetricsPort),
+ },
+ },
+ InitialDelaySeconds: 20,
+ TimeoutSeconds: 1,
+ PeriodSeconds: 10,
+ SuccessThreshold: 1,
+ FailureThreshold: 3,
+ }, deployment.Spec.Template.Spec.Containers[0].LivenessProbe, "liveness")
+ testPodProbe(t, testProbeStartup, deployment.Spec.Template.Spec.Containers[0].StartupProbe, "startup")
+ assert.Nilf(t, deployment.Spec.Template.Spec.Containers[0].ReadinessProbe, "%s probe should be nil since it was not specified", "readiness")
+
+ // Check the Service
expectedServiceLabels := util.MergeLabelsOrAnnotations(instance.SharedLabelsWith(instance.Labels), map[string]string{"service-type": "metrics"})
expectedServiceAnnotations := map[string]string{"prometheus.io/path": "/metrics", "prometheus.io/port": "80", "prometheus.io/scheme": "http", "prometheus.io/scrape": "true"}
service := expectService(t, g, requests, expectedMetricsRequest, metricsSKey, expectedDeploymentLabels)
diff --git a/controllers/suite_test.go b/controllers/suite_test.go
index 4752253..77581c5 100644
--- a/controllers/suite_test.go
+++ b/controllers/suite_test.go
@@ -51,6 +51,14 @@
}
func TestMain(m *testing.M) {
+ // TODO: We can probably remove this once we upgrade our minimum supported version of Kubernetes
+ customApiServerFlags := []string{
+ "--feature-gates=StartupProbe=true",
+ }
+
+ apiServerFlags := append([]string(nil), envtest.DefaultKubeAPIServerFlags...)
+ apiServerFlags = append(apiServerFlags, customApiServerFlags...)
+
ctrl.SetLogger(zap.New(zap.UseDevMode(true)))
t := &envtest.Environment{
CRDDirectoryPaths: []string{
@@ -58,6 +66,7 @@
filepath.Join("..", "config", "dependencies"),
},
AttachControlPlaneOutput: false, // set to true to get more logging from the control plane
+ KubeAPIServerFlags: apiServerFlags,
}
solrv1beta1.AddToScheme(scheme.Scheme)
zkOp.AddToScheme(scheme.Scheme)
diff --git a/controllers/util/common.go b/controllers/util/common.go
index 7a3fd50..45074d4 100644
--- a/controllers/util/common.go
+++ b/controllers/util/common.go
@@ -132,6 +132,35 @@
return int32(ordinal) >= replicas
}
+// customizeProbe builds the probe logic used for pod liveness, readiness, startup checks
+func customizeProbe(initialProbe *corev1.Probe, customProbe corev1.Probe) *corev1.Probe {
+ if customProbe.InitialDelaySeconds != 0 {
+ initialProbe.InitialDelaySeconds = customProbe.InitialDelaySeconds
+ }
+
+ if customProbe.TimeoutSeconds != 0 {
+ initialProbe.TimeoutSeconds = customProbe.TimeoutSeconds
+ }
+
+ if customProbe.SuccessThreshold != 0 {
+ initialProbe.SuccessThreshold = customProbe.SuccessThreshold
+ }
+
+ if customProbe.FailureThreshold != 0 {
+ initialProbe.FailureThreshold = customProbe.FailureThreshold
+ }
+
+ if customProbe.PeriodSeconds != 0 {
+ initialProbe.PeriodSeconds = customProbe.PeriodSeconds
+ }
+
+ if customProbe.Handler.Exec != nil || customProbe.Handler.HTTPGet != nil || customProbe.Handler.TCPSocket != nil {
+ initialProbe.Handler = customProbe.Handler
+ }
+
+ return initialProbe
+}
+
// CopyConfigMapFields copies the owned fields from one ConfigMap to another
func CopyConfigMapFields(from, to *corev1.ConfigMap, logger logr.Logger) bool {
logger = logger.WithValues("kind", "configMap")
diff --git a/controllers/util/prometheus_exporter_util.go b/controllers/util/prometheus_exporter_util.go
index f7aef36..03636fb 100644
--- a/controllers/util/prometheus_exporter_util.go
+++ b/controllers/util/prometheus_exporter_util.go
@@ -303,6 +303,7 @@
deployment.Spec.Template.Spec.ImagePullSecrets = imagePullSecrets
if nil != customPodOptions {
+ metricsContainer := &deployment.Spec.Template.Spec.Containers[0]
if customPodOptions.ServiceAccountName != "" {
deployment.Spec.Template.Spec.ServiceAccountName = customPodOptions.ServiceAccountName
}
@@ -311,12 +312,8 @@
deployment.Spec.Template.Spec.Affinity = customPodOptions.Affinity
}
- if customPodOptions.Resources.Requests != nil {
- deployment.Spec.Template.Spec.Containers[0].Resources.Requests = customPodOptions.Resources.Requests
- }
-
- if customPodOptions.Resources.Limits != nil {
- deployment.Spec.Template.Spec.Containers[0].Resources.Limits = customPodOptions.Resources.Limits
+ if customPodOptions.Resources.Limits != nil || customPodOptions.Resources.Requests != nil {
+ metricsContainer.Resources = customPodOptions.Resources
}
if customPodOptions.PodSecurityContext != nil {
@@ -338,6 +335,22 @@
if customPodOptions.TerminationGracePeriodSeconds != nil {
deployment.Spec.Template.Spec.TerminationGracePeriodSeconds = customPodOptions.TerminationGracePeriodSeconds
}
+
+ if customPodOptions.ReadinessProbe != nil {
+ // Default Prometheus Exporter container does not contain a readinessProbe, so copy the livenessProbe
+ baseProbe := metricsContainer.LivenessProbe.DeepCopy()
+ metricsContainer.ReadinessProbe = customizeProbe(baseProbe, *customPodOptions.ReadinessProbe)
+ }
+
+ if customPodOptions.StartupProbe != nil {
+ // Default Prometheus Exporter container does not contain a startupProbe, so copy the livenessProbe
+ baseProbe := metricsContainer.LivenessProbe.DeepCopy()
+ metricsContainer.StartupProbe = customizeProbe(baseProbe, *customPodOptions.StartupProbe)
+ }
+
+ if customPodOptions.LivenessProbe != nil {
+ metricsContainer.LivenessProbe = customizeProbe(metricsContainer.LivenessProbe, *customPodOptions.LivenessProbe)
+ }
}
return deployment
diff --git a/controllers/util/solr_util.go b/controllers/util/solr_util.go
index 5c44f79..14276cb 100644
--- a/controllers/util/solr_util.go
+++ b/controllers/util/solr_util.go
@@ -63,24 +63,6 @@
DefaultStatefulSetPodManagementPolicy = appsv1.ParallelPodManagement
- DefaultLivenessProbeInitialDelaySeconds = 20
- DefaultLivenessProbeTimeoutSeconds = 1
- DefaultLivenessProbeSuccessThreshold = 1
- DefaultLivenessProbeFailureThreshold = 3
- DefaultLivenessProbePeriodSeconds = 10
-
- DefaultReadinessProbeInitialDelaySeconds = 15
- DefaultReadinessProbeTimeoutSeconds = 1
- DefaultReadinessProbeSuccessThreshold = 1
- DefaultReadinessProbeFailureThreshold = 3
- DefaultReadinessProbePeriodSeconds = 5
-
- DefaultStartupProbeInitialDelaySeconds = 20
- DefaultStartupProbeTimeoutSeconds = 30
- DefaultStartupProbeSuccessThreshold = 1
- DefaultStartupProbeFailureThreshold = 15
- DefaultStartupProbePeriodSeconds = 10
-
DefaultKeyStorePath = "/var/solr/tls"
Pkcs12KeystoreFile = "keystore.p12"
DefaultWritableKeyStorePath = "/var/solr/tls/pkcs12"
@@ -445,19 +427,19 @@
},
},
LivenessProbe: &corev1.Probe{
- InitialDelaySeconds: DefaultLivenessProbeInitialDelaySeconds,
- TimeoutSeconds: DefaultLivenessProbeTimeoutSeconds,
- SuccessThreshold: DefaultLivenessProbeSuccessThreshold,
- FailureThreshold: DefaultLivenessProbeFailureThreshold,
- PeriodSeconds: DefaultLivenessProbePeriodSeconds,
+ InitialDelaySeconds: 20,
+ TimeoutSeconds: 1,
+ SuccessThreshold: 1,
+ FailureThreshold: 3,
+ PeriodSeconds: 10,
Handler: defaultHandler,
},
ReadinessProbe: &corev1.Probe{
- InitialDelaySeconds: DefaultReadinessProbeInitialDelaySeconds,
- TimeoutSeconds: DefaultReadinessProbeTimeoutSeconds,
- SuccessThreshold: DefaultReadinessProbeSuccessThreshold,
- FailureThreshold: DefaultReadinessProbeFailureThreshold,
- PeriodSeconds: DefaultReadinessProbePeriodSeconds,
+ InitialDelaySeconds: 15,
+ TimeoutSeconds: 1,
+ SuccessThreshold: 1,
+ FailureThreshold: 3,
+ PeriodSeconds: 5,
Handler: defaultHandler,
},
VolumeMounts: volumeMounts,
@@ -552,6 +534,8 @@
stateful.Spec.Template.Spec.ImagePullSecrets = imagePullSecrets
if nil != customPodOptions {
+ solrContainer := &stateful.Spec.Template.Spec.Containers[0]
+
if customPodOptions.ServiceAccountName != "" {
stateful.Spec.Template.Spec.ServiceAccountName = customPodOptions.ServiceAccountName
}
@@ -561,7 +545,7 @@
}
if customPodOptions.Resources.Limits != nil || customPodOptions.Resources.Requests != nil {
- stateful.Spec.Template.Spec.Containers[0].Resources = customPodOptions.Resources
+ solrContainer.Resources = customPodOptions.Resources
}
if customPodOptions.PodSecurityContext != nil {
@@ -576,16 +560,21 @@
stateful.Spec.Template.Spec.NodeSelector = customPodOptions.NodeSelector
}
+ if customPodOptions.StartupProbe != nil {
+ // Default Solr container does not contain a startupProbe, so copy the livenessProbe
+ baseProbe := solrContainer.LivenessProbe.DeepCopy()
+ // Two options are different by default from the livenessProbe
+ baseProbe.TimeoutSeconds = 30
+ baseProbe.FailureThreshold = 15
+ solrContainer.StartupProbe = customizeProbe(baseProbe, *customPodOptions.StartupProbe)
+ }
+
if customPodOptions.LivenessProbe != nil {
- stateful.Spec.Template.Spec.Containers[0].LivenessProbe = fillProbe(*customPodOptions.LivenessProbe, DefaultLivenessProbeInitialDelaySeconds, DefaultLivenessProbeTimeoutSeconds, DefaultLivenessProbeSuccessThreshold, DefaultLivenessProbeFailureThreshold, DefaultLivenessProbePeriodSeconds, &defaultHandler)
+ solrContainer.LivenessProbe = customizeProbe(solrContainer.LivenessProbe, *customPodOptions.LivenessProbe)
}
if customPodOptions.ReadinessProbe != nil {
- stateful.Spec.Template.Spec.Containers[0].ReadinessProbe = fillProbe(*customPodOptions.ReadinessProbe, DefaultReadinessProbeInitialDelaySeconds, DefaultReadinessProbeTimeoutSeconds, DefaultReadinessProbeSuccessThreshold, DefaultReadinessProbeFailureThreshold, DefaultReadinessProbePeriodSeconds, &defaultHandler)
- }
-
- if customPodOptions.StartupProbe != nil {
- stateful.Spec.Template.Spec.Containers[0].StartupProbe = fillProbe(*customPodOptions.StartupProbe, DefaultStartupProbeInitialDelaySeconds, DefaultStartupProbeTimeoutSeconds, DefaultStartupProbeSuccessThreshold, DefaultStartupProbeFailureThreshold, DefaultStartupProbePeriodSeconds, &defaultHandler)
+ solrContainer.ReadinessProbe = customizeProbe(solrContainer.ReadinessProbe, *customPodOptions.ReadinessProbe)
}
if customPodOptions.PriorityClassName != "" {
@@ -686,44 +675,6 @@
return configMap
}
-// fillProbe builds the probe logic used for pod liveness, readiness, startup checks
-func fillProbe(customProbe corev1.Probe, defaultInitialDelaySeconds int32, defaultTimeoutSeconds int32, defaultSuccessThreshold int32, defaultFailureThreshold int32, defaultPeriodSeconds int32, defaultHandler *corev1.Handler) *corev1.Probe {
- probe := &corev1.Probe{
- InitialDelaySeconds: defaultInitialDelaySeconds,
- TimeoutSeconds: defaultTimeoutSeconds,
- SuccessThreshold: defaultSuccessThreshold,
- FailureThreshold: defaultFailureThreshold,
- PeriodSeconds: defaultPeriodSeconds,
- Handler: *defaultHandler,
- }
-
- if customProbe.InitialDelaySeconds != 0 {
- probe.InitialDelaySeconds = customProbe.InitialDelaySeconds
- }
-
- if customProbe.TimeoutSeconds != 0 {
- probe.TimeoutSeconds = customProbe.TimeoutSeconds
- }
-
- if customProbe.SuccessThreshold != 0 {
- probe.SuccessThreshold = customProbe.SuccessThreshold
- }
-
- if customProbe.FailureThreshold != 0 {
- probe.FailureThreshold = customProbe.FailureThreshold
- }
-
- if customProbe.PeriodSeconds != 0 {
- probe.PeriodSeconds = customProbe.PeriodSeconds
- }
-
- if customProbe.Handler.Exec != nil || customProbe.Handler.HTTPGet != nil || customProbe.Handler.TCPSocket != nil {
- probe.Handler = customProbe.Handler
- }
-
- return probe
-}
-
// GenerateCommonService returns a new corev1.Service pointer generated for the entire SolrCloud instance
// solrCloud: SolrCloud instance
func GenerateCommonService(solrCloud *solr.SolrCloud) *corev1.Service {
diff --git a/helm/solr-operator/Chart.yaml b/helm/solr-operator/Chart.yaml
index 9d61acd..6d791f6 100644
--- a/helm/solr-operator/Chart.yaml
+++ b/helm/solr-operator/Chart.yaml
@@ -98,6 +98,13 @@
url: https://github.com/apache/solr-operator/issues/294
- name: Github PR
url: https://github.com/apache/solr-operator/pull/295
+ - kind: added
+ description: Ability to customize probes for PrometheusExporter
+ links:
+ - name: Github Issue
+ url: https://github.com/apache/solr-operator/issues/282
+ - name: Github PR
+ url: https://github.com/apache/solr-operator/pull/297
artifacthub.io/images: |
- name: solr-operator
image: apache/solr-operator:v0.4.0-prerelease
