Use Solr's /admin/info/health for pod readiness checks (#629)
Now that the operator expects Solr versions >=8.11, we can use Solr's
/admin/info/health endpoint as our default readiness probe.
(/admin/info/system remains in use for liveness and startup probes.)
diff --git a/controllers/solrcloud_controller_basic_auth_test.go b/controllers/solrcloud_controller_basic_auth_test.go
index 7a4cee8..a91846f 100644
--- a/controllers/solrcloud_controller_basic_auth_test.go
+++ b/controllers/solrcloud_controller_basic_auth_test.go
@@ -208,13 +208,17 @@
func expectStatefulSetBasicAuthConfig(ctx context.Context, sc *solrv1beta1.SolrCloud, expectBootstrapSecret bool) *appsv1.StatefulSet {
Expect(sc.Spec.SolrSecurity).To(Not(BeNil()), "solrSecurity is not configured for this SolrCloud instance!")
- expProbePath := "/solr/admin/info/system"
+ expLivenessProbePath := "/solr/admin/info/system"
+ expReadinessProbePath := "/solr/admin/info/health"
if sc.Spec.CustomSolrKubeOptions.PodOptions != nil && sc.Spec.CustomSolrKubeOptions.PodOptions.LivenessProbe != nil {
- expProbePath = sc.Spec.CustomSolrKubeOptions.PodOptions.LivenessProbe.HTTPGet.Path
+ expLivenessProbePath = sc.Spec.CustomSolrKubeOptions.PodOptions.LivenessProbe.HTTPGet.Path
+ }
+ if sc.Spec.CustomSolrKubeOptions.PodOptions != nil && sc.Spec.CustomSolrKubeOptions.PodOptions.ReadinessProbe != nil {
+ expReadinessProbePath = sc.Spec.CustomSolrKubeOptions.PodOptions.ReadinessProbe.HTTPGet.Path
}
stateful := expectStatefulSetWithChecks(ctx, sc, sc.StatefulSetName(), func(g Gomega, found *appsv1.StatefulSet) {
- expectBasicAuthConfigOnPodTemplateWithGomega(g, sc, &found.Spec.Template, expectBootstrapSecret, expProbePath)
+ expectBasicAuthConfigOnPodTemplateWithGomega(g, sc, &found.Spec.Template, expectBootstrapSecret, expLivenessProbePath, expReadinessProbePath)
})
expectSecretWithChecks(ctx, sc, sc.BasicAuthSecretName(), func(innerG Gomega, found *corev1.Secret) {
@@ -237,7 +241,9 @@
probePaths := util.GetCustomProbePaths(sc)
if len(probePaths) > 0 {
securityJson := string(bootstrapSecret.Data["security.json"])
- Expect(securityJson).To(ContainSubstring(util.DefaultProbePath), "bootstrapped security.json should have an authz rule for probe path: %s", util.DefaultProbePath)
+ Expect(securityJson).To(ContainSubstring(util.DefaultLivenessProbePath), "bootstrapped security.json should have an authz rule for liveness probe path: %s", util.DefaultLivenessProbePath)
+ Expect(securityJson).To(ContainSubstring(util.DefaultReadinessProbePath), "bootstrapped security.json should have an authz rule for readiness probe path: %s", util.DefaultReadinessProbePath)
+
for _, p := range probePaths {
p = p[len("/solr"):] // drop the /solr part on the path
Expect(securityJson).To(ContainSubstring(p), "bootstrapped security.json should have an authz rule for probe path: %s", p)
@@ -250,7 +256,7 @@
}
// Ensures config is setup for basic-auth enabled Solr pods
-func expectBasicAuthConfigOnPodTemplateWithGomega(g Gomega, solrCloud *solrv1beta1.SolrCloud, podTemplate *corev1.PodTemplateSpec, expectBootstrapSecret bool, expProbePath string) *corev1.Container {
+func expectBasicAuthConfigOnPodTemplateWithGomega(g Gomega, solrCloud *solrv1beta1.SolrCloud, podTemplate *corev1.PodTemplateSpec, expectBootstrapSecret bool, expLivenessProbePath string, expReadinessProbePath string) *corev1.Container {
// check the env vars needed for the probes to work with auth
g.Expect(podTemplate.Spec.Containers).To(Not(BeEmpty()), "Solr Pod requires containers")
mainContainer := podTemplate.Spec.Containers[0]
@@ -282,18 +288,22 @@
g.Expect(basicAuthSecretVolMount).To(Not(BeNil()), "No Basic Auth volume mount used in Solr container")
g.Expect(basicAuthSecretVolMount.MountPath).To(Equal("/etc/secrets/"+secretName), "Wrong path used to mount Basic Auth volume")
- expProbeCmd := fmt.Sprintf("JAVA_TOOL_OPTIONS=\"-Dbasicauth=$(cat /etc/secrets/%s-solrcloud-basic-auth/username):$(cat /etc/secrets/%s-solrcloud-basic-auth/password) -Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory\" "+
+ expLivenessProbeCmd := fmt.Sprintf("JAVA_TOOL_OPTIONS=\"-Dbasicauth=$(cat /etc/secrets/%s-solrcloud-basic-auth/username):$(cat /etc/secrets/%s-solrcloud-basic-auth/password) -Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory\" "+
"solr api -get \"http://${SOLR_HOST}:8983%s\"",
- solrCloud.Name, solrCloud.Name, expProbePath)
+ solrCloud.Name, solrCloud.Name, expLivenessProbePath)
+ expReadinessProbeCmd := fmt.Sprintf("JAVA_TOOL_OPTIONS=\"-Dbasicauth=$(cat /etc/secrets/%s-solrcloud-basic-auth/username):$(cat /etc/secrets/%s-solrcloud-basic-auth/password) -Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory\" "+
+ "solr api -get \"http://${SOLR_HOST}:8983%s\"",
+ solrCloud.Name, solrCloud.Name, expReadinessProbePath)
+
g.Expect(mainContainer.LivenessProbe).To(Not(BeNil()), "main container should have a liveness probe defined")
g.Expect(mainContainer.LivenessProbe.Exec).To(Not(BeNil()), "liveness probe should have an exec when auth is enabled")
g.Expect(mainContainer.LivenessProbe.Exec.Command).To(Not(BeEmpty()), "liveness probe command cannot be empty")
- g.Expect(mainContainer.LivenessProbe.Exec.Command[2]).To(Equal(expProbeCmd), "liveness probe should invoke java with auth opts")
+ g.Expect(mainContainer.LivenessProbe.Exec.Command[2]).To(Equal(expLivenessProbeCmd), "liveness probe should invoke java with auth opts")
g.Expect(mainContainer.LivenessProbe.TimeoutSeconds).To(BeEquivalentTo(5), "liveness probe default timeout should be increased when using basicAuth")
g.Expect(mainContainer.ReadinessProbe).To(Not(BeNil()), "main container should have a readiness probe defined")
g.Expect(mainContainer.ReadinessProbe.Exec).To(Not(BeNil()), "readiness probe should have an exec when auth is enabled")
g.Expect(mainContainer.ReadinessProbe.Exec.Command).To(Not(BeEmpty()), "readiness probe command cannot be empty")
- g.Expect(mainContainer.ReadinessProbe.Exec.Command[2]).To(Equal(expProbeCmd), "readiness probe should invoke java with auth opts")
+ g.Expect(mainContainer.ReadinessProbe.Exec.Command[2]).To(Equal(expReadinessProbeCmd), "readiness probe should invoke java with auth opts")
g.Expect(mainContainer.ReadinessProbe.TimeoutSeconds).To(BeEquivalentTo(5), "readiness probe default timeout should be increased when using basicAuth")
}
diff --git a/controllers/solrcloud_controller_tls_test.go b/controllers/solrcloud_controller_tls_test.go
index 22daa96..c24013f 100644
--- a/controllers/solrcloud_controller_tls_test.go
+++ b/controllers/solrcloud_controller_tls_test.go
@@ -124,7 +124,7 @@
ProbeHandler: corev1.ProbeHandler{
HTTPGet: &corev1.HTTPGetAction{
Scheme: corev1.URISchemeHTTPS,
- Path: "/solr/admin/info/health",
+ Path: "/solr/admin/customreadiness",
Port: intstr.FromInt(8983),
},
},
@@ -134,7 +134,7 @@
solrCloud.Spec.SolrTLS = createTLSOptions(tlsSecretName, keystorePassKey, false)
})
FIt("has the correct resources", func(ctx context.Context) {
- Expect(util.GetCustomProbePaths(solrCloud)).To(ConsistOf("/solr/admin/info/health"), "Utility Probe paths command gives wrong result")
+ Expect(util.GetCustomProbePaths(solrCloud)).To(ConsistOf("/solr/admin/customreadiness"), "Utility Probe paths command gives wrong result")
verifyUserSuppliedTLSConfig(solrCloud.Spec.SolrTLS, tlsSecretName, keystorePassKey, tlsSecretName)
By("checking that the User supplied TLS Config is correct in the generated StatefulSet")
@@ -536,7 +536,7 @@
g.Expect(mainContainer.LivenessProbe.Exec).To(Not(BeNil()), "liveness probe should have an exec when mTLS is enabled")
g.Expect(mainContainer.LivenessProbe.Exec.Command).To(Not(BeNil()), "liveness probe should have an exec when mTLS is enabled")
g.Expect(mainContainer.LivenessProbe.Exec.Command).To(HaveLen(3), "liveness probe command has wrong number of args")
- path := "/solr" + util.DefaultProbePath
+ path := "/solr" + util.DefaultLivenessProbePath
if solrCloud.Spec.CustomSolrKubeOptions.PodOptions != nil && solrCloud.Spec.CustomSolrKubeOptions.PodOptions.LivenessProbe != nil {
path = solrCloud.Spec.CustomSolrKubeOptions.PodOptions.LivenessProbe.HTTPGet.Path
}
@@ -544,7 +544,7 @@
g.Expect(mainContainer.ReadinessProbe).To(Not(BeNil()), "main container should have a readiness probe defined")
g.Expect(mainContainer.ReadinessProbe.Exec).To(Not(BeNil()), "readiness probe should have an exec when mTLS is enabled")
g.Expect(mainContainer.ReadinessProbe.Exec.Command).To(HaveLen(3), "readiness probe command has wrong number of args")
- path = "/solr" + util.DefaultProbePath
+ path = "/solr" + util.DefaultReadinessProbePath
if solrCloud.Spec.CustomSolrKubeOptions.PodOptions != nil && solrCloud.Spec.CustomSolrKubeOptions.PodOptions.ReadinessProbe != nil {
path = solrCloud.Spec.CustomSolrKubeOptions.PodOptions.ReadinessProbe.HTTPGet.Path
}
@@ -553,7 +553,7 @@
g.Expect(mainContainer.StartupProbe).To(Not(BeNil()), "main container should have a startup probe defined")
g.Expect(mainContainer.StartupProbe.Exec).To(Not(BeNil()), "startup probe should have an exec when auth is enabled")
g.Expect(mainContainer.StartupProbe.Exec.Command).To(HaveLen(3), "startup probe command has wrong number of args")
- path = "/solr" + util.DefaultProbePath
+ path = "/solr" + util.DefaultStartupProbePath
if solrCloud.Spec.CustomSolrKubeOptions.PodOptions != nil && solrCloud.Spec.CustomSolrKubeOptions.PodOptions.StartupProbe != nil {
path = solrCloud.Spec.CustomSolrKubeOptions.PodOptions.StartupProbe.HTTPGet.Path
}
diff --git a/controllers/util/solr_security_util.go b/controllers/util/solr_security_util.go
index 8e76dc0..51caa31 100644
--- a/controllers/util/solr_security_util.go
+++ b/controllers/util/solr_security_util.go
@@ -39,9 +39,11 @@
)
const (
- SecurityJsonFile = "security.json"
- BasicAuthMd5Annotation = "solr.apache.org/basicAuthMd5"
- DefaultProbePath = "/admin/info/system"
+ SecurityJsonFile = "security.json"
+ BasicAuthMd5Annotation = "solr.apache.org/basicAuthMd5"
+ DefaultStartupProbePath = "/admin/info/system"
+ DefaultLivenessProbePath = DefaultStartupProbePath
+ DefaultReadinessProbePath = "/admin/info/health"
)
// Utility struct holding security related config and objects resolved at runtime needed during reconciliation,
@@ -433,7 +435,8 @@
// Gets a list of probe paths we need to setup authz for
func getProbePaths(solrCloud *solr.SolrCloud) []string {
- probePaths := []string{DefaultProbePath}
+ // Current startup and liveness probes use the same API, so liveness needn't be explicitly specified here
+ probePaths := []string{DefaultStartupProbePath, DefaultReadinessProbePath}
probePaths = append(probePaths, GetCustomProbePaths(solrCloud)...)
return uniqueProbePaths(probePaths)
}
diff --git a/controllers/util/solr_util.go b/controllers/util/solr_util.go
index 62a99c8..c6ede52 100644
--- a/controllers/util/solr_util.go
+++ b/controllers/util/solr_util.go
@@ -89,13 +89,9 @@
}
defaultProbeTimeout := int32(1)
- defaultHandler := corev1.ProbeHandler{
- HTTPGet: &corev1.HTTPGetAction{
- Scheme: probeScheme,
- Path: "/solr" + DefaultProbePath,
- Port: intstr.FromInt(solrPodPort),
- },
- }
+ defaultStartupProbe := createDefaultProbeHandlerForPath(probeScheme, solrPodPort, DefaultStartupProbePath)
+ defaultLivenessProbe := createDefaultProbeHandlerForPath(probeScheme, solrPodPort, DefaultLivenessProbePath)
+ defaultReadinessProbe := createDefaultProbeHandlerForPath(probeScheme, solrPodPort, DefaultReadinessProbePath)
labels := solrCloud.SharedLabelsWith(solrCloud.GetLabels())
selectorLabels := solrCloud.SharedLabels()
@@ -468,7 +464,7 @@
SuccessThreshold: 1,
FailureThreshold: 10,
PeriodSeconds: 5,
- ProbeHandler: defaultHandler,
+ ProbeHandler: defaultStartupProbe,
},
// Kill Solr if it is unavailable for any 60-second period
LivenessProbe: &corev1.Probe{
@@ -476,7 +472,7 @@
SuccessThreshold: 1,
FailureThreshold: 3,
PeriodSeconds: 20,
- ProbeHandler: defaultHandler,
+ ProbeHandler: defaultLivenessProbe,
},
// Do not route requests to solr if it is not available for any 20-second period
ReadinessProbe: &corev1.Probe{
@@ -484,7 +480,7 @@
SuccessThreshold: 1,
FailureThreshold: 2,
PeriodSeconds: 10,
- ProbeHandler: defaultHandler,
+ ProbeHandler: defaultReadinessProbe,
},
VolumeMounts: volumeMounts,
Env: envVars,
@@ -782,6 +778,16 @@
return containers
}
+func createDefaultProbeHandlerForPath(probeScheme corev1.URIScheme, solrPodPort int, path string) corev1.ProbeHandler {
+ return corev1.ProbeHandler{
+ HTTPGet: &corev1.HTTPGetAction{
+ Scheme: probeScheme,
+ Path: "/solr" + path,
+ Port: intstr.FromInt(solrPodPort),
+ },
+ }
+}
+
const DefaultSolrXML = `<?xml version="1.0" encoding="UTF-8" ?>
<solr>
%s
diff --git a/docs/solr-cloud/solr-cloud-crd.md b/docs/solr-cloud/solr-cloud-crd.md
index b30ea67..8c8874e 100644
--- a/docs/solr-cloud/solr-cloud-crd.md
+++ b/docs/solr-cloud/solr-cloud-crd.md
@@ -930,10 +930,10 @@
With a command, we can load the username and password from a secret; Kubernetes will
[update the mounted secret files](https://kubernetes.io/docs/concepts/configuration/secret/#mounted-secrets-are-updated-automatically) when the secret changes automatically.
-If you customize the HTTP path for any probes (under `spec.customSolrKubeOptions.podOptions`),
-then you must use `probesRequireAuth=false` as the operator does not reconfigure custom HTTP probes to use the command needed to support `probesRequireAuth=true`.
+By default, the operator will use Solr's `/admin/info/system` endpoint for startup and liveness probes, and the `/admin/info/health` endpoint for readiness probes.
+This default can be customized by changing the HTTP path for any probes (under `spec.customSolrKubeOptions.podOptions`), however this also requires users to set `probesRequireAuth=false` as the operator does not reconfigure custom HTTP probes to use the command needed to support `probesRequireAuth=true`.
-If you're running Solr 8+, then we recommend using the `/admin/info/health` endpoint for your probes using the following config:
+Custom readiness and liveness probes can be specified using configuration like the following:
```yaml
spec:
...
@@ -942,21 +942,28 @@
livenessProbe:
httpGet:
scheme: HTTP
- path: /solr/admin/info/health
+ path: /solr/admin/customliveness
port: 8983
readinessProbe:
httpGet:
scheme: HTTP
- path: /solr/admin/info/health
+ path: /solr/admin/customreadiness
port: 8983
```
-Consequently, the bootstrapped `security.json` will include an additional rule to allow access to the `/admin/info/health` endpoint:
+
+Consequently, the bootstrapped `security.json` will include additional rules to allow access to the endpoints used by the startup, liveness, and readiness probes:
```json
{
"name": "k8s-probe-1",
"role": null,
"collection": null,
- "path": "/admin/info/health"
+ "path": "/admin/customliveness"
+ },
+ {
+ "name": "k8s-probe-2",
+ "role": null,
+ "collection": null,
+ "path": "/admin/customreadiness"
}
```
@@ -993,7 +1000,7 @@
"name": "k8s-probe-0",
"role": null,
"collection": null,
- "path": "/admin/info/system"
+ "path": "/admin/info/health"
},
{
"name": "k8s-status",
@@ -1050,7 +1057,7 @@
"name": "k8s-probe-0",
"role": null,
"collection": null,
- "path": "/admin/info/system"
+ "path": "/admin/info/health"
}
```
In this case, the `"role":null` indicates this endpoint allows anonymous access by unknown users.
@@ -1146,7 +1153,6 @@
Users need to ensure their `security.json` contains the user supplied in the `basicAuthSecret` has read access to the following endpoints:
```
-/admin/info/system
/admin/info/health
/admin/collections
/admin/metrics
diff --git a/docs/upgrade-notes.md b/docs/upgrade-notes.md
index 128be56..3019e48 100644
--- a/docs/upgrade-notes.md
+++ b/docs/upgrade-notes.md
@@ -130,7 +130,11 @@
- The `POD_HOSTNAME` envVar in SolrCloud Pods has been deprecated. Use `POD_NAME` instead.
-- Use of the `hostPort` system property placeholder in custom solr.xml files has been deprecated. Use `<int name="hostPort">${solr.port.advertise:80}</int>`, the default value used by Solr, instead.
+- Use of the `hostPort` system property placeholder in custom solr.xml files has been deprecated.
+ Use `<int name="hostPort">${solr.port.advertise:80}</int>`, the default value used by Solr, instead.
+
+- By default `solrcloud` resources will now use `/admin/info/system` and `/admin/info/health` for liveness and readiness checks, respectively.
+ Administrators that provide custom `security.json` files for their clusters should either exempt both of these endpoints from authentication entirely, or configure permissions ensuring the relevant Solr user account can access them without issue.
### v0.7.0
- **Kubernetes support is now limited to 1.21+.**
diff --git a/hack/release/smoke_test/test_cluster.sh b/hack/release/smoke_test/test_cluster.sh
index 26d7ed4..27b3aa1 100755
--- a/hack/release/smoke_test/test_cluster.sh
+++ b/hack/release/smoke_test/test_cluster.sh
@@ -195,7 +195,7 @@
sleep 2
printf "\nCheck the admin URL to make sure it works\n"
-curl --silent "http://localhost:18983/solr/admin/info/system" | grep '"status":0' > /dev/null
+curl --silent "http://localhost:18983/solr/admin/info/health" | grep '"status":0' > /dev/null
printf "\nCreating a test collection\n"
curl --silent "http://localhost:18983/solr/admin/collections?action=CREATE&name=smoke-test&replicationFactor=1&numShards=2" | grep '"status":0' > /dev/null
diff --git a/helm/solr/Chart.yaml b/helm/solr/Chart.yaml
index f265d4b..5fda1c1 100644
--- a/helm/solr/Chart.yaml
+++ b/helm/solr/Chart.yaml
@@ -62,6 +62,13 @@
url: https://github.com/apache/solr-operator/issues/635
- name: Github PR
url: https://github.com/apache/solr-operator/pull/636
+ - kind: changed
+ description: Solr's `/admin/info/health` API is now used for readiness probes by default.
+ links:
+ - name: Github Issue
+ url: https://github.com/apache/solr-operator/issues/628
+ - name: Github PR
+ url: https://github.com/apache/solr-operator/pull/629
artifacthub.io/containsSecurityUpdates: "false"
artifacthub.io/recommendations: |
- url: https://artifacthub.io/packages/helm/apache-solr/solr-operator
