| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package controllers |
| |
| import ( |
| "crypto/md5" |
| b64 "encoding/base64" |
| "fmt" |
| solr "github.com/apache/solr-operator/api/v1beta1" |
| "github.com/apache/solr-operator/controllers/util" |
| "github.com/onsi/gomega" |
| "github.com/stretchr/testify/assert" |
| "golang.org/x/net/context" |
| appsv1 "k8s.io/api/apps/v1" |
| corev1 "k8s.io/api/core/v1" |
| netv1 "k8s.io/api/networking/v1beta1" |
| "k8s.io/apimachinery/pkg/api/errors" |
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| "k8s.io/apimachinery/pkg/types" |
| "k8s.io/apimachinery/pkg/util/intstr" |
| ctrl "sigs.k8s.io/controller-runtime" |
| "sigs.k8s.io/controller-runtime/pkg/client" |
| "sigs.k8s.io/controller-runtime/pkg/manager" |
| "sigs.k8s.io/controller-runtime/pkg/reconcile" |
| "strings" |
| "sync" |
| "testing" |
| "time" |
| ) |
| |
| var _ reconcile.Reconciler = &SolrCloudReconciler{} |
| |
| var ( |
| expectedCloudWithTLSRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: "foo-tls", Namespace: "default"}} |
| expectedIngressWithTLS = types.NamespacedName{Name: "foo-tls-solrcloud-common", Namespace: "default"} |
| expectedStatefulSetName = types.NamespacedName{Name: "foo-tls-solrcloud", Namespace: "default"} |
| ) |
| |
| func TestBasicAuthBootstrapSecurityJson(t *testing.T) { |
| instance := buildTestSolrCloud() |
| instance.Spec.SolrSecurity = &solr.SolrSecurityOptions{AuthenticationType: solr.Basic, ProbesRequireAuth: true} |
| verifyReconcileWithSecurity(t, instance, false, false) |
| } |
| |
| func TestBasicAuthBootstrapSecurityJsonWithZkACLs(t *testing.T) { |
| instance := buildTestSolrCloud() |
| instance.Spec.SolrSecurity = &solr.SolrSecurityOptions{AuthenticationType: solr.Basic, ProbesRequireAuth: true} |
| zkReplicas := int32(1) |
| instance.Spec.ZookeeperRef = &solr.ZookeeperRef{ |
| ProvidedZookeeper: &solr.ZookeeperSpec{ |
| Replicas: &zkReplicas, |
| AllACL: &solr.ZookeeperACL{ |
| SecretRef: "secret-name", |
| UsernameKey: "user", |
| PasswordKey: "pass", |
| }, |
| ReadOnlyACL: &solr.ZookeeperACL{ |
| SecretRef: "read-secret-name", |
| UsernameKey: "read-only-user", |
| PasswordKey: "read-only-pass", |
| }, |
| }, |
| } |
| verifyReconcileWithSecurity(t, instance, false, true) |
| } |
| |
| func TestBasicAuthBootstrapSecurityJsonDeleteSecret(t *testing.T) { |
| instance := buildTestSolrCloud() |
| instance.Spec.SolrSecurity = &solr.SolrSecurityOptions{AuthenticationType: solr.Basic} |
| verifyReconcileWithSecurity(t, instance, true, false) |
| } |
| |
| func TestBasicAuthWithUserProvidedCreds(t *testing.T) { |
| instance := buildTestSolrCloud() |
| instance.Spec.SolrSecurity = &solr.SolrSecurityOptions{AuthenticationType: solr.Basic, BasicAuthSecret: "my-basic-auth-secret"} |
| verifyReconcileWithSecurity(t, instance, false, false) |
| } |
| |
| func TestBasicAuthBootstrapWithTLS(t *testing.T) { |
| instance := buildTestSolrCloud() |
| |
| // custom probe endpoint too |
| instance.Spec.CustomSolrKubeOptions.PodOptions = &solr.PodOptions{ |
| ReadinessProbe: &corev1.Probe{ |
| Handler: corev1.Handler{ |
| HTTPGet: &corev1.HTTPGetAction{ |
| Scheme: corev1.URISchemeHTTPS, |
| Path: "/solr/admin/info/health", |
| Port: intstr.FromInt(8983), |
| }, |
| }, |
| }, |
| } |
| instance.Spec.SolrSecurity = &solr.SolrSecurityOptions{AuthenticationType: solr.Basic} |
| |
| probePaths := util.GetCustomProbePaths(instance) |
| assert.EqualValues(t, []string{"/solr/admin/info/health"}, probePaths) |
| |
| tlsSecretName := "tls-cert-secret-from-user" |
| keystorePassKey := "some-password-key-thingy" |
| instance.Spec.SolrTLS = createTLSOptions(tlsSecretName, keystorePassKey, false) |
| verifyUserSuppliedTLSConfig(t, instance.Spec.SolrTLS, tlsSecretName, keystorePassKey, tlsSecretName) |
| verifyReconcileUserSuppliedTLS(t, instance, false, false) |
| } |
| |
| func mountedTLSDir(path string) *solr.MountedTLSDirectory { |
| return &solr.MountedTLSDirectory{ |
| Path: path, |
| KeystoreFile: util.DefaultPkcs12KeystoreFile, |
| TruststoreFile: util.DefaultPkcs12TruststoreFile, |
| KeystorePasswordFile: util.DefaultKeystorePasswordFile, |
| } |
| } |
| |
| func TestMountedTLSDir(t *testing.T) { |
| instance := buildTestSolrCloud() |
| mountedDir := mountedTLSDir("/mounted-tls-dir") |
| instance.Spec.SolrTLS = &solr.SolrTLSOptions{MountedTLSDir: mountedDir, CheckPeerName: true, ClientAuth: "Need", VerifyClientHostname: true} |
| verifyReconcileMountedTLSDir(t, instance) |
| } |
| |
| func TestMountedTLSDirNonDefaultFileNames(t *testing.T) { |
| instance := buildTestSolrCloud() |
| mountedDir := &solr.MountedTLSDirectory{ |
| Path: "/mounted-non-default", |
| KeystoreFile: "ks.p12", |
| TruststoreFile: "ts.p12", |
| KeystorePasswordFile: "ks-password", |
| TruststorePasswordFile: "ts-password", |
| } |
| instance.Spec.SolrTLS = &solr.SolrTLSOptions{MountedTLSDir: mountedDir, CheckPeerName: true, ClientAuth: "Need", VerifyClientHostname: true} |
| verifyReconcileMountedTLSDir(t, instance) |
| } |
| |
| func TestMountedTLSDirWithBasicAuth(t *testing.T) { |
| instance := buildTestSolrCloud() |
| mountedDir := mountedTLSDir("/mounted-tls-dir") |
| instance.Spec.SolrTLS = &solr.SolrTLSOptions{MountedTLSDir: mountedDir, CheckPeerName: true, ClientAuth: "Need", VerifyClientHostname: true} |
| instance.Spec.SolrSecurity = &solr.SolrSecurityOptions{AuthenticationType: solr.Basic} // with basic-auth too |
| verifyReconcileMountedTLSDir(t, instance) |
| } |
| |
| func TestMountedTLSServerAndClientDirs(t *testing.T) { |
| instance := buildTestSolrCloud() |
| mountedSrvrTLSDir := mountedTLSDir("/mounted-tls-dir") |
| mountedClientTLSDir := mountedTLSDir("/mounted-client-tls-dir") |
| instance.Spec.SolrTLS = &solr.SolrTLSOptions{MountedTLSDir: mountedSrvrTLSDir, CheckPeerName: true, ClientAuth: "Need", VerifyClientHostname: true} |
| instance.Spec.SolrClientTLS = &solr.SolrTLSOptions{MountedTLSDir: mountedClientTLSDir, CheckPeerName: true} |
| verifyReconcileMountedTLSDir(t, instance) |
| } |
| |
| // For TLS, all we really need is a secret holding the keystore password and a secret holding the pkcs12 keystore, |
| // which can come from anywhere really, so this method tests handling of user-supplied secrets |
| func TestUserSuppliedTLSSecretWithPkcs12Keystore(t *testing.T) { |
| tlsSecretName := "tls-cert-secret-from-user" |
| keystorePassKey := "some-password-key-thingy" |
| instance := buildTestSolrCloud() |
| instance.Spec.SolrSecurity = &solr.SolrSecurityOptions{AuthenticationType: solr.Basic} // with basic-auth too |
| instance.Spec.SolrTLS = createTLSOptions(tlsSecretName, keystorePassKey, false) |
| verifyUserSuppliedTLSConfig(t, instance.Spec.SolrTLS, tlsSecretName, keystorePassKey, tlsSecretName) |
| verifyReconcileUserSuppliedTLS(t, instance, false, false) |
| } |
| |
| // User wants a different trust store than the keystore |
| func TestUserSuppliedTLSSecretWithSeparateTrustStore(t *testing.T) { |
| tlsSecretName := "tls-cert-secret-from-user" |
| keystorePassKey := "some-password-key-thingy" |
| instance := buildTestSolrCloud() |
| instance.Spec.SolrSecurity = &solr.SolrSecurityOptions{AuthenticationType: solr.Basic} // with basic-auth too |
| instance.Spec.SolrTLS = createTLSOptions(tlsSecretName, keystorePassKey, false) |
| |
| trustStoreSecretName := "custom-truststore-secret" |
| trustStoreFile := "truststore.p12" |
| instance.Spec.SolrTLS.TrustStoreSecret = &corev1.SecretKeySelector{ |
| LocalObjectReference: corev1.LocalObjectReference{Name: trustStoreSecretName}, |
| Key: trustStoreFile, |
| } |
| |
| instance.Spec.SolrTLS.TrustStorePasswordSecret = &corev1.SecretKeySelector{ |
| LocalObjectReference: corev1.LocalObjectReference{Name: trustStoreSecretName}, |
| Key: "truststore-pass", |
| } |
| |
| instance.Spec.SolrTLS.ClientAuth = solr.Need // require client auth too (mTLS between the pods) |
| |
| verifyUserSuppliedTLSConfig(t, instance.Spec.SolrTLS, tlsSecretName, keystorePassKey, tlsSecretName) |
| verifyReconcileUserSuppliedTLS(t, instance, false, false) |
| } |
| |
| // Test upgrade from non-TLS cluster to TLS enabled cluster |
| func TestEnableTLSOnExistingCluster(t *testing.T) { |
| instance := buildTestSolrCloud() |
| instance.Spec.SolrSecurity = &solr.SolrSecurityOptions{AuthenticationType: solr.Basic} |
| |
| changed := instance.WithDefaults() |
| assert.True(t, changed, "WithDefaults should have changed the test SolrCloud instance") |
| |
| g := gomega.NewGomegaWithT(t) |
| helper := NewTLSTestHelper(g) |
| defer func() { |
| helper.StopTest() |
| }() |
| |
| ctx := context.TODO() |
| cleanupTest(g, instance.Namespace) |
| helper.ReconcileSolrCloud(ctx, instance, 2, util.DefaultPkcs12KeystoreFile) |
| defer testClient.Delete(ctx, instance) |
| |
| // now, update the config to enable TLS |
| tlsSecretName := "tls-cert-secret-from-user" |
| keystorePassKey := "some-password-key-thingy" |
| |
| err := testClient.Get(ctx, types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}, instance) |
| g.Expect(err).NotTo(gomega.HaveOccurred()) |
| instance.Spec.SolrTLS = createTLSOptions(tlsSecretName, keystorePassKey, true) |
| foundTLSSecret := &corev1.Secret{} |
| lookupErr := testClient.Get(ctx, types.NamespacedName{Name: instance.Spec.SolrTLS.PKCS12Secret.Name, Namespace: instance.Namespace}, foundTLSSecret) |
| // TLS secret should not exist |
| assert.True(t, errors.IsNotFound(lookupErr)) |
| |
| // apply the update to trigger the upgrade to https |
| err = testClient.Update(ctx, instance) |
| g.Expect(err).NotTo(gomega.HaveOccurred()) |
| |
| // Create a mock secret in the background so the isCert ready function returns |
| var mockTLSSecret *corev1.Secret |
| var wg sync.WaitGroup |
| wg.Add(1) |
| go func() { |
| secret, _ := createMockTLSSecret(ctx, testClient, instance.Spec.SolrTLS.PKCS12Secret.Name, "keystore.p12", instance.Namespace, instance.Spec.SolrTLS.KeyStorePasswordSecret.Key, "") |
| mockTLSSecret = &secret |
| wg.Done() |
| }() |
| helper.WaitForReconcile(3) |
| wg.Wait() |
| |
| expectStatefulSetTLSConfig(t, g, instance, false) |
| expectPassthroughIngressTLSConfig(t, g, tlsSecretName, instance.Spec.SolrTLS) |
| |
| defer testClient.Delete(ctx, mockTLSSecret) |
| } |
| |
| func TestUserSuppliedTLSSecretWithPkcs12Conversion(t *testing.T) { |
| tlsSecretName := "tls-cert-secret-from-user-no-pkcs12" |
| keystorePassKey := "some-password-key-thingy" |
| instance := buildTestSolrCloud() |
| instance.Spec.SolrSecurity = &solr.SolrSecurityOptions{AuthenticationType: solr.Basic} |
| instance.Spec.SolrTLS = createTLSOptions(tlsSecretName, keystorePassKey, false) |
| verifyUserSuppliedTLSConfig(t, instance.Spec.SolrTLS, tlsSecretName, keystorePassKey, tlsSecretName) |
| verifyReconcileUserSuppliedTLS(t, instance, true, false) |
| } |
| |
| func TestTLSSecretUpdate(t *testing.T) { |
| tlsSecretName := "tls-cert-secret-update" |
| keystorePassKey := "some-password-key-thingy" |
| instance := buildTestSolrCloud() |
| instance.Spec.SolrSecurity = &solr.SolrSecurityOptions{AuthenticationType: solr.Basic} |
| instance.Spec.SolrTLS = createTLSOptions(tlsSecretName, keystorePassKey, true) |
| instance.Spec.SolrTLS.ClientAuth = solr.Need |
| verifyUserSuppliedTLSConfig(t, instance.Spec.SolrTLS, tlsSecretName, keystorePassKey, tlsSecretName) |
| verifyReconcileUserSuppliedTLS(t, instance, false, true) |
| } |
| |
| func TestTLSServerAndClientFromSecret(t *testing.T) { |
| serverCertSecret := "tls-server-cert" |
| clientCertSecret := "tls-client-cert" |
| keystorePassKey := "some-password-key-thingy" |
| instance := buildTestSolrCloud() |
| instance.Spec.SolrTLS = createTLSOptions(serverCertSecret, keystorePassKey, true) |
| instance.Spec.SolrTLS.ClientAuth = solr.Need |
| |
| // Additional client cert |
| instance.Spec.SolrClientTLS = &solr.SolrTLSOptions{ |
| KeyStorePasswordSecret: &corev1.SecretKeySelector{ |
| LocalObjectReference: corev1.LocalObjectReference{Name: clientCertSecret}, |
| Key: keystorePassKey, |
| }, |
| PKCS12Secret: &corev1.SecretKeySelector{ |
| LocalObjectReference: corev1.LocalObjectReference{Name: clientCertSecret}, |
| Key: util.DefaultPkcs12KeystoreFile, |
| }, |
| TrustStoreSecret: &corev1.SecretKeySelector{ |
| LocalObjectReference: corev1.LocalObjectReference{Name: clientCertSecret}, |
| Key: util.DefaultPkcs12TruststoreFile, |
| }, |
| RestartOnTLSSecretUpdate: true, |
| } |
| |
| verifyReconcileUserSuppliedTLS(t, instance, false, true) |
| } |
| |
| func verifyReconcileMountedTLSDir(t *testing.T, instance *solr.SolrCloud) { |
| g := gomega.NewGomegaWithT(t) |
| ctx := context.TODO() |
| helper := NewTLSTestHelper(g) |
| defer func() { |
| helper.StopTest() |
| }() |
| // start with a clean namespace |
| cleanupTest(g, instance.Namespace) |
| |
| helper.ReconcileSolrCloud(ctx, instance, 3, util.DefaultPkcs12KeystoreFile) |
| defer testClient.Delete(ctx, instance) |
| |
| expectStatefulSetMountedTLSDirConfig(t, g, instance) |
| } |
| |
| func verifyReconcileUserSuppliedTLS(t *testing.T, instance *solr.SolrCloud, needsPkcs12InitContainer bool, restartOnTLSSecretUpdate bool) { |
| g := gomega.NewGomegaWithT(t) |
| ctx := context.TODO() |
| helper := NewTLSTestHelper(g) |
| defer func() { |
| helper.StopTest() |
| }() |
| |
| // start with a clean namespace |
| cleanupTest(g, instance.Namespace) |
| |
| // Custom truststore? |
| if instance.Spec.SolrTLS.TrustStoreSecret != nil { |
| // create the mock truststore secret |
| secretData := map[string][]byte{} |
| secretData[instance.Spec.SolrTLS.TrustStoreSecret.Key] = []byte(b64.StdEncoding.EncodeToString([]byte("mock truststore"))) |
| secretData[instance.Spec.SolrTLS.TrustStorePasswordSecret.Key] = []byte(b64.StdEncoding.EncodeToString([]byte("mock truststore password"))) |
| trustStoreSecret := corev1.Secret{ |
| ObjectMeta: metav1.ObjectMeta{Name: instance.Spec.SolrTLS.TrustStoreSecret.Name, Namespace: instance.Namespace}, |
| Data: secretData, |
| Type: corev1.SecretTypeOpaque, |
| } |
| err := testClient.Create(ctx, &trustStoreSecret) |
| g.Expect(err).NotTo(gomega.HaveOccurred()) |
| defer testClient.Delete(ctx, &trustStoreSecret) |
| } |
| |
| // create the secret required for reconcile, it has both keys ... |
| tlsKey := "keystore.p12" |
| if needsPkcs12InitContainer { |
| tlsKey = "tls.key" // to trigger the initContainer creation, don't want keystore.p12 in the secret |
| } |
| |
| helper.ReconcileSolrCloud(ctx, instance, 3, tlsKey) |
| defer testClient.Delete(ctx, instance) |
| |
| sts := expectStatefulSetTLSConfig(t, g, instance, needsPkcs12InitContainer) |
| |
| if !restartOnTLSSecretUpdate { |
| assert.Empty(t, sts.Spec.Template.ObjectMeta.Annotations[util.SolrTlsCertMd5Annotation], |
| "Shouldn't have a cert MD5 as we're not tracking updates to the secret") |
| return |
| } |
| |
| // let's trigger an update to the TLS secret to simulate the cert getting renewed and the pods getting restarted |
| expectRestartOnTLSSecretUpdate(t, helper, instance.Spec.SolrTLS.PKCS12Secret.Name, instance, sts, needsPkcs12InitContainer, util.SolrTlsCertMd5Annotation) |
| |
| // does basic-auth work with TLS? That's the most common so we test both here |
| if instance.Spec.SolrSecurity != nil { |
| expectBootstrapSecret := instance.Spec.SolrSecurity.BasicAuthSecret == "" |
| expectStatefulSetBasicAuthConfig(t, g, instance, expectBootstrapSecret) |
| } |
| |
| // Update the client cert and see the change get picked up |
| if instance.Spec.SolrClientTLS != nil && instance.Spec.SolrClientTLS.PKCS12Secret != nil && instance.Spec.SolrClientTLS.RestartOnTLSSecretUpdate { |
| expectRestartOnTLSSecretUpdate(t, helper, instance.Spec.SolrClientTLS.PKCS12Secret.Name, instance, sts, false, util.SolrClientTlsCertMd5Annotation) |
| } |
| } |
| |
| func expectRestartOnTLSSecretUpdate(t *testing.T, helper *TLSTestHelper, secretName string, instance *solr.SolrCloud, sts *appsv1.StatefulSet, needsPkcs12InitContainer bool, certAnnotation string) { |
| g := helper.g |
| ctx := context.TODO() |
| |
| // let's trigger an update to the TLS secret to simulate the cert getting renewed and the pods getting restarted |
| foundTLSSecret := &corev1.Secret{} |
| err := testClient.Get(ctx, types.NamespacedName{Name: secretName, Namespace: instance.Namespace}, foundTLSSecret) |
| g.Expect(err).NotTo(gomega.HaveOccurred()) |
| expectedCertMd5 := fmt.Sprintf("%x", md5.Sum(foundTLSSecret.Data[util.TLSCertKey])) |
| assert.Equal(t, expectedCertMd5, sts.Spec.Template.ObjectMeta.Annotations[certAnnotation], |
| "TLS cert MD5 annotation on STS does not match the secret") |
| |
| // change the tls.crt which should trigger a rolling restart |
| updatedTlsCertData := "certificate renewed" |
| foundTLSSecret.Data[util.TLSCertKey] = []byte(updatedTlsCertData) |
| err = testClient.Update(context.TODO(), foundTLSSecret) |
| g.Expect(err).NotTo(gomega.HaveOccurred()) |
| |
| // capture all reconcile requests |
| helper.WaitForReconcile(2) |
| |
| // Check the annotation on the pod template to make sure a rolling restart will take place |
| time.Sleep(time.Millisecond * 250) |
| sts = expectStatefulSetTLSConfig(t, g, instance, needsPkcs12InitContainer) |
| expectedCertMd5 = fmt.Sprintf("%x", md5.Sum(foundTLSSecret.Data[util.TLSCertKey])) |
| assert.Equal(t, expectedCertMd5, sts.Spec.Template.ObjectMeta.Annotations[certAnnotation], |
| "TLS cert MD5 annotation on STS does not match the UPDATED secret") |
| } |
| |
| func TestTLSCommonIngressTermination(t *testing.T) { |
| // now, update the config to enable TLS |
| tlsSecretName := "tls-cert-secret-from-user" |
| |
| instance := buildTestSolrCloud() |
| instance.Spec.SolrSecurity = &solr.SolrSecurityOptions{AuthenticationType: solr.Basic} |
| instance.Spec.SolrAddressability.External.IngressTLSTerminationSecret = tlsSecretName |
| |
| changed := instance.WithDefaults() |
| assert.True(t, changed, "WithDefaults should have changed the test SolrCloud instance") |
| |
| g := gomega.NewGomegaWithT(t) |
| helper := NewTLSTestHelper(g) |
| defer func() { |
| helper.StopTest() |
| }() |
| |
| ctx := context.TODO() |
| cleanupTest(g, instance.Namespace) |
| helper.ReconcileSolrCloud(ctx, instance, 2, util.DefaultPkcs12KeystoreFile) |
| defer testClient.Delete(ctx, instance) |
| |
| expectTerminateIngressTLSConfig(t, g, tlsSecretName, false) |
| |
| // Check that the Addresses in the status are correct |
| g.Eventually(func() error { |
| return testClient.Get(context.TODO(), types.NamespacedName{Name: instance.Name, Namespace: instance.Namespace}, instance) |
| }, timeout).Should(gomega.Succeed()) |
| assert.Equal(t, "http://"+instance.Name+"-solrcloud-common."+instance.Namespace, instance.Status.InternalCommonAddress, "Wrong internal common address in status") |
| if assert.NotNil(t, instance.Status.ExternalCommonAddress, "External common address in Status should not be nil.") { |
| assert.EqualValues(t, "https://"+instance.Namespace+"-"+instance.Name+"-solrcloud."+testDomain, *instance.Status.ExternalCommonAddress, "Wrong external common address in status") |
| } |
| } |
| |
| func expectStatefulSetMountedTLSDirConfig(t *testing.T, g *gomega.GomegaWithT, sc *solr.SolrCloud) *appsv1.StatefulSet { |
| ctx := context.TODO() |
| stateful := &appsv1.StatefulSet{} |
| g.Eventually(func() error { return testClient.Get(ctx, expectedStatefulSetName, stateful) }, timeout).Should(gomega.Succeed()) |
| podTemplate := &stateful.Spec.Template |
| expectMountedTLSDirConfigOnPodTemplate(t, podTemplate, sc) |
| |
| // Check HTTPS cluster prop setup container |
| assert.NotNil(t, podTemplate.Spec.InitContainers) |
| expectZkSetupInitContainerForTLS(t, sc, stateful) |
| |
| // should have a mount for the initdb on main container |
| expectInitdbVolumeMount(t, podTemplate) |
| |
| // verify initContainer to create initdb script to export the keystore & truststore passwords before launching the main container |
| name := "export-tls-password" |
| expInitContainer := expectInitContainer(t, podTemplate, name, "initdb", util.InitdbPath) |
| assert.Equal(t, 3, len(expInitContainer.Command), "Wrong command length for "+name+" init container") |
| assert.Contains(t, expInitContainer.Command[2], "SOLR_SSL_KEY_STORE_PASSWORD", "Wrong shell command for "+name+": "+expInitContainer.Command[2]) |
| assert.Contains(t, expInitContainer.Command[2], "SOLR_SSL_TRUST_STORE_PASSWORD", "Wrong shell command for "+name+": "+expInitContainer.Command[2]) |
| |
| if sc.Spec.SolrClientTLS != nil && sc.Spec.SolrClientTLS.MountedTLSDir != nil { |
| assert.Contains(t, expInitContainer.Command[2], "SOLR_SSL_CLIENT_KEY_STORE_PASSWORD", "Wrong shell command for "+name+": "+expInitContainer.Command[2]) |
| assert.Contains(t, expInitContainer.Command[2], "SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD", "Wrong shell command for "+name+": "+expInitContainer.Command[2]) |
| } else { |
| assert.NotContains(t, expInitContainer.Command[2], "SOLR_SSL_CLIENT_KEY_STORE_PASSWORD", "Wrong shell command for "+name+": "+expInitContainer.Command[2]) |
| assert.NotContains(t, expInitContainer.Command[2], "SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD", "Wrong shell command for "+name+": "+expInitContainer.Command[2]) |
| } |
| |
| return stateful |
| } |
| |
| func expectMountedTLSDirConfigOnPodTemplate(t *testing.T, podTemplate *corev1.PodTemplateSpec, sc *solr.SolrCloud) { |
| assert.NotNil(t, podTemplate.Spec.Containers) |
| assert.True(t, len(podTemplate.Spec.Containers) > 0) |
| mainContainer := podTemplate.Spec.Containers[0] |
| assert.NotNil(t, mainContainer, "Didn't find the main solrcloud-node container in the sts!") |
| assert.NotNil(t, mainContainer.Env, "No Env vars for main solrcloud-node container in the sts!") |
| expectMountedTLSDirEnvVars(t, mainContainer.Env, sc) |
| |
| // verify the probes use a command with SSL opts |
| tlsJavaToolOpts, tlsJavaSysProps := "", "" |
| // if there's a client cert, then the probe should use that, else uses the server cert |
| if sc.Spec.SolrClientTLS != nil && sc.Spec.SolrClientTLS.MountedTLSDir != nil { |
| expectedKeystorePasswordFile := sc.Spec.SolrClientTLS.MountedTLSDir.Path + "/" + sc.Spec.SolrClientTLS.MountedTLSDir.KeystorePasswordFile |
| expectedTruststorePasswordFile := sc.Spec.SolrClientTLS.MountedTLSDir.Path + "/" |
| if sc.Spec.SolrClientTLS.MountedTLSDir.TruststorePasswordFile != "" { |
| expectedTruststorePasswordFile += sc.Spec.SolrClientTLS.MountedTLSDir.TruststorePasswordFile |
| } else { |
| expectedTruststorePasswordFile += sc.Spec.SolrClientTLS.MountedTLSDir.KeystorePasswordFile |
| } |
| |
| tlsJavaToolOpts = "-Djavax.net.ssl.keyStorePassword=$(cat " + expectedKeystorePasswordFile + ") " + |
| "-Djavax.net.ssl.trustStorePassword=$(cat " + expectedTruststorePasswordFile + ")" |
| tlsJavaSysProps = "-Djavax.net.ssl.trustStore=$SOLR_SSL_CLIENT_TRUST_STORE -Djavax.net.ssl.keyStore=$SOLR_SSL_CLIENT_KEY_STORE" |
| } else { |
| expectedKeystorePasswordFile := sc.Spec.SolrTLS.MountedTLSDir.Path + "/" + sc.Spec.SolrTLS.MountedTLSDir.KeystorePasswordFile |
| expectedTruststorePasswordFile := sc.Spec.SolrTLS.MountedTLSDir.Path + "/" |
| if sc.Spec.SolrTLS.MountedTLSDir.TruststorePasswordFile != "" { |
| expectedTruststorePasswordFile += sc.Spec.SolrTLS.MountedTLSDir.TruststorePasswordFile |
| } else { |
| expectedTruststorePasswordFile += sc.Spec.SolrTLS.MountedTLSDir.KeystorePasswordFile |
| } |
| |
| tlsJavaToolOpts = "-Djavax.net.ssl.keyStorePassword=$(cat " + expectedKeystorePasswordFile + ") " + |
| "-Djavax.net.ssl.trustStorePassword=$(cat " + expectedTruststorePasswordFile + ")" |
| tlsJavaSysProps = "-Djavax.net.ssl.trustStore=$SOLR_SSL_TRUST_STORE -Djavax.net.ssl.keyStore=$SOLR_SSL_KEY_STORE" |
| } |
| |
| assert.NotNil(t, mainContainer.LivenessProbe, "main container should have a liveness probe defined") |
| assert.NotNil(t, mainContainer.LivenessProbe.Exec, "liveness probe should have an exec when mTLS is enabled") |
| assert.True(t, strings.Contains(mainContainer.LivenessProbe.Exec.Command[2], tlsJavaToolOpts), "liveness probe should invoke java with SSL opts") |
| assert.True(t, strings.Contains(mainContainer.LivenessProbe.Exec.Command[2], tlsJavaSysProps), "liveness probe should invoke java with SSL opts") |
| assert.NotNil(t, mainContainer.ReadinessProbe, "main container should have a readiness probe defined") |
| assert.NotNil(t, mainContainer.ReadinessProbe.Exec, "readiness probe should have an exec when mTLS is enabled") |
| assert.True(t, strings.Contains(mainContainer.ReadinessProbe.Exec.Command[2], tlsJavaToolOpts), "readiness probe should invoke java with SSL opts") |
| assert.True(t, strings.Contains(mainContainer.ReadinessProbe.Exec.Command[2], tlsJavaSysProps), "readiness probe should invoke java with SSL opts") |
| } |
| |
| // ensures the TLS settings are applied correctly to the STS |
| func expectStatefulSetTLSConfig(t *testing.T, g *gomega.GomegaWithT, sc *solr.SolrCloud, needsPkcs12InitContainer bool) *appsv1.StatefulSet { |
| ctx := context.TODO() |
| stateful := &appsv1.StatefulSet{} |
| g.Eventually(func() error { return testClient.Get(ctx, expectedStatefulSetName, stateful) }, timeout).Should(gomega.Succeed()) |
| podTemplate := &stateful.Spec.Template |
| expectTLSConfigOnPodTemplate(t, sc.Spec.SolrTLS, podTemplate, needsPkcs12InitContainer, false, sc.Spec.SolrClientTLS) |
| |
| // Check HTTPS cluster prop setup container |
| assert.NotNil(t, podTemplate.Spec.InitContainers) |
| expectZkSetupInitContainerForTLS(t, sc, stateful) |
| |
| return stateful |
| } |
| |
| func expectZkSetupInitContainerForTLS(t *testing.T, sc *solr.SolrCloud, sts *appsv1.StatefulSet) { |
| var zkSetupInitContainer *corev1.Container = nil |
| podTemplate := &sts.Spec.Template |
| for _, cnt := range podTemplate.Spec.InitContainers { |
| if cnt.Name == "setup-zk" { |
| zkSetupInitContainer = &cnt |
| break |
| } |
| } |
| expCmd := "/opt/solr/server/scripts/cloud-scripts/zkcli.sh -zkhost ${ZK_HOST} -cmd clusterprop -name urlScheme -val https" |
| if sc.Spec.SolrTLS != nil { |
| assert.NotNil(t, zkSetupInitContainer, "Didn't find the zk-setup InitContainer in the sts!") |
| if zkSetupInitContainer != nil { |
| assert.Equal(t, sts.Spec.Template.Spec.Containers[0].Image, zkSetupInitContainer.Image, "The zk-setup init container should use the same image as the Solr container") |
| assert.Equal(t, 3, len(zkSetupInitContainer.Command), "Wrong command length for zk-setup init container") |
| assert.Contains(t, zkSetupInitContainer.Command[2], expCmd, "ZK Setup command does not set urlScheme") |
| expNumVars := 3 |
| if sc.Spec.SolrSecurity != nil && sc.Spec.SolrSecurity.BasicAuthSecret == "" { |
| expNumVars = 4 // one more for SECURITY_JSON |
| } |
| assert.Equal(t, expNumVars, len(zkSetupInitContainer.Env), "Wrong number of envVars for zk-setup init container") |
| } |
| } else { |
| assert.Nil(t, zkSetupInitContainer, "Shouldn't find the zk-setup InitContainer in the sts, when not using https!") |
| } |
| } |
| |
| func verifyReconcileWithSecurity(t *testing.T, instance *solr.SolrCloud, deleteBootstrapSecret bool, useZkCRD bool) { |
| g := gomega.NewGomegaWithT(t) |
| ctx := context.TODO() |
| helper := NewTLSTestHelper(g, useZkCRD) |
| defer func() { |
| helper.StopTest() |
| }() |
| |
| cleanupTest(g, instance.Namespace) |
| |
| // is there a user-provided secret for basic auth creds? |
| if instance.Spec.SolrSecurity != nil && instance.Spec.SolrSecurity.BasicAuthSecret != "" { |
| basicAuthSecret := createBasicAuthSecret(instance.Spec.SolrSecurity.BasicAuthSecret, solr.DefaultBasicAuthUsername, instance.Namespace) |
| err := testClient.Create(ctx, basicAuthSecret) |
| g.Expect(err).NotTo(gomega.HaveOccurred()) |
| defer testClient.Delete(ctx, basicAuthSecret) |
| } |
| |
| // now try to reconcile |
| helper.ReconcileSolrCloud(ctx, instance, 2, util.DefaultPkcs12KeystoreFile) |
| defer testClient.Delete(ctx, instance) |
| |
| expectBootstrapSecret := instance.Spec.SolrSecurity.BasicAuthSecret == "" |
| expectStatefulSetBasicAuthConfig(t, g, instance, expectBootstrapSecret) |
| |
| if deleteBootstrapSecret { |
| bootstrapSecret := &corev1.Secret{} |
| err := testClient.Get(ctx, types.NamespacedName{Name: instance.SecurityBootstrapSecretName(), Namespace: instance.Namespace}, bootstrapSecret) |
| g.Expect(err).NotTo(gomega.HaveOccurred()) |
| // this should trigger a reconcile ... |
| testClient.Delete(ctx, bootstrapSecret) |
| helper.WaitForReconcile(3) |
| expectStatefulSetBasicAuthConfig(t, g, instance, false /* bootstrap secret not found */) |
| } |
| } |
| |
| func expectStatefulSetBasicAuthConfig(t *testing.T, g *gomega.GomegaWithT, sc *solr.SolrCloud, expectBootstrapSecret bool) *appsv1.StatefulSet { |
| assert.NotNil(t, sc.Spec.SolrSecurity, "solrSecurity is not configured for this SolrCloud instance!") |
| |
| ctx := context.TODO() |
| stateful := &appsv1.StatefulSet{} |
| g.Eventually(func() error { return testClient.Get(ctx, expectedStatefulSetName, stateful) }, timeout).Should(gomega.Succeed()) |
| podTemplate := &stateful.Spec.Template |
| expectBasicAuthConfigOnPodTemplate(t, sc, podTemplate, expectBootstrapSecret) |
| |
| basicAuthSecret := &corev1.Secret{} |
| err := testClient.Get(ctx, types.NamespacedName{Name: sc.BasicAuthSecretName(), Namespace: sc.Namespace}, basicAuthSecret) |
| g.Expect(err).NotTo(gomega.HaveOccurred()) |
| assert.Equal(t, solr.DefaultBasicAuthUsername, string(basicAuthSecret.Data[corev1.BasicAuthUsernameKey]), "password should be set for k8s-oper in the basic auth secret") |
| assert.True(t, basicAuthSecret.Data[corev1.BasicAuthPasswordKey] != nil && len(basicAuthSecret.Data[corev1.BasicAuthPasswordKey]) > 0, "password should be set for k8s-oper in the basic auth secret") |
| |
| // verify a security.json gets bootstrapped if not using a user-provided secret |
| if sc.Spec.SolrSecurity.BasicAuthSecret == "" && expectBootstrapSecret { |
| bootstrapSecret := &corev1.Secret{} |
| err := testClient.Get(ctx, types.NamespacedName{Name: sc.SecurityBootstrapSecretName(), Namespace: sc.Namespace}, bootstrapSecret) |
| g.Expect(err).NotTo(gomega.HaveOccurred()) |
| assert.True(t, bootstrapSecret.Data["admin"] != nil && len(bootstrapSecret.Data["admin"]) > 0, "password should be set for admin in the bootstrap security secret") |
| assert.True(t, bootstrapSecret.Data["solr"] != nil && len(bootstrapSecret.Data["solr"]) > 0, "password should be set for solr in the bootstrap security secret") |
| assert.True(t, bootstrapSecret.Data["security.json"] != nil && len(bootstrapSecret.Data["security.json"]) > 0, "security.json not found in the bootstrap security secret") |
| |
| if sc.Spec.CustomSolrKubeOptions.PodOptions != nil { |
| probePaths := util.GetCustomProbePaths(sc) |
| if len(probePaths) > 0 { |
| securityJson := string(bootstrapSecret.Data["security.json"]) |
| assert.True(t, strings.Contains(securityJson, util.DefaultProbePath), "bootstrapped security.json should have an authz rule for "+util.DefaultProbePath) |
| for _, p := range probePaths { |
| p = p[len("/solr"):] // drop the /solr part on the path |
| assert.True(t, strings.Contains(securityJson, p), "bootstrapped security.json should have an authz rule for "+p) |
| } |
| } |
| } |
| } |
| |
| return stateful |
| } |
| |
| func expectPassthroughIngressTLSConfig(t *testing.T, g *gomega.GomegaWithT, expectedTLSSecretName string, solrTLS *solr.SolrTLSOptions) { |
| ingress := &netv1.Ingress{} |
| g.Eventually(func() error { return testClient.Get(context.TODO(), expectedIngressWithTLS, ingress) }, timeout).Should(gomega.Succeed()) |
| assert.True(t, ingress.Spec.TLS != nil && len(ingress.Spec.TLS) == 1, "Wrong number of TLS Secrets for ingress") |
| assert.Equal(t, expectedTLSSecretName, ingress.Spec.TLS[0].SecretName, "Wrong secretName for ingress TLS") |
| if solrTLS != nil { |
| assert.Equal(t, "HTTPS", ingress.ObjectMeta.Annotations["nginx.ingress.kubernetes.io/backend-protocol"], "Ingress Backend Protocol annotation incorrect") |
| } else { |
| assert.Equal(t, "HTTP", ingress.ObjectMeta.Annotations["nginx.ingress.kubernetes.io/backend-protocol"], "Ingress Backend Protocol annotation incorrect") |
| } |
| if len(ingress.Spec.TLS) > 0 { |
| assert.Equal(t, "true", ingress.ObjectMeta.Annotations["nginx.ingress.kubernetes.io/ssl-redirect"], "Ingress SSL Redirect annotation incorrect") |
| } else { |
| assert.Equal(t, "", ingress.ObjectMeta.Annotations["nginx.ingress.kubernetes.io/ssl-redirect"], "Ingress SSL Redirect annotation incorrect") |
| } |
| } |
| |
| func expectTerminateIngressTLSConfig(t *testing.T, g *gomega.GomegaWithT, expectedTLSSecretName string, isBackendTls bool) { |
| ingress := &netv1.Ingress{} |
| g.Eventually(func() error { return testClient.Get(context.TODO(), expectedIngressWithTLS, ingress) }, timeout).Should(gomega.Succeed()) |
| assert.Equal(t, 1, len(ingress.Spec.TLS), "Wrong number of TLS Secrets for ingress") |
| assert.Equal(t, expectedTLSSecretName, ingress.Spec.TLS[0].SecretName, "Wrong secretName for ingress TLS") |
| assert.Equal(t, 2, len(ingress.Spec.TLS[0].Hosts), "Wrong number of hosts for Ingress TLS termination") |
| assert.Equal(t, "default-foo-tls-solrcloud."+testDomain, ingress.Spec.TLS[0].Hosts[0], "Wrong common-host name for Ingress TLS termination") |
| assert.Equal(t, "default-foo-tls-solrcloud-0."+testDomain, ingress.Spec.TLS[0].Hosts[1], "Wrong common-host name for Ingress TLS termination") |
| |
| if isBackendTls { |
| assert.Equal(t, "HTTPS", ingress.ObjectMeta.Annotations["nginx.ingress.kubernetes.io/backend-protocol"], "Ingress Backend Protocol annotation incorrect") |
| } else { |
| assert.Equal(t, "HTTP", ingress.ObjectMeta.Annotations["nginx.ingress.kubernetes.io/backend-protocol"], "Ingress Backend Protocol annotation incorrect") |
| } |
| assert.Equal(t, "true", ingress.ObjectMeta.Annotations["nginx.ingress.kubernetes.io/ssl-redirect"], "Ingress SSL Redirect annotation incorrect") |
| } |
| |
| // Ensures config is setup for basic-auth enabled Solr pods |
| func expectBasicAuthConfigOnPodTemplate(t *testing.T, instance *solr.SolrCloud, podTemplate *corev1.PodTemplateSpec, expectBootstrapSecret bool) *corev1.Container { |
| |
| // check the env vars needed for the probes to work with auth |
| assert.NotNil(t, podTemplate.Spec.Containers) |
| assert.True(t, len(podTemplate.Spec.Containers) > 0) |
| mainContainer := podTemplate.Spec.Containers[0] |
| assert.NotNil(t, mainContainer, "Didn't find the main solrcloud-node container in the sts!") |
| assert.NotNil(t, mainContainer.Env, "Didn't find the main solrcloud-node container in the sts!") |
| |
| // probes with auth |
| if instance.Spec.SolrSecurity.ProbesRequireAuth { |
| assert.NotNil(t, podTemplate.Spec.Volumes) |
| secretName := instance.BasicAuthSecretName() |
| var basicAuthSecretVol *corev1.Volume = nil |
| for _, vol := range podTemplate.Spec.Volumes { |
| if vol.Name == secretName { |
| basicAuthSecretVol = &vol |
| break |
| } |
| } |
| assert.NotNil(t, basicAuthSecretVol) |
| assert.NotNil(t, basicAuthSecretVol.VolumeSource.Secret, "Didn't find the basic auth secret volume in sts config!") |
| assert.Equal(t, instance.BasicAuthSecretName(), basicAuthSecretVol.VolumeSource.Secret.SecretName) |
| |
| var basicAuthSecretVolMount *corev1.VolumeMount = nil |
| for _, m := range podTemplate.Spec.Containers[0].VolumeMounts { |
| if m.Name == secretName { |
| basicAuthSecretVolMount = &m |
| break |
| } |
| } |
| assert.NotNil(t, basicAuthSecretVolMount) |
| assert.Equal(t, "/etc/secrets/"+secretName, basicAuthSecretVolMount.MountPath) |
| |
| expProbeCmd := "JAVA_TOOL_OPTIONS=\"-Dbasicauth=$(cat /etc/secrets/foo-tls-solrcloud-basic-auth/username):$(cat /etc/secrets/foo-tls-solrcloud-basic-auth/password)\" java " + |
| "-Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory " + |
| "-Dsolr.install.dir=\"/opt/solr\" -Dlog4j.configurationFile=\"/opt/solr/server/resources/log4j2-console.xml\" " + |
| "-classpath \"/opt/solr/server/solr-webapp/webapp/WEB-INF/lib/*:/opt/solr/server/lib/ext/*:/opt/solr/server/lib/*\" " + |
| "org.apache.solr.util.SolrCLI api -get http://localhost:8983/solr/admin/info/system" |
| if assert.NotNil(t, mainContainer.LivenessProbe, "main container should have a liveness probe defined") { |
| if assert.NotNil(t, mainContainer.LivenessProbe.Exec, "liveness probe should have an exec when auth is enabled") { |
| assert.Equal(t, expProbeCmd, mainContainer.LivenessProbe.Exec.Command[2], "liveness probe should invoke java with auth opts") |
| } |
| assert.EqualValues(t, 5, mainContainer.LivenessProbe.TimeoutSeconds, "liveness probe default timeout should be increased when using basicAuth") |
| } |
| if assert.NotNil(t, mainContainer.ReadinessProbe, "main container should have a readiness probe defined") { |
| if assert.NotNil(t, mainContainer.ReadinessProbe.Exec, "readiness probe should have an exec when auth is enabled") { |
| assert.Equal(t, expProbeCmd, mainContainer.ReadinessProbe.Exec.Command[2], "readiness probe should invoke java with auth opts") |
| } |
| } |
| } |
| |
| // if no user-provided auth secret, then check that security.json gets bootstrapped correctly |
| if instance.Spec.SolrSecurity.BasicAuthSecret == "" { |
| // initContainers |
| assert.NotNil(t, podTemplate.Spec.InitContainers) |
| var expInitContainer *corev1.Container = nil |
| for _, cnt := range podTemplate.Spec.InitContainers { |
| if cnt.Name == "setup-zk" { |
| expInitContainer = &cnt |
| break |
| } |
| } |
| |
| if expectBootstrapSecret { |
| // if the zookeeperRef has ACLs set, verify the env vars were set correctly for this initContainer |
| allACL, _ := instance.Spec.ZookeeperRef.GetACLs() |
| if allACL != nil { |
| assert.Equal(t, 10, len(expInitContainer.Env)) |
| assert.Equal(t, "SOLR_OPTS", expInitContainer.Env[len(expInitContainer.Env)-2].Name) |
| assert.Equal(t, "SECURITY_JSON", expInitContainer.Env[len(expInitContainer.Env)-1].Name) |
| testACLEnvVars(t, expInitContainer.Env[3:len(expInitContainer.Env)-2]) |
| } // else this ref not using ACLs |
| |
| assert.NotNil(t, expInitContainer, "Didn't find the setup-zk InitContainer in the sts!") |
| expCmd := "ZK_SECURITY_JSON=$(/opt/solr/server/scripts/cloud-scripts/zkcli.sh -zkhost ${ZK_HOST} -cmd get /security.json); " + |
| "if [ ${#ZK_SECURITY_JSON} -lt 3 ]; then " + |
| "echo $SECURITY_JSON > /tmp/security.json; " + |
| "/opt/solr/server/scripts/cloud-scripts/zkcli.sh -zkhost ${ZK_HOST} -cmd putfile /security.json /tmp/security.json; echo \"put security.json in ZK\"; fi" |
| assert.True(t, strings.Contains(expInitContainer.Command[2], expCmd), "setup-zk initContainer not configured to bootstrap security.json!") |
| } else { |
| assert.True(t, expInitContainer == nil || !strings.Contains(expInitContainer.Command[2], "SECURITY_JSON"), |
| "setup-zk initContainer not reconciled after bootstrap secret deleted") |
| } |
| |
| } |
| |
| return &mainContainer // return as a convenience in case tests want to do more checking on the main container |
| } |
| |
| func buildTestSolrCloud() *solr.SolrCloud { |
| replicas := int32(1) |
| instance := &solr.SolrCloud{ |
| ObjectMeta: metav1.ObjectMeta{Name: expectedCloudWithTLSRequest.Name, Namespace: expectedCloudWithTLSRequest.Namespace}, |
| Spec: solr.SolrCloudSpec{ |
| Replicas: &replicas, |
| ZookeeperRef: &solr.ZookeeperRef{ |
| ConnectionInfo: &solr.ZookeeperConnectionInfo{ |
| InternalConnectionString: "host:7271", |
| }, |
| }, |
| SolrAddressability: solr.SolrAddressabilityOptions{ |
| External: &solr.ExternalAddressability{ |
| Method: solr.Ingress, |
| UseExternalAddress: true, |
| DomainName: testDomain, |
| }, |
| }, |
| }, |
| } |
| return instance |
| } |
| |
| // Consolidate common reconcile TLS test setup and behavior |
| type TLSTestHelper struct { |
| g *gomega.GomegaWithT |
| mgr manager.Manager |
| requests chan reconcile.Request |
| stopMgr chan struct{} |
| mgrStopped *sync.WaitGroup |
| } |
| |
| func NewTLSTestHelper(g *gomega.GomegaWithT, useZkCRD ...bool) *TLSTestHelper { |
| if useZkCRD != nil && useZkCRD[0] { |
| UseZkCRD(true) |
| } else { |
| UseZkCRD(false) |
| } |
| |
| // Setup the Manager and Controller. Wrap the Controller Reconcile function so it writes each request to a |
| // channel when it is finished. |
| mgr, err := manager.New(testCfg, manager.Options{}) |
| g.Expect(err).NotTo(gomega.HaveOccurred()) |
| testClient = mgr.GetClient() |
| |
| solrCloudReconciler := &SolrCloudReconciler{ |
| Client: testClient, |
| Log: ctrl.Log.WithName("controllers").WithName("SolrCloud"), |
| } |
| newRec, requests := SetupTestReconcile(solrCloudReconciler) |
| |
| g.Expect(solrCloudReconciler.SetupWithManagerAndReconciler(mgr, newRec)).NotTo(gomega.HaveOccurred()) |
| stopMgr, mgrStopped := StartTestManager(mgr, g) |
| |
| return &TLSTestHelper{ |
| g: g, |
| mgr: mgr, |
| requests: requests, |
| stopMgr: stopMgr, |
| mgrStopped: mgrStopped, |
| } |
| } |
| |
| func (helper *TLSTestHelper) StopTest() { |
| close(helper.stopMgr) |
| helper.mgrStopped.Wait() |
| } |
| |
| func (helper *TLSTestHelper) WaitForReconcile(expectedRequests int) { |
| g := helper.g |
| requests := helper.requests |
| for r := 0; r < expectedRequests; r++ { |
| g.Eventually(requests, timeout).Should(gomega.Receive(gomega.Equal(expectedCloudWithTLSRequest))) |
| } |
| // clear so we can wait for update reconcile to occur |
| emptyRequests(requests) |
| } |
| |
| func (helper *TLSTestHelper) ReconcileSolrCloud(ctx context.Context, instance *solr.SolrCloud, expectedRequests int, keyInSecret string) { |
| g := helper.g |
| |
| // trigger the reconcile process and then wait for reconcile to finish |
| // expectedRequests gives the expected number of requests created during the reconcile process |
| err := testClient.Create(ctx, instance) |
| g.Expect(err).NotTo(gomega.HaveOccurred()) |
| |
| // Create a mock secret in the background so the isCert ready function returns |
| var wg sync.WaitGroup |
| if instance.Spec.SolrTLS != nil && instance.Spec.SolrTLS.PKCS12Secret != nil { |
| wg.Add(1) |
| go func() { |
| createMockTLSSecret(ctx, testClient, instance.Spec.SolrTLS.PKCS12Secret.Name, keyInSecret, |
| instance.Namespace, instance.Spec.SolrTLS.KeyStorePasswordSecret.Key, "") |
| wg.Done() |
| }() |
| } |
| |
| // need a secret for the client cert too? |
| if instance.Spec.SolrClientTLS != nil && instance.Spec.SolrClientTLS.PKCS12Secret != nil { |
| truststoreKey := "" |
| if instance.Spec.SolrClientTLS.TrustStoreSecret == instance.Spec.SolrClientTLS.PKCS12Secret { |
| truststoreKey = instance.Spec.SolrClientTLS.TrustStoreSecret.Key |
| } |
| wg.Add(1) |
| go func() { |
| createMockTLSSecret(ctx, testClient, instance.Spec.SolrClientTLS.PKCS12Secret.Name, util.DefaultPkcs12KeystoreFile, |
| instance.Namespace, instance.Spec.SolrClientTLS.KeyStorePasswordSecret.Key, truststoreKey) |
| wg.Done() |
| }() |
| } |
| // reconcile the SolrCloud ~ the TLS secrets get created in the background so it should eventually reconcile ... |
| helper.WaitForReconcile(expectedRequests) |
| |
| wg.Wait() |
| |
| stateful := &appsv1.StatefulSet{} |
| g.Eventually(func() error { return testClient.Get(ctx, expectedStatefulSetName, stateful) }, timeout).Should(gomega.Succeed()) |
| } |
| |
| // Ensures all the TLS env vars, volume mounts and initContainers are setup for the PodTemplateSpec |
| func expectTLSConfigOnPodTemplate(t *testing.T, tls *solr.SolrTLSOptions, podTemplate *corev1.PodTemplateSpec, needsPkcs12InitContainer bool, clientOnly bool, clientTLS *solr.SolrTLSOptions) *corev1.Container { |
| assert.NotNil(t, podTemplate.Spec.Volumes) |
| |
| if tls.PKCS12Secret != nil { |
| var keystoreVol *corev1.Volume = nil |
| for _, vol := range podTemplate.Spec.Volumes { |
| if vol.Name == "keystore" { |
| keystoreVol = &vol |
| break |
| } |
| } |
| assert.NotNil(t, keystoreVol, fmt.Sprintf("keystore volume not found in pod template; volumes: %v", podTemplate.Spec.Volumes)) |
| assert.NotNil(t, keystoreVol.VolumeSource.Secret, "Didn't find TLS keystore volume in sts config! keystoreVol: "+keystoreVol.String()) |
| assert.Equal(t, tls.PKCS12Secret.Name, keystoreVol.VolumeSource.Secret.SecretName) |
| } |
| |
| // check the SOLR_SSL_ related env vars on the sts |
| assert.NotNil(t, podTemplate.Spec.Containers) |
| assert.True(t, len(podTemplate.Spec.Containers) > 0) |
| mainContainer := podTemplate.Spec.Containers[0] |
| assert.NotNil(t, mainContainer, "Didn't find the main solrcloud-node container in the sts!") |
| assert.NotNil(t, mainContainer.Env, "Didn't find the main solrcloud-node container in the sts!") |
| |
| // is there a separate truststore? |
| expectedTrustStorePath := "" |
| if tls.TrustStoreSecret != nil { |
| expectedTrustStorePath = util.DefaultTrustStorePath + "/" + tls.TrustStoreSecret.Key |
| } |
| |
| if tls.PKCS12Secret != nil { |
| expectTLSEnvVars(t, mainContainer.Env, tls.KeyStorePasswordSecret.Name, tls.KeyStorePasswordSecret.Key, needsPkcs12InitContainer, expectedTrustStorePath, clientOnly, clientTLS) |
| } else if tls.TrustStoreSecret != nil { |
| envVars := mainContainer.Env |
| assert.NotNil(t, envVars) |
| envVars = filterVarsByName(envVars, func(n string) bool { |
| return strings.HasPrefix(n, "SOLR_SSL_") |
| }) |
| assert.Equal(t, 3, len(envVars)) |
| for _, envVar := range envVars { |
| if envVar.Name == "SOLR_SSL_CLIENT_TRUST_STORE" { |
| assert.Equal(t, expectedTrustStorePath, envVar.Value) |
| } |
| if envVar.Name == "SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD" { |
| assert.NotNil(t, envVar.ValueFrom) |
| assert.NotNil(t, envVar.ValueFrom.SecretKeyRef) |
| assert.Equal(t, tls.TrustStorePasswordSecret.Name, envVar.ValueFrom.SecretKeyRef.Name) |
| assert.Equal(t, tls.TrustStorePasswordSecret.Key, envVar.ValueFrom.SecretKeyRef.Key) |
| } |
| } |
| } |
| |
| // different trust store? |
| if tls.TrustStoreSecret != nil { |
| var truststoreVol *corev1.Volume = nil |
| for _, vol := range podTemplate.Spec.Volumes { |
| if vol.Name == "truststore" { |
| truststoreVol = &vol |
| break |
| } |
| } |
| assert.NotNil(t, truststoreVol, fmt.Sprintf("truststore volume not found in pod template; volumes: %v", podTemplate.Spec.Volumes)) |
| assert.NotNil(t, truststoreVol.VolumeSource.Secret, "Didn't find TLS truststore volume in sts config!") |
| assert.Equal(t, tls.TrustStoreSecret.Name, truststoreVol.VolumeSource.Secret.SecretName) |
| } |
| |
| // initContainers |
| if needsPkcs12InitContainer { |
| var pkcs12Vol *corev1.Volume = nil |
| for _, vol := range podTemplate.Spec.Volumes { |
| if vol.Name == "pkcs12" { |
| pkcs12Vol = &vol |
| break |
| } |
| } |
| |
| assert.NotNil(t, pkcs12Vol, "Didn't find TLS keystore volume in sts config!") |
| assert.NotNil(t, pkcs12Vol.EmptyDir, "pkcs12 vol should by an emptyDir") |
| |
| assert.NotNil(t, podTemplate.Spec.InitContainers) |
| var expInitContainer *corev1.Container = nil |
| for _, cnt := range podTemplate.Spec.InitContainers { |
| if cnt.Name == "gen-pkcs12-keystore" { |
| expInitContainer = &cnt |
| break |
| } |
| } |
| expCmd := "openssl pkcs12 -export -in /var/solr/tls/tls.crt -in /var/solr/tls/ca.crt -inkey /var/solr/tls/tls.key -out /var/solr/tls/pkcs12/keystore.p12 -passout pass:${SOLR_SSL_KEY_STORE_PASSWORD}" |
| assert.NotNil(t, expInitContainer, "Didn't find the gen-pkcs12-keystore InitContainer in the sts!") |
| assert.Equal(t, expCmd, expInitContainer.Command[2]) |
| } |
| |
| if tls.ClientAuth == solr.Need { |
| // verify the probes use a command with SSL opts |
| tlsProps := "" |
| if clientTLS != nil { |
| tlsProps = "-Djavax.net.ssl.trustStore=$SOLR_SSL_CLIENT_TRUST_STORE -Djavax.net.ssl.keyStore=$SOLR_SSL_CLIENT_KEY_STORE" + |
| " -Djavax.net.ssl.keyStorePassword=$SOLR_SSL_CLIENT_KEY_STORE_PASSWORD -Djavax.net.ssl.trustStorePassword=$SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD" |
| } else { |
| tlsProps = "-Djavax.net.ssl.trustStore=$SOLR_SSL_TRUST_STORE -Djavax.net.ssl.keyStore=$SOLR_SSL_KEY_STORE" + |
| " -Djavax.net.ssl.keyStorePassword=$SOLR_SSL_KEY_STORE_PASSWORD -Djavax.net.ssl.trustStorePassword=$SOLR_SSL_TRUST_STORE_PASSWORD" |
| } |
| assert.NotNil(t, mainContainer.LivenessProbe, "main container should have a liveness probe defined") |
| assert.NotNil(t, mainContainer.LivenessProbe.Exec, "liveness probe should have an exec when auth is enabled") |
| assert.True(t, strings.Contains(mainContainer.LivenessProbe.Exec.Command[2], tlsProps), "liveness probe should invoke java with SSL opts") |
| assert.NotNil(t, mainContainer.ReadinessProbe, "main container should have a readiness probe defined") |
| assert.NotNil(t, mainContainer.ReadinessProbe.Exec, "readiness probe should have an exec when auth is enabled") |
| assert.True(t, strings.Contains(mainContainer.ReadinessProbe.Exec.Command[2], tlsProps), "readiness probe should invoke java with SSL opts") |
| } |
| |
| return &mainContainer // return as a convenience in case tests want to do more checking on the main container |
| } |
| |
| func expectMountedTLSDirEnvVars(t *testing.T, envVars []corev1.EnvVar, sc *solr.SolrCloud) { |
| assert.NotNil(t, envVars) |
| envVars = filterVarsByName(envVars, func(n string) bool { |
| return strings.HasPrefix(n, "SOLR_SSL_") |
| }) |
| |
| if sc.Spec.SolrClientTLS != nil { |
| assert.Equal(t, 8, len(envVars), "expected SOLR_SSL and SOLR_SSL_CLIENT related env vars not found") |
| } else { |
| assert.Equal(t, 6, len(envVars), "expected SOLR_SSL related env vars not found") |
| } |
| |
| expectedKeystorePath := sc.Spec.SolrTLS.MountedTLSDir.Path + "/" + sc.Spec.SolrTLS.MountedTLSDir.KeystoreFile |
| expectedTruststorePath := sc.Spec.SolrTLS.MountedTLSDir.Path + "/" + sc.Spec.SolrTLS.MountedTLSDir.TruststoreFile |
| |
| for _, envVar := range envVars { |
| if envVar.Name == "SOLR_SSL_ENABLED" { |
| assert.Equal(t, "true", envVar.Value) |
| } |
| |
| if envVar.Name == "SOLR_SSL_KEY_STORE" { |
| assert.Equal(t, expectedKeystorePath, envVar.Value) |
| } |
| |
| if envVar.Name == "SOLR_SSL_TRUST_STORE" { |
| assert.Equal(t, expectedTruststorePath, envVar.Value) |
| } |
| |
| if envVar.Name == "SOLR_SSL_WANT_CLIENT_AUTH" { |
| assert.Equal(t, "false", envVar.Value) |
| } |
| |
| if envVar.Name == "SOLR_SSL_NEED_CLIENT_AUTH" { |
| assert.Equal(t, "true", envVar.Value) |
| } |
| |
| if envVar.Name == "SOLR_SSL_CHECK_PEER_NAME" { |
| assert.Equal(t, "true", envVar.Value) |
| } |
| } |
| |
| if sc.Spec.SolrClientTLS != nil && sc.Spec.SolrClientTLS.MountedTLSDir != nil { |
| for _, envVar := range envVars { |
| if envVar.Name == "SOLR_SSL_CLIENT_KEY_STORE" { |
| expectedKeystorePath = sc.Spec.SolrClientTLS.MountedTLSDir.Path + "/" + sc.Spec.SolrClientTLS.MountedTLSDir.KeystoreFile |
| assert.Equal(t, expectedKeystorePath, envVar.Value) |
| } |
| |
| if envVar.Name == "SOLR_SSL_CLIENT_TRUST_STORE" { |
| expectedTruststorePath = sc.Spec.SolrClientTLS.MountedTLSDir.Path + "/" + sc.Spec.SolrClientTLS.MountedTLSDir.TruststoreFile |
| assert.Equal(t, expectedTruststorePath, envVar.Value) |
| } |
| |
| } |
| } |
| } |
| |
| // ensure the TLS related env vars are set for the Solr pod |
| func expectTLSEnvVars(t *testing.T, envVars []corev1.EnvVar, expectedKeystorePasswordSecretName string, expectedKeystorePasswordSecretKey string, needsPkcs12InitContainer bool, expectedTruststorePath string, clientOnly bool, clientTLS *solr.SolrTLSOptions) { |
| assert.NotNil(t, envVars) |
| envVars = filterVarsByName(envVars, func(n string) bool { |
| return strings.HasPrefix(n, "SOLR_SSL_") |
| }) |
| |
| if clientOnly { |
| assert.Equal(t, 5, len(envVars)) |
| } else { |
| if clientTLS != nil { |
| assert.Equal(t, 13, len(envVars)) |
| } else { |
| assert.Equal(t, 9, len(envVars)) |
| } |
| } |
| |
| expectedKeystorePath := util.DefaultKeyStorePath + "/keystore.p12" |
| if needsPkcs12InitContainer { |
| expectedKeystorePath = util.DefaultWritableKeyStorePath + "/keystore.p12" |
| } |
| |
| if expectedTruststorePath == "" { |
| expectedTruststorePath = expectedKeystorePath |
| } |
| |
| for _, envVar := range envVars { |
| if envVar.Name == "SOLR_SSL_ENABLED" { |
| assert.Equal(t, "true", envVar.Value) |
| } |
| |
| if envVar.Name == "SOLR_SSL_KEY_STORE" { |
| assert.Equal(t, expectedKeystorePath, envVar.Value) |
| } |
| |
| if envVar.Name == "SOLR_SSL_TRUST_STORE" { |
| assert.Equal(t, expectedTruststorePath, envVar.Value) |
| } |
| |
| if envVar.Name == "SOLR_SSL_KEY_STORE_PASSWORD" { |
| assert.NotNil(t, envVar.ValueFrom) |
| assert.NotNil(t, envVar.ValueFrom.SecretKeyRef) |
| assert.Equal(t, expectedKeystorePasswordSecretName, envVar.ValueFrom.SecretKeyRef.Name) |
| assert.Equal(t, expectedKeystorePasswordSecretKey, envVar.ValueFrom.SecretKeyRef.Key) |
| } |
| } |
| |
| if clientTLS != nil { |
| for _, envVar := range envVars { |
| |
| if envVar.Name == "SOLR_SSL_CLIENT_KEY_STORE" { |
| assert.Equal(t, "/var/solr/client-tls/keystore.p12", envVar.Value) |
| } |
| |
| if envVar.Name == "SOLR_SSL_CLIENT_TRUST_STORE" { |
| assert.Equal(t, "/var/solr/client-tls/truststore.p12", envVar.Value) |
| } |
| |
| if envVar.Name == "SOLR_SSL_CLIENT_KEY_STORE_PASSWORD" || envVar.Name == "SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD" { |
| assert.NotNil(t, envVar.ValueFrom) |
| assert.NotNil(t, envVar.ValueFrom.SecretKeyRef) |
| } |
| } |
| } |
| } |
| |
| func verifyUserSuppliedTLSConfig(t *testing.T, tls *solr.SolrTLSOptions, expectedKeystorePasswordSecretName string, expectedKeystorePasswordSecretKey string, expectedTlsSecretName string) { |
| assert.NotNil(t, tls) |
| assert.Equal(t, expectedKeystorePasswordSecretName, tls.KeyStorePasswordSecret.Name) |
| assert.Equal(t, expectedKeystorePasswordSecretKey, tls.KeyStorePasswordSecret.Key) |
| assert.Equal(t, expectedTlsSecretName, tls.PKCS12Secret.Name) |
| assert.Equal(t, "keystore.p12", tls.PKCS12Secret.Key) |
| } |
| |
| func createTLSOptions(tlsSecretName string, keystorePassKey string, restartOnTLSSecretUpdate bool) *solr.SolrTLSOptions { |
| return &solr.SolrTLSOptions{ |
| KeyStorePasswordSecret: &corev1.SecretKeySelector{ |
| LocalObjectReference: corev1.LocalObjectReference{Name: tlsSecretName}, |
| Key: keystorePassKey, |
| }, |
| PKCS12Secret: &corev1.SecretKeySelector{ |
| LocalObjectReference: corev1.LocalObjectReference{Name: tlsSecretName}, |
| Key: util.DefaultPkcs12KeystoreFile, |
| }, |
| RestartOnTLSSecretUpdate: restartOnTLSSecretUpdate, |
| } |
| } |
| |
| func createMockTLSSecret(ctx context.Context, apiClient client.Client, secretName string, secretKey string, ns string, keystorePasswordKey string, truststoreKey string) (corev1.Secret, error) { |
| secretData := map[string][]byte{} |
| secretData[secretKey] = []byte(b64.StdEncoding.EncodeToString([]byte("mock keystore"))) |
| secretData[util.TLSCertKey] = []byte(b64.StdEncoding.EncodeToString([]byte("mock tls.crt"))) |
| |
| if keystorePasswordKey != "" { |
| secretData[keystorePasswordKey] = []byte(b64.StdEncoding.EncodeToString([]byte("mock keystore password"))) |
| } |
| |
| if truststoreKey != "" { |
| secretData[truststoreKey] = []byte(b64.StdEncoding.EncodeToString([]byte("mock truststore"))) |
| } |
| |
| mockTLSSecret := corev1.Secret{ |
| ObjectMeta: metav1.ObjectMeta{Name: secretName, Namespace: ns}, |
| Data: secretData, |
| Type: corev1.SecretTypeOpaque, |
| } |
| err := apiClient.Create(ctx, &mockTLSSecret) |
| return mockTLSSecret, err |
| } |