submarine-cloud-v3/controllers/submarine_controller_test.go - submarine - Git at Google

 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
  * this work for additional information regarding copyright ownership.
  * The ASF licenses this file to You under the Apache License, Version 2.0
  * (the "License"); you may not use this file except in compliance with
  * the License.  You may obtain a copy of the License at
  *
  *    http://www.apache.org/licenses/LICENSE-2.0
  *
  * Unless required by applicable law or agreed to in writing, software
  * distributed under the License is distributed on an "AS IS" BASIS,
  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */

 package controllers

 import (
 	"context"
 	"fmt"
 	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"os"
 	"path/filepath"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"time"

 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/pkg/errors"
 	istio "istio.io/client-go/pkg/apis/networking/v1alpha3"
 	istiov1alpha3 "istio.io/client-go/pkg/apis/networking/v1alpha3"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/yaml"
 	"sigs.k8s.io/controller-runtime/pkg/client"

 	submarineapacheorgv1 "github.com/apache/submarine/submarine-cloud-v3/api/v1"
 )

 func createScheme() *runtime.Scheme {
 	scheme := runtime.NewScheme()
 	_ = clientgoscheme.AddToScheme(scheme)
 	_ = appsv1.AddToScheme(scheme)
 	_ = corev1.AddToScheme(scheme)
 	_ = submarineapacheorgv1.AddToScheme(scheme)
 	_ = istio.AddToScheme(scheme)
 	_ = serializer.NewCodecFactory(scheme).UniversalDeserializer().Decode
 	return scheme
 }

 // We are mainly testing created resources, so for the time being we have not declared k8s client and recoder
 func createSubmarineReconciler(config ...*SubmarineReconciler) *SubmarineReconciler {
 	namespace := "submarine"
 	istioEnable := false
 	submarineGateway := "submarine/submarine-gateway"
 	seldonIstioEnable := false
 	seldonGateway := "submarine/seldon-gateway"
 	clusterType := "kubernetes"
 	createPodSecurityPolicy := false
 	if len(config) > 0 {
 		if config[0].Namespace != "" {
 			namespace = config[0].Namespace
 		}
 		if &config[0].IstioEnable != nil {
 			istioEnable = config[0].IstioEnable
 		}
 		if config[0].SubmarineGateway != "" {
 			submarineGateway = config[0].SubmarineGateway
 		}
 		if &config[0].SeldonIstioEnable != nil {
 			seldonIstioEnable = config[0].SeldonIstioEnable
 		}
 		if config[0].SeldonGateway != "" {
 			seldonGateway = config[0].SeldonGateway
 		}
 		if config[0].ClusterType != "" {
 			clusterType = config[0].ClusterType
 		}
 		if &config[0].CreatePodSecurityPolicy != nil {
 			createPodSecurityPolicy = config[0].CreatePodSecurityPolicy
 		}
 	}
 	return &SubmarineReconciler{
 		Scheme:                  createScheme(),
 		Log:                     ctrl.Log.WithName("submarine-test"),
 		Namespace:               namespace,
 		IstioEnable:             istioEnable,
 		SubmarineGateway:        submarineGateway,
 		SeldonIstioEnable:       seldonIstioEnable,
 		SeldonGateway:           seldonGateway,
 		ClusterType:             clusterType,
 		CreatePodSecurityPolicy: createPodSecurityPolicy,
 	}
 }

 var _ = Describe("Submarine controller", func() {

 	// Define utility constants and variables
 	const (
 		// The namespaces where the Submarine CRs are created
 		submarineNamespaceDefaultCR = "submarine-test-submit-default-cr"
 		submarineNamespaceCustomCR  = "submarine-test-submit-custom-cr"
 		customHost                  = "submarine-custom-host"
 		customGateway               = "submarine-custom-gateway"

 		createNsTimeout         = time.Second * 10
 		createNsInterval        = time.Second * 2
 		createSubmarineTimeout  = time.Second * 1200
 		createSubmarineInterval = time.Second * 10
 		deleteSubmarineTimeout  = time.Second * 600
 		deleteSubmarineInterval = time.Second * 10
 		deleteNsTimeout         = time.Second * 120
 		deleteNsInterval        = time.Second * 2
 	)
 	var (
 		// The name of Submarine is specified in the YAML file.
 		// Storing name to call k8sClient.Get with NamespacedName
 		submarineNameDefaultCR  string
 		submarineNameCustomCR   string
 		submarineCustomHosts    []string
 		submarineCustomGateways []string

 		ctx = context.Background()
 	)

 	Context("Create test namespaces", func() {
 		It(fmt.Sprintf("Should create namespace %s", submarineNamespaceDefaultCR), func() {
 			By(fmt.Sprintf("Creating the test namespace %s", submarineNamespaceDefaultCR))
 			ns := &corev1.Namespace{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: submarineNamespaceDefaultCR, // Namespace to test default CR
 					Labels: map[string]string{
 						"istio-injection": "enabled",
 					},
 				},
 			}
 			Expect(k8sClient.Create(ctx, ns)).Should(Succeed())

 			// We'll need to retry getting this newly created namespace, given that creation may not immediately happen.
 			createdNs := &corev1.Namespace{} // stub
 			Eventually(func() bool {
 				err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNamespaceDefaultCR, Namespace: "default"}, createdNs)
 				if err != nil {
 					return false
 				}
 				return true
 			}, createNsTimeout, createNsInterval).Should(BeTrue())

 			// The namespace should have Istio label
 			Expect(createdNs.Labels["istio-injection"]).To(Equal("enabled"))
 		})
 		It(fmt.Sprintf("Should create namespace %s", submarineNamespaceCustomCR), func() {
 			By(fmt.Sprintf("Creating the test namespace %s", submarineNamespaceCustomCR))
 			ns := &corev1.Namespace{
 				ObjectMeta: metav1.ObjectMeta{
 					Name: submarineNamespaceCustomCR, // Namespace to test custom CR
 					Labels: map[string]string{
 						"istio-injection": "enabled",
 					},
 				},
 			}
 			Expect(k8sClient.Create(ctx, ns)).Should(Succeed())

 			// We'll need to retry getting this newly created namespace, given that creation may not immediately happen.
 			createdNs := &corev1.Namespace{} // stub
 			Eventually(func() bool {
 				err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNamespaceCustomCR, Namespace: "default"}, createdNs)
 				if err != nil {
 					return false
 				}
 				return true
 			}, createNsTimeout, createNsInterval).Should(BeTrue())

 			// The namespace should have Istio label
 			Expect(createdNs.Labels["istio-injection"]).To(Equal("enabled"))
 		})
 	})

 	Context("Create Submarines", func() {
 		It(fmt.Sprintf("Should create Submarine in %s and it should become RUNNING", submarineNamespaceDefaultCR), func() {
 			By(fmt.Sprintf("Creating new Submarine in %s", submarineNamespaceDefaultCR))
 			submarine, err := MakeSubmarineFromYaml("../config/samples/_v1_submarine.yaml")
 			Expect(err).To(BeNil())

 			// Leave Spec.Virtualservice.Host empty to test default value
 			// Leave Spec.Virtualservice.Gateways empty to test default value

 			// The name of Submarine is specified in the YAML file.
 			// Storing name to call k8sClient.Get with NamespacedName
 			submarineNameDefaultCR = submarine.Name

 			// Create Submarines in our namespace
 			submarine.Namespace = submarineNamespaceDefaultCR
 			Expect(k8sClient.Create(ctx, submarine)).Should(Succeed())

 			// We'll need to retry getting this newly created Submarine, given that creation may not immediately happen.
 			createdSubmarine := &submarineapacheorgv1.Submarine{} // stub
 			Eventually(func() bool {
 				err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNameDefaultCR, Namespace: submarineNamespaceDefaultCR}, createdSubmarine)
 				if err != nil {
 					return false
 				}
 				return true
 			}, createNsTimeout, createNsInterval).Should(BeTrue())

 			// Wait for Submarine to be in RUNNING state
 			By(fmt.Sprintf("Waiting until Submarine %s/%s become RUNNING", submarineNameDefaultCR, submarineNamespaceDefaultCR))
 			Eventually(func() bool {
 				err = k8sClient.Get(ctx, types.NamespacedName{Name: submarineNameDefaultCR, Namespace: submarineNamespaceDefaultCR}, createdSubmarine)
 				Expect(err).To(BeNil())

 				state := createdSubmarine.Status.SubmarineState.State
 				Expect(state).ToNot(Equal(submarineapacheorgv1.FailedState))
 				if createdSubmarine.Status.SubmarineState.State == submarineapacheorgv1.RunningState {
 					return true
 				}
 				return false
 			}, createSubmarineTimeout, createSubmarineInterval).Should(BeTrue())
 		})
 		It(fmt.Sprintf("Should create Submarine in %s and it should become RUNNING", submarineNamespaceCustomCR), func() {
 			By(fmt.Sprintf("Creating new Submarine in %s", submarineNamespaceCustomCR))
 			submarine, err := MakeSubmarineFromYaml("../config/samples/_v1_submarine.yaml")
 			Expect(err).To(BeNil())

 			// Set Spec.Virtualservice.Hosts to [submarineCustomHosts] to test custom value
 			submarineCustomHosts = make([]string, 1, 1)
 			submarineCustomHosts[0] = customHost
 			submarine.Spec.Virtualservice.Hosts = submarineCustomHosts

 			// Set Spec.Virtualservice.Gateways to [submarineCustomGateway] to test custom value
 			submarineCustomGateways = make([]string, 1, 1)
 			submarineCustomGateways[0] = customGateway
 			submarine.Spec.Virtualservice.Gateways = submarineCustomGateways

 			// The name of Submarine is specified in the YAML file.
 			// Storing name to call k8sClient.Get with NamespacedName
 			submarineNameCustomCR = submarine.Name

 			// Create Submarines in our namespace
 			submarine.Namespace = submarineNamespaceCustomCR
 			Expect(k8sClient.Create(ctx, submarine)).Should(Succeed())

 			// We'll need to retry getting this newly created Submarine, given that creation may not immediately happen.
 			createdSubmarine := &submarineapacheorgv1.Submarine{} // stub
 			Eventually(func() bool {
 				err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNameCustomCR, Namespace: submarineNamespaceCustomCR}, createdSubmarine)
 				if err != nil {
 					return false
 				}
 				return true
 			}, createNsTimeout, createNsInterval).Should(BeTrue())

 			// Wait for Submarine to be in RUNNING state
 			By(fmt.Sprintf("Waiting until Submarine %s/%s become RUNNING", submarineNameCustomCR, submarineNamespaceCustomCR))
 			Eventually(func() bool {
 				err = k8sClient.Get(ctx, types.NamespacedName{Name: submarineNameCustomCR, Namespace: submarineNamespaceCustomCR}, createdSubmarine)
 				Expect(err).To(BeNil())

 				state := createdSubmarine.Status.SubmarineState.State
 				Expect(state).ToNot(Equal(submarineapacheorgv1.FailedState))
 				if createdSubmarine.Status.SubmarineState.State == submarineapacheorgv1.RunningState {
 					return true
 				}
 				return false
 			}, createSubmarineTimeout, createSubmarineInterval).Should(BeTrue())
 		})
 	})

 	Context("Verify Virtual Service Spec", func() {
 		It(fmt.Sprintf("Hosts and Gateways should have default values In %s", submarineNamespaceDefaultCR), func() {
 			By(fmt.Sprintf("Getting Virtual Service In %s", submarineNamespaceDefaultCR))
 			createdVirtualService := &istiov1alpha3.VirtualService{} // stub
 			err := k8sClient.Get(ctx, types.NamespacedName{Name: virtualServiceName, Namespace: submarineNamespaceDefaultCR}, createdVirtualService)
 			Expect(err).To(BeNil())

 			// The default value for host is *
 			Expect(createdVirtualService.Spec.Hosts[0]).To(Equal("*"))
 			// The default value for gateway is ${namespace}/submarine-gateway
 			Expect(createdVirtualService.Spec.Gateways[0]).To(Equal("submarine-cloud-v3-system/submarine-gateway"))
 		})
 		It(fmt.Sprintf("Hosts and Gateways should have custom values In %s", submarineNamespaceCustomCR), func() {
 			By(fmt.Sprintf("Getting Virtual Service In %s", submarineNamespaceCustomCR))
 			createdVirtualService := &istiov1alpha3.VirtualService{} // stub
 			err := k8sClient.Get(ctx, types.NamespacedName{Name: virtualServiceName, Namespace: submarineNamespaceCustomCR}, createdVirtualService)
 			Expect(err).To(BeNil())

 			// The custom value for hosts matches the submarine CR
 			Expect(createdVirtualService.Spec.Hosts).To(Equal(submarineCustomHosts))
 			// The custom value for gateways matches the submarine CR
 			Expect(createdVirtualService.Spec.Gateways).To(Equal(submarineCustomGateways))
 		})
 	})

 	Context("Delete Submarine", func() {
 		It(fmt.Sprintf("Should delete the Submarine In %s", submarineNamespaceDefaultCR), func() {
 			Expect(submarineNameDefaultCR).ToNot(BeNil())

 			By(fmt.Sprintf("Deleting Submarine %s/%s", submarineNameDefaultCR, submarineNamespaceDefaultCR))
 			createdSubmarine := &submarineapacheorgv1.Submarine{} // stub
 			err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNameDefaultCR, Namespace: submarineNamespaceDefaultCR}, createdSubmarine)
 			Expect(err).To(BeNil())

 			foreground := metav1.DeletePropagationForeground
 			err = k8sClient.Delete(ctx, createdSubmarine, &client.DeleteOptions{
 				PropagationPolicy: &foreground,
 			})
 			Expect(err).To(BeNil())

 			// Wait for Submarine to be deleted entirely
 			Eventually(func() bool {
 				err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNameDefaultCR, Namespace: submarineNamespaceDefaultCR}, createdSubmarine)
 				if apierrors.IsNotFound(err) {
 					return true
 				}
 				Expect(err).To(BeNil())
 				return false
 			}, deleteSubmarineTimeout, deleteSubmarineInterval).Should(BeTrue())
 		})
 		It(fmt.Sprintf("Should delete the Submarine In %s", submarineNamespaceCustomCR), func() {
 			Expect(submarineNameCustomCR).ToNot(BeNil())

 			By(fmt.Sprintf("Deleting Submarine %s/%s", submarineNameCustomCR, submarineNamespaceCustomCR))
 			createdSubmarine := &submarineapacheorgv1.Submarine{} // stub
 			err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNameCustomCR, Namespace: submarineNamespaceCustomCR}, createdSubmarine)
 			Expect(err).To(BeNil())

 			foreground := metav1.DeletePropagationForeground
 			err = k8sClient.Delete(ctx, createdSubmarine, &client.DeleteOptions{
 				PropagationPolicy: &foreground,
 			})
 			Expect(err).To(BeNil())

 			// Wait for Submarine to be deleted entirely
 			Eventually(func() bool {
 				err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNameCustomCR, Namespace: submarineNamespaceCustomCR}, createdSubmarine)
 				if apierrors.IsNotFound(err) {
 					return true
 				}
 				Expect(err).To(BeNil())
 				return false
 			}, deleteSubmarineTimeout, deleteSubmarineInterval).Should(BeTrue())
 		})
 	})

 	Context("Delete the test namespace", func() {
 		It(fmt.Sprintf("Should delete namespace %s", submarineNamespaceDefaultCR), func() {
 			By(fmt.Sprintf("Deleting the test namespace %s", submarineNamespaceDefaultCR))

 			createdNs := &corev1.Namespace{} // stub
 			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: submarineNamespaceDefaultCR, Namespace: "default"}, createdNs)).Should(Succeed())
 			Expect(k8sClient.Delete(ctx, createdNs)).Should(Succeed())

 			// Wait for submarine to be deleted entirely

 			Eventually(func() bool {
 				err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNamespaceDefaultCR, Namespace: "default"}, createdNs)
 				if apierrors.IsNotFound(err) {
 					return true
 				}
 				Expect(err).To(BeNil())
 				return false
 			}, deleteNsTimeout, deleteNsInterval).Should(BeTrue())
 		})
 		It(fmt.Sprintf("Should delete namespace %s", submarineNamespaceCustomCR), func() {
 			By(fmt.Sprintf("Deleting the test namespace %s", submarineNamespaceCustomCR))

 			createdNs := &corev1.Namespace{} // stub
 			Expect(k8sClient.Get(ctx, types.NamespacedName{Name: submarineNamespaceCustomCR, Namespace: "default"}, createdNs)).Should(Succeed())
 			Expect(k8sClient.Delete(ctx, createdNs)).Should(Succeed())

 			// Wait for submarine to be deleted entirely

 			Eventually(func() bool {
 				err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNamespaceCustomCR, Namespace: "default"}, createdNs)
 				if apierrors.IsNotFound(err) {
 					return true
 				}
 				Expect(err).To(BeNil())
 				return false
 			}, deleteNsTimeout, deleteNsInterval).Should(BeTrue())
 		})
 	})
 })

 func MakeSubmarineFromYaml(pathToYaml string) (*submarineapacheorgv1.Submarine, error) {
 	manifest, err := PathToOSFile(pathToYaml)
 	if err != nil {
 		return nil, err
 	}
 	tmp := submarineapacheorgv1.Submarine{}
 	if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&tmp); err != nil {
 		return nil, errors.Wrap(err, fmt.Sprintf("failed to decode file %s", pathToYaml))
 	}
 	return &tmp, err
 }

 func MakeSubmarineFromYamlByNamespace(pathToYaml string, namespace string) (*submarineapacheorgv1.Submarine, error) {
 	submarine, err := MakeSubmarineFromYaml(pathToYaml)
 	if err != nil {
 		return nil, err
 	}
 	submarine.Namespace = namespace
 	return submarine, nil
 }

 // PathToOSFile gets the absolute path from relative path.
 func PathToOSFile(relativePath string) (*os.File, error) {
 	path, err := filepath.Abs(relativePath)
 	if err != nil {
 		return nil, errors.Wrap(err, fmt.Sprintf("failed generate absolute file path of %s", relativePath))
 	}

 	manifest, err := os.Open(path)
 	if err != nil {
 		return nil, errors.Wrap(err, fmt.Sprintf("failed to open file %s", path))
 	}

 	return manifest, nil
 }
	/*
	* Licensed to the Apache Software Foundation (ASF) under one or more
	* contributor license agreements. See the NOTICE file distributed with
	* this work for additional information regarding copyright ownership.
	* The ASF licenses this file to You under the Apache License, Version 2.0
	* (the "License"); you may not use this file except in compliance with
	* the License. You may obtain a copy of the License at
	*
	* http://www.apache.org/licenses/LICENSE-2.0
	*
	* Unless required by applicable law or agreed to in writing, software
	* distributed under the License is distributed on an "AS IS" BASIS,
	* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	* See the License for the specific language governing permissions and
	* limitations under the License.
	*/

	package controllers

	import (
	"context"
	"fmt"
	appsv1 "k8s.io/api/apps/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
	"os"
	"path/filepath"
	ctrl "sigs.k8s.io/controller-runtime"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/pkg/errors"
	istio "istio.io/client-go/pkg/apis/networking/v1alpha3"
	istiov1alpha3 "istio.io/client-go/pkg/apis/networking/v1alpha3"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/yaml"
	"sigs.k8s.io/controller-runtime/pkg/client"

	submarineapacheorgv1 "github.com/apache/submarine/submarine-cloud-v3/api/v1"
	)

	func createScheme() *runtime.Scheme {
	scheme := runtime.NewScheme()
	_ = clientgoscheme.AddToScheme(scheme)
	_ = appsv1.AddToScheme(scheme)
	_ = corev1.AddToScheme(scheme)
	_ = submarineapacheorgv1.AddToScheme(scheme)
	_ = istio.AddToScheme(scheme)
	_ = serializer.NewCodecFactory(scheme).UniversalDeserializer().Decode
	return scheme
	}

	// We are mainly testing created resources, so for the time being we have not declared k8s client and recoder
	func createSubmarineReconciler(config ...SubmarineReconciler) SubmarineReconciler {
	namespace := "submarine"
	istioEnable := false
	submarineGateway := "submarine/submarine-gateway"
	seldonIstioEnable := false
	seldonGateway := "submarine/seldon-gateway"
	clusterType := "kubernetes"
	createPodSecurityPolicy := false
	if len(config) > 0 {
	if config[0].Namespace != "" {
	namespace = config[0].Namespace
	}
	if &config[0].IstioEnable != nil {
	istioEnable = config[0].IstioEnable
	}
	if config[0].SubmarineGateway != "" {
	submarineGateway = config[0].SubmarineGateway
	}
	if &config[0].SeldonIstioEnable != nil {
	seldonIstioEnable = config[0].SeldonIstioEnable
	}
	if config[0].SeldonGateway != "" {
	seldonGateway = config[0].SeldonGateway
	}
	if config[0].ClusterType != "" {
	clusterType = config[0].ClusterType
	}
	if &config[0].CreatePodSecurityPolicy != nil {
	createPodSecurityPolicy = config[0].CreatePodSecurityPolicy
	}
	}
	return &SubmarineReconciler{
	Scheme: createScheme(),
	Log: ctrl.Log.WithName("submarine-test"),
	Namespace: namespace,
	IstioEnable: istioEnable,
	SubmarineGateway: submarineGateway,
	SeldonIstioEnable: seldonIstioEnable,
	SeldonGateway: seldonGateway,
	ClusterType: clusterType,
	CreatePodSecurityPolicy: createPodSecurityPolicy,
	}
	}

	var _ = Describe("Submarine controller", func() {

	// Define utility constants and variables
	const (
	// The namespaces where the Submarine CRs are created
	submarineNamespaceDefaultCR = "submarine-test-submit-default-cr"
	submarineNamespaceCustomCR = "submarine-test-submit-custom-cr"
	customHost = "submarine-custom-host"
	customGateway = "submarine-custom-gateway"

	createNsTimeout = time.Second * 10
	createNsInterval = time.Second * 2
	createSubmarineTimeout = time.Second * 1200
	createSubmarineInterval = time.Second * 10
	deleteSubmarineTimeout = time.Second * 600
	deleteSubmarineInterval = time.Second * 10
	deleteNsTimeout = time.Second * 120
	deleteNsInterval = time.Second * 2
	)
	var (
	// The name of Submarine is specified in the YAML file.
	// Storing name to call k8sClient.Get with NamespacedName
	submarineNameDefaultCR string
	submarineNameCustomCR string
	submarineCustomHosts []string
	submarineCustomGateways []string

	ctx = context.Background()
	)

	Context("Create test namespaces", func() {
	It(fmt.Sprintf("Should create namespace %s", submarineNamespaceDefaultCR), func() {
	By(fmt.Sprintf("Creating the test namespace %s", submarineNamespaceDefaultCR))
	ns := &corev1.Namespace{
	ObjectMeta: metav1.ObjectMeta{
	Name: submarineNamespaceDefaultCR, // Namespace to test default CR
	Labels: map[string]string{
	"istio-injection": "enabled",
	},
	},
	}
	Expect(k8sClient.Create(ctx, ns)).Should(Succeed())

	// We'll need to retry getting this newly created namespace, given that creation may not immediately happen.
	createdNs := &corev1.Namespace{} // stub
	Eventually(func() bool {
	err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNamespaceDefaultCR, Namespace: "default"}, createdNs)
	if err != nil {
	return false
	}
	return true
	}, createNsTimeout, createNsInterval).Should(BeTrue())

	// The namespace should have Istio label
	Expect(createdNs.Labels["istio-injection"]).To(Equal("enabled"))
	})
	It(fmt.Sprintf("Should create namespace %s", submarineNamespaceCustomCR), func() {
	By(fmt.Sprintf("Creating the test namespace %s", submarineNamespaceCustomCR))
	ns := &corev1.Namespace{
	ObjectMeta: metav1.ObjectMeta{
	Name: submarineNamespaceCustomCR, // Namespace to test custom CR
	Labels: map[string]string{
	"istio-injection": "enabled",
	},
	},
	}
	Expect(k8sClient.Create(ctx, ns)).Should(Succeed())

	// We'll need to retry getting this newly created namespace, given that creation may not immediately happen.
	createdNs := &corev1.Namespace{} // stub
	Eventually(func() bool {
	err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNamespaceCustomCR, Namespace: "default"}, createdNs)
	if err != nil {
	return false
	}
	return true
	}, createNsTimeout, createNsInterval).Should(BeTrue())

	// The namespace should have Istio label
	Expect(createdNs.Labels["istio-injection"]).To(Equal("enabled"))
	})
	})

	Context("Create Submarines", func() {
	It(fmt.Sprintf("Should create Submarine in %s and it should become RUNNING", submarineNamespaceDefaultCR), func() {
	By(fmt.Sprintf("Creating new Submarine in %s", submarineNamespaceDefaultCR))
	submarine, err := MakeSubmarineFromYaml("../config/samples/_v1_submarine.yaml")
	Expect(err).To(BeNil())

	// Leave Spec.Virtualservice.Host empty to test default value
	// Leave Spec.Virtualservice.Gateways empty to test default value

	// The name of Submarine is specified in the YAML file.
	// Storing name to call k8sClient.Get with NamespacedName
	submarineNameDefaultCR = submarine.Name

	// Create Submarines in our namespace
	submarine.Namespace = submarineNamespaceDefaultCR
	Expect(k8sClient.Create(ctx, submarine)).Should(Succeed())

	// We'll need to retry getting this newly created Submarine, given that creation may not immediately happen.
	createdSubmarine := &submarineapacheorgv1.Submarine{} // stub
	Eventually(func() bool {
	err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNameDefaultCR, Namespace: submarineNamespaceDefaultCR}, createdSubmarine)
	if err != nil {
	return false
	}
	return true
	}, createNsTimeout, createNsInterval).Should(BeTrue())

	// Wait for Submarine to be in RUNNING state
	By(fmt.Sprintf("Waiting until Submarine %s/%s become RUNNING", submarineNameDefaultCR, submarineNamespaceDefaultCR))
	Eventually(func() bool {
	err = k8sClient.Get(ctx, types.NamespacedName{Name: submarineNameDefaultCR, Namespace: submarineNamespaceDefaultCR}, createdSubmarine)
	Expect(err).To(BeNil())

	state := createdSubmarine.Status.SubmarineState.State
	Expect(state).ToNot(Equal(submarineapacheorgv1.FailedState))
	if createdSubmarine.Status.SubmarineState.State == submarineapacheorgv1.RunningState {
	return true
	}
	return false
	}, createSubmarineTimeout, createSubmarineInterval).Should(BeTrue())
	})
	It(fmt.Sprintf("Should create Submarine in %s and it should become RUNNING", submarineNamespaceCustomCR), func() {
	By(fmt.Sprintf("Creating new Submarine in %s", submarineNamespaceCustomCR))
	submarine, err := MakeSubmarineFromYaml("../config/samples/_v1_submarine.yaml")
	Expect(err).To(BeNil())

	// Set Spec.Virtualservice.Hosts to [submarineCustomHosts] to test custom value
	submarineCustomHosts = make([]string, 1, 1)
	submarineCustomHosts[0] = customHost
	submarine.Spec.Virtualservice.Hosts = submarineCustomHosts

	// Set Spec.Virtualservice.Gateways to [submarineCustomGateway] to test custom value
	submarineCustomGateways = make([]string, 1, 1)
	submarineCustomGateways[0] = customGateway
	submarine.Spec.Virtualservice.Gateways = submarineCustomGateways

	// The name of Submarine is specified in the YAML file.
	// Storing name to call k8sClient.Get with NamespacedName
	submarineNameCustomCR = submarine.Name

	// Create Submarines in our namespace
	submarine.Namespace = submarineNamespaceCustomCR
	Expect(k8sClient.Create(ctx, submarine)).Should(Succeed())

	// We'll need to retry getting this newly created Submarine, given that creation may not immediately happen.
	createdSubmarine := &submarineapacheorgv1.Submarine{} // stub
	Eventually(func() bool {
	err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNameCustomCR, Namespace: submarineNamespaceCustomCR}, createdSubmarine)
	if err != nil {
	return false
	}
	return true
	}, createNsTimeout, createNsInterval).Should(BeTrue())

	// Wait for Submarine to be in RUNNING state
	By(fmt.Sprintf("Waiting until Submarine %s/%s become RUNNING", submarineNameCustomCR, submarineNamespaceCustomCR))
	Eventually(func() bool {
	err = k8sClient.Get(ctx, types.NamespacedName{Name: submarineNameCustomCR, Namespace: submarineNamespaceCustomCR}, createdSubmarine)
	Expect(err).To(BeNil())

	state := createdSubmarine.Status.SubmarineState.State
	Expect(state).ToNot(Equal(submarineapacheorgv1.FailedState))
	if createdSubmarine.Status.SubmarineState.State == submarineapacheorgv1.RunningState {
	return true
	}
	return false
	}, createSubmarineTimeout, createSubmarineInterval).Should(BeTrue())
	})
	})

	Context("Verify Virtual Service Spec", func() {
	It(fmt.Sprintf("Hosts and Gateways should have default values In %s", submarineNamespaceDefaultCR), func() {
	By(fmt.Sprintf("Getting Virtual Service In %s", submarineNamespaceDefaultCR))
	createdVirtualService := &istiov1alpha3.VirtualService{} // stub
	err := k8sClient.Get(ctx, types.NamespacedName{Name: virtualServiceName, Namespace: submarineNamespaceDefaultCR}, createdVirtualService)
	Expect(err).To(BeNil())

	// The default value for host is *
	Expect(createdVirtualService.Spec.Hosts[0]).To(Equal("*"))
	// The default value for gateway is ${namespace}/submarine-gateway
	Expect(createdVirtualService.Spec.Gateways[0]).To(Equal("submarine-cloud-v3-system/submarine-gateway"))
	})
	It(fmt.Sprintf("Hosts and Gateways should have custom values In %s", submarineNamespaceCustomCR), func() {
	By(fmt.Sprintf("Getting Virtual Service In %s", submarineNamespaceCustomCR))
	createdVirtualService := &istiov1alpha3.VirtualService{} // stub
	err := k8sClient.Get(ctx, types.NamespacedName{Name: virtualServiceName, Namespace: submarineNamespaceCustomCR}, createdVirtualService)
	Expect(err).To(BeNil())

	// The custom value for hosts matches the submarine CR
	Expect(createdVirtualService.Spec.Hosts).To(Equal(submarineCustomHosts))
	// The custom value for gateways matches the submarine CR
	Expect(createdVirtualService.Spec.Gateways).To(Equal(submarineCustomGateways))
	})
	})

	Context("Delete Submarine", func() {
	It(fmt.Sprintf("Should delete the Submarine In %s", submarineNamespaceDefaultCR), func() {
	Expect(submarineNameDefaultCR).ToNot(BeNil())

	By(fmt.Sprintf("Deleting Submarine %s/%s", submarineNameDefaultCR, submarineNamespaceDefaultCR))
	createdSubmarine := &submarineapacheorgv1.Submarine{} // stub
	err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNameDefaultCR, Namespace: submarineNamespaceDefaultCR}, createdSubmarine)
	Expect(err).To(BeNil())

	foreground := metav1.DeletePropagationForeground
	err = k8sClient.Delete(ctx, createdSubmarine, &client.DeleteOptions{
	PropagationPolicy: &foreground,
	})
	Expect(err).To(BeNil())

	// Wait for Submarine to be deleted entirely
	Eventually(func() bool {
	err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNameDefaultCR, Namespace: submarineNamespaceDefaultCR}, createdSubmarine)
	if apierrors.IsNotFound(err) {
	return true
	}
	Expect(err).To(BeNil())
	return false
	}, deleteSubmarineTimeout, deleteSubmarineInterval).Should(BeTrue())
	})
	It(fmt.Sprintf("Should delete the Submarine In %s", submarineNamespaceCustomCR), func() {
	Expect(submarineNameCustomCR).ToNot(BeNil())

	By(fmt.Sprintf("Deleting Submarine %s/%s", submarineNameCustomCR, submarineNamespaceCustomCR))
	createdSubmarine := &submarineapacheorgv1.Submarine{} // stub
	err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNameCustomCR, Namespace: submarineNamespaceCustomCR}, createdSubmarine)
	Expect(err).To(BeNil())

	foreground := metav1.DeletePropagationForeground
	err = k8sClient.Delete(ctx, createdSubmarine, &client.DeleteOptions{
	PropagationPolicy: &foreground,
	})
	Expect(err).To(BeNil())

	// Wait for Submarine to be deleted entirely
	Eventually(func() bool {
	err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNameCustomCR, Namespace: submarineNamespaceCustomCR}, createdSubmarine)
	if apierrors.IsNotFound(err) {
	return true
	}
	Expect(err).To(BeNil())
	return false
	}, deleteSubmarineTimeout, deleteSubmarineInterval).Should(BeTrue())
	})
	})

	Context("Delete the test namespace", func() {
	It(fmt.Sprintf("Should delete namespace %s", submarineNamespaceDefaultCR), func() {
	By(fmt.Sprintf("Deleting the test namespace %s", submarineNamespaceDefaultCR))

	createdNs := &corev1.Namespace{} // stub
	Expect(k8sClient.Get(ctx, types.NamespacedName{Name: submarineNamespaceDefaultCR, Namespace: "default"}, createdNs)).Should(Succeed())
	Expect(k8sClient.Delete(ctx, createdNs)).Should(Succeed())

	// Wait for submarine to be deleted entirely

	Eventually(func() bool {
	err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNamespaceDefaultCR, Namespace: "default"}, createdNs)
	if apierrors.IsNotFound(err) {
	return true
	}
	Expect(err).To(BeNil())
	return false
	}, deleteNsTimeout, deleteNsInterval).Should(BeTrue())
	})
	It(fmt.Sprintf("Should delete namespace %s", submarineNamespaceCustomCR), func() {
	By(fmt.Sprintf("Deleting the test namespace %s", submarineNamespaceCustomCR))

	createdNs := &corev1.Namespace{} // stub
	Expect(k8sClient.Get(ctx, types.NamespacedName{Name: submarineNamespaceCustomCR, Namespace: "default"}, createdNs)).Should(Succeed())
	Expect(k8sClient.Delete(ctx, createdNs)).Should(Succeed())

	// Wait for submarine to be deleted entirely

	Eventually(func() bool {
	err := k8sClient.Get(ctx, types.NamespacedName{Name: submarineNamespaceCustomCR, Namespace: "default"}, createdNs)
	if apierrors.IsNotFound(err) {
	return true
	}
	Expect(err).To(BeNil())
	return false
	}, deleteNsTimeout, deleteNsInterval).Should(BeTrue())
	})
	})
	})

	func MakeSubmarineFromYaml(pathToYaml string) (*submarineapacheorgv1.Submarine, error) {
	manifest, err := PathToOSFile(pathToYaml)
	if err != nil {
	return nil, err
	}
	tmp := submarineapacheorgv1.Submarine{}
	if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&tmp); err != nil {
	return nil, errors.Wrap(err, fmt.Sprintf("failed to decode file %s", pathToYaml))
	}
	return &tmp, err
	}

	func MakeSubmarineFromYamlByNamespace(pathToYaml string, namespace string) (*submarineapacheorgv1.Submarine, error) {
	submarine, err := MakeSubmarineFromYaml(pathToYaml)
	if err != nil {
	return nil, err
	}
	submarine.Namespace = namespace
	return submarine, nil
	}

	// PathToOSFile gets the absolute path from relative path.
	func PathToOSFile(relativePath string) (*os.File, error) {
	path, err := filepath.Abs(relativePath)
	if err != nil {
	return nil, errors.Wrap(err, fmt.Sprintf("failed generate absolute file path of %s", relativePath))
	}

	manifest, err := os.Open(path)
	if err != nil {
	return nil, errors.Wrap(err, fmt.Sprintf("failed to open file %s", path))
	}

	return manifest, nil
	}