Fix#376 - E2E tests are failing in nightly deploy job (#384)
diff --git a/.ci/jenkins/Jenkinsfile.e2e.cluster b/.ci/jenkins/Jenkinsfile.e2e.cluster
index 3b90cab..9719256 100644
--- a/.ci/jenkins/Jenkinsfile.e2e.cluster
+++ b/.ci/jenkins/Jenkinsfile.e2e.cluster
@@ -2,9 +2,12 @@
helper = null
-minikubeClusterPlatform = 'minikube'
+KIND_VERSION = "v0.20.0"
+kindClusterPlatform = 'kind'
openshiftClusterPlatform = 'openshift'
+kindLogsFolder = "/tmp/kind-logs"
+
pipeline {
agent {
docker {
@@ -47,6 +50,31 @@
}
}
}
+ stage('Load image into Kind') {
+ when {
+ expression {
+ return getClusterName() == kindClusterPlatform
+ }
+ }
+ steps {
+ script {
+ kind.loadImage(getTestImage())
+ }
+ }
+ }
+ stage('Deploy the operator') {
+ when {
+ expression {
+ return getClusterName() == kindClusterPlatform
+ }
+ }
+ steps {
+ script {
+ sh "make deploy IMG=${getTestImage()}"
+ sh "kubectl wait pod -A -l control-plane=sonataflow-operator --for condition=Ready --timeout=120s"
+ }
+ }
+ }
stage('Prepare for e2e tests') {
when {
expression {
@@ -83,16 +111,25 @@
make test-e2e
"""
} catch (err) {
+ kind.exportLogs(kindLogsFolder)
sh 'make undeploy'
+ deleteKindCluster()
throw err
}
sh 'kubectl get pods -A'
+ deleteKindCluster()
}
}
}
}
}
post {
+ always {
+ script {
+ archiveArtifacts(artifacts: "**${kindLogsFolder}/**/*.*,**/e2e-test-report.xml")
+ junit '**/e2e-test-report.xml'
+ }
+ }
cleanup {
script {
clean()
@@ -104,7 +141,6 @@
void clean() {
helper.cleanGoPath()
util.cleanNode(containerEngine)
- cleanupCluster()
}
String getTestImage() {
@@ -121,10 +157,12 @@
void setupCluster() {
switch (getClusterName()) {
- case minikubeClusterPlatform:
- setupMinikube()
+ case kindClusterPlatform:
+ echo 'Creating kind cluster'
+ createKindCluster()
break
case openshiftClusterPlatform:
+ echo 'Setting up Openshift'
setupOpenshift()
break
default:
@@ -132,13 +170,12 @@
}
}
-void setupMinikube() {
- // Start minikube
- minikube.minikubeMemory = '12g'
- minikube.start()
+void createKindCluster() {
+ sh(script: "make KIND_VERSION=${KIND_VERSION} create-cluster", returnStdout: true)
+}
- minikube.waitForMinikubeStarted()
- minikube.waitForMinikubeRegistry()
+void deleteKindCluster() {
+ sh(script: "make delete-cluster", returnStdout: true)
}
void setupOpenshift() {
@@ -148,8 +185,9 @@
void cleanupCluster() {
switch (getClusterName()) {
- case minikubeClusterPlatform:
- minikube.stop()
+ case kindClusterPlatform:
+ echo 'Deleting kind cluster'
+ deleteKindCluster()
break
case openshiftClusterPlatform:
echo 'Nothing to cleanup on openshift. All good !'
@@ -161,8 +199,8 @@
void executeInCluster(Closure executeClosure) {
switch (getClusterName()) {
- case minikubeClusterPlatform:
- echo "Execute in minikube"
+ case kindClusterPlatform:
+ echo "Execute in kind"
executeClosure()
break
case openshiftClusterPlatform:
@@ -178,7 +216,7 @@
void getPlatformCRFilePath() {
switch (getClusterName()) {
- case minikubeClusterPlatform:
+ case kindClusterPlatform:
return 'test/testdata/sonataflow.org_v1alpha08_sonataflowplatform_withCache_minikube.yaml'
case openshiftClusterPlatform:
return 'test/testdata/sonataflow.org_v1alpha08_sonataflowplatform_openshift.yaml'
diff --git a/.ci/jenkins/scripts/helper.groovy b/.ci/jenkins/scripts/helper.groovy
index 38542b7..85326e7 100644
--- a/.ci/jenkins/scripts/helper.groovy
+++ b/.ci/jenkins/scripts/helper.groovy
@@ -1,7 +1,7 @@
openshift = null
container = null
properties = null
-minikube = null
+kind = null
defaultImageParamsPrefix = 'IMAGE'
baseImageParamsPrefix = 'BASE_IMAGE'
@@ -19,7 +19,7 @@
container.containerEngineTlsOptions = env.CONTAINER_ENGINE_TLS_OPTIONS ?: ''
container.containerOpenshift = openshift
- minikube = load '.ci/jenkins/scripts/minikube.groovy'
+ kind = load '.ci/jenkins/scripts/kind.groovy'
}
void updateDisplayName() {
diff --git a/.ci/jenkins/scripts/kind.groovy b/.ci/jenkins/scripts/kind.groovy
new file mode 100644
index 0000000..bd28dfb
--- /dev/null
+++ b/.ci/jenkins/scripts/kind.groovy
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+void exportLogs(String destination) {
+ println "Exporting kind logs to ${destination}"
+ def exportKindLogs = sh(returnStatus: true, script: """
+ mkdir -p .${destination}
+ kind export logs --loglevel=debug .${destination}
+ """)
+}
+
+void loadImage(String imageName) {
+ sh "kind load docker-image ${imageName}"
+}
+
+return this
diff --git a/Makefile b/Makefile
index 2d178f6..f7ac155 100644
--- a/Makefile
+++ b/Makefile
@@ -347,7 +347,7 @@
.PHONY: test-e2e # You will need to have a Minikube/Kind cluster up in running to run this target, and run container-builder before the test
test-e2e: install-operator-sdk
- go test ./test/e2e/* -v -ginkgo.v -timeout 30m
+ go test ./test/e2e/* -v -ginkgo.v -ginkgo.no-color -ginkgo.junit-report=./e2e-test-report.xml -timeout 60m
.PHONY: before-pr
before-pr: test generate-all
diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go
index 28e747d..0b87fa8 100644
--- a/test/e2e/helpers.go
+++ b/test/e2e/helpers.go
@@ -84,6 +84,7 @@
Expect(h.Status).To(Equal(upStatus))
return
}
+
if len(errs.Error()) > 0 {
errs = fmt.Errorf("%v; %w", err, errs)
} else {
@@ -99,7 +100,17 @@
cmd := exec.Command("kubectl", "exec", "-t", podName, "-n", ns, "-c", containerName, "--", "curl", "-s", "localhost:8080/q/health")
output, err := utils.Run(cmd)
Expect(err).NotTo(HaveOccurred())
- err = json.Unmarshal(output, &h)
+ // On Apache CI Nodes, does not return valid JSON, hence we match first and last brackets by index and extract it
+ stringOutput := string(output)
+ startIndex := strings.Index(stringOutput, "{")
+ endIndex := strings.LastIndex(stringOutput, "}")
+ if startIndex == 0 {
+ stringOutput = stringOutput[startIndex : endIndex+1]
+ } else {
+ stringOutput = stringOutput[startIndex-1 : endIndex+1]
+ }
+ fmt.Printf("Parsed following JSON object from health Endpoint response: %v\n", stringOutput)
+ err = json.Unmarshal([]byte(stringOutput), &h)
if err != nil {
return nil, fmt.Errorf("failed to execute curl command against health endpoint in container %s:%v with output %s", containerName, err, output)
}
diff --git a/test/e2e/platform_test.go b/test/e2e/platform_test.go
index 49afd5a..83cfb6e 100644
--- a/test/e2e/platform_test.go
+++ b/test/e2e/platform_test.go
@@ -87,7 +87,7 @@
cmd = exec.Command("kubectl", "wait", "pod", "-n", targetNamespace, "-l", "app=sonataflow-platform", "--for", "condition=Ready", "--timeout=5s")
_, err = utils.Run(cmd)
return err
- }, 10*time.Minute, 5).Should(Succeed())
+ }, 20*time.Minute, 5).Should(Succeed())
By("Evaluate status of service's health endpoint")
cmd = exec.Command("kubectl", "get", "pod", "-l", "app=sonataflow-platform", "-n", targetNamespace, "-ojsonpath={.items[*].metadata.name}")
output, err := utils.Run(cmd)
@@ -113,7 +113,7 @@
Expect(sf).NotTo(BeEmpty(), "sonataflow name is empty")
EventuallyWithOffset(1, func() bool {
return verifyWorkflowIsInRunningStateInNamespace(sf, targetNamespace)
- }, 5*time.Minute, 5).Should(BeTrue())
+ }, 10*time.Minute, 5).Should(BeTrue())
}
},
Entry("with both Job Service and Data Index and ephemeral persistence and the workflow in a dev profile", test.GetSonataFlowE2EPlatformServicesDirectory(), dev, ephemeral),
diff --git a/test/e2e/workflow_test.go b/test/e2e/workflow_test.go
index e05c843..cb64698 100644
--- a/test/e2e/workflow_test.go
+++ b/test/e2e/workflow_test.go
@@ -68,7 +68,7 @@
"test/testdata/"+test.SonataFlowSimpleOpsYamlCR), "-n", targetNamespace)
_, err := utils.Run(cmd)
return err
- }, 2*time.Minute, time.Second).Should(Succeed())
+ }, 3*time.Minute, time.Second).Should(Succeed())
By("check the workflow is in running state")
EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("simple", targetNamespace) }, 15*time.Minute, 30*time.Second).Should(BeTrue())
@@ -78,7 +78,7 @@
"test/testdata/"+test.SonataFlowSimpleOpsYamlCR), "-n", targetNamespace)
_, err := utils.Run(cmd)
return err
- }, 2*time.Minute, time.Second).Should(Succeed())
+ }, 3*time.Minute, time.Second).Should(Succeed())
})
It("should successfully deploy the Greeting Workflow in prod mode and verify if it's running", func() {
@@ -88,7 +88,7 @@
"test/testdata/"+test.SonataFlowGreetingsDataInputSchemaConfig), "-n", targetNamespace)
_, err := utils.Run(cmd)
return err
- }, 2*time.Minute, time.Second).Should(Succeed())
+ }, 3*time.Minute, time.Second).Should(Succeed())
By("creating an instance of the SonataFlow Operand(CR)")
EventuallyWithOffset(1, func() error {
@@ -96,7 +96,7 @@
"test/testdata/"+test.SonataFlowGreetingsWithDataInputSchemaCR), "-n", targetNamespace)
_, err := utils.Run(cmd)
return err
- }, 2*time.Minute, time.Second).Should(Succeed())
+ }, 3*time.Minute, time.Second).Should(Succeed())
By("check the workflow is in running state")
EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("greeting", targetNamespace) }, 15*time.Minute, 30*time.Second).Should(BeTrue())
@@ -106,7 +106,7 @@
"test/testdata/"+test.SonataFlowGreetingsWithDataInputSchemaCR), "-n", targetNamespace)
_, err := utils.Run(cmd)
return err
- }, 2*time.Minute, time.Second).Should(Succeed())
+ }, 3*time.Minute, time.Second).Should(Succeed())
})
It("should successfully deploy the orderprocessing workflow in devmode and verify if it's running", func() {
@@ -117,7 +117,7 @@
test.GetSonataFlowE2eOrderProcessingFolder()), "-n", targetNamespace)
_, err := utils.Run(cmd)
return err
- }, 2*time.Minute, time.Second).Should(Succeed())
+ }, 3*time.Minute, time.Second).Should(Succeed())
By("check the workflow is in running state")
EventuallyWithOffset(1, func() bool { return verifyWorkflowIsInRunningState("orderprocessing", targetNamespace) }, 10*time.Minute, 30*time.Second).Should(BeTrue())
@@ -135,7 +135,7 @@
test.GetSonataFlowE2eOrderProcessingFolder()), "-n", targetNamespace)
_, err := utils.Run(cmd)
return err
- }, 2*time.Minute, time.Second).Should(Succeed())
+ }, 3*time.Minute, time.Second).Should(Succeed())
})
})
@@ -184,7 +184,7 @@
out, err := utils.Run(cmd)
GinkgoWriter.Printf("%s\n", string(out))
return err
- }, 10*time.Minute, 5).Should(Succeed())
+ }, 12*time.Minute, 5).Should(Succeed())
By("Evaluate status of the workflow's pod database connection health endpoint")
cmd = exec.Command("kubectl", "get", "pod", "-l", "sonataflow.org/workflow-app=callbackstatetimeouts", "-n", ns, "-ojsonpath={.items[*].metadata.name}")
@@ -205,7 +205,7 @@
}
}
return false
- }, 10*time.Minute).Should(BeTrue())
+ }, 12*time.Minute).Should(BeTrue())
},
Entry("defined in the workflow from an existing kubernetes service as a reference", test.GetSonataFlowE2EWorkflowPersistenceSampleDataDirectory("by_service")),
)
