Integration Test improvements (#613)
- Save Solr Operator logs for each test (filtered to only the applicable lines for that test)
- Print the log directory on failure
- Include the KUBERNETES_VERSION on failure-retry scripts.
diff --git a/.gitignore b/.gitignore
index bf3ff90..54bb603 100644
--- a/.gitignore
+++ b/.gitignore
@@ -50,6 +50,9 @@
# Directory to test generated files
generated-check
+# Integration test outputs
+tests/**/output
+
# Python for the release wizard
venv
__pycache__
diff --git a/go.mod b/go.mod
index eef7232..6193a4c 100644
--- a/go.mod
+++ b/go.mod
@@ -11,6 +11,7 @@
github.com/robfig/cron/v3 v3.0.1
github.com/stretchr/testify v1.8.2
golang.org/x/net v0.14.0
+ golang.org/x/text v0.12.0
helm.sh/helm/v3 v3.11.1
k8s.io/api v0.27.2
k8s.io/apimachinery v0.27.2
@@ -124,7 +125,6 @@
golang.org/x/sync v0.3.0 // indirect
golang.org/x/sys v0.11.0 // indirect
golang.org/x/term v0.11.0 // indirect
- golang.org/x/text v0.12.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.12.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect
diff --git a/tests/e2e/suite_test.go b/tests/e2e/suite_test.go
index 770965b..40ebaed 100644
--- a/tests/e2e/suite_test.go
+++ b/tests/e2e/suite_test.go
@@ -18,6 +18,8 @@
package e2e
import (
+ "bufio"
+ "bytes"
"context"
"fmt"
solrv1beta1 "github.com/apache/solr-operator/api/v1beta1"
@@ -25,14 +27,23 @@
"github.com/go-logr/logr"
"github.com/onsi/ginkgo/v2/types"
zkApi "github.com/pravega/zookeeper-operator/api/v1beta1"
+ "golang.org/x/text/cases"
+ "golang.org/x/text/language"
+ "io"
+ corev1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/labels"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"math/rand"
+ "os"
+ "path/filepath"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/config"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
+ "strings"
"testing"
"time"
@@ -44,6 +55,7 @@
// Available environment variables to customize tests
operatorImageEnv = "OPERATOR_IMAGE"
solrImageEnv = "SOLR_IMAGE"
+ kubeVersionEnv = "KUBERNETES_VERSION"
backupDirHostPath = "/tmp/backup"
@@ -64,6 +76,9 @@
operatorImage = getEnvWithDefault(operatorImageEnv, defaultOperatorImage)
solrImage = getEnvWithDefault(solrImageEnv, defaultSolrImage)
+ kubeVersion = getEnvWithDefault(kubeVersionEnv, defaultSolrImage)
+
+ outputDir = "output"
)
// Run e2e tests using the Ginkgo runner.
@@ -81,6 +96,9 @@
logger = zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))
logf.SetLogger(logger)
+ Expect(os.RemoveAll(outputDir+"/")).To(Succeed(), "Could not delete existing output directory before tests start: %s", outputDir)
+ Expect(os.Mkdir(outputDir, os.ModeDir|os.ModePerm)).To(Succeed(), "Could not create directory for test output: %s", outputDir)
+
var err error
k8sConfig, err = config.GetConfig()
Expect(err).NotTo(HaveOccurred(), "Could not load in default kubernetes config")
@@ -139,6 +157,7 @@
randomSeed int64
operatorImage string
solrImage string
+ kubeVersion string
}
// ColorableString for ReportEntry to use
@@ -149,18 +168,20 @@
// non-colorable String() is used by go's string formatting support but ignored by ReportEntry
func (rc RetryCommand) String() string {
return fmt.Sprintf(
- "make e2e-tests TEST_FILES=%q TEST_FILTER=%q TEST_SEED=%d TEST_PARALLELISM=%d %s=%q %s=%q",
+ "make e2e-tests TEST_FILES=%q TEST_FILTER=%q TEST_SEED=%d TEST_PARALLELISM=%d %s=%q %s=%q %s=%q",
rc.report.FileName(),
rc.report.FullText(),
rc.randomSeed,
rc.parallelism,
solrImageEnv, rc.solrImage,
operatorImageEnv, rc.operatorImage,
+ kubeVersionEnv, rc.kubeVersion,
)
}
type FailureInformation struct {
- namespace string
+ namespace string
+ outputDirectory string
}
// ColorableString for ReportEntry to use
@@ -171,18 +192,38 @@
// non-colorable String() is used by go's string formatting support but ignored by ReportEntry
func (fi FailureInformation) String() string {
return fmt.Sprintf(
- "Namespace: %s\n",
+ "Namespace: %s\nLogs Directory: %s\n",
fi.namespace,
+ fi.outputDirectory,
)
}
var _ = ReportAfterEach(func(report SpecReport) {
+ testName := cases.Title(language.AmericanEnglish, cases.NoLower).String(report.FullText())
+ testOutputDir := outputDir + "/" + strings.ReplaceAll(strings.ReplaceAll(testName, " ", "-"), " ", "")
+ // We count "ran" as "passed" or "failed"
+ if report.State.Is(types.SpecStatePassed | types.SpecStateFailureStates) {
+ Expect(os.Mkdir(testOutputDir, os.ModeDir|os.ModePerm)).To(Succeed(), "Could not create directory for test output: %s", testOutputDir)
+ testOutputDir += "/"
+ // Always save the logs of the Solr Operator for the test
+ writePodLogsToFile(
+ testOutputDir+"solr-operator.log",
+ getSolrOperatorPodName(solrOperatorReleaseNamespace),
+ solrOperatorReleaseNamespace,
+ report.StartTime,
+ fmt.Sprintf("%q: %q", "namespace", testNamespace()),
+ )
+ }
+
if report.Failed() {
ginkgoConfig, _ := GinkgoConfiguration()
+ testOutputDir, _ := filepath.Abs(testOutputDir)
AddReportEntry(
"Failure Information",
+ types.CodeLocation{},
FailureInformation{
- namespace: testNamespace(),
+ namespace: testNamespace(),
+ outputDirectory: testOutputDir,
},
)
AddReportEntry(
@@ -194,7 +235,57 @@
randomSeed: GinkgoRandomSeed(),
operatorImage: operatorImage,
solrImage: solrImage,
+ kubeVersion: kubeVersion,
},
)
}
})
+
+func getSolrOperatorPodName(namespace string) string {
+ labelSelector := labels.SelectorFromSet(map[string]string{"control-plane": "solr-operator"})
+ listOps := &client.ListOptions{
+ Namespace: namespace,
+ LabelSelector: labelSelector,
+ Limit: int64(1),
+ }
+
+ foundPods := &corev1.PodList{}
+ Expect(k8sClient.List(context.TODO(), foundPods, listOps)).To(Succeed(), "Could not fetch Solr Operator pod")
+ Expect(foundPods).ToNot(BeNil(), "No Solr Operator pods could be found")
+ Expect(foundPods.Items).ToNot(BeEmpty(), "No Solr Operator pods could be found")
+ return foundPods.Items[0].Name
+}
+
+func writePodLogsToFile(filename string, podName string, podNamespace string, startTimeRaw time.Time, filterLinesWithString string) {
+ logFile, err := os.Create(filename)
+ defer logFile.Close()
+ Expect(err).ToNot(HaveOccurred(), "Could not open file to save logs: %s", filename)
+
+ startTime := metav1.NewTime(startTimeRaw)
+ podLogOpts := corev1.PodLogOptions{
+ SinceTime: &startTime,
+ }
+ req := rawK8sClient.CoreV1().Pods(podNamespace).GetLogs(podName, &podLogOpts)
+ podLogs, logsErr := req.Stream(context.Background())
+ defer podLogs.Close()
+ Expect(logsErr).ToNot(HaveOccurred(), "Could not open stream to fetch pod logs. namespace: %s, pod: %s", podNamespace, podName)
+
+ var logReader io.Reader
+ logReader = podLogs
+
+ if filterLinesWithString != "" {
+ filteredWriter := bytes.NewBufferString("")
+ scanner := bufio.NewScanner(podLogs)
+ for scanner.Scan() {
+ line := scanner.Text()
+ if strings.Contains(line, filterLinesWithString) {
+ io.WriteString(filteredWriter, line)
+ io.WriteString(filteredWriter, "\n")
+ }
+ }
+ logReader = filteredWriter
+ }
+
+ _, err = io.Copy(logFile, logReader)
+ Expect(err).ToNot(HaveOccurred(), "Could not write podLogs to file: %s", filename)
+}
diff --git a/tests/e2e/test_utils_test.go b/tests/e2e/test_utils_test.go
index 2330e99..cc4a724 100644
--- a/tests/e2e/test_utils_test.go
+++ b/tests/e2e/test_utils_test.go
@@ -21,6 +21,7 @@
"bytes"
"context"
"encoding/json"
+ "errors"
"fmt"
solrv1beta1 "github.com/apache/solr-operator/api/v1beta1"
"github.com/apache/solr-operator/controllers/util"
@@ -87,7 +88,7 @@
histClient := action.NewHistory(actionConfig)
histClient.Max = 1
var solrOperatorHelmRelease *release.Release
- if _, err = histClient.Run(solrOperatorReleaseName); err == driver.ErrReleaseNotFound {
+ if _, err = histClient.Run(solrOperatorReleaseName); errors.Is(err, driver.ErrReleaseNotFound) {
installClient := action.NewInstall(actionConfig)
installClient.ReleaseName = solrOperatorReleaseName
