e2e/common/operator_metrics_test.go - camel-k - Git at Google

 // +build integration

 // To enable compilation of this file in Goland, go to "Settings -> Go -> Vendoring & Build Tags -> Custom Tags" and add "integration"

 /*
 Licensed to the Apache Software Foundation (ASF) under one or more
 contributor license agreements.  See the NOTICE file distributed with
 this work for additional information regarding copyright ownership.
 The ASF licenses this file to You under the Apache License, Version 2.0
 (the "License"); you may not use this file except in compliance with
 the License.  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */

 package common

 import (
 	"bytes"
 	"fmt"
 	"math"
 	"testing"
 	"time"

 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gstruct"
 	"github.com/onsi/gomega/types"

 	corev1 "k8s.io/api/core/v1"

 	prometheus "github.com/prometheus/client_model/go"
 	"github.com/prometheus/common/expfmt"

 	. "github.com/apache/camel-k/e2e/support"
 	. "github.com/apache/camel-k/e2e/support/util"
 	v1 "github.com/apache/camel-k/pkg/apis/camel/v1"
 )

 func TestMetrics(t *testing.T) {
 	WithNewTestNamespace(t, func(ns string) {
 		name := "java"
 		Expect(Kamel("install", "-n", ns).Execute()).To(Succeed())
 		Expect(Kamel("run", "-n", ns, "files/Java.java",
 			"-t", "prometheus.enabled=true",
 			"-t", "prometheus.pod-monitor=false").Execute()).To(Succeed())
 		Eventually(IntegrationPodPhase(ns, name), TestTimeoutMedium).Should(Equal(corev1.PodRunning))
 		Eventually(IntegrationCondition(ns, name, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue))
 		Eventually(IntegrationLogs(ns, "java"), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))

 		pod := OperatorPod(ns)()
 		Expect(pod).NotTo(BeNil())

 		logs := StructuredLogs(ns, pod.Name, corev1.PodLogOptions{})
 		Expect(logs).NotTo(BeEmpty())

 		response, err := TestClient().CoreV1().RESTClient().Get().
 			AbsPath(fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/proxy/metrics", ns, pod.Name)).DoRaw(TestContext)
 		Expect(err).To(BeNil())
 		metrics, err := parsePrometheusData(response)
 		Expect(err).To(BeNil())

 		it := Integration(ns, name)()
 		Expect(it).NotTo(BeNil())
 		build := Build(ns, it.Status.IntegrationKit.Name)()
 		Expect(build).NotTo(BeNil())

 		t.Run("Build duration metric", func(t *testing.T) {
 			// Get the duration from the Build status
 			duration, err := time.ParseDuration(build.Status.Duration)
 			Expect(err).To(BeNil())

 			// Check it's consistent with the duration observed from logs
 			var ts1, ts2 time.Time
 			err = NewLogWalker(&logs).
 				AddStep(MatchFields(IgnoreExtras, Fields{
 					"LoggerName":  Equal("camel-k.controller.build"),
 					"Message":     Equal("Build state transition"),
 					"Phase":       Equal(string(v1.BuildPhasePending)),
 					"RequestName": Equal(build.Name),
 				}), LogEntryNoop).
 				AddStep(MatchFields(IgnoreExtras, Fields{
 					"LoggerName":  Equal("camel-k.controller.build"),
 					"Message":     Equal("Reconciling Build"),
 					"RequestName": Equal(build.Name),
 				}), func(l *LogEntry) { ts1 = l.Timestamp.Time }).
 				AddStep(MatchFields(IgnoreExtras, Fields{
 					"LoggerName": Equal("camel-k.builder"),
 					"Message":    HavePrefix("resolved base image:"),
 				}), LogEntryNoop).
 				AddStep(MatchFields(IgnoreExtras, Fields{
 					"LoggerName":  Equal("camel-k.controller.build"),
 					"Message":     Equal("Reconciling Build"),
 					"RequestName": Equal(build.Name),
 				}), func(l *LogEntry) { ts2 = l.Timestamp.Time }).
 				Walk()
 			Expect(err).To(BeNil())
 			Expect(ts1).NotTo(BeZero())
 			Expect(ts2).NotTo(BeZero())
 			Expect(ts2).To(BeTemporally(">", ts1))

 			durationFromLogs := ts2.Sub(ts1)
 			Expect(math.Abs((durationFromLogs - duration).Seconds())).To(BeNumerically("<", 1))

 			// Check the duration is observed in the corresponding metric
 			Expect(metrics).To(HaveKeyWithValue("camel_k_build_duration_seconds",
 				EqualP(prometheus.MetricFamily{
 					Name: stringP("camel_k_build_duration_seconds"),
 					Help: stringP("Camel K build duration"),
 					Type: metricTypeP(prometheus.MetricType_HISTOGRAM),
 					Metric: []*prometheus.Metric{
 						{
 							Label: []*prometheus.LabelPair{
 								label("result", "Succeeded"),
 							},
 							Histogram: &prometheus.Histogram{
 								SampleCount: uint64P(1),
 								SampleSum:   float64P(duration.Seconds()),
 								Bucket:      buckets(duration.Seconds(), []float64{30, 60, 90, 120, 300, 600, math.Inf(1)}),
 							},
 						},
 					},
 				}),
 			))
 		})

 		t.Run("Build recovery attempts metric", func(t *testing.T) {
 			// Check there are no failures reported in the Build status
 			Expect(build.Status.Failure).To(BeNil())

 			// Check no recovery attempts are reported in the logs
 			recoveryAttempts, err := NewLogCounter(&logs).Count(MatchFields(IgnoreExtras, Fields{
 				"LoggerName":  Equal("camel-k.controller.build"),
 				"Message":     HavePrefix("Recovery attempt"),
 				"Kind":        Equal("Build"),
 				"RequestName": Equal(build.Name),
 			}))
 			Expect(err).To(BeNil())
 			Expect(recoveryAttempts).To(BeNumerically("==", 0))

 			// Check no recovery attempts are observed in the corresponding metric
 			Expect(metrics).To(HaveKeyWithValue("camel_k_build_recovery_attempts",
 				EqualP(prometheus.MetricFamily{
 					Name: stringP("camel_k_build_recovery_attempts"),
 					Help: stringP("Camel K build recovery attempts"),
 					Type: metricTypeP(prometheus.MetricType_HISTOGRAM),
 					Metric: []*prometheus.Metric{
 						{
 							Label: []*prometheus.LabelPair{
 								label("result", "Succeeded"),
 							},
 							Histogram: &prometheus.Histogram{
 								SampleCount: uint64P(1),
 								SampleSum:   float64P(0),
 								Bucket:      buckets(0, []float64{0, 1, 2, 3, 4, 5, math.Inf(1)}),
 							},
 						},
 					},
 				}),
 			))
 		})

 		t.Run("reconciliation duration metric", func(t *testing.T) {
 			Expect(metrics).To(HaveKeyWithValue("camel_k_reconciliation_duration_seconds",
 				PointTo(MatchFields(IgnoreExtras, Fields{
 					"Name": EqualP("camel_k_reconciliation_duration_seconds"),
 					"Help": EqualP("Camel K reconciliation loop duration"),
 					"Type": EqualP(prometheus.MetricType_HISTOGRAM),
 				})),
 			))

 			counter := NewLogCounter(&logs)

 			// Count the number of IntegrationPlatform reconciliations
 			platformReconciliations, err := counter.Count(MatchFields(IgnoreExtras, Fields{
 				"LoggerName":       Equal("camel-k.controller.integrationplatform"),
 				"Message":          Equal("Reconciling IntegrationPlatform"),
 				"RequestNamespace": Equal(ns),
 				"RequestName":      Equal("camel-k"),
 			}))
 			Expect(err).To(BeNil())

 			// Check it matches the observation in the corresponding metric
 			platformReconciled := getMetric(metrics["camel_k_reconciliation_duration_seconds"],
 				MatchFieldsP(IgnoreExtras, Fields{
 					"Label": ConsistOf(
 						label("group", v1.SchemeGroupVersion.Group),
 						label("version", v1.SchemeGroupVersion.Version),
 						label("kind", "IntegrationPlatform"),
 						label("namespace", ns),
 						label("result", "Reconciled"),
 						label("tag", ""),
 					),
 				}))
 			Expect(platformReconciled).NotTo(BeNil())
 			platformReconciledCount := *platformReconciled.Histogram.SampleCount
 			Expect(platformReconciledCount).To(BeNumerically(">", 0))

 			platformRequeued := getMetric(metrics["camel_k_reconciliation_duration_seconds"],
 				MatchFieldsP(IgnoreExtras, Fields{
 					"Label": ConsistOf(
 						label("group", v1.SchemeGroupVersion.Group),
 						label("version", v1.SchemeGroupVersion.Version),
 						label("kind", "IntegrationPlatform"),
 						label("namespace", ns),
 						label("result", "Requeued"),
 						label("tag", ""),
 					),
 				}))
 			platformRequeuedCount := uint64(0)
 			if platformRequeued != nil {
 				platformRequeuedCount = *platformRequeued.Histogram.SampleCount
 			}

 			Expect(platformReconciliations).To(BeNumerically("==", platformReconciledCount+platformRequeuedCount))

 			// Count the number of Integration reconciliations
 			integrationReconciliations, err := counter.Count(MatchFields(IgnoreExtras, Fields{
 				"LoggerName":       Equal("camel-k.controller.integration"),
 				"Message":          Equal("Reconciling Integration"),
 				"RequestNamespace": Equal(it.Namespace),
 				"RequestName":      Equal(it.Name),
 			}))
 			Expect(err).To(BeNil())
 			Expect(integrationReconciliations).To(BeNumerically(">", 0))

 			// Check it matches the observation in the corresponding metric
 			Expect(metrics).To(HaveKeyWithValue("camel_k_reconciliation_duration_seconds",
 				PointTo(MatchFields(IgnoreExtras, Fields{
 					"Name": EqualP("camel_k_reconciliation_duration_seconds"),
 					"Help": EqualP("Camel K reconciliation loop duration"),
 					"Type": EqualP(prometheus.MetricType_HISTOGRAM),
 					"Metric": ContainElement(MatchFieldsP(IgnoreExtras, Fields{
 						"Label": ConsistOf(
 							label("group", v1.SchemeGroupVersion.Group),
 							label("version", v1.SchemeGroupVersion.Version),
 							label("kind", "Integration"),
 							label("namespace", it.Namespace),
 							label("result", "Reconciled"),
 							label("tag", ""),
 						),
 						"Histogram": MatchFieldsP(IgnoreExtras, Fields{
 							"SampleCount": EqualP(uint64(integrationReconciliations)),
 						}),
 					})),
 				})),
 			))

 			// Count the number of IntegrationKit reconciliations
 			integrationKitReconciliations, err := counter.Count(MatchFields(IgnoreExtras, Fields{
 				"LoggerName":       Equal("camel-k.controller.integrationkit"),
 				"Message":          Equal("Reconciling IntegrationKit"),
 				"RequestNamespace": Equal(it.Status.IntegrationKit.Namespace),
 				"RequestName":      Equal(it.Status.IntegrationKit.Name),
 			}))
 			Expect(err).To(BeNil())
 			Expect(integrationKitReconciliations).To(BeNumerically(">", 0))

 			// Check it matches the observation in the corresponding metric
 			Expect(metrics).To(HaveKeyWithValue("camel_k_reconciliation_duration_seconds",
 				PointTo(MatchFields(IgnoreExtras, Fields{
 					"Name": EqualP("camel_k_reconciliation_duration_seconds"),
 					"Help": EqualP("Camel K reconciliation loop duration"),
 					"Type": EqualP(prometheus.MetricType_HISTOGRAM),
 					"Metric": ContainElement(MatchFieldsP(IgnoreExtras, Fields{
 						"Label": ConsistOf(
 							label("group", v1.SchemeGroupVersion.Group),
 							label("version", v1.SchemeGroupVersion.Version),
 							label("kind", "IntegrationKit"),
 							label("namespace", it.Status.IntegrationKit.Namespace),
 							label("result", "Reconciled"),
 							label("tag", ""),
 						),
 						"Histogram": MatchFieldsP(IgnoreExtras, Fields{
 							"SampleCount": EqualP(uint64(integrationKitReconciliations)),
 						}),
 					})),
 				})),
 			))

 			// Count the number of Build reconciliations
 			buildReconciliations, err := counter.Count(MatchFields(IgnoreExtras, Fields{
 				"LoggerName":       Equal("camel-k.controller.build"),
 				"Message":          Equal("Reconciling Build"),
 				"RequestNamespace": Equal(build.Namespace),
 				"RequestName":      Equal(build.Name),
 			}))
 			Expect(err).To(BeNil())

 			// Check it matches the observation in the corresponding metric
 			buildReconciled := getMetric(metrics["camel_k_reconciliation_duration_seconds"],
 				MatchFieldsP(IgnoreExtras, Fields{
 					"Label": ConsistOf(
 						label("group", v1.SchemeGroupVersion.Group),
 						label("version", v1.SchemeGroupVersion.Version),
 						label("kind", "Build"),
 						label("namespace", build.Namespace),
 						label("result", "Reconciled"),
 						label("tag", ""),
 					),
 				}))
 			Expect(buildReconciled).NotTo(BeNil())
 			buildReconciledCount := *buildReconciled.Histogram.SampleCount
 			Expect(buildReconciledCount).To(BeNumerically(">", 0))

 			buildRequeued := getMetric(metrics["camel_k_reconciliation_duration_seconds"],
 				MatchFieldsP(IgnoreExtras, Fields{
 					"Label": ConsistOf(
 						label("group", v1.SchemeGroupVersion.Group),
 						label("version", v1.SchemeGroupVersion.Version),
 						label("kind", "Build"),
 						label("namespace", build.Namespace),
 						label("result", "Requeued"),
 						label("tag", ""),
 					),
 				}))
 			buildRequeuedCount := uint64(0)
 			if buildRequeued != nil {
 				buildRequeuedCount = *buildRequeued.Histogram.SampleCount
 			}

 			Expect(buildReconciliations).To(BeNumerically("==", buildReconciledCount+buildRequeuedCount))
 		})

 		t.Run("Build queue duration metric", func(t *testing.T) {
 			var ts1, ts2 time.Time
 			// The start queuing time is taken from the creation time
 			ts1 = build.CreationTimestamp.Time

 			// Retrieve the end queuing time from the logs
 			err = NewLogWalker(&logs).
 				AddStep(MatchFields(IgnoreExtras, Fields{
 					"LoggerName":  Equal("camel-k.controller.build"),
 					"Message":     Equal("Build state transition"),
 					"Phase":       Equal(string(v1.BuildPhasePending)),
 					"RequestName": Equal(build.Name),
 				}), func(l *LogEntry) { ts2 = l.Timestamp.Time }).
 				Walk()
 			Expect(err).To(BeNil())
 			Expect(ts1).NotTo(BeZero())
 			Expect(ts2).NotTo(BeZero())

 			durationFromLogs := ts2.Sub(ts1)

 			// Retrieve the queuing duration from the metric
 			Expect(metrics).To(HaveKey("camel_k_build_queue_duration_seconds"))
 			metric := metrics["camel_k_build_queue_duration_seconds"].Metric
 			Expect(metric).To(HaveLen(1))
 			histogram := metric[0].Histogram
 			Expect(histogram).NotTo(BeNil())
 			Expect(histogram.SampleSum).NotTo(BeNil())

 			duration := *histogram.SampleSum

 			// Check both durations match
 			Expect(math.Abs(durationFromLogs.Seconds() - duration)).To(BeNumerically("<", 1))

 			// Check the queuing duration is correctly observed in the corresponding metric
 			Expect(metrics).To(HaveKeyWithValue("camel_k_build_queue_duration_seconds",
 				EqualP(prometheus.MetricFamily{
 					Name: stringP("camel_k_build_queue_duration_seconds"),
 					Help: stringP("Camel K build queue duration"),
 					Type: metricTypeP(prometheus.MetricType_HISTOGRAM),
 					Metric: []*prometheus.Metric{
 						{
 							Histogram: &prometheus.Histogram{
 								SampleCount: uint64P(1),
 								SampleSum:   histogram.SampleSum,
 								Bucket:      buckets(0, []float64{5, 15, 30, 60, 300, math.Inf(1)}),
 							},
 						},
 					},
 				}),
 			))
 		})

 		t.Run("Integration first readiness metric", func(t *testing.T) {
 			var ts1, ts2 time.Time

 			// The start time is taken from the Integration status initialization timestamp
 			ts1 = it.Status.InitializationTimestamp.Time
 			Expect(ts1).NotTo(BeZero())
 			// The end time is reported into the ready condition first truthy time
 			ts2 = it.Status.GetCondition(v1.IntegrationConditionReady).FirstTruthyTime.Time
 			Expect(ts2).NotTo(BeZero())

 			duration := ts2.Sub(ts1)

 			// Retrieve these start and end times from the logs
 			err = NewLogWalker(&logs).
 				AddStep(MatchFields(IgnoreExtras, Fields{
 					"LoggerName":  Equal("camel-k.controller.integration"),
 					"Message":     Equal("Reconciling Integration"),
 					"RequestName": Equal(it.Name),
 					"PhaseFrom":   Equal(string(v1.IntegrationPhaseInitialization)),
 					"PhaseTo":     Equal(string(v1.IntegrationPhaseBuildingKit)),
 				}), func(l *LogEntry) { ts1 = l.Timestamp.Time }).
 				AddStep(MatchFields(IgnoreExtras, Fields{
 					"LoggerName":  Equal("camel-k.controller.integration"),
 					"Message":     HavePrefix("First readiness"),
 					"RequestName": Equal(it.Name),
 				}), func(l *LogEntry) { ts2 = l.Timestamp.Time }).
 				Walk()
 			Expect(err).To(BeNil())
 			Expect(ts1).NotTo(BeZero())
 			Expect(ts2).NotTo(BeZero())
 			Expect(ts2).To(BeTemporally(">", ts1))
 			durationFromLogs := ts2.Sub(ts1)

 			// Check both durations match
 			Expect(math.Abs((durationFromLogs - duration).Seconds())).To(BeNumerically("<", 1))

 			// Check the duration is correctly observed in the corresponding metric
 			Expect(metrics).To(HaveKeyWithValue("camel_k_integration_first_readiness_seconds",
 				EqualP(prometheus.MetricFamily{
 					Name: stringP("camel_k_integration_first_readiness_seconds"),
 					Help: stringP("Camel K integration time to first readiness"),
 					Type: metricTypeP(prometheus.MetricType_HISTOGRAM),
 					Metric: []*prometheus.Metric{
 						{
 							Histogram: &prometheus.Histogram{
 								SampleCount: uint64P(1),
 								SampleSum:   float64P(duration.Seconds()),
 								Bucket:      buckets(duration.Seconds(), []float64{5, 10, 30, 60, 120, math.Inf(1)}),
 							},
 						},
 					},
 				}),
 			))
 		})

 		// Clean up
 		Expect(Kamel("delete", "--all", "-n", ns).Execute()).To(Succeed())
 	})
 }

 func getMetric(family *prometheus.MetricFamily, matcher types.GomegaMatcher) *prometheus.Metric {
 	for _, metric := range family.Metric {
 		if match, err := matcher.Match(metric); err != nil {
 			panic(err)
 		} else if match {
 			return metric
 		}
 	}
 	return nil
 }

 func label(name, value string) *prometheus.LabelPair {
 	return &prometheus.LabelPair{
 		Name:  &name,
 		Value: &value,
 	}
 }

 func bucket(value float64, upperBound float64) *prometheus.Bucket {
 	var count uint64
 	if value <= upperBound {
 		count++
 	}
 	return &prometheus.Bucket{
 		UpperBound:      float64P(upperBound),
 		CumulativeCount: &count,
 	}
 }

 func buckets(value float64, upperBounds []float64) []*prometheus.Bucket {
 	var buckets []*prometheus.Bucket
 	for _, upperBound := range upperBounds {
 		buckets = append(buckets, bucket(value, upperBound))
 	}
 	return buckets
 }

 // https://prometheus.io/docs/instrumenting/exposition_formats/
 func parsePrometheusData(data []byte) (map[string]*prometheus.MetricFamily, error) {
 	var parser expfmt.TextParser
 	return parser.TextToMetricFamilies(bytes.NewReader(data))
 }

 func EqualP(expected interface{}) types.GomegaMatcher {
 	return PointTo(Equal(expected))
 }

 func MatchFieldsP(options Options, fields Fields) types.GomegaMatcher {
 	return PointTo(MatchFields(options, fields))
 }

 func stringP(s string) *string {
 	return &s
 }

 func metricTypeP(t prometheus.MetricType) *prometheus.MetricType {
 	return &t
 }

 func uint64P(i uint64) *uint64 {
 	return &i
 }

 func float64P(f float64) *float64 {
 	return &f
 }
	// +build integration

	// To enable compilation of this file in Goland, go to "Settings -> Go -> Vendoring & Build Tags -> Custom Tags" and add "integration"

	/*
	Licensed to the Apache Software Foundation (ASF) under one or more
	contributor license agreements. See the NOTICE file distributed with
	this work for additional information regarding copyright ownership.
	The ASF licenses this file to You under the Apache License, Version 2.0
	(the "License"); you may not use this file except in compliance with
	the License. You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

	Unless required by applicable law or agreed to in writing, software
	distributed under the License is distributed on an "AS IS" BASIS,
	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	See the License for the specific language governing permissions and
	limitations under the License.
	*/

	package common

	import (
	"bytes"
	"fmt"
	"math"
	"testing"
	"time"

	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gstruct"
	"github.com/onsi/gomega/types"

	corev1 "k8s.io/api/core/v1"

	prometheus "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"

	. "github.com/apache/camel-k/e2e/support"
	. "github.com/apache/camel-k/e2e/support/util"
	v1 "github.com/apache/camel-k/pkg/apis/camel/v1"
	)

	func TestMetrics(t *testing.T) {
	WithNewTestNamespace(t, func(ns string) {
	name := "java"
	Expect(Kamel("install", "-n", ns).Execute()).To(Succeed())
	Expect(Kamel("run", "-n", ns, "files/Java.java",
	"-t", "prometheus.enabled=true",
	"-t", "prometheus.pod-monitor=false").Execute()).To(Succeed())
	Eventually(IntegrationPodPhase(ns, name), TestTimeoutMedium).Should(Equal(corev1.PodRunning))
	Eventually(IntegrationCondition(ns, name, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue))
	Eventually(IntegrationLogs(ns, "java"), TestTimeoutShort).Should(ContainSubstring("Magicstring!"))

	pod := OperatorPod(ns)()
	Expect(pod).NotTo(BeNil())

	logs := StructuredLogs(ns, pod.Name, corev1.PodLogOptions{})
	Expect(logs).NotTo(BeEmpty())

	response, err := TestClient().CoreV1().RESTClient().Get().
	AbsPath(fmt.Sprintf("/api/v1/namespaces/%s/pods/%s/proxy/metrics", ns, pod.Name)).DoRaw(TestContext)
	Expect(err).To(BeNil())
	metrics, err := parsePrometheusData(response)
	Expect(err).To(BeNil())

	it := Integration(ns, name)()
	Expect(it).NotTo(BeNil())
	build := Build(ns, it.Status.IntegrationKit.Name)()
	Expect(build).NotTo(BeNil())

	t.Run("Build duration metric", func(t *testing.T) {
	// Get the duration from the Build status
	duration, err := time.ParseDuration(build.Status.Duration)
	Expect(err).To(BeNil())

	// Check it's consistent with the duration observed from logs
	var ts1, ts2 time.Time
	err = NewLogWalker(&logs).
	AddStep(MatchFields(IgnoreExtras, Fields{
	"LoggerName": Equal("camel-k.controller.build"),
	"Message": Equal("Build state transition"),
	"Phase": Equal(string(v1.BuildPhasePending)),
	"RequestName": Equal(build.Name),
	}), LogEntryNoop).
	AddStep(MatchFields(IgnoreExtras, Fields{
	"LoggerName": Equal("camel-k.controller.build"),
	"Message": Equal("Reconciling Build"),
	"RequestName": Equal(build.Name),
	}), func(l *LogEntry) { ts1 = l.Timestamp.Time }).
	AddStep(MatchFields(IgnoreExtras, Fields{
	"LoggerName": Equal("camel-k.builder"),
	"Message": HavePrefix("resolved base image:"),
	}), LogEntryNoop).
	AddStep(MatchFields(IgnoreExtras, Fields{
	"LoggerName": Equal("camel-k.controller.build"),
	"Message": Equal("Reconciling Build"),
	"RequestName": Equal(build.Name),
	}), func(l *LogEntry) { ts2 = l.Timestamp.Time }).
	Walk()
	Expect(err).To(BeNil())
	Expect(ts1).NotTo(BeZero())
	Expect(ts2).NotTo(BeZero())
	Expect(ts2).To(BeTemporally(">", ts1))

	durationFromLogs := ts2.Sub(ts1)
	Expect(math.Abs((durationFromLogs - duration).Seconds())).To(BeNumerically("<", 1))

	// Check the duration is observed in the corresponding metric
	Expect(metrics).To(HaveKeyWithValue("camel_k_build_duration_seconds",
	EqualP(prometheus.MetricFamily{
	Name: stringP("camel_k_build_duration_seconds"),
	Help: stringP("Camel K build duration"),
	Type: metricTypeP(prometheus.MetricType_HISTOGRAM),
	Metric: []*prometheus.Metric{
	{
	Label: []*prometheus.LabelPair{
	label("result", "Succeeded"),
	},
	Histogram: &prometheus.Histogram{
	SampleCount: uint64P(1),
	SampleSum: float64P(duration.Seconds()),
	Bucket: buckets(duration.Seconds(), []float64{30, 60, 90, 120, 300, 600, math.Inf(1)}),
	},
	},
	},
	}),
	))
	})

	t.Run("Build recovery attempts metric", func(t *testing.T) {
	// Check there are no failures reported in the Build status
	Expect(build.Status.Failure).To(BeNil())

	// Check no recovery attempts are reported in the logs
	recoveryAttempts, err := NewLogCounter(&logs).Count(MatchFields(IgnoreExtras, Fields{
	"LoggerName": Equal("camel-k.controller.build"),
	"Message": HavePrefix("Recovery attempt"),
	"Kind": Equal("Build"),
	"RequestName": Equal(build.Name),
	}))
	Expect(err).To(BeNil())
	Expect(recoveryAttempts).To(BeNumerically("==", 0))

	// Check no recovery attempts are observed in the corresponding metric
	Expect(metrics).To(HaveKeyWithValue("camel_k_build_recovery_attempts",
	EqualP(prometheus.MetricFamily{
	Name: stringP("camel_k_build_recovery_attempts"),
	Help: stringP("Camel K build recovery attempts"),
	Type: metricTypeP(prometheus.MetricType_HISTOGRAM),
	Metric: []*prometheus.Metric{
	{
	Label: []*prometheus.LabelPair{
	label("result", "Succeeded"),
	},
	Histogram: &prometheus.Histogram{
	SampleCount: uint64P(1),
	SampleSum: float64P(0),
	Bucket: buckets(0, []float64{0, 1, 2, 3, 4, 5, math.Inf(1)}),
	},
	},
	},
	}),
	))
	})

	t.Run("reconciliation duration metric", func(t *testing.T) {
	Expect(metrics).To(HaveKeyWithValue("camel_k_reconciliation_duration_seconds",
	PointTo(MatchFields(IgnoreExtras, Fields{
	"Name": EqualP("camel_k_reconciliation_duration_seconds"),
	"Help": EqualP("Camel K reconciliation loop duration"),
	"Type": EqualP(prometheus.MetricType_HISTOGRAM),
	})),
	))

	counter := NewLogCounter(&logs)

	// Count the number of IntegrationPlatform reconciliations
	platformReconciliations, err := counter.Count(MatchFields(IgnoreExtras, Fields{
	"LoggerName": Equal("camel-k.controller.integrationplatform"),
	"Message": Equal("Reconciling IntegrationPlatform"),
	"RequestNamespace": Equal(ns),
	"RequestName": Equal("camel-k"),
	}))
	Expect(err).To(BeNil())

	// Check it matches the observation in the corresponding metric
	platformReconciled := getMetric(metrics["camel_k_reconciliation_duration_seconds"],
	MatchFieldsP(IgnoreExtras, Fields{
	"Label": ConsistOf(
	label("group", v1.SchemeGroupVersion.Group),
	label("version", v1.SchemeGroupVersion.Version),
	label("kind", "IntegrationPlatform"),
	label("namespace", ns),
	label("result", "Reconciled"),
	label("tag", ""),
	),
	}))
	Expect(platformReconciled).NotTo(BeNil())
	platformReconciledCount := *platformReconciled.Histogram.SampleCount
	Expect(platformReconciledCount).To(BeNumerically(">", 0))

	platformRequeued := getMetric(metrics["camel_k_reconciliation_duration_seconds"],
	MatchFieldsP(IgnoreExtras, Fields{
	"Label": ConsistOf(
	label("group", v1.SchemeGroupVersion.Group),
	label("version", v1.SchemeGroupVersion.Version),
	label("kind", "IntegrationPlatform"),
	label("namespace", ns),
	label("result", "Requeued"),
	label("tag", ""),
	),
	}))
	platformRequeuedCount := uint64(0)
	if platformRequeued != nil {
	platformRequeuedCount = *platformRequeued.Histogram.SampleCount
	}

	Expect(platformReconciliations).To(BeNumerically("==", platformReconciledCount+platformRequeuedCount))

	// Count the number of Integration reconciliations
	integrationReconciliations, err := counter.Count(MatchFields(IgnoreExtras, Fields{
	"LoggerName": Equal("camel-k.controller.integration"),
	"Message": Equal("Reconciling Integration"),
	"RequestNamespace": Equal(it.Namespace),
	"RequestName": Equal(it.Name),
	}))
	Expect(err).To(BeNil())
	Expect(integrationReconciliations).To(BeNumerically(">", 0))

	// Check it matches the observation in the corresponding metric
	Expect(metrics).To(HaveKeyWithValue("camel_k_reconciliation_duration_seconds",
	PointTo(MatchFields(IgnoreExtras, Fields{
	"Name": EqualP("camel_k_reconciliation_duration_seconds"),
	"Help": EqualP("Camel K reconciliation loop duration"),
	"Type": EqualP(prometheus.MetricType_HISTOGRAM),
	"Metric": ContainElement(MatchFieldsP(IgnoreExtras, Fields{
	"Label": ConsistOf(
	label("group", v1.SchemeGroupVersion.Group),
	label("version", v1.SchemeGroupVersion.Version),
	label("kind", "Integration"),
	label("namespace", it.Namespace),
	label("result", "Reconciled"),
	label("tag", ""),
	),
	"Histogram": MatchFieldsP(IgnoreExtras, Fields{
	"SampleCount": EqualP(uint64(integrationReconciliations)),
	}),
	})),
	})),
	))

	// Count the number of IntegrationKit reconciliations
	integrationKitReconciliations, err := counter.Count(MatchFields(IgnoreExtras, Fields{
	"LoggerName": Equal("camel-k.controller.integrationkit"),
	"Message": Equal("Reconciling IntegrationKit"),
	"RequestNamespace": Equal(it.Status.IntegrationKit.Namespace),
	"RequestName": Equal(it.Status.IntegrationKit.Name),
	}))
	Expect(err).To(BeNil())
	Expect(integrationKitReconciliations).To(BeNumerically(">", 0))

	// Check it matches the observation in the corresponding metric
	Expect(metrics).To(HaveKeyWithValue("camel_k_reconciliation_duration_seconds",
	PointTo(MatchFields(IgnoreExtras, Fields{
	"Name": EqualP("camel_k_reconciliation_duration_seconds"),
	"Help": EqualP("Camel K reconciliation loop duration"),
	"Type": EqualP(prometheus.MetricType_HISTOGRAM),
	"Metric": ContainElement(MatchFieldsP(IgnoreExtras, Fields{
	"Label": ConsistOf(
	label("group", v1.SchemeGroupVersion.Group),
	label("version", v1.SchemeGroupVersion.Version),
	label("kind", "IntegrationKit"),
	label("namespace", it.Status.IntegrationKit.Namespace),
	label("result", "Reconciled"),
	label("tag", ""),
	),
	"Histogram": MatchFieldsP(IgnoreExtras, Fields{
	"SampleCount": EqualP(uint64(integrationKitReconciliations)),
	}),
	})),
	})),
	))

	// Count the number of Build reconciliations
	buildReconciliations, err := counter.Count(MatchFields(IgnoreExtras, Fields{
	"LoggerName": Equal("camel-k.controller.build"),
	"Message": Equal("Reconciling Build"),
	"RequestNamespace": Equal(build.Namespace),
	"RequestName": Equal(build.Name),
	}))
	Expect(err).To(BeNil())

	// Check it matches the observation in the corresponding metric
	buildReconciled := getMetric(metrics["camel_k_reconciliation_duration_seconds"],
	MatchFieldsP(IgnoreExtras, Fields{
	"Label": ConsistOf(
	label("group", v1.SchemeGroupVersion.Group),
	label("version", v1.SchemeGroupVersion.Version),
	label("kind", "Build"),
	label("namespace", build.Namespace),
	label("result", "Reconciled"),
	label("tag", ""),
	),
	}))
	Expect(buildReconciled).NotTo(BeNil())
	buildReconciledCount := *buildReconciled.Histogram.SampleCount
	Expect(buildReconciledCount).To(BeNumerically(">", 0))

	buildRequeued := getMetric(metrics["camel_k_reconciliation_duration_seconds"],
	MatchFieldsP(IgnoreExtras, Fields{
	"Label": ConsistOf(
	label("group", v1.SchemeGroupVersion.Group),
	label("version", v1.SchemeGroupVersion.Version),
	label("kind", "Build"),
	label("namespace", build.Namespace),
	label("result", "Requeued"),
	label("tag", ""),
	),
	}))
	buildRequeuedCount := uint64(0)
	if buildRequeued != nil {
	buildRequeuedCount = *buildRequeued.Histogram.SampleCount
	}

	Expect(buildReconciliations).To(BeNumerically("==", buildReconciledCount+buildRequeuedCount))
	})

	t.Run("Build queue duration metric", func(t *testing.T) {
	var ts1, ts2 time.Time
	// The start queuing time is taken from the creation time
	ts1 = build.CreationTimestamp.Time

	// Retrieve the end queuing time from the logs
	err = NewLogWalker(&logs).
	AddStep(MatchFields(IgnoreExtras, Fields{
	"LoggerName": Equal("camel-k.controller.build"),
	"Message": Equal("Build state transition"),
	"Phase": Equal(string(v1.BuildPhasePending)),
	"RequestName": Equal(build.Name),
	}), func(l *LogEntry) { ts2 = l.Timestamp.Time }).
	Walk()
	Expect(err).To(BeNil())
	Expect(ts1).NotTo(BeZero())
	Expect(ts2).NotTo(BeZero())

	durationFromLogs := ts2.Sub(ts1)

	// Retrieve the queuing duration from the metric
	Expect(metrics).To(HaveKey("camel_k_build_queue_duration_seconds"))
	metric := metrics["camel_k_build_queue_duration_seconds"].Metric
	Expect(metric).To(HaveLen(1))
	histogram := metric[0].Histogram
	Expect(histogram).NotTo(BeNil())
	Expect(histogram.SampleSum).NotTo(BeNil())

	duration := *histogram.SampleSum

	// Check both durations match
	Expect(math.Abs(durationFromLogs.Seconds() - duration)).To(BeNumerically("<", 1))

	// Check the queuing duration is correctly observed in the corresponding metric
	Expect(metrics).To(HaveKeyWithValue("camel_k_build_queue_duration_seconds",
	EqualP(prometheus.MetricFamily{
	Name: stringP("camel_k_build_queue_duration_seconds"),
	Help: stringP("Camel K build queue duration"),
	Type: metricTypeP(prometheus.MetricType_HISTOGRAM),
	Metric: []*prometheus.Metric{
	{
	Histogram: &prometheus.Histogram{
	SampleCount: uint64P(1),
	SampleSum: histogram.SampleSum,
	Bucket: buckets(0, []float64{5, 15, 30, 60, 300, math.Inf(1)}),
	},
	},
	},
	}),
	))
	})

	t.Run("Integration first readiness metric", func(t *testing.T) {
	var ts1, ts2 time.Time

	// The start time is taken from the Integration status initialization timestamp
	ts1 = it.Status.InitializationTimestamp.Time
	Expect(ts1).NotTo(BeZero())
	// The end time is reported into the ready condition first truthy time
	ts2 = it.Status.GetCondition(v1.IntegrationConditionReady).FirstTruthyTime.Time
	Expect(ts2).NotTo(BeZero())

	duration := ts2.Sub(ts1)

	// Retrieve these start and end times from the logs
	err = NewLogWalker(&logs).
	AddStep(MatchFields(IgnoreExtras, Fields{
	"LoggerName": Equal("camel-k.controller.integration"),
	"Message": Equal("Reconciling Integration"),
	"RequestName": Equal(it.Name),
	"PhaseFrom": Equal(string(v1.IntegrationPhaseInitialization)),
	"PhaseTo": Equal(string(v1.IntegrationPhaseBuildingKit)),
	}), func(l *LogEntry) { ts1 = l.Timestamp.Time }).
	AddStep(MatchFields(IgnoreExtras, Fields{
	"LoggerName": Equal("camel-k.controller.integration"),
	"Message": HavePrefix("First readiness"),
	"RequestName": Equal(it.Name),
	}), func(l *LogEntry) { ts2 = l.Timestamp.Time }).
	Walk()
	Expect(err).To(BeNil())
	Expect(ts1).NotTo(BeZero())
	Expect(ts2).NotTo(BeZero())
	Expect(ts2).To(BeTemporally(">", ts1))
	durationFromLogs := ts2.Sub(ts1)

	// Check both durations match
	Expect(math.Abs((durationFromLogs - duration).Seconds())).To(BeNumerically("<", 1))

	// Check the duration is correctly observed in the corresponding metric
	Expect(metrics).To(HaveKeyWithValue("camel_k_integration_first_readiness_seconds",
	EqualP(prometheus.MetricFamily{
	Name: stringP("camel_k_integration_first_readiness_seconds"),
	Help: stringP("Camel K integration time to first readiness"),
	Type: metricTypeP(prometheus.MetricType_HISTOGRAM),
	Metric: []*prometheus.Metric{
	{
	Histogram: &prometheus.Histogram{
	SampleCount: uint64P(1),
	SampleSum: float64P(duration.Seconds()),
	Bucket: buckets(duration.Seconds(), []float64{5, 10, 30, 60, 120, math.Inf(1)}),
	},
	},
	},
	}),
	))
	})

	// Clean up
	Expect(Kamel("delete", "--all", "-n", ns).Execute()).To(Succeed())
	})
	}

	func getMetric(family prometheus.MetricFamily, matcher types.GomegaMatcher) prometheus.Metric {
	for _, metric := range family.Metric {
	if match, err := matcher.Match(metric); err != nil {
	panic(err)
	} else if match {
	return metric
	}
	}
	return nil
	}

	func label(name, value string) *prometheus.LabelPair {
	return &prometheus.LabelPair{
	Name: &name,
	Value: &value,
	}
	}

	func bucket(value float64, upperBound float64) *prometheus.Bucket {
	var count uint64
	if value <= upperBound {
	count++
	}
	return &prometheus.Bucket{
	UpperBound: float64P(upperBound),
	CumulativeCount: &count,
	}
	}

	func buckets(value float64, upperBounds []float64) []*prometheus.Bucket {
	var buckets []*prometheus.Bucket
	for _, upperBound := range upperBounds {
	buckets = append(buckets, bucket(value, upperBound))
	}
	return buckets
	}

	// https://prometheus.io/docs/instrumenting/exposition_formats/
	func parsePrometheusData(data []byte) (map[string]*prometheus.MetricFamily, error) {
	var parser expfmt.TextParser
	return parser.TextToMetricFamilies(bytes.NewReader(data))
	}

	func EqualP(expected interface{}) types.GomegaMatcher {
	return PointTo(Equal(expected))
	}

	func MatchFieldsP(options Options, fields Fields) types.GomegaMatcher {
	return PointTo(MatchFields(options, fields))
	}

	func stringP(s string) *string {
	return &s
	}

	func metricTypeP(t prometheus.MetricType) *prometheus.MetricType {
	return &t
	}

	func uint64P(i uint64) *uint64 {
	return &i
	}

	func float64P(f float64) *float64 {
	return &f
	}