| /* |
| * Licensed to the Apache Software Foundation (ASF) under one or more |
| * contributor license agreements. See the NOTICE file distributed with |
| * this work for additional information regarding copyright ownership. |
| * The ASF licenses this file to You under the Apache License, Version 2.0 |
| * (the "License"); you may not use this file except in compliance with |
| * the License. You may obtain a copy of the License at |
| * |
| * http://www.apache.org/licenses/LICENSE-2.0 |
| * |
| * Unless required by applicable law or agreed to in writing, software |
| * distributed under the License is distributed on an "AS IS" BASIS, |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| * See the License for the specific language governing permissions and |
| * limitations under the License. |
| */ |
| |
| package util |
| |
| import ( |
| "strconv" |
| "strings" |
| |
| solr "github.com/apache/solr-operator/api/v1beta1" |
| appsv1 "k8s.io/api/apps/v1" |
| corev1 "k8s.io/api/core/v1" |
| netv1 "k8s.io/api/networking/v1beta1" |
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
| "k8s.io/apimachinery/pkg/util/intstr" |
| ) |
| |
| const ( |
| SolrMetricsPort = 8080 |
| SolrMetricsPortName = "solr-metrics" |
| ExtSolrMetricsPort = 80 |
| |
| SolrPrometheusExporterContainer = "solr-prometheus-exporter" |
| DefaultPrometheusExporterEntrypoint = "/opt/solr/contrib/prometheus-exporter/bin/solr-exporter" |
| PrometheusExporterConfigMapKey = "solr-prometheus-exporter.xml" |
| PrometheusExporterConfigXmlMd5Annotation = "solr.apache.org/exporterConfigXmlMd5" |
| ) |
| |
| // SolrConnectionInfo defines how to connect to a cloud or standalone solr instance. |
| // One, and only one, of Cloud or Standalone must be provided. |
| type SolrConnectionInfo struct { |
| CloudZkConnnectionInfo *solr.ZookeeperConnectionInfo |
| StandaloneAddress string |
| } |
| |
| // GenerateSolrPrometheusExporterDeployment returns a new appsv1.Deployment pointer generated for the SolrCloud Prometheus Exporter instance |
| // solrPrometheusExporter: SolrPrometheusExporter instance |
| func GenerateSolrPrometheusExporterDeployment(solrPrometheusExporter *solr.SolrPrometheusExporter, solrConnectionInfo SolrConnectionInfo, solrCloudImage *solr.ContainerImage, configXmlMd5 string, tls *TLSCerts, basicAuthMd5 string) *appsv1.Deployment { |
| gracePeriodTerm := int64(10) |
| singleReplica := int32(1) |
| fsGroup := int64(SolrMetricsPort) |
| |
| labels := solrPrometheusExporter.SharedLabelsWith(solrPrometheusExporter.GetLabels()) |
| var annotations map[string]string |
| selectorLabels := solrPrometheusExporter.SharedLabels() |
| |
| labels["technology"] = solr.SolrPrometheusExporterTechnologyLabel |
| selectorLabels["technology"] = solr.SolrPrometheusExporterTechnologyLabel |
| |
| podLabels := labels |
| var podAnnotations map[string]string |
| var imagePullSecrets []corev1.LocalObjectReference |
| |
| customDeploymentOptions := solrPrometheusExporter.Spec.CustomKubeOptions.DeploymentOptions |
| if nil != customDeploymentOptions { |
| labels = MergeLabelsOrAnnotations(labels, customDeploymentOptions.Labels) |
| annotations = customDeploymentOptions.Annotations |
| } |
| |
| customPodOptions := solrPrometheusExporter.Spec.CustomKubeOptions.PodOptions.DeepCopy() |
| if nil != customPodOptions { |
| podLabels = MergeLabelsOrAnnotations(podLabels, customPodOptions.Labels) |
| podAnnotations = customPodOptions.Annotations |
| imagePullSecrets = customPodOptions.ImagePullSecrets |
| } |
| |
| var envVars []corev1.EnvVar |
| var allJavaOpts []string |
| |
| var solrVolumes []corev1.Volume |
| var volumeMounts []corev1.VolumeMount |
| exporterArgs := []string{ |
| "-p", strconv.Itoa(SolrMetricsPort), |
| "-n", strconv.Itoa(int(solrPrometheusExporter.Spec.NumThreads)), |
| } |
| |
| if solrPrometheusExporter.Spec.ScrapeInterval > 0 { |
| exporterArgs = append(exporterArgs, "-s", strconv.Itoa(int(solrPrometheusExporter.Spec.ScrapeInterval))) |
| } |
| |
| // Setup the solrConnectionInfo |
| if solrConnectionInfo.CloudZkConnnectionInfo != nil { |
| exporterArgs = append(exporterArgs, "-z", solrConnectionInfo.CloudZkConnnectionInfo.ZkConnectionString()) |
| |
| // Add ACL information, if given, through Env Vars |
| if hasACLs, aclEnvs := AddACLsToEnv(solrConnectionInfo.CloudZkConnnectionInfo.AllACL, solrConnectionInfo.CloudZkConnnectionInfo.ReadOnlyACL); hasACLs { |
| envVars = append(envVars, aclEnvs...) |
| |
| // The $SOLR_ZK_CREDS_AND_ACLS parameter does not get picked up when running the Prometheus Exporter, it must be added to the JAVA_OPTS. |
| allJavaOpts = append(allJavaOpts, "$(SOLR_ZK_CREDS_AND_ACLS)") |
| } |
| } else if solrConnectionInfo.StandaloneAddress != "" { |
| exporterArgs = append(exporterArgs, "-b", solrConnectionInfo.StandaloneAddress) |
| } |
| |
| // Only add the config if it is passed in from the user. Otherwise, use the default. |
| if solrPrometheusExporter.Spec.Config != "" || |
| (solrPrometheusExporter.Spec.CustomKubeOptions.ConfigMapOptions != nil && solrPrometheusExporter.Spec.CustomKubeOptions.ConfigMapOptions.ProvidedConfigMap != "") { |
| configMapName := solrPrometheusExporter.MetricsConfigMapName() |
| if solrPrometheusExporter.Spec.CustomKubeOptions.ConfigMapOptions != nil && solrPrometheusExporter.Spec.CustomKubeOptions.ConfigMapOptions.ProvidedConfigMap != "" { |
| configMapName = solrPrometheusExporter.Spec.CustomKubeOptions.ConfigMapOptions.ProvidedConfigMap |
| } |
| solrVolumes = []corev1.Volume{{ |
| Name: "solr-prometheus-exporter-xml", |
| VolumeSource: corev1.VolumeSource{ |
| ConfigMap: &corev1.ConfigMapVolumeSource{ |
| LocalObjectReference: corev1.LocalObjectReference{ |
| Name: configMapName, |
| }, |
| Items: []corev1.KeyToPath{ |
| { |
| Key: PrometheusExporterConfigMapKey, |
| Path: PrometheusExporterConfigMapKey, |
| }, |
| }, |
| }, |
| }, |
| }} |
| |
| volumeMounts = []corev1.VolumeMount{{Name: "solr-prometheus-exporter-xml", MountPath: "/opt/solr-exporter", ReadOnly: true}} |
| |
| exporterArgs = append(exporterArgs, "-f", "/opt/solr-exporter/"+PrometheusExporterConfigMapKey) |
| } else { |
| exporterArgs = append(exporterArgs, "-f", "/opt/solr/contrib/prometheus-exporter/conf/solr-exporter-config.xml") |
| } |
| |
| entrypoint := DefaultPrometheusExporterEntrypoint |
| if solrPrometheusExporter.Spec.ExporterEntrypoint != "" { |
| entrypoint = solrPrometheusExporter.Spec.ExporterEntrypoint |
| } |
| |
| // Add Custom EnvironmentVariables to the solr container |
| if nil != customPodOptions { |
| // Add environment variables to container |
| envVars = append(envVars, customPodOptions.EnvVariables...) |
| |
| // Add Custom Volumes to pod |
| for _, volume := range customPodOptions.Volumes { |
| // Only add the container mount if one has been provided. |
| if volume.DefaultContainerMount != nil { |
| volume.DefaultContainerMount.Name = volume.Name |
| volumeMounts = append(volumeMounts, *volume.DefaultContainerMount) |
| } |
| |
| solrVolumes = append(solrVolumes, corev1.Volume{ |
| Name: volume.Name, |
| VolumeSource: volume.Source, |
| }) |
| } |
| } |
| |
| // basic auth enabled? |
| if solrPrometheusExporter.Spec.SolrReference.BasicAuthSecret != "" { |
| envVars = append(envVars, BasicAuthEnvVars(solrPrometheusExporter.Spec.SolrReference.BasicAuthSecret)...) |
| allJavaOpts = append(allJavaOpts, "-Dbasicauth=$(BASIC_AUTH_USER):$(BASIC_AUTH_PASS)") |
| allJavaOpts = append(allJavaOpts, "-Dsolr.httpclient.builder.factory=org.apache.solr.client.solrj.impl.PreemptiveBasicAuthClientBuilderFactory") |
| } |
| |
| // the order of env vars in the array is important for the $(var) syntax to work |
| // since JAVA_OPTS refers to $(SOLR_SSL_*) if TLS is enabled, it needs to be last |
| if len(allJavaOpts) > 0 { |
| envVars = append(envVars, corev1.EnvVar{Name: "JAVA_OPTS", Value: strings.Join(allJavaOpts, " ")}) |
| } |
| |
| containerImage := solrPrometheusExporter.Spec.Image |
| if containerImage == nil { |
| containerImage = solrCloudImage |
| } |
| |
| defaultProbeHandler := corev1.ProbeHandler{ |
| HTTPGet: &corev1.HTTPGetAction{ |
| Scheme: corev1.URISchemeHTTP, |
| Path: "/metrics", |
| Port: intstr.FromInt(SolrMetricsPort), |
| }, |
| } |
| |
| containers := []corev1.Container{ |
| { |
| Name: SolrPrometheusExporterContainer, |
| Image: containerImage.ToImageName(), |
| ImagePullPolicy: containerImage.PullPolicy, |
| Ports: []corev1.ContainerPort{{ContainerPort: SolrMetricsPort, Name: SolrMetricsPortName, Protocol: corev1.ProtocolTCP}}, |
| VolumeMounts: volumeMounts, |
| Command: []string{entrypoint}, |
| Args: exporterArgs, |
| Env: envVars, |
| |
| StartupProbe: &corev1.Probe{ |
| ProbeHandler: defaultProbeHandler, |
| InitialDelaySeconds: 2, |
| TimeoutSeconds: 2, |
| PeriodSeconds: 2, |
| SuccessThreshold: 1, |
| FailureThreshold: 10, |
| }, |
| LivenessProbe: &corev1.Probe{ |
| ProbeHandler: defaultProbeHandler, |
| TimeoutSeconds: 2, |
| PeriodSeconds: 10, |
| SuccessThreshold: 1, |
| FailureThreshold: 3, |
| }, |
| ReadinessProbe: &corev1.Probe{ |
| ProbeHandler: defaultProbeHandler, |
| TimeoutSeconds: 2, |
| PeriodSeconds: 5, |
| SuccessThreshold: 1, |
| FailureThreshold: 3, |
| }, |
| }, |
| } |
| |
| var initContainers []corev1.Container |
| if customPodOptions != nil { |
| if len(customPodOptions.SidecarContainers) > 0 { |
| containers = append(containers, customPodOptions.SidecarContainers...) |
| } |
| |
| if len(customPodOptions.InitContainers) > 0 { |
| initContainers = customPodOptions.InitContainers |
| } |
| } |
| |
| // track the MD5 of the custom exporter config in the pod spec annotations, |
| // so we get a rolling restart when the configMap changes |
| if configXmlMd5 != "" { |
| if podAnnotations == nil { |
| podAnnotations = make(map[string]string, 1) |
| } |
| podAnnotations[PrometheusExporterConfigXmlMd5Annotation] = configXmlMd5 |
| } |
| |
| // if the basic-auth secret changes, we want to restart the deployment pods |
| if basicAuthMd5 != "" { |
| if podAnnotations == nil { |
| podAnnotations = make(map[string]string, 1) |
| } |
| podAnnotations[BasicAuthMd5Annotation] = basicAuthMd5 |
| } |
| |
| deployment := &appsv1.Deployment{ |
| ObjectMeta: metav1.ObjectMeta{ |
| Name: solrPrometheusExporter.MetricsDeploymentName(), |
| Namespace: solrPrometheusExporter.GetNamespace(), |
| Labels: labels, |
| Annotations: annotations, |
| }, |
| Spec: appsv1.DeploymentSpec{ |
| Selector: &metav1.LabelSelector{ |
| MatchLabels: selectorLabels, |
| }, |
| Replicas: &singleReplica, |
| Template: corev1.PodTemplateSpec{ |
| ObjectMeta: metav1.ObjectMeta{ |
| Labels: podLabels, |
| Annotations: podAnnotations, |
| }, |
| Spec: corev1.PodSpec{ |
| TerminationGracePeriodSeconds: &gracePeriodTerm, |
| SecurityContext: &corev1.PodSecurityContext{ |
| FSGroup: &fsGroup, |
| }, |
| Volumes: solrVolumes, |
| InitContainers: initContainers, |
| Containers: containers, |
| }, |
| }, |
| }, |
| } |
| |
| if containerImage.ImagePullSecret != "" { |
| imagePullSecrets = append( |
| imagePullSecrets, |
| corev1.LocalObjectReference{Name: containerImage.ImagePullSecret}, |
| ) |
| } |
| |
| deployment.Spec.Template.Spec.ImagePullSecrets = imagePullSecrets |
| |
| if nil != customPodOptions { |
| metricsContainer := &deployment.Spec.Template.Spec.Containers[0] |
| if customPodOptions.ServiceAccountName != "" { |
| deployment.Spec.Template.Spec.ServiceAccountName = customPodOptions.ServiceAccountName |
| } |
| |
| if customPodOptions.Affinity != nil { |
| deployment.Spec.Template.Spec.Affinity = customPodOptions.Affinity |
| } |
| |
| if customPodOptions.Resources.Limits != nil || customPodOptions.Resources.Requests != nil { |
| metricsContainer.Resources = customPodOptions.Resources |
| } |
| |
| if customPodOptions.PodSecurityContext != nil { |
| deployment.Spec.Template.Spec.SecurityContext = customPodOptions.PodSecurityContext |
| } |
| |
| if customPodOptions.Tolerations != nil { |
| deployment.Spec.Template.Spec.Tolerations = customPodOptions.Tolerations |
| } |
| |
| if customPodOptions.NodeSelector != nil { |
| deployment.Spec.Template.Spec.NodeSelector = customPodOptions.NodeSelector |
| } |
| |
| if customPodOptions.PriorityClassName != "" { |
| deployment.Spec.Template.Spec.PriorityClassName = customPodOptions.PriorityClassName |
| } |
| |
| if customPodOptions.TerminationGracePeriodSeconds != nil { |
| deployment.Spec.Template.Spec.TerminationGracePeriodSeconds = customPodOptions.TerminationGracePeriodSeconds |
| } |
| |
| if customPodOptions.ReadinessProbe != nil { |
| // Default Prometheus Exporter container does not contain a readinessProbe, so copy the livenessProbe |
| baseProbe := metricsContainer.LivenessProbe.DeepCopy() |
| metricsContainer.ReadinessProbe = customizeProbe(baseProbe, *customPodOptions.ReadinessProbe) |
| } |
| |
| if customPodOptions.StartupProbe != nil { |
| // Default Prometheus Exporter container does not contain a startupProbe, so copy the livenessProbe |
| baseProbe := metricsContainer.LivenessProbe.DeepCopy() |
| metricsContainer.StartupProbe = customizeProbe(baseProbe, *customPodOptions.StartupProbe) |
| } |
| |
| if customPodOptions.LivenessProbe != nil { |
| metricsContainer.LivenessProbe = customizeProbe(metricsContainer.LivenessProbe, *customPodOptions.LivenessProbe) |
| } |
| |
| if customPodOptions.Lifecycle != nil { |
| metricsContainer.Lifecycle = customPodOptions.Lifecycle |
| } |
| |
| if len(customPodOptions.TopologySpreadConstraints) > 0 { |
| deployment.Spec.Template.Spec.TopologySpreadConstraints = customPodOptions.TopologySpreadConstraints |
| |
| // Set the label selector for constraints to the statefulSet label selector, if none is provided |
| for i := range deployment.Spec.Template.Spec.TopologySpreadConstraints { |
| if deployment.Spec.Template.Spec.TopologySpreadConstraints[i].LabelSelector == nil { |
| deployment.Spec.Template.Spec.TopologySpreadConstraints[i].LabelSelector = deployment.Spec.Selector.DeepCopy() |
| } |
| } |
| } |
| } |
| |
| // Enrich the deployment definition to allow the exporter to make requests to TLS enabled Solr pods |
| if tls != nil && tls.ClientConfig != nil { |
| tls.enableTLSOnExporterDeployment(deployment) |
| } |
| |
| return deployment |
| } |
| |
| // GenerateMetricsConfigMap returns a new corev1.ConfigMap pointer generated for the Solr Prometheus Exporter instance solr-prometheus-exporter.xml |
| // solrPrometheusExporter: SolrPrometheusExporter instance |
| func GenerateMetricsConfigMap(solrPrometheusExporter *solr.SolrPrometheusExporter) *corev1.ConfigMap { |
| labels := solrPrometheusExporter.SharedLabelsWith(solrPrometheusExporter.GetLabels()) |
| var annotations map[string]string |
| |
| customOptions := solrPrometheusExporter.Spec.CustomKubeOptions.ConfigMapOptions |
| if nil != customOptions { |
| labels = MergeLabelsOrAnnotations(labels, customOptions.Labels) |
| annotations = customOptions.Annotations |
| } |
| |
| configMap := &corev1.ConfigMap{ |
| ObjectMeta: metav1.ObjectMeta{ |
| Name: solrPrometheusExporter.MetricsConfigMapName(), |
| Namespace: solrPrometheusExporter.GetNamespace(), |
| Labels: labels, |
| Annotations: annotations, |
| }, |
| Data: map[string]string{ |
| PrometheusExporterConfigMapKey: solrPrometheusExporter.Spec.Config, |
| }, |
| } |
| return configMap |
| } |
| |
| // GenerateSolrMetricsService returns a new corev1.Service pointer generated for the SolrCloud Prometheus Exporter deployment |
| // Metrics will be collected on this service endpoint, as we don't want to double-tick data if multiple exporters are runnning. |
| // solrPrometheusExporter: solrPrometheusExporter instance |
| func GenerateSolrMetricsService(solrPrometheusExporter *solr.SolrPrometheusExporter) *corev1.Service { |
| copyLabels := solrPrometheusExporter.GetLabels() |
| if copyLabels == nil { |
| copyLabels = map[string]string{} |
| } |
| labels := solrPrometheusExporter.SharedLabelsWith(solrPrometheusExporter.GetLabels()) |
| labels["service-type"] = "metrics" |
| annotations := map[string]string{ |
| "prometheus.io/scrape": "true", |
| "prometheus.io/scheme": "http", |
| "prometheus.io/path": "/metrics", |
| } |
| |
| selectorLabels := solrPrometheusExporter.SharedLabels() |
| selectorLabels["technology"] = solr.SolrPrometheusExporterTechnologyLabel |
| |
| customOptions := solrPrometheusExporter.Spec.CustomKubeOptions.ServiceOptions |
| if nil != customOptions { |
| labels = MergeLabelsOrAnnotations(labels, customOptions.Labels) |
| annotations = MergeLabelsOrAnnotations(annotations, customOptions.Annotations) |
| } |
| |
| appProtocol := "http" |
| |
| service := &corev1.Service{ |
| ObjectMeta: metav1.ObjectMeta{ |
| Name: solrPrometheusExporter.MetricsServiceName(), |
| Namespace: solrPrometheusExporter.GetNamespace(), |
| Labels: labels, |
| Annotations: annotations, |
| }, |
| Spec: corev1.ServiceSpec{ |
| Ports: []corev1.ServicePort{ |
| { |
| Name: SolrMetricsPortName, |
| Port: ExtSolrMetricsPort, |
| Protocol: corev1.ProtocolTCP, |
| TargetPort: intstr.FromInt(SolrMetricsPort), |
| AppProtocol: &appProtocol, |
| }, |
| }, |
| Selector: selectorLabels, |
| }, |
| } |
| return service |
| } |
| |
| // CreateMetricsIngressRule returns a new Ingress Rule generated for the solr metrics endpoint |
| // This is not currently used, as an ingress is not created for the metrics endpoint. |
| // solrCloud: SolrCloud instance |
| // nodeName: string Name of the node |
| // ingressBaseDomain: string base domain for the ingress controller |
| func CreateMetricsIngressRule(solrPrometheusExporter *solr.SolrPrometheusExporter, ingressBaseDomain string) netv1.IngressRule { |
| pathType := netv1.PathTypeImplementationSpecific |
| externalAddress := solrPrometheusExporter.MetricsIngressUrl(ingressBaseDomain) |
| return netv1.IngressRule{ |
| Host: externalAddress, |
| IngressRuleValue: netv1.IngressRuleValue{ |
| HTTP: &netv1.HTTPIngressRuleValue{ |
| Paths: []netv1.HTTPIngressPath{ |
| { |
| Backend: netv1.IngressBackend{ |
| ServiceName: solrPrometheusExporter.MetricsServiceName(), |
| ServicePort: intstr.FromInt(ExtSolrMetricsPort), |
| }, |
| PathType: &pathType, |
| }, |
| }, |
| }, |
| }, |
| } |
| } |