Add basic helm charts for Log Search (developement) (#51)
diff --git a/k8s/helm-charts/infra-solr/Chart.yaml b/k8s/helm-charts/infra-solr/Chart.yaml
new file mode 100644
index 0000000..abd08e7
--- /dev/null
+++ b/k8s/helm-charts/infra-solr/Chart.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+name: infra-solr
+version: 1.0.0
+description: Infra solr service for Log Search components
+keywords:
+ - infra-solr
+maintainers:
+ - name: Apache
+deprecated: false
\ No newline at end of file
diff --git a/k8s/helm-charts/infra-solr/templates/infra-solr.yaml b/k8s/helm-charts/infra-solr/templates/infra-solr.yaml
new file mode 100644
index 0000000..5693d7d
--- /dev/null
+++ b/k8s/helm-charts/infra-solr/templates/infra-solr.yaml
@@ -0,0 +1,97 @@
+{{- if .Values.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ .Release.Name }}-infra-solr
+ namespace: {{.Values.global.namespace.logging}}
+ labels:
+ app: infra-solr
+spec:
+ ports:
+ - port: 8886
+ name: infra-solr
+ externalIPs:
+ []
+ clusterIP: None
+ selector:
+ app: infra-solr
+ release: {{ .Release.Name }}
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+ name: {{ .Release.Name }}-infra-solr
+ namespace: {{.Values.global.namespace.logging}}
+ labels:
+ chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ release: {{ .Release.Name }}
+spec:
+ selector:
+ matchLabels:
+ app: infra-solr
+ serviceName: {{ .Release.Name }}-infra-solr
+ replicas: {{.Values.global.solr.replicas}}
+ updateStrategy:
+ type: RollingUpdate
+ podManagementPolicy: Parallel
+ template:
+ metadata:
+ labels:
+ app: infra-solr
+ release: {{ .Release.Name }}
+ spec:
+{{- if .Values.affinity }}
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: "app"
+ operator: In
+ values:
+ - infra-solr
+ topologyKey: "kubernetes.io/hostname"
+{{end}}
+ containers:
+ - name: infra-solr
+ image: "{{.Values.infraSolrImage}}"
+ env:
+ - name: ZK_CONNECT_STRING
+ value: "{{.Values.zkRelease}}-zookeeper-cs.{{.Values.global.namespace.logging}}.svc.{{.Values.global.clusterDomain}}:2181"
+ - name: SOLR_PORT
+ value: "8886"
+ - name: CLOUD_MODE
+ value: "true"
+ readinessProbe:
+ httpGet:
+ path: /
+ port: 8886
+ initialDelaySeconds: 30
+ timeoutSeconds: 15
+ livenessProbe:
+ httpGet:
+ path: /
+ port: 8886
+ initialDelaySeconds: 30
+ timeoutSeconds: 15
+ volumeMounts:
+ - name: data
+ mountPath: /var/lib/ambari-infra-solr/data
+ securityContext:
+ runAsUser: 8983
+{{- if .Values.persistence.enabled }}
+ volumeClaimTemplates:
+ - metadata:
+ name: solr-home
+ spec:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: {{ .Values.persistence.size }}
+{{- else }}
+ volumes:
+ - name: data
+ emptyDir: {}
+{{- end }}
+{{end}}
\ No newline at end of file
diff --git a/k8s/helm-charts/infra-solr/values.yaml b/k8s/helm-charts/infra-solr/values.yaml
new file mode 100644
index 0000000..f23b6d3
--- /dev/null
+++ b/k8s/helm-charts/infra-solr/values.yaml
@@ -0,0 +1,18 @@
+enabled: true
+infraSolrImage: "apache/ambari-infra-solr:latest"
+heapSize: "512m"
+global:
+ clusterDomain: "cluster.local"
+ namespace:
+ logging: "default"
+ solr:
+ replicas: 3
+
+loglevel: INFO
+affinity: true
+
+persistence:
+ enabled: false
+ size: 100G
+
+zkRelease: test-zk
\ No newline at end of file
diff --git a/k8s/helm-charts/logsearch-helm.sh b/k8s/helm-charts/logsearch-helm.sh
new file mode 100755
index 0000000..f6b949d
--- /dev/null
+++ b/k8s/helm-charts/logsearch-helm.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+sdir="`dirname \"$0\"`"
+: ${1:?"argument is missing: (install|delete)"}
+command="$1"
+shift
+
+function install_charts() {
+ helm install --name test-zk $sdir/zookeeper/
+ helm install --name test-solr $sdir/infra-solr/ --set zkRelease=test-zk
+ helm install --name test-logsearch $sdir/logsearch/ --set zkRelease=test-zk
+}
+
+function purge_charts() {
+ helm del --purge test-logsearch
+ helm del --purge test-solr
+ helm del --purge test-zk
+}
+
+case $command in
+ "install")
+ install_charts
+ ;;
+ "delete")
+ purge_charts
+ ;;
+ *)
+ echo "Available commands: (install|delete)"
+ ;;
+esac
\ No newline at end of file
diff --git a/k8s/helm-charts/logsearch/Chart.yaml b/k8s/helm-charts/logsearch/Chart.yaml
new file mode 100644
index 0000000..42d4316
--- /dev/null
+++ b/k8s/helm-charts/logsearch/Chart.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+name: logsearch
+version: 1.0.0
+description: Apache Ambari Log Search service
+keywords:
+ - logsearch
+maintainers:
+ - name: Apache
+deprecated: false
\ No newline at end of file
diff --git a/k8s/helm-charts/logsearch/templates/logsearch-configmap.yaml b/k8s/helm-charts/logsearch/templates/logsearch-configmap.yaml
new file mode 100644
index 0000000..eeacb2a
--- /dev/null
+++ b/k8s/helm-charts/logsearch/templates/logsearch-configmap.yaml
@@ -0,0 +1,100 @@
+{{ if .Values.enabled }}
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ .Release.Name }}-logsearch-configmap
+ namespace: {{.Values.global.namespace.logging}}
+ labels:
+ chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ release: {{ .Release.Name }}
+ heritage: {{ .Release.Service }}
+data:
+ logsearch.properties: |-
+ ---
+ logsearch.solr.zk_connect_string={{.Values.zkRelease}}-zookeeper-cs.{{.Values.global.namespace.logging}}.svc.{{.Values.global.clusterDomain}}:2181
+ logsearch.config.zk_connect_string={{.Values.zkRelease}}-zookeeper-cs.{{.Values.global.namespace.logging}}.svc.{{.Values.global.clusterDomain}}:2181
+ logsearch.config.api.enabled={{.Values.configApiEnabled }}
+ logsearch.config.api.filter.zk.enabled=true
+
+ # Service Logs
+ logsearch.solr.service.logs.collection=service_logs
+ logsearch.solr.service.logs.config.name=hadoop_logs
+
+ logsearch.solr.service.logs.numshards=6
+ logsearch.solr.service.logs.replication.factor=2
+ logsearch.web.service_logs.field.visible=sdi_java_logger_name,sdi_java_method
+
+ # Audit logs
+ logsearch.solr.audit.logs.zk_connect_string={{.Release.Name}}-zookeeper-cs.{{.Values.global.namespace.logging}}.svc.{{.Values.global.clusterDomain}}:2181
+
+ logsearch.solr.audit.logs.url=
+ logsearch.solr.audit.logs.collection=audit_logs
+ logsearch.solr.audit.logs.numshards=2
+ logsearch.solr.audit.logs.replication.factor=2
+
+ logsearch.solr.config_set.folder=/usr/lib/ambari-logsearch-portal/conf/solr_configsets
+ logsearch.solr.audit.logs.config_set.folder=/usr/lib/ambari-logsearch-portal/conf/solr_configsets
+
+ # Metadata collection
+ logsearch.solr.logsearch_metadata.collection=logsearch_metadata
+ logsearch.solr.logsearch_metadata.config.name=logsearch_metadata
+ logsearch.solr.logsearch_metadata.replication.factor=2
+
+ # Logfeeder Settings
+ logsearch.logfeeder.include.default.level=FATAL,ERROR,WARN,INFO
+
+ # logsearch-admin.json
+ logsearch.auth.file.enable=true
+ logsearch.login.credentials.file=users.json
+
+ logsearch.auth.ldap.enable=false
+ logsearch.auth.simple.enable=false
+ logsearch.auth.external_auth.enable=false
+ logsearch.auth.redirect.forward=true
+
+ logsearch.protocol=http
+ log4j2.yml: |-
+ Configutation:
+ name: LogSearchConfig
+ packages: org.apache.ambari.logsearch.layout
+ Properties:
+ Property:
+ name: log-path
+ value: "/var/log/ambari-logsearch-portal"
+ Appenders:
+ Console:
+ name: Console_Appender
+ target: SYSTEM_OUT
+ PatternLayout:
+ pattern: "%d [%t] %-5p %C{6} (%F:%L) - %m%n"
+ Loggers:
+ Root:
+ level: info
+ AppenderRef:
+ - ref: Console_Appender
+
+ users.json: |-
+ {
+ "users": [{
+ "name": "Logsearch Admin",
+ "username": "{{.Values.username}}",
+ "password": "{{.Values.password}}",
+ "en_password": ""
+ }]
+ }
+
+ logsearch-env.sh: |-
+ set -e
+ export JAVA_HOME=/usr/java/default
+ export LOGFILE=/var/log/ambari-logsearch-portal/logsearch.log
+ export LOGSEARCH_PORT=61888
+ export LOGSEARCH_PATH=/usr/lib/ambari-logsearch-portal
+ export LOGSEARCH_CONF_DIR=/logsearch-conf
+ export LOGSEARCH_PID_FILE=/var/run/ambari-logsearch-portal/logsearch.pid
+ export LOGSEARCH_JAVA_MEM=${LOGSEARCH_JAVA_MEM:-"-Xmx1024m"}
+ export LOGSEARCH_DEBUG=false
+ export LOGSEARCH_DEBUG_PORT=5005
+ export LOGSEARCH_SSL="false"
+
+{{ end }}
\ No newline at end of file
diff --git a/k8s/helm-charts/logsearch/templates/logsearch.yaml b/k8s/helm-charts/logsearch/templates/logsearch.yaml
new file mode 100755
index 0000000..cf71579
--- /dev/null
+++ b/k8s/helm-charts/logsearch/templates/logsearch.yaml
@@ -0,0 +1,65 @@
+{{ if .Values.enabled }}
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: {{ .Release.Name }}-logsearch
+ namespace: {{.Values.global.namespace.logging}}
+spec:
+ strategy:
+ rollingUpdate:
+ maxUnavailable: 0
+ replicas: 1
+ selector:
+ matchLabels:
+ app: logsearch
+ release: {{ .Release.Name }}
+ template:
+ metadata:
+ labels:
+ app: logsearch
+ release: {{ .Release.Name }}
+ spec:
+{{- if .Values.rbac.enabled}}
+ serviceAccountName: {{ .Release.Name }}-logsearch-service-account
+{{end}}
+ terminationGracePeriodSeconds: 0
+ containers:
+ - name: logsearch
+ image: {{ .Values.image }}
+ imagePullPolicy: Always
+ env:
+ - name: ZK_CONNECT_STRING
+ value: "{{.Values.zkRelease}}-zookeeper-cs.{{.Values.global.namespace.logging}}.svc.{{.Values.global.clusterDomain}}:2181/infra-solr"
+ ports:
+ - containerPort: 61888
+ name: logsearchweb
+ volumeMounts:
+ - name: logsearch-configmap
+ mountPath: {{ .Values.configMountPath }}
+ readOnly: false
+ resources:
+{{ toYaml .Values.resources | indent 12 }}
+ readinessProbe:
+ httpGet:
+ path: /
+ port: 61888
+ initialDelaySeconds: 10
+ timeoutSeconds: 5
+ volumes:
+ - name: logsearch-configmap
+ configMap:
+ name: {{ .Release.Name }}-logsearch-configmap
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ .Release.Name }}-logsearch
+ namespace: {{.Values.global.namespace.logging}}
+spec:
+ ports:
+ - port: 61888
+ name: logsearchweb
+ selector:
+ app: logsearch
+ release: {{ .Release.Name }}
+{{- end }}
\ No newline at end of file
diff --git a/k8s/helm-charts/logsearch/values.yaml b/k8s/helm-charts/logsearch/values.yaml
new file mode 100755
index 0000000..001cdfd
--- /dev/null
+++ b/k8s/helm-charts/logsearch/values.yaml
@@ -0,0 +1,18 @@
+enabled: true
+image: apache/ambari-logsearch-portal:latest
+configMountPath: /logsearch-conf
+
+global:
+ clusterDomain: "cluster.local"
+ namespace:
+ logging: "default"
+ kubernetesApi: "default"
+ solr:
+ replicas: 3
+
+username: admin
+password: admin
+configApiEnabled: true
+
+rbac:
+ enabled: false
\ No newline at end of file
diff --git a/k8s/helm-charts/zookeeper/Chart.yaml b/k8s/helm-charts/zookeeper/Chart.yaml
new file mode 100644
index 0000000..5e2847a
--- /dev/null
+++ b/k8s/helm-charts/zookeeper/Chart.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+name: zookeeper
+version: 1.0.0
+description: zookeeper service for our components
+keywords:
+ - zookeeper
+maintainers:
+ - name: Apache
+deprecated: false
\ No newline at end of file
diff --git a/k8s/helm-charts/zookeeper/templates/zookeeper.yaml b/k8s/helm-charts/zookeeper/templates/zookeeper.yaml
new file mode 100644
index 0000000..dfe3646
--- /dev/null
+++ b/k8s/helm-charts/zookeeper/templates/zookeeper.yaml
@@ -0,0 +1,142 @@
+{{- if .Values.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ .Release.Name }}-zookeeper-hs
+ labels:
+ app: {{ .Release.Name }}-zookeeper
+spec:
+ ports:
+ - port: 2888
+ name: server
+ - port: 3888
+ name: leader-election
+ clusterIP: None
+ selector:
+ app: {{ .Release.Name }}-zookeeper
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ .Release.Name }}-zookeeper-cs
+ labels:
+ app: {{ .Release.Name }}-zookeeper
+spec:
+ ports:
+ - port: 2181
+ name: client
+ selector:
+ app: {{ .Release.Name }}-zookeeper
+---
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+ name: {{ .Release.Name }}-zookeeper-pdb
+spec:
+ selector:
+ matchLabels:
+ app: {{ .Release.Name }}-zookeeper
+ maxUnavailable: 1
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+ name: {{ .Release.Name }}-zookeeper
+ labels:
+ chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+ release: {{ .Release.Name }}
+spec:
+ selector:
+ matchLabels:
+ app: {{ .Release.Name }}-zookeeper
+ serviceName: {{ .Release.Name }}-zookeeper-hs
+ replicas: {{.Values.replicas}}
+ updateStrategy:
+ type: RollingUpdate
+ podManagementPolicy: Parallel
+ template:
+ metadata:
+ labels:
+ app: {{ .Release.Name }}-zookeeper
+ spec:
+{{- if .Values.affinity }}
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: "app"
+ operator: In
+ values:
+ - {{ .Release.Name }}-zookeeper
+ topologyKey: "kubernetes.io/hostname"
+{{end}}
+ containers:
+ - name: kubernetes-zookeeper
+ image: "k8s.gcr.io/kubernetes-zookeeper:1.0-3.4.10"
+ resources:
+ requests:
+ memory: "1Gi"
+ cpu: "0.5"
+ ports:
+ - containerPort: 2181
+ name: client
+ - containerPort: 2888
+ name: server
+ - containerPort: 3888
+ name: leader-election
+ command:
+ - sh
+ - -c
+ - "start-zookeeper \
+ --servers={{.Values.replicas}} \
+ --data_dir=/var/lib/zookeeper/data \
+ --data_log_dir=/var/lib/zookeeper/data/log \
+ --conf_dir=/opt/zookeeper/conf \
+ --client_port=2181 \
+ --election_port=3888 \
+ --server_port=2888 \
+ --tick_time=2000 \
+ --init_limit=10 \
+ --sync_limit=5 \
+ --heap=512M \
+ --max_client_cnxns=60 \
+ --snap_retain_count=3 \
+ --purge_interval=12 \
+ --max_session_timeout=40000 \
+ --min_session_timeout=4000 \
+ --log_level=INFO"
+ readinessProbe:
+ exec:
+ command:
+ - sh
+ - -c
+ - "zookeeper-ready 2181"
+ initialDelaySeconds: 10
+ timeoutSeconds: 5
+ livenessProbe:
+ exec:
+ command:
+ - sh
+ - -c
+ - "zookeeper-ready 2181"
+ initialDelaySeconds: 10
+ timeoutSeconds: 5
+ volumeMounts:
+ - name: zookeeper-datadir
+ mountPath: /var/lib/zookeeper
+{{- if .Values.persistence.enabled }}
+ volumeClaimTemplates:
+ - metadata:
+ name: zookeeper-datadir
+ spec:
+ accessModes: [ "ReadWriteOnce" ]
+ resources:
+ requests:
+ storage: {{ .Values.persistence.size }}
+{{- else }}
+ volumes:
+ - name: zookeeper-datadir
+ emptyDir: {}
+{{- end }}
+{{end}}
\ No newline at end of file
diff --git a/k8s/helm-charts/zookeeper/values.yaml b/k8s/helm-charts/zookeeper/values.yaml
new file mode 100644
index 0000000..2cdfc01
--- /dev/null
+++ b/k8s/helm-charts/zookeeper/values.yaml
@@ -0,0 +1,9 @@
+enabled: true
+zookeeperImage: "k8s.gcr.io/kubernetes-zookeeper:1.0-3.4.10"
+
+affinity: false
+persistence:
+ enabled: false
+ size: 100G
+
+replicas: 1
\ No newline at end of file
diff --git a/k8s/localstack/localstack.yaml b/k8s/localstack/localstack.yaml
new file mode 100644
index 0000000..6558387
--- /dev/null
+++ b/k8s/localstack/localstack.yaml
@@ -0,0 +1,62 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: localstack
+ labels:
+ app: localstack
+spec:
+ selector:
+ matchLabels:
+ app: localstack
+ template:
+ metadata:
+ labels:
+ app: localstack
+ spec:
+ hostname: localstack
+ containers:
+ - name: localstack
+ image: localstack/localstack
+ env:
+ - name: SERVICES
+ value: "s3:4569"
+ - name: POD_IP
+ valueFrom: { fieldRef: { fieldPath: status.podIP } }
+ ports:
+ - name: http
+ containerPort: 4569
+---
+kind: NetworkPolicy
+apiVersion: networking.k8s.io/v1
+metadata:
+ name: localstack
+ labels:
+spec:
+ policyTypes:
+ - Ingress
+ - Egress
+ podSelector:
+ matchLabels:
+ app: localstack
+ ingress:
+ - ports:
+ - port: 4569
+ egress:
+ - {}
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: localstack
+ labels:
+spec:
+ type: NodePort
+ ports:
+ - name: http
+ port: 4569
+ nodePort: 31000
+ targetPort: http
+ externalIPs:
+ []
+ selector:
+ app: localstack
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 23a5d8e..23b36cd 100644
--- a/pom.xml
+++ b/pom.xml
@@ -375,6 +375,7 @@
<exclude>**/node_modules/**</exclude>
<exclude>**/dist/**</exclude>
<exclude>.repository/**</exclude>
+ <exclude>k8s/**</exclude>
</excludes>
</configuration>
<executions>
