METRON-97: Create data purging script for logfiles closes apache/incubator-metron#82
diff --git a/deployment/roles/ambari_common/defaults/main.yml b/deployment/roles/ambari_common/defaults/main.yml
new file mode 100644
index 0000000..65c83d9
--- /dev/null
+++ b/deployment/roles/ambari_common/defaults/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+hadoop_logrotate_frequency: daily
+hadoop_logrotate_retention: 30
diff --git a/deployment/roles/ambari_common/tasks/main.yml b/deployment/roles/ambari_common/tasks/main.yml
index b1b1659..2526923 100644
--- a/deployment/roles/ambari_common/tasks/main.yml
+++ b/deployment/roles/ambari_common/tasks/main.yml
@@ -44,3 +44,9 @@
- name: Download Ambari repo
get_url: url="{{ rhel_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo
+- name: Create Logrotate Script for Hadoop Services
+ template:
+ src: "metron-hadoop-logrotate.yml"
+ dest: "/etc/logrotate.d/metron-ambari"
+ mode: 0644
+
diff --git a/deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml b/deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml
new file mode 100644
index 0000000..042b490
--- /dev/null
+++ b/deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml
@@ -0,0 +1,135 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#Hadoop HDFS Logs
+/var/log/hadoop/hdfs/*.log* {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+}
+
+/var/log/hadoop/hdfs/*.out {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+}
+
+/var/log/hadoop/hdfs/*.audit {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+}
+
+#Hadoop Yarn Logs
+/var/log/hadoop/yarn/*.log {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+}
+
+#Hadoop Mapreduce Logs
+/var/log/hadoop/mapreduce/*.log {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+}
+
+#Storm Logs
+/var/log/storm/*.log {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+}
+
+/var/log/storm/*.out {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+}
+
+#Kafka Logs
+/var/log/kafka/*.log {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+}
+
+/var/log/kafka/*.err {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+}
+
+#HBase Logs
+/var/log/hbase/*.log* {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+}
+
+/var/log/hbase/*.out {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+}
+
+/var/log/hbase/*.audit {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+}
+
+#Zookeeper Logs
+/var/log/zookeeper/*.log {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+}
+
+/var/log/zookeeper/*.out {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+}
diff --git a/deployment/roles/elasticsearch/defaults/main.yml b/deployment/roles/elasticsearch/defaults/main.yml
index d91fa1a..0026717 100644
--- a/deployment/roles/elasticsearch/defaults/main.yml
+++ b/deployment/roles/elasticsearch/defaults/main.yml
@@ -17,4 +17,6 @@
---
elasticsearch_data_dir: /var/lib/elasticsearch
elasticsearch_network_interface: eth0
+elasticsearch_logrotate_frequency: daily
+elasticsearch_logrotate_retention: 30
diff --git a/deployment/roles/elasticsearch/tasks/main.yml b/deployment/roles/elasticsearch/tasks/main.yml
index d9b2f44..26554aa 100644
--- a/deployment/roles/elasticsearch/tasks/main.yml
+++ b/deployment/roles/elasticsearch/tasks/main.yml
@@ -64,3 +64,10 @@
service: name=elasticsearch state=started enabled=yes
- include: configure_index.yml
+
+- name: Create Logrotate Script for Elasticsearch
+ template:
+ src: "metron-elasticsearch-logrotate.yml"
+ dest: "/etc/logrotate.d/metron-elasticsearch"
+ mode: 0644
+
diff --git a/deployment/roles/elasticsearch/templates/metron-elasticsearch-logrotate.yml b/deployment/roles/elasticsearch/templates/metron-elasticsearch-logrotate.yml
new file mode 100644
index 0000000..5504ce1
--- /dev/null
+++ b/deployment/roles/elasticsearch/templates/metron-elasticsearch-logrotate.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#Elasticsearch
+/var/log/elasticsearch/*.log {
+ {{ elasticsearch_logrotate_frequency }}
+ rotate {{ elasticsearch_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+}
+
