AMBARI-25668 Integrate the Apache released HBase as back store of Ambari Metrics (#3305)
Integrate the Apache released HBase as back store of Ambari Metrics - embedded mode
Co-authored-by: Mariann Varga-Vakula <mariannvakula@mariannvakula-MBP16.local>
Co-authored-by: Mariann Vakula <mariann.vakula@cloudera.com>
diff --git a/ambari-metrics/ambari-metrics-assembly/pom.xml b/ambari-metrics/ambari-metrics-assembly/pom.xml
index d25b292..78d3362 100644
--- a/ambari-metrics/ambari-metrics-assembly/pom.xml
+++ b/ambari-metrics/ambari-metrics-assembly/pom.xml
@@ -266,11 +266,53 @@
<exclude>lib/*tests.jar</exclude>
<exclude>lib/findbugs*.jar</exclude>
<exclude>lib/jdk.tools*.jar</exclude>
+ <!-- hadoop, guava and commons-beanutils are excluded because they will be replaced with different versions -->
+ <exclude>lib/hadoop*.jar</exclude>
+ <exclude>lib/guava-11.0.2.jar</exclude>
+ <exclude>lib/commons-beanutils-core-1.8.0.jar</exclude>
</excludes>
</source>
</sources>
</mapping>
<mapping>
+ <!--embedded applications-->
+ <directory>/usr/lib/ams-hbase/lib/</directory>
+ <directoryIncluded>false</directoryIncluded>
+ <sources>
+ <source>
+ <location>${project.build.directory}/embedded/${hadoop.folder}/share/hadoop/hdfs</location>
+ <includes>
+ <include>hadoop-hdfs-${hadoop.version}.jar</include>
+ <include>hadoop-hdfs-client-${hadoop.version}.jar</include>
+ </includes>
+ </source>
+ <source>
+ <location>${project.build.directory}/embedded/${hadoop.folder}/share/hadoop/common/lib</location>
+ <includes>
+ <include>commons-beanutils-*.jar</include>
+ </includes>
+ </source>
+ <source>
+ <location>${project.build.directory}/embedded/${hadoop.folder}/share/hadoop/tools/lib</location>
+ <includes>
+ <include>hadoop-aws*.jar</include>
+ <include>hadoop-distcp-*.jar</include>
+ </includes>
+ </source>
+ <source>
+ <location>${project.build.directory}/embedded/${hadoop.folder}/share/hadoop/mapreduce</location>
+ <includes>
+ <include>hadoop-mapreduce-client-app-${hadoop.version}.jar</include>
+ <include>hadoop-mapreduce-client-common-${hadoop.version}.jar</include>
+ <include>hadoop-mapreduce-client-core-${hadoop.version}.jar</include>
+ <include>hadoop-mapreduce-client-hs-${hadoop.version}.jar</include>
+ <include>hadoop-mapreduce-client-jobclient-${hadoop.version}.jar</include>
+ <include>hadoop-mapreduce-client-shuffle-${hadoop.version}.jar</include>
+ </includes>
+ </source>
+ </sources>
+ </mapping>
+ <mapping>
<directory>/usr/lib/ams-hbase/bin</directory>
<filemode>755</filemode>
<sources>
@@ -287,6 +329,25 @@
<includes>
<include>phoenix*.jar</include>
<include>antlr*.jar</include>
+ <include>re2j*.jar</include>
+ <include>failureaccess*.jar</include>
+ <include>guava-28*.jar</include>
+ <include>stax2-api*.jar</include>
+ <include>woodstox-core*.jar</include>
+ <include>hadoop-annotations-${hadoop.version}.jar</include>
+ <include>hadoop-auth-${hadoop.version}.jar</include>
+ <include>hadoop-common-${hadoop.version}.jar</include>
+ <include>commons-configuration2-*.jar</include>
+ <include>hadoop-yarn-api-${hadoop.version}.jar</include>
+ <include>hadoop-yarn-client-${hadoop.version}.jar</include>
+ <include>hadoop-yarn-common-${hadoop.version}.jar</include>
+ <include>hadoop-yarn-registry-${hadoop.version}.jar</include>
+ <include>hadoop-yarn-server-applicationhistoryservice-${hadoop.version}.jar</include>
+ <include>hadoop-yarn-server-common-${hadoop.version}.jar</include>
+ <include>hadoop-yarn-server-nodemanager-${hadoop.version}.jar</include>
+ <include>hadoop-yarn-server-resourcemanager-${hadoop.version}.jar</include>
+ <include>hadoop-yarn-server-timelineservice-${hadoop.version}.jar</include>
+ <include>hadoop-yarn-server-web-proxy-${hadoop.version}.jar</include>
</includes>
</source>
</sources>
@@ -837,7 +898,8 @@
<data>
<src>${collector.dir}/target/embedded/${hbase.folder}</src>
<type>directory</type>
- <excludes>bin/**,bin/*,lib/*tests.jar</excludes>
+ <!-- hadoop, guava and commons-beanutils are excluded because they will be replaced with different versions -->
+ <excludes>bin/**,bin/*,lib/*tests.jar,lib/hadoop*jar,lib/guava-11.0.2.jar,lib/commons-beanutils-core-1.8.0.jar</excludes>
<mapper>
<type>perm</type>
<prefix>/usr/lib/ams-hbase</prefix>
@@ -845,6 +907,46 @@
</mapper>
</data>
<data>
+ <src>${project.build.directory}/embedded/${hadoop.folder}/share/hadoop/hdfs</src>
+ <type>directory</type>
+ <includes>hadoop-hdfs-${hadoop.version}.jar,hadoop-hdfs-client-${hadoop.version}.jar</includes>
+ <mapper>
+ <type>perm</type>
+ <prefix>/usr/lib/ams-hbase/lib</prefix>
+ <filemode>644</filemode>
+ </mapper>
+ </data>
+ <data>
+ <src>${project.build.directory}/embedded/${hadoop.folder}/share/hadoop/common/lib</src>
+ <type>directory</type>
+ <includes>commons-beanutils-*.jar</includes>
+ <mapper>
+ <type>perm</type>
+ <prefix>/usr/lib/ams-hbase/lib</prefix>
+ <filemode>644</filemode>
+ </mapper>
+ </data>
+ <data>
+ <src>${project.build.directory}/embedded/${hadoop.folder}/share/hadoop/tools/lib</src>
+ <type>directory</type>
+ <includes>hadoop-aws*.jar,hadoop-distcp-*.jar</includes>
+ <mapper>
+ <type>perm</type>
+ <prefix>/usr/lib/ams-hbase/lib</prefix>
+ <filemode>644</filemode>
+ </mapper>
+ </data>
+ <data>
+ <src>${project.build.directory}/embedded/${hadoop.folder}/share/hadoop/mapreduce</src>
+ <type>directory</type>
+ <includes>hadoop-mapreduce-client-app-${hadoop.version}.jar,hadoop-mapreduce-client-common-${hadoop.version}.jar,hadoop-mapreduce-client-core-${hadoop.version}.jar,hadoop-mapreduce-client-hs-${hadoop.version}.jar,hadoop-mapreduce-client-jobclient-${hadoop.version}.jar,hadoop-mapreduce-client-shuffle-${hadoop.version}.jar</includes>
+ <mapper>
+ <type>perm</type>
+ <prefix>/usr/lib/ams-hbase/lib</prefix>
+ <filemode>644</filemode>
+ </mapper>
+ </data>
+ <data>
<src>${collector.dir}/target/embedded/${hbase.folder}/bin</src>
<type>directory</type>
<mapper>
@@ -865,7 +967,7 @@
<data>
<src>${collector.dir}/target/lib</src>
<type>directory</type>
- <includes>phoenix*.jar,antlr*.jar</includes>
+ <includes>phoenix*.jar,antlr*.jar,re2j*.jar,failureaccess*.jar,guava-28*.jar,stax2-api*.jar,woodstox-core*.jar,hadoop-annotations*.jar,hadoop-auth*.jar,hadoop-common*.jar,commons-configuration2*.jar,hadoop-yarn-api-*.jar,hadoop-yarn-client-*.jar,hadoop-yarn-common-*.jar,hadoop-yarn-registry-*.jar,hadoop-yarn-server-applicationhistoryservice-*.jar,hadoop-yarn-server-common-*.jar,hadoop-yarn-server-nodemanager-*.jar,hadoop-yarn-server-resourcemanager-*.jar,hadoop-yarn-server-timelineservice-*.jar,hadoop-yarn-server-web-proxy-*.jar</includes>
<mapper>
<type>perm</type>
<filemode>644</filemode>
diff --git a/ambari-metrics/ambari-metrics-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
index 95f9cac..ead50d0 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/pom.xml
+++ b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
@@ -34,7 +34,6 @@
<!-- Needed for generating FindBugs warnings using parent pom -->
<!--<yarn.basedir>${project.parent.parent.basedir}</yarn.basedir>-->
<protobuf.version>2.5.0</protobuf.version>
- <hadoop.version>3.1.1</hadoop.version>
<phoenix.version>5.0.0-HBase-2.0</phoenix.version>
<hbase.version>2.0.2</hbase.version>
</properties>
@@ -738,6 +737,7 @@
<version>3.2</version>
<scope>test</scope>
</dependency>
+
<!-- for unit tests only -->
<dependency>
<groupId>org.apache.phoenix</groupId>
diff --git a/ambari-metrics/pom.xml b/ambari-metrics/pom.xml
index 282041f..5ff578c 100644
--- a/ambari-metrics/pom.xml
+++ b/ambari-metrics/pom.xml
@@ -44,6 +44,7 @@
<hbase.folder>hbase-2.0.2</hbase.folder>
<hadoop.tar>https://archive.apache.org/dist/hadoop/common/hadoop-3.1.1/hadoop-3.1.1.tar.gz</hadoop.tar>
<hadoop.folder>hadoop-3.1.1</hadoop.folder>
+ <hadoop.version>3.1.1</hadoop.version>
<grafana.folder>grafana-6.7.4</grafana.folder>
<grafana.tar>https://dl.grafana.com/oss/release/grafana-6.7.4.linux-amd64.tar.gz</grafana.tar>
<phoenix.tar>https://downloads.apache.org/phoenix/apache-phoenix-5.0.0-HBase-2.0/bin/apache-phoenix-5.0.0-HBase-2.0-bin.tar.gz</phoenix.tar>
