HDFS-12110: libhdfs++: Rebase HDFS-8707 on trunk, minor conflict resolution in this patch. Contributed by Deepak Majeti and Anatoli Shein.
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index 56db12a..d6bf779 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -38,7 +38,6 @@
# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
# Ubuntu Java. See Java section below!
######
-<<<<<<< 5f556eacf0fedbaec3cdb819847c7c6c3bacc048
RUN apt-get -q update && apt-get -q install -y \
apt-utils \
build-essential \
@@ -56,6 +55,7 @@
libfuse-dev \
libprotobuf-dev \
libprotoc-dev \
+ libsasl2-dev \
libsnappy-dev \
libssl-dev \
libtool \
@@ -65,7 +65,6 @@
pkg-config \
python \
python2.7 \
- pylint \
python-pip \
python-pkg-resources \
python-setuptools \
@@ -74,8 +73,8 @@
software-properties-common \
snappy \
sudo \
- zlib1g-dev \
- valgrind
+ valgrind \
+ zlib1g-dev
#######
# OpenJDK 8
@@ -203,3 +202,4 @@
ADD hadoop_env_checks.sh /root/hadoop_env_checks.sh
RUN chmod 755 /root/hadoop_env_checks.sh
RUN echo '~/hadoop_env_checks.sh' >> /root/.bashrc
+
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
index f596eac..52d9257 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/pom.xml
@@ -31,12 +31,12 @@
<properties>
<require.fuse>false</require.fuse>
- <hadoop.component>hdfs</hadoop.component>
<require.libwebhdfs>false</require.libwebhdfs>
<require.valgrind>false</require.valgrind>
<native_ctest_args></native_ctest_args>
<native_cmake_args></native_cmake_args>
<native_make_args></native_make_args>
+ <hadoop.component>hdfs</hadoop.component>
</properties>
<dependencies>
@@ -168,17 +168,21 @@
<goals><goal>run</goal></goals>
<configuration>
<skip>${skipTests}</skip>
- <target>
- <property name="compile_classpath" refid="maven.compile.classpath"/>
- <property name="test_classpath" refid="maven.test.classpath"/>
- <exec executable="ctest" failonerror="true" dir="${project.build.directory}/">
- <arg line="--output-on-failure"/>
- <arg line="${native_ctest_args}"/>
- <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/>
- <!-- Make sure libhadoop.so is on LD_LIBRARY_PATH. -->
- <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/>
- </exec>
- </target>
+ <target>
+ <property name="compile_classpath" refid="maven.compile.classpath"/>
+ <property name="test_classpath" refid="maven.test.classpath"/>
+ <exec executable="ctest" failonerror="true" dir="${project.build.directory}/native">
+ <arg line="--output-on-failure"/>
+ <arg line="${native_ctest_args}"/>
+ <env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/>
+ <!-- HADOOP_HOME required to find winutils. -->
+ <env key="HADOOP_HOME" value="${hadoop.common.build.dir}"/>
+ <!-- Make sure hadoop.dll and jvm.dll are on PATH. -->
+ <env key="PATH" value="${env.PATH};${hadoop.common.build.dir}/bin;${java.home}/jre/bin/server;${java.home}/bin/server"/>
+ <!-- Make sure libhadoop.so is on LD_LIBRARY_PATH. -->
+ <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/>
+ </exec>
+ </target>
</configuration>
</execution>
</executions>
@@ -197,13 +201,13 @@
<build>
<plugins>
<plugin>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-maven-plugins</artifactId>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
<executions>
<execution>
- <id>cmake-compile</id>
+ <id>make</id>
<phase>compile</phase>
- <goals><goal>cmake-compile</goal></goals>
+ <goals><goal>run</goal></goals>
<configuration>
<target>
<mkdir dir="${project.build.directory}"/>
@@ -217,12 +221,6 @@
</target>
</configuration>
</execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
<execution>
<id>native_tests</id>
<phase>test</phase>
@@ -237,7 +235,7 @@
<arg line="${native_ctest_args}"/>
<env key="CLASSPATH" value="${test_classpath}:${compile_classpath}"/>
<!-- Make sure libhadoop.so is on LD_LIBRARY_PATH. -->
- <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/>
+ <env key="LD_LIBRARY_PATH" value="${env.LD_LIBRARY_PATH}:${project.build.directory}/native/target/usr/local/lib:${hadoop.common.build.dir}/native/target/usr/local/lib"/>
</exec>
</target>
</configuration>
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
index eaec0ec..a3f8f2d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/CMakeLists.txt
@@ -58,19 +58,11 @@
# Omit unneeded headers.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWIN32_LEAN_AND_MEAN")
set(OS_DIR ${CMAKE_SOURCE_DIR}/main/native/libhdfs/os/windows)
-
- # IMPORTANT: OUT_DIR MUST be relative to maven's
- # project.build.directory (=target) and match dist-copynativelibs
- # in order to be in a release
- set(OUT_DIR bin)
+ set(OUT_DIR target/bin)
else()
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
set(OS_DIR ${CMAKE_SOURCE_DIR}/main/native/libhdfs/os/posix)
-
- # IMPORTANT: OUT_DIR MUST be relative to maven's
- # project.build.directory (=target) and match dist-copynativelibs
- # in order to be in a release
- set(OUT_DIR native/target/usr/local/lib)
+ set(OUT_DIR target/usr/local/lib)
endif()
# Configure JNI.
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
index e9d87c1..94b1b56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/CMakeLists.txt
@@ -35,7 +35,10 @@
SET(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMake" ${CMAKE_MODULE_PATH})
# If there's a better way to inform FindCyrusSASL.cmake, let's make this cleaner:
-SET(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};${CYRUS_SASL_DIR};${GSASL_DIR}")
+SET(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};${CYRUS_SASL_DIR};${GSASL_DIR};$ENV{PROTOBUF_HOME}")
+
+# Specify PROTOBUF_HOME so that find_package picks up the correct version
+SET(CMAKE_PREFIX_PATH "${CMAKE_PREFIX_PATH};$ENV{PROTOBUF_HOME}")
find_package(Doxygen)
find_package(OpenSSL REQUIRED)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt
index a5b5c4c..2eff301 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/proto/CMakeLists.txt
@@ -29,6 +29,7 @@
${PROTO_HDFS_DIR}/hdfs.proto
${PROTO_HDFS_DIR}/inotify.proto
${PROTO_HDFS_DIR}/xattr.proto
+ ${PROTO_HDFS_DIR}/ReconfigurationProtocol.proto
${PROTO_HADOOP_DIR}/IpcConnectionContext.proto
${PROTO_HADOOP_DIR}/ProtobufRpcEngine.proto
${PROTO_HADOOP_DIR}/RpcHeader.proto
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
index 1edfedc..54d4cf6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/hdfs_shim.c
@@ -470,6 +470,13 @@
return libhdfs_hadoopRzBufferFree(file->libhdfsRep, buffer);
}
+int hdfsGetHedgedReadMetrics(hdfsFS fs, struct hdfsHedgedReadMetrics **metrics) {
+ return libhdfs_hdfsGetHedgedReadMetrics(fs->libhdfsRep, (struct libhdfs_hdfsHedgedReadMetrics **) metrics);
+}
+
+void hdfsFreeHedgedReadMetrics(struct hdfsHedgedReadMetrics *metrics) {
+ return libhdfs_hdfsFreeHedgedReadMetrics((struct libhdfs_hdfsHedgedReadMetrics *) metrics);
+}
/*************
* hdfs_ext functions
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h
index fcf3185..b907768 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_defines.h
@@ -90,3 +90,8 @@
#define kObjectKindDirectory libhdfs_kObjectKindDirectory
#define hdfsReadStatistics libhdfs_hdfsReadStatistics
#define hdfsFileInfo libhdfs_hdfsFileInfo
+#define hdfsHedgedReadMetrics libhdfs_hdfsHedgedReadMetrics
+#define hdfsGetHedgedReadMetrics libhdfs_hdfsGetHedgedReadMetrics
+#define hdfsFreeHedgedReadMetrics libhdfs_hdfsFreeHedgedReadMetrics
+#define hdfsStreamBuilderAlloc libhdfs_hdfsStreamBuilderAlloc
+#define hdfsStreamBuilderBuild libhdfs_hdfsStreamBuilderBuild
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h
index 5a67fce..fce0e82 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfs_wrapper_undefs.h
@@ -100,3 +100,8 @@
#undef hdfsRenameSnapshot
#undef hdfsAllowSnapshot
#undef hdfsDisallowSnapshot
+#undef hdfsHedgedReadMetrics
+#undef hdfsGetHedgedReadMetrics
+#undef hdfsFreeHedgedReadMetrics
+#undef hdfsStreamBuilderAlloc
+#undef hdfsStreamBuilderBuild
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h
index c9bed90..d0411c21 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/libhdfspp_wrapper_defines.h
@@ -100,3 +100,8 @@
#define hdfsRenameSnapshot libhdfspp_hdfsRenameSnapshot
#define hdfsAllowSnapshot libhdfspp_hdfsAllowSnapshot
#define hdfsDisallowSnapshot libhdfspp_hdfsDisallowSnapshot
+#define hdfsHedgedReadMetrics libhdfspp_hdfsHedgedReadMetrics
+#define hdfsGetHedgedReadMetrics libhdfspp_hdfsGetHedgedReadMetrics
+#define hdfsFreeHedgedReadMetrics libhdfspp_hdfsFreeHedgedReadMetrics
+#define hdfsStreamBuilderAlloc libhdfspp_hdfsStreamBuilderAlloc
+#define hdfsStreamBuilderBuild libhdfspp_hdfsStreamBuilderBuild
