HDFS-15909. Make fnmatch cross platform (#2792)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt
index 624cda5..d6ea248 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/CMakeLists.txt
@@ -16,6 +16,7 @@
# limitations under the License.
#
-add_library(fs_obj OBJECT filesystem.cc filesystem_sync.cc filehandle.cc bad_datanode_tracker.cc namenode_operations.cc)
+add_library(fs_obj OBJECT $<TARGET_OBJECTS:x_platform_obj> filesystem.cc filesystem_sync.cc filehandle.cc bad_datanode_tracker.cc namenode_operations.cc)
+target_include_directories(fs_obj PRIVATE ../lib)
add_dependencies(fs_obj proto)
-add_library(fs $<TARGET_OBJECTS:fs_obj>)
+add_library(fs $<TARGET_OBJECTS:fs_obj> $<TARGET_OBJECTS:x_platform_obj>)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
index ba75e86..741d6c7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/fs/filesystem.cc
@@ -25,12 +25,12 @@
#include <limits>
#include <future>
#include <tuple>
-#include <iostream>
#include <pwd.h>
-#include <fnmatch.h>
#include <boost/asio/ip/tcp.hpp>
+#include "x-platform/syscall.h"
+
#define FMT_THIS_ADDR "this=" << (void*)this
namespace hdfs {
@@ -722,8 +722,8 @@
for (StatInfo const& si : stat_infos) {
//If we are at the last depth and it matches both path and name, we need to output it.
if (operational_state->depth == shared_state->dirs.size() - 2
- && !fnmatch(shared_state->dirs[operational_state->depth + 1].c_str(), si.path.c_str(), 0)
- && !fnmatch(shared_state->name.c_str(), si.path.c_str(), 0)) {
+ && XPlatform::Syscall::FnMatch(shared_state->dirs[operational_state->depth + 1], si.path)
+ && XPlatform::Syscall::FnMatch(shared_state->name, si.path)) {
outputs.push_back(si);
}
//Skip if not directory
@@ -731,7 +731,7 @@
continue;
}
//Checking for a match with the path at the current depth
- if(!fnmatch(shared_state->dirs[operational_state->depth + 1].c_str(), si.path.c_str(), 0)){
+ if(XPlatform::Syscall::FnMatch(shared_state->dirs[operational_state->depth + 1], si.path)) {
//Launch a new requests for every matched directory
shared_state->outstanding_requests++;
auto callback = [this, si, operational_state, shared_state](const Status &stat, const std::vector<StatInfo> & stat_infos, bool has_more) {
@@ -755,7 +755,7 @@
nn_.GetListing(si.full_path, callback);
}
//All names that match the specified name are saved to outputs
- if(!fnmatch(shared_state->name.c_str(), si.path.c_str(), 0)){
+ if(XPlatform::Syscall::FnMatch(shared_state->name, si.path)) {
outputs.push_back(si);
}
}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
index d162f6f..297aceb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall.h
@@ -48,6 +48,17 @@
*/
static int WriteToStdout(const char* message);
+ /**
+ * Checks whether the {@link str} argument matches the {@link pattern}
+ * argument, which is a shell wildcard pattern.
+ *
+ * @param pattern The wildcard pattern to use.
+ * @param str The string to match.
+ * @returns A boolean indicating whether the given {@link str}
+ * matches {@link pattern}.
+ */
+ static bool FnMatch(const std::string& pattern, const std::string& str);
+
private:
static bool WriteToStdoutImpl(const char* message);
};
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
index e556d997..2c51dbf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_linux.cc
@@ -16,6 +16,7 @@
* limitations under the License.
*/
+#include <fnmatch.h>
#include <unistd.h>
#include <cstring>
@@ -30,6 +31,11 @@
return WriteToStdoutImpl(message) ? 1 : 0;
}
+bool XPlatform::Syscall::FnMatch(const std::string& pattern,
+ const std::string& str) {
+ return fnmatch(pattern.c_str(), str.c_str(), 0) == 0;
+}
+
bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) {
const auto message_len = strlen(message);
const auto result = write(1, message, message_len);
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
index 06b0031..dc9ba63 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/x-platform/syscall_windows.cc
@@ -16,10 +16,13 @@
* limitations under the License.
*/
+#include <Shlwapi.h>
#include <Windows.h>
#include "syscall.h"
+#pragma comment(lib, "Shlwapi.lib")
+
bool XPlatform::Syscall::WriteToStdout(const std::string& message) {
return WriteToStdoutImpl(message.c_str());
}
@@ -28,6 +31,12 @@
return WriteToStdoutImpl(message) ? 1 : 0;
}
+bool XPlatform::Syscall::FnMatch(const std::string& pattern,
+ const std::string& str) {
+ return PathMatchSpecA(static_cast<LPCSTR>(str.c_str()),
+ static_cast<LPCSTR>(pattern.c_str())) == TRUE;
+}
+
bool XPlatform::Syscall::WriteToStdoutImpl(const char* message) {
auto* const stdout_handle = GetStdHandle(STD_OUTPUT_HANDLE);
if (stdout_handle == INVALID_HANDLE_VALUE || stdout_handle == nullptr) {
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt
index ac9f8fb..6a7d0be 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/CMakeLists.txt
@@ -18,9 +18,16 @@
if(WIN32)
add_executable(x_platform_utils_test $<TARGET_OBJECTS:x_platform_obj> utils_common_test.cc utils_test_main.cc utils_win_test.cc)
+ add_executable(x_platform_syscall_test $<TARGET_OBJECTS:x_platform_obj> syscall_common_test.cc utils_test_main.cc syscall_win_test.cc)
else(WIN32)
add_executable(x_platform_utils_test $<TARGET_OBJECTS:x_platform_obj> utils_common_test.cc utils_test_main.cc utils_nix_test.cc)
+ add_executable(x_platform_syscall_test $<TARGET_OBJECTS:x_platform_obj> syscall_common_test.cc utils_test_main.cc syscall_nix_test.cc)
endif(WIN32)
+
target_include_directories(x_platform_utils_test PRIVATE ${LIBHDFSPP_LIB_DIR})
target_link_libraries(x_platform_utils_test gmock_main)
add_test(x_platform_utils_test x_platform_utils_test)
+
+target_include_directories(x_platform_syscall_test PRIVATE ${LIBHDFSPP_LIB_DIR})
+target_link_libraries(x_platform_syscall_test gmock_main)
+add_test(x_platform_syscall_test x_platform_syscall_test)
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
new file mode 100644
index 0000000..04da29a
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_common_test.cc
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "x-platform/syscall.h"
+
+TEST(XPlatformSyscall, FnMatchBasicAsterisk) {
+ const std::string pattern("a*.doc");
+ const std::string str("abcd.doc");
+ EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str));
+}
+
+TEST(XPlatformSyscall, FnMatchBasicQuestionMark) {
+ const std::string pattern("a?.doc");
+ const std::string str("ab.doc");
+ EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str));
+}
+
+TEST(XPlatformSyscall, FnMatchNegativeAsterisk) {
+ const std::string pattern("a*.doc");
+ const std::string str("bcd.doc");
+ EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
+}
+
+TEST(XPlatformSyscall, FnMatchNegativeQuestionMark) {
+ const std::string pattern("a?.doc");
+ const std::string str("abc.doc");
+ EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_nix_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_nix_test.cc
new file mode 100644
index 0000000..f2c753f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_nix_test.cc
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "x-platform/syscall.h"
+
+TEST(XPlatformSyscall, FnMatchBasicPath) {
+ const std::string pattern("*.doc");
+ const std::string str("some/path/abcd.doc");
+ EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str));
+}
+
+TEST(XPlatformSyscall, FnMatchNegativePath) {
+ const std::string pattern("x*.doc");
+ const std::string str("y/abcd.doc");
+ EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
+}
diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_win_test.cc b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_win_test.cc
new file mode 100644
index 0000000..2d7393f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/tests/x-platform/syscall_win_test.cc
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "x-platform/syscall.h"
+
+TEST(XPlatformSyscall, FnMatchBasicPath) {
+ const std::string pattern("*.doc");
+ const std::string str(R"(some\path\abcd.doc)");
+ EXPECT_TRUE(XPlatform::Syscall::FnMatch(pattern, str));
+}
+
+TEST(XPlatformSyscall, FnMatchNegativePath) {
+ const std::string pattern("x*.doc");
+ const std::string str(R"(y\abcd.doc)");
+ EXPECT_FALSE(XPlatform::Syscall::FnMatch(pattern, str));
+}
