blob: b9d3f6b4664ae1ffb11df8d8e9a90d59bf8d1a05 [file] [log] [blame]
/********************************************************************
* 2014 -
* open source under Apache License Version 2.0
********************************************************************/
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "client/FileSystem.h"
#include "client/FileSystemInter.h"
#include "client/OutputStream.h"
#include "client/Permission.h"
#include "DateTime.h"
#include "Exception.h"
#include "ExceptionInternal.h"
#include "gtest/gtest.h"
#include "TestUtil.h"
#include "Thread.h"
#include "XmlConfig.h"
#include <ctime>
#ifndef TEST_HDFS_PREFIX
#define TEST_HDFS_PREFIX "./"
#endif
#define BASE_DIR TEST_HDFS_PREFIX"/testFileSystem/"
using namespace Hdfs;
using namespace Hdfs::Internal;
class TestFileSystem: public ::testing::Test {
public:
TestFileSystem() :
conf("function-test.xml") {
conf.set("output.default.packetsize", 1024);
conf.set("rpc.client.ping.interval", 1000);
fs = shared_ptr<FileSystem>(new FileSystem(conf));
superfs = shared_ptr<FileSystem>(new FileSystem(conf));
fs->connect();
superfs->connect(conf.getString("dfs.default.uri"), HDFS_SUPERUSER, NULL);
superfs->setWorkingDirectory(fs->getWorkingDirectory().c_str());
try {
superfs->deletePath(BASE_DIR, true);
} catch (...) {
}
superfs->mkdirs(BASE_DIR, 0755);
superfs->setOwner(BASE_DIR, USER, NULL);
}
~TestFileSystem() {
try {
superfs->deletePath(BASE_DIR, true);
fs->disconnect();
superfs->disconnect();
} catch (...) {
}
}
protected:
Config conf;
shared_ptr<FileSystem> fs;
shared_ptr<FileSystem> superfs;
};
static inline bool CheckPermission(const FileSystem & fs, const char * path,
const Permission & mode) {
FileStatus status = fs.getFileStatus(path);
return status.getPermission() == mode;
}
TEST_F(TestFileSystem, mkdir) {
Permission p(ALL, READ_WRITE, READ_WRITE);
ASSERT_THROW(fs->mkdir(NULL, p), InvalidParameter);
ASSERT_THROW(fs->mkdir("", p), InvalidParameter);
EXPECT_NO_THROW(fs->mkdir(BASE_DIR"testa", p));
EXPECT_NO_THROW(fs->mkdir(BASE_DIR"testb", p));
}
TEST_F(TestFileSystem, mkdirs) {
EXPECT_THROW(fs->mkdirs("", 0644), InvalidParameter);
EXPECT_NO_THROW(fs->mkdirs(BASE_DIR"testmkdirs", 0644));
EXPECT_NO_THROW(fs->mkdirs(BASE_DIR"m/testmkdirs", 0644));
}
TEST_F(TestFileSystem, rename) {
ASSERT_THROW(fs->rename(NULL, "retest"), InvalidParameter);
ASSERT_THROW(fs->rename("/test1/testa", NULL), InvalidParameter);
ASSERT_THROW(fs->rename(NULL, NULL), InvalidParameter);
}
TEST_F(TestFileSystem, getDefaultReplication) {
ASSERT_NO_THROW(fs->getDefaultReplication());
ASSERT_EQ(fs->getDefaultReplication(), 3);
}
TEST_F(TestFileSystem, getFileStatus) {
OutputStream ous;
EXPECT_THROW(fs->getFileStatus(BASE_DIR"mm"), FileNotFoundException);
ous.open(*fs, BASE_DIR"testGetFileStatus", Create, 0644, false, 0, 2048);
char buffer[2048];
FillBuffer(buffer, sizeof(buffer), 0);
ous.append(buffer, sizeof(buffer));
ous.close();
EXPECT_NO_THROW(fs->getFileStatus(BASE_DIR"testGetFileStatus"));
}
TEST_F(TestFileSystem, listDirectory) {
fs->disconnect();
ASSERT_THROW(fs->listDirectory(BASE_DIR), HdfsIOException);
fs->connect();
ASSERT_THROW(fs->listDirectory(""), InvalidParameter);
const int dirs = 10000;
for (int i = 0; i < dirs; ++i) {
std::stringstream ss;
ss.imbue(std::locale::classic());
ss << BASE_DIR << "testListDir/" << i;
fs->mkdirs(ss.str().c_str(), 0777);
}
DirectoryIterator it;
EXPECT_NO_THROW(it = fs->listDirectory(BASE_DIR"testListDir/"));
int count = 0;
while (it.hasNext()) {
count ++;
it.getNext();
}
ASSERT_EQ(dirs, count);
ASSERT_THROW(it.getNext(), HdfsIOException);
}
TEST_F(TestFileSystem, DISABLED_listEncryptionZone) {
fs->disconnect();
fs->connect();
const int dirs = 201;
for (int i = 0; i < dirs; i++){
std::stringstream newstr;
newstr << i;
std::string tde = "/TDE" + newstr.str();
std::string key = "keytde" + newstr.str();
std::string rmTde = "hadoop fs -rmr /TDE" + newstr.str();
std::string tdeKey = "hadoop key create keytde" + newstr.str();
std::string mkTde = "hadoop fs -mkdir /TDE" + newstr.str();
std::string tdeZone = "hdfs crypto -createZone -keyName " + key + "-path " + tde;
system(rmTde.c_str());
system(tdeKey.c_str());
system(mkTde.c_str());
system(tdeZone.c_str());
}
EncryptionZoneIterator it;
EXPECT_NO_THROW(it = fs->listEncryptionZone());
int count = 0;
while (it.hasNext()) {
count ++;
it.getNext();
}
ASSERT_EQ(dirs, count);
ASSERT_THROW(it.getNext(), HdfsIOException);
}
TEST_F(TestFileSystem, setOwner) {
fs->disconnect();
ASSERT_THROW(fs->setOwner(BASE_DIR, "setOwner", ""), HdfsIOException);
fs->connect();
ASSERT_THROW(fs->setOwner(NULL, "", ""), InvalidParameter);
EXPECT_THROW(fs->setOwner(BASE_DIR, "", ""), InvalidParameter);
EXPECT_NO_THROW(superfs->setOwner(BASE_DIR, "setOwner", ""));
}
TEST_F(TestFileSystem, setTimes) {
time_t now_time;
now_time = time(NULL);
fs->disconnect();
ASSERT_THROW(fs->setTimes("", now_time, now_time), HdfsIOException);
fs->connect();
ASSERT_THROW(fs->setTimes(NULL, now_time, now_time), InvalidParameter);
EXPECT_NO_THROW(fs->setTimes(BASE_DIR, now_time, now_time));
}
TEST_F(TestFileSystem, setPermission) {
fs->disconnect();
ASSERT_THROW(fs->setPermission("", 0644), HdfsIOException);
fs->connect();
ASSERT_THROW(fs->setPermission(NULL, 0644), InvalidParameter);
EXPECT_THROW(superfs->setPermission("qq", 0644), FileNotFoundException);
}
TEST_F(TestFileSystem, setReplication) {
fs->disconnect();
ASSERT_THROW(fs->setReplication("", 4), HdfsIOException);
fs->connect();
ASSERT_THROW(fs->setReplication(NULL, 4), InvalidParameter);
EXPECT_TRUE(false == fs->setReplication("qq", 0644));
}
TEST_F(TestFileSystem, setWorkingDirectory) {
EXPECT_THROW(fs->setWorkingDirectory(""), InvalidParameter);
EXPECT_THROW(fs->setWorkingDirectory("ddd"), InvalidParameter);
EXPECT_NO_THROW(fs->setWorkingDirectory("/user/test1"));
}
TEST_F(TestFileSystem, getWorkingDirectory) {
EXPECT_NO_THROW(fs->setWorkingDirectory("/user/test1"));
EXPECT_STREQ("/user/test1", fs->getWorkingDirectory().c_str());
}
TEST_F(TestFileSystem, exist) {
fs->disconnect();
EXPECT_THROW(fs->exist(""), HdfsIOException);
fs->connect();
EXPECT_THROW(fs->exist(""), InvalidParameter);
//EXPECT_NO_THROW(fs->exist(BASE_DIR"mm"));
EXPECT_EQ(fs->exist("/mm"), false);
EXPECT_EQ(fs->exist(BASE_DIR), true);
}
TEST_F(TestFileSystem, getStats) {
fs->disconnect();
EXPECT_THROW(fs->getStats(), HdfsIOException);
fs->connect();
EXPECT_NO_THROW(fs->getStats());
}
TEST_F(TestFileSystem, truncate) {
OutputStream os;
try {
fs->truncate("NOTEXIST", 20);
} catch (RpcNoSuchMethodException & e) {
return;
} catch (UnsupportedOperationException & e) {
return;
} catch (...) {
}
fs->disconnect();
EXPECT_THROW(fs->truncate(BASE_DIR"description", 20), HdfsIOException);
fs->connect();
EXPECT_THROW(fs->truncate("", 20), InvalidParameter);
EXPECT_THROW(fs->truncate(BASE_DIR"mm", 20), FileNotFoundException);
EXPECT_NO_THROW(os.open(*fs, BASE_DIR"testTruncate", Create, 0644, false, 0, 2048));
EXPECT_NO_THROW(os.close());
EXPECT_THROW(fs->truncate(BASE_DIR"testTruncate", 20), InvalidParameter);
EXPECT_NO_THROW(os.open(*fs, BASE_DIR"testTruncate", Append, 0644, false, 0, 2048));
char buffer[512];
FillBuffer(buffer, sizeof(buffer), 0);
EXPECT_NO_THROW(os.append(buffer, sizeof(buffer)));
EXPECT_NO_THROW(os.close());
EXPECT_NO_THROW(fs->truncate(BASE_DIR"testTruncate", 20));
}
TEST_F(TestFileSystem, testPing) {
EXPECT_NO_THROW(fs->listDirectory(BASE_DIR));
sleep_for(seconds(5));
EXPECT_NO_THROW(fs->listDirectory(BASE_DIR));
}
TEST_F(TestFileSystem, getFileBlockLocations) {
EXPECT_THROW(fs->getFileBlockLocations(NULL, 0, 0), InvalidParameter);
EXPECT_THROW(fs->getFileBlockLocations("", 0, 0), InvalidParameter);
EXPECT_THROW(fs->getFileBlockLocations("NOTEXIST", -1, 0), InvalidParameter);
EXPECT_THROW(fs->getFileBlockLocations("NOTEXIST", 0, -1), InvalidParameter);
EXPECT_THROW(fs->getFileBlockLocations("NOTEXIST", 0, 0), FileNotFoundException);
OutputStream os;
std::vector<BlockLocation> retval;
/**
* empty file
*/
os.open(*fs, BASE_DIR"TestGetFileBlockLocations", Create | Overwrite, 0777, true, 1, 1024);
os.close();
EXPECT_NO_THROW(DebugException(retval = fs->getFileBlockLocations(BASE_DIR"TestGetFileBlockLocations", 0, 100)));
ASSERT_EQ(0u, retval.size());
/**
* opened file
*/
os.open(*fs, BASE_DIR"TestGetFileBlockLocations", Create | Overwrite, 0777, true, 1, 1024);
std::vector<char> buffer(1025);
os.append(&buffer[0], buffer.size());
os.sync();
EXPECT_NO_THROW(DebugException(retval = fs->getFileBlockLocations(BASE_DIR"TestGetFileBlockLocations", 0, buffer.size())));
ASSERT_EQ(2u, retval.size());
for (size_t i = 0; i < retval.size(); ++i) {
EXPECT_EQ(1u, retval[i].getHosts().size());
EXPECT_EQ(1u, retval[i].getNames().size());
EXPECT_EQ(1u, retval[i].getTopologyPaths().size());
EXPECT_FALSE(retval[0].isCorrupt());
}
EXPECT_EQ(0, retval[0].getOffset());
EXPECT_EQ(1024, retval[1].getOffset());
EXPECT_EQ(1024, retval[0].getLength());
//EXPECT_EQ(1, retval[1].getLength());
os.close();
/*
* nonexistent regions
*/
EXPECT_NO_THROW(DebugException(retval = fs->getFileBlockLocations(BASE_DIR"TestGetFileBlockLocations", buffer.size(), 100)));
ASSERT_EQ(0u, retval.size());
}