HAWQ-1799. delete useless codes from libhdfs3
diff --git a/depends/libhdfs3/CMake/FindCurl.cmake b/depends/libhdfs3/CMake/FindCurl.cmake
deleted file mode 100644
index e93b01d..0000000
--- a/depends/libhdfs3/CMake/FindCurl.cmake
+++ /dev/null
@@ -1,26 +0,0 @@
-# - Try to find the CURL library (curl)
-#
-# Once done this will define
-#
-# CURL_FOUND - System has gnutls
-# CURL_INCLUDE_DIR - The gnutls include directory
-# CURL_LIBRARIES - The libraries needed to use gnutls
-# CURL_DEFINITIONS - Compiler switches required for using gnutls
-
-
-IF (CURL_INCLUDE_DIR AND CURL_LIBRARIES)
- # in cache already
- SET(CURL_FIND_QUIETLY TRUE)
-ENDIF (CURL_INCLUDE_DIR AND CURL_LIBRARIES)
-
-FIND_PATH(CURL_INCLUDE_DIR curl/curl.h)
-
-FIND_LIBRARY(CURL_LIBRARIES curl)
-
-INCLUDE(FindPackageHandleStandardArgs)
-
-# handle the QUIETLY and REQUIRED arguments and set CURL_FOUND to TRUE if
-# all listed variables are TRUE
-FIND_PACKAGE_HANDLE_STANDARD_ARGS(CURL DEFAULT_MSG CURL_LIBRARIES CURL_INCLUDE_DIR)
-
-MARK_AS_ADVANCED(CURL_INCLUDE_DIR CURL_LIBRARIES)
diff --git a/depends/libhdfs3/CMake/FindSSL.cmake b/depends/libhdfs3/CMake/FindSSL.cmake
deleted file mode 100644
index bcbc5d8..0000000
--- a/depends/libhdfs3/CMake/FindSSL.cmake
+++ /dev/null
@@ -1,26 +0,0 @@
-# - Try to find the Open ssl library (ssl)
-#
-# Once done this will define
-#
-# SSL_FOUND - System has gnutls
-# SSL_INCLUDE_DIR - The gnutls include directory
-# SSL_LIBRARIES - The libraries needed to use gnutls
-# SSL_DEFINITIONS - Compiler switches required for using gnutls
-
-
-IF (SSL_INCLUDE_DIR AND SSL_LIBRARIES)
- # in cache already
- SET(SSL_FIND_QUIETLY TRUE)
-ENDIF (SSL_INCLUDE_DIR AND SSL_LIBRARIES)
-
-FIND_PATH(SSL_INCLUDE_DIR openssl/opensslv.h)
-
-FIND_LIBRARY(SSL_LIBRARIES crypto)
-
-INCLUDE(FindPackageHandleStandardArgs)
-
-# handle the QUIETLY and REQUIRED arguments and set SSL_FOUND to TRUE if
-# all listed variables are TRUE
-FIND_PACKAGE_HANDLE_STANDARD_ARGS(SSL DEFAULT_MSG SSL_LIBRARIES SSL_INCLUDE_DIR)
-
-MARK_AS_ADVANCED(SSL_INCLUDE_DIR SSL_LIBRARIES)
\ No newline at end of file
diff --git a/depends/libhdfs3/mock/MockCryptoCodec.h b/depends/libhdfs3/mock/MockCryptoCodec.h
deleted file mode 100644
index a9a220e..0000000
--- a/depends/libhdfs3/mock/MockCryptoCodec.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/********************************************************************
- * 2014 -
- * open source under Apache License Version 2.0
- ********************************************************************/
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef _HDFS_LIBHDFS3_MOCK_CRYPTOCODEC_H_
-#define _HDFS_LIBHDFS3_MOCK_CRYPTOCODEC_H_
-
-#include "gmock/gmock.h"
-
-#include "client/CryptoCodec.h"
-#include "client/KmsClientProvider.h"
-
-class MockCryptoCodec: public Hdfs::CryptoCodec {
-public:
- MockCryptoCodec(FileEncryptionInfo *encryptionInfo, shared_ptr<KmsClientProvider> kcp, int32_t bufSize) : CryptoCodec(encryptionInfo, kcp, bufSize) {}
-
- MOCK_METHOD2(init, int(CryptoMethod crypto_method, int64_t stream_offset));
- MOCK_METHOD2(cipher_wrap, std::string(const char * buffer,int64_t size));
-};
-
-#endif /* _HDFS_LIBHDFS3_MOCK_CRYPTOCODEC_H_ */
diff --git a/depends/libhdfs3/mock/MockHttpClient.h b/depends/libhdfs3/mock/MockHttpClient.h
deleted file mode 100644
index 9da1186..0000000
--- a/depends/libhdfs3/mock/MockHttpClient.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/********************************************************************
- * 2014 -
- * open source under Apache License Version 2.0
- ********************************************************************/
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef _HDFS_LIBHDFS3_MOCK_HTTPCLIENT_H_
-#define _HDFS_LIBHDFS3_MOCK_HTTPCLIENT_H_
-
-#include "gmock/gmock.h"
-
-#include "client/HttpClient.h"
-#include "client/KmsClientProvider.h"
-#include <boost/property_tree/ptree.hpp>
-
-using boost::property_tree::ptree;
-
-class MockHttpClient: public Hdfs::HttpClient {
-public:
- MOCK_METHOD0(post, std::string());
- MOCK_METHOD0(del, std::string());
- MOCK_METHOD0(put, std::string());
- MOCK_METHOD0(get, std::string());
-
- std::string getPostResult(FileEncryptionInfo &encryptionInfo) {
- ptree map;
- map.put("name", encryptionInfo.getKeyName());
- map.put("iv", encryptionInfo.getIv());
- map.put("material", encryptionInfo.getKey());
-
- std::string json = KmsClientProvider::toJson(map);
- return json;
- }
-
-};
-
-#endif /* _HDFS_LIBHDFS3_MOCK_HTTPCLIENT_H_ */
diff --git a/depends/libhdfs3/mock/MockKmsClientProvider.h b/depends/libhdfs3/mock/MockKmsClientProvider.h
deleted file mode 100644
index 81fb8f3..0000000
--- a/depends/libhdfs3/mock/MockKmsClientProvider.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/********************************************************************
- * 2014 -
- * open source under Apache License Version 2.0
- ********************************************************************/
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef _HDFS_LIBHDFS3_MOCK_KMSCLIENTPROVIDER_H_
-#define _HDFS_LIBHDFS3_MOCK_KMSCLIENTPROVIDER_H_
-
-#include "gmock/gmock.h"
-
-#include "client/KmsClientProvider.h"
-
-using namespace Hdfs::Internal;
-
-class MockKmsClientProvider: public Hdfs::KmsClientProvider {
-public:
- MockKmsClientProvider(shared_ptr<RpcAuth> auth, shared_ptr<SessionConfig> conf) : KmsClientProvider(auth, conf) {}
- MOCK_METHOD1(setHttpClient, void(shared_ptr<HttpClient> hc));
- MOCK_METHOD1(getKeyMetadata, ptree(const FileEncryptionInfo &encryptionInfo));
- MOCK_METHOD1(deleteKey, void(const FileEncryptionInfo &encryptionInfo));
- MOCK_METHOD1(decryptEncryptedKey, ptree(const FileEncryptionInfo &encryptionInfo));
- MOCK_METHOD5(createKey, void(const std::string &keyName, const std::string &cipher, const int length, const std::string &material, const std::string &description));
-
- ptree getEDKResult(FileEncryptionInfo &encryptionInfo) {
- ptree map;
- map.put("name", encryptionInfo.getKeyName());
- map.put("iv", encryptionInfo.getIv());
- map.put("material", KmsClientProvider::base64Encode(encryptionInfo.getKey()));
- return map;
- }
-
-};
-
-#endif /* _HDFS_LIBHDFS3_MOCK_KMSCLIENTPROVIDER_H_ */
diff --git a/depends/libhdfs3/src/client/CryptoCodec.cpp b/depends/libhdfs3/src/client/CryptoCodec.cpp
deleted file mode 100644
index bd4443f..0000000
--- a/depends/libhdfs3/src/client/CryptoCodec.cpp
+++ /dev/null
@@ -1,216 +0,0 @@
-/********************************************************************
- * 2014 -
- * open source under Apache License Version 2.0
- ********************************************************************/
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "CryptoCodec.h"
-#include "Logger.h"
-
-using namespace Hdfs::Internal;
-
-
-namespace Hdfs {
-
- //copy from java HDFS code
- std::string CryptoCodec::calculateIV(const std::string& initIV, unsigned long counter) {
- char IV[initIV.length()];
-
- int i = initIV.length(); // IV length
- int j = 0; // counter bytes index
- unsigned int sum = 0;
- while (i-- > 0) {
- // (sum >>> Byte.SIZE) is the carry for addition
- sum = (initIV[i] & 0xff) + (sum >> 8);
- if (j++ < 8) { // Big-endian, and long is 8 bytes length
- sum += (char) counter & 0xff;
- counter >>= 8;
- }
- IV[i] = (char) sum;
- }
-
- return std::string(IV, initIV.length());
- }
-
- CryptoCodec::CryptoCodec(FileEncryptionInfo *encryptionInfo, shared_ptr<KmsClientProvider> kcp, int32_t bufSize) :
- encryptionInfo(encryptionInfo), kcp(kcp), bufSize(bufSize)
- {
-
- // Init global status
- ERR_load_crypto_strings();
- OpenSSL_add_all_algorithms();
- OPENSSL_config(NULL);
-
- // Create cipher context
- cipherCtx = EVP_CIPHER_CTX_new();
- cipher = NULL;
-
- padding = 0;
- counter = 0;
- is_init = false;
- }
-
- CryptoCodec::~CryptoCodec()
- {
- if (cipherCtx)
- EVP_CIPHER_CTX_free(cipherCtx);
- }
-
- std::string CryptoCodec::getDecryptedKeyFromKms()
- {
- ptree map = kcp->decryptEncryptedKey(*encryptionInfo);
- std::string key;
- try {
- key = map.get < std::string > ("material");
- } catch (...) {
- THROW(HdfsIOException, "CryptoCodec : Can not get key from kms.");
- }
-
- int rem = key.length() % 4;
- if (rem) {
- rem = 4 - rem;
- while (rem != 0) {
- key = key + "=";
- rem--;
- }
- }
-
- std::replace(key.begin(), key.end(), '-', '+');
- std::replace(key.begin(), key.end(), '_', '/');
-
- LOG(DEBUG3, "CryptoCodec : getDecryptedKeyFromKms material is :%s", key.c_str());
-
- key = KmsClientProvider::base64Decode(key);
- return key;
- }
-
- int CryptoCodec::init(CryptoMethod crypto_method, int64_t stream_offset) {
- // Check CryptoCodec init or not.
- if (is_init)
- return 0;
-
- // Get decrypted key from KMS.
- decryptedKey = getDecryptedKeyFromKms();
-
- // Select cipher method based on the decrypted key length.
- AlgorithmBlockSize = decryptedKey.length();
- if (AlgorithmBlockSize == KEY_LENGTH_256) {
- cipher = EVP_aes_256_ctr();
- } else if (AlgorithmBlockSize == KEY_LENGTH_128) {
- cipher = EVP_aes_128_ctr();
- } else {
- LOG(WARNING, "CryptoCodec : Invalid key length.");
- return -1;
- }
-
- is_init = true;
- // Calculate iv and counter in order to init cipher context with cipher method. Default value is 0.
- if ((resetStreamOffset(crypto_method, stream_offset)) < 0) {
- is_init = false;
- return -1;
- }
-
- LOG(DEBUG3, "CryptoCodec init success, length of the decrypted key is : %llu, crypto method is : %d", AlgorithmBlockSize, crypto_method);
- return 1;
-
- }
-
- int CryptoCodec::resetStreamOffset(CryptoMethod crypto_method, int64_t stream_offset) {
- // Check CryptoCodec init or not.
- if (is_init == false)
- return -1;
- // Calculate new IV when appending an existed file.
- std::string iv = encryptionInfo->getIv();
- if (stream_offset > 0) {
- counter = stream_offset / AlgorithmBlockSize;
- padding = stream_offset % AlgorithmBlockSize;
- iv = this->calculateIV(iv, counter);
- }
-
- // Judge the crypto method is encrypt or decrypt.
- int enc = (method == CryptoMethod::ENCRYPT) ? 1 : 0;
-
- // Init cipher context with cipher method.
- if (!EVP_CipherInit_ex(cipherCtx, cipher, NULL,
- (const unsigned char *) decryptedKey.c_str(), (const unsigned char *) iv.c_str(),
- enc)) {
- LOG(WARNING, "EVP_CipherInit_ex failed");
- return -1;
- }
-
- // AES/CTR/NoPadding, set padding to 0.
- EVP_CIPHER_CTX_set_padding(cipherCtx, 0);
-
- return 1;
- }
-
- std::string CryptoCodec::cipher_wrap(const char * buffer, int64_t size) {
- if (!is_init)
- THROW(InvalidParameter, "CryptoCodec isn't init");
-
- int offset = 0;
- int remaining = size;
- int len = 0;
- int ret = 0;
-
- std::string in_buf(buffer,size);
- std::string out_buf(size, 0);
- //set necessary padding when appending a existed file
- if (padding > 0) {
- in_buf.insert(0, padding, 0);
- out_buf.resize(padding+size);
- remaining += padding;
- }
-
- // If the encode/decode buffer size larger than crypto buffer size, encode/decode buffer one by one
- while (remaining > bufSize) {
- ret = EVP_CipherUpdate(cipherCtx, (unsigned char *) &out_buf[offset], &len,
- (const unsigned char *)in_buf.data() + offset, bufSize);
-
- if (!ret) {
- std::string err = ERR_lib_error_string(ERR_get_error());
- THROW(HdfsIOException, "CryptoCodec : cipher_wrap AES data failed:%s, crypto_method:%d", err.c_str(), method);
- }
- offset += len;
- remaining -= len;
- LOG(DEBUG3, "CryptoCodec : EVP_CipherUpdate successfully, len:%d", len);
- }
-
- if (remaining) {
- ret = EVP_CipherUpdate(cipherCtx, (unsigned char *) &out_buf[offset], &len,
- (const unsigned char *) in_buf.data() + offset, remaining);
-
- if (!ret) {
- std::string err = ERR_lib_error_string(ERR_get_error());
- THROW(HdfsIOException, "CryptoCodec : cipher_wrap AES data failed:%s, crypto_method:%d", err.c_str(), method);
- }
-
- }
-
- //cut off padding when necessary
- if (padding > 0) {
- out_buf.erase(0, padding);
- padding = 0;
- }
-
- return out_buf;
- }
-
-}
-
diff --git a/depends/libhdfs3/src/client/CryptoCodec.h b/depends/libhdfs3/src/client/CryptoCodec.h
deleted file mode 100644
index f5070fe..0000000
--- a/depends/libhdfs3/src/client/CryptoCodec.h
+++ /dev/null
@@ -1,112 +0,0 @@
-/********************************************************************
- * 2014 -
- * open source under Apache License Version 2.0
- ********************************************************************/
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef _HDFS_LIBHDFS3_CLIENT_CRYPTOCODEC_H_
-#define _HDFS_LIBHDFS3_CLIENT_CRYPTOCODEC_H_
-
-#include <string>
-
-#include "openssl/conf.h"
-#include "openssl/evp.h"
-#include "openssl/err.h"
-#include "FileEncryptionInfo.h"
-#include "KmsClientProvider.h"
-
-#define KEY_LENGTH_256 32
-#define KEY_LENGTH_128 16
-
-namespace Hdfs {
-
- enum CryptoMethod {
- DECRYPT = 0,
- ENCRYPT = 1
- };
-
- class CryptoCodec {
- public:
- /**
- * Construct a CryptoCodec instance.
- * @param encryptionInfo the encryption info of file.
- * @param kcp a KmsClientProvider instance to get key from kms server.
- * @param bufSize crypto buffer size.
- */
- CryptoCodec(FileEncryptionInfo *encryptionInfo, shared_ptr<KmsClientProvider> kcp, int32_t bufSize);
-
- /**
- * Destroy a CryptoCodec instance.
- */
- virtual ~CryptoCodec();
-
- /**
- * encrypt/decrypt(depends on init()) buffer data
- * @param buffer
- * @param size
- * @return encrypt/decrypt result string
- */
- virtual std::string cipher_wrap(const char * buffer, int64_t size);
-
- /**
- * init CryptoCodec
- * @param method CryptoMethod
- * @param stream_offset 0 when open a new file; file_lenght when append a existed file
- * @return 1 success; 0 no need(already inited); -1 failed
- */
- virtual int init(CryptoMethod crypto_method, int64_t stream_offset = 0);
-
- /**
- * Reset iv and padding value when seek file.
- * @param crypto_method do encrypt/decrypt work according to crypto_method.
- * @param stream_offset the offset of the current file.
- * @return 1 sucess; -1 failed.
- */
- virtual int resetStreamOffset(CryptoMethod crypto_method, int64_t stream_offset);
-
- private:
-
- /**
- * Get decrypted key from kms.
- */
- std::string getDecryptedKeyFromKms();
-
- /**
- * calculate new IV for appending a existed file
- * @param initIV
- * @param counter
- * @return new IV string
- */
- std::string calculateIV(const std::string& initIV, unsigned long counter);
-
- shared_ptr<KmsClientProvider> kcp;
- FileEncryptionInfo* encryptionInfo;
- EVP_CIPHER_CTX* cipherCtx;
- const EVP_CIPHER* cipher;
- CryptoMethod method;
-
- bool is_init;
- int32_t bufSize;
- int64_t padding;
- int64_t counter;
- std::string decryptedKey;
- uint64_t AlgorithmBlockSize;
- };
-
-}
-#endif
diff --git a/depends/libhdfs3/src/client/HttpClient.cpp b/depends/libhdfs3/src/client/HttpClient.cpp
deleted file mode 100644
index 09a74a6..0000000
--- a/depends/libhdfs3/src/client/HttpClient.cpp
+++ /dev/null
@@ -1,349 +0,0 @@
-/********************************************************************
- * 2014 -
- * open source under Apache License Version 2.0
- ********************************************************************/
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "HttpClient.h"
-#include "Logger.h"
-
-using namespace Hdfs::Internal;
-
-namespace Hdfs {
-
-#define CURL_SETOPT(handle, option, optarg, fmt, ...) \
- res = curl_easy_setopt(handle, option, optarg); \
- if (res != CURLE_OK) { \
- THROW(HdfsIOException, fmt, ##__VA_ARGS__); \
- }
-
-#define CURL_SETOPT_ERROR1(handle, option, optarg, fmt) \
- CURL_SETOPT(handle, option, optarg, fmt, curl_easy_strerror(res));
-
-#define CURL_SETOPT_ERROR2(handle, option, optarg, fmt) \
- CURL_SETOPT(handle, option, optarg, fmt, curl_easy_strerror(res), \
- errorString().c_str())
-
-#define CURL_PERFORM(handle, fmt) \
- res = curl_easy_perform(handle); \
- if (res != CURLE_OK) { \
- THROW(HdfsIOException, fmt, curl_easy_strerror(res), errorString().c_str()); \
- }
-
-#define CURL_GETOPT_ERROR2(handle, option, optarg, fmt) \
- res = curl_easy_getinfo(handle, option, optarg); \
- if (res != CURLE_OK) { \
- THROW(HdfsIOException, fmt, curl_easy_strerror(res), errorString().c_str()); \
- }
-
-#define CURL_GET_RESPONSE(handle, code, fmt) \
- CURL_GETOPT_ERROR2(handle, CURLINFO_RESPONSE_CODE, code, fmt);
-
-HttpClient::HttpClient() : curl(NULL), list(NULL) {
-}
-
-/**
- * Construct a HttpClient instance.
- * @param url a url which is the address to send the request to the corresponding http server.
- */
-HttpClient::HttpClient(const std::string &url) {
- curl = NULL;
- list = NULL;
- this->url = url;
-}
-
-/**
- * Destroy a HttpClient instance.
- */
-HttpClient::~HttpClient()
-{
- destroy();
-}
-
-/**
- * Receive error string from curl.
- */
-std::string HttpClient::errorString() {
- if (strlen(errbuf) == 0) {
- return "";
- }
- return errbuf;
-}
-
-/**
- * Curl call back function to receive the reponse messages.
- * @return return the size of reponse messages.
- */
-size_t HttpClient::CurlWriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp)
-{
- size_t realsize = size * nmemb;
- if (userp == NULL || contents == NULL) {
- return 0;
- }
- ((std::string *) userp)->append((const char *) contents, realsize);
- LOG(DEBUG3, "HttpClient : Http response is : %s", ((std::string * )userp)->c_str());
- return realsize;
-}
-
-/**
- * Init curl handler and set curl options.
- */
-void HttpClient::init() {
- if (!initialized) {
- initialized = true;
- if (curl_global_init (CURL_GLOBAL_ALL)) {
- THROW(HdfsIOException, "Cannot initialize curl client for KMS");
- }
- }
-
- curl = curl_easy_init();
- if (!curl) {
- THROW(HdfsIOException, "Cannot initialize curl handle for KMS");
- }
-
- CURL_SETOPT_ERROR1(curl, CURLOPT_ERRORBUFFER, errbuf,
- "Cannot initialize curl error buffer for KMS: %s");
-
- errbuf[0] = 0;
-
- CURL_SETOPT_ERROR2(curl, CURLOPT_NOPROGRESS, 1,
- "Cannot initialize no progress in HttpClient: %s: %s");
-
- CURL_SETOPT_ERROR2(curl, CURLOPT_VERBOSE, 0,
- "Cannot initialize no verbose in HttpClient: %s: %s");
-
- CURL_SETOPT_ERROR2(curl, CURLOPT_COOKIEFILE, "",
- "Cannot initialize cookie behavior in HttpClient: %s: %s");
-
- CURL_SETOPT_ERROR2(curl, CURLOPT_HTTPHEADER, list,
- "Cannot initialize headers in HttpClient: %s: %s");
-
- CURL_SETOPT_ERROR2(curl, CURLOPT_WRITEFUNCTION, HttpClient::CurlWriteMemoryCallback,
- "Cannot initialize body reader in HttpClient: %s: %s");
-
- CURL_SETOPT_ERROR2(curl, CURLOPT_WRITEDATA, (void *)&response,
- "Cannot initialize body reader data in HttpClient: %s: %s");
-
-
- /* Some servers don't like requests that are made without a user-agent
- * field, so we provide one
- */
- CURL_SETOPT_ERROR2(curl, CURLOPT_USERAGENT, "libcurl-agent/1.0",
- "Cannot initialize user agent in HttpClient: %s: %s");
- list = NULL;
-}
-
-/**
- * Do clean up for curl.
- */
-void HttpClient::destroy() {
- if (curl) {
- curl_easy_cleanup(curl);
- curl = NULL;
- }
- if (list) {
- curl_slist_free_all(list);
- list = NULL;
- }
- initialized = false;
-}
-
-/**
- * Set url for http client.
- */
-void HttpClient::setURL(const std::string &url) {
- this->url = url;
-}
-
-/**
- * Set retry times for http request which can be configured in config file.
- */
-void HttpClient::setRequestRetryTimes(int request_retry_times) {
- if (request_retry_times < 0) {
- THROW(InvalidParameter, "HttpClient : Invalid value for request_retry_times.");
- }
- this->request_retry_times = request_retry_times;
-}
-
-/**
- * Set request timeout which can be configured in config file.
- */
-void HttpClient::setRequestTimeout(int64_t curl_timeout) {
- if (curl_timeout < 0) {
- THROW(InvalidParameter, "HttpClient : Invalid value for curl_timeout.");
- }
- this->curl_timeout = curl_timeout;
-}
-
-/**
- * Set headers for http client.
- */
-void HttpClient::setHeaders(const std::vector<std::string> &headers) {
- if (!headers.empty()) {
- this->headers = headers;
- for (std::string header : headers) {
- list = curl_slist_append(list, header.c_str());
- if (!list) {
- THROW(HdfsIOException, "Cannot add header in HttpClient.");
- }
- }
- } else {
- LOG(DEBUG3, "HttpClient : Header is empty.");
- }
-}
-
-
-/**
- * Set body for http client.
- */
-void HttpClient::setBody(const std::string &body) {
- this->body = body;
-}
-
-/**
- * Set expected response code.
- */
-void HttpClient::setExpectedResponseCode(int64_t response_code_ok) {
- this->response_code_ok = response_code_ok;
-}
-
-/**
- * Http common method to get response info by sending request to http server.
- * @param method : define different http methods.
- * @return return response info.
- */
-std::string HttpClient::httpCommon(httpMethod method) {
- /* Set headers and url. */
- if (list != NULL) {
- CURL_SETOPT_ERROR2(curl, CURLOPT_HTTPHEADER, list,
- "Cannot initialize headers in HttpClient: %s: %s");
- } else {
- LOG(DEBUG3, "HttpClient : Http Header is NULL");
- }
-
- if (curl != NULL) {
- CURL_SETOPT_ERROR2(curl, CURLOPT_URL, url.c_str(),
- "Cannot initialize url in HttpClient: %s: %s");
- } else {
- LOG(LOG_ERROR, "HttpClient : Http URL is NULL");
- }
-
- /* Set body based on different http method. */
- switch (method) {
- case HTTP_GET:
- {
- break;
- }
- case HTTP_POST:
- {
- CURL_SETOPT_ERROR2(curl, CURLOPT_COPYPOSTFIELDS, body.c_str(),
- "Cannot initialize post data in HttpClient: %s: %s");
- break;
- }
- case HTTP_DELETE:
- {
- CURL_SETOPT_ERROR2(curl, CURLOPT_CUSTOMREQUEST, "DELETE",
- "Cannot initialize set customer request in HttpClient: %s: %s");
- break;
- }
- case HTTP_PUT:
- {
- CURL_SETOPT_ERROR2(curl, CURLOPT_CUSTOMREQUEST, "PUT",
- "Cannot initialize set customer request in HttpClient: %s: %s");
- CURL_SETOPT_ERROR2(curl, CURLOPT_COPYPOSTFIELDS, body.c_str(),
- "Cannot initialize post data in HttpClient: %s: %s");
- break;
- }
- default:
- {
- LOG(LOG_ERROR, "HttpClient : unknown method: %d", method);
- }
- }
-
- /* Do several http request try according to request_retry_times
- * until got the right response code.
- */
- int64_t response_code = -1;
-
- while (request_retry_times >= 0 && response_code != response_code_ok) {
- request_retry_times -= 1;
- response = "";
- CURL_SETOPT_ERROR2(curl, CURLOPT_TIMEOUT, curl_timeout,
- "Send request to http server timeout: %s: %s");
- CURL_PERFORM(curl, "Could not send request in HttpClient: %s %s");
- CURL_GET_RESPONSE(curl, &response_code,
- "Cannot get response code in HttpClient: %s: %s");
- }
- LOG(DEBUG3, "HttpClient : The http method is %d. The http url is %s. The http response is %s.",
- method, url.c_str(), response.c_str());
- return response;
-}
-
-/**
- * Http GET method.
- */
-std::string HttpClient::get() {
- return httpCommon(HTTP_GET);
-}
-
-/**
- * Http POST method.
- */
-std::string HttpClient::post() {
- return httpCommon(HTTP_POST);
-}
-
-/**
- * Http DELETE method.
- */
-std::string HttpClient::del() {
- return httpCommon(HTTP_DELETE);
-}
-
-/**
- * Http PUT method.
- */
-std::string HttpClient::put() {
- return httpCommon(HTTP_PUT);
-}
-
-
-/**
- * URL encodes the given string.
- */
-std::string HttpClient::escape(const std::string &data) {
- if (curl) {
- char *output = curl_easy_escape(curl, data.c_str(), data.length());
- if (output) {
- std::string out(output);
- return out;
- } else {
- THROW(HdfsIOException, "HttpClient : Curl escape failed.");
- }
- } else {
- LOG(WARNING, "HttpClient : Curl in escape method is NULL");
- }
- std::string empty;
- return empty;
-}
-}
-
-/* Curl global init only can be done once. */
-bool Hdfs::HttpClient::initialized = false;
-
diff --git a/depends/libhdfs3/src/client/HttpClient.h b/depends/libhdfs3/src/client/HttpClient.h
deleted file mode 100644
index c77789b..0000000
--- a/depends/libhdfs3/src/client/HttpClient.h
+++ /dev/null
@@ -1,155 +0,0 @@
-/********************************************************************
- * 2014 -
- * open source under Apache License Version 2.0
- ********************************************************************/
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef _HDFS_LIBHDFS3_CLIENT_HTTPCLIENT_H_
-#define _HDFS_LIBHDFS3_CLIENT_HTTPCLIENT_H_
-
-#include <string>
-#include <vector>
-#include <curl/curl.h>
-#include "Exception.h"
-#include "ExceptionInternal.h"
-
-typedef enum httpMethod {
- HTTP_GET = 0,
- HTTP_POST = 1,
- HTTP_DELETE = 2,
- HTTP_PUT = 3
-} httpMethod;
-
-namespace Hdfs {
-
-class HttpClient {
-public:
- HttpClient();
-
- /**
- * Construct a HttpClient instance.
- * @param url a url which is the address to send the request to the corresponding http server.
- */
- HttpClient(const std::string &url);
-
- /**
- * Destroy a HttpClient instance.
- */
- virtual ~HttpClient();
-
- /**
- * Set url for http client.
- */
- void setURL(const std::string &url);
-
- /**
- * Set headers for http client.
- */
- void setHeaders(const std::vector<std::string> &headers);
-
- /**
- * Set body for http client.
- */
- void setBody(const std::string &body);
-
- /**
- * Set retry times for http request which can be configured in config file.
- */
- void setRequestRetryTimes(int requst_retry_times);
-
- /**
- * Set request timeout which can be configured in config file.
- */
- void setRequestTimeout(int64_t curl_timeout);
-
- /**
- * Set expected response code.
- */
- void setExpectedResponseCode(int64_t response_code_ok);
-
- /**
- * Init curl handler and set options for curl.
- */
- void init();
-
- /**
- * Do clean up for curl.
- */
- void destroy();
-
- /**
- * Http POST method.
- */
- virtual std::string post();
-
- /**
- * Http DELETE method.
- */
- virtual std::string del();
-
- /**
- * Http PUT method.
- */
- virtual std::string put();
-
- /**
- * Http GET method.
- */
- virtual std::string get();
-
- /**
- * URL encodes the given string.
- */
- std::string escape(const std::string &data);
-
- /**
- * Receive error string from curl.
- */
- std::string errorString();
-
-private:
-
- /**
- * Http common method to get response info by sending request to http server.
- * @param method : define different http methods.
- * @return return response info.
- */
- std::string httpCommon(httpMethod method);
-
- /**
- * Curl call back function to receive the reponse messages.
- * @return return the size of reponse messages.
- */
- static size_t CurlWriteMemoryCallback(void *contents, size_t size, size_t nmemb, void *userp);
-
- static bool initialized;
- CURLcode res;
- std::string url;
- std::vector<std::string> headers;
- std::string body;
- int64_t response_code_ok;
- int request_retry_times;
- int64_t curl_timeout;
- CURL *curl;
- struct curl_slist *list;
- std::string response;
- char errbuf[CURL_ERROR_SIZE] = { 0 };
-};
-
-}
-#endif
diff --git a/depends/libhdfs3/src/client/KmsClientProvider.cpp b/depends/libhdfs3/src/client/KmsClientProvider.cpp
deleted file mode 100644
index ac59570..0000000
--- a/depends/libhdfs3/src/client/KmsClientProvider.cpp
+++ /dev/null
@@ -1,325 +0,0 @@
-/********************************************************************
- * 2014 -
- * open source under Apache License Version 2.0
- ********************************************************************/
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "KmsClientProvider.h"
-#include "Logger.h"
-#include <gsasl.h>
-#include <map>
-#include <boost/property_tree/json_parser.hpp>
-using namespace Hdfs::Internal;
-using boost::property_tree::read_json;
-using boost::property_tree::write_json;
-
-namespace Hdfs {
-
-/**
- * Convert ptree format to json format
- */
-std::string KmsClientProvider::toJson(const ptree &data) {
- std::ostringstream buf;
- try {
- write_json(buf, data, false);
- std::string json = buf.str();
- return json;
- } catch (...) {
- THROW(HdfsIOException, "KmsClientProvider : Write json failed.");
- }
-}
-
-/**
- * Convert json format to ptree format
- */
-ptree KmsClientProvider::fromJson(const std::string &data) {
- ptree pt2;
- try {
- std::istringstream is(data);
- read_json(is, pt2);
- return pt2;
- } catch (...) {
- THROW(HdfsIOException, "KmsClientProvider : Read json failed.");
- }
-}
-
-/**
- * Encode string to base64.
- */
-std::string KmsClientProvider::base64Encode(const std::string &data) {
- char * buffer = NULL;
- size_t len = 0;
- int rc = 0;
- std::string result;
-
- LOG(DEBUG3, "KmsClientProvider : Encode data is %s", data.c_str());
-
- if (GSASL_OK != (rc = gsasl_base64_to(data.data(), data.size(), &buffer, &len))) {
- assert(GSASL_MALLOC_ERROR == rc);
- throw std::bad_alloc();
- }
-
- if (buffer) {
- result.assign(buffer, len);
- free(buffer);
- }
-
- if (!buffer || result.length() != len) {
- THROW(HdfsIOException,
- "KmsClientProvider: Failed to encode string to base64");
- }
-
- return result;
-}
-
-/**
- * Decode base64 to string.
- */
-std::string KmsClientProvider::base64Decode(const std::string &data) {
- char * buffer = NULL;
- size_t len = 0;
- int rc = 0;
- std::string result;
-
- if (GSASL_OK != (rc = gsasl_base64_from(data.data(), data.size(), &buffer, &len))) {
- assert(GSASL_MALLOC_ERROR == rc);
- throw std::bad_alloc();
- }
-
- if (buffer) {
- result.assign(buffer, len);
- free(buffer);
- }
-
- if (!buffer || result.length() != len) {
- THROW(HdfsIOException,
- "KmsClientProvider: Failed to decode base64 to string");
- }
-
- return result;
-}
-
-/**
- * Construct a KmsClientProvider instance.
- * @param auth RpcAuth to get the auth method and user info.
- * @param conf a SessionConfig to get the configuration.
- */
-KmsClientProvider::KmsClientProvider(shared_ptr<RpcAuth> rpcAuth, shared_ptr<SessionConfig> config) : auth(rpcAuth), conf(config)
-{
- hc.reset(new HttpClient());
- method = RpcAuth::ParseMethod(conf->getKmsMethod());
-}
-
-/**
- * Set HttpClient object.
- */
-void KmsClientProvider::setHttpClient(shared_ptr<HttpClient> hc)
-{
- this->hc = hc;
-}
-
-/**
- * Parse kms url from configure file.
- */
-std::string KmsClientProvider::parseKmsUrl()
-{
- std::string start = "kms://";
- std::string http = "http@";
- std::string https = "https@";
- std::string urlParse = conf->getKmsUrl();
- LOG(DEBUG3, "KmsClientProvider : Get kms url from conf : %s.",
- urlParse.c_str());
- if (urlParse.compare(0, start.length(), start) == 0) {
- start = urlParse.substr(start.length());
- if (start.compare(0, http.length(), http) == 0) {
- return "http://" + start.substr(http.length());
- } else if (start.compare(0, https.length(), https) == 0) {
- return "https://" + start.substr(https.length());
- } else
- THROW(HdfsIOException, "Bad KMS provider URL: %s", urlParse.c_str());
- } else
- THROW(HdfsIOException, "Bad KMS provider URL: %s", urlParse.c_str());
-
-}
-
-/**
- * Build kms url based on urlSuffix and different auth method.
- */
-std::string KmsClientProvider::buildKmsUrl(const std::string &url, const std::string &urlSuffix)
-{
- std::string baseUrl = url;
- baseUrl = url + "/v1/" + urlSuffix;
- std::size_t found = urlSuffix.find('?');
-
- if (method == AuthMethod::KERBEROS) {
- // todo
- THROW(InvalidParameter, "KmsClientProvider : Not support kerberos yet.");
- } else if (method == AuthMethod::SIMPLE) {
- std::string user = auth->getUser().getRealUser();
- LOG(DEBUG3,
- "KmsClientProvider : Kms urlSuffix is : %s. Auth real user is : %s.",
- urlSuffix.c_str(), user.c_str());
- if (user.length() == 0)
- user = auth->getUser().getKrbName();
- if (found != std::string::npos)
- return baseUrl + "&user.name=" + user;
- else
- return baseUrl + "?user.name=" + user;
- } else {
- return baseUrl;
- }
-}
-
-/**
- * Set common headers for kms API.
- */
-void KmsClientProvider::setCommonHeaders(std::vector<std::string>& headers)
-{
- headers.push_back("Content-Type: application/json");
- headers.push_back("Accept: *");
-}
-
-
-/**
- * Create an encryption key from kms.
- * @param keyName the name of this key.
- * @param cipher the ciphertext of this key. e.g. "AES/CTR/NoPadding" .
- * @param length the length of this key.
- * @param material will be encode to base64.
- * @param description key's info.
- */
-void KmsClientProvider::createKey(const std::string &keyName, const std::string &cipher, const int length, const std::string &material, const std::string &description)
-{
- hc->init();
- /* Prepare url for HttpClient.*/
- url = parseKmsUrl();
- std::string urlSuffix = "keys";
- url = buildKmsUrl(url, urlSuffix);
- /* Prepare headers for HttpClient.*/
- std::vector<std::string> headers;
- setCommonHeaders(headers);
- /* Prepare body for HttpClient. */
- ptree map;
- map.put("name", keyName);
- map.put("cipher", cipher);
- map.put("description", description);
- std::string body = toJson(map);
- /* Set options for HttpClient to get response. */
- hc->setURL(url);
- hc->setHeaders(headers);
- hc->setBody(body);
- hc->setRequestRetryTimes(conf->getHttpRequestRetryTimes());
- hc->setRequestTimeout(conf->getCurlTimeOut());
- hc->setExpectedResponseCode(201);
- std::string response = hc->post();
-
- LOG(DEBUG3,
- "KmsClientProvider::createKey : The key name, key cipher, key length, key material, description are : %s, %s, %d, %s, %s. The kms url is : %s . The kms body is : %s. The response of kms server is : %s .",
- keyName.c_str(), cipher.c_str(), length, material.c_str(),
- description.c_str(), url.c_str(), body.c_str(), response.c_str());
-
-}
-
-/**
- * Get key metadata based on encrypted file's key name.
- * @param encryptionInfo the encryption info of file.
- * @return return response info about key metadata from kms server.
- */
-ptree KmsClientProvider::getKeyMetadata(const FileEncryptionInfo &encryptionInfo)
-{
- hc->init();
- url = parseKmsUrl();
- std::string urlSuffix = "key/" + hc->escape(encryptionInfo.getKeyName()) + "/_metadata";
- url = buildKmsUrl(url, urlSuffix);
-
- hc->setURL(url);
- hc->setExpectedResponseCode(200);
- hc->setRequestRetryTimes(conf->getHttpRequestRetryTimes());
- hc->setRequestTimeout(conf->getCurlTimeOut());
- std::string response = hc->get();
-
- LOG(DEBUG3,
- "KmsClientProvider::getKeyMetadata : The kms url is : %s. The response of kms server is : %s .",
- url.c_str(), response.c_str());
-
- return fromJson(response);
-
-}
-
-/**
- * Delete an encryption key from kms.
- * @param encryptionInfo the encryption info of file.
- */
-void KmsClientProvider::deleteKey(const FileEncryptionInfo &encryptionInfo)
-{
- hc->init();
- url = parseKmsUrl();
- std::string urlSuffix = "key/" + hc->escape(encryptionInfo.getKeyName());
- url = buildKmsUrl(url, urlSuffix);
-
- hc->setURL(url);
- hc->setExpectedResponseCode(200);
- hc->setRequestRetryTimes(conf->getHttpRequestRetryTimes());
- hc->setRequestTimeout(conf->getCurlTimeOut());
- std::string response = hc->del();
-
- LOG(DEBUG3,
- "KmsClientProvider::deleteKey : The kms url is : %s. The response of kms server is : %s .",
- url.c_str(), response.c_str());
-}
-
-/**
- * Decrypt an encrypted key from kms.
- * @param encryptionInfo the encryption info of file.
- * @return return decrypted key.
- */
-ptree KmsClientProvider::decryptEncryptedKey(const FileEncryptionInfo &encryptionInfo)
-{
- hc->init();
- /* Prepare HttpClient url. */
- url = parseKmsUrl();
- std::string urlSuffix = "keyversion/" + hc->escape(encryptionInfo.getEzKeyVersionName()) + "/_eek?eek_op=decrypt";
- url = buildKmsUrl(url, urlSuffix);
- /* Prepare HttpClient headers. */
- std::vector<std::string> headers;
- setCommonHeaders(headers);
- /* Prepare HttpClient body with json format. */
- ptree map;
- map.put("name", encryptionInfo.getKeyName());
- map.put("iv", base64Encode(encryptionInfo.getIv()));
- map.put("material", base64Encode(encryptionInfo.getKey()));
- std::string body = toJson(map);
-
- /* Set options for HttpClient. */
- hc->setURL(url);
- hc->setHeaders(headers);
- hc->setBody(body);
- hc->setExpectedResponseCode(200);
- hc->setRequestRetryTimes(conf->getHttpRequestRetryTimes());
- hc->setRequestTimeout(conf->getCurlTimeOut());
- std::string response = hc->post();
-
- LOG(DEBUG3,
- "KmsClientProvider::decryptEncryptedKey : The kms url is : %s . The kms body is : %s. The response of kms server is : %s .",
- url.c_str(), body.c_str(), response.c_str());
- return fromJson(response);
-}
-
-}
-
diff --git a/depends/libhdfs3/src/client/KmsClientProvider.h b/depends/libhdfs3/src/client/KmsClientProvider.h
deleted file mode 100644
index a6c4336..0000000
--- a/depends/libhdfs3/src/client/KmsClientProvider.h
+++ /dev/null
@@ -1,142 +0,0 @@
-/********************************************************************
- * 2014 -
- * open source under Apache License Version 2.0
- ********************************************************************/
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef _HDFS_LIBHDFS3_CLIENT_KMSCLIENTPROVIDER_H_
-#define _HDFS_LIBHDFS3_CLIENT_KMSCLIENTPROVIDER_H_
-
-#include <string>
-#include <gsasl.h>
-
-#include "openssl/conf.h"
-#include "openssl/evp.h"
-#include "openssl/err.h"
-#include "FileEncryptionInfo.h"
-#include "HttpClient.h"
-#include <vector>
-#include "common/SessionConfig.h"
-#include "rpc/RpcAuth.h"
-#include "common/Memory.h"
-#include <boost/property_tree/ptree.hpp>
-
-using boost::property_tree::ptree;
-using namespace Hdfs::Internal;
-
-
-namespace Hdfs {
-
-class KmsClientProvider {
-public:
-
- /**
- * Construct a KmsClientProvider instance.
- * @param auth RpcAuth to get the auth method and user info.
- * @param conf a SessionConfig to get the configuration.
- */
- KmsClientProvider(shared_ptr<RpcAuth> auth, shared_ptr<SessionConfig> conf);
-
- /**
- * Destroy a KmsClientProvider instance.
- */
- virtual ~KmsClientProvider() {
- }
-
- /**
- * Set HttpClient object.
- */
- void setHttpClient(shared_ptr<HttpClient> hc);
-
- /**
- * Create an encryption key from kms.
- * @param keyName the name of this key.
- * @param cipher the ciphertext of this key. e.g. "AES/CTR/NoPadding" .
- * @param length the length of this key.
- * @param material will be encode to base64.
- * @param description key's info.
- */
- virtual void createKey(const std::string &keyName, const std::string &cipher, const int length, const std::string &material, const std::string &description);
-
- /**
- * Get key metadata based on encrypted file's key name.
- * @param encryptionInfo the encryption info of file.
- * @return return response info about key metadata from kms server.
- */
- virtual ptree getKeyMetadata(const FileEncryptionInfo &encryptionInfo);
-
- /**
- * Delete an encryption key from kms.
- * @param encryptionInfo the encryption info of file.
- */
- virtual void deleteKey(const FileEncryptionInfo &encryptionInfo);
-
- /**
- * Decrypt an encrypted key from kms.
- * @param encryptionInfo the encryption info of file.
- * @return return decrypted key.
- */
- virtual ptree decryptEncryptedKey(const FileEncryptionInfo &encryptionInfo);
-
- /**
- * Encode string to base64.
- */
- static std::string base64Encode(const std::string &data);
-
- /**
- * Decode base64 to string.
- */
- static std::string base64Decode(const std::string &data);
-
-private:
-
- /**
- * Convert ptree format to json format.
- */
- static std::string toJson(const ptree &data);
-
- /**
- * Convert json format to ptree format.
- */
- static ptree fromJson(const std::string &data);
-
- /**
- * Parse kms url from configure file.
- */
- std::string parseKmsUrl();
-
- /**
- * Build kms url based on urlSuffix and different auth method.
- */
- std::string buildKmsUrl(const std::string &url, const std::string &urlSuffix);
- /**
- * Set common headers for kms API.
- */
- void setCommonHeaders(std::vector<std::string>& headers);
-
- shared_ptr<HttpClient> hc;
- std::string url;
-
- shared_ptr<RpcAuth> auth;
- AuthMethod method;
- shared_ptr<SessionConfig> conf;
-
-};
-
-}
-#endif
diff --git a/depends/libhdfs3/test/function/TestKmsClient.cpp b/depends/libhdfs3/test/function/TestKmsClient.cpp
deleted file mode 100644
index 0295866..0000000
--- a/depends/libhdfs3/test/function/TestKmsClient.cpp
+++ /dev/null
@@ -1,178 +0,0 @@
-/********************************************************************
- * 2014 -
- * open source under Apache License Version 2.0
- ********************************************************************/
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "client/FileSystem.h"
-#include "client/FileSystemInter.h"
-#include "DateTime.h"
-#include "Exception.h"
-#include "ExceptionInternal.h"
-#include "gtest/gtest.h"
-#include "TestUtil.h"
-#include "Thread.h"
-#include "XmlConfig.h"
-#include "client/KmsClientProvider.h"
-#include "client/HttpClient.h"
-#include "client/hdfs.h"
-
-#include <ctime>
-
-#ifndef TEST_HDFS_PREFIX
-#define TEST_HDFS_PREFIX "./"
-#endif
-
-#define BASE_DIR TEST_HDFS_PREFIX"/testKmsClient/"
-
-using namespace Hdfs;
-using namespace Hdfs::Internal;
-
-class TestKmsClient: public ::testing::Test {
-public:
- TestKmsClient() :
- conf("function-test.xml") {
- conf.set("hadoop.kms.authentication.type", "simple");
- conf.set("dfs.encryption.key.provider.uri",
- "kms://http@0.0.0.0:16000/kms");
- sconf.reset(new SessionConfig(conf));
- userInfo.setRealUser("abai");
- auth.reset(new RpcAuth(userInfo, RpcAuth::ParseMethod(sconf->getKmsMethod())));
- hc.reset(new HttpClient());
- kcp.reset(new KmsClientProvider(auth, sconf));
- kcp->setHttpClient(hc);
- fs.reset(new FileSystem(conf));
- fs->connect();
- }
-
- ~TestKmsClient() {
- try {
- fs->disconnect();
- } catch (...) {
- }
- }
-protected:
- Config conf;
- UserInfo userInfo;
- shared_ptr<RpcAuth> auth;
- shared_ptr<HttpClient> hc;
- shared_ptr<KmsClientProvider> kcp;
- shared_ptr<SessionConfig> sconf;
- shared_ptr<FileSystem> fs;
-};
-
-TEST_F(TestKmsClient, CreateKeySuccess) {
- std::string keyName = "testcreatekeyname";
- std::string cipher = "AES/CTR/NoPadding";
- int length = 128;
- std::string material = "testCreateKey";
- std::string description = "Test create key success.";
- ASSERT_NO_THROW(
- kcp->createKey(keyName, cipher, length, material, description));
-}
-
-TEST_F(TestKmsClient, GetKeyMetadataSuccess) {
- FileEncryptionInfo encryptionInfo;
- encryptionInfo.setKeyName("testcreatekeyname");
- ptree map = kcp->getKeyMetadata(encryptionInfo);
- std::string keyName = map.get < std::string > ("name");
- ASSERT_STREQ("testcreatekeyname", keyName.c_str());
-}
-
-TEST_F(TestKmsClient, DeleteKeySuccess) {
- FileEncryptionInfo encryptionInfo;
- encryptionInfo.setKeyName("testcreatekeyname");
- ASSERT_NO_THROW(kcp->deleteKey(encryptionInfo));
-}
-
-
-TEST_F(TestKmsClient, DecryptEncryptedKeySuccess) {
- hdfsFS hfs = NULL;
- struct hdfsBuilder * bld = hdfsNewBuilder();
- assert(bld != NULL);
- hdfsBuilderSetNameNode(bld, "default");
- hfs = hdfsBuilderConnect(bld);
-
- //create key
- hc.reset(new HttpClient());
- kcp.reset(new KmsClientProvider(auth, sconf));
- kcp->setHttpClient(hc);
- std::string keyName = "testdekeyname";
- std::string cipher = "AES/CTR/NoPadding";
- int length = 128;
- std::string material = "test DEK";
- std::string description = "Test DEK create key success.";
- kcp->createKey(keyName, cipher, length, material, description);
-
- //delete dir
- hdfsDelete(hfs, BASE_DIR"/testDEKey", true);
-
- //create dir
- EXPECT_EQ(0, hdfsCreateDirectory(hfs, BASE_DIR"/testDEKey"));
-
- //create encryption zone and encrypted file
- ASSERT_EQ(0,
- hdfsCreateEncryptionZone(hfs, BASE_DIR"/testDEKey", "testdekeyname"));
- std::string hadoop_command = "hadoop fs -touchz ";
- std::string tdeFile = BASE_DIR"/testDEKey/tdefile";
- std::string createFile = hadoop_command + tdeFile;
- std::system(createFile.c_str());
-
- //decrypt encrypted key
- hc.reset(new HttpClient());
- kcp.reset(new KmsClientProvider(auth, sconf));
- kcp->setHttpClient(hc);
- FileStatus fileStatus = fs->getFileStatus(tdeFile.c_str());
- FileEncryptionInfo *enInfo = fileStatus.getFileEncryption();
- ptree map = kcp->decryptEncryptedKey(*enInfo);
- std::string versionName = map.get < std::string > ("versionName");
- ASSERT_STREQ("EK", versionName.c_str());
-
- //delete key
- hc.reset(new HttpClient());
- kcp.reset(new KmsClientProvider(auth, sconf));
- kcp->setHttpClient(hc);
- FileEncryptionInfo encryptionInfo;
- encryptionInfo.setKeyName("testdekeyname");
- kcp->deleteKey(encryptionInfo);
-
-}
-
-TEST_F(TestKmsClient, CreateKeyFailediBadUrl) {
- std::string keyName = "testcreatekeyfailname";
- std::string cipher = "AES/CTR/NoPadding";
- std::string material = "testCreateKey";
-
- std::string url[4] = { "ftp:///http@localhost:16000/kms",
- "kms://htttp@localhost:16000/kms",
- "kms:///httpss@localhost:16000/kms",
- "kms:///http@localhost:16000/kms" };
- for (int i = 0; i < 4; i++) {
- conf.set("hadoop.kms.authentication.type", "simple");
- conf.set("dfs.encryption.key.provider.uri", url[i]);
- sconf.reset(new SessionConfig(conf));
- userInfo.setRealUser("abai");
- auth.reset(new RpcAuth(userInfo, RpcAuth::ParseMethod(sconf->getKmsMethod())));
- hc.reset(new HttpClient());
- kcp.reset(new KmsClientProvider(auth, sconf));
- ASSERT_THROW(kcp->createKey("tesTdeBadUrl", "AES/CTR/NoPadding", 128,
- "test DEK", "test DEK description"), HdfsIOException);
- }
-}
-
-
diff --git a/depends/libhdfs3/test/unit/UnitTestCryptoCodec.cpp b/depends/libhdfs3/test/unit/UnitTestCryptoCodec.cpp
deleted file mode 100644
index 92e9403..0000000
--- a/depends/libhdfs3/test/unit/UnitTestCryptoCodec.cpp
+++ /dev/null
@@ -1,141 +0,0 @@
-/********************************************************************
- * 2014 -
- * open source under Apache License Version 2.0
- ********************************************************************/
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "gtest/gtest.h"
-#include "gmock/gmock.h"
-
-#include "client/FileSystem.h"
-#include "client/FileSystemImpl.h"
-#include "client/FileSystemInter.h"
-#include "client/OutputStream.h"
-#include "client/OutputStreamImpl.h"
-#include "client/Packet.h"
-#include "client/Pipeline.h"
-#include "DateTime.h"
-#include "MockFileSystemInter.h"
-#include "MockCryptoCodec.h"
-#include "MockKmsClientProvider.h"
-#include "MockHttpClient.h"
-#include "MockLeaseRenewer.h"
-#include "MockPipeline.h"
-#include "NamenodeStub.h"
-#include "server/ExtendedBlock.h"
-#include "TestDatanodeStub.h"
-#include "TestUtil.h"
-#include "Thread.h"
-#include "XmlConfig.h"
-#include "client/KmsClientProvider.h"
-#include <string>
-
-using namespace Hdfs;
-using namespace Hdfs::Internal;
-using namespace Hdfs::Mock;
-using namespace testing;
-using ::testing::AtLeast;
-
-
-class TestCryptoCodec: public ::testing::Test {
-public:
- TestCryptoCodec() {
-
- }
-
- ~TestCryptoCodec() {
- }
-
-protected:
-};
-
-TEST_F(TestCryptoCodec, KmsGetKey_Success) {
- FileEncryptionInfo encryptionInfo;
- encryptionInfo.setKeyName("KmsName");
- encryptionInfo.setIv("KmsIv");
- encryptionInfo.setEzKeyVersionName("KmsVersionName");
- encryptionInfo.setKey("KmsKey");
- Config conf;
- conf.set("hadoop.kms.authentication.type", "simple");
- conf.set("dfs.encryption.key.provider.uri", "kms://http@0.0.0.0:16000/kms");
- shared_ptr<SessionConfig> sconf(new SessionConfig(conf));
- UserInfo userInfo;
- userInfo.setRealUser("abai");
- shared_ptr<RpcAuth> auth(new RpcAuth(userInfo, RpcAuth::ParseMethod(sconf->getKmsMethod())));
-
- KmsClientProvider kcp(auth, sconf);
- shared_ptr<MockHttpClient> hc(new MockHttpClient());
- kcp.setHttpClient(hc);
-
- EXPECT_CALL(*hc, post()).Times(1).WillOnce(
- Return(hc->getPostResult(encryptionInfo)));
-
- ptree map = kcp.decryptEncryptedKey(encryptionInfo);
- std::string KmsKey = map.get < std::string > ("material");
-
- ASSERT_STREQ("KmsKey", KmsKey.c_str());
-}
-
-TEST_F(TestCryptoCodec, encode_Success) {
- FileEncryptionInfo encryptionInfo;
- encryptionInfo.setKeyName("ESKeyName");
- encryptionInfo.setIv("ESIv");
- encryptionInfo.setEzKeyVersionName("ESVersionName");
-
- Config conf;
- conf.set("hadoop.kms.authentication.type", "simple");
- conf.set("dfs.encryption.key.provider.uri", "kms://http@0.0.0.0:16000/kms");
- shared_ptr<SessionConfig> sconf(new SessionConfig(conf));
- UserInfo userInfo;
- userInfo.setRealUser("abai");
- shared_ptr<RpcAuth> auth(
- new RpcAuth(userInfo, RpcAuth::ParseMethod(sconf->getKmsMethod())));
-
- shared_ptr<MockKmsClientProvider> kcp(
- new MockKmsClientProvider(auth, sconf));
-
- //char buf[1024] = "encode hello world";
- char buf[1024];
- Hdfs::FillBuffer(buf, sizeof(buf)-1, 2048);
- buf[sizeof(buf)-1] = 0;
-
- int32_t bufSize = 1024;
-
- std::string Key[2] = { "012345678901234567890123456789ab",
- "0123456789012345"};
- for (int i = 0; i < 2; i++) {
- encryptionInfo.setKey(Key[i]);
- shared_ptr<MockHttpClient> hc(new MockHttpClient());
- kcp->setHttpClient(hc);
-
- EXPECT_CALL(*kcp, decryptEncryptedKey(_)).Times(2).WillRepeatedly(
- Return(kcp->getEDKResult(encryptionInfo)));
-
- CryptoCodec es(&encryptionInfo, kcp, bufSize);
- es.init(CryptoMethod::ENCRYPT);
- CryptoCodec ds(&encryptionInfo, kcp, bufSize);
- ds.init(CryptoMethod::DECRYPT);
-
-
- std::string encodeStr = es.cipher_wrap(buf, strlen(buf));
- ASSERT_NE(0, memcmp(buf, encodeStr.c_str(), strlen(buf)));
-
- std::string decodeStr = ds.cipher_wrap(encodeStr.c_str(), strlen(buf));
- ASSERT_STREQ(decodeStr.c_str(), buf);
- }
-}
