Repository: incubator-hawq Updated Branches: refs/heads/master efa1230a9 -> 517e6d26c
HAWQ-1193. Add createEncryption, getEZForPath, listEncryptionZones RPC for libhdfs3. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/517e6d26 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/517e6d26 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/517e6d26 Branch: refs/heads/master Commit: 517e6d26cadff3a0fc03af1f36e8302bf454e573 Parents: efa1230 Author: ivan <iw...@pivotal.io> Authored: Tue Dec 20 16:51:18 2016 +0800 Committer: ivan <iw...@pivotal.io> Committed: Thu Jan 19 16:10:32 2017 +0800 ---------------------------------------------------------------------- depends/libhdfs3/mock/MockFileSystemInter.h | 5 + .../libhdfs3/src/client/EncryptionZoneInfo.h | 80 +++++++++++++ .../src/client/EncryptionZoneIterator.cpp | 86 ++++++++++++++ .../src/client/EncryptionZoneIterator.h | 56 +++++++++ .../libhdfs3/src/client/FileEncryptionInfo.h | 93 +++++++++++++++ depends/libhdfs3/src/client/FileStatus.h | 19 +++- depends/libhdfs3/src/client/FileSystem.cpp | 54 +++++++++ depends/libhdfs3/src/client/FileSystem.h | 32 ++++++ depends/libhdfs3/src/client/FileSystemImpl.cpp | 88 ++++++++++++++ depends/libhdfs3/src/client/FileSystemImpl.h | 40 +++++++ depends/libhdfs3/src/client/FileSystemInter.h | 40 +++++++ depends/libhdfs3/src/client/Hdfs.cpp | 114 ++++++++++++++++++- depends/libhdfs3/src/client/hdfs.h | 63 ++++++++++ .../src/proto/ClientNamenodeProtocol.proto | 8 ++ depends/libhdfs3/src/proto/datatransfer.proto | 1 + depends/libhdfs3/src/proto/encryption.proto | 67 +++++++++++ depends/libhdfs3/src/proto/hdfs.proto | 60 ++++++++++ depends/libhdfs3/src/rpc/RpcAuth.h | 2 +- depends/libhdfs3/src/rpc/RpcChannel.cpp | 4 +- depends/libhdfs3/src/server/Namenode.h | 35 +++++- depends/libhdfs3/src/server/NamenodeImpl.cpp | 73 ++++++++++++ depends/libhdfs3/src/server/NamenodeImpl.h | 9 ++ depends/libhdfs3/src/server/NamenodeProxy.cpp | 24 ++++ depends/libhdfs3/src/server/NamenodeProxy.h | 7 ++ depends/libhdfs3/src/server/RpcHelper.h | 31 +++++ .../libhdfs3/test/function/TestCInterface.cpp | 42 +++++++ .../libhdfs3/test/function/TestFileSystem.cpp | 33 ++++++ .../libhdfs3/test/function/TestOutputStream.cpp | 12 ++ 28 files changed, 1171 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/mock/MockFileSystemInter.h ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/mock/MockFileSystemInter.h b/depends/libhdfs3/mock/MockFileSystemInter.h index 4d91c73..336db7e 100644 --- a/depends/libhdfs3/mock/MockFileSystemInter.h +++ b/depends/libhdfs3/mock/MockFileSystemInter.h @@ -101,6 +101,11 @@ public: MOCK_METHOD3(getFileBlockLocations, std::vector<Hdfs::BlockLocation> (const char * path, int64_t start, int64_t len)); MOCK_METHOD2(listAllDirectoryItems, std::vector<Hdfs::FileStatus> (const char * path, bool needLocation)); MOCK_METHOD0(getPeerCache, Hdfs::Internal::PeerCache &()); + MOCK_METHOD2(createEncryptionZone, bool(const char * path, const char * keyName)); + MOCK_METHOD1(getEZForPath, Hdfs::EncryptionZoneInfo(const char * path)); + MOCK_METHOD2(listEncryptionZones, bool(const int64_t id, std::vector<Hdfs::EncryptionZoneInfo> &)); + MOCK_METHOD0(listEncryptionZone, Hdfs::EncryptionZoneIterator()); + MOCK_METHOD0(listAllEncryptionZoneItems, std::vector<Hdfs::EncryptionZoneInfo>()); }; #endif /* _HDFS_LIBHDFS3_MOCK_MOCKSOCKET_H_ */ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/EncryptionZoneInfo.h ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/client/EncryptionZoneInfo.h b/depends/libhdfs3/src/client/EncryptionZoneInfo.h new file mode 100644 index 0000000..d436ae7 --- /dev/null +++ b/depends/libhdfs3/src/client/EncryptionZoneInfo.h @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef _HDFS_LIBHDFS3_CLIENT_ENCRYPTIONZONEINFO_H_ +#define _HDFS_LIBHDFS3_CLIENT_ENCRYPTIONZONEINFO_H_ + +#include <string> + +namespace Hdfs { + +class EncryptionZoneInfo { +public: + EncryptionZoneInfo() : + suite(0), cryptoProtocolVersion(0), id(0) { + } + + int getSuite() const { + return suite; + } + + void setSuite(int suite) { + this->suite = suite; + } + + int getCryptoProtocolVersion() const { + return cryptoProtocolVersion; + } + + void setCryptoProtocolVersion(int cryptoProtocolVersion) { + this->cryptoProtocolVersion = cryptoProtocolVersion; + } + + int getId() const { + return id; + } + + void setId(int id) { + this->id = id; + } + + const char * getPath() const{ + return path.c_str(); + } + + void setPath(const char * path){ + this->path = path; + } + + const char * getKeyName() const{ + return keyName.c_str(); + } + + void setKeyName(const char * keyName){ + this->keyName = keyName; + } + +private: + int suite; + int cryptoProtocolVersion; + int64_t id; + std::string path; + std::string keyName; +}; + +} +#endif http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/EncryptionZoneIterator.cpp ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/client/EncryptionZoneIterator.cpp b/depends/libhdfs3/src/client/EncryptionZoneIterator.cpp new file mode 100644 index 0000000..085541a --- /dev/null +++ b/depends/libhdfs3/src/client/EncryptionZoneIterator.cpp @@ -0,0 +1,86 @@ +/******************************************************************** + * 2014 - + * open source under Apache License Version 2.0 + ********************************************************************/ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "EncryptionZoneIterator.h" +#include "Exception.h" +#include "ExceptionInternal.h" +#include "EncryptionZoneInfo.h" +#include "FileSystemImpl.h" + +namespace Hdfs { +EncryptionZoneIterator::EncryptionZoneIterator() :filesystem(NULL), id(0), next(0) { +} + +EncryptionZoneIterator::EncryptionZoneIterator(Hdfs::Internal::FileSystemImpl * const fs, + const int64_t id) :filesystem(fs), id(id), next(0) { +} + +EncryptionZoneIterator::EncryptionZoneIterator(const EncryptionZoneIterator & it) : + filesystem(it.filesystem), id(it.id), next(it.next), lists(it.lists) { +} + +EncryptionZoneIterator & EncryptionZoneIterator::operator =(const EncryptionZoneIterator & it) { + if (this == &it) { + return *this; + } + + filesystem = it.filesystem; + id = it.id; + next = it.next; + lists = it.lists; + return *this; +} + +bool EncryptionZoneIterator::listEncryptionZones() { + bool more; + + if (NULL == filesystem) { + return false; + } + + next = 0; + lists.clear(); + more = filesystem->listEncryptionZones(id, lists); + if (!lists.empty()){ + id = lists.back().getId(); + } + + return more || !lists.empty(); +} + +bool EncryptionZoneIterator::hasNext() { + if (next >= lists.size()) { + return listEncryptionZones(); + } + + return true; +} + +Hdfs::EncryptionZoneInfo EncryptionZoneIterator::getNext() { + if (next >= lists.size()) { + if (!listEncryptionZones()) { + THROW(HdfsIOException, "End of the dir flow"); + } + } + return lists[next++]; +} + +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/EncryptionZoneIterator.h ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/client/EncryptionZoneIterator.h b/depends/libhdfs3/src/client/EncryptionZoneIterator.h new file mode 100644 index 0000000..9e37559 --- /dev/null +++ b/depends/libhdfs3/src/client/EncryptionZoneIterator.h @@ -0,0 +1,56 @@ +/******************************************************************** + * 2014 - + * open source under Apache License Version 2.0 + ********************************************************************/ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef _HDFS_LIBHFDS3_CLIENT_ENCRYPTIONZONE_ITERATOR_H_ +#define _HDFS_LIBHFDS3_CLIENT_ENCRYPTIONZONE_ITERATOR_H_ + +#include "FileStatus.h" +#include "EncryptionZoneInfo.h" +#include <vector> + +namespace Hdfs { +namespace Internal { +class FileSystemImpl; +} + +class EncryptionZoneIterator { +public: + EncryptionZoneIterator(); + EncryptionZoneIterator(Hdfs::Internal::FileSystemImpl * const fs, + const int64_t id); + EncryptionZoneIterator(const EncryptionZoneIterator & it); + EncryptionZoneIterator & operator = (const EncryptionZoneIterator & it); + bool hasNext(); + EncryptionZoneInfo getNext(); + +private: + bool listEncryptionZones(); + +private: + Hdfs::Internal::FileSystemImpl * filesystem; + int64_t id; + size_t next; + std::vector<EncryptionZoneInfo> lists; +}; + +} + +#endif /* _HDFS_LIBHFDS3_CLIENT_ENCRYPTIONZONE_ITERATOR_H_ */ http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/FileEncryptionInfo.h ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/client/FileEncryptionInfo.h b/depends/libhdfs3/src/client/FileEncryptionInfo.h new file mode 100644 index 0000000..32ead6c --- /dev/null +++ b/depends/libhdfs3/src/client/FileEncryptionInfo.h @@ -0,0 +1,93 @@ +/******************************************************************** + * 2014 - + * open source under Apache License Version 2.0 + ********************************************************************/ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef _HDFS_LIBHDFS3_CLIENT_FILEENCRYPTIONINFO_H_ +#define _HDFS_LIBHDFS3_CLIENT_FILEENCRYPTIONINFO_H_ + +#include <string> + +namespace Hdfs { + +class FileEncryptionInfo { +public: + FileEncryptionInfo() : + cryptoProtocolVersion(0), suite(0){ + } + + int getSuite() const { + return suite; + } + + void setSuite(int suite) { + this->suite = suite; + } + + int getCryptoProtocolVersion() const { + return cryptoProtocolVersion; + } + + void setCryptoProtocolVersion(int cryptoProtocolVersion) { + this->cryptoProtocolVersion = cryptoProtocolVersion; + } + + const std::string & getKey() const{ + return key; + } + + void setKey(const std::string & key){ + this->key = key; + } + + const std::string & getKeyName() const{ + return keyName; + } + + void setKeyName(const std::string & keyName){ + this->keyName = keyName; + } + + const std::string & getIv() const{ + return iv; + } + + void setIv(const std::string & iv){ + this->iv = iv; + } + + const std::string & getEzKeyVersionName() const{ + return ezKeyVersionName; + } + + void setEzKeyVersionName(const std::string & ezKeyVersionName){ + this->ezKeyVersionName = ezKeyVersionName; + } + +private: + int suite; + int cryptoProtocolVersion; + std::string key; + std::string iv; + std::string keyName; + std::string ezKeyVersionName; +}; + +} +#endif http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/FileStatus.h ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/client/FileStatus.h b/depends/libhdfs3/src/client/FileStatus.h index 51b5096..1033b80 100644 --- a/depends/libhdfs3/src/client/FileStatus.h +++ b/depends/libhdfs3/src/client/FileStatus.h @@ -23,8 +23,9 @@ #define _HDFS_LIBHDFS3_CLIENT_FILESTATUS_H_ #include "Permission.h" +#include "client/FileEncryptionInfo.h" -#include <string> +#include <string.h> namespace Hdfs { @@ -143,6 +144,21 @@ public: return !symlink.empty(); } + /** + * Get encryption information for a file. + */ + FileEncryptionInfo* getFileEncryption(){ + return &fileEncryption; + } + + /** + * Is an encryption file? + * @return true is this is an encryption file + */ + bool isFileEncrypted() const { + return fileEncryption.getKey().length() > 0 && fileEncryption.getKeyName().length() > 0; + } + private: bool isdir; int64_t atime; @@ -155,6 +171,7 @@ private: std::string owner; std::string path; std::string symlink; + FileEncryptionInfo fileEncryption; }; } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/FileSystem.cpp ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/client/FileSystem.cpp b/depends/libhdfs3/src/client/FileSystem.cpp index 6a660a4..8c18590 100644 --- a/depends/libhdfs3/src/client/FileSystem.cpp +++ b/depends/libhdfs3/src/client/FileSystem.cpp @@ -20,6 +20,7 @@ * limitations under the License. */ #include "DirectoryIterator.h" +#include "EncryptionZoneIterator.h" #include "Exception.h" #include "ExceptionInternal.h" #include "FileSystem.h" @@ -582,4 +583,57 @@ void FileSystem::cancelDelegationToken(const std::string & token) { impl->filesystem->cancelDelegationToken(token); } + +/** + * Create encryption zone for the directory with specific key name + * @param path the directory path which is to be created. + * @param keyname The key name of the encryption zone + * @return return true if success. + */ +bool FileSystem::createEncryptionZone(const char * path, const char * keyName) { + if (!impl) { + THROW(HdfsIOException, "FileSystem: not connected."); + } + + return impl->filesystem->createEncryptionZone(path, keyName); +} + +/** +* To get encryption zone information. +* @param path the path which information is to be returned. +* @return the encryption zone information. +*/ +EncryptionZoneInfo FileSystem::getEZForPath(const char * path) { + if (!impl) { + THROW(HdfsIOException, "FileSystem: not connected."); + } + + return impl->filesystem->getEZForPath(path); +} + +/** + * list the contents of an encryption zone. + * @return Return a iterator to visit all elements in this encryption zone. + */ +EncryptionZoneIterator FileSystem::listEncryptionZone() { + if (!impl) { + THROW(HdfsIOException, "FileSystem: not connected."); + } + + return impl->filesystem->listEncryptionZone(); +} + +/** +* list all the contents of encryption zones. +* @param id the index of encryption zones. +* @return Return a vector of encryption zones information.. +*/ +std::vector<EncryptionZoneInfo> FileSystem::listAllEncryptionZoneItems() { + if (!impl) { + THROW(HdfsIOException, "FileSystem: not connected."); + } + + return impl->filesystem->listAllEncryptionZoneItems(); +} + } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/FileSystem.h ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/client/FileSystem.h b/depends/libhdfs3/src/client/FileSystem.h index 4f99c3c..e1f4dd2 100644 --- a/depends/libhdfs3/src/client/FileSystem.h +++ b/depends/libhdfs3/src/client/FileSystem.h @@ -24,8 +24,10 @@ #include "BlockLocation.h" #include "DirectoryIterator.h" +#include "EncryptionZoneIterator.h" #include "FileStatus.h" #include "FileSystemStats.h" +#include "EncryptionZoneInfo.h" #include "Permission.h" #include "XmlConfig.h" @@ -276,6 +278,36 @@ public: */ void cancelDelegationToken(const std::string & token); + /** + * Create encryption zone for the directory with specific key name + * @param path the directory path which is to be created. + * @param keyname The key name of the encryption zone + * @return return true if success. + */ + bool createEncryptionZone(const char * path, const char * keyName); + + /** + * To get encryption zone information. + * @param path the path which information is to be returned. + * @return the encryption zone information. + */ + EncryptionZoneInfo getEZForPath(const char * path); + + /** + * list the contents of an encryption zone; + * @return Return a iterator to visit all elements in this encryption zone. + */ + EncryptionZoneIterator listEncryptionZone(); + + + /** + * list all the contents of encryption zones. + * @param id the index of encryption zones. + * @return Return a vector of encryption zones information.. + */ + std::vector<EncryptionZoneInfo> listAllEncryptionZoneItems(); + + private: Config conf; Internal::FileSystemWrapper * impl; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/FileSystemImpl.cpp ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/client/FileSystemImpl.cpp b/depends/libhdfs3/src/client/FileSystemImpl.cpp index 7b0f20a..6ee2b91 100644 --- a/depends/libhdfs3/src/client/FileSystemImpl.cpp +++ b/depends/libhdfs3/src/client/FileSystemImpl.cpp @@ -22,11 +22,13 @@ #include "Atomic.h" #include "BlockLocation.h" #include "DirectoryIterator.h" +#include "EncryptionZoneIterator.h" #include "Exception.h" #include "ExceptionInternal.h" #include "FileStatus.h" #include "FileSystemImpl.h" #include "FileSystemStats.h" +#include "EncryptionZoneInfo.h" #include "InputStream.h" #include "LeaseRenewer.h" #include "Logger.h" @@ -775,5 +777,91 @@ bool FileSystemImpl::unregisterOpenedOutputStream() { return openedOutputStream == 0; } +/** + * Create encryption zone for the directory with specific key name + * @param path the directory path which is to be created. + * @param keyname The key name of the encryption zone + * @return return true if success. + */ + +bool FileSystemImpl::createEncryptionZone(const char * path, const char * keyName) { + if (!nn) { + THROW(HdfsIOException, "FileSystemImpl: not connected."); + } + + if (NULL == path || !strlen(path)) { + THROW(InvalidParameter, "Invalid input: path should not be empty"); + } + + if (NULL == keyName || !strlen(keyName)) { + THROW(InvalidParameter, "Invalid input: key name should not be empty"); + } + + return nn->createEncryptionZone(getStandardPath(path), keyName); +} + + +/** + * To get encryption zone information. + * @param path the path which information is to be returned. + * @return the encryption zone information. + */ + +EncryptionZoneInfo FileSystemImpl::getEZForPath(const char * path) { + if (!nn) { + THROW(HdfsIOException, "FileSystemImpl: not connected."); + } + + if (NULL == path || !strlen(path)) { + THROW(InvalidParameter, "Invalid input: path should not be empty"); + } + + return nn->getEncryptionZoneInfo(getStandardPath(path), NULL); +} + +bool FileSystemImpl::listEncryptionZones(const int64_t id, + std::vector<EncryptionZoneInfo> & ezl) { + if (!nn) { + THROW(HdfsIOException, "FileSystemImpl: not connected."); + } + + return nn->listEncryptionZones(id, ezl); +} + +/** + * list the contents of an encryption zone. + * @return return the encryption zone information. + */ +EncryptionZoneIterator FileSystemImpl::listEncryptionZone() { + if (!nn) { + THROW(HdfsIOException, "FileSystemImpl: not connected."); + } + + return EncryptionZoneIterator(this, 0); +} +/** + * list all the contents of encryption zones. + * @param id the index of the encyrption zones. + * @return Return a vector of encryption zones information. + */ + +std::vector<EncryptionZoneInfo> FileSystemImpl::listAllEncryptionZoneItems() { + if (!nn) { + THROW(HdfsIOException, "FileSystemImpl: not connected."); + } + + std::vector<EncryptionZoneInfo> retval; + retval.clear(); + int64_t id = 0; + + EncryptionZoneIterator it; + it = FileSystemImpl::listEncryptionZone(); + + while (it.hasNext()) { + retval.push_back(it.getNext()); + } + return retval; +} + } } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/FileSystemImpl.h ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/client/FileSystemImpl.h b/depends/libhdfs3/src/client/FileSystemImpl.h index bd590bd..1c11a61 100644 --- a/depends/libhdfs3/src/client/FileSystemImpl.h +++ b/depends/libhdfs3/src/client/FileSystemImpl.h @@ -24,10 +24,12 @@ #include "BlockLocation.h" #include "DirectoryIterator.h" +#include "EncryptionZoneIterator.h" #include "FileStatus.h" #include "FileSystemInter.h" #include "FileSystemKey.h" #include "FileSystemStats.h" +#include "EncryptionZoneInfo.h" #include "Permission.h" #include "server/Namenode.h" #include "SessionConfig.h" @@ -477,6 +479,44 @@ public: return *peerCache; } + /** + * Create encryption zone for the directory with specific key name + * @param path the directory path which is to be created. + * @param keyname The key name of the encryption zone + * @return return true if success. + */ + bool createEncryptionZone(const char * path, const char * keyName); + + /** + * To get encryption zone information. + * @param path the path which information is to be returned. + * @return the encryption zone information. + */ + EncryptionZoneInfo getEZForPath(const char * path); + + /** + * Get a partial listing of the indicated encryption zones + * + * @param id the index of encryption zones. + * @param ezl append the returned encryption zones. + * @return return true if there are more items. + */ + bool listEncryptionZones(const int64_t id, std::vector<EncryptionZoneInfo> & ezl); + + /** + * list the contents of an encryption zone. + * @return Return a iterator to visit all elements in this encryption zone. + */ + EncryptionZoneIterator listEncryptionZone(); + + + /** + * list all the contents of encryption zones. + * @param id the index of encryption zones. + * @return Return a vector of encryption zones information.. + */ + std::vector<EncryptionZoneInfo> listAllEncryptionZoneItems(); + private: Config conf; FileSystemKey key; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/FileSystemInter.h ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/client/FileSystemInter.h b/depends/libhdfs3/src/client/FileSystemInter.h index 352c37f..45df347 100644 --- a/depends/libhdfs3/src/client/FileSystemInter.h +++ b/depends/libhdfs3/src/client/FileSystemInter.h @@ -27,9 +27,11 @@ #include "BlockLocation.h" #include "DirectoryIterator.h" +#include "EncryptionZoneIterator.h" #include "FileStatus.h" #include "FileSystemKey.h" #include "FileSystemStats.h" +#include "EncryptionZoneInfo.h" #include "PeerCache.h" #include "Permission.h" #include "server/LocatedBlocks.h" @@ -481,6 +483,44 @@ public: * @return return the peer cache. */ virtual PeerCache& getPeerCache() = 0; + + /** + * Create encryption zone for the directory with specific key name + * @param path the directory path which is to be created. + * @param keyname The key name of the encryption zone + * @return return true if success. + */ + virtual bool createEncryptionZone(const char * path, const char * keyName) = 0; + + /** + * To get encryption zone information. + * @param path the path which information is to be returned. + * @return the encryption zone information. + */ + virtual EncryptionZoneInfo getEZForPath(const char * path) = 0; + + /** + * Get a partial listing of the indicated encryption zones + * + * @param id the index of encryption zones. + * @param ezl append the returned encryption zones. + * @return return true if there are more items. + */ + virtual bool listEncryptionZones(const int64_t id, std::vector<EncryptionZoneInfo> & ezl) = 0; + + /** + * list the contents of an encryption zone. + * @return Return a iterator to visit all elements in this encryption zone. + */ + virtual EncryptionZoneIterator listEncryptionZone() = 0; + + + /** + * list all the contents of encryption zones. + * @param id the index of encryption zones. + * @return Return a vector of encryption zones information.. + */ + virtual std::vector<EncryptionZoneInfo> listAllEncryptionZoneItems() = 0; }; } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/Hdfs.cpp ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/client/Hdfs.cpp b/depends/libhdfs3/src/client/Hdfs.cpp index 395f4f8..2840adc 100644 --- a/depends/libhdfs3/src/client/Hdfs.cpp +++ b/depends/libhdfs3/src/client/Hdfs.cpp @@ -989,8 +989,31 @@ int hdfsSetReplication(hdfsFS fs, const char * path, int16_t replication) { return -1; } +static void ConstructHdfsEncryptionZoneInfo(hdfsEncryptionZoneInfo * infoEn, + std::vector<Hdfs::EncryptionZoneInfo> & enStatus) { + size_t size = enStatus.size(); + + for (size_t i = 0; i < size; ++i) { + infoEn[i].mSuite = enStatus[i].getSuite(); + infoEn[i].mCryptoProtocolVersion = enStatus[i].getCryptoProtocolVersion(); + infoEn[i].mId = enStatus[i].getId(); + infoEn[i].mPath = Strdup(enStatus[i].getPath()); + infoEn[i].mKeyName = Strdup(enStatus[i].getKeyName()); + } +} + +static void ConstructHdfsEncryptionFileInfo(hdfsEncryptionFileInfo * infoEn, + Hdfs::FileEncryptionInfo* enStatus) { + infoEn->mSuite = enStatus->getSuite(); + infoEn->mCryptoProtocolVersion = enStatus->getCryptoProtocolVersion(); + infoEn->mKey = const_cast<char*>(enStatus->getKey().c_str()); + infoEn->mKeyName = const_cast<char*>(enStatus->getKeyName().c_str()); + infoEn->mIv = const_cast<char*>(enStatus->getIv().c_str()); + infoEn->mEzKeyVersionName = const_cast<char*>(enStatus->getEzKeyVersionName().c_str()); +} + static void ConstructHdfsFileInfo(hdfsFileInfo * infos, - const std::vector<Hdfs::FileStatus> & status) { + std::vector<Hdfs::FileStatus> & status) { size_t size = status.size(); for (size_t i = 0; i < size; ++i) { @@ -1006,6 +1029,13 @@ static void ConstructHdfsFileInfo(hdfsFileInfo * infos, infos[i].mPermissions = status[i].getPermission().toShort(); infos[i].mReplication = status[i].getReplication(); infos[i].mSize = status[i].getLength(); + infos[i].mHdfsEncryptionFileInfo = NULL; + if (status[i].isFileEncrypted()) { + infos[i].mHdfsEncryptionFileInfo = new hdfsEncryptionFileInfo[1]; + memset(infos[i].mHdfsEncryptionFileInfo, 0, sizeof(hdfsEncryptionFileInfo)); + ConstructHdfsEncryptionFileInfo(infos[i].mHdfsEncryptionFileInfo, status[i].getFileEncryption()); + + } } } @@ -1021,7 +1051,7 @@ hdfsFileInfo * hdfsListDirectory(hdfsFS fs, const char * path, size = status.size(); retval = new hdfsFileInfo[size]; memset(retval, 0, sizeof(hdfsFileInfo) * size); - ConstructHdfsFileInfo(&retval[0], status); + ConstructHdfsFileInfo(retval, status); *numEntries = size; return retval; } catch (const std::bad_alloc & e) { @@ -1061,11 +1091,22 @@ hdfsFileInfo * hdfsGetPathInfo(hdfsFS fs, const char * path) { return NULL; } +void hdfsFreeEncryptionZoneInfo(hdfsEncryptionZoneInfo * infos, int numEntries) { + for (int i = 0; infos != NULL && i < numEntries; ++i) { + delete [] infos[i].mPath; + delete [] infos[i].mKeyName; + } + delete[] infos; +} + void hdfsFreeFileInfo(hdfsFileInfo * infos, int numEntries) { for (int i = 0; infos != NULL && i < numEntries; ++i) { delete [] infos[i].mGroup; delete [] infos[i].mName; delete [] infos[i].mOwner; + if (infos[i].mHdfsEncryptionFileInfo != NULL) { + delete [] infos[i].mHdfsEncryptionFileInfo; + } } delete[] infos; @@ -1450,6 +1491,75 @@ void hdfsFreeFileBlockLocations(BlockLocation * locations, int numOfBlock) { delete [] locations; } +int hdfsCreateEncryptionZone(hdfsFS fs, const char * path, const char * keyName) { + PARAMETER_ASSERT(fs && path && strlen(path) > 0 && keyName && strlen(keyName) > 0, -1, EINVAL); + + try { + return fs->getFilesystem().createEncryptionZone(path, keyName) ? 0 : -1; + } catch (const std::bad_alloc & e) { + SetErrorMessage("Out of memory"); + errno = ENOMEM; + } catch (...) { + SetLastException(Hdfs::current_exception()); + handleException(Hdfs::current_exception()); + } + + return -1; +} + +hdfsEncryptionZoneInfo * hdfsGetEZForPath(hdfsFS fs, const char * path) { + PARAMETER_ASSERT(fs && path && strlen(path) > 0, NULL, EINVAL); + hdfsEncryptionZoneInfo * retval = NULL; + + try { + retval = new hdfsEncryptionZoneInfo[1]; + memset(retval, 0, sizeof(hdfsEncryptionZoneInfo)); + std::vector<Hdfs::EncryptionZoneInfo> enStatus(1); + enStatus[0] = fs->getFilesystem().getEZForPath(path); + ConstructHdfsEncryptionZoneInfo(retval, enStatus); + return retval; + } catch (const std::bad_alloc & e) { + SetErrorMessage("Out of memory"); + hdfsFreeEncryptionZoneInfo(retval, 1); + /* If out of memory error occurred, free hdfsEncryptionZoneInfo array's memory. */ + errno = ENOMEM; + } catch (...) { + SetLastException(Hdfs::current_exception()); + hdfsFreeEncryptionZoneInfo(retval, 1); + /* If any exceptions throw out, free hdfsEncryptionZoneInfo array's memory. */ + handleException(Hdfs::current_exception()); + } + + return NULL; +} + + +hdfsEncryptionZoneInfo * hdfsListEncryptionZones(hdfsFS fs, int * numEntries) { + PARAMETER_ASSERT(fs, NULL, EINVAL); + hdfsEncryptionZoneInfo * retval = NULL; + int size = 0; + + try { + std::vector<Hdfs::EncryptionZoneInfo> enStatus = + fs->getFilesystem().listAllEncryptionZoneItems(); + size = enStatus.size(); + retval = new hdfsEncryptionZoneInfo[size]; + memset(retval, 0, sizeof(hdfsEncryptionZoneInfo) * size); + ConstructHdfsEncryptionZoneInfo(&retval[0], enStatus); + *numEntries = size; + return retval; + } catch (const std::bad_alloc & e) { + SetErrorMessage("Out of memory"); + /* If out of memory error occurred, free hdfsEncryptionZoneInfo array's memory. */ + hdfsFreeEncryptionZoneInfo(retval, size); + } catch (...) { + SetLastException(Hdfs::current_exception()); + /* If any exceptions throw out, free hdfsEncryptionZoneInfo array's memory. */ + hdfsFreeEncryptionZoneInfo(retval, size); + handleException(Hdfs::current_exception()); + } + return NULL; +} #ifdef __cplusplus } #endif http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/client/hdfs.h ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/client/hdfs.h b/depends/libhdfs3/src/client/hdfs.h index 5d356f9..f8b61ea 100644 --- a/depends/libhdfs3/src/client/hdfs.h +++ b/depends/libhdfs3/src/client/hdfs.h @@ -482,6 +482,30 @@ int hdfsCreateDirectory(hdfsFS fs, const char * path); int hdfsSetReplication(hdfsFS fs, const char * path, int16_t replication); /** + * hdfsEncryptionZoneInfo- Information about an encryption zone. + */ +typedef struct { + int mSuite; /* the suite of encryption zone */ + int mCryptoProtocolVersion; /* the version of crypto protocol */ + int64_t mId; /* the id of encryption zone */ + char * mPath; /* the path of encryption zone */ + char * mKeyName; /* the key name of encryption zone */ +} hdfsEncryptionZoneInfo; + + +/** + * hdfsEncryptionFileInfo - Information about an encryption file/directory. + */ +typedef struct { + int mSuite; /* the suite of encryption file/directory */ + int mCryptoProtocolVersion; /* the version of crypto protocol */ + char * mKey; /* the key of encryption file/directory */ + char * mKeyName; /* the key name of encryption file/directory */ + char * mIv; /* the iv of encryption file/directory */ + char * mEzKeyVersionName; /* the version encryption file/directory */ +} hdfsEncryptionFileInfo; + +/** * hdfsFileInfo - Information about a file/directory. */ typedef struct { @@ -495,6 +519,7 @@ typedef struct { char * mGroup; /* the group associated with the file */ short mPermissions; /* the permissions associated with the file */ tTime mLastAccess; /* the last access time for the file in seconds */ + hdfsEncryptionFileInfo * mHdfsEncryptionFileInfo; /* the encryption info of the file/directory */ } hdfsFileInfo; /** @@ -528,6 +553,15 @@ hdfsFileInfo * hdfsGetPathInfo(hdfsFS fs, const char * path); void hdfsFreeFileInfo(hdfsFileInfo * infos, int numEntries); /** + * hdfsFreeEncryptionZoneInfo - Free up the hdfsEncryptionZoneInfo array (including fields) + * @param infos The array of dynamically-allocated hdfsEncryptionZoneInfo + * objects. + * @param numEntries The size of the array. + */ +void hdfsFreeEncryptionZoneInfo(hdfsEncryptionZoneInfo * infos, int numEntries); + + +/** * hdfsGetHosts - Get hostnames where a particular block (determined by * pos & blocksize) of a file is stored. The last element in the array * is NULL. Due to replication, a single block could be present on @@ -723,6 +757,35 @@ BlockLocation * hdfsGetFileBlockLocations(hdfsFS fs, const char * path, */ void hdfsFreeFileBlockLocations(BlockLocation * locations, int numOfBlock); +/** + * Create encryption zone for the directory with specific key name + * @param fs The configured filesystem handle. + * @param path The path of the directory. + * @param keyname The key name of the encryption zone + * @return Returns 0 on success, -1 on error. + */ +int hdfsCreateEncryptionZone(hdfsFS fs, const char * path, const char * keyName); + +/** + * hdfsEncryptionZoneInfo - Get information about a path as a (dynamically + * allocated) single hdfsEncryptionZoneInfo struct. hdfsEncryptionZoneInfo should be + * called when the pointer is no longer needed. + * @param fs The configured filesystem handle. + * @param path The path of the encryption zone. + * @return Returns a dynamically-allocated hdfsEncryptionZoneInfo object; + * NULL on error. + */ +hdfsEncryptionZoneInfo * hdfsGetEZForPath(hdfsFS fs, const char * path); + +/** + * hdfsEncryptionZoneInfo - Get list of all the encryption zones. + * hdfsFreeEncryptionZoneInfo should be called to deallocate memory. + * @param fs The configured filesystem handle. + * @return Returns a dynamically-allocated array of hdfsEncryptionZoneInfo objects; + * NULL on error. + */ +hdfsEncryptionZoneInfo * hdfsListEncryptionZones(hdfsFS fs, int * numEntries); + #ifdef __cplusplus } #endif http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/proto/ClientNamenodeProtocol.proto ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/proto/ClientNamenodeProtocol.proto b/depends/libhdfs3/src/proto/ClientNamenodeProtocol.proto index 5362246..1ab69cd 100644 --- a/depends/libhdfs3/src/proto/ClientNamenodeProtocol.proto +++ b/depends/libhdfs3/src/proto/ClientNamenodeProtocol.proto @@ -33,6 +33,7 @@ package Hdfs.Internal; import "hdfs.proto"; import "Security.proto"; +import "encryption.proto"; /** * The ClientNamenodeProtocol Service defines the interface between a client @@ -74,6 +75,7 @@ message CreateRequestProto { required bool createParent = 5; required uint32 replication = 6; // Short: Only 16 bits used required uint64 blockSize = 7; + repeated CryptoProtocolVersionProto cryptoProtocolVersion = 8; } message CreateResponseProto { @@ -752,4 +754,10 @@ service ClientNamenodeProtocol { returns(GetSnapshotDiffReportResponseProto); rpc isFileClosed(IsFileClosedRequestProto) returns(IsFileClosedResponseProto); + rpc createEncryptionZone(CreateEncryptionZoneRequestProto) + returns(CreateEncryptionZoneResponseProto); + rpc listEncryptionZones(ListEncryptionZonesRequestProto) + returns(ListEncryptionZonesResponseProto); + rpc getEZForPath(GetEZForPathRequestProto) + returns(GetEZForPathResponseProto); } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/proto/datatransfer.proto ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/proto/datatransfer.proto b/depends/libhdfs3/src/proto/datatransfer.proto index 5d8013e..d787d8a 100644 --- a/depends/libhdfs3/src/proto/datatransfer.proto +++ b/depends/libhdfs3/src/proto/datatransfer.proto @@ -43,6 +43,7 @@ message DataTransferEncryptorMessageProto { required DataTransferEncryptorStatus status = 1; optional bytes payload = 2; optional string message = 3; + repeated CipherOptionProto cipherOption = 4; } message BaseHeaderProto { http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/proto/encryption.proto ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/proto/encryption.proto b/depends/libhdfs3/src/proto/encryption.proto new file mode 100644 index 0000000..53206f8 --- /dev/null +++ b/depends/libhdfs3/src/proto/encryption.proto @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * These .proto interfaces are private and stable. + * Please see http://wiki.apache.org/hadoop/Compatibility + * for what changes are allowed for a *stable* .proto interface. + */ + +/** This file contains protocol buffers that are used throughout HDFS -- i.e. + * by the client, server, and data transfer protocols. + */ + +option java_package = "org.apache.hadoop.hdfs.protocol.proto"; +option java_outer_classname = "EncryptionZonesProtos"; +option java_generate_equals_and_hash = true; +package Hdfs.Internal; + +import "hdfs.proto"; + +message CreateEncryptionZoneRequestProto { + required string src = 1; + optional string keyName = 2; +} + +message CreateEncryptionZoneResponseProto { +} + +message ListEncryptionZonesRequestProto { + required int64 id = 1; +} + +message EncryptionZoneProto { + required int64 id = 1; + required string path = 2; + required CipherSuiteProto suite = 3; + required CryptoProtocolVersionProto cryptoProtocolVersion = 4; + required string keyName = 5; +} + +message ListEncryptionZonesResponseProto { + repeated EncryptionZoneProto zones = 1; + required bool hasMore = 2; +} + +message GetEZForPathRequestProto { + required string src = 1; +} + +message GetEZForPathResponseProto { + optional EncryptionZoneProto zone = 1; +} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/proto/hdfs.proto ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/proto/hdfs.proto b/depends/libhdfs3/src/proto/hdfs.proto index 19e3f79..d85f217 100644 --- a/depends/libhdfs3/src/proto/hdfs.proto +++ b/depends/libhdfs3/src/proto/hdfs.proto @@ -160,6 +160,64 @@ message DataEncryptionKeyProto { optional string encryptionAlgorithm = 6; } +/** + * Cipher suite. + */ +enum CipherSuiteProto { + UNKNOWN = 1; + AES_CTR_NOPADDING = 2; +} + +/** + * Crypto protocol version used to access encrypted files. + */ +enum CryptoProtocolVersionProto { + UNKNOWN_PROTOCOL_VERSION = 1; + ENCRYPTION_ZONES = 2; +} + +/** + * Encryption information for a file. + */ +message FileEncryptionInfoProto { + required CipherSuiteProto suite = 1; + required CryptoProtocolVersionProto cryptoProtocolVersion = 2; + required bytes key = 3; + required bytes iv = 4; + required string keyName = 5; + required string ezKeyVersionName = 6; +} + +/** + * Encryption information for an individual + * file within an encryption zone + */ +message PerFileEncryptionInfoProto { + required bytes key = 1; + required bytes iv = 2; + required string ezKeyVersionName = 3; +} + +/** + * Encryption information for an encryption + * zone + */ +message ZoneEncryptionInfoProto { + required CipherSuiteProto suite = 1; + required CryptoProtocolVersionProto cryptoProtocolVersion = 2; + required string keyName = 3; +} + +/** + * Cipher option + */ +message CipherOptionProto { + required CipherSuiteProto suite = 1; + optional bytes inKey = 2; + optional bytes inIv = 3; + optional bytes outKey = 4; + optional bytes outIv = 5; +} /** * A set of file blocks and their locations. @@ -203,6 +261,8 @@ message HdfsFileStatusProto { // Optional field for fileId optional uint64 fileId = 13 [default = 0]; // default as an invalid id optional int32 childrenNum = 14 [default = -1]; + // Optional field for file encryption + optional FileEncryptionInfoProto fileEncryptionInfo = 15; } /** http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/rpc/RpcAuth.h ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/rpc/RpcAuth.h b/depends/libhdfs3/src/rpc/RpcAuth.h index df503bb..5075f08 100644 --- a/depends/libhdfs3/src/rpc/RpcAuth.h +++ b/depends/libhdfs3/src/rpc/RpcAuth.h @@ -33,7 +33,7 @@ namespace Internal { enum AuthMethod { SIMPLE = 80, KERBEROS = 81, //"GSSAPI" TOKEN = 82, //"DIGEST-MD5" - UNKNOWN = 255 + UNSURENESS = 255 }; enum AuthProtocol { http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/rpc/RpcChannel.cpp ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/rpc/RpcChannel.cpp b/depends/libhdfs3/src/rpc/RpcChannel.cpp index 974e86c..7f9ef5d 100644 --- a/depends/libhdfs3/src/rpc/RpcChannel.cpp +++ b/depends/libhdfs3/src/rpc/RpcChannel.cpp @@ -121,7 +121,7 @@ const RpcSaslProto_SaslAuth * RpcChannelImpl::createSaslClient( break; } else if (method.getMethod() == AuthMethod::SIMPLE) { return auth; - } else if (method.getMethod() == AuthMethod::UNKNOWN) { + } else if (method.getMethod() == AuthMethod::UNSURENESS) { return auth; } else { auth = NULL; @@ -187,7 +187,7 @@ RpcAuth RpcChannelImpl::setupSaslConnection() { if (retval.getMethod() == AuthMethod::SIMPLE) { done = true; - } else if (retval.getMethod() == AuthMethod::UNKNOWN) { + } else if (retval.getMethod() == AuthMethod::UNSURENESS) { THROW(AccessControlException, "Unknown auth mechanism"); } else { std::string respToken; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/server/Namenode.h ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/server/Namenode.h b/depends/libhdfs3/src/server/Namenode.h index 9577b6d..ad213e8 100644 --- a/depends/libhdfs3/src/server/Namenode.h +++ b/depends/libhdfs3/src/server/Namenode.h @@ -23,6 +23,7 @@ #define _HDFS_LIBHDFS3_SERVER_NAMENODE_H_ #include "client/FileStatus.h" +#include "client/EncryptionZoneInfo.h" #include "client/Permission.h" #include "DatanodeInfo.h" #include "Exception.h" @@ -809,8 +810,40 @@ public: * close the namenode connection. */ virtual void close() {}; -}; + /** + * Create encryption zone for the directory with specific key name + * @param path the directory path which is to be created. + * @param keyname The key name of the encryption zone + * @return return true if success. + * @throw HdfsIOException If an I/O error occurred + */ + virtual bool createEncryptionZone(const std::string & src, const std::string & keyName) = 0; + + /** + * To get encryption zone information. + * @param path the path which information is to be returned. + * @return the encryption zone information. + * @throw FileNotFoundException If file <code>src</code> does not exist + * @throw UnresolvedLinkException If <code>src</code> contains a symlink + * @throw HdfsIOException If an I/O error occurred + */ + virtual EncryptionZoneInfo getEncryptionZoneInfo(const std::string & src, bool *exist) = 0; + + /** + * Get a partial listing of the indicated encryption zones + * + * @param id the index of encryption zone + * @param ezl append the returned encryption zones. + * + * @throw AccessControlException permission denied + * @throw UnresolvedLinkException If <code>src</code> contains a symlink + * @throw HdfsIOException If an I/O error occurred + */ + virtual bool listEncryptionZones(const int64_t id, std::vector<EncryptionZoneInfo> & ezl) + /* throw (AccessControlException, UnresolvedLinkException, HdfsIOException) */ = 0; + +}; } } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/server/NamenodeImpl.cpp ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/server/NamenodeImpl.cpp b/depends/libhdfs3/src/server/NamenodeImpl.cpp index ae55e9d..958f6b1 100644 --- a/depends/libhdfs3/src/server/NamenodeImpl.cpp +++ b/depends/libhdfs3/src/server/NamenodeImpl.cpp @@ -85,6 +85,7 @@ void NamenodeImpl::create(const std::string & src, const Permission & masked, FileAlreadyExistsException, FileNotFoundException, NSQuotaExceededException, ParentNotDirectoryException, UnresolvedLinkException, HdfsIOException) */{ + try { CreateRequestProto request; CreateResponseProto response; @@ -94,6 +95,7 @@ void NamenodeImpl::create(const std::string & src, const Permission & masked, request.set_createparent(createParent); request.set_replication(replication); request.set_src(src); + request.add_cryptoprotocolversion(CryptoProtocolVersionProto::ENCRYPTION_ZONES); Build(masked, request.mutable_masked()); invoke(RpcCall(false, "create", &request, &response)); } catch (const HdfsRpcServerException & e) { @@ -792,5 +794,76 @@ void NamenodeImpl::cancelDelegationToken(const Token & token) { } } +bool NamenodeImpl::createEncryptionZone(const std::string & src, const std::string & keyName) { + try { + CreateEncryptionZoneRequestProto request; + CreateEncryptionZoneResponseProto response; + request.set_src(src); + request.set_keyname(keyName); + invoke(RpcCall(true, "createEncryptionZone",&request, &response)); + return true; + } catch (const HdfsRpcServerException & e) { + UnWrapper < HdfsIOException > unwrapper(e); + unwrapper.unwrap(__FILE__, __LINE__); + } +} + +EncryptionZoneInfo NamenodeImpl::getEncryptionZoneInfo(const std::string & src, bool *exist) +/* throw (FileNotFoundException, + UnresolvedLinkException, HdfsIOException) */{ + EncryptionZoneInfo retval; + + try { + GetEZForPathRequestProto request; + GetEZForPathResponseProto response; + request.set_src(src); + invoke(RpcCall(true, "getEZForPath", &request, &response)); + + if (response.has_zone()) { + Convert(retval, response.zone()); + retval.setPath(src.c_str()); + + if (exist) { + *exist = true; + } + + return retval; + } + + if (!exist) { + THROW(FileNotFoundException, "Path %s does not exist.", src.c_str()); + } + + *exist = false; + } catch (const HdfsRpcServerException & e) { + UnWrapper < FileNotFoundException, + UnresolvedLinkException, HdfsIOException > unwrapper(e); + unwrapper.unwrap(__FILE__, __LINE__); + } + + return retval; +} + +//Idempotent +bool NamenodeImpl::listEncryptionZones(const int64_t id, std::vector<EncryptionZoneInfo> & ezl) + /* throw (AccessControlException,FileNotFoundException, UnresolvedLinkException, HdfsIOException) */{ + try { + ListEncryptionZonesRequestProto request; + ListEncryptionZonesResponseProto response; + request.set_id(id); + invoke(RpcCall(true, "listEncryptionZones", &request, &response)); + + if (response.zones_size() != 0) { + Convert(ezl, response); + return response.hasmore(); + } + + } catch (const HdfsRpcServerException & e) { + UnWrapper < FileNotFoundException, + UnresolvedLinkException, HdfsIOException > unwrapper(e); + unwrapper.unwrap(__FILE__, __LINE__); + } +} + } } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/server/NamenodeImpl.h ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/server/NamenodeImpl.h b/depends/libhdfs3/src/server/NamenodeImpl.h index 2d915da..1abe6d9 100644 --- a/depends/libhdfs3/src/server/NamenodeImpl.h +++ b/depends/libhdfs3/src/server/NamenodeImpl.h @@ -218,6 +218,15 @@ public: void cancelDelegationToken(const Token & token) /*throws IOException*/; + bool createEncryptionZone(const std::string & src, const std::string & keyName); + /* throws HdfsIOException If an I/O error occurred */ + + EncryptionZoneInfo getEncryptionZoneInfo(const std::string & src, bool *exist); + /* throw (FileNotFoundException, UnresolvedLinkException, HdfsIOException) */ + bool listEncryptionZones(const int64_t id, std::vector<EncryptionZoneInfo> & ezl); + /* throw (AccessControlException, UnresolvedLinkException, HdfsIOException) */ + + private: void invoke(const RpcCall & call); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/server/NamenodeProxy.cpp ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/server/NamenodeProxy.cpp b/depends/libhdfs3/src/server/NamenodeProxy.cpp index 893b44f..81581ef 100644 --- a/depends/libhdfs3/src/server/NamenodeProxy.cpp +++ b/depends/libhdfs3/src/server/NamenodeProxy.cpp @@ -524,5 +524,29 @@ void NamenodeProxy::close() { namenodes.clear(); } +bool NamenodeProxy::createEncryptionZone(const std::string & src, const std::string & keyName) { + NAMENODE_HA_RETRY_BEGIN(); + return namenode->createEncryptionZone(src, keyName); + NAMENODE_HA_RETRY_END(); + assert(!"should not reach here"); + return false; +} + +EncryptionZoneInfo NamenodeProxy::getEncryptionZoneInfo(const std::string & src, bool *exist) { + NAMENODE_HA_RETRY_BEGIN(); + return namenode->getEncryptionZoneInfo(src, exist); + NAMENODE_HA_RETRY_END(); + assert(!"should not reach here"); + return EncryptionZoneInfo(); +} + +bool NamenodeProxy::listEncryptionZones(const int64_t id, std::vector<EncryptionZoneInfo> & ezl) { + NAMENODE_HA_RETRY_BEGIN(); + return namenode->listEncryptionZones(id, ezl); + NAMENODE_HA_RETRY_END(); + assert(!"should not reach here"); + return false; +} + } } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/server/NamenodeProxy.h ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/server/NamenodeProxy.h b/depends/libhdfs3/src/server/NamenodeProxy.h index 7d22713..c5b5142 100644 --- a/depends/libhdfs3/src/server/NamenodeProxy.h +++ b/depends/libhdfs3/src/server/NamenodeProxy.h @@ -139,6 +139,13 @@ public: void close(); + bool createEncryptionZone(const std::string & path, const std::string & keyName); + + EncryptionZoneInfo getEncryptionZoneInfo(const std::string & src, bool *exist); + + bool listEncryptionZones(const int64_t id, std::vector<EncryptionZoneInfo> & ezl); + + private: shared_ptr<Namenode> getActiveNamenode(uint32_t & oldValue); void failoverToNextNamenode(uint32_t oldValue); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/src/server/RpcHelper.h ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/src/server/RpcHelper.h b/depends/libhdfs3/src/server/RpcHelper.h index c6d9a4e..571ffc0 100644 --- a/depends/libhdfs3/src/server/RpcHelper.h +++ b/depends/libhdfs3/src/server/RpcHelper.h @@ -23,6 +23,7 @@ #define _HDFS_LIBHDFS3_SERVER_RPCHELPER_H_ #include "client/FileStatus.h" +#include "client/EncryptionZoneInfo.h" #include "client/Permission.h" #include "ClientDatanodeProtocol.pb.h" #include "ClientNamenodeProtocol.pb.h" @@ -182,6 +183,36 @@ static inline void Convert(const std::string & src, FileStatus & fs, fs.setSymlink(proto.symlink().c_str()); fs.setPermission(Permission(proto.permission().perm())); fs.setIsdir(proto.filetype() == HdfsFileStatusProto::IS_DIR); + + if (proto.has_fileencryptioninfo()){ + const FileEncryptionInfoProto &encrypt = proto.fileencryptioninfo(); + FileEncryptionInfo* convert = fs.getFileEncryption(); + convert->setSuite(encrypt.suite()); + convert->setCryptoProtocolVersion(encrypt.cryptoprotocolversion()); + convert->setKey(encrypt.key()); + convert->setKeyName(encrypt.keyname()); + convert->setIv(encrypt.iv()); + convert->setEzKeyVersionName(encrypt.ezkeyversionname()); + } +} + +static inline void Convert(EncryptionZoneInfo & enZone, + const EncryptionZoneProto & proto) { + enZone.setSuite(proto.suite()); + enZone.setCryptoProtocolVersion(proto.cryptoprotocolversion()); + enZone.setId(proto.id()); + enZone.setPath(proto.path().c_str()); + enZone.setKeyName(proto.keyname().c_str()); +} + +static inline void Convert(std::vector<EncryptionZoneInfo> & ezl, + const ListEncryptionZonesResponseProto & proto) { + RepeatedPtrField<EncryptionZoneProto> ptrproto = proto.zones(); + for (int i=0; i < ptrproto.size(); i++) { + EncryptionZoneInfo enZoneInfo; + Convert(enZoneInfo, ptrproto.Get(i)); + ezl.push_back(enZoneInfo); + } } static inline void Convert(const std::string & src, http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/test/function/TestCInterface.cpp ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/test/function/TestCInterface.cpp b/depends/libhdfs3/test/function/TestCInterface.cpp index 2b81067..e45aaee 100644 --- a/depends/libhdfs3/test/function/TestCInterface.cpp +++ b/depends/libhdfs3/test/function/TestCInterface.cpp @@ -30,6 +30,9 @@ #include <fcntl.h> #include <stdlib.h> #include <limits> +#include <stdlib.h> +#include <sstream> +#include <iostream> using namespace Hdfs::Internal; @@ -198,6 +201,45 @@ TEST(TestCInterfaceConnect, TestConnect_Success) { ASSERT_EQ(hdfsDisconnect(fs), 0); } +TEST(TestCInterfaceTDE, DISABLED_TestCreateEnRPC_Success) { + hdfsFS fs = NULL; + hdfsEncryptionZoneInfo * enInfo = NULL; + char * uri = NULL; + setenv("LIBHDFS3_CONF", "function-test.xml", 1); + struct hdfsBuilder * bld = hdfsNewBuilder(); + assert(bld != NULL); + hdfsBuilderSetNameNode(bld, "default"); + fs = hdfsBuilderConnect(bld); + ASSERT_TRUE(fs != NULL); + system("hadoop fs -rmr /TDE"); + system("hadoop key create keytde"); + system("hadoop fs -mkdir /TDE"); + ASSERT_EQ(0, hdfsCreateEncryptionZone(fs, "/TDE", "keytde")); + enInfo = hdfsGetEZForPath(fs, "/TDE"); + ASSERT_TRUE(enInfo != NULL); + EXPECT_TRUE(enInfo->mKeyName != NULL); + std::cout << "----hdfsEncryptionZoneInfo----:" << " KeyName : " << enInfo->mKeyName << " Suite : " << enInfo->mSuite << " CryptoProtocolVersion : " << enInfo->mCryptoProtocolVersion << " Id : " << enInfo->mId << " Path : " << enInfo->mPath << std::endl; + hdfsFreeEncryptionZoneInfo(enInfo, 1); + for (int i = 0; i <= 201; i++){ + std::stringstream newstr; + newstr << i; + std::string tde = "/TDE" + newstr.str(); + std::string key = "keytde" + newstr.str(); + std::string rmTde = "hadoop fs -rmr /TDE" + newstr.str(); + std::string tdeKey = "hadoop key create keytde" + newstr.str(); + std::string mkTde = "hadoop fs -mkdir /TDE" + newstr.str(); + system(rmTde.c_str()); + system(tdeKey.c_str()); + system(mkTde.c_str()); + ASSERT_EQ(0, hdfsCreateEncryptionZone(fs, tde.c_str(), key.c_str())); + } + hdfsEncryptionZoneInfo * enZoneInfos = NULL; + int num = 0; + hdfsListEncryptionZones(fs, &num); + EXPECT_EQ(num, 203); + ASSERT_EQ(hdfsDisconnect(fs), 0); + hdfsFreeBuilder(bld); +} TEST(TestErrorMessage, TestErrorMessage) { EXPECT_NO_THROW(hdfsGetLastError()); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/test/function/TestFileSystem.cpp ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/test/function/TestFileSystem.cpp b/depends/libhdfs3/test/function/TestFileSystem.cpp index 3191adb..b9d3f6b 100644 --- a/depends/libhdfs3/test/function/TestFileSystem.cpp +++ b/depends/libhdfs3/test/function/TestFileSystem.cpp @@ -147,6 +147,39 @@ TEST_F(TestFileSystem, listDirectory) { ASSERT_THROW(it.getNext(), HdfsIOException); } +TEST_F(TestFileSystem, DISABLED_listEncryptionZone) { + fs->disconnect(); + fs->connect(); + const int dirs = 201; + + for (int i = 0; i < dirs; i++){ + std::stringstream newstr; + newstr << i; + std::string tde = "/TDE" + newstr.str(); + std::string key = "keytde" + newstr.str(); + std::string rmTde = "hadoop fs -rmr /TDE" + newstr.str(); + std::string tdeKey = "hadoop key create keytde" + newstr.str(); + std::string mkTde = "hadoop fs -mkdir /TDE" + newstr.str(); + std::string tdeZone = "hdfs crypto -createZone -keyName " + key + "-path " + tde; + system(rmTde.c_str()); + system(tdeKey.c_str()); + system(mkTde.c_str()); + system(tdeZone.c_str()); + } + + EncryptionZoneIterator it; + EXPECT_NO_THROW(it = fs->listEncryptionZone()); + int count = 0; + + while (it.hasNext()) { + count ++; + it.getNext(); + } + + ASSERT_EQ(dirs, count); + ASSERT_THROW(it.getNext(), HdfsIOException); +} + TEST_F(TestFileSystem, setOwner) { fs->disconnect(); ASSERT_THROW(fs->setOwner(BASE_DIR, "setOwner", ""), HdfsIOException); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/517e6d26/depends/libhdfs3/test/function/TestOutputStream.cpp ---------------------------------------------------------------------- diff --git a/depends/libhdfs3/test/function/TestOutputStream.cpp b/depends/libhdfs3/test/function/TestOutputStream.cpp index faf84e7..e57df34 100644 --- a/depends/libhdfs3/test/function/TestOutputStream.cpp +++ b/depends/libhdfs3/test/function/TestOutputStream.cpp @@ -517,6 +517,18 @@ TEST_F(TestOutputStream, TestOpenFileForWrite) { } +TEST_F(TestOutputStream, DISABLE_TestOpenFileForWriteTDE){ + conf.set("output.default.packetsize", 1024); + fs = new FileSystem(conf); + fs->connect(); + fs->mkdirs("/testTDE", 0755); + system("hadoop key create amy"); + system("hdfs crypto -createZone -keyName amy -path /testTDE"); + OutputStream other; + ASSERT_NO_THROW(other.open(*fs, "/testTDE/amy", Create | Append)); + other.close(); + fs->disconnect(); +} TEST_F(TestOutputStream, TestWriteChunkPacket) { //test create a file and write a block