http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h new file mode 100644 index 0000000..2fdeec9 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/hdfspp.h @@ -0,0 +1,492 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBHDFSPP_HDFSPP_H_ +#define LIBHDFSPP_HDFSPP_H_ + +#include "hdfspp/options.h" +#include "hdfspp/status.h" +#include "hdfspp/events.h" +#include "hdfspp/block_location.h" +#include "hdfspp/statinfo.h" +#include "hdfspp/fsinfo.h" +#include "hdfspp/content_summary.h" +#include "hdfspp/uri.h" +#include "hdfspp/config_parser.h" +#include "hdfspp/locks.h" + +#include <functional> +#include <memory> +#include <set> +#include <iostream> + +namespace hdfs { + +/** + * An IoService manages a queue of asynchronous tasks. All libhdfs++ + * operations are filed against a particular IoService. + * + * When an operation is queued into an IoService, the IoService will + * run the callback handler associated with the operation. Note that + * the IoService must be stopped before destructing the objects that + * post the operations. + * + * From an implementation point of view the hdfs::IoService provides + * a thin wrapper over an asio::io_service object so that additional + * instrumentation and functionality can be added. + **/ + +class IoService : public std::enable_shared_from_this<IoService> +{ + public: + static IoService *New(); + static std::shared_ptr<IoService> MakeShared(); + virtual ~IoService(); + + /** + * Start up as many threads as there are logical processors. + * Return number of threads created. + **/ + virtual unsigned int InitDefaultWorkers() = 0; + + /** + * Initialize with thread_count handler threads. + * If thread count is less than one print a log message and default to one thread. + * Return number of threads created. + **/ + virtual unsigned int InitWorkers(unsigned int thread_count) = 0; + + /** + * Place an item on the execution queue. Will be invoked from outside of the calling context. + **/ + virtual void PostTask(std::function<void(void)>& asyncTask) = 0; + + /** + * Run the asynchronous tasks associated with this IoService. + **/ + virtual void Run() = 0; + /** + * Stop running asynchronous tasks associated with this IoService. + * All worker threads will return as soon as they finish executing their current task. + **/ + virtual void Stop() = 0; +}; + +/** + * A node exclusion rule provides a simple way of testing if the + * client should attempt to connect to a node based on the node's + * UUID. The FileSystem and FileHandle use the BadDataNodeTracker + * by default. AsyncPreadSome takes an optional NodeExclusionRule + * that will override the BadDataNodeTracker. + **/ +class NodeExclusionRule { + public: + virtual ~NodeExclusionRule(); + virtual bool IsBadNode(const std::string &node_uuid) = 0; +}; + +/** + * Applications opens a FileHandle to read files in HDFS. + **/ +class FileHandle { +public: + /** + * Read data from a specific position. The current implementation + * stops at the block boundary. + * + * @param buf the pointer to the buffer + * @param buf_size the size of the buffer + * @param offset the offset the file + * + * The handler returns the datanode that serves the block and the number of + * bytes has read. Status::InvalidOffset is returned when trying to begin + * a read past the EOF. + **/ + virtual void + PositionRead(void *buf, size_t buf_size, uint64_t offset, + const std::function<void(const Status &, size_t)> &handler) = 0; + virtual Status PositionRead(void *buf, size_t buf_size, off_t offset, size_t *bytes_read) = 0; + virtual Status Read(void *buf, size_t buf_size, size_t *bytes_read) = 0; + virtual Status Seek(off_t *offset, std::ios_base::seekdir whence) = 0; + + /** + * Cancel outstanding file operations. This is not reversable, once called + * the handle should be disposed of. + **/ + virtual void CancelOperations(void) = 0; + + /** + * Determine if a datanode should be excluded from future operations + * based on the return Status. + * + * @param status the Status object returned by InputStream::PositionRead + * @return true if the status indicates a failure that is not recoverable + * by the client and false otherwise. + **/ + static bool ShouldExclude(const Status &status); + + + /** + * Sets an event callback for file-level event notifications (such as connecting + * to the DataNode, communications errors, etc.) + * + * Many events are defined in hdfspp/events.h; the consumer should also expect + * to be called with many private events, which can be ignored. + * + * @param callback The function to call when a reporting event occurs. + */ + virtual void SetFileEventCallback(file_event_callback callback) = 0; + + /* how many bytes have been successfully read */ + virtual uint64_t get_bytes_read() = 0; + + /* resets the number of bytes read to zero */ + virtual void clear_bytes_read() = 0; + + virtual ~FileHandle(); +}; + +/** + * FileSystem implements APIs to interact with HDFS. + **/ +class FileSystem { + public: + //Returns the default maximum depth for recursive Find tool + static uint32_t GetDefaultFindMaxDepth(); + + //Returns the default permission mask + static uint16_t GetDefaultPermissionMask(); + + //Checks if the given permission mask is valid + static Status CheckValidPermissionMask(uint16_t permissions); + + //Checks if replication value is valid + static Status CheckValidReplication(uint16_t replication); + + /** + * Create a new instance of the FileSystem object. The call + * initializes the RPC connections to the NameNode and returns an + * FileSystem object. + * + * Note: The FileSystem takes ownership of the IoService passed in the + * constructor. The FileSystem destructor will call delete on it. + * + * If user_name is blank, the current user will be used for a default. + **/ + static FileSystem *New( + IoService *&io_service, const std::string &user_name, const Options &options); + + /** + * Works the same as the other FileSystem::New but takes a copy of an existing IoService. + * The shared IoService is expected to already have worker threads initialized. + **/ + static FileSystem *New( + std::shared_ptr<IoService>, const std::string &user_name, const Options &options); + + /** + * Returns a new instance with default user and option, with the default IOService. + **/ + static FileSystem *New(); + + /** + * Callback type for async FileSystem::Connect calls. + * Provides the result status and instance pointer to the handler. + **/ + typedef std::function<void(const Status& result_status, FileSystem *created_fs)> AsyncConnectCallback; + + /** + * Connect directly to the specified namenode using the host and port (service). + **/ + virtual void Connect(const std::string &server, const std::string &service, + const AsyncConnectCallback &handler) = 0; + + /* Synchronous call of Connect */ + virtual Status Connect(const std::string &server, const std::string &service) = 0; + + + /** + * Connects to the hdfs instance indicated by the defaultFs value of the + * Options structure. + * + * If no defaultFs is defined, returns an error. + */ + virtual void ConnectToDefaultFs( + const AsyncConnectCallback& handler) = 0; + virtual Status ConnectToDefaultFs() = 0; + + /** + * Cancels any attempts to connect to the HDFS cluster. + * FileSystem is expected to be destroyed after invoking this. + */ + virtual bool CancelPendingConnect() = 0; + + /** + * Open a file on HDFS. The call issues an RPC to the NameNode to + * gather the locations of all blocks in the file and to return a + * new instance of the @ref InputStream object. + **/ + virtual void + Open(const std::string &path, + const std::function<void(const Status &, FileHandle *)> &handler) = 0; + virtual Status Open(const std::string &path, FileHandle **handle) = 0; + + /** + * Get the block size for the given file. + * @param path The path to the file + */ + virtual void GetPreferredBlockSize(const std::string &path, + const std::function<void(const Status &, const uint64_t &)> &handler) = 0; + virtual Status GetPreferredBlockSize(const std::string &path, uint64_t & block_size) = 0; + + /** + * Set replication for an existing file. + * <p> + * The NameNode sets replication to the new value and returns. + * The actual block replication is not expected to be performed during + * this method call. The blocks will be populated or removed in the + * background as the result of the routine block maintenance procedures. + * + * @param path file name + * @param replication new replication + */ + virtual void SetReplication(const std::string & path, int16_t replication, std::function<void(const Status &)> handler) = 0; + virtual Status SetReplication(const std::string & path, int16_t replication) = 0; + + /** + * Sets the modification and access time of the file to the specified time. + * @param path The string representation of the path + * @param mtime The number of milliseconds since Jan 1, 1970. + * Setting mtime to -1 means that modification time should not + * be set by this call. + * @param atime The number of milliseconds since Jan 1, 1970. + * Setting atime to -1 means that access time should not be set + * by this call. + */ + virtual void SetTimes(const std::string & path, uint64_t mtime, uint64_t atime, std::function<void(const Status &)> handler) = 0; + virtual Status SetTimes(const std::string & path, uint64_t mtime, uint64_t atime) = 0; + + /** + * Returns metadata about the file if the file/directory exists. + **/ + virtual void + GetFileInfo(const std::string &path, + const std::function<void(const Status &, const StatInfo &)> &handler) = 0; + virtual Status GetFileInfo(const std::string &path, StatInfo & stat_info) = 0; + + /** + * Returns the number of directories, files and bytes under the given path + **/ + virtual void + GetContentSummary(const std::string &path, + const std::function<void(const Status &, const ContentSummary &)> &handler) = 0; + virtual Status GetContentSummary(const std::string &path, ContentSummary & stat_info) = 0; + + /** + * Retrieves the file system information as a whole, such as the total raw size of all files in the filesystem + * and the raw capacity of the filesystem + * + * FsInfo struct is populated by GetFsStats + **/ + virtual void GetFsStats( + const std::function<void(const Status &, const FsInfo &)> &handler) = 0; + virtual Status GetFsStats(FsInfo & fs_info) = 0; + + /** + * Retrieves the files contained in a directory and returns the metadata + * for each of them. + * + * The asynchronous method will return batches of files; the consumer must + * return true if they want more files to be delivered. The final bool + * parameter in the callback will be set to false if this is the final + * batch of files. + * + * The synchronous method will return all files in the directory. + * + * Path must be an absolute path in the hdfs filesytem (e.g. /tmp/foo/bar) + **/ + virtual void + GetListing(const std::string &path, + const std::function<bool(const Status &, const std::vector<StatInfo> &, bool)> &handler) = 0; + virtual Status GetListing(const std::string &path, std::vector<StatInfo> * stat_infos) = 0; + + /** + * Returns the locations of all known blocks for the indicated file (or part of it), or an error + * if the information could not be found + */ + virtual void GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length, + const std::function<void(const Status &, std::shared_ptr<FileBlockLocation> locations)> ) = 0; + virtual Status GetBlockLocations(const std::string & path, uint64_t offset, uint64_t length, + std::shared_ptr<FileBlockLocation> * locations) = 0; + + /** + * Creates a new directory + * + * @param path Path to the directory to be created (must be non-empty) + * @param permissions Permissions for the new directory (negative value for the default permissions) + * @param createparent Create parent directories if they do not exist (may not be empty) + */ + virtual void Mkdirs(const std::string & path, uint16_t permissions, bool createparent, + std::function<void(const Status &)> handler) = 0; + virtual Status Mkdirs(const std::string & path, uint16_t permissions, bool createparent) = 0; + + /** + * Delete the given file or directory from the file system. + * <p> + * same as delete but provides a way to avoid accidentally + * deleting non empty directories programmatically. + * @param path existing name (must be non-empty) + * @param recursive if true deletes a non empty directory recursively + */ + virtual void Delete(const std::string &path, bool recursive, + const std::function<void(const Status &)> &handler) = 0; + virtual Status Delete(const std::string &path, bool recursive) = 0; + + /** + * Rename - Rename file. + * @param oldPath The path of the source file. (must be non-empty) + * @param newPath The path of the destination file. (must be non-empty) + */ + virtual void Rename(const std::string &oldPath, const std::string &newPath, + const std::function<void(const Status &)> &handler) = 0; + virtual Status Rename(const std::string &oldPath, const std::string &newPath) = 0; + + /** + * Set permissions for an existing file/directory. + * + * @param path the path to the file or directory + * @param permissions the bitmask to set it to (should be between 0 and 01777) + */ + virtual void SetPermission(const std::string & path, uint16_t permissions, + const std::function<void(const Status &)> &handler) = 0; + virtual Status SetPermission(const std::string & path, uint16_t permissions) = 0; + + /** + * Set Owner of a path (i.e. a file or a directory). + * The parameters username and groupname can be empty. + * @param path file path + * @param username If it is empty, the original username remains unchanged. + * @param groupname If it is empty, the original groupname remains unchanged. + */ + virtual void SetOwner(const std::string & path, const std::string & username, + const std::string & groupname, const std::function<void(const Status &)> &handler) = 0; + virtual Status SetOwner(const std::string & path, + const std::string & username, const std::string & groupname) = 0; + + /** + * Finds all files matching the specified name recursively starting from the + * specified directory. Returns metadata for each of them. + * + * Example: Find("/dir?/tree*", "some?file*name") + * + * @param path Absolute path at which to begin search, can have wild cards (must be non-blank) + * @param name Name to find, can also have wild cards (must be non-blank) + * + * The asynchronous method will return batches of files; the consumer must + * return true if they want more files to be delivered. The final bool + * parameter in the callback will be set to false if this is the final + * batch of files. + * + * The synchronous method will return matching files. + **/ + virtual void + Find(const std::string &path, const std::string &name, const uint32_t maxdepth, + const std::function<bool(const Status &, const std::vector<StatInfo> & , bool)> &handler) = 0; + virtual Status Find(const std::string &path, const std::string &name, + const uint32_t maxdepth, std::vector<StatInfo> * stat_infos) = 0; + + + /***************************************************************************** + * FILE SYSTEM SNAPSHOT FUNCTIONS + ****************************************************************************/ + + /** + * Creates a snapshot of a snapshottable directory specified by path + * + * @param path Path to the directory to be snapshotted (must be non-empty) + * @param name Name to be given to the created snapshot (may be empty) + **/ + virtual void CreateSnapshot(const std::string &path, const std::string &name, + const std::function<void(const Status &)> &handler) = 0; + virtual Status CreateSnapshot(const std::string &path, + const std::string &name) = 0; + + /** + * Deletes the directory snapshot specified by path and name + * + * @param path Path to the snapshotted directory (must be non-empty) + * @param name Name of the snapshot to be deleted (must be non-empty) + **/ + virtual void DeleteSnapshot(const std::string &path, const std::string &name, + const std::function<void(const Status &)> &handler) = 0; + virtual Status DeleteSnapshot(const std::string &path, + const std::string &name) = 0; + + /** + * Renames the directory snapshot specified by path from old_name to new_name + * + * @param path Path to the snapshotted directory (must be non-blank) + * @param old_name Current name of the snapshot (must be non-blank) + * @param new_name New name of the snapshot (must be non-blank) + **/ + virtual void RenameSnapshot(const std::string &path, const std::string &old_name, + const std::string &new_name, const std::function<void(const Status &)> &handler) = 0; + virtual Status RenameSnapshot(const std::string &path, const std::string &old_name, + const std::string &new_name) = 0; + + /** + * Allows snapshots to be made on the specified directory + * + * @param path Path to the directory to be made snapshottable (must be non-empty) + **/ + virtual void AllowSnapshot(const std::string &path, + const std::function<void(const Status &)> &handler) = 0; + virtual Status AllowSnapshot(const std::string &path) = 0; + + /** + * Disallows snapshots to be made on the specified directory + * + * @param path Path to the directory to be made non-snapshottable (must be non-empty) + **/ + virtual void DisallowSnapshot(const std::string &path, + const std::function<void(const Status &)> &handler) = 0; + virtual Status DisallowSnapshot(const std::string &path) = 0; + + /** + * Note that it is an error to destroy the filesystem from within a filesystem + * callback. It will lead to a deadlock and the termination of the process. + */ + virtual ~FileSystem(); + + + /** + * Sets an event callback for fs-level event notifications (such as connecting + * to the NameNode, communications errors with the NN, etc.) + * + * Many events are defined in hdfspp/events.h; the consumer should also expect + * to be called with many private events, which can be ignored. + * + * @param callback The function to call when a reporting event occurs. + */ + virtual void SetFsEventCallback(fs_event_callback callback) = 0; + + virtual Options get_options() = 0; + + virtual std::string get_cluster_name() = 0; +}; +} + +#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/locks.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/locks.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/locks.h new file mode 100644 index 0000000..3dfeab4 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/locks.h @@ -0,0 +1,110 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COMMON_HDFS_LOCKS_H_ +#define COMMON_HDFS_LOCKS_H_ + +#include <stdexcept> +#include <string> +#include <atomic> +#include <mutex> +#include <memory> + +namespace hdfs +{ + +// +// Thrown by LockGuard to indicate that it was unable to acquire a mutex +// what_str should contain info about what caused the failure +// +class LockFailure : public std::runtime_error { + public: + LockFailure(const char *what_str) : std::runtime_error(what_str) {}; + LockFailure(const std::string& what_str) : std::runtime_error(what_str) {}; +}; + +// +// A pluggable mutex type to allow client code to share mutexes it may +// already use to protect certain system resources. Certain shared +// libraries have some procedures that aren't always implemented in a thread +// safe manner. If libhdfs++ and the code linking it depend on the same +// library this provides a mechanism to coordinate safe access. +// +// Interface provided is intended to be similar to std::mutex. If the lock +// can't be aquired it may throw LockFailure from the lock method. If lock +// does fail libhdfs++ is expected fail as cleanly as possible e.g. +// FileSystem::Mkdirs might return a MutexError but a subsequent call may be +// successful. +// +class Mutex { + public: + virtual ~Mutex() {}; + virtual void lock() = 0; + virtual void unlock() = 0; + virtual std::string str() = 0; +}; + +// +// LockGuard works in a similar manner to std::lock_guard: it locks the mutex +// in the constructor and unlocks it in the destructor. +// Failure to acquire the mutex in the constructor will result in throwing a +// LockFailure exception. +// +class LockGuard { + public: + LockGuard(Mutex *m); + ~LockGuard(); + private: + Mutex *_mtx; +}; + +// +// Manage instances of hdfs::Mutex that are intended to be global to the +// process. +// +// LockManager's InitLocks method provides a mechanism for the calling +// application to share its own implementations of hdfs::Mutex. It must be +// called prior to instantiating any FileSystem objects and can only be +// called once. If a lock is not provided a default mutex type wrapping +// std::mutex is used as a default. +// + +class LockManager { + public: + // Initializes with a default set of C++11 style mutexes + static bool InitLocks(Mutex *gssapi); + static Mutex *getGssapiMutex(); + + // Tests only, implementation may no-op on release builds. + // Reset _finalized to false and set all Mutex* members to default values. + static void TEST_reset_manager(); + static Mutex *TEST_get_default_mutex(); + private: + // Used only in tests. + static Mutex *TEST_default_mutex; + // Use to synchronize calls into GSSAPI/Kerberos libs + static Mutex *gssapiMtx; + + // Prevent InitLocks from being called more than once + // Allows all locks to be set a single time atomically + static std::mutex _state_lock; + static bool _finalized; +}; + +} // end namespace hdfs +#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/log.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/log.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/log.h new file mode 100644 index 0000000..0371951 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/log.h @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LIBHDFSPP_HDFS_LOG +#define LIBHDFSPP_HDFS_LOG + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * Things that are part of the public API but are specific to logging live here. + * Added to avoid including the whole public API into the implementation of the logger. + **/ + +/* logging levels, compatible with enum in lib/common/logging.cc */ +#define HDFSPP_LOG_LEVEL_TRACE 0 +#define HDFSPP_LOG_LEVEL_DEBUG 1 +#define HDFSPP_LOG_LEVEL_INFO 2 +#define HDFSPP_LOG_LEVEL_WARN 3 +#define HDFSPP_LOG_LEVEL_ERROR 4 + +/* components emitting messages, compatible with enum lib/common/logging.cc */ +#define HDFSPP_LOG_COMPONENT_UNKNOWN 1 << 0 +#define HDFSPP_LOG_COMPONENT_RPC 1 << 1 +#define HDFSPP_LOG_COMPONENT_BLOCKREADER 1 << 2 +#define HDFSPP_LOG_COMPONENT_FILEHANDLE 1 << 3 +#define HDFSPP_LOG_COMPONENT_FILESYSTEM 1 << 4 + +/** + * POD struct for C to consume (C++ interface gets to take advantage of RAII) + **/ +typedef struct { + const char *msg; + int level; + int component; + const char *file_name; + int file_line; +} LogData; + +#ifdef __cplusplus +} // end extern C +#endif + +#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h new file mode 100644 index 0000000..8d29a40 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/options.h @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBHDFSPP_OPTIONS_H_ +#define LIBHDFSPP_OPTIONS_H_ + +#include "hdfspp/uri.h" + +#include <string> +#include <vector> +#include <map> + +namespace hdfs { + + +struct NamenodeInfo { + NamenodeInfo(const std::string &nameservice_, const std::string &nodename_, const URI &uri_) : + nameservice(nameservice_), name(nodename_), uri(uri_) {} + NamenodeInfo(){} + //nameservice this belongs to + std::string nameservice; + //node name + std::string name; + //host:port + URI uri; + + //get server hostname and port (aka service) + std::string get_host() const; + std::string get_port() const; +}; + +/** + * Options to control the behavior of the libhdfspp library. + **/ +struct Options { + /** + * Time out of RPC requests in milliseconds. + * Default: 30000 + **/ + int rpc_timeout; + static const int kDefaultRpcTimeout = 30000; + + /** + * Time to wait for an RPC connection before failing + * Default: 30000 + **/ + int rpc_connect_timeout; + static const int kDefaultRpcConnectTimeout = 30000; + + /** + * Maximum number of retries for RPC operations + **/ + int max_rpc_retries; + static const int kNoRetry = 0; + static const int kDefaultMaxRpcRetries = kNoRetry; + + /** + * Number of ms to wait between retry of RPC operations + **/ + int rpc_retry_delay_ms; + static const int kDefaultRpcRetryDelayMs = 10000; + + /** + * Exclusion time for failed datanodes in milliseconds. + * Default: 60000 + **/ + unsigned int host_exclusion_duration; + static const unsigned int kDefaultHostExclusionDuration = 600000; + + /** + * URI to connect to if no host:port are specified in connect + */ + URI defaultFS; + + /** + * Namenodes used to provide HA for this cluster if applicable + **/ + std::map<std::string, std::vector<NamenodeInfo>> services; + + + /** + * Client failover attempts before failover gives up + **/ + int failover_max_retries; + static const unsigned int kDefaultFailoverMaxRetries = 4; + + /** + * Client failover attempts before failover gives up if server + * connection is timing out. + **/ + int failover_connection_max_retries; + static const unsigned int kDefaultFailoverConnectionMaxRetries = 0; + + /* + * Which form of authentication to use with the server + * Default: simple + */ + enum Authentication { + kSimple, + kKerberos + }; + Authentication authentication; + static const Authentication kDefaultAuthentication = kSimple; + + /** + * Block size in bytes. + * Default: 128 * 1024 * 1024 = 134217728 + **/ + long block_size; + static const long kDefaultBlockSize = 128*1024*1024; + + /** + * Asio worker thread count + * default: -1, indicates number of hardware threads + **/ + int io_threads_; + static const int kDefaultIoThreads = -1; + + Options(); +}; +} +#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h new file mode 100644 index 0000000..1da1f18 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/statinfo.h @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef HDFSPP_STATINFO_H_ +#define HDFSPP_STATINFO_H_ + +#include <string> + +namespace hdfs { + +/** + * Information that is assumed to be unchanging about a file for the duration of + * the operations. + */ +struct StatInfo { + enum FileType { + IS_DIR = 1, + IS_FILE = 2, + IS_SYMLINK = 3 + }; + + int file_type; + std::string path; + std::string full_path; + uint64_t length; + uint64_t permissions; //Octal number as in POSIX permissions; e.g. 0777 + std::string owner; + std::string group; + uint64_t modification_time; + uint64_t access_time; + std::string symlink; + uint32_t block_replication; + uint64_t blocksize; + uint64_t fileid; + uint64_t children_num; + + StatInfo(); + + //Converts StatInfo object to std::string (hdfs_ls format) + std::string str() const; +}; + +} + +#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h new file mode 100644 index 0000000..718e530 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/status.h @@ -0,0 +1,111 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef LIBHDFSPP_STATUS_H_ +#define LIBHDFSPP_STATUS_H_ + +#include <string> +#include <system_error> + +namespace hdfs { + +class Status { + public: + // Create a success status. + Status() : code_(0) {} + + // Note: Avoid calling the Status constructors directly, call the factory methods instead + + // Used for common status types + Status(int code, const char *msg); + // Used for server side exceptions reported through RpcResponseProto and similar + Status(int code, const char *exception_class, const char *exception_details); + + // Factory methods + static Status OK(); + static Status InvalidArgument(const char *msg); + static Status ResourceUnavailable(const char *msg); + static Status Unimplemented(); + static Status Exception(const char *exception_class_name, const char *exception_details); + static Status Error(const char *error_message); + static Status AuthenticationFailed(); + static Status AuthenticationFailed(const char *msg); + static Status AuthorizationFailed(); + static Status AuthorizationFailed(const char *msg); + static Status Canceled(); + static Status PathNotFound(const char *msg); + static Status InvalidOffset(const char *msg); + static Status PathIsNotDirectory(const char *msg); + static Status MutexError(const char *msg); + + // success + bool ok() const { return code_ == 0; } + + bool is_invalid_offset() const { return code_ == kInvalidOffset; } + + // contains ENOENT error + bool pathNotFound() const { return code_ == kPathNotFound; } + + // Returns the string "OK" for success. + std::string ToString() const; + + // get error code + int code() const { return code_; } + + // if retry can possibly recover an error + bool notWorthRetry() const; + + enum Code { + kOk = 0, + kInvalidArgument = static_cast<unsigned>(std::errc::invalid_argument), + kResourceUnavailable = static_cast<unsigned>(std::errc::resource_unavailable_try_again), + kUnimplemented = static_cast<unsigned>(std::errc::function_not_supported), + kOperationCanceled = static_cast<unsigned>(std::errc::operation_canceled), + kPermissionDenied = static_cast<unsigned>(std::errc::permission_denied), + kPathNotFound = static_cast<unsigned>(std::errc::no_such_file_or_directory), + kNotADirectory = static_cast<unsigned>(std::errc::not_a_directory), + kFileAlreadyExists = static_cast<unsigned>(std::errc::file_exists), + kPathIsNotEmptyDirectory = static_cast<unsigned>(std::errc::directory_not_empty), + kBusy = static_cast<unsigned>(std::errc::device_or_resource_busy), + + // non-errc codes start at 256 + kException = 256, + kAuthenticationFailed = 257, + kAccessControlException = 258, + kStandbyException = 259, + kSnapshotProtocolException = 260, + kInvalidOffset = 261, + }; + + std::string get_exception_class_str() const { + return exception_class_; + } + + int get_server_exception_type() const { + return code_; + } + + private: + int code_; + std::string msg_; + + std::string exception_class_; +}; + +} + +#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h new file mode 100644 index 0000000..d8574d1 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/include/hdfspp/uri.h @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef COMMON_HDFS_URI_H_ +#define COMMON_HDFS_URI_H_ + +#include <iostream> +#include <string> +#include <vector> +#include <stdexcept> + +namespace hdfs +{ + +class uri_parse_error : public std::invalid_argument { + public: + uri_parse_error(const char *what_str) : std::invalid_argument(what_str) {} + uri_parse_error(const std::string& what_str) : std::invalid_argument(what_str) {} +}; + +class URI { +public: + // Parse a string into a URI. Throw a hdfs::uri_parse_error if URI is malformed. + static URI parse_from_string(const std::string &str); + + // URI encode/decode strings + static std::string encode (const std::string &input); + static std::string decode (const std::string &input); + + URI(); + + std::string get_scheme(bool encoded_output=false) const; + + void set_scheme(const std::string &s, bool encoded_input=false); + + // empty if none. + std::string get_host(bool encoded_output=false) const; + + void set_host(const std::string& h, bool encoded_input=false); + + // true if port has been set + bool has_port() const; + + // undefined if port hasn't been set + uint16_t get_port() const; + + // use default if port hasn't been set + uint16_t get_port_or_default(uint16_t default_val) const; + + void set_port(uint16_t p); + + void clear_port(); + + std::string get_path(bool encoded_output=false) const; + + void set_path(const std::string &p, bool encoded_input=false); + + void add_path(const std::string &p, bool encoded_input=false); + + std::vector<std::string> get_path_elements(bool encoded_output=false) const; + + struct Query { + Query(const std::string& key, const std::string& val); + std::string key; + std::string value; + }; + + std::string get_query(bool encoded_output=false) const; + + std::vector<Query> get_query_elements(bool encoded_output=false) const; + + // Not that set_query must always pass in encoded strings + void set_query(const std::string &q); + + // Adds a parameter onto the query; does not check if it already exists + // e.g. parseFromString("foo?bar=baz").addQuery("bing","bang") + // would leave "bar=baz&bing=bang" as the query + void add_query(const std::string &name, const std::string & value, bool encoded_input=false); + + // Removes the query part if exists + // e.g. parseFromString("foo?bar=baz&bing=bang&bar=bong").removeQueries("bar") + // would leave bing=bang as the query + void remove_query(const std::string &q_name, bool encoded_input=false); + + std::string get_fragment(bool encoded_output=false) const; + + void set_fragment(const std::string &f, bool encoded_input=false); + + std::string str(bool encoded_output=true) const; + + // Get a string with each URI field printed on a seperate line + std::string GetDebugString() const; +private: + // These are stored in encoded form + std::string scheme; + std::string user; + std::string pass; + std::string host; + std::vector<std::string> path; + std::vector<Query> queries; + std::string fragment; + // implicitly narrowed to uint16_t if positive + // -1 to indicate uninitialized + int32_t _port; + + // URI encoding helpers + static std::string from_encoded(bool encoded_output, const std::string & input); + static std::string to_encoded(bool encoded_input, const std::string & input); + + bool has_authority() const; + std::string build_authority(bool encoded_output) const; + + std::string build_path(bool encoded_output) const; + void parse_path(bool input_encoded, const std::string &input_path); +}; + +inline std::ostream& operator<<(std::ostream &out, const URI &uri) { + return out << uri.str(); +} + +} +#endif http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/CMakeLists.txt new file mode 100644 index 0000000..c851597 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/CMakeLists.txt @@ -0,0 +1,25 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_subdirectory(common) +add_subdirectory(fs) +add_subdirectory(reader) +add_subdirectory(rpc) +add_subdirectory(proto) +add_subdirectory(connection) +add_subdirectory(bindings) http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/CMakeLists.txt new file mode 100644 index 0000000..93139ce --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/CMakeLists.txt @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +add_subdirectory(c) http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt new file mode 100644 index 0000000..98b081f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-native-client/src/main/native/libhdfspp/lib/bindings/c/CMakeLists.txt @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +add_library(bindings_c_obj OBJECT hdfs.cc) +add_dependencies(bindings_c_obj fs rpc reader proto common fs rpc reader proto common) +add_library(bindings_c $<TARGET_OBJECTS:bindings_c_obj>) --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
