Revert "Revert "Merge branch 'trunk' into HDFS-7240"" After testing it was confirmed that these changes work as expected.
This reverts commit 7a542fb3270953fff039c9b1bd7ba7afa35a842c. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b78c94f4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b78c94f4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b78c94f4 Branch: refs/heads/HDFS-7240 Commit: b78c94f44c454f35eeadb34d48fea649ea491de5 Parents: d0488c7 Author: Anu Engineer <aengin...@apache.org> Authored: Mon Apr 2 19:58:09 2018 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Mon Apr 2 19:58:09 2018 -0700 ---------------------------------------------------------------------- BUILDING.txt | 32 +- dev-support/bin/dist-layout-stitching | 1 + dev-support/bin/win-vs-upgrade.cmd | 39 + dev-support/docker/Dockerfile | 3 + dev-support/win-paths-eg.cmd | 49 + .../hadoop-annotations/pom.xml | 15 - hadoop-common-project/hadoop-common/pom.xml | 28 + .../org/apache/hadoop/conf/Configuration.java | 5 +- .../java/org/apache/hadoop/util/ThreadUtil.java | 26 +- .../org/apache/hadoop/util/VersionInfo.java | 3 +- .../src/main/native/native.vcxproj | 2 + .../org/apache/hadoop/crypto/OpensslCipher.c | 33 + .../apache/hadoop/conf/TestConfiguration.java | 31 + .../apache/hadoop/fs/FileContextTestHelper.java | 4 +- .../hadoop/fs/contract/ContractTestUtils.java | 32 +- .../apache/hadoop/test/GenericTestUtils.java | 7 +- .../hadoop/crypto/key/kms/server/KMSWebApp.java | 6 + .../java/org/apache/hadoop/hdfs/DFSClient.java | 2 +- .../hadoop-hdfs-native-client/pom.xml | 116 +- .../src/CMakeLists.txt | 17 +- .../main/native/libhdfs-tests/native_mini_dfs.c | 22 +- .../main/native/libhdfs-tests/native_mini_dfs.h | 11 +- .../libhdfs-tests/test_libhdfs_mini_stress.c | 350 + .../libhdfs-tests/test_libhdfs_threaded.c | 70 +- .../native/libhdfspp/CMake/FindCyrusSASL.cmake | 49 + .../main/native/libhdfspp/CMake/FindGSasl.cmake | 44 + .../src/main/native/libhdfspp/CMakeLists.txt | 297 + .../src/main/native/libhdfspp/CONTRIBUTING.md | 161 + .../src/main/native/libhdfspp/doc/Doxyfile.in | 35 + .../src/main/native/libhdfspp/doc/mainpage.dox | 26 + .../native/libhdfspp/examples/CMakeLists.txt | 20 + .../native/libhdfspp/examples/c/CMakeLists.txt | 20 + .../libhdfspp/examples/c/cat/CMakeLists.txt | 27 + .../main/native/libhdfspp/examples/c/cat/cat.c | 121 + .../examples/c/connect_cancel/CMakeLists.txt | 27 + .../examples/c/connect_cancel/connect_cancel.c | 107 + .../native/libhdfspp/examples/cc/CMakeLists.txt | 24 + .../libhdfspp/examples/cc/cat/CMakeLists.txt | 27 + .../native/libhdfspp/examples/cc/cat/cat.cc | 89 + .../examples/cc/connect_cancel/CMakeLists.txt | 27 + .../cc/connect_cancel/connect_cancel.cc | 154 + .../libhdfspp/examples/cc/find/CMakeLists.txt | 27 + .../native/libhdfspp/examples/cc/find/find.cc | 140 + .../examples/cc/gendirs/CMakeLists.txt | 27 + .../libhdfspp/examples/cc/gendirs/gendirs.cc | 122 + .../libhdfspp/include/hdfspp/block_location.h | 177 + .../libhdfspp/include/hdfspp/config_parser.h | 68 + .../libhdfspp/include/hdfspp/content_summary.h | 48 + .../native/libhdfspp/include/hdfspp/events.h | 141 + .../native/libhdfspp/include/hdfspp/fsinfo.h | 48 + .../native/libhdfspp/include/hdfspp/hdfs_ext.h | 394 + .../native/libhdfspp/include/hdfspp/hdfspp.h | 492 + .../native/libhdfspp/include/hdfspp/locks.h | 110 + .../main/native/libhdfspp/include/hdfspp/log.h | 60 + .../native/libhdfspp/include/hdfspp/options.h | 136 + .../native/libhdfspp/include/hdfspp/statinfo.h | 59 + .../native/libhdfspp/include/hdfspp/status.h | 111 + .../main/native/libhdfspp/include/hdfspp/uri.h | 137 + .../main/native/libhdfspp/lib/CMakeLists.txt | 25 + .../libhdfspp/lib/bindings/CMakeLists.txt | 19 + .../libhdfspp/lib/bindings/c/CMakeLists.txt | 21 + .../native/libhdfspp/lib/bindings/c/hdfs.cc | 2007 ++ .../native/libhdfspp/lib/common/CMakeLists.txt | 24 + .../native/libhdfspp/lib/common/async_stream.h | 49 + .../native/libhdfspp/lib/common/auth_info.cc | 18 + .../native/libhdfspp/lib/common/auth_info.h | 90 + .../libhdfspp/lib/common/cancel_tracker.cc | 37 + .../libhdfspp/lib/common/cancel_tracker.h | 40 + .../libhdfspp/lib/common/config_parser.cc | 219 + .../libhdfspp/lib/common/configuration.cc | 169 + .../native/libhdfspp/lib/common/configuration.h | 108 + .../lib/common/configuration_loader.cc | 328 + .../libhdfspp/lib/common/configuration_loader.h | 138 + .../lib/common/configuration_loader_impl.h | 122 + .../libhdfspp/lib/common/content_summary.cc | 55 + .../libhdfspp/lib/common/continuation/asio.h | 65 + .../lib/common/continuation/continuation.h | 137 + .../lib/common/continuation/protobuf.h | 129 + .../main/native/libhdfspp/lib/common/fsinfo.cc | 61 + .../libhdfspp/lib/common/hdfs_configuration.cc | 210 + .../libhdfspp/lib/common/hdfs_configuration.h | 70 + .../libhdfspp/lib/common/hdfs_ioservice.cc | 146 + .../libhdfspp/lib/common/hdfs_ioservice.h | 79 + .../libhdfspp/lib/common/libhdfs_events_impl.cc | 89 + .../libhdfspp/lib/common/libhdfs_events_impl.h | 59 + .../main/native/libhdfspp/lib/common/locks.cc | 100 + .../main/native/libhdfspp/lib/common/logging.cc | 227 + .../main/native/libhdfspp/lib/common/logging.h | 217 + .../libhdfspp/lib/common/namenode_info.cc | 178 + .../native/libhdfspp/lib/common/namenode_info.h | 49 + .../native/libhdfspp/lib/common/new_delete.h | 52 + .../libhdfspp/lib/common/optional_wrapper.h | 43 + .../main/native/libhdfspp/lib/common/options.cc | 61 + .../native/libhdfspp/lib/common/retry_policy.cc | 87 + .../native/libhdfspp/lib/common/retry_policy.h | 160 + .../libhdfspp/lib/common/sasl_authenticator.h | 66 + .../libhdfspp/lib/common/sasl_digest_md5.cc | 240 + .../native/libhdfspp/lib/common/statinfo.cc | 74 + .../main/native/libhdfspp/lib/common/status.cc | 192 + .../src/main/native/libhdfspp/lib/common/uri.cc | 454 + .../main/native/libhdfspp/lib/common/util.cc | 170 + .../src/main/native/libhdfspp/lib/common/util.h | 185 + .../main/native/libhdfspp/lib/common/util_c.h | 31 + .../libhdfspp/lib/connection/CMakeLists.txt | 21 + .../lib/connection/datanodeconnection.cc | 72 + .../lib/connection/datanodeconnection.h | 97 + .../main/native/libhdfspp/lib/fs/CMakeLists.txt | 21 + .../libhdfspp/lib/fs/bad_datanode_tracker.cc | 71 + .../libhdfspp/lib/fs/bad_datanode_tracker.h | 75 + .../main/native/libhdfspp/lib/fs/filehandle.cc | 370 + .../main/native/libhdfspp/lib/fs/filehandle.h | 147 + .../main/native/libhdfspp/lib/fs/filesystem.cc | 859 + .../main/native/libhdfspp/lib/fs/filesystem.h | 291 + .../native/libhdfspp/lib/fs/filesystem_sync.cc | 607 + .../libhdfspp/lib/fs/namenode_operations.cc | 727 + .../libhdfspp/lib/fs/namenode_operations.h | 136 + .../native/libhdfspp/lib/proto/CMakeLists.txt | 87 + .../libhdfspp/lib/proto/protoc_gen_hrpc.cc | 95 + .../native/libhdfspp/lib/reader/CMakeLists.txt | 21 + .../native/libhdfspp/lib/reader/block_reader.cc | 571 + .../native/libhdfspp/lib/reader/block_reader.h | 159 + .../native/libhdfspp/lib/reader/datatransfer.cc | 52 + .../native/libhdfspp/lib/reader/datatransfer.h | 75 + .../libhdfspp/lib/reader/datatransfer_impl.h | 136 + .../main/native/libhdfspp/lib/reader/fileinfo.h | 38 + .../native/libhdfspp/lib/reader/readergroup.cc | 55 + .../native/libhdfspp/lib/reader/readergroup.h | 52 + .../native/libhdfspp/lib/rpc/CMakeLists.txt | 31 + .../libhdfspp/lib/rpc/cyrus_sasl_engine.cc | 469 + .../libhdfspp/lib/rpc/cyrus_sasl_engine.h | 50 + .../native/libhdfspp/lib/rpc/gsasl_engine.cc | 230 + .../native/libhdfspp/lib/rpc/gsasl_engine.h | 47 + .../libhdfspp/lib/rpc/namenode_tracker.cc | 142 + .../native/libhdfspp/lib/rpc/namenode_tracker.h | 84 + .../main/native/libhdfspp/lib/rpc/request.cc | 179 + .../src/main/native/libhdfspp/lib/rpc/request.h | 87 + .../native/libhdfspp/lib/rpc/rpc_connection.h | 182 + .../libhdfspp/lib/rpc/rpc_connection_impl.cc | 506 + .../libhdfspp/lib/rpc/rpc_connection_impl.h | 463 + .../main/native/libhdfspp/lib/rpc/rpc_engine.cc | 334 + .../main/native/libhdfspp/lib/rpc/rpc_engine.h | 190 + .../native/libhdfspp/lib/rpc/sasl_engine.cc | 80 + .../main/native/libhdfspp/lib/rpc/sasl_engine.h | 104 + .../native/libhdfspp/lib/rpc/sasl_protocol.cc | 407 + .../native/libhdfspp/lib/rpc/sasl_protocol.h | 84 + .../main/native/libhdfspp/tests/CMakeLists.txt | 170 + .../native/libhdfspp/tests/bad_datanode_test.cc | 330 + .../libhdfspp/tests/configuration_test.cc | 573 + .../native/libhdfspp/tests/configuration_test.h | 158 + .../native/libhdfspp/tests/hdfs_builder_test.cc | 137 + .../libhdfspp/tests/hdfs_config_connect_bugs.cc | 136 + .../libhdfspp/tests/hdfs_configuration_test.cc | 172 + .../native/libhdfspp/tests/hdfs_ext_test.cc | 771 + .../libhdfspp/tests/hdfs_ioservice_test.cc | 105 + .../src/main/native/libhdfspp/tests/hdfs_shim.c | 524 + .../native/libhdfspp/tests/hdfspp_errors.cc | 116 + .../native/libhdfspp/tests/hdfspp_mini_dfs.h | 194 + .../libhdfspp/tests/hdfspp_mini_dfs_smoke.cc | 49 + .../native/libhdfspp/tests/libhdfs_wrapper.c | 22 + .../native/libhdfspp/tests/libhdfs_wrapper.h | 28 + .../libhdfspp/tests/libhdfs_wrapper_defines.h | 97 + .../libhdfspp/tests/libhdfs_wrapper_undefs.h | 107 + .../native/libhdfspp/tests/libhdfspp_wrapper.cc | 22 + .../native/libhdfspp/tests/libhdfspp_wrapper.h | 29 + .../libhdfspp/tests/libhdfspp_wrapper_defines.h | 107 + .../main/native/libhdfspp/tests/logging_test.cc | 374 + .../native/libhdfspp/tests/mock_connection.cc | 40 + .../native/libhdfspp/tests/mock_connection.h | 140 + .../libhdfspp/tests/node_exclusion_test.cc | 86 + .../libhdfspp/tests/remote_block_reader_test.cc | 536 + .../native/libhdfspp/tests/retry_policy_test.cc | 63 + .../native/libhdfspp/tests/rpc_engine_test.cc | 505 + .../libhdfspp/tests/sasl_digest_md5_test.cc | 47 + .../src/main/native/libhdfspp/tests/uri_test.cc | 293 + .../native/libhdfspp/tests/user_lock_test.cc | 225 + .../libhdfspp/third_party/asio-1.10.2/COPYING | 4 + .../third_party/asio-1.10.2/include/asio.hpp | 122 + .../asio-1.10.2/include/asio/async_result.hpp | 94 + .../include/asio/basic_datagram_socket.hpp | 949 + .../include/asio/basic_deadline_timer.hpp | 520 + .../include/asio/basic_io_object.hpp | 240 + .../include/asio/basic_raw_socket.hpp | 940 + .../include/asio/basic_seq_packet_socket.hpp | 565 + .../include/asio/basic_serial_port.hpp | 695 + .../include/asio/basic_signal_set.hpp | 384 + .../asio-1.10.2/include/asio/basic_socket.hpp | 1518 ++ .../include/asio/basic_socket_acceptor.hpp | 1136 + .../include/asio/basic_socket_iostream.hpp | 286 + .../include/asio/basic_socket_streambuf.hpp | 567 + .../include/asio/basic_stream_socket.hpp | 852 + .../include/asio/basic_streambuf.hpp | 369 + .../include/asio/basic_streambuf_fwd.hpp | 33 + .../include/asio/basic_waitable_timer.hpp | 519 + .../asio-1.10.2/include/asio/buffer.hpp | 2239 ++ .../include/asio/buffered_read_stream.hpp | 244 + .../include/asio/buffered_read_stream_fwd.hpp | 25 + .../include/asio/buffered_stream.hpp | 258 + .../include/asio/buffered_stream_fwd.hpp | 25 + .../include/asio/buffered_write_stream.hpp | 236 + .../include/asio/buffered_write_stream_fwd.hpp | 25 + .../include/asio/buffers_iterator.hpp | 481 + .../include/asio/completion_condition.hpp | 218 + .../asio-1.10.2/include/asio/connect.hpp | 823 + .../asio-1.10.2/include/asio/coroutine.hpp | 328 + .../include/asio/datagram_socket_service.hpp | 432 + .../asio-1.10.2/include/asio/deadline_timer.hpp | 63 + .../include/asio/deadline_timer_service.hpp | 173 + .../include/asio/detail/addressof.hpp | 38 + .../asio-1.10.2/include/asio/detail/array.hpp | 38 + .../include/asio/detail/array_fwd.hpp | 34 + .../asio-1.10.2/include/asio/detail/assert.hpp | 32 + .../include/asio/detail/atomic_count.hpp | 45 + .../asio/detail/base_from_completion_cond.hpp | 68 + .../include/asio/detail/bind_handler.hpp | 489 + .../include/asio/detail/buffer_resize_guard.hpp | 66 + .../asio/detail/buffer_sequence_adapter.hpp | 383 + .../asio/detail/buffered_stream_storage.hpp | 126 + .../include/asio/detail/call_stack.hpp | 125 + .../include/asio/detail/chrono_time_traits.hpp | 190 + .../include/asio/detail/completion_handler.hpp | 81 + .../asio-1.10.2/include/asio/detail/config.hpp | 895 + .../include/asio/detail/consuming_buffers.hpp | 292 + .../asio-1.10.2/include/asio/detail/cstdint.hpp | 46 + .../include/asio/detail/date_time_fwd.hpp | 34 + .../asio/detail/deadline_timer_service.hpp | 227 + .../include/asio/detail/dependent_type.hpp | 36 + .../include/asio/detail/descriptor_ops.hpp | 117 + .../include/asio/detail/descriptor_read_op.hpp | 119 + .../include/asio/detail/descriptor_write_op.hpp | 119 + .../include/asio/detail/dev_poll_reactor.hpp | 210 + .../include/asio/detail/epoll_reactor.hpp | 242 + .../asio-1.10.2/include/asio/detail/event.hpp | 48 + .../asio/detail/eventfd_select_interrupter.hpp | 83 + .../include/asio/detail/fd_set_adapter.hpp | 39 + .../include/asio/detail/fenced_block.hpp | 76 + .../include/asio/detail/function.hpp | 38 + .../asio/detail/gcc_arm_fenced_block.hpp | 89 + .../asio/detail/gcc_hppa_fenced_block.hpp | 66 + .../asio/detail/gcc_sync_fenced_block.hpp | 63 + .../asio/detail/gcc_x86_fenced_block.hpp | 89 + .../asio/detail/handler_alloc_helpers.hpp | 82 + .../asio/detail/handler_cont_helpers.hpp | 45 + .../asio/detail/handler_invoke_helpers.hpp | 57 + .../include/asio/detail/handler_tracking.hpp | 159 + .../asio/detail/handler_type_requirements.hpp | 488 + .../include/asio/detail/hash_map.hpp | 331 + .../detail/impl/buffer_sequence_adapter.ipp | 118 + .../include/asio/detail/impl/descriptor_ops.ipp | 451 + .../asio/detail/impl/dev_poll_reactor.hpp | 78 + .../asio/detail/impl/dev_poll_reactor.ipp | 445 + .../include/asio/detail/impl/epoll_reactor.hpp | 76 + .../include/asio/detail/impl/epoll_reactor.ipp | 662 + .../detail/impl/eventfd_select_interrupter.ipp | 165 + .../asio/detail/impl/handler_tracking.ipp | 305 + .../include/asio/detail/impl/kqueue_reactor.hpp | 80 + .../include/asio/detail/impl/kqueue_reactor.ipp | 463 + .../detail/impl/pipe_select_interrupter.ipp | 124 + .../include/asio/detail/impl/posix_event.ipp | 47 + .../include/asio/detail/impl/posix_mutex.ipp | 46 + .../include/asio/detail/impl/posix_thread.ipp | 74 + .../include/asio/detail/impl/posix_tss_ptr.ipp | 46 + .../detail/impl/reactive_descriptor_service.ipp | 208 + .../impl/reactive_serial_port_service.ipp | 151 + .../impl/reactive_socket_service_base.ipp | 267 + .../asio/detail/impl/resolver_service_base.ipp | 130 + .../include/asio/detail/impl/select_reactor.hpp | 87 + .../include/asio/detail/impl/select_reactor.ipp | 313 + .../asio/detail/impl/service_registry.hpp | 88 + .../asio/detail/impl/service_registry.ipp | 188 + .../asio/detail/impl/signal_set_service.ipp | 647 + .../include/asio/detail/impl/socket_ops.ipp | 3394 +++ .../detail/impl/socket_select_interrupter.ipp | 175 + .../include/asio/detail/impl/strand_service.hpp | 118 + .../include/asio/detail/impl/strand_service.ipp | 176 + .../asio/detail/impl/task_io_service.hpp | 78 + .../asio/detail/impl/task_io_service.ipp | 474 + .../include/asio/detail/impl/throw_error.ipp | 60 + .../asio/detail/impl/timer_queue_ptime.ipp | 84 + .../asio/detail/impl/timer_queue_set.ipp | 101 + .../include/asio/detail/impl/win_event.ipp | 67 + .../detail/impl/win_iocp_handle_service.ipp | 528 + .../asio/detail/impl/win_iocp_io_service.hpp | 130 + .../asio/detail/impl/win_iocp_io_service.ipp | 531 + .../impl/win_iocp_serial_port_service.ipp | 180 + .../impl/win_iocp_socket_service_base.ipp | 728 + .../include/asio/detail/impl/win_mutex.ipp | 78 + .../detail/impl/win_object_handle_service.ipp | 444 + .../asio/detail/impl/win_static_mutex.ipp | 118 + .../include/asio/detail/impl/win_thread.ipp | 139 + .../include/asio/detail/impl/win_tss_ptr.ipp | 57 + .../detail/impl/winrt_ssocket_service_base.ipp | 612 + .../asio/detail/impl/winrt_timer_scheduler.hpp | 79 + .../asio/detail/impl/winrt_timer_scheduler.ipp | 122 + .../include/asio/detail/impl/winsock_init.ipp | 82 + .../include/asio/detail/io_control.hpp | 134 + .../include/asio/detail/keyword_tss_ptr.hpp | 70 + .../include/asio/detail/kqueue_reactor.hpp | 219 + .../asio-1.10.2/include/asio/detail/limits.hpp | 26 + .../asio/detail/local_free_on_block_exit.hpp | 57 + .../include/asio/detail/macos_fenced_block.hpp | 61 + .../asio-1.10.2/include/asio/detail/mutex.hpp | 48 + .../include/asio/detail/noncopyable.hpp | 43 + .../include/asio/detail/null_event.hpp | 88 + .../include/asio/detail/null_fenced_block.hpp | 45 + .../include/asio/detail/null_mutex.hpp | 64 + .../include/asio/detail/null_reactor.hpp | 67 + .../include/asio/detail/null_signal_blocker.hpp | 69 + .../include/asio/detail/null_socket_service.hpp | 497 + .../include/asio/detail/null_static_mutex.hpp | 60 + .../include/asio/detail/null_thread.hpp | 61 + .../include/asio/detail/null_tss_ptr.hpp | 68 + .../include/asio/detail/object_pool.hpp | 146 + .../include/asio/detail/old_win_sdk_compat.hpp | 214 + .../include/asio/detail/op_queue.hpp | 156 + .../include/asio/detail/operation.hpp | 38 + .../asio/detail/pipe_select_interrupter.hpp | 89 + .../include/asio/detail/pop_options.hpp | 105 + .../include/asio/detail/posix_event.hpp | 126 + .../asio/detail/posix_fd_set_adapter.hpp | 118 + .../include/asio/detail/posix_mutex.hpp | 76 + .../asio/detail/posix_signal_blocker.hpp | 85 + .../include/asio/detail/posix_static_mutex.hpp | 64 + .../include/asio/detail/posix_thread.hpp | 105 + .../include/asio/detail/posix_tss_ptr.hpp | 79 + .../include/asio/detail/push_options.hpp | 138 + .../asio/detail/reactive_descriptor_service.hpp | 322 + .../asio/detail/reactive_null_buffers_op.hpp | 88 + .../detail/reactive_serial_port_service.hpp | 234 + .../asio/detail/reactive_socket_accept_op.hpp | 136 + .../asio/detail/reactive_socket_connect_op.hpp | 106 + .../asio/detail/reactive_socket_recv_op.hpp | 123 + .../asio/detail/reactive_socket_recvfrom_op.hpp | 133 + .../asio/detail/reactive_socket_recvmsg_op.hpp | 125 + .../asio/detail/reactive_socket_send_op.hpp | 120 + .../asio/detail/reactive_socket_sendto_op.hpp | 123 + .../asio/detail/reactive_socket_service.hpp | 457 + .../detail/reactive_socket_service_base.hpp | 450 + .../asio-1.10.2/include/asio/detail/reactor.hpp | 32 + .../include/asio/detail/reactor_fwd.hpp | 40 + .../include/asio/detail/reactor_op.hpp | 61 + .../include/asio/detail/reactor_op_queue.hpp | 168 + .../include/asio/detail/regex_fwd.hpp | 35 + .../include/asio/detail/resolve_endpoint_op.hpp | 121 + .../include/asio/detail/resolve_op.hpp | 131 + .../include/asio/detail/resolver_service.hpp | 129 + .../asio/detail/resolver_service_base.hpp | 129 + .../include/asio/detail/scoped_lock.hpp | 101 + .../include/asio/detail/scoped_ptr.hpp | 79 + .../include/asio/detail/select_interrupter.hpp | 46 + .../include/asio/detail/select_reactor.hpp | 219 + .../include/asio/detail/service_registry.hpp | 156 + .../include/asio/detail/shared_ptr.hpp | 38 + .../include/asio/detail/signal_blocker.hpp | 44 + .../include/asio/detail/signal_handler.hpp | 82 + .../include/asio/detail/signal_init.hpp | 47 + .../include/asio/detail/signal_op.hpp | 49 + .../include/asio/detail/signal_set_service.hpp | 216 + .../include/asio/detail/socket_holder.hpp | 98 + .../include/asio/detail/socket_ops.hpp | 334 + .../include/asio/detail/socket_option.hpp | 316 + .../asio/detail/socket_select_interrupter.hpp | 91 + .../include/asio/detail/socket_types.hpp | 404 + .../asio/detail/solaris_fenced_block.hpp | 61 + .../include/asio/detail/static_mutex.hpp | 52 + .../include/asio/detail/std_event.hpp | 176 + .../include/asio/detail/std_mutex.hpp | 73 + .../include/asio/detail/std_static_mutex.hpp | 81 + .../include/asio/detail/std_thread.hpp | 65 + .../include/asio/detail/strand_service.hpp | 142 + .../include/asio/detail/task_io_service.hpp | 201 + .../asio/detail/task_io_service_operation.hpp | 76 + .../asio/detail/task_io_service_thread_info.hpp | 40 + .../asio-1.10.2/include/asio/detail/thread.hpp | 56 + .../include/asio/detail/thread_info_base.hpp | 91 + .../include/asio/detail/throw_error.hpp | 53 + .../include/asio/detail/throw_exception.hpp | 51 + .../include/asio/detail/timer_queue.hpp | 332 + .../include/asio/detail/timer_queue_base.hpp | 68 + .../include/asio/detail/timer_queue_ptime.hpp | 93 + .../include/asio/detail/timer_queue_set.hpp | 66 + .../include/asio/detail/timer_scheduler.hpp | 35 + .../include/asio/detail/timer_scheduler_fwd.hpp | 40 + .../asio-1.10.2/include/asio/detail/tss_ptr.hpp | 69 + .../include/asio/detail/type_traits.hpp | 58 + .../include/asio/detail/variadic_templates.hpp | 63 + .../include/asio/detail/wait_handler.hpp | 83 + .../asio-1.10.2/include/asio/detail/wait_op.hpp | 45 + .../include/asio/detail/weak_ptr.hpp | 38 + .../include/asio/detail/win_event.hpp | 126 + .../include/asio/detail/win_fd_set_adapter.hpp | 149 + .../include/asio/detail/win_fenced_block.hpp | 89 + .../asio/detail/win_iocp_handle_read_op.hpp | 109 + .../asio/detail/win_iocp_handle_service.hpp | 322 + .../asio/detail/win_iocp_handle_write_op.hpp | 101 + .../include/asio/detail/win_iocp_io_service.hpp | 315 + .../asio/detail/win_iocp_null_buffers_op.hpp | 119 + .../include/asio/detail/win_iocp_operation.hpp | 95 + .../asio/detail/win_iocp_overlapped_op.hpp | 88 + .../asio/detail/win_iocp_overlapped_ptr.hpp | 144 + .../detail/win_iocp_serial_port_service.hpp | 228 + .../asio/detail/win_iocp_socket_accept_op.hpp | 165 + .../asio/detail/win_iocp_socket_connect_op.hpp | 124 + .../asio/detail/win_iocp_socket_recv_op.hpp | 115 + .../asio/detail/win_iocp_socket_recvfrom_op.hpp | 123 + .../asio/detail/win_iocp_socket_recvmsg_op.hpp | 116 + .../asio/detail/win_iocp_socket_send_op.hpp | 109 + .../asio/detail/win_iocp_socket_service.hpp | 525 + .../detail/win_iocp_socket_service_base.hpp | 524 + .../asio/detail/win_iocp_thread_info.hpp | 34 + .../include/asio/detail/win_mutex.hpp | 78 + .../asio/detail/win_object_handle_service.hpp | 183 + .../include/asio/detail/win_static_mutex.hpp | 74 + .../include/asio/detail/win_thread.hpp | 139 + .../include/asio/detail/win_tss_ptr.hpp | 79 + .../include/asio/detail/wince_thread.hpp | 116 + .../include/asio/detail/winrt_async_manager.hpp | 294 + .../include/asio/detail/winrt_async_op.hpp | 65 + .../include/asio/detail/winrt_resolve_op.hpp | 117 + .../asio/detail/winrt_resolver_service.hpp | 183 + .../asio/detail/winrt_socket_connect_op.hpp | 90 + .../asio/detail/winrt_socket_recv_op.hpp | 110 + .../asio/detail/winrt_socket_send_op.hpp | 101 + .../asio/detail/winrt_ssocket_service.hpp | 232 + .../asio/detail/winrt_ssocket_service_base.hpp | 355 + .../asio/detail/winrt_timer_scheduler.hpp | 131 + .../include/asio/detail/winrt_utils.hpp | 106 + .../include/asio/detail/winsock_init.hpp | 128 + .../include/asio/detail/wrapped_handler.hpp | 291 + .../asio-1.10.2/include/asio/error.hpp | 331 + .../asio-1.10.2/include/asio/error_code.hpp | 188 + .../include/asio/generic/basic_endpoint.hpp | 193 + .../include/asio/generic/datagram_protocol.hpp | 123 + .../include/asio/generic/detail/endpoint.hpp | 133 + .../asio/generic/detail/impl/endpoint.ipp | 109 + .../include/asio/generic/raw_protocol.hpp | 121 + .../asio/generic/seq_packet_protocol.hpp | 122 + .../include/asio/generic/stream_protocol.hpp | 127 + .../include/asio/handler_alloc_hook.hpp | 81 + .../include/asio/handler_continuation_hook.hpp | 54 + .../include/asio/handler_invoke_hook.hpp | 85 + .../asio-1.10.2/include/asio/handler_type.hpp | 112 + .../include/asio/high_resolution_timer.hpp | 63 + .../include/asio/impl/buffered_read_stream.hpp | 358 + .../include/asio/impl/buffered_write_stream.hpp | 338 + .../asio-1.10.2/include/asio/impl/connect.hpp | 428 + .../asio-1.10.2/include/asio/impl/error.ipp | 128 + .../include/asio/impl/error_code.ipp | 128 + .../include/asio/impl/handler_alloc_hook.ipp | 77 + .../include/asio/impl/io_service.hpp | 152 + .../include/asio/impl/io_service.ipp | 155 + .../asio-1.10.2/include/asio/impl/read.hpp | 753 + .../asio-1.10.2/include/asio/impl/read_at.hpp | 810 + .../include/asio/impl/read_until.hpp | 1147 + .../include/asio/impl/serial_port_base.hpp | 59 + .../include/asio/impl/serial_port_base.ipp | 554 + .../asio-1.10.2/include/asio/impl/spawn.hpp | 336 + .../asio-1.10.2/include/asio/impl/src.cpp | 25 + .../asio-1.10.2/include/asio/impl/src.hpp | 74 + .../include/asio/impl/use_future.hpp | 172 + .../asio-1.10.2/include/asio/impl/write.hpp | 765 + .../asio-1.10.2/include/asio/impl/write_at.hpp | 825 + .../asio-1.10.2/include/asio/io_service.hpp | 770 + .../asio-1.10.2/include/asio/ip/address.hpp | 200 + .../asio-1.10.2/include/asio/ip/address_v4.hpp | 241 + .../asio-1.10.2/include/asio/ip/address_v6.hpp | 246 + .../include/asio/ip/basic_endpoint.hpp | 263 + .../include/asio/ip/basic_resolver.hpp | 268 + .../include/asio/ip/basic_resolver_entry.hpp | 94 + .../include/asio/ip/basic_resolver_iterator.hpp | 260 + .../include/asio/ip/basic_resolver_query.hpp | 244 + .../include/asio/ip/detail/endpoint.hpp | 139 + .../include/asio/ip/detail/impl/endpoint.ipp | 204 + .../include/asio/ip/detail/socket_option.hpp | 569 + .../asio-1.10.2/include/asio/ip/host_name.hpp | 42 + .../asio-1.10.2/include/asio/ip/icmp.hpp | 115 + .../include/asio/ip/impl/address.hpp | 53 + .../include/asio/ip/impl/address.ipp | 226 + .../include/asio/ip/impl/address_v4.hpp | 53 + .../include/asio/ip/impl/address_v4.ipp | 178 + .../include/asio/ip/impl/address_v6.hpp | 53 + .../include/asio/ip/impl/address_v6.ipp | 298 + .../include/asio/ip/impl/basic_endpoint.hpp | 55 + .../include/asio/ip/impl/host_name.ipp | 54 + .../asio-1.10.2/include/asio/ip/multicast.hpp | 191 + .../include/asio/ip/resolver_query_base.hpp | 130 + .../include/asio/ip/resolver_service.hpp | 176 + .../asio-1.10.2/include/asio/ip/tcp.hpp | 155 + .../asio-1.10.2/include/asio/ip/udp.hpp | 111 + .../asio-1.10.2/include/asio/ip/unicast.hpp | 70 + .../asio-1.10.2/include/asio/ip/v6_only.hpp | 69 + .../include/asio/is_read_buffered.hpp | 59 + .../include/asio/is_write_buffered.hpp | 59 + .../include/asio/local/basic_endpoint.hpp | 239 + .../include/asio/local/connect_pair.hpp | 104 + .../include/asio/local/datagram_protocol.hpp | 80 + .../include/asio/local/detail/endpoint.hpp | 133 + .../include/asio/local/detail/impl/endpoint.ipp | 128 + .../include/asio/local/stream_protocol.hpp | 90 + .../asio-1.10.2/include/asio/placeholders.hpp | 123 + .../include/asio/posix/basic_descriptor.hpp | 490 + .../asio/posix/basic_stream_descriptor.hpp | 362 + .../include/asio/posix/descriptor_base.hpp | 97 + .../include/asio/posix/stream_descriptor.hpp | 37 + .../asio/posix/stream_descriptor_service.hpp | 260 + .../include/asio/raw_socket_service.hpp | 432 + .../asio-1.10.2/include/asio/read.hpp | 631 + .../asio-1.10.2/include/asio/read_at.hpp | 664 + .../asio-1.10.2/include/asio/read_until.hpp | 923 + .../include/asio/seq_packet_socket_service.hpp | 380 + .../asio-1.10.2/include/asio/serial_port.hpp | 36 + .../include/asio/serial_port_base.hpp | 167 + .../include/asio/serial_port_service.hpp | 253 + .../asio-1.10.2/include/asio/signal_set.hpp | 28 + .../include/asio/signal_set_service.hpp | 134 + .../include/asio/socket_acceptor_service.hpp | 302 + .../asio-1.10.2/include/asio/socket_base.hpp | 520 + .../asio-1.10.2/include/asio/spawn.hpp | 265 + .../asio-1.10.2/include/asio/ssl.hpp | 30 + .../include/asio/ssl/basic_context.hpp | 40 + .../asio-1.10.2/include/asio/ssl/context.hpp | 787 + .../include/asio/ssl/context_base.hpp | 167 + .../include/asio/ssl/context_service.hpp | 40 + .../asio/ssl/detail/buffered_handshake_op.hpp | 110 + .../include/asio/ssl/detail/engine.hpp | 164 + .../include/asio/ssl/detail/handshake_op.hpp | 68 + .../include/asio/ssl/detail/impl/engine.ipp | 326 + .../asio/ssl/detail/impl/openssl_init.ipp | 145 + .../asio-1.10.2/include/asio/ssl/detail/io.hpp | 347 + .../include/asio/ssl/detail/openssl_init.hpp | 101 + .../include/asio/ssl/detail/openssl_types.hpp | 28 + .../asio/ssl/detail/password_callback.hpp | 72 + .../include/asio/ssl/detail/read_op.hpp | 73 + .../include/asio/ssl/detail/shutdown_op.hpp | 60 + .../include/asio/ssl/detail/stream_core.hpp | 126 + .../include/asio/ssl/detail/verify_callback.hpp | 68 + .../include/asio/ssl/detail/write_op.hpp | 73 + .../asio-1.10.2/include/asio/ssl/error.hpp | 68 + .../include/asio/ssl/impl/context.hpp | 71 + .../include/asio/ssl/impl/context.ipp | 950 + .../asio-1.10.2/include/asio/ssl/impl/error.ipp | 57 + .../asio/ssl/impl/rfc2818_verification.ipp | 166 + .../asio-1.10.2/include/asio/ssl/impl/src.hpp | 28 + .../include/asio/ssl/old/basic_context.hpp | 434 + .../include/asio/ssl/old/context_service.hpp | 174 + .../ssl/old/detail/openssl_context_service.hpp | 386 + .../asio/ssl/old/detail/openssl_operation.hpp | 524 + .../ssl/old/detail/openssl_stream_service.hpp | 571 + .../asio-1.10.2/include/asio/ssl/old/stream.hpp | 501 + .../include/asio/ssl/old/stream_service.hpp | 184 + .../include/asio/ssl/rfc2818_verification.hpp | 100 + .../asio-1.10.2/include/asio/ssl/stream.hpp | 756 + .../include/asio/ssl/stream_base.hpp | 52 + .../include/asio/ssl/stream_service.hpp | 40 + .../include/asio/ssl/verify_context.hpp | 73 + .../include/asio/ssl/verify_mode.hpp | 63 + .../asio-1.10.2/include/asio/steady_timer.hpp | 61 + .../asio-1.10.2/include/asio/strand.hpp | 251 + .../include/asio/stream_socket_service.hpp | 376 + .../asio-1.10.2/include/asio/streambuf.hpp | 33 + .../asio-1.10.2/include/asio/system_error.hpp | 131 + .../asio-1.10.2/include/asio/system_timer.hpp | 57 + .../asio-1.10.2/include/asio/thread.hpp | 92 + .../asio-1.10.2/include/asio/time_traits.hpp | 96 + .../asio-1.10.2/include/asio/unyield.hpp | 21 + .../asio-1.10.2/include/asio/use_future.hpp | 92 + .../asio-1.10.2/include/asio/version.hpp | 23 + .../asio-1.10.2/include/asio/wait_traits.hpp | 41 + .../include/asio/waitable_timer_service.hpp | 168 + .../include/asio/windows/basic_handle.hpp | 281 + .../asio/windows/basic_object_handle.hpp | 178 + .../asio/windows/basic_random_access_handle.hpp | 376 + .../asio/windows/basic_stream_handle.hpp | 359 + .../include/asio/windows/object_handle.hpp | 38 + .../asio/windows/object_handle_service.hpp | 177 + .../include/asio/windows/overlapped_ptr.hpp | 116 + .../asio/windows/random_access_handle.hpp | 37 + .../windows/random_access_handle_service.hpp | 220 + .../include/asio/windows/stream_handle.hpp | 37 + .../asio/windows/stream_handle_service.hpp | 218 + .../asio-1.10.2/include/asio/write.hpp | 618 + .../asio-1.10.2/include/asio/write_at.hpp | 670 + .../asio-1.10.2/include/asio/yield.hpp | 23 + .../third_party/asio-1.10.2/src/asio.cpp | 11 + .../third_party/asio-1.10.2/src/asio_ssl.cpp | 11 + .../third_party/gmock-1.7.0/CMakeLists.txt | 21 + .../libhdfspp/third_party/gmock-1.7.0/LICENSE | 28 + .../third_party/gmock-1.7.0/gmock-gtest-all.cc | 11443 ++++++++++ .../third_party/gmock-1.7.0/gmock/gmock.h | 14198 ++++++++++++ .../third_party/gmock-1.7.0/gmock_main.cc | 54 + .../third_party/gmock-1.7.0/gtest/gtest.h | 20061 +++++++++++++++++ .../third_party/protobuf/protobuf/cpp_helpers.h | 82 + .../rapidxml-1.13/rapidxml/license.txt | 52 + .../rapidxml-1.13/rapidxml/rapidxml.hpp | 2596 +++ .../rapidxml/rapidxml_iterators.hpp | 174 + .../rapidxml-1.13/rapidxml/rapidxml_print.hpp | 421 + .../rapidxml-1.13/rapidxml/rapidxml_utils.hpp | 122 + .../libhdfspp/third_party/tr2/optional.hpp | 1068 + .../third_party/uriparser2/CMakeLists.txt | 27 + .../uriparser2/uriparser2/.gitignore | 4 + .../third_party/uriparser2/uriparser2/LICENSE | 19 + .../third_party/uriparser2/uriparser2/Makefile | 20 + .../third_party/uriparser2/uriparser2/README.md | 103 + .../uriparser2/uriparser2/test-uriparser2.c | 108 + .../uriparser2/uriparser2/uriparser/AUTHORS | 2 + .../uriparser2/uriparser2/uriparser/COPYING | 36 + .../uriparser2/uriparser2/uriparser/Uri.h | 752 + .../uriparser2/uriparser2/uriparser/UriBase.h | 188 + .../uriparser2/uriparser2/uriparser/UriCommon.c | 527 + .../uriparser2/uriparser2/uriparser/UriCommon.h | 96 + .../uriparser2/uriparser/UriCompare.c | 191 + .../uriparser2/uriparser/UriDefsAnsi.h | 82 + .../uriparser2/uriparser/UriDefsConfig.h | 105 + .../uriparser2/uriparser/UriDefsUnicode.h | 82 + .../uriparser2/uriparser2/uriparser/UriEscape.c | 449 + .../uriparser2/uriparser2/uriparser/UriFile.c | 182 + .../uriparser2/uriparser2/uriparser/UriIp4.c | 325 + .../uriparser2/uriparser2/uriparser/UriIp4.h | 87 + .../uriparser2/uriparser/UriIp4Base.c | 96 + .../uriparser2/uriparser/UriIp4Base.h | 59 + .../uriparser2/uriparser/UriNormalize.c | 722 + .../uriparser2/uriparser/UriNormalizeBase.c | 119 + .../uriparser2/uriparser/UriNormalizeBase.h | 53 + .../uriparser2/uriparser2/uriparser/UriParse.c | 2205 ++ .../uriparser2/uriparser/UriParseBase.c | 90 + .../uriparser2/uriparser/UriParseBase.h | 55 + .../uriparser2/uriparser2/uriparser/UriQuery.c | 456 + .../uriparser2/uriparser/UriRecompose.c | 573 + .../uriparser2/uriparser/UriResolve.c | 255 + .../uriparser2/uriparser/UriShorten.c | 316 + .../uriparser2/uriparser2/uriparser2.c | 273 + .../uriparser2/uriparser2/uriparser2.h | 101 + .../main/native/libhdfspp/tools/CMakeLists.txt | 93 + .../libhdfspp/tools/hdfs_allowSnapshot.cc | 86 + .../src/main/native/libhdfspp/tools/hdfs_cat.cc | 83 + .../main/native/libhdfspp/tools/hdfs_chgrp.cc | 185 + .../main/native/libhdfspp/tools/hdfs_chmod.cc | 183 + .../main/native/libhdfspp/tools/hdfs_chown.cc | 195 + .../native/libhdfspp/tools/hdfs_copyToLocal.cc | 88 + .../main/native/libhdfspp/tools/hdfs_count.cc | 93 + .../libhdfspp/tools/hdfs_createSnapshot.cc | 95 + .../libhdfspp/tools/hdfs_deleteSnapshot.cc | 87 + .../src/main/native/libhdfspp/tools/hdfs_df.cc | 89 + .../libhdfspp/tools/hdfs_disallowSnapshot.cc | 86 + .../src/main/native/libhdfspp/tools/hdfs_du.cc | 176 + .../main/native/libhdfspp/tools/hdfs_find.cc | 146 + .../src/main/native/libhdfspp/tools/hdfs_get.cc | 88 + .../src/main/native/libhdfspp/tools/hdfs_ls.cc | 130 + .../main/native/libhdfspp/tools/hdfs_mkdir.cc | 98 + .../native/libhdfspp/tools/hdfs_moveToLocal.cc | 90 + .../libhdfspp/tools/hdfs_renameSnapshot.cc | 88 + .../src/main/native/libhdfspp/tools/hdfs_rm.cc | 90 + .../main/native/libhdfspp/tools/hdfs_setrep.cc | 172 + .../main/native/libhdfspp/tools/hdfs_stat.cc | 87 + .../main/native/libhdfspp/tools/hdfs_tail.cc | 124 + .../main/native/libhdfspp/tools/tools_common.cc | 142 + .../main/native/libhdfspp/tools/tools_common.h | 37 + .../dev-support/findbugsExcludeFile.xml | 22 + hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 236 + .../hdfs/protocolPB/RouterAdminProtocolPB.java | 44 + ...uterAdminProtocolServerSideTranslatorPB.java | 211 + .../RouterAdminProtocolTranslatorPB.java | 208 + .../hadoop/hdfs/protocolPB/package-info.java | 18 + .../federation/metrics/FederationMBean.java | 223 + .../federation/metrics/FederationMetrics.java | 766 + .../federation/metrics/FederationRPCMBean.java | 92 + .../metrics/FederationRPCMetrics.java | 250 + .../FederationRPCPerformanceMonitor.java | 221 + .../federation/metrics/NamenodeBeanMetrics.java | 634 + .../federation/metrics/StateStoreMBean.java | 45 + .../federation/metrics/StateStoreMetrics.java | 149 + .../server/federation/metrics/package-info.java | 27 + .../resolver/ActiveNamenodeResolver.java | 118 + .../resolver/FederationNamenodeContext.java | 87 + .../FederationNamenodeServiceState.java | 46 + .../resolver/FederationNamespaceInfo.java | 73 + .../resolver/FileSubclusterResolver.java | 75 + .../resolver/MembershipNamenodeResolver.java | 317 + .../federation/resolver/MountTableManager.java | 80 + .../federation/resolver/MountTableResolver.java | 597 + .../MultipleDestinationMountTableResolver.java | 116 + .../resolver/NamenodePriorityComparator.java | 66 + .../resolver/NamenodeStatusReport.java | 406 + .../federation/resolver/PathLocation.java | 212 + .../federation/resolver/RemoteLocation.java | 76 + .../resolver/order/DestinationOrder.java | 30 + .../resolver/order/HashFirstResolver.java | 71 + .../federation/resolver/order/HashResolver.java | 137 + .../resolver/order/LocalResolver.java | 297 + .../resolver/order/OrderedResolver.java | 37 + .../resolver/order/RandomResolver.java | 62 + .../federation/resolver/order/package-info.java | 29 + .../federation/resolver/package-info.java | 41 + .../federation/router/ConnectionContext.java | 104 + .../federation/router/ConnectionManager.java | 435 + .../federation/router/ConnectionPool.java | 337 + .../federation/router/ConnectionPoolId.java | 123 + .../server/federation/router/DFSRouter.java | 76 + .../server/federation/router/ErasureCoding.java | 198 + .../federation/router/FederationUtil.java | 208 + .../router/NamenodeHeartbeatService.java | 362 + .../federation/router/PeriodicService.java | 198 + .../hdfs/server/federation/router/Quota.java | 208 + .../server/federation/router/RBFConfigKeys.java | 229 + .../router/RemoteLocationContext.java | 68 + .../server/federation/router/RemoteMethod.java | 164 + .../server/federation/router/RemoteParam.java | 71 + .../hdfs/server/federation/router/Router.java | 655 + .../federation/router/RouterAdminServer.java | 298 + .../server/federation/router/RouterClient.java | 80 + .../router/RouterHeartbeatService.java | 166 + .../federation/router/RouterHttpServer.java | 124 + .../server/federation/router/RouterMetrics.java | 73 + .../federation/router/RouterMetricsService.java | 108 + .../router/RouterPermissionChecker.java | 82 + .../federation/router/RouterQuotaManager.java | 172 + .../router/RouterQuotaUpdateService.java | 227 + .../federation/router/RouterQuotaUsage.java | 119 + .../federation/router/RouterRpcClient.java | 1081 + .../federation/router/RouterRpcMonitor.java | 108 + .../federation/router/RouterRpcServer.java | 2330 ++ .../router/RouterSafeModeException.java | 53 + .../router/RouterSafemodeService.java | 149 + .../federation/router/RouterServiceState.java | 31 + .../federation/router/RouterStateManager.java | 50 + .../server/federation/router/package-info.java | 31 + .../federation/store/CachedRecordStore.java | 242 + .../federation/store/MembershipStore.java | 126 + .../federation/store/MountTableStore.java | 49 + .../server/federation/store/RecordStore.java | 100 + .../server/federation/store/RouterStore.java | 78 + .../federation/store/StateStoreCache.java | 36 + .../store/StateStoreCacheUpdateService.java | 70 + .../StateStoreConnectionMonitorService.java | 67 + .../federation/store/StateStoreService.java | 450 + .../store/StateStoreUnavailableException.java | 33 + .../federation/store/StateStoreUtils.java | 109 + .../store/driver/StateStoreDriver.java | 203 + .../driver/StateStoreRecordOperations.java | 148 + .../store/driver/StateStoreSerializer.java | 119 + .../store/driver/impl/StateStoreBaseImpl.java | 86 + .../driver/impl/StateStoreFileBaseImpl.java | 462 + .../store/driver/impl/StateStoreFileImpl.java | 148 + .../driver/impl/StateStoreFileSystemImpl.java | 178 + .../driver/impl/StateStoreSerializableImpl.java | 98 + .../driver/impl/StateStoreSerializerPBImpl.java | 115 + .../driver/impl/StateStoreZooKeeperImpl.java | 328 + .../store/driver/impl/package-info.java | 39 + .../federation/store/driver/package-info.java | 37 + .../store/impl/MembershipStoreImpl.java | 310 + .../store/impl/MountTableStoreImpl.java | 154 + .../federation/store/impl/RouterStoreImpl.java | 93 + .../federation/store/impl/package-info.java | 31 + .../server/federation/store/package-info.java | 62 + .../protocol/AddMountTableEntryRequest.java | 47 + .../protocol/AddMountTableEntryResponse.java | 42 + .../store/protocol/EnterSafeModeRequest.java | 32 + .../store/protocol/EnterSafeModeResponse.java | 50 + .../protocol/GetMountTableEntriesRequest.java | 49 + .../protocol/GetMountTableEntriesResponse.java | 53 + .../GetNamenodeRegistrationsRequest.java | 52 + .../GetNamenodeRegistrationsResponse.java | 55 + .../store/protocol/GetNamespaceInfoRequest.java | 30 + .../protocol/GetNamespaceInfoResponse.java | 52 + .../protocol/GetRouterRegistrationRequest.java | 47 + .../protocol/GetRouterRegistrationResponse.java | 44 + .../protocol/GetRouterRegistrationsRequest.java | 34 + .../GetRouterRegistrationsResponse.java | 55 + .../store/protocol/GetSafeModeRequest.java | 31 + .../store/protocol/GetSafeModeResponse.java | 49 + .../store/protocol/LeaveSafeModeRequest.java | 32 + .../store/protocol/LeaveSafeModeResponse.java | 50 + .../protocol/NamenodeHeartbeatRequest.java | 52 + .../protocol/NamenodeHeartbeatResponse.java | 49 + .../protocol/RemoveMountTableEntryRequest.java | 49 + .../protocol/RemoveMountTableEntryResponse.java | 42 + .../store/protocol/RouterHeartbeatRequest.java | 50 + .../store/protocol/RouterHeartbeatResponse.java | 49 + .../protocol/UpdateMountTableEntryRequest.java | 51 + .../protocol/UpdateMountTableEntryResponse.java | 43 + .../UpdateNamenodeRegistrationRequest.java | 72 + .../UpdateNamenodeRegistrationResponse.java | 51 + .../pb/AddMountTableEntryRequestPBImpl.java | 84 + .../pb/AddMountTableEntryResponsePBImpl.java | 76 + .../impl/pb/EnterSafeModeRequestPBImpl.java | 62 + .../impl/pb/EnterSafeModeResponsePBImpl.java | 73 + .../impl/pb/FederationProtocolPBTranslator.java | 145 + .../pb/GetMountTableEntriesRequestPBImpl.java | 76 + .../pb/GetMountTableEntriesResponsePBImpl.java | 104 + .../GetNamenodeRegistrationsRequestPBImpl.java | 87 + .../GetNamenodeRegistrationsResponsePBImpl.java | 99 + .../impl/pb/GetNamespaceInfoRequestPBImpl.java | 60 + .../impl/pb/GetNamespaceInfoResponsePBImpl.java | 95 + .../pb/GetRouterRegistrationRequestPBImpl.java | 70 + .../pb/GetRouterRegistrationResponsePBImpl.java | 79 + .../pb/GetRouterRegistrationsRequestPBImpl.java | 61 + .../GetRouterRegistrationsResponsePBImpl.java | 102 + .../impl/pb/GetSafeModeRequestPBImpl.java | 62 + .../impl/pb/GetSafeModeResponsePBImpl.java | 73 + .../impl/pb/LeaveSafeModeRequestPBImpl.java | 62 + .../impl/pb/LeaveSafeModeResponsePBImpl.java | 73 + .../impl/pb/NamenodeHeartbeatRequestPBImpl.java | 93 + .../pb/NamenodeHeartbeatResponsePBImpl.java | 71 + .../pb/RemoveMountTableEntryRequestPBImpl.java | 76 + .../pb/RemoveMountTableEntryResponsePBImpl.java | 76 + .../impl/pb/RouterHeartbeatRequestPBImpl.java | 78 + .../impl/pb/RouterHeartbeatResponsePBImpl.java | 70 + .../pb/UpdateMountTableEntryRequestPBImpl.java | 96 + .../pb/UpdateMountTableEntryResponsePBImpl.java | 76 + ...UpdateNamenodeRegistrationRequestPBImpl.java | 95 + ...pdateNamenodeRegistrationResponsePBImpl.java | 73 + .../store/protocol/impl/pb/package-info.java | 29 + .../federation/store/protocol/package-info.java | 31 + .../federation/store/records/BaseRecord.java | 215 + .../store/records/MembershipState.java | 336 + .../store/records/MembershipStats.java | 130 + .../federation/store/records/MountTable.java | 437 + .../server/federation/store/records/Query.java | 66 + .../federation/store/records/QueryResult.java | 56 + .../federation/store/records/RouterState.java | 168 + .../store/records/StateStoreVersion.java | 91 + .../records/impl/pb/MembershipStatePBImpl.java | 335 + .../records/impl/pb/MembershipStatsPBImpl.java | 201 + .../store/records/impl/pb/MountTablePBImpl.java | 317 + .../store/records/impl/pb/PBRecord.java | 47 + .../records/impl/pb/RouterStatePBImpl.java | 202 + .../impl/pb/StateStoreVersionPBImpl.java | 79 + .../store/records/impl/pb/package-info.java | 29 + .../federation/store/records/package-info.java | 36 + .../federation/utils/ConsistentHashRing.java | 144 + .../server/federation/utils/package-info.java | 28 + .../hdfs/tools/federation/RouterAdmin.java | 640 + .../hdfs/tools/federation/package-info.java | 28 + .../src/main/proto/FederationProtocol.proto | 244 + .../src/main/proto/RouterProtocol.proto | 62 + .../src/main/resources/hdfs-rbf-default.xml | 434 + .../src/main/webapps/proto-web.xml | 16 + .../main/webapps/router/federationhealth.html | 424 + .../src/main/webapps/router/federationhealth.js | 381 + .../src/main/webapps/router/index.html | 24 + .../src/main/webapps/router/robots.txt | 2 + .../src/site/markdown/HDFSRouterFederation.md | 373 + .../src/site/resources/css/site.css | 30 + .../site/resources/images/routerfederation.png | Bin 0 -> 24961 bytes .../hadoop-hdfs-rbf/src/site/site.xml | 28 + .../fs/contract/router/RouterHDFSContract.java | 98 + .../router/TestRouterHDFSContractAppend.java | 44 + .../router/TestRouterHDFSContractConcat.java | 51 + .../router/TestRouterHDFSContractCreate.java | 48 + .../router/TestRouterHDFSContractDelete.java | 48 + .../TestRouterHDFSContractGetFileStatus.java | 49 + .../router/TestRouterHDFSContractMkdir.java | 48 + .../router/TestRouterHDFSContractOpen.java | 48 + .../router/TestRouterHDFSContractRename.java | 48 + .../TestRouterHDFSContractRootDirectory.java | 64 + .../router/TestRouterHDFSContractSeek.java | 49 + .../router/TestRouterHDFSContractSetTimes.java | 49 + .../server/federation/FederationTestUtils.java | 277 + .../server/federation/MiniRouterDFSCluster.java | 1028 + .../hdfs/server/federation/MockResolver.java | 313 + .../server/federation/RouterConfigBuilder.java | 163 + .../server/federation/StateStoreDFSCluster.java | 148 + .../metrics/TestFederationMetrics.java | 293 + .../federation/metrics/TestMetricsBase.java | 196 + .../resolver/TestMountTableResolver.java | 528 + .../TestMultipleDestinationResolver.java | 419 + .../resolver/TestNamenodeResolver.java | 284 + .../resolver/order/TestLocalResolver.java | 143 + .../router/TestConnectionManager.java | 157 + .../federation/router/TestRBFConfigFields.java | 51 + .../server/federation/router/TestRouter.java | 150 + .../federation/router/TestRouterAdmin.java | 340 + .../federation/router/TestRouterAdminCLI.java | 479 + .../router/TestRouterAllResolver.java | 402 + .../router/TestRouterHeartbeatService.java | 146 + .../federation/router/TestRouterMountTable.java | 143 + .../router/TestRouterNamenodeHeartbeat.java | 168 + .../router/TestRouterNamenodeMonitoring.java | 143 + .../federation/router/TestRouterQuota.java | 451 + .../router/TestRouterQuotaManager.java | 125 + .../router/TestRouterRPCClientRetries.java | 160 + .../server/federation/router/TestRouterRpc.java | 1072 + .../router/TestRouterRpcMultiDestination.java | 216 + .../federation/router/TestRouterSafemode.java | 200 + .../store/FederationStateStoreTestUtils.java | 274 + .../federation/store/TestStateStoreBase.java | 81 + .../store/TestStateStoreMembershipState.java | 463 + .../store/TestStateStoreMountTable.java | 250 + .../store/TestStateStoreRouterState.java | 195 + .../store/driver/TestStateStoreDriverBase.java | 613 + .../store/driver/TestStateStoreFile.java | 76 + .../store/driver/TestStateStoreFileBase.java | 47 + .../store/driver/TestStateStoreFileSystem.java | 94 + .../store/driver/TestStateStoreZK.java | 105 + .../store/records/TestMembershipState.java | 129 + .../store/records/TestMountTable.java | 259 + .../store/records/TestRouterState.java | 85 + .../src/test/resources/contract/hdfs.xml | 114 + .../src/test/resources/hdfs-site.xml | 29 + .../src/test/resources/log4j.properties | 23 + .../dev-support/findbugsExcludeFile.xml | 3 - hadoop-hdfs-project/hadoop-hdfs/pom.xml | 6 - .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 204 - .../hdfs/protocolPB/RouterAdminProtocolPB.java | 44 - ...uterAdminProtocolServerSideTranslatorPB.java | 211 - .../RouterAdminProtocolTranslatorPB.java | 208 - .../server/blockmanagement/BlockManager.java | 7 + .../BlockUnderConstructionFeature.java | 6 +- .../server/datanode/web/DatanodeHttpServer.java | 1 + .../federation/metrics/FederationMBean.java | 223 - .../federation/metrics/FederationMetrics.java | 766 - .../federation/metrics/FederationRPCMBean.java | 92 - .../metrics/FederationRPCMetrics.java | 250 - .../FederationRPCPerformanceMonitor.java | 221 - .../federation/metrics/NamenodeBeanMetrics.java | 634 - .../federation/metrics/StateStoreMBean.java | 45 - .../federation/metrics/StateStoreMetrics.java | 149 - .../server/federation/metrics/package-info.java | 27 - .../resolver/ActiveNamenodeResolver.java | 118 - .../resolver/FederationNamenodeContext.java | 87 - .../FederationNamenodeServiceState.java | 46 - .../resolver/FederationNamespaceInfo.java | 73 - .../resolver/FileSubclusterResolver.java | 75 - .../resolver/MembershipNamenodeResolver.java | 317 - .../federation/resolver/MountTableManager.java | 80 - .../federation/resolver/MountTableResolver.java | 597 - .../resolver/NamenodePriorityComparator.java | 63 - .../resolver/NamenodeStatusReport.java | 406 - .../federation/resolver/PathLocation.java | 212 - .../federation/resolver/RemoteLocation.java | 76 - .../resolver/order/DestinationOrder.java | 29 - .../federation/resolver/order/package-info.java | 29 - .../federation/resolver/package-info.java | 41 - .../federation/router/ConnectionContext.java | 104 - .../federation/router/ConnectionManager.java | 436 - .../federation/router/ConnectionPool.java | 337 - .../federation/router/ConnectionPoolId.java | 123 - .../server/federation/router/DFSRouter.java | 76 - .../server/federation/router/ErasureCoding.java | 198 - .../federation/router/FederationUtil.java | 209 - .../router/NamenodeHeartbeatService.java | 362 - .../federation/router/PeriodicService.java | 198 - .../hdfs/server/federation/router/Quota.java | 208 - .../router/RemoteLocationContext.java | 68 - .../server/federation/router/RemoteMethod.java | 164 - .../server/federation/router/RemoteParam.java | 71 - .../hdfs/server/federation/router/Router.java | 656 - .../federation/router/RouterAdminServer.java | 298 - .../server/federation/router/RouterClient.java | 80 - .../router/RouterHeartbeatService.java | 167 - .../federation/router/RouterHttpServer.java | 124 - .../server/federation/router/RouterMetrics.java | 73 - .../federation/router/RouterMetricsService.java | 108 - .../router/RouterPermissionChecker.java | 82 - .../federation/router/RouterQuotaManager.java | 172 - .../router/RouterQuotaUpdateService.java | 228 - .../federation/router/RouterQuotaUsage.java | 119 - .../federation/router/RouterRpcClient.java | 1022 - .../federation/router/RouterRpcMonitor.java | 108 - .../federation/router/RouterRpcServer.java | 2189 -- .../router/RouterSafeModeException.java | 53 - .../router/RouterSafemodeService.java | 150 - .../federation/router/RouterServiceState.java | 31 - .../federation/router/RouterStateManager.java | 50 - .../server/federation/router/package-info.java | 31 - .../federation/store/CachedRecordStore.java | 242 - .../federation/store/MembershipStore.java | 126 - .../federation/store/MountTableStore.java | 49 - .../server/federation/store/RecordStore.java | 100 - .../server/federation/store/RouterStore.java | 78 - .../federation/store/StateStoreCache.java | 36 - .../store/StateStoreCacheUpdateService.java | 70 - .../StateStoreConnectionMonitorService.java | 67 - .../federation/store/StateStoreService.java | 450 - .../store/StateStoreUnavailableException.java | 33 - .../federation/store/StateStoreUtils.java | 109 - .../store/driver/StateStoreDriver.java | 203 - .../driver/StateStoreRecordOperations.java | 148 - .../store/driver/StateStoreSerializer.java | 119 - .../store/driver/impl/StateStoreBaseImpl.java | 86 - .../driver/impl/StateStoreFileBaseImpl.java | 462 - .../store/driver/impl/StateStoreFileImpl.java | 148 - .../driver/impl/StateStoreFileSystemImpl.java | 178 - .../driver/impl/StateStoreSerializableImpl.java | 98 - .../driver/impl/StateStoreSerializerPBImpl.java | 115 - .../driver/impl/StateStoreZooKeeperImpl.java | 328 - .../store/driver/impl/package-info.java | 39 - .../federation/store/driver/package-info.java | 37 - .../store/impl/MembershipStoreImpl.java | 310 - .../store/impl/MountTableStoreImpl.java | 154 - .../federation/store/impl/RouterStoreImpl.java | 93 - .../federation/store/impl/package-info.java | 31 - .../server/federation/store/package-info.java | 62 - .../protocol/AddMountTableEntryRequest.java | 47 - .../protocol/AddMountTableEntryResponse.java | 42 - .../store/protocol/EnterSafeModeRequest.java | 32 - .../store/protocol/EnterSafeModeResponse.java | 50 - .../protocol/GetMountTableEntriesRequest.java | 49 - .../protocol/GetMountTableEntriesResponse.java | 53 - .../GetNamenodeRegistrationsRequest.java | 52 - .../GetNamenodeRegistrationsResponse.java | 55 - .../store/protocol/GetNamespaceInfoRequest.java | 30 - .../protocol/GetNamespaceInfoResponse.java | 52 - .../protocol/GetRouterRegistrationRequest.java | 47 - .../protocol/GetRouterRegistrationResponse.java | 44 - .../protocol/GetRouterRegistrationsRequest.java | 34 - .../GetRouterRegistrationsResponse.java | 55 - .../store/protocol/GetSafeModeRequest.java | 31 - .../store/protocol/GetSafeModeResponse.java | 49 - .../store/protocol/LeaveSafeModeRequest.java | 32 - .../store/protocol/LeaveSafeModeResponse.java | 50 - .../protocol/NamenodeHeartbeatRequest.java | 52 - .../protocol/NamenodeHeartbeatResponse.java | 49 - .../protocol/RemoveMountTableEntryRequest.java | 49 - .../protocol/RemoveMountTableEntryResponse.java | 42 - .../store/protocol/RouterHeartbeatRequest.java | 50 - .../store/protocol/RouterHeartbeatResponse.java | 49 - .../protocol/UpdateMountTableEntryRequest.java | 51 - .../protocol/UpdateMountTableEntryResponse.java | 43 - .../UpdateNamenodeRegistrationRequest.java | 72 - .../UpdateNamenodeRegistrationResponse.java | 51 - .../pb/AddMountTableEntryRequestPBImpl.java | 84 - .../pb/AddMountTableEntryResponsePBImpl.java | 76 - .../impl/pb/EnterSafeModeRequestPBImpl.java | 62 - .../impl/pb/EnterSafeModeResponsePBImpl.java | 73 - .../impl/pb/FederationProtocolPBTranslator.java | 145 - .../pb/GetMountTableEntriesRequestPBImpl.java | 76 - .../pb/GetMountTableEntriesResponsePBImpl.java | 104 - .../GetNamenodeRegistrationsRequestPBImpl.java | 87 - .../GetNamenodeRegistrationsResponsePBImpl.java | 99 - .../impl/pb/GetNamespaceInfoRequestPBImpl.java | 60 - .../impl/pb/GetNamespaceInfoResponsePBImpl.java | 95 - .../pb/GetRouterRegistrationRequestPBImpl.java | 70 - .../pb/GetRouterRegistrationResponsePBImpl.java | 79 - .../pb/GetRouterRegistrationsRequestPBImpl.java | 61 - .../GetRouterRegistrationsResponsePBImpl.java | 102 - .../impl/pb/GetSafeModeRequestPBImpl.java | 62 - .../impl/pb/GetSafeModeResponsePBImpl.java | 73 - .../impl/pb/LeaveSafeModeRequestPBImpl.java | 62 - .../impl/pb/LeaveSafeModeResponsePBImpl.java | 73 - .../impl/pb/NamenodeHeartbeatRequestPBImpl.java | 93 - .../pb/NamenodeHeartbeatResponsePBImpl.java | 71 - .../pb/RemoveMountTableEntryRequestPBImpl.java | 76 - .../pb/RemoveMountTableEntryResponsePBImpl.java | 76 - .../impl/pb/RouterHeartbeatRequestPBImpl.java | 78 - .../impl/pb/RouterHeartbeatResponsePBImpl.java | 70 - .../pb/UpdateMountTableEntryRequestPBImpl.java | 96 - .../pb/UpdateMountTableEntryResponsePBImpl.java | 76 - ...UpdateNamenodeRegistrationRequestPBImpl.java | 95 - ...pdateNamenodeRegistrationResponsePBImpl.java | 73 - .../store/protocol/impl/pb/package-info.java | 29 - .../federation/store/protocol/package-info.java | 31 - .../federation/store/records/BaseRecord.java | 215 - .../store/records/MembershipState.java | 336 - .../store/records/MembershipStats.java | 130 - .../federation/store/records/MountTable.java | 430 - .../server/federation/store/records/Query.java | 66 - .../federation/store/records/QueryResult.java | 56 - .../federation/store/records/RouterState.java | 168 - .../store/records/StateStoreVersion.java | 91 - .../records/impl/pb/MembershipStatePBImpl.java | 335 - .../records/impl/pb/MembershipStatsPBImpl.java | 201 - .../store/records/impl/pb/MountTablePBImpl.java | 313 - .../store/records/impl/pb/PBRecord.java | 47 - .../records/impl/pb/RouterStatePBImpl.java | 202 - .../impl/pb/StateStoreVersionPBImpl.java | 79 - .../store/records/impl/pb/package-info.java | 29 - .../federation/store/records/package-info.java | 36 - .../hdfs/server/namenode/FSDirTruncateOp.java | 5 +- .../hdfs/server/namenode/FSNamesystem.java | 2 +- .../hadoop/hdfs/server/namenode/INode.java | 6 +- .../server/namenode/snapshot/FileDiffList.java | 3 +- .../hdfs/tools/federation/RouterAdmin.java | 626 - .../hdfs/tools/federation/package-info.java | 28 - .../hdfs/tools/snapshot/SnapshotDiff.java | 1 + .../org/apache/hadoop/hdfs/web/JsonUtil.java | 3 + .../src/main/proto/FederationProtocol.proto | 243 - .../src/main/proto/RouterProtocol.proto | 62 - .../src/main/resources/hdfs-default.xml | 408 - .../main/webapps/router/federationhealth.html | 424 - .../src/main/webapps/router/federationhealth.js | 381 - .../src/main/webapps/router/index.html | 24 - .../src/main/webapps/router/robots.txt | 2 - .../src/site/markdown/HDFSRouterFederation.md | 373 - .../site/resources/images/routerfederation.png | Bin 24961 -> 0 bytes .../org/apache/hadoop/hdfs/DFSTestUtil.java | 37 + .../apache/hadoop/hdfs/TestEncryptionZones.java | 41 + .../hdfs/server/datanode/TestBlockRecovery.java | 83 + .../server/federation/FederationTestUtils.java | 246 - .../hdfs/server/federation/MockResolver.java | 313 - .../server/federation/RouterConfigBuilder.java | 163 - .../server/federation/RouterDFSCluster.java | 1005 - .../server/federation/StateStoreDFSCluster.java | 148 - .../metrics/TestFederationMetrics.java | 293 - .../federation/metrics/TestMetricsBase.java | 196 - .../resolver/TestMountTableResolver.java | 528 - .../resolver/TestNamenodeResolver.java | 284 - .../router/TestConnectionManager.java | 157 - .../router/TestNamenodeHeartbeat.java | 168 - .../server/federation/router/TestRouter.java | 150 - .../federation/router/TestRouterAdmin.java | 292 - .../federation/router/TestRouterAdminCLI.java | 444 - .../router/TestRouterHeartbeatService.java | 147 - .../federation/router/TestRouterMountTable.java | 143 - .../router/TestRouterNamenodeMonitoring.java | 143 - .../federation/router/TestRouterQuota.java | 452 - .../router/TestRouterQuotaManager.java | 125 - .../router/TestRouterRPCClientRetries.java | 160 - .../server/federation/router/TestRouterRpc.java | 1072 - .../router/TestRouterRpcMultiDestination.java | 216 - .../federation/router/TestRouterSafemode.java | 201 - .../store/FederationStateStoreTestUtils.java | 274 - .../federation/store/TestStateStoreBase.java | 81 - .../store/TestStateStoreMembershipState.java | 463 - .../store/TestStateStoreMountTable.java | 250 - .../store/TestStateStoreRouterState.java | 195 - .../store/driver/TestStateStoreDriverBase.java | 613 - .../store/driver/TestStateStoreFile.java | 76 - .../store/driver/TestStateStoreFileBase.java | 47 - .../store/driver/TestStateStoreFileSystem.java | 94 - .../store/driver/TestStateStoreZK.java | 105 - .../store/records/TestMembershipState.java | 129 - .../store/records/TestMountTable.java | 259 - .../store/records/TestRouterState.java | 85 - .../snapshot/TestSnapshotDiffReport.java | 25 +- .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 6 +- .../hadoop/hdfs/web/TestWebHdfsTimeouts.java | 6 +- hadoop-hdfs-project/pom.xml | 1 + .../org/apache/hadoop/mapred/TestQueue.java | 7 +- hadoop-project/pom.xml | 7 +- hadoop-project/src/site/site.xml | 3 +- .../fs/aliyun/oss/AliyunOSSCopyFileContext.java | 70 + .../fs/aliyun/oss/AliyunOSSCopyFileTask.java | 65 + .../fs/aliyun/oss/AliyunOSSFileSystem.java | 51 +- .../apache/hadoop/fs/aliyun/oss/Constants.java | 16 + .../oss/TestAliyunOSSFileSystemContract.java | 130 + .../apache/hadoop/tools/HadoopArchiveLogs.java | 214 +- .../hadoop/tools/TestHadoopArchiveLogs.java | 82 +- .../tools/hadoop-aws/committer_architecture.md | 76 +- .../markdown/tools/hadoop-aws/committers.md | 18 +- .../markdown/tools/hadoop-aws/encryption.md | 8 +- .../src/site/markdown/tools/hadoop-aws/index.md | 34 +- .../site/markdown/tools/hadoop-aws/s3guard.md | 4 +- .../site/markdown/tools/hadoop-aws/testing.md | 20 +- .../tools/hadoop-aws/troubleshooting_s3a.md | 2 +- .../fs/contract/s3a/ITestS3AContractDistCp.java | 17 + hadoop-tools/hadoop-azure-datalake/pom.xml | 19 + .../fs/adl/live/TestAdlContractDistCpLive.java | 36 + .../org/apache/hadoop/tools/CopyListing.java | 3 +- .../hadoop/tools/CopyListingFileStatus.java | 13 +- .../apache/hadoop/tools/DistCpConstants.java | 20 +- .../apache/hadoop/tools/DistCpOptionSwitch.java | 20 +- .../org/apache/hadoop/tools/DistCpOptions.java | 23 + .../org/apache/hadoop/tools/OptionsParser.java | 6 + .../hadoop/tools/mapred/CopyCommitter.java | 219 +- .../hadoop/tools/mapred/DeletedDirTracker.java | 181 + .../apache/hadoop/tools/util/DistCpUtils.java | 41 +- .../contract/AbstractContractDistCpTest.java | 453 +- .../tools/contract/TestLocalContractDistCp.java | 36 + .../hadoop/tools/mapred/TestCopyCommitter.java | 165 +- .../tools/mapred/TestDeletedDirTracker.java | 250 + .../hadoop/tools/util/TestDistCpUtils.java | 46 +- .../src/test/resources/contract/localfs.xml | 128 + .../src/test/resources/log4j.properties | 13 +- .../api/records/AllocationTagNamespace.java | 336 - .../api/records/AllocationTagNamespaceType.java | 29 - .../hadoop/yarn/api/records/AllocationTags.java | 50 - .../hadoop/yarn/api/records/Evaluable.java | 38 - .../yarn/api/records/TargetApplications.java | 53 - .../yarn/api/resource/PlacementConstraints.java | 58 +- .../hadoop/yarn/conf/YarnConfiguration.java | 5 +- .../InvalidAllocationTagException.java | 34 - .../src/main/resources/yarn-default.xml | 4 +- .../RegisterNodeManagerRequest.java | 5 + .../pb/RegisterNodeManagerRequestPBImpl.java | 80 + .../yarn_server_common_service_protos.proto | 1 + .../hadoop/yarn/server/nodemanager/Context.java | 4 +- .../nodemanager/LocalDirsHandlerService.java | 7 +- .../yarn/server/nodemanager/NodeManager.java | 24 + .../nodemanager/NodeStatusUpdaterImpl.java | 14 + .../runtime/DockerLinuxContainerRuntime.java | 10 +- .../logaggregation/AppLogAggregatorImpl.java | 11 +- .../tracker/NMLogAggregationStatusTracker.java | 270 + .../TestLinuxContainerExecutorWithMocks.java | 4 +- .../amrmproxy/BaseAMRMProxyTest.java | 6 + .../BaseContainerManagerTest.java | 19 + .../runtime/TestDockerContainerRuntime.java | 98 +- .../TestNMLogAggregationStatusTracker.java | 151 + .../resourcemanager/ResourceTrackerService.java | 17 +- .../resourcemanager/rmnode/RMNodeImpl.java | 6 + .../rmnode/RMNodeStartedEvent.java | 11 + .../scheduler/SchedulerApplicationAttempt.java | 15 +- .../constraint/AllocationTagNamespace.java | 312 + .../scheduler/constraint/AllocationTags.java | 82 + .../constraint/AllocationTagsManager.java | 146 +- .../scheduler/constraint/Evaluable.java | 38 + .../constraint/PlacementConstraintsUtil.java | 55 +- .../constraint/TargetApplications.java | 51 + .../algorithm/LocalAllocationTagsManager.java | 27 +- .../SingleConstraintAppPlacementAllocator.java | 17 +- .../security/QueueACLsManager.java | 1 - .../resourcemanager/webapp/RMWSConsts.java | 8 + .../webapp/RMWebServiceProtocol.java | 18 + .../resourcemanager/webapp/RMWebServices.java | 55 +- .../yarn/server/resourcemanager/MockAM.java | 11 +- .../rmcontainer/TestRMContainerImpl.java | 47 +- ...estSchedulingRequestContainerAllocation.java | 126 + .../constraint/TestAllocationTagsManager.java | 272 +- .../constraint/TestAllocationTagsNamespace.java | 36 +- .../TestPlacementConstraintsUtil.java | 247 +- .../TestLocalAllocationTagsManager.java | 33 +- ...stSingleConstraintAppPlacementAllocator.java | 9 +- .../webapp/TestRMWebServices.java | 84 + .../webapp/DefaultRequestInterceptorREST.java | 9 + .../webapp/FederationInterceptorREST.java | 6 + .../server/router/webapp/RouterWebServices.java | 17 + .../webapp/MockRESTRequestInterceptor.java | 6 + .../PassThroughRESTRequestInterceptor.java | 8 + .../pom.xml | 8 + .../pom.xml | 4 + .../storage/reader/ApplicationEntityReader.java | 14 +- .../reader/FlowActivityEntityReader.java | 4 +- .../storage/reader/FlowRunEntityReader.java | 7 +- .../storage/reader/GenericEntityReader.java | 12 +- .../reader/SubApplicationEntityReader.java | 15 +- .../storage/reader/TimelineEntityReader.java | 46 +- .../pom.xml | 4 + .../src/main/webapp/app/initializers/loader.js | 7 +- 1227 files changed, 219630 insertions(+), 37636 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/BUILDING.txt ---------------------------------------------------------------------- diff --git a/BUILDING.txt b/BUILDING.txt index 6c266e5..f1df194 100644 --- a/BUILDING.txt +++ b/BUILDING.txt @@ -11,6 +11,8 @@ Requirements: * Zlib devel (if compiling native code) * openssl devel (if compiling native hadoop-pipes and to get the best HDFS encryption performance) * Linux FUSE (Filesystem in Userspace) version 2.6 or above (if compiling fuse_dfs) +* Jansson C XML parsing library ( if compiling libwebhdfs ) +* Doxygen ( if compiling libhdfspp and generating the documents ) * Internet connection for first build (to fetch all Maven and Hadoop dependencies) * python (for releasedocs) * bats (for shell code testing) @@ -348,7 +350,7 @@ Requirements: * Maven 3.0 or later * ProtocolBuffer 2.5.0 * CMake 3.1 or newer -* Windows SDK 7.1 or Visual Studio 2010 Professional +* Visual Studio 2010 Professional or Higher * Windows SDK 8.1 (if building CPU rate control for the container executor) * zlib headers (if building native code bindings for zlib) * Internet connection for first build (to fetch all Maven and Hadoop dependencies) @@ -359,18 +361,15 @@ Requirements: Unix command-line tools are also included with the Windows Git package which can be downloaded from http://git-scm.com/downloads -If using Visual Studio, it must be Visual Studio 2010 Professional (not 2012). +If using Visual Studio, it must be Professional level or higher. Do not use Visual Studio Express. It does not support compiling for 64-bit, -which is problematic if running a 64-bit system. The Windows SDK 7.1 is free to -download here: - -http://www.microsoft.com/en-us/download/details.aspx?id=8279 +which is problematic if running a 64-bit system. The Windows SDK 8.1 is available to download at: http://msdn.microsoft.com/en-us/windows/bg162891.aspx -Cygwin is neither required nor supported. +Cygwin is not required. ---------------------------------------------------------------------------------- Building: @@ -378,21 +377,12 @@ Building: Keep the source code tree in a short path to avoid running into problems related to Windows maximum path length limitation (for example, C:\hdc). -Run builds from a Windows SDK Command Prompt. (Start, All Programs, -Microsoft Windows SDK v7.1, Windows SDK 7.1 Command Prompt). - -JAVA_HOME must be set, and the path must not contain spaces. If the full path -would contain spaces, then use the Windows short path instead. - -You must set the Platform environment variable to either x64 or Win32 depending -on whether you're running a 64-bit or 32-bit system. Note that this is -case-sensitive. It must be "Platform", not "PLATFORM" or "platform". -Environment variables on Windows are usually case-insensitive, but Maven treats -them as case-sensitive. Failure to set this environment variable correctly will -cause msbuild to fail while building the native code in hadoop-common. +There is one support command file located in dev-support called win-paths-eg.cmd. +It should be copied somewhere convenient and modified to fit your needs. -set Platform=x64 (when building on a 64-bit system) -set Platform=Win32 (when building on a 32-bit system) +win-paths-eg.cmd sets up the environment for use. You will need to modify this +file. It will put all of the required components in the command path, +configure the bit-ness of the build, and set several optional components. Several tests require that the user must have the Create Symbolic Links privilege. http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/dev-support/bin/dist-layout-stitching ---------------------------------------------------------------------- diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching index c0787be..cff2ce7 100755 --- a/dev-support/bin/dist-layout-stitching +++ b/dev-support/bin/dist-layout-stitching @@ -128,6 +128,7 @@ run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}" run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${VERSION}" . run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-${VERSION}" . run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-native-client/target/hadoop-hdfs-native-client-${VERSION}" . +run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-rbf/target/hadoop-hdfs-rbf-${VERSION}" . run copy "${ROOT}/hadoop-yarn-project/target/hadoop-yarn-project-${VERSION}" . run copy "${ROOT}/hadoop-mapreduce-project/target/hadoop-mapreduce-${VERSION}" . http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/dev-support/bin/win-vs-upgrade.cmd ---------------------------------------------------------------------- diff --git a/dev-support/bin/win-vs-upgrade.cmd b/dev-support/bin/win-vs-upgrade.cmd new file mode 100644 index 0000000..d8c9d73 --- /dev/null +++ b/dev-support/bin/win-vs-upgrade.cmd @@ -0,0 +1,39 @@ +@ECHO OFF +@REM Licensed to the Apache Software Foundation (ASF) under one or more +@REM contributor license agreements. See the NOTICE file distributed with +@REM this work for additional information regarding copyright ownership. +@REM The ASF licenses this file to You under the Apache License, Version 2.0 +@REM (the "License"); you may not use this file except in compliance with +@REM the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, software +@REM distributed under the License is distributed on an "AS IS" BASIS, +@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@REM See the License for the specific language governing permissions and +@REM limitations under the License. + +@WHERE devenv +IF %ERRORLEVEL% NEQ 0 ( + @ECHO "devenv command was not found. Verify your compiler installation level." + EXIT /b 1 +) + +@REM Need to save output to a file because for loop will just +@REM loop forever... :( + +SET srcdir=%1 +SET workdir=%2 + +IF EXIST %srcdir%\Backup ( + @ECHO "Solution files already upgraded." + EXIT /b 0 +) + +CD %srcdir% +DIR /B *.sln > %workdir%\HADOOP-SLN-UPGRADE.TXT + +FOR /F %%f IN (%workdir%\HADOOP-SLN-UPGRADE.TXT) DO ( + devenv %%f /upgrade +) http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/dev-support/docker/Dockerfile ---------------------------------------------------------------------- diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile index cefb3dd..369c606 100644 --- a/dev-support/docker/Dockerfile +++ b/dev-support/docker/Dockerfile @@ -42,6 +42,7 @@ RUN apt-get -q update && apt-get -q install -y \ apt-utils \ build-essential \ bzip2 \ + clang \ curl \ doxygen \ fuse \ @@ -54,6 +55,7 @@ RUN apt-get -q update && apt-get -q install -y \ libfuse-dev \ libprotobuf-dev \ libprotoc-dev \ + libsasl2-dev \ libsnappy-dev \ libssl-dev \ libtool \ @@ -71,6 +73,7 @@ RUN apt-get -q update && apt-get -q install -y \ software-properties-common \ snappy \ sudo \ + valgrind \ zlib1g-dev ####### http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/dev-support/win-paths-eg.cmd ---------------------------------------------------------------------- diff --git a/dev-support/win-paths-eg.cmd b/dev-support/win-paths-eg.cmd new file mode 100644 index 0000000..f9acae0 --- /dev/null +++ b/dev-support/win-paths-eg.cmd @@ -0,0 +1,49 @@ +@ECHO OFF +@REM Licensed to the Apache Software Foundation (ASF) under one or more +@REM contributor license agreements. See the NOTICE file distributed with +@REM this work for additional information regarding copyright ownership. +@REM The ASF licenses this file to You under the Apache License, Version 2.0 +@REM (the "License"); you may not use this file except in compliance with +@REM the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, software +@REM distributed under the License is distributed on an "AS IS" BASIS, +@REM WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@REM See the License for the specific language governing permissions and +@REM limitations under the License. + +@REM ************************************************* +@REM JDK and these settings MUST MATCH +@REM +@REM 64-bit : Platform = x64, VCVARSPLAT = amd64 +@REM +@REM 32-bit : Platform = Win32, VCVARSPLAT = x86 +@REM + +SET Platform=x64 +SET VCVARSPLAT=amd64 + +@REM ****************** +@REM Forcibly move the Maven local repo + +SET MAVEN_OPTS=-Dmaven.repo.local=C:\Tools\m2 + +@REM ******************************************* +@REM +@REM Locations of your bits and pieces +@REM +@REM NOTE: cmake is assumed to already be on the +@REM command path +@REM + +SET MAVEN_HOME=C:\Tools\apache-maven-3.5.0 +SET JAVA_HOME=C:\Tools\jdk +SET MSVS=C:\Program Files (x86)\Microsoft Visual Studio 12.0 +SET PROTO_BIN=C:\Tools\protobuf-2.5.0 +SET GIT_HOME=C:\Program Files\Git + +SET PATH=%JAVA_HOME%\bin;%MAVEN_HOME%\bin;%PROTO_BIN%;%GIT_HOME%\bin;%PATH% + +CALL "%MSVS%\VC\vcvarsall.bat" %VCVARSPLAT% http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/hadoop-common-project/hadoop-annotations/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-annotations/pom.xml b/hadoop-common-project/hadoop-annotations/pom.xml index 4506551..64dde71 100644 --- a/hadoop-common-project/hadoop-annotations/pom.xml +++ b/hadoop-common-project/hadoop-annotations/pom.xml @@ -39,21 +39,6 @@ <profiles> <profile> - <id>jdk1.7</id> - <activation> - <jdk>1.7</jdk> - </activation> - <dependencies> - <dependency> - <groupId>jdk.tools</groupId> - <artifactId>jdk.tools</artifactId> - <version>1.7</version> - <scope>system</scope> - <systemPath>${java.home}/../lib/tools.jar</systemPath> - </dependency> - </dependencies> - </profile> - <profile> <id>jdk1.8</id> <activation> <jdk>1.8</jdk> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/hadoop-common-project/hadoop-common/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml index 49d3575..1a16dc4 100644 --- a/hadoop-common-project/hadoop-common/pom.xml +++ b/hadoop-common-project/hadoop-common/pom.xml @@ -839,6 +839,20 @@ <artifactId>exec-maven-plugin</artifactId> <executions> <execution> + <id>convert-ms-winutils</id> + <phase>generate-sources</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>${basedir}\..\..\dev-support\bin\win-vs-upgrade.cmd</executable> + <arguments> + <argument>${basedir}\src\main\winutils</argument> + <argument>${project.build.directory}</argument> + </arguments> + </configuration> + </execution> + <execution> <id>compile-ms-winutils</id> <phase>compile</phase> <goals> @@ -858,6 +872,20 @@ </configuration> </execution> <execution> + <id>convert-ms-native-dll</id> + <phase>generate-sources</phase> + <goals> + <goal>exec</goal> + </goals> + <configuration> + <executable>${basedir}\..\..\dev-support\bin\win-vs-upgrade.cmd</executable> + <arguments> + <argument>${basedir}\src\main\native</argument> + <argument>${project.build.directory}</argument> + </arguments> + </configuration> + </execution> + <execution> <id>compile-ms-native-dll</id> <phase>compile</phase> <goals> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java index 00b4702..e0a5866 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java @@ -816,8 +816,11 @@ public class Configuration implements Iterable<Map.Entry<String,String>>, */ @SuppressWarnings("unchecked") public Configuration(Configuration other) { - this.resources = (ArrayList<Resource>) other.resources.clone(); synchronized(other) { + // Make sure we clone a finalized state + // Resources like input streams can be processed only once + other.getProps(); + this.resources = (ArrayList<Resource>) other.resources.clone(); if (other.properties != null) { this.properties = (Properties)other.properties.clone(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/b78c94f4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ThreadUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ThreadUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ThreadUtil.java index 6444428..2cda8a4 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ThreadUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/ThreadUtil.java @@ -53,8 +53,7 @@ public class ThreadUtil { * Convenience method that returns a resource as inputstream from the * classpath. * <p> - * It first attempts to use the Thread's context classloader and if not - * set it uses the class' classloader. + * Uses the Thread's context classloader to load resource. * * @param resourceName resource to retrieve. * @@ -68,6 +67,27 @@ public class ThreadUtil { throw new IOException("Can not read resource file '" + resourceName + "' because class loader of the current thread is null"); } + return getResourceAsStream(cl, resourceName); + } + + /** + * Convenience method that returns a resource as inputstream from the + * classpath using given classloader. + * <p> + * + * @param cl ClassLoader to be used to retrieve resource. + * @param resourceName resource to retrieve. + * + * @throws IOException thrown if resource cannot be loaded + * @return inputstream with the resource. + */ + public static InputStream getResourceAsStream(ClassLoader cl, + String resourceName) + throws IOException { + if (cl == null) { + throw new IOException("Can not read resource file '" + resourceName + + "' because given class loader is null"); + } InputStream is = cl.getResourceAsStream(resourceName); if (is == null) { throw new IOException("Can not read resource file '" + @@ -75,4 +95,6 @@ public class ThreadUtil { } return is; } + + } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
