Merge branch 'trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3a7f3b2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3a7f3b2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3a7f3b2 Branch: refs/heads/HDFS-7240 Commit: b3a7f3b2dfffdd83abcfbe630e31acfcf68f5521 Parents: 84e11c7 c21c260 Author: Xiaoyu Yao <x...@apache.org> Authored: Wed Jul 19 14:52:35 2017 -0700 Committer: Xiaoyu Yao <x...@apache.org> Committed: Wed Jul 19 14:52:35 2017 -0700 ---------------------------------------------------------------------- hadoop-client-modules/hadoop-client/pom.xml | 4 - .../util/TestCertificateUtil.java | 6 +- hadoop-common-project/hadoop-common/pom.xml | 4 - .../org/apache/hadoop/conf/Configuration.java | 27 +- .../apache/hadoop/conf/ReconfigurableBase.java | 7 +- .../hadoop/conf/ReconfigurationServlet.java | 8 +- .../hadoop/crypto/JceAesCtrCryptoCodec.java | 8 +- .../hadoop/crypto/OpensslAesCtrCryptoCodec.java | 8 +- .../org/apache/hadoop/crypto/OpensslCipher.java | 8 +- .../crypto/key/kms/KMSClientProvider.java | 39 +- .../key/kms/LoadBalancingKMSClientProvider.java | 90 +- .../crypto/random/OpensslSecureRandom.java | 8 +- .../hadoop/crypto/random/OsSecureRandom.java | 9 +- .../apache/hadoop/fs/AbstractFileSystem.java | 6 +- .../java/org/apache/hadoop/fs/ChecksumFs.java | 8 +- .../fs/CommonConfigurationKeysPublic.java | 29 + .../hadoop/fs/DelegationTokenRenewer.java | 10 +- .../org/apache/hadoop/fs/FSInputChecker.java | 9 +- .../java/org/apache/hadoop/fs/FileContext.java | 10 +- .../java/org/apache/hadoop/fs/FileUtil.java | 10 +- .../main/java/org/apache/hadoop/fs/FsShell.java | 6 +- .../apache/hadoop/fs/FsShellPermissions.java | 4 +- .../main/java/org/apache/hadoop/fs/Globber.java | 7 +- .../org/apache/hadoop/fs/HarFileSystem.java | 11 +- .../org/apache/hadoop/fs/LocalDirAllocator.java | 9 +- .../main/java/org/apache/hadoop/fs/Trash.java | 7 +- .../apache/hadoop/fs/TrashPolicyDefault.java | 8 +- .../org/apache/hadoop/fs/ftp/FTPFileSystem.java | 8 +- .../hadoop/fs/permission/FsPermission.java | 6 +- .../hadoop/fs/sftp/SFTPConnectionPool.java | 7 +- .../apache/hadoop/fs/sftp/SFTPFileSystem.java | 7 +- .../org/apache/hadoop/fs/shell/Command.java | 6 +- .../apache/hadoop/fs/shell/CopyCommands.java | 112 ++- .../apache/hadoop/fs/shell/MoveCommands.java | 4 +- .../apache/hadoop/ha/ActiveStandbyElector.java | 15 +- .../apache/hadoop/ha/FailoverController.java | 10 +- .../main/java/org/apache/hadoop/ha/HAAdmin.java | 8 +- .../org/apache/hadoop/ha/HealthMonitor.java | 8 +- .../java/org/apache/hadoop/ha/NodeFencer.java | 6 +- .../org/apache/hadoop/ha/PowerShellFencer.java | 7 +- .../apache/hadoop/ha/ShellCommandFencer.java | 7 +- .../org/apache/hadoop/ha/SshFenceByTcpPort.java | 18 +- .../java/org/apache/hadoop/ha/StreamPumper.java | 8 +- .../apache/hadoop/ha/ZKFailoverController.java | 20 +- ...HAServiceProtocolServerSideTranslatorPB.java | 6 +- .../org/apache/hadoop/http/HttpServer2.java | 6 +- .../hadoop/http/lib/StaticUserWebFilter.java | 7 +- .../java/org/apache/hadoop/io/BloomMapFile.java | 6 +- .../apache/hadoop/io/FastByteComparisons.java | 7 +- .../main/java/org/apache/hadoop/io/IOUtils.java | 5 +- .../main/java/org/apache/hadoop/io/MapFile.java | 8 +- .../org/apache/hadoop/io/ReadaheadPool.java | 6 +- .../java/org/apache/hadoop/io/SequenceFile.java | 7 +- .../main/java/org/apache/hadoop/io/UTF8.java | 5 +- .../apache/hadoop/io/compress/CodecPool.java | 6 +- .../io/compress/CompressionCodecFactory.java | 8 +- .../apache/hadoop/io/compress/DefaultCodec.java | 6 +- .../io/compress/bzip2/Bzip2Compressor.java | 8 +- .../io/compress/bzip2/Bzip2Decompressor.java | 8 +- .../hadoop/io/compress/bzip2/Bzip2Factory.java | 6 +- .../hadoop/io/compress/lz4/Lz4Compressor.java | 8 +- .../hadoop/io/compress/lz4/Lz4Decompressor.java | 8 +- .../io/compress/snappy/SnappyCompressor.java | 8 +- .../io/compress/snappy/SnappyDecompressor.java | 8 +- .../io/compress/zlib/BuiltInZlibDeflater.java | 8 +- .../hadoop/io/compress/zlib/ZlibCompressor.java | 8 +- .../hadoop/io/compress/zlib/ZlibFactory.java | 8 +- .../apache/hadoop/io/erasurecode/CodecUtil.java | 6 +- .../io/erasurecode/ErasureCodeNative.java | 8 +- .../org/apache/hadoop/io/file/tfile/BCFile.java | 6 +- .../hadoop/io/file/tfile/Compression.java | 6 +- .../org/apache/hadoop/io/file/tfile/TFile.java | 8 +- .../hadoop/io/file/tfile/TFileDumper.java | 8 +- .../org/apache/hadoop/io/nativeio/NativeIO.java | 16 +- .../nativeio/SharedFileDescriptorFactory.java | 7 +- .../apache/hadoop/io/retry/RetryPolicies.java | 6 +- .../org/apache/hadoop/io/retry/RetryUtils.java | 6 +- .../io/serializer/SerializationFactory.java | 8 +- .../org/apache/hadoop/ipc/CallQueueManager.java | 7 +- .../main/java/org/apache/hadoop/ipc/Client.java | 6 +- .../org/apache/hadoop/ipc/FairCallQueue.java | 6 +- .../apache/hadoop/ipc/ProtobufRpcEngine.java | 7 +- .../main/java/org/apache/hadoop/ipc/RPC.java | 6 +- .../org/apache/hadoop/ipc/RefreshRegistry.java | 7 +- .../java/org/apache/hadoop/ipc/RetryCache.java | 6 +- .../main/java/org/apache/hadoop/ipc/Server.java | 16 +- .../ipc/WeightedRoundRobinMultiplexer.java | 8 +- .../apache/hadoop/ipc/WritableRpcEngine.java | 6 +- .../hadoop/ipc/metrics/RetryCacheMetrics.java | 6 +- .../hadoop/ipc/metrics/RpcDetailedMetrics.java | 8 +- .../apache/hadoop/ipc/metrics/RpcMetrics.java | 6 +- .../org/apache/hadoop/jmx/JMXJsonServlet.java | 7 +- .../hadoop/metrics2/impl/MBeanInfoBuilder.java | 2 +- .../hadoop/metrics2/impl/MetricsConfig.java | 8 +- .../metrics2/impl/MetricsSinkAdapter.java | 9 +- .../metrics2/impl/MetricsSourceAdapter.java | 7 +- .../hadoop/metrics2/impl/MetricsSystemImpl.java | 6 +- .../hadoop/metrics2/lib/MethodMetric.java | 7 +- .../metrics2/lib/MetricsSourceBuilder.java | 7 +- .../metrics2/lib/MutableMetricsFactory.java | 7 +- .../hadoop/metrics2/lib/MutableRates.java | 7 +- .../lib/MutableRatesWithAggregation.java | 7 +- .../hadoop/metrics2/sink/GraphiteSink.java | 7 +- .../sink/ganglia/AbstractGangliaSink.java | 10 +- .../metrics2/sink/ganglia/GangliaSink30.java | 6 +- .../metrics2/sink/ganglia/GangliaSink31.java | 7 +- .../org/apache/hadoop/metrics2/util/MBeans.java | 6 +- .../hadoop/metrics2/util/MetricsCache.java | 6 +- .../main/java/org/apache/hadoop/net/DNS.java | 6 +- .../java/org/apache/hadoop/net/NetUtils.java | 6 +- .../apache/hadoop/net/ScriptBasedMapping.java | 8 +- .../apache/hadoop/net/SocketIOWithTimeout.java | 6 +- .../org/apache/hadoop/net/TableMapping.java | 6 +- .../apache/hadoop/net/unix/DomainSocket.java | 10 +- .../hadoop/net/unix/DomainSocketWatcher.java | 12 +- .../AuthenticationWithProxyUserFilter.java | 8 +- .../hadoop/security/CompositeGroupsMapping.java | 7 +- .../org/apache/hadoop/security/Credentials.java | 10 +- .../java/org/apache/hadoop/security/Groups.java | 7 +- .../HttpCrossOriginFilterInitializer.java | 8 +- .../security/JniBasedUnixGroupsMapping.java | 8 +- .../JniBasedUnixGroupsMappingWithFallback.java | 8 +- .../JniBasedUnixGroupsNetgroupMapping.java | 6 +- ...edUnixGroupsNetgroupMappingWithFallback.java | 8 +- .../hadoop/security/LdapGroupsMapping.java | 7 +- .../apache/hadoop/security/ProviderUtils.java | 7 +- .../apache/hadoop/security/SaslInputStream.java | 7 +- .../apache/hadoop/security/SaslRpcClient.java | 7 +- .../apache/hadoop/security/SaslRpcServer.java | 6 +- .../apache/hadoop/security/SecurityUtil.java | 9 +- .../hadoop/security/ShellBasedIdMapping.java | 8 +- .../ShellBasedUnixGroupsNetgroupMapping.java | 8 +- .../hadoop/security/WhitelistBasedResolver.java | 7 +- .../alias/AbstractJavaKeyStoreProvider.java | 6 +- .../alias/CredentialProviderFactory.java | 10 + .../authorize/ServiceAuthorizationManager.java | 9 +- .../hadoop/security/http/CrossOriginFilter.java | 7 +- .../security/ssl/FileBasedKeyStoresFactory.java | 8 +- .../security/ssl/ReloadingX509TrustManager.java | 7 +- .../hadoop/security/token/DtFileOperations.java | 7 +- .../hadoop/security/token/DtUtilShell.java | 6 +- .../org/apache/hadoop/security/token/Token.java | 6 +- .../AbstractDelegationTokenSecretManager.java | 8 +- .../apache/hadoop/service/AbstractService.java | 9 +- .../apache/hadoop/service/CompositeService.java | 7 +- .../service/LoggingStateChangeListener.java | 11 +- .../hadoop/service/ServiceOperations.java | 5 +- .../tracing/TracerConfigurationManager.java | 8 +- .../hadoop/util/ApplicationClassLoader.java | 10 +- .../apache/hadoop/util/AsyncDiskService.java | 7 +- .../apache/hadoop/util/CombinedIPWhiteList.java | 7 +- .../org/apache/hadoop/util/FileBasedIPList.java | 11 +- .../main/java/org/apache/hadoop/util/GSet.java | 6 +- .../hadoop/util/GenericOptionsParser.java | 7 +- .../org/apache/hadoop/util/HostsFileReader.java | 9 +- .../apache/hadoop/util/IntrusiveCollection.java | 7 +- .../org/apache/hadoop/util/JvmPauseMonitor.java | 6 +- .../org/apache/hadoop/util/MachineList.java | 6 +- .../apache/hadoop/util/NativeCodeLoader.java | 8 +- .../hadoop/util/NodeHealthScriptRunner.java | 7 +- .../java/org/apache/hadoop/util/Progress.java | 6 +- .../apache/hadoop/util/ShutdownHookManager.java | 7 +- .../hadoop/util/ShutdownThreadsHelper.java | 7 +- .../org/apache/hadoop/util/SysInfoLinux.java | 8 +- .../org/apache/hadoop/util/SysInfoWindows.java | 7 +- .../java/org/apache/hadoop/util/ThreadUtil.java | 7 +- .../org/apache/hadoop/util/VersionInfo.java | 8 +- .../hadoop/util/concurrent/AsyncGetFuture.java | 7 +- .../hadoop/util/concurrent/ExecutorHelper.java | 8 +- .../HadoopScheduledThreadPoolExecutor.java | 8 +- .../concurrent/HadoopThreadPoolExecutor.java | 8 +- .../src/main/resources/core-default.xml | 37 +- .../3.0.0-alpha4/CHANGES.3.0.0-alpha4.md | 880 +++++++++++++++++++ .../3.0.0-alpha4/RELEASENOTES.3.0.0-alpha4.md | 492 +++++++++++ .../conf/TestCommonConfigurationFields.java | 1 + .../apache/hadoop/conf/TestDeprecatedKeys.java | 2 +- .../hadoop/crypto/CryptoStreamsTestBase.java | 6 +- .../apache/hadoop/crypto/TestCryptoCodec.java | 7 +- .../crypto/key/TestKeyProviderFactory.java | 11 +- .../kms/TestLoadBalancingKMSClientProvider.java | 378 +++++++- .../apache/hadoop/fs/FCStatisticsBaseTest.java | 7 +- .../fs/FileContextCreateMkdirBaseTest.java | 2 +- .../fs/FileContextMainOperationsBaseTest.java | 66 +- .../hadoop/fs/FileContextPermissionBase.java | 5 +- .../apache/hadoop/fs/FileContextUtilBase.java | 5 +- .../org/apache/hadoop/fs/TestFileContext.java | 7 +- .../org/apache/hadoop/fs/TestFileStatus.java | 8 +- .../java/org/apache/hadoop/fs/TestFileUtil.java | 6 +- .../org/apache/hadoop/fs/TestFsShellCopy.java | 6 +- .../apache/hadoop/fs/TestFsShellReturnCode.java | 8 +- .../org/apache/hadoop/fs/TestFsShellTouch.java | 6 +- .../org/apache/hadoop/fs/TestHarFileSystem.java | 7 +- .../java/org/apache/hadoop/fs/TestHardLink.java | 2 +- .../org/apache/hadoop/fs/TestListFiles.java | 4 +- .../fs/TestLocalFileSystemPermission.java | 2 +- .../fs/contract/AbstractBondedFSContract.java | 8 +- .../hadoop/fs/contract/ContractTestUtils.java | 8 + .../hadoop/fs/loadGenerator/LoadGenerator.java | 10 +- .../hadoop/fs/shell/TestCopyFromLocal.java | 173 ++++ .../hadoop/fs/shell/TestCopyPreserveFlag.java | 19 + .../hadoop/ha/ActiveStandbyElectorTestUtil.java | 6 +- .../org/apache/hadoop/ha/DummyHAService.java | 7 +- .../org/apache/hadoop/ha/MiniZKFCCluster.java | 7 +- .../ha/TestActiveStandbyElectorRealZK.java | 7 +- .../java/org/apache/hadoop/ha/TestHAAdmin.java | 6 +- .../org/apache/hadoop/ha/TestHealthMonitor.java | 6 +- .../hadoop/ha/TestShellCommandFencer.java | 55 +- .../apache/hadoop/ha/TestSshFenceByTcpPort.java | 6 +- .../hadoop/ha/TestZKFailoverController.java | 5 +- .../apache/hadoop/http/TestGlobalFilter.java | 6 +- .../org/apache/hadoop/http/TestHttpServer.java | 6 +- .../apache/hadoop/http/TestHttpServerLogs.java | 6 +- .../hadoop/http/TestHttpServerWebapps.java | 9 +- .../hadoop/http/TestHttpServerWithSpengo.java | 7 +- .../org/apache/hadoop/http/TestPathFilter.java | 6 +- .../apache/hadoop/http/TestSSLHttpServer.java | 7 +- .../apache/hadoop/http/TestServletFilter.java | 6 +- .../hadoop/http/resource/JerseyResource.java | 6 +- .../org/apache/hadoop/io/TestArrayFile.java | 7 +- .../hadoop/io/TestDefaultStringifier.java | 8 +- .../org/apache/hadoop/io/TestSequenceFile.java | 7 +- .../java/org/apache/hadoop/io/TestSetFile.java | 7 +- .../org/apache/hadoop/io/TestWritableUtils.java | 7 +- .../apache/hadoop/io/compress/TestCodec.java | 8 +- .../io/compress/TestCompressionStreamReuse.java | 9 +- .../apache/hadoop/io/nativeio/TestNativeIO.java | 10 +- .../TestSharedFileDescriptorFactory.java | 7 +- .../io/serializer/TestSerializationFactory.java | 6 +- .../org/apache/hadoop/ipc/MiniRPCBenchmark.java | 13 +- .../org/apache/hadoop/ipc/TestAsyncIPC.java | 10 +- .../java/org/apache/hadoop/ipc/TestIPC.java | 22 +- .../hadoop/ipc/TestIPCServerResponder.java | 10 +- .../apache/hadoop/ipc/TestMiniRPCBenchmark.java | 2 +- .../ipc/TestProtoBufRpcServerHandoff.java | 12 +- .../java/org/apache/hadoop/ipc/TestRPC.java | 8 +- .../apache/hadoop/ipc/TestRPCCompatibility.java | 8 +- .../hadoop/ipc/TestRPCServerShutdown.java | 7 +- .../apache/hadoop/ipc/TestRpcServerHandoff.java | 8 +- .../java/org/apache/hadoop/ipc/TestSaslRPC.java | 23 +- .../java/org/apache/hadoop/ipc/TestServer.java | 4 +- .../ipc/TestWeightedRoundRobinMultiplexer.java | 7 +- .../metrics2/impl/TestGangliaMetrics.java | 7 +- .../hadoop/metrics2/impl/TestMetricsConfig.java | 7 +- .../metrics2/impl/TestMetricsSystemImpl.java | 13 +- .../hadoop/metrics2/impl/TestSinkQueue.java | 10 +- .../hadoop/metrics2/lib/TestMutableMetrics.java | 7 +- .../hadoop/metrics2/util/TestMetricsCache.java | 8 +- .../org/apache/hadoop/net/ServerSocketUtil.java | 9 +- .../java/org/apache/hadoop/net/TestDNS.java | 6 +- .../org/apache/hadoop/net/TestNetUtils.java | 6 +- .../hadoop/net/TestSocketIOWithTimeout.java | 8 +- .../apache/hadoop/net/TestStaticMapping.java | 7 +- .../hadoop/net/unix/TestDomainSocket.java | 2 +- .../net/unix/TestDomainSocketWatcher.java | 15 +- .../security/TestCompositeGroupMapping.java | 7 +- .../hadoop/security/TestDoAsEffectiveUser.java | 8 +- .../hadoop/security/TestGroupFallback.java | 19 +- .../hadoop/security/TestGroupsCaching.java | 8 +- .../TestShellBasedUnixGroupsMapping.java | 8 +- .../hadoop/security/TestUGIWithMiniKdc.java | 2 +- .../security/TestUserGroupInformation.java | 2 +- .../alias/TestCredentialProviderFactory.java | 17 +- .../authorize/TestAccessControlList.java | 16 +- .../security/authorize/TestProxyUsers.java | 8 +- .../hadoop/security/ssl/TestSSLFactory.java | 2 +- .../token/delegation/TestDelegationToken.java | 7 +- .../delegation/web/TestWebDelegationToken.java | 2 +- .../hadoop/service/TestCompositeService.java | 7 +- .../hadoop/service/TestServiceLifecycle.java | 7 +- .../apache/hadoop/test/GenericTestUtils.java | 71 +- .../org/apache/hadoop/test/MetricsAsserts.java | 6 +- .../hadoop/test/MultithreadedTestUtil.java | 8 +- .../hadoop/test/TestGenericTestUtils.java | 15 +- .../org/apache/hadoop/test/TestJUnitSetup.java | 7 +- .../hadoop/util/Crc32PerformanceTest.java | 11 +- .../hadoop/util/TestAsyncDiskService.java | 7 +- .../org/apache/hadoop/util/TestClasspath.java | 9 +- .../org/apache/hadoop/util/TestFindClass.java | 7 +- .../hadoop/util/TestIdentityHashStore.java | 7 +- .../apache/hadoop/util/TestLightWeightGSet.java | 7 +- .../util/TestLightWeightResizableGSet.java | 7 +- .../hadoop/util/TestNativeCodeLoader.java | 6 +- .../hadoop/util/TestReadWriteDiskValidator.java | 5 +- .../apache/hadoop/util/TestSignalLogger.java | 11 +- .../org/apache/hadoop/util/TestWinUtils.java | 6 +- .../src/test/resources/testConf.xml | 44 +- hadoop-common-project/hadoop-nfs/pom.xml | 6 + .../org/apache/hadoop/mount/MountdBase.java | 12 +- .../java/org/apache/hadoop/nfs/NfsExports.java | 6 +- .../org/apache/hadoop/nfs/nfs3/FileHandle.java | 6 +- .../org/apache/hadoop/nfs/nfs3/Nfs3Base.java | 10 +- .../hadoop/oncrpc/RegistrationClient.java | 7 +- .../java/org/apache/hadoop/oncrpc/RpcCall.java | 10 +- .../org/apache/hadoop/oncrpc/RpcProgram.java | 6 +- .../java/org/apache/hadoop/oncrpc/RpcUtil.java | 11 +- .../hadoop/oncrpc/SimpleTcpClientHandler.java | 7 +- .../apache/hadoop/oncrpc/SimpleTcpServer.java | 7 +- .../apache/hadoop/oncrpc/SimpleUdpServer.java | 7 +- .../hadoop/oncrpc/security/Credentials.java | 6 +- .../hadoop/oncrpc/security/SecurityHandler.java | 7 +- .../java/org/apache/hadoop/portmap/Portmap.java | 8 +- .../hadoop/portmap/RpcProgramPortmap.java | 7 +- .../apache/hadoop/oncrpc/TestFrameDecoder.java | 6 +- .../hadoop/fs/http/client/HttpFSFileSystem.java | 42 + .../hadoop/fs/http/server/FSOperations.java | 52 +- .../http/server/HttpFSAuthenticationFilter.java | 6 +- .../http/server/HttpFSParametersProvider.java | 3 +- .../hadoop/fs/http/server/HttpFSServer.java | 21 +- .../fs/http/server/HttpFSServerWebServer.java | 15 +- .../libexec/shellprofile.d/hadoop-httpfs.sh | 17 - .../src/main/resources/httpfs-default.xml | 8 +- .../src/site/markdown/ServerSetup.md.vm | 11 +- .../fs/http/client/BaseTestHttpFSWith.java | 115 ++- ...KerberosAuthenticationHandlerForTesting.java | 1 - .../hadoop/fs/http/server/TestHttpFSServer.java | 264 +++--- .../http/server/TestHttpFSServerWebServer.java | 2 +- .../jdiff/Apache_Hadoop_HDFS_3.0.0-alpha4.xml | 322 +++++++ .../server/blockmanagement/BlockCollection.java | 24 +- .../hdfs/server/datanode/BPOfferService.java | 47 +- .../hadoop/hdfs/server/datanode/DataNode.java | 8 + .../hdfs/server/datanode/DataNodeMXBean.java | 7 + .../datanode/fsdataset/impl/FsDatasetImpl.java | 2 + .../ContentSummaryComputationContext.java | 23 +- .../namenode/DirectoryWithQuotaFeature.java | 4 +- .../server/namenode/FSDirStatAndListingOp.java | 9 +- .../server/namenode/FSPermissionChecker.java | 32 + .../hadoop/hdfs/server/namenode/INode.java | 9 +- .../hdfs/server/namenode/INodeDirectory.java | 9 +- .../hdfs/server/namenode/INodeReference.java | 3 +- .../hadoop/hdfs/server/namenode/NameNode.java | 5 +- .../hdfs/server/namenode/SecondaryNameNode.java | 10 + .../snapshot/DirectorySnapshottableFeature.java | 3 +- .../snapshot/DirectoryWithSnapshotFeature.java | 3 +- .../hdfs/server/namenode/snapshot/Snapshot.java | 4 +- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 57 +- .../hadoop-hdfs/src/main/webapps/datanode/dn.js | 2 +- .../fs/TestHDFSFileContextMainOperations.java | 46 +- .../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 19 +- .../blockmanagement/TestBlockManager.java | 4 +- .../server/datanode/TestBPOfferService.java | 29 + .../server/datanode/TestDataNodeMXBean.java | 4 + .../TestGetContentSummaryWithPermission.java | 201 +++++ .../namenode/TestNameNodeHttpServerXFrame.java | 22 + .../namenode/TestNameNodeReconfigure.java | 11 + .../hadoop/mapreduce/v2/util/MRWebAppUtil.java | 9 +- .../java/org/apache/hadoop/mapreduce/Job.java | 13 +- .../hadoop/mapreduce/lib/db/DBOutputFormat.java | 15 +- .../webapp/TestMapReduceTrackingUriPlugin.java | 26 +- .../mapreduce/TestMapperReducerCleanup.java | 4 + .../mapreduce/lib/db/TestDBOutputFormat.java | 45 + hadoop-project/pom.xml | 6 +- .../fs/azure/AzureNativeFileSystemStore.java | 78 +- .../hadoop/fs/azure/BlockBlobInputStream.java | 396 +++++++++ .../hadoop/fs/azure/CachingAuthorizer.java | 232 +++++ .../fs/azure/LocalSASKeyGeneratorImpl.java | 28 +- .../hadoop/fs/azure/NativeAzureFileSystem.java | 78 +- .../fs/azure/NativeAzureFileSystemHelper.java | 28 + .../hadoop/fs/azure/NativeFileSystemStore.java | 6 +- .../fs/azure/RemoteSASKeyGeneratorImpl.java | 54 +- .../fs/azure/RemoteWasbAuthorizerImpl.java | 46 +- .../hadoop/fs/azure/SASKeyGeneratorImpl.java | 4 +- .../fs/azure/SecureStorageInterfaceImpl.java | 5 + .../fs/azure/SecureWasbRemoteCallHelper.java | 86 +- .../hadoop/fs/azure/StorageInterface.java | 11 +- .../hadoop/fs/azure/StorageInterfaceImpl.java | 5 + .../hadoop/fs/azure/WasbRemoteCallHelper.java | 61 +- .../hadoop/fs/azure/security/Constants.java | 19 +- .../RemoteWasbDelegationTokenManager.java | 27 +- .../hadoop/fs/azure/security/SpnegoToken.java | 49 ++ .../hadoop-azure/src/site/markdown/index.md | 50 ++ .../hadoop/fs/azure/AbstractWasbTestBase.java | 5 + .../fs/azure/AzureBlobStorageTestAccount.java | 40 +- .../hadoop/fs/azure/MockStorageInterface.java | 36 +- .../hadoop/fs/azure/MockWasbAuthorizerImpl.java | 22 +- .../azure/TestAzureConcurrentOutOfBandIo.java | 2 +- .../fs/azure/TestBlockBlobInputStream.java | 756 ++++++++++++++++ .../TestNativeAzureFSAuthorizationCaching.java | 60 ++ .../TestNativeAzureFileSystemAuthorization.java | 231 +++-- ...veAzureFileSystemAuthorizationWithOwner.java | 2 +- .../fs/azure/TestWasbRemoteCallHelper.java | 64 +- .../src/test/resources/azure-test.xml | 3 +- .../ams/ApplicationMasterServiceContext.java | 29 + .../ams/ApplicationMasterServiceProcessor.java | 81 ++ .../yarn/ams/ApplicationMasterServiceUtils.java | 89 ++ .../apache/hadoop/yarn/ams/package-info.java | 24 + .../hadoop/yarn/conf/YarnConfiguration.java | 11 +- .../yarn/client/api/impl/TimelineWriter.java | 8 +- .../resource/DominantResourceCalculator.java | 22 +- .../src/main/resources/yarn-default.xml | 20 + .../timeline/RollingLevelDBTimelineStore.java | 50 +- .../hadoop-yarn-server-common/pom.xml | 6 - .../hadoop/yarn/server/webapp/dao/AppInfo.java | 14 + .../nodemanager/LinuxContainerExecutor.java | 19 +- .../PrivilegedOperationException.java | 10 +- .../runtime/ContainerExecutionException.java | 10 +- .../scheduler/ContainerScheduler.java | 135 ++- .../recovery/NMLeveldbStateStoreService.java | 6 +- .../TestLinuxContainerExecutorWithMocks.java | 89 ++ .../TestContainerManagerRecovery.java | 2 +- .../TestContainerSchedulerQueuing.java | 85 ++ .../resourcemanager/AMSProcessingChain.java | 102 +++ .../ApplicationMasterService.java | 446 +--------- .../resourcemanager/DefaultAMSProcessor.java | 456 ++++++++++ ...pportunisticContainerAllocatorAMService.java | 184 ++-- .../yarn/server/resourcemanager/RMContext.java | 3 +- .../capacity/FifoCandidatesSelector.java | 6 +- .../ProportionalCapacityPreemptionPolicy.java | 7 +- .../placement/PlacementFactory.java | 45 + .../resourcemanager/recovery/RMStateStore.java | 5 +- .../resource/ResourceWeights.java | 4 +- .../resourcemanager/rmapp/RMAppEvent.java | 24 - .../resourcemanager/rmapp/RMAppEventType.java | 1 + .../server/resourcemanager/rmapp/RMAppImpl.java | 8 +- .../rmapp/attempt/RMAppAttemptImpl.java | 20 +- .../scheduler/SchedulerUtils.java | 18 - .../scheduler/activities/ActivitiesLogger.java | 33 +- .../scheduler/capacity/CapacityScheduler.java | 42 +- .../CapacitySchedulerConfiguration.java | 27 + .../scheduler/capacity/LeafQueue.java | 83 +- .../scheduler/fair/FSContext.java | 2 +- .../scheduler/fair/FairScheduler.java | 11 +- .../fair/policies/FairSharePolicy.java | 17 +- .../webapp/FairSchedulerAppsBlock.java | 6 + .../resourcemanager/webapp/NodeIDsInfo.java | 5 +- .../resourcemanager/webapp/RMAppsBlock.java | 8 + .../resourcemanager/webapp/dao/AppInfo.java | 13 + .../webapp/dao/LabelsToNodesInfo.java | 6 +- .../TestApplicationMasterService.java | 163 +++- ...pportunisticContainerAllocatorAMService.java | 8 + .../server/resourcemanager/TestRMRestart.java | 4 +- .../recovery/TestZKRMStateStore.java | 14 +- .../rmapp/TestRMAppTransitions.java | 17 + .../attempt/TestRMAppAttemptTransitions.java | 33 +- .../CapacitySchedulerPreemptionTestBase.java | 7 +- .../capacity/TestCapacityScheduler.java | 186 +++- .../TestCapacitySchedulerAsyncScheduling.java | 149 ++++ ...TestCapacitySchedulerSurgicalPreemption.java | 99 ++- .../scheduler/capacity/TestLeafQueue.java | 10 +- .../scheduler/fair/TestSchedulingPolicy.java | 19 +- .../webapp/TestRMWebServicesApps.java | 2 +- .../server/webproxy/amfilter/AmIpFilter.java | 60 +- .../server/webproxy/amfilter/TestAmFilter.java | 70 +- .../src/site/markdown/ResourceManagerHA.md | 2 +- .../src/site/markdown/TimelineServer.md | 4 +- 444 files changed, 9525 insertions(+), 2377 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3a7f3b2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java ---------------------------------------------------------------------- diff --cc hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java index fe1cc6e,eb8a5c3..2055afa --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java @@@ -18,24 -18,6 +18,24 @@@ package org.apache.hadoop.fs; +import com.google.common.base.Preconditions; +import org.apache.commons.collections.map.CaseInsensitiveMap; +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.apache.commons.io.FileUtils; - import org.apache.commons.logging.Log; - import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.permission.FsAction; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.io.IOUtils; +import org.apache.hadoop.io.nativeio.NativeIO; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.Shell.ShellCommandExecutor; +import org.apache.hadoop.util.StringUtils; ++import org.slf4j.Logger; ++import org.slf4j.LoggerFactory; + import java.io.BufferedInputStream; import java.io.BufferedOutputStream; import java.io.File; http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3a7f3b2/hadoop-common-project/hadoop-nfs/src/main/java/org/apache/hadoop/oncrpc/SimpleTcpServer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3a7f3b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3a7f3b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3a7f3b2/hadoop-project/pom.xml ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
