This is an automated email from the ASF dual-hosted git repository. avijayan pushed a commit to branch HDDS-3698-upgrade in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit b8dcf6b2d63918ea28864b264a4a1768a312d5b3 Merge: 0d014c0 5e5635f Author: Aravindan Vijayan <avija...@cloudera.com> AuthorDate: Tue Oct 27 06:26:03 2020 -0700 Merge remote-tracking branch 'upstream/master' into HDDS-3698-upgrade .github/workflows/post-commit.yml | 3 + LICENSE.txt | 4 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 8 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 15 +- .../hadoop/hdds/scm/storage/ChunkInputStream.java | 10 +- .../hadoop/hdds/scm/storage/CommitWatcher.java | 2 +- .../hdds/scm/storage/DummyChunkInputStream.java | 2 +- .../storage/TestBlockOutputStreamCorrectness.java | 2 +- .../org/apache/hadoop/hdds/ratis/RatisHelper.java | 19 + .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 3 + .../hadoop/hdds/scm/container/ContainerInfo.java | 2 + .../hadoop/hdds/scm/exceptions/SCMException.java | 3 +- .../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 42 ++- .../hdds/scm/storage/ContainerProtocolCalls.java | 139 +++---- .../common/src/main/resources/ozone-default.xml | 30 +- .../hadoop/hdds/conf/SimpleConfiguration.java | 13 + .../hadoop/hdds/conf/TestOzoneConfiguration.java | 7 +- .../org/apache/hadoop/hdds/conf/ConfigType.java | 3 +- .../hdds/conf/ConfigurationReflectionUtil.java | 12 + .../common/statemachine/EndpointStateMachine.java | 16 + .../CreatePipelineCommandHandler.java | 10 +- .../commandhandler/DeleteBlocksCommandHandler.java | 65 ++-- .../states/datanode/RunningDatanodeState.java | 8 +- .../common/transport/server/XceiverServerSpi.java | 10 +- .../transport/server/ratis/XceiverServerRatis.java | 55 ++- .../container/common/utils/HddsVolumeUtil.java | 5 + .../ozone/container/common/volume/HddsVolume.java | 8 +- .../container/common/volume/VolumeIOStats.java | 72 ++-- .../container/keyvalue/impl/BlockManagerImpl.java | 85 ++--- .../background/BlockDeletingService.java | 45 +-- .../protocol/commands/CreatePipelineCommand.java | 46 ++- .../main/resources/webapps/hddsDatanode/index.html | 4 +- .../TestCreatePipelineCommandHandler.java | 7 +- hadoop-hdds/dev-support/checkstyle/checkstyle.xml | 4 +- hadoop-hdds/docs/content/design/typesafeconfig.md | 10 +- hadoop-hdds/docs/content/feature/Recon.md | 2 +- .../certificates/utils/CertificateSignRequest.java | 2 +- .../certificates/utils/SelfSignedCertificate.java | 4 +- .../resources/webapps/static/angular-1.7.9.min.js | 350 ----------------- .../resources/webapps/static/angular-1.8.0.min.js | 350 +++++++++++++++++ ...ute-1.7.9.min.js => angular-route-1.8.0.min.js} | 6 +- .../interface-client/src/main/proto/hdds.proto | 1 + .../proto/ScmServerDatanodeHeartbeatProtocol.proto | 1 + .../src/main/proto/ScmServerProtocol.proto | 1 + hadoop-hdds/pom.xml | 4 +- .../block/DatanodeDeletedBlockTransactions.java | 32 +- .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 19 +- .../hdds/scm/container/ReplicationManager.java | 3 +- .../hdds/scm/container/SCMContainerManager.java | 28 +- .../hdds/scm/pipeline/PipelineStateManager.java | 3 +- .../hdds/scm/pipeline/RatisPipelineProvider.java | 29 +- .../algorithms/DefaultLeaderChoosePolicy.java | 42 +++ .../choose/algorithms/LeaderChoosePolicy.java | 55 +++ .../algorithms/LeaderChoosePolicyFactory.java | 75 ++++ .../algorithms/MinLeaderCountChoosePolicy.java | 91 +++++ .../leader/choose/algorithms/package-info.java | 19 + .../src/main/resources/webapps/scm/index.html | 4 +- .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 17 +- .../choose/algorithms/TestLeaderChoosePolicy.java | 74 ++++ .../ozone/client/io/BlockOutputStreamEntry.java | 8 +- .../client/io/BlockOutputStreamEntryPool.java | 5 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 + .../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 6 +- ...OzoneManagerProtocolClientSideTranslatorPB.java | 4 +- .../hadoop/ozone/security/acl/OzoneObjInfo.java | 10 + .../hadoop/ozone/security/acl/RequestContext.java | 48 ++- hadoop-ozone/dev-support/checks/findbugs.sh | 7 +- .../dist/src/main/compose/ozone-mr/hadoop32/.env | 6 +- .../dist/src/main/compose/ozone/docker-config | 2 +- .../dist/src/main/compose/ozonesecure-mr/.env | 3 +- .../compose/ozonesecure-mr/docker-compose.yaml | 8 +- .../dist/src/main/compose/ozonesecure-mr/test.sh | 1 + .../main/compose/ozonesecure-om-ha/docker-config | 4 +- .../src/main/compose/ozonesecure/docker-config | 4 +- .../dist/src/main/compose/upgrade/docker-config | 2 +- hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 4 +- .../dist/src/main/smoketest/basic/getconf.robot | 37 +- .../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 216 +++++++++++ .../TestRatisPipelineCreateAndDestroy.java | 2 +- .../java/org/apache/hadoop/ozone/TestDataUtil.java | 13 +- .../apache/hadoop/ozone/TestMiniOzoneCluster.java | 4 +- .../hadoop/ozone/TestOzoneConfigurationFields.java | 11 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 67 +++- .../container/metrics/TestContainerMetrics.java | 14 + .../org/apache/hadoop/ozone/om/TestOmAcls.java | 9 +- .../ozone/om/TestOzoneManagerHAMetadataOnly.java | 76 ++++ .../ozone/om/TestOzoneManagerListVolumes.java | 36 +- .../ozone/om/TestOzoneManagerRocksDBLogging.java | 2 +- .../ozone/recon/TestReconWithOzoneManager.java | 32 +- .../hadoop/ozone/scm/TestCloseContainer.java | 148 ++++++++ .../hadoop/ozone/scm/TestContainerSmallFile.java | 20 +- .../scm/TestGetCommittedBlockLengthAndPutKey.java | 4 +- .../hadoop/ozone/scm/TestXceiverClientGrpc.java | 6 +- .../src/main/proto/OmClientProtocol.proto | 20 + .../dev-support/findbugsExcludeFile.xml | 21 ++ hadoop-ozone/interface-storage/pom.xml | 9 +- .../java/org/apache/hadoop/ozone/om/OMMetrics.java | 37 ++ .../org/apache/hadoop/ozone/om/OzoneManager.java | 87 +++-- .../ozone/om/ratis/OzoneManagerRatisServer.java | 11 +- .../hadoop/ozone/om/request/OMClientRequest.java | 26 +- .../om/request/bucket/acl/OMBucketAclRequest.java | 15 + .../request/bucket/acl/OMBucketAddAclRequest.java | 15 + .../bucket/acl/OMBucketRemoveAclRequest.java | 15 + .../request/bucket/acl/OMBucketSetAclRequest.java | 15 + .../ozone/om/request/file/OMFileCreateRequest.java | 3 +- .../om/request/key/OMAllocateBlockRequest.java | 3 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 3 +- .../ozone/om/request/key/OMKeyDeleteRequest.java | 11 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 90 ++++- .../ozone/om/request/key/OMKeysDeleteRequest.java | 16 +- .../ozone/om/request/key/OMKeysRenameRequest.java | 9 +- .../ozone/om/request/key/acl/OMKeyAclRequest.java | 15 + .../om/request/key/acl/OMKeyAddAclRequest.java | 15 + .../om/request/key/acl/OMKeyRemoveAclRequest.java | 15 + .../om/request/key/acl/OMKeySetAclRequest.java | 15 + .../request/key/acl/prefix/OMPrefixAclRequest.java | 7 + .../hadoop/ozone/om/request/util/ObjectParser.java | 6 +- .../om/request/volume/OMVolumeCreateRequest.java | 6 + .../om/request/volume/OMVolumeSetOwnerRequest.java | 2 +- .../om/request/volume/acl/OMVolumeAclRequest.java | 14 + .../request/volume/acl/OMVolumeAddAclRequest.java | 15 + .../volume/acl/OMVolumeRemoveAclRequest.java | 15 + .../request/volume/acl/OMVolumeSetAclRequest.java | 15 + ...ponse.java => AbstractOMKeyDeleteResponse.java} | 68 ++-- .../ozone/om/response/key/OMKeyDeleteResponse.java | 73 +--- .../om/response/key/OMKeysDeleteResponse.java | 38 +- .../om/response/key/OMOpenKeysDeleteRequest.java | 192 ++++++++++ .../om/response/key/OMOpenKeysDeleteResponse.java | 72 ++++ .../ozone/security/acl/OzoneNativeAuthorizer.java | 44 ++- .../main/resources/webapps/ozoneManager/index.html | 4 +- .../ozone/om/request/TestOMRequestUtils.java | 100 ++++- .../bucket/acl/TestOMBucketAddAclRequest.java | 119 ++++++ .../bucket/acl/TestOMBucketRemoveAclRequest.java | 137 +++++++ .../bucket/acl/TestOMBucketSetAclRequest.java | 125 ++++++ .../ozone/om/request/bucket/acl/package-info.java | 17 +- .../ozone/om/request/key/TestOMKeyAclRequest.java | 155 +++++++- .../key/TestOMKeyPurgeRequestAndResponse.java | 15 +- .../request/key/TestOMOpenKeysDeleteRequest.java | 419 +++++++++++++++++++++ .../volume/acl/TestOMVolumeAddAclRequest.java | 7 + .../volume/acl/TestOMVolumeRemoveAclRequest.java | 8 + .../volume/acl/TestOMVolumeSetAclRequest.java | 7 + .../bucket/TestOMBucketCreateResponse.java | 8 + .../bucket/TestOMBucketDeleteResponse.java | 8 + .../bucket/TestOMBucketSetPropertyResponse.java | 9 + .../file/TestOMDirectoryCreateResponse.java | 8 + .../ozone/om/response/key/TestOMKeyResponse.java | 3 + .../om/response/key/TestOMKeysDeleteResponse.java | 12 +- .../response/key/TestOMOpenKeysDeleteResponse.java | 185 +++++++++ .../s3/multipart/TestS3MultipartResponse.java | 9 + .../security/TestOMDelegationTokenResponse.java | 8 + .../volume/TestOMVolumeCreateResponse.java | 8 + .../volume/TestOMVolumeDeleteResponse.java | 8 + .../volume/TestOMVolumeSetOwnerResponse.java | 9 + .../volume/TestOMVolumeSetQuotaResponse.java | 9 + .../hadoop/ozone/security/acl/TestVolumeOwner.java | 298 +++++++++++++++ hadoop-ozone/pom.xml | 4 +- .../hadoop/ozone/recon/ReconServerConfigKeys.java | 48 ++- .../spi/impl/OzoneManagerServiceProviderImpl.java | 52 ++- .../spi/impl/PrometheusServiceProviderImpl.java | 17 +- .../recon/tasks/TestContainerKeyMapperTask.java | 2 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 5 +- .../ozone/conf/PrintConfKeyCommandHandler.java | 3 +- .../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 11 +- .../org/apache/hadoop/ozone/debug/DBScanner.java | 18 +- .../org/apache/hadoop/ozone/debug/DropTable.java | 81 ++++ .../apache/hadoop/ozone/debug/RocksDBUtils.java | 49 +++ pom.xml | 4 +- 167 files changed, 4718 insertions(+), 1105 deletions(-) diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java index 91efc49,b757eb9..bda4067 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java @@@ -272,7 -271,7 +272,8 @@@ public final class OmUtils case AddAcl: case PurgeKeys: case RecoverTrash: + case FinalizeUpgrade: + case DeleteOpenKeys: return false; default: LOG.error("CmdType {} is not categorized as readOnly or not.", cmdType); diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAddAclRequest.java index 45cfdcf,6b3d614..4a727d1 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAddAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketAddAclRequest.java @@@ -25,9 -23,10 +25,11 @@@ import java.util.List import com.google.common.collect.Lists; import org.apache.hadoop.ozone.om.OMMetrics; + import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneObj.ObjectType; import org.apache.hadoop.ozone.util.BooleanBiFunction; + import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketRemoveAclRequest.java index 1c8af01,6552a18..b9d924b --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketRemoveAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketRemoveAclRequest.java @@@ -23,8 -21,9 +23,10 @@@ import static org.apache.hadoop.ozone.p import java.io.IOException; import java.util.List; + import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneObj.ObjectType; + import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketSetAclRequest.java index 915342e,09fa7da..5d4bfcf --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketSetAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/acl/OMBucketSetAclRequest.java @@@ -24,8 -22,9 +24,10 @@@ import java.io.IOException import java.util.ArrayList; import java.util.List; + import org.apache.hadoop.ozone.om.OzoneManager; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneObj.ObjectType; + import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java index 5527742,c475d6e..9a0a3fa --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyAddAclRequest.java @@@ -29,7 -28,7 +30,8 @@@ import org.apache.hadoop.ozone.om.helpe import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneObj.ObjectType; + import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java index cb3d16a7,d9fbc35..8b17fe4 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeyRemoveAclRequest.java @@@ -29,7 -28,7 +30,8 @@@ import org.apache.hadoop.ozone.om.helpe import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneObj.ObjectType; + import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java index b76862f,e07ac09..c6da84f --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/acl/OMKeySetAclRequest.java @@@ -30,7 -29,7 +31,8 @@@ import org.apache.hadoop.ozone.om.helpe import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; import org.apache.hadoop.ozone.om.response.key.acl.OMKeyAclResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneObj.ObjectType; + import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java index ce1b2cb,9c81c36..94f0d54 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/OMVolumeCreateRequest.java @@@ -203,9 -203,10 +204,14 @@@ public class OMVolumeCreateRequest exte return omClientResponse; } + public static String getRequestType() { + return CreateVolume.name(); + } ++ + @VisibleForTesting + public static Logger getLogger() { + return LOG; + } } diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAddAclRequest.java index d65e0df,15f0c67..0a5fadd --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAddAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeAddAclRequest.java @@@ -30,7 -29,7 +31,8 @@@ import org.apache.hadoop.ozone.om.respo import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneObj.ObjectType; + import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeRemoveAclRequest.java index e386e20,ea03503..d20e1a9 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeRemoveAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeRemoveAclRequest.java @@@ -30,7 -29,7 +31,8 @@@ import org.apache.hadoop.ozone.om.respo import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneObj.ObjectType; + import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeSetAclRequest.java index 6a0d0f1,787cf76..848a243 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeSetAclRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/volume/acl/OMVolumeSetAclRequest.java @@@ -29,7 -28,7 +30,8 @@@ import org.apache.hadoop.ozone.om.respo import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneObj.ObjectType; + import org.apache.hadoop.util.Time; import org.slf4j.Logger; import org.slf4j.LoggerFactory; --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org