This is an automated email from the ASF dual-hosted git repository. rakeshr pushed a change to branch HDDS-2939 in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
from bfa2801 HDDS-4211. [OFS] Better owner and group display for listing Ozone volumes and buckets (#1397) add a7c72fb HDDS-4196. Add an endpoint in Recon to query Prometheus (#1390) add 4b96d83 HDDS-4228: add field 'num' to ALLOCATE_BLOCK of scm audit log. (#1413) add 04ac1ef HDDS-4129. change MAX_QUOTA_IN_BYTES to Long.MAX_VALUE. (#1337) add 2134c2e HDDS-4218.Remove test TestRatisManager (#1409) add ab7481d HDDS-4217.Remove test TestOzoneContainerRatis (#1408) add 72e3215 HDDS-4119. Improve performance of the BufferPool management of Ozone client (#1336) add 48e8e50 HDDS-3927. Rename Ozone OM,DN,SCM runtime options to conform to naming conventions (#1401) add 68869d1 HDDS-4155. Directory and filename can end up with same name in a path. (#1361) add def697f HDDS-4039. Reduce the number of fields in hdds.proto to improve performance (#1289) add 8e71e81 HDDS-4166. Documentation index page redirects to the wrong address (#1372) add 045aa71 HDDS-4075. Retry request on different OM on AccessControlException (#1303) add e84f5ce HDDS-4210. ResolveBucket during checkAcls fails. (#1398) add 7beb2d0 HDDS-4053. Volume space: add quotaUsageInBytes and update it when write and delete key. (#1296) add 570d34c HDDS-4244. Container deleted wrong replica cause mis-replicated. (#1423) add 69c3e0e HDDS-4250. Fix wrong logger name (#1429) add 079ee7f HDDS-4104. Provide a way to get the default value and key of java-based-configuration easily (#1369) add 241de5a HDDS-4241. Support HADOOP_TOKEN_FILE_LOCATION for Ozone token CLI. (#1422) add 0da6cfd HDDS-4247. Fixed log4j usage in some places (#1426) add 410a246 HDDS-4255. Remove unused Ant and Jdiff dependency versions (#1433) add 68d1ab0 HDDS-3981. Add more debug level log to XceiverClientGrpc for debug purpose (#1214) add ce0c072 HDDS-3102. ozone getconf command should use the GenericCli parent class (#1410) add 1e9ff6c HDDS-3947: Sort DNs for client when the key is a file for #getFileStatus #listStatus APIs (#1385) add f3a60dc HDDS-4233. Interrupted exeception printed out from DatanodeStateMachine (#1416) add b281d62 HDDS-4206. Attempt pipeline creation more frequently in acceptance tests (#1389) No new revisions were added by this update. Summary of changes: hadoop-hdds/client/pom.xml | 11 + .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 18 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 123 +++++---- .../apache/hadoop/hdds/scm/storage/BufferPool.java | 47 ++-- .../hadoop/hdds/scm/storage/CommitWatcher.java | 37 +-- .../storage/TestBlockOutputStreamCorrectness.java | 224 +++++++++++++++++ .../hadoop/hdds/scm/storage/TestBufferPool.java} | 29 ++- hadoop-hdds/common/src/main/conf/hadoop-env.sh | 13 +- .../hadoop/hdds/protocol/DatanodeDetails.java | 83 ++++-- .../apache/hadoop/hdds/recon/ReconConfigKeys.java | 4 + .../apache/hadoop/hdds/scm/XceiverClientSpi.java | 4 +- .../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 2 +- .../hdds/scm/storage/ContainerProtocolCalls.java | 80 +++--- .../org/apache/hadoop/hdds/utils/Scheduler.java | 2 +- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 5 +- .../apache/hadoop/ozone/common/ChunkBuffer.java | 14 +- .../common/ChunkBufferImplWithByteBuffer.java | 10 +- .../hadoop/hdds/protocol/MockDatanodeDetails.java | 2 +- .../hadoop/hdds/scm/pipeline/MockPipeline.java | 29 ++- .../hadoop/ozone/common/TestChunkBuffer.java | 16 +- .../hdds/conf/ConfigurationReflectionUtil.java | 46 ++++ .../hdds/conf/TestConfigurationReflectionUtil.java | 111 ++++++++ .../common/statemachine/DatanodeStateMachine.java | 7 +- .../states/endpoint/RegisterEndpointTask.java | 6 +- .../container/common/utils/HddsVolumeUtil.java | 2 +- .../container/keyvalue/KeyValueContainer.java | 3 +- .../protocol/StorageContainerDatanodeProtocol.java | 14 +- ...inerDatanodeProtocolClientSideTranslatorPB.java | 12 +- ...inerDatanodeProtocolServerSideTranslatorPB.java | 2 +- .../hadoop/ozone/container/common/ScmTestMock.java | 9 +- hadoop-hdds/docs/content/concept/Overview.zh.md | 2 +- hadoop-hdds/docs/content/interface/O3fs.zh.md | 5 +- .../docs/content/security/SecureOzone.zh.md | 3 +- .../themes/ozonedoc/layouts/_default/single.html | 2 +- .../ozonedoc/layouts/partials/languages.html | 3 +- .../themes/ozonedoc/layouts/partials/navbar.html | 6 +- .../themes/ozonedoc/layouts/partials/sidebar.html | 4 +- .../x509/certificate/authority/BaseApprover.java | 2 +- .../interface-client/src/main/proto/hdds.proto | 17 +- .../interface-client/src/main/resources/proto.lock | 40 ++- .../proto/ScmServerDatanodeHeartbeatProtocol.proto | 2 +- .../interface-server/src/main/resources/proto.lock | 4 +- .../hdds/scm/container/ContainerReportHandler.java | 2 +- .../hdds/scm/container/ReplicationManager.java | 18 +- .../hdds/scm/container/SCMContainerManager.java | 4 +- .../hadoop/hdds/scm/node/NodeStateManager.java | 30 +-- .../scm/pipeline/BackgroundPipelineCreator.java | 2 +- .../hdds/scm/pipeline/PipelineReportHandler.java | 2 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 1 + .../hdds/scm/server/SCMDatanodeProtocolServer.java | 4 +- .../hdds/scm/container/TestReplicationManager.java | 92 +++++++ .../ozone/container/common/TestEndPoint.java | 2 +- .../apache/hadoop/ozone/client/OzoneVolume.java | 17 ++ .../apache/hadoop/ozone/client/rpc/RpcClient.java | 8 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 26 ++ .../apache/hadoop/ozone/freon/OzoneGetConf.java | 278 --------------------- .../apache/hadoop/ozone/freon/package-info.java | 21 -- .../ozone/om/ha/OMFailoverProxyProvider.java | 187 +++++++++++++- .../hadoop/ozone/om/helpers/OmVolumeArgs.java | 34 ++- .../ozone/om/protocolPB/Hadoop3OmTransport.java | 175 +------------ ...OzoneManagerProtocolClientSideTranslatorPB.java | 2 + .../ozone/security/OzoneTokenIdentifier.java | 5 +- .../dist/src/main/compose/ozone-csi/docker-config | 1 + .../dist/src/main/compose/ozone-mr/common-config | 1 + .../src/main/compose/ozone-om-ha-s3/docker-config | 1 + .../dist/src/main/compose/ozone-om-ha-s3/test.sh | 2 + .../src/main/compose/ozone-topology/docker-config | 5 +- .../dist/src/main/compose/ozone/docker-config | 1 + .../src/main/compose/ozones3-haproxy/docker-config | 1 + .../src/main/compose/ozonesecure-mr/docker-config | 1 + .../main/compose/ozonesecure-om-ha/docker-config | 3 +- .../src/main/compose/ozonesecure-om-ha/test.sh | 2 + .../src/main/compose/ozonesecure/docker-config | 1 + .../dist/src/main/compose/upgrade/docker-config | 1 + .../readdata.robot => basic/getconf.robot} | 17 +- .../smoketest/security/ozone-secure-token.robot | 16 +- hadoop-ozone/dist/src/shell/ozone/ozone | 15 +- hadoop-ozone/dist/src/shell/ozone/stop-ozone.sh | 8 +- .../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 120 +++++++++ .../hadoop/ozone/TestOzoneConfigurationFields.java | 4 +- .../hadoop/ozone/client/rpc/TestCommitWatcher.java | 31 ++- .../client/rpc/TestOzoneRpcClientAbstract.java | 269 +++++++++++++++++++- .../ozoneimpl/TestOzoneContainerRatis.java | 138 ---------- .../container/ozoneimpl/TestRatisManager.java | 124 --------- .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 7 +- .../src/main/proto/OmClientProtocol.proto | 2 + .../apache/hadoop/ozone/om/BucketManagerImpl.java | 4 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 72 +++++- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 3 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 55 +++- .../apache/hadoop/ozone/om/fs/OzoneManagerFS.java | 53 +++- .../hadoop/ozone/om/request/OMClientRequest.java | 18 +- .../bucket/acl/OMBucketRemoveAclRequest.java | 2 +- .../request/bucket/acl/OMBucketSetAclRequest.java | 2 +- .../ozone/om/request/file/OMFileCreateRequest.java | 39 +-- .../om/request/key/OMAllocateBlockRequest.java | 34 +-- .../ozone/om/request/key/OMKeyCommitRequest.java | 43 +++- .../ozone/om/request/key/OMKeyCreateRequest.java | 42 ++-- .../ozone/om/request/key/OMKeyDeleteRequest.java | 29 ++- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 39 ++- .../ozone/om/request/key/OMKeysDeleteRequest.java | 22 +- .../ozone/om/request/key/OMKeysRenameRequest.java | 2 +- .../om/request/key/OMTrashRecoverRequest.java | 2 +- .../om/request/key/acl/OMKeyRemoveAclRequest.java | 2 +- .../om/request/key/acl/OMKeySetAclRequest.java | 2 +- .../key/acl/prefix/OMPrefixRemoveAclRequest.java | 2 +- .../key/acl/prefix/OMPrefixSetAclRequest.java | 2 +- .../multipart/S3MultipartUploadAbortRequest.java | 37 ++- .../S3MultipartUploadCommitPartRequest.java | 17 +- .../S3MultipartUploadCompleteRequest.java | 10 + .../om/response/file/OMFileCreateResponse.java | 8 +- .../om/response/key/OMAllocateBlockResponse.java | 10 +- .../ozone/om/response/key/OMKeyCommitResponse.java | 11 +- .../ozone/om/response/key/OMKeyCreateResponse.java | 12 +- .../ozone/om/response/key/OMKeyDeleteResponse.java | 11 +- .../om/response/key/OMKeysDeleteResponse.java | 12 +- .../multipart/S3MultipartUploadAbortResponse.java | 12 +- .../S3MultipartUploadCommitPartResponse.java | 10 +- .../OzoneDelegationTokenSecretManager.java | 2 +- .../hadoop/ozone/om/failover/TestOMFailovers.java | 152 +++++++++++ .../request/file/TestOMDirectoryCreateRequest.java | 4 +- .../ozone/om/request/key/TestOMKeyRequest.java | 7 +- .../s3/multipart/TestS3MultipartRequest.java | 4 +- .../response/key/TestOMAllocateBlockResponse.java | 14 +- .../om/response/key/TestOMKeyCommitResponse.java | 12 +- .../om/response/key/TestOMKeyCreateResponse.java | 16 +- .../om/response/key/TestOMKeyDeleteResponse.java | 17 +- .../om/response/key/TestOMKeysDeleteResponse.java | 23 +- .../s3/multipart/TestS3MultipartResponse.java | 5 +- .../TestS3MultipartUploadAbortResponse.java | 14 +- hadoop-ozone/pom.xml | 3 +- .../ozone/recon/MetricsServiceProviderFactory.java | 86 +++++++ .../hadoop/ozone/recon/ReconControllerModule.java | 1 + .../hadoop/ozone/recon/ReconSchemaManager.java | 2 +- .../hadoop/ozone/recon/ReconServerConfigKeys.java | 10 + .../org/apache/hadoop/ozone/recon/ReconUtils.java | 15 +- .../ozone/recon/api/MetricsProxyEndpoint.java | 118 +++++++++ .../hadoop/ozone/recon/api/PipelineEndpoint.java | 59 ++++- .../ozone/recon/codec/DatanodeDetailsCodec.java | 5 +- .../apache/hadoop/ozone/recon/metrics/Metric.java | 38 +-- .../ReconIncrementalContainerReportHandler.java | 2 +- .../ozone/recon/spi/MetricsServiceProvider.java | 60 +++++ .../spi/impl/OzoneManagerServiceProviderImpl.java | 2 +- .../spi/impl/PrometheusServiceProviderImpl.java | 212 ++++++++++++++++ .../hadoop/ozone/recon/ReconTestInjector.java | 3 + .../apache/hadoop/ozone/recon/TestReconUtils.java | 6 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 108 +++++++- .../impl/TestOzoneManagerServiceProviderImpl.java | 9 +- .../test/resources/prometheus-test-response.txt | 21 ++ .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 21 ++ .../org/apache/hadoop/ozone/conf/OzoneGetConf.java | 86 +++++++ .../OzoneManagersCommandHandler.java} | 37 ++- .../PrintConfKeyCommandHandler.java} | 32 +-- .../StorageContainerManagersCommandHandler.java} | 38 ++- .../apache/hadoop/ozone/conf}/package-info.java | 4 +- .../hadoop/ozone/shell/token/GetTokenHandler.java | 8 +- .../ozone/shell/token/PrintTokenHandler.java | 3 +- .../ozone/shell/token/RenewTokenHandler.java | 4 +- .../hadoop/ozone/shell/token/TokenOption.java | 38 ++- pom.xml | 16 -- 160 files changed, 3291 insertions(+), 1381 deletions(-) create mode 100644 hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBlockOutputStreamCorrectness.java copy hadoop-hdds/{common/src/main/java/org/apache/hadoop/hdds/scm/storage/CheckedBiFunction.java => client/src/test/java/org/apache/hadoop/hdds/scm/storage/TestBufferPool.java} (58%) create mode 100644 hadoop-hdds/config/src/test/java/org/apache/hadoop/hdds/conf/TestConfigurationReflectionUtil.java delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/freon/OzoneGetConf.java delete mode 100644 hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/freon/package-info.java copy hadoop-ozone/dist/src/main/smoketest/{topology/readdata.robot => basic/getconf.robot} (65%) delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerRatis.java delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestRatisManager.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/failover/TestOMFailovers.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/MetricsServiceProviderFactory.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/api/MetricsProxyEndpoint.java copy hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/NodeManagerMXBean.java => hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/metrics/Metric.java (57%) create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/MetricsServiceProvider.java create mode 100644 hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/PrometheusServiceProviderImpl.java create mode 100644 hadoop-ozone/recon/src/test/resources/prometheus-test-response.txt create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/conf/OzoneGetConf.java copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/{audit/parser/handler/LoadCommandHandler.java => conf/OzoneManagersCommandHandler.java} (58%) copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/{audit/parser/handler/LoadCommandHandler.java => conf/PrintConfKeyCommandHandler.java} (59%) copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/{audit/parser/handler/LoadCommandHandler.java => conf/StorageContainerManagersCommandHandler.java} (57%) copy hadoop-ozone/{ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils => tools/src/main/java/org/apache/hadoop/ozone/conf}/package-info.java (90%) --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org