This is an automated email from the ASF dual-hosted git repository. rakeshr pushed a change to branch HDDS-2939 in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git.
from b8d1e3d HDDS-4099. No Log4j 2 configuration file found error appears in CLI (#1318) add e5e89e0 HDDS-4114. Bump log4j2 version (#1325) add 59fc0bb HDDS-4127. Components with web interface should depend on hdds-docs. (#1335) add 1abbfed HDDS-4094. Support byte-level write in Freon HadoopFsGenerator (#1310) add 1c7003e HDDS-4139. Update version number in upgrade tests (#1347) add c656feb HDDS-4144. Update version info in hadoop client dependency readme (#1348) add 122eac5 HDDS-4074. [OFS] Implement AbstractFileSystem for RootedOzoneFileSystem (#1330) add 854fdc4 HDDS-4112. Improve SCM webui page performance (#1323) add c0084a1 HDDS-3654. Let backgroundCreator create pipeline for the support replication factors alternately (#984) add a2080cf HDDS-4111. Keep the CSI.zh.md consistent with CSI.md (#1320) add 8102ac7 HDDS-4062. Non rack aware pipelines should not be created if multiple racks are alive. (#1291) add 9292b39 HDDS-4068. Client should not retry same OM on network connection failure (#1324) add 7f674fd HDDS-3972. Add option to limit number of items displaying through ldb tool. (#1206) add bc7786a HDDS-4056. Convert OzoneAdmin to pluggable model (#1285) add 5fab834 HDDS-4152. Archive container logs for kubernetes check (#1355) add 5523636 HDDS-4140. Auto-close /pending pull requests after 21 days of inactivity (#1344) add dcb1c6e HDDS-2411. add a datanode chunk validator fo datanode chunk generator (#1312) add 2f3edd9 HDDS-4153. Increase default timeout in kubernetes tests (#1357) add da61c4a HDDS-4149. Implement OzoneFileStatus#toString (#1356) add d064230 HDDS-4109. Tests in TestOzoneFileSystem should use the existing MiniOzoneCluster (#1316) add f6e4417 HDDS-4145. Bump version to 1.1.0-SNAPSHOT on master (#1349) add 02289ce HDDS-4146. Show the ScmId and ClusterId in the scm web ui. (#1350) add f64bc6e HDDS-4137. Turn on the verbose mode of safe mode check on testlib (#1343) add 44acf78 HDDS-4147. Add OFS to FileSystem META-INF (#1352) add 8e98977 HDDS-4151. Skip the inputstream while offset larger than zero in s3g (#1354) add d34ab29 HDDS-3903. OzoneRpcClient support batch rename keys. (#1150) add 78ca8bf HDDS-4077. Incomplete OzoneFileSystem statistics (#1329) add 0ec1a8a HDDS-3867. Extend the chunkinfo tool to display information from all nodes in the pipeline. (#1154) add 34ee831 HDDS-4121. Implement OmMetadataMangerImpl#getExpiredOpenKeys. (#1351) add 13fe31b HDDS-4167. Acceptance test logs missing if fails during cluster startup (#1366) add 9cef3f6 HDDS-4176. Fix failed UT: test2WayCommitForTimeoutException (#1370) add 199512b HDDS-4131. Container report should update container key count and bytes used if they differ in SCM (#1339) add 77d56e6 HDDS-4165. GitHub Actions cache does not work outside of workspace (#1364) add c77e7ba HDDS-3804. Recon start fails with SQL exception with MySQL DB. (#1377) add b58054e HDDS-4169. Fix some minor errors in StorageContainerManager.md (#1367) add d2fb937 HDDS-4200. Fix missing right bracket in HA doc (#1380) add dc5a997 HDDS-1889. Add support for verifying multiline log entry (#1308) add 79f9fab HDDS-3762. Intermittent failure in TestDeleteWithSlowFollower (#1376) add fd63aac HDDS-4150. Disabling flaky unit test until HDDS-4150 is fixed. add b2fca43 HDDS-4189. Add alias `roles` for `ozone admin om` subcommand `getserviceroles` (#1375) add c8d5334 HDDS-4199. Fix failed UT: TestOMAllocateBlockRequest#testValidateAndUpdateCache (#1379) add ead6371 HDDS-3840. Use OFS in MapReduce acceptance test (#1365) No new revisions were added by this update. Summary of changes: .github/close-pending.sh | 41 ++++ .github/closing-message.txt | 7 + .github/comment-commands/close.sh | 10 +- .github/comment-commands/pending.sh | 1 + .../{comments.yaml => close-pending.yaml} | 19 +- .github/workflows/post-commit.yml | 6 +- hadoop-hdds/client/pom.xml | 4 +- .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 30 +++ .../apache/hadoop/hdds/scm/XceiverClientRatis.java | 7 + hadoop-hdds/common/pom.xml | 9 +- .../org/apache/hadoop/hdds/cli/package-info.java | 4 +- .../apache/hadoop/hdds/scm/XceiverClientSpi.java | 11 + .../hdds/scm/storage/ContainerProtocolCalls.java | 34 +++ .../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 + .../hadoop/ozone/audit/TestOzoneAuditLogger.java | 62 ++++- hadoop-hdds/config/pom.xml | 4 +- hadoop-hdds/container-service/pom.xml | 11 +- .../commandhandler/DeleteBlocksCommandHandler.java | 2 +- .../ozone/container/ContainerTestHelper.java | 5 + .../content/concept/StorageContainerManager.md | 5 +- hadoop-hdds/docs/content/feature/HA.md | 2 +- hadoop-hdds/docs/content/interface/CSI.zh.md | 13 +- hadoop-hdds/docs/pom.xml | 4 +- hadoop-hdds/framework/pom.xml | 4 +- hadoop-hdds/hadoop-dependency-client/README.md | 18 +- hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +- hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +- hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +- hadoop-hdds/interface-admin/pom.xml | 4 +- hadoop-hdds/interface-client/pom.xml | 4 +- hadoop-hdds/interface-server/pom.xml | 4 +- hadoop-hdds/pom.xml | 4 +- hadoop-hdds/server-scm/pom.xml | 5 +- .../container/AbstractContainerReportHandler.java | 47 +++- .../hdds/scm/container/ReplicationManager.java | 34 --- .../scm/pipeline/BackgroundPipelineCreator.java | 33 ++- .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 35 +++ .../apache/hadoop/hdds/scm/server/SCMMXBean.java | 4 + .../hdds/scm/server/StorageContainerManager.java | 8 + .../main/resources/webapps/scm/scm-overview.html | 18 +- .../src/main/resources/webapps/scm/scm.js | 4 - .../hadoop/hdds/scm/container/MockNodeManager.java | 13 + .../scm/container/TestContainerReportHandler.java | 164 ++++++++++++- .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 5 +- .../scm/pipeline/TestPipelinePlacementPolicy.java | 81 ++++++ hadoop-hdds/test-utils/pom.xml | 4 +- hadoop-hdds/tools/pom.xml | 12 +- .../org/apache/hadoop/hdds/cli/OzoneAdmin.java | 67 +++++ .../org/apache/hadoop/hdds/cli}/package-info.java | 4 +- .../hdds/scm/cli/ReplicationManagerCommands.java | 23 +- .../scm/cli/ReplicationManagerStartSubcommand.java | 21 +- .../cli/ReplicationManagerStatusSubcommand.java | 32 +-- .../scm/cli/ReplicationManagerStopSubcommand.java | 25 +- .../hdds/scm/cli/SafeModeCheckSubcommand.java | 40 ++- .../hadoop/hdds/scm/cli/SafeModeCommands.java | 27 +- .../hdds/scm/cli/SafeModeExitSubcommand.java | 22 +- .../hdds/scm/cli/SafeModeWaitSubcommand.java | 13 +- .../org/apache/hadoop/hdds/scm/cli/ScmOption.java | 72 ++++++ .../apache/hadoop/hdds/scm/cli/ScmSubcommand.java | 24 +- .../hadoop/hdds/scm/cli/TopologySubcommand.java | 65 +++-- .../hdds/scm/cli/container/CloseSubcommand.java | 20 +- .../hdds/scm/cli/container/ContainerCommands.java | 21 +- .../hdds/scm/cli/container/CreateSubcommand.java | 26 +- .../hdds/scm/cli/container/DeleteSubcommand.java | 20 +- .../hdds/scm/cli/container/InfoSubcommand.java | 40 ++- .../hdds/scm/cli/container/ListSubcommand.java | 32 +-- .../hdds/scm/cli/container/WithScmClient.java | 29 --- .../hdds/scm/cli/datanode/DatanodeCommands.java | 21 +- .../hdds/scm/cli/datanode/ListInfoSubcommand.java | 48 ++-- .../cli/pipeline/ActivatePipelineSubcommand.java | 19 +- .../scm/cli/pipeline/ClosePipelineSubcommand.java | 19 +- .../scm/cli/pipeline/CreatePipelineSubcommand.java | 38 ++- .../cli/pipeline/DeactivatePipelineSubcommand.java | 19 +- .../scm/cli/pipeline/ListPipelinesSubcommand.java | 40 ++- .../hdds/scm/cli/pipeline/PipelineCommands.java | 22 +- hadoop-ozone/client/pom.xml | 4 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 10 + .../hadoop/ozone/client/io/OzoneInputStream.java | 5 + .../ozone/client/protocol/ClientProtocol.java | 12 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 13 + hadoop-ozone/common/pom.xml | 4 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 + .../org/apache/hadoop/ozone/audit/OMAction.java | 1 + .../hadoop/ozone/om/exceptions/OMException.java | 5 +- .../ozone/om/ha/OMFailoverProxyProvider.java | 16 +- .../{OmDeleteKeys.java => OmRenameKeys.java} | 28 ++- .../hadoop/ozone/om/helpers/OzoneFileStatus.java | 18 ++ .../ozone/om/protocol/OzoneManagerProtocol.java | 9 + ...OzoneManagerProtocolClientSideTranslatorPB.java | 32 +++ hadoop-ozone/csi/pom.xml | 4 +- hadoop-ozone/datanode/pom.xml | 4 +- hadoop-ozone/dev-support/checks/kubernetes.sh | 2 +- hadoop-ozone/dist/pom.xml | 4 +- .../main/compose/ozone-mr/hadoop27/docker-config | 1 + .../src/main/compose/ozone-mr/hadoop27/test.sh | 7 +- .../main/compose/ozone-mr/hadoop31/docker-config | 1 + .../src/main/compose/ozone-mr/hadoop31/test.sh | 7 +- .../main/compose/ozone-mr/hadoop32/docker-config | 1 + .../src/main/compose/ozone-mr/hadoop32/test.sh | 7 +- .../dist/src/main/compose/ozone-mr/test.sh | 22 +- .../src/main/compose/ozone-om-ha-s3/docker-config | 1 - .../src/main/compose/ozone-om-ha/docker-config | 1 - .../dist/src/main/compose/ozone/docker-config | 2 - .../src/main/compose/ozonesecure-mr/docker-config | 1 + .../dist/src/main/compose/ozonesecure-mr/test.sh | 5 +- .../main/compose/ozonesecure-om-ha/docker-config | 1 - .../src/main/compose/ozonesecure/docker-config | 2 - hadoop-ozone/dist/src/main/compose/test-all.sh | 21 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 40 ++- .../dist/src/main/compose/upgrade/README.md | 2 +- hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 2 +- .../src/main/compose/upgrade/versions/README.md | 2 +- .../main/compose/upgrade/versions/ozone-0.5.0.sh | 1 + .../versions/{ozone-0.6.0.sh => ozone-1.0.0.sh} | 1 + .../src/main/k8s/examples/getting-started/test.sh | 2 + .../dist/src/main/k8s/examples/minikube/test.sh | 2 + .../dist/src/main/k8s/examples/ozone-dev/test.sh | 2 + .../dist/src/main/k8s/examples/ozone/test.sh | 2 + .../dist/src/main/k8s/examples/test-all.sh | 11 +- hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 11 +- .../{topology/cli.robot => admincli/admin.robot} | 23 +- .../src/main/smoketest/admincli/container.robot | 68 ++++++ .../src/main/smoketest/admincli/datanode.robot | 19 +- .../src/main/smoketest/admincli/pipeline.robot | 49 +++- .../smoketest/admincli/replicationmanager.robot | 53 ++++ .../src/main/smoketest/admincli/safemode.robot | 45 ++++ .../dist/src/main/smoketest/createmrenv.robot | 13 +- .../src/main/smoketest/debug/ozone-debug.robot | 4 +- .../dist/src/main/smoketest/mapreduce.robot | 11 +- .../main/smoketest/om-ratis/testOMAdminCmd.robot | 2 +- .../dist/src/main/smoketest/omha/testOMHA.robot | 2 +- .../src/main/smoketest/ozonefs/hadoopo3fs.robot | 12 +- hadoop-ozone/dist/src/shell/ozone/ozone | 2 +- .../dist/src/shell/upgrade/{0.6.0.sh => 1.0.0.sh} | 2 +- .../upgrade/{0.6.0 => 1.0.0}/01-migrate-scm-db.sh | 0 .../fault-injection-test/mini-chaos-tests/pom.xml | 4 +- .../services/org.apache.hadoop.fs.FileSystem | 1 + .../fault-injection-test/network-tests/pom.xml | 2 +- hadoop-ozone/fault-injection-test/pom.xml | 4 +- hadoop-ozone/insight/pom.xml | 4 +- hadoop-ozone/integration-test/pom.xml | 4 +- .../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 15 +- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 22 +- .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 9 +- .../ozone/contract/rooted/RootedOzoneContract.java | 7 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 106 +++++++- .../ozone/client/rpc/TestWatchForCommit.java | 5 +- .../commandhandler/TestBlockDeletion.java | 5 +- .../org/apache/hadoop/ozone/om/TestOmLDBCli.java | 120 +++++++++ .../org/apache/hadoop/ozone/om/TestOmSQLCli.java | 235 ------------------ .../hadoop/ozone/shell/TestOzoneDatanodeShell.java | 2 +- .../hadoop/ozone/shell/TestOzoneShellHA.java | 3 +- hadoop-ozone/interface-client/pom.xml | 4 +- .../src/main/proto/OmClientProtocol.proto | 26 ++ hadoop-ozone/ozone-manager/pom.xml | 5 +- .../org/apache/hadoop/ozone/om/KeyManager.java | 11 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 5 +- .../apache/hadoop/ozone/om/OMMetadataManager.java | 10 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 35 ++- .../hadoop/ozone/om/OpenKeyCleanupService.java | 42 +--- .../org/apache/hadoop/ozone/om/OzoneManager.java | 12 +- .../om/ratis/utils/OzoneManagerRatisUtils.java | 3 + .../ozone/om/request/key/OMKeysRenameRequest.java | 271 +++++++++++++++++++++ ...nameResponse.java => OMKeysRenameResponse.java} | 55 +++-- .../hadoop/ozone/om/TestOmMetadataManager.java | 77 ++++++ .../ozone/om/request/TestOMRequestUtils.java | 33 ++- .../om/request/key/TestOMAllocateBlockRequest.java | 22 +- .../om/request/key/TestOMKeysRenameRequest.java | 160 ++++++++++++ .../om/response/key/TestOMKeysRenameResponse.java | 131 ++++++++++ hadoop-ozone/ozonefs-common/pom.xml | 4 +- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 71 ++++++ .../fs/ozone/BasicRootedOzoneFileSystem.java | 71 ++++++ .../services/org.apache.hadoop.fs.FileSystem | 1 + hadoop-ozone/ozonefs-hadoop2/pom.xml | 4 +- .../org/apache/hadoop/fs/ozone/RootedOzFs.java} | 12 +- .../services/org.apache.hadoop.fs.FileSystem | 1 + hadoop-ozone/ozonefs-hadoop3/pom.xml | 4 +- .../org/apache/hadoop/fs/ozone/RootedOzFs.java} | 12 +- .../services/org.apache.hadoop.fs.FileSystem | 1 + hadoop-ozone/ozonefs-shaded/pom.xml | 4 +- hadoop-ozone/ozonefs/pom.xml | 4 +- .../hadoop/fs/ozone/{OzFs.java => RootedOzFs.java} | 12 +- .../services/org.apache.hadoop.fs.FileSystem | 1 + hadoop-ozone/pom.xml | 4 +- hadoop-ozone/recon-codegen/pom.xml | 2 +- .../recon/schema/ContainerSchemaDefinition.java | 4 +- .../recon/schema/ReconTaskSchemaDefinition.java | 2 +- .../ozone/recon/schema/StatsSchemaDefinition.java | 2 +- .../recon/schema/UtilizationSchemaDefinition.java | 10 +- hadoop-ozone/recon/pom.xml | 2 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 2 + hadoop-ozone/s3gateway/pom.xml | 9 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 14 +- hadoop-ozone/tools/pom.xml | 6 +- .../org/apache/hadoop/ozone/admin/OzoneAdmin.java | 122 ---------- .../ozone/admin/om/GetServiceRolesSubcommand.java | 4 +- .../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 2 +- .../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 149 ++++++----- .../hadoop/ozone/debug/ContainerChunkInfo.java | 21 +- .../org/apache/hadoop/ozone/debug/DBScanner.java | 62 +++-- .../org/apache/hadoop/ozone/debug/RDBParser.java | 4 + .../hadoop/ozone/freon/ContentGenerator.java | 31 ++- .../hadoop/ozone/freon/DatanodeChunkValidator.java | 244 +++++++++++++++++++ .../java/org/apache/hadoop/ozone/freon/Freon.java | 1 + .../hadoop/ozone/freon/HadoopFsGenerator.java | 12 +- .../services/org.apache.hadoop.fs.FileSystem | 1 + .../hadoop/ozone/freon/TestContentGenerator.java | 82 +++++++ .../TestGenerateOzoneRequiredConfigurations.java | 5 +- pom.xml | 14 +- 209 files changed, 3412 insertions(+), 1345 deletions(-) create mode 100755 .github/close-pending.sh create mode 100644 .github/closing-message.txt copy .github/workflows/{comments.yaml => close-pending.yaml} (75%) create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/cli/OzoneAdmin.java copy hadoop-hdds/{test-utils/src/main/java/org/apache/hadoop/test => tools/src/main/java/org/apache/hadoop/hdds/cli}/package-info.java (92%) create mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmOption.java copy hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/volume/VolumeHandler.java => hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/ScmSubcommand.java (61%) delete mode 100644 hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/container/WithScmClient.java copy hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/{OmDeleteKeys.java => OmRenameKeys.java} (61%) rename hadoop-ozone/dist/src/main/compose/upgrade/versions/{ozone-0.6.0.sh => ozone-1.0.0.sh} (91%) copy hadoop-ozone/dist/src/main/smoketest/{topology/cli.robot => admincli/admin.robot} (58%) create mode 100644 hadoop-ozone/dist/src/main/smoketest/admincli/container.robot create mode 100644 hadoop-ozone/dist/src/main/smoketest/admincli/replicationmanager.robot create mode 100644 hadoop-ozone/dist/src/main/smoketest/admincli/safemode.robot rename hadoop-ozone/dist/src/shell/upgrade/{0.6.0.sh => 1.0.0.sh} (94%) rename hadoop-ozone/dist/src/shell/upgrade/{0.6.0 => 1.0.0}/01-migrate-scm-db.sh (100%) create mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmLDBCli.java delete mode 100644 hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmSQLCli.java create mode 100644 hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeysRenameRequest.java copy hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/{OMKeyRenameResponse.java => OMKeysRenameResponse.java} (56%) create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeysRenameRequest.java create mode 100644 hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeysRenameResponse.java copy hadoop-ozone/{ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzFs.java => ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/RootedOzFs.java} (81%) copy hadoop-ozone/{ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzFs.java => ozonefs-hadoop3/src/main/java/org/apache/hadoop/fs/ozone/RootedOzFs.java} (80%) copy hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/{OzFs.java => RootedOzFs.java} (80%) delete mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/admin/OzoneAdmin.java create mode 100644 hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/DatanodeChunkValidator.java create mode 100644 hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestContentGenerator.java --------------------------------------------------------------------- To unsubscribe, e-mail: ozone-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: ozone-commits-h...@hadoop.apache.org