This is an automated email from the ASF dual-hosted git repository. ChenSammi pushed a commit to branch HDDS-10685 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 1ea7e9e7985cf5d5938df4db253d6e78fada1a52 Merge: 277737474cd cca31ff7206 Author: Sammi Chen <[email protected]> AuthorDate: Wed Jul 22 17:30:44 2026 +0800 Merge remote-tracking branch 'origin/master' into HDDS-10685 .github/workflows/build-ratis.yml | 2 +- .github/workflows/check.yml | 2 +- .github/workflows/ci.yml | 2 +- .github/workflows/close-stale-prs.yaml | 2 +- .github/workflows/intermittent-test-check.yml | 4 +- .github/workflows/populate-cache.yml | 4 +- .github/workflows/repeat-acceptance.yml | 4 +- .run/CsiServer.run.xml | 33 - .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 40 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 16 +- .../hdds/scm/storage/ECBlockOutputStream.java | 25 +- .../hdds/scm/storage/StreamBlockInputStream.java | 9 +- .../TestXceiverClientManagerCloseContention.java | 161 + .../hdds/scm/client/TestHddsClientUtils.java | 37 + .../storage/TestBlockOutputStreamCorrectness.java | 52 +- .../scm/storage/TestStreamBlockInputStream.java | 156 + hadoop-hdds/common/src/main/conf/ozone-env.sh | 19 +- .../java/org/apache/hadoop/hdds/HddsUtils.java | 21 +- .../hadoop/hdds/conf/OzoneConfiguration.java | 1 - .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 3 + .../hdds/scm/storage/ContainerProtocolCalls.java | 35 +- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 9 + .../common/src/main/resources/ozone-default.xml | 117 + .../java/org/apache/hadoop/hdds/TestHddsUtils.java | 19 + .../container/common/helpers/ContainerUtils.java | 26 + .../ozone/container/common/impl/ContainerSet.java | 95 +- .../container/common/impl/HddsDispatcher.java | 16 +- .../common/interfaces/ContainerDispatcher.java | 6 +- .../ozone/container/common/interfaces/Handler.java | 4 +- .../server/ratis/ContainerStateMachine.java | 20 +- .../diskbalancer/DiskBalancerService.java | 17 +- .../ECReconstructionCoordinator.java | 3 +- .../ozone/container/keyvalue/KeyValueHandler.java | 15 +- .../keyvalue/impl/ChunkManagerDispatcher.java | 8 +- .../keyvalue/impl/FilePerBlockStrategy.java | 8 +- .../keyvalue/impl/KeyValueStreamDataChannel.java | 104 +- .../keyvalue/impl/StreamDataChannelBase.java | 16 + .../keyvalue/interfaces/ChunkManager.java | 5 +- .../StaleRecoveringContainerScrubbingService.java | 21 +- .../ContainerTableSchemaFinalizeAction.java | 2 +- ...stStaleRecoveringContainerScrubbingService.java | 50 + .../container/common/impl/TestHddsDispatcher.java | 75 + .../diskbalancer/TestDiskBalancerService.java | 43 + .../impl/TestKeyValueStreamDataChannel.java | 139 +- hadoop-hdds/docs/content/design/diskbalancer.md | 12 +- .../docs/content/design/lifecycle-task-resume.md | 80 + .../design/s3-object-lifecycle-management.md | 29 + hadoop-hdds/docs/content/feature/DiskBalancer.md | 37 +- .../docs/content/feature/DiskBalancer.zh.md | 34 +- hadoop-hdds/docs/content/feature/Lifecycle.md | 397 + hadoop-hdds/docs/content/feature/Lifecycle.zh.md | 396 + hadoop-hdds/docs/content/interface/CSI.md | 93 - hadoop-hdds/docs/content/interface/CSI.png | Bin 27210 -> 0 bytes hadoop-hdds/docs/content/interface/CSI.zh.md | 92 - hadoop-hdds/docs/content/tools/Admin.md | 4 +- hadoop-hdds/docs/content/tools/Repair.md | 4 +- .../org/apache/hadoop/hdds/utils/Archiver.java | 76 + .../hadoop/hdds/utils/DBCheckpointServlet.java | 10 +- .../hadoop/hdds/utils/RDBSnapshotProvider.java | 2 +- .../org/apache/hadoop/hdds/utils/TestArchiver.java | 40 + .../src/main/proto/ScmAdminProtocol.proto | 3 + .../interface-admin/src/main/resources/proto.lock | 18 + .../src/main/proto/DatanodeClientProtocol.proto | 4 + .../utils/{TestUtils.java => RocksTestUtils.java} | 4 +- .../utils/db/TestManagedRawSSTFileIterator.java | 4 +- .../hadoop/hdds/utils/db/TestSstFileSetReader.java | 6 +- .../container/AbstractContainerReportHandler.java | 26 +- .../hdds/scm/container/ContainerManager.java | 4 +- .../hdds/scm/container/ContainerManagerImpl.java | 5 +- .../hdds/scm/container/ContainerStateManager.java | 5 +- .../scm/container/ContainerStateManagerImpl.java | 5 +- .../scm/container/balancer/ContainerBalancer.java | 19 +- .../balancer/ContainerBalancerStatusInfo.java | 43 +- .../balancer/ContainerBalancerStopReason.java | 63 + .../container/balancer/ContainerBalancerTask.java | 143 +- .../SCMContainerPlacementRackScatter.java | 49 +- .../scm/container/states/ContainerStateMap.java | 49 +- .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 135 +- .../ha/invoker/ContainerStateManagerInvoker.java | 83 +- .../scm/pipeline/PipelineStateManagerImpl.java | 2 +- .../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 263 +- .../hdds/scm/server/SCMClientProtocolServer.java | 15 +- .../hdds/scm/server/StorageContainerManager.java | 9 +- .../scm/container/TestContainerStateManager.java | 2 +- .../container/balancer/TestContainerBalancer.java | 43 + .../balancer/TestContainerBalancerStatusInfo.java | 66 + .../balancer/TestContainerBalancerTask.java | 58 + .../TestSCMContainerPlacementRackScatter.java | 34 + .../container/states/TestContainerStateMap.java | 102 +- hadoop-ozone/cli-admin/pom.xml | 4 + .../scm/cli/ContainerBalancerStatusSubcommand.java | 133 +- .../datanode/AbstractDiskBalancerSubCommand.java | 162 +- .../hdds/scm/cli/datanode/DatanodeParameters.java | 3 +- .../scm/cli/datanode/DiskBalancerCommands.java | 31 +- .../cli/datanode/DiskBalancerCommonOptions.java | 12 + .../cli/datanode/DiskBalancerReportSubcommand.java | 35 +- .../cli/datanode/DiskBalancerStatusSubcommand.java | 46 +- .../cli/datanode/DiskBalancerSubCommandUtil.java | 139 +- .../ozone/admin/om/LifecycleResumeSubCommand.java | 73 + .../ozone/admin/om/LifecycleStatusSubCommand.java | 89 + .../ozone/admin/om/LifecycleSubCommand.java} | 37 +- .../ozone/admin/om/LifecycleSuspendSubCommand.java | 76 + .../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 3 +- .../ozone/admin/om/snapshot/DefragSubCommand.java | 11 +- .../datanode/TestContainerBalancerSubCommand.java | 242 +- .../datanode/TestDiskBalancerSubCommandUtil.java | 135 + .../cli/datanode/TestDiskBalancerSubCommands.java | 345 +- .../admin/om/snapshot/TestDefragSubCommand.java | 22 + .../container/analyze/AnalyzeSubcommand.java | 234 +- .../analyze/ScmContainerMetadataReader.java | 155 + .../ozone/debug/om/ContainerToKeyMapping.java | 19 +- .../apache/hadoop/ozone/fsck/ContainerMapper.java | 3 +- .../analyze/ContainerAnalyzeTestHelper.java | 37 + .../container/analyze/TestAnalyzeSubcommand.java | 188 +- .../analyze/TestScmContainerMetadataReader.java | 100 + .../schemaupgrade/UpgradeContainerSchema.java | 3 +- .../ozone/repair/ldb/RocksDBManualCompaction.java | 15 +- .../apache/hadoop/ozone/repair/om/CompactOMDB.java | 25 +- .../hadoop/ozone/repair/om/FSORepairTool.java | 25 +- .../hadoop/ozone/repair/ldb/TestLdbRepair.java | 2 +- .../hadoop/ozone/repair/om/TestCompactOMDB.java | 12 + .../hadoop/ozone/shell/keys/DeleteKeyHandler.java | 4 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 50 + .../ozone/client/OzoneLifecycleConfiguration.java | 230 + .../ozone/client/protocol/ClientProtocol.java | 67 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 60 +- .../hadoop/ozone/client/TestOzoneBucket.java | 72 + .../client/TestRpcClientGetFileStatusHeadOp.java | 93 + hadoop-ozone/common/pom.xml | 4 + .../apache/hadoop/fs/ozone/OzoneTrashPolicy.java | 2 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 12 + .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 65 + .../hadoop/ozone/om/exceptions/OMException.java | 3 + .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 71 +- .../OmLCAbortIncompleteMultipartUpload.java | 139 + .../apache/hadoop/ozone/om/helpers/OmLCAction.java | 59 + .../hadoop/ozone/om/helpers/OmLCExpiration.java | 215 + .../apache/hadoop/ozone/om/helpers/OmLCFilter.java | 235 + .../apache/hadoop/ozone/om/helpers/OmLCRule.java | 411 + .../ozone/om/helpers/OmLifecycleConfiguration.java | 334 + .../om/helpers/OmLifecycleRuleAndOperator.java | 214 + .../ozone/om/helpers/OmLifecycleScanState.java | 255 + .../hadoop/ozone/om/helpers/OmLifecycleUtils.java | 131 + .../hadoop/ozone/om/helpers/OzoneFSUtils.java | 72 + .../hadoop/ozone/om/helpers/RepeatedOmKeyInfo.java | 44 +- .../ozone/om/protocol/OzoneManagerProtocol.java | 65 + ...OzoneManagerProtocolClientSideTranslatorPB.java | 103 + .../apache/hadoop/ozone/om/helpers/OMLCUtils.java | 131 + .../TestOmLCAbortIncompleteMultipartUpload.java | 171 + .../ozone/om/helpers/TestOmLCExpiration.java | 198 + .../hadoop/ozone/om/helpers/TestOmLCFilter.java | 159 + .../hadoop/ozone/om/helpers/TestOmLCRule.java | 459 + .../om/helpers/TestOmLifeCycleConfiguration.java | 266 + .../om/helpers/TestOmLifecycleRuleAndOperator.java | 139 + .../ozone/om/helpers/TestOmLifecycleScanState.java | 79 + .../csi/dev-support/findbugsExcludeFile.xml | 22 - hadoop-ozone/csi/pom.xml | 290 - .../apache/hadoop/ozone/csi/ControllerService.java | 116 - .../org/apache/hadoop/ozone/csi/CsiServer.java | 177 - .../apache/hadoop/ozone/csi/IdentityService.java | 72 - .../org/apache/hadoop/ozone/csi/NodeService.java | 148 - hadoop-ozone/csi/src/main/proto/csi.proto | 1323 --- hadoop-ozone/csi/src/main/resources/proto.lock | 1479 --- .../dev-support/intellij/ozone-site-ha.xml | 10 +- hadoop-ozone/dev-support/intellij/ozone-site.xml | 8 - .../dist/dev-support/bin/dist-layout-stitching | 4 +- hadoop-ozone/dist/pom.xml | 5 - .../Datanode Chunk Read_Write Dashboard.json | 1601 ---- .../dashboards/Ozone - DataNode Overview.json | 9417 ++++++++++++++++++++ hadoop-ozone/dist/src/main/compose/ozone-csi/.env | 20 - .../src/main/compose/ozone-csi/docker-compose.yaml | 64 - .../dist/src/main/compose/ozone-csi/docker-config | 44 - .../dist/src/main/compose/ozone-ha/docker-config | 26 + .../dist/src/main/compose/ozone/docker-config | 14 +- .../src/main/compose/ozonesecure-ha/docker-config | 6 + .../test-om-lifecycle.sh} | 15 +- .../compose/ozonesecure-ha/test-repair-tools.sh | 2 +- .../src/main/compose/ozonesecure/docker-config | 26 + hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 7 +- .../dist/src/main/compose/xcompat/clients.yaml | 4 + hadoop-ozone/dist/src/main/compose/xcompat/lib.sh | 2 +- .../k8s/definitions/ozone-csi/csi-controller.yaml | 53 - .../main/k8s/definitions/ozone-csi/csi-crd.yaml | 21 - .../main/k8s/definitions/ozone-csi/csi-node.yaml | 95 - .../main/k8s/definitions/ozone-csi/csi-rbac.yaml | 66 - .../definitions/ozone-csi/csi-storageclass.yaml | 20 - .../k8s/definitions/ozone-csi/definitions/csi.yaml | 28 - .../k8s/definitions/test-webserver/flekszible.yaml | 16 - .../test-webserver/webserver-deployment.yaml | 50 - .../test-webserver/webserver-service.yaml | 29 - .../test-webserver/webserver-volume.yaml | 29 - .../src/main/k8s/examples/ozone-dev/Flekszible | 7 - .../k8s/examples/ozone-dev/config-configmap.yaml | 6 +- .../examples/ozone-dev/csi/csi-node-daemonset.yaml | 97 - .../ozone-dev/csi/csi-ozone-clusterrole.yaml | 98 - .../csi/csi-ozone-clusterrolebinding.yaml | 28 - .../ozone-dev/csi/csi-ozone-serviceaccount.yaml | 21 - .../ozone-dev/csi/csi-provisioner-deployment.yaml | 54 - .../csi/org.apache.hadoop.ozone-csidriver.yaml | 22 - .../examples/ozone-dev/csi/ozone-storageclass.yaml | 21 - .../ozone-csi-test-webserver-deployment.yaml | 50 - ...e-csi-test-webserver-persistentvolumeclaim.yaml | 29 - .../pv-test/ozone-csi-test-webserver-service.yaml | 29 - .../dist/src/main/k8s/examples/ozone/Flekszible | 5 - .../main/k8s/examples/ozone/config-configmap.yaml | 6 +- .../k8s/examples/ozone/csi/csi-node-daemonset.yaml | 97 - .../examples/ozone/csi/csi-ozone-clusterrole.yaml | 98 - .../ozone/csi/csi-ozone-clusterrolebinding.yaml | 28 - .../ozone/csi/csi-ozone-serviceaccount.yaml | 21 - .../ozone/csi/csi-provisioner-deployment.yaml | 54 - .../csi/org.apache.hadoop.ozone-csidriver.yaml | 22 - .../k8s/examples/ozone/csi/ozone-storageclass.yaml | 21 - .../ozone-csi-test-webserver-deployment.yaml | 50 - ...e-csi-test-webserver-persistentvolumeclaim.yaml | 29 - .../pv-test/ozone-csi-test-webserver-service.yaml | 29 - hadoop-ozone/dist/src/main/license/jar-report.txt | 2 - .../src/main/smoketest/balancer/testBalancer.robot | 2 +- .../main/smoketest/lifecycle/om-lifecycle.robot | 119 + .../src/main/smoketest/s3/bucketlifecycle.robot | 57 + .../dist/src/main/smoketest/s3/objectputget.robot | 10 +- hadoop-ozone/dist/src/shell/ozone/ozone | 9 +- .../dist/src/shell/ozone/ozone-functions.sh | 25 +- .../csi.robot => test/shell/ozone_bootstrap.bats} | 43 +- hadoop-ozone/iceberg/pom.xml | 2 +- .../AbstractTestStorageDistributionEndpoint.java | 14 +- ...ndpointUtil.java => ReconEndpointTestUtil.java} | 6 +- .../ozone/recon/TestReconAndAdminContainerCLI.java | 36 +- .../ozone/recon/TestReconWithOzoneManagerHA.java | 9 - hadoop-ozone/integration-test-s3/pom.xml | 12 +- .../hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java | 11 + .../hadoop/ozone/s3/awssdk/S3SDKTestUtils.java | 59 + .../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 508 +- .../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 475 +- hadoop-ozone/integration-test/pom.xml | 5 - .../fs/ozone/AbstractOzoneFileSystemTest.java | 44 + .../ozone/AbstractRootedOzoneFileSystemTest.java | 89 +- .../AbstractRootedOzoneFileSystemTestWithFSO.java | 3 +- .../java/org/apache/hadoop/fs/ozone/TestHSync.java | 4 +- .../fs/ozone/TestOFSIsDirectoryBenchmark.java | 153 + .../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 6 +- .../hdds/scm/TestContainerReportWithKeys.java | 7 +- .../TestSCMFollowerCatchupWithContainerReport.java | 336 + .../hdds/scm/TestStorageContainerManager.java | 4 +- .../TestReplicationManagerIntegration.java | 8 +- .../node/{TestNodeUtil.java => NodeTestUtil.java} | 4 +- .../scm/node/TestDecommissionAndMaintenance.java | 16 +- .../hdds/scm/safemode/TestSafeModeSCMHA.java | 18 +- ...UpgradeUtils.java => HddsUpgradeTestUtils.java} | 6 +- .../TestDNDataDistributionFinalization.java | 4 +- .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 24 +- .../TestScmDataDistributionFinalization.java | 12 +- .../hadoop/hdds/upgrade/TestScmHAFinalization.java | 20 +- .../java/org/apache/hadoop/ozone/TestDataUtil.java | 9 +- .../ozone/client/rpc/OzoneRpcClientTests.java | 224 +- .../client/rpc/TestFailureHandlingByClient.java | 13 +- .../client/rpc/TestOzoneAtRestEncryption.java | 2 +- .../rpc/TestOzoneClientMultipartUploadWithFSO.java | 67 +- .../rpc/TestOzoneRpcClientForAclAuditLog.java | 6 +- .../client/rpc/read/TestChunkInputStream.java | 10 +- .../ozone/client/rpc/read/TestKeyInputStream.java | 28 +- .../client/rpc/read/TestLocalChunkInputStream.java | 6 +- .../rpc/read/TestStreamBlockInputStream.java | 10 +- .../ozone/client/rpc/read/TestStreamRead.java | 6 +- .../rpc/read/TestStreamReadDatanodeFailover.java | 4 +- .../commandhandler/TestDeleteContainerHandler.java | 2 +- .../ozone/local/TestLocalOzoneClusterRuntime.java | 32 +- .../ozone/om/AbstractOzoneManagerHATest.java | 11 +- .../om/{TestBucket.java => BucketForTesting.java} | 16 +- .../hadoop/ozone/om/TestAddRemoveOzoneManager.java | 6 +- .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 12 +- .../hadoop/ozone/om/TestListKeysWithFSO.java | 12 +- .../apache/hadoop/ozone/om/TestOMBootstrap.java | 5 +- .../hadoop/ozone/om/TestOMDbCheckpointServlet.java | 35 +- .../TestOMDbCheckpointServletInodeBasedXfer.java | 80 +- .../ozone/om/TestOMRatisSnapshotTransfer.java | 5 +- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 5 +- .../hadoop/ozone/om/TestObjectStoreWithFSO.java | 8 +- .../hadoop/ozone/om/TestOmBlockVersioning.java | 13 +- .../org/apache/hadoop/ozone/om/TestOmMetrics.java | 14 +- ...stOzoneManagerHAFollowerReadWithAllRunning.java | 7 +- .../ozone/om/TestOzoneManagerHAWithAllRunning.java | 13 +- .../om/TestOzoneManagerHAWithStoppedNodes.java | 9 +- .../hadoop/ozone/om/TestOzoneManagerRestart.java | 20 +- .../hadoop/ozone/om/TestRecursiveAclWithFSO.java | 6 +- .../apache/hadoop/ozone/om/TestScmSafeMode.java | 8 +- .../ozone/om/service/TestBlockDeletionService.java | 4 +- .../ozone/om/service/TestRootedDDSWithFSO.java | 9 +- ...TestSnapshotDeletingServiceIntegrationTest.java | 8 +- .../hadoop/ozone/om/snapshot/OmSnapshotTests.java | 43 +- .../ozone/om/snapshot/TestOMDBCheckpointUtils.java | 17 + .../ozone/om/snapshot/TestOmSnapshotDisabled.java | 8 +- .../om/snapshot/TestOmSnapshotDisabledRestart.java | 8 +- .../om/snapshot/TestOzoneManagerHASnapshot.java | 25 +- .../snapshot/TestOzoneManagerSnapshotProvider.java | 5 +- .../hadoop/ozone/scm/node/TestDiskBalancer.java | 4 +- ...skBalancerDuringDecommissionAndMaintenance.java | 4 +- .../hadoop/ozone/shell/TestOzoneShellHA.java | 76 + .../src/main/proto/OmClientProtocol.proto | 177 +- .../apache/hadoop/ozone/om/OMMetadataManager.java | 30 +- .../ozone/om/helpers/TestOmKeyInfoCodec.java | 132 +- .../om/helpers/TestRepeatedOmKeyInfoCodec.java | 109 +- .../org/apache/hadoop/ozone/audit/OMAction.java | 8 +- .../org/apache/hadoop/ozone/om/KeyManager.java | 11 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 74 +- .../org/apache/hadoop/ozone/om/ListIterator.java | 3 +- .../hadoop/ozone/om/OMDBCheckpointServlet.java | 67 +- .../om/OMDBCheckpointServletInodeBasedXfer.java | 28 +- .../java/org/apache/hadoop/ozone/om/OMMetrics.java | 56 + .../hadoop/ozone/om/OMMultiTenantManagerImpl.java | 4 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 99 +- .../apache/hadoop/ozone/om/OmSnapshotManager.java | 4 + .../org/apache/hadoop/ozone/om/OzoneManager.java | 92 +- .../hadoop/ozone/om/SnapshotChainManager.java | 2 +- .../hadoop/ozone/om/SstFilteringService.java | 3 +- .../hadoop/ozone/om/TrashOzoneFileSystem.java | 1 + .../hadoop/ozone/om/codec/OMDBDefinition.java | 70 +- .../hadoop/ozone/om/helpers/OMAuditLogger.java | 1 + .../om/ratis/utils/OzoneManagerRatisUtils.java | 12 + .../om/ratis_snapshot/OmRatisSnapshotProvider.java | 300 +- .../hadoop/ozone/om/request/OMClientRequest.java | 39 +- .../ozone/om/request/OMClientRequestUtils.java | 2 +- .../om/request/bucket/OMBucketDeleteRequest.java | 18 +- .../file/OMDirectoryCreateRequestWithFSO.java | 4 +- .../ozone/om/request/file/OMFileRequest.java | 2 +- .../om/request/key/OMAllocateBlockRequest.java | 4 +- .../key/OMDirectoriesPurgeRequestWithFSO.java | 2 + .../ozone/om/request/key/OMKeyCommitRequest.java | 3 - .../om/request/key/OMKeyCommitRequestWithFSO.java | 4 +- .../ozone/om/request/key/OMKeyDeleteRequest.java | 2 +- .../ozone/om/request/key/OMKeyRenameRequest.java | 2 +- .../om/request/key/OMKeyRenameRequestWithFSO.java | 8 + .../ozone/om/request/key/OMKeysDeleteRequest.java | 109 +- .../om/request/key/OmKeysDeleteRequestWithFSO.java | 6 +- .../OMLifecycleConfigurationDeleteRequest.java | 209 + .../OMLifecycleConfigurationSetRequest.java | 238 + .../lifecycle/OMLifecycleSaveScanStateRequest.java | 74 + .../OMLifecycleSetServiceStatusRequest.java | 125 + .../ozone/om/request/lifecycle}/package-info.java | 4 +- .../om/request/util/OMMultipartUploadUtils.java | 3 +- .../om/request/volume/OMQuotaRepairRequest.java | 2 +- .../om/response/bucket/OMBucketDeleteResponse.java | 11 +- .../om/response/key/OMKeysDeleteResponse.java | 17 +- .../response/key/OMKeysDeleteResponseWithFSO.java | 15 +- .../OMLifecycleConfigurationDeleteResponse.java | 64 + .../OMLifecycleConfigurationSetResponse.java | 67 + .../OMLifecycleSaveScanStateResponse.java | 51 + .../OMLifecycleSetServiceStatusResponse.java | 46 + .../ozone/om/response/lifecycle}/package-info.java | 4 +- .../hadoop/ozone/om/service/CompactDBUtil.java | 19 + .../hadoop/ozone/om/service/CompactionService.java | 12 +- .../ozone/om/service/DirectoryDeletingService.java | 5 +- .../ozone/om/service/KeyLifecycleService.java | 2131 +++++ .../om/service/KeyLifecycleServiceMetrics.java | 196 + .../ozone/om/service/OMRangerBGSyncService.java | 4 +- .../om/snapshot/FSODirectoryPathResolver.java | 3 +- .../ozone/om/snapshot/OMDBCheckpointUtils.java | 74 +- .../om/snapshot/defrag/SnapshotDefragService.java | 35 +- .../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 3 +- .../protocolPB/OzoneManagerRequestHandler.java | 47 +- .../apache/hadoop/ozone/om/FaultInjectorImpl.java | 98 + .../hadoop/ozone/om/TestOmMetadataManager.java | 39 +- .../TestOmRatisSnapshotProvider.java | 59 + .../TestOMClientRequestUserInfoFallback.java | 106 - .../om/request/key/TestOMKeyCommitRequest.java | 6 +- .../om/request/key/TestOMKeysDeleteRequest.java | 158 +- .../key/TestOMKeysDeleteRequestWithFSO.java | 77 +- .../TestOMLifecycleConfigurationDeleteRequest.java | 300 + .../TestOMLifecycleConfigurationRequest.java | 153 + .../TestOMLifecycleConfigurationSetRequest.java | 390 + .../TestOMLifecycleSaveScanStateRequest.java | 118 + .../TestS3MultipartUploadCompleteRequest.java | 2 +- .../snapshot/TestOMSnapshotCreateRequest.java | 6 +- .../ozone/om/response/TestCleanupTableInfo.java | 2 + .../om/response/key/TestOMKeyCommitResponse.java | 2 +- .../om/response/key/TestOMKeyDeleteResponse.java | 2 +- .../om/response/key/TestOMKeysDeleteResponse.java | 2 +- .../key/TestOMKeysDeleteResponseWithFSO.java | 2 +- ...TestOMLifecycleConfigurationDeleteResponse.java | 117 + .../TestOMLifecycleConfigurationSetResponse.java | 124 + .../TestOMLifecycleSaveScanStateResponse.java | 61 + .../TestS3MultipartUploadCommitPartResponse.java | 2 +- .../hadoop/ozone/om/service/TestCompactDBUtil.java | 92 + .../ozone/om/service/TestCompactionService.java | 33 + .../ozone/om/service/TestKeyDeletingService.java | 2 +- .../ozone/om/service/TestKeyLifecycleService.java | 3343 +++++++ .../om/snapshot/TestFSODirectoryPathResolver.java | 2 +- .../ozone/om/snapshot/TestSnapshotDiffManager.java | 2 + .../om/snapshot/TestSnapshotDiffValueParser.java | 22 +- .../snapshot/defrag/TestSnapshotDefragService.java | 216 +- .../protocolPB/TestOzoneManagerRequestHandler.java | 83 + .../fs/ozone/BasicOzoneClientAdapterImpl.java | 29 +- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 125 +- .../ozone/BasicRootedOzoneClientAdapterImpl.java | 73 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 67 +- .../apache/hadoop/fs/ozone/FileStatusAdapter.java | 20 + .../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 11 + .../ozone/TestBasicOzoneClientAdapterHeadOp.java | 136 + .../TestBasicRootedOzoneClientAdapterHeadOp.java | 176 + .../hadoop/fs/ozone/TestOzoneFileSystemHeadOp.java | 149 + .../fs/ozone/TestRootedOzoneFileSystemHeadOp.java | 174 + .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 8 + .../hadoop/fs/ozone/RootedOzoneFileSystem.java | 8 + .../apache/hadoop/fs/ozone/OzoneFileSystem.java | 8 + .../hadoop/fs/ozone/RootedOzoneFileSystem.java | 8 + hadoop-ozone/pom.xml | 1 - .../org/apache/hadoop/ozone/recon/ReconUtils.java | 2 +- .../ozone/recon/api/OMDBInsightEndpoint.java | 3 +- .../ozone/recon/api/ReconGlobalMetricsService.java | 4 +- .../ozone/recon/api/handlers/FSOBucketHandler.java | 3 +- .../recon/api/handlers/LegacyBucketHandler.java | 5 +- .../ozone/recon/api/handlers/OBSBucketHandler.java | 8 +- .../ozone/recon/chatbot/ChatbotConfigKeys.java | 20 + .../recon/chatbot/llm/LangChain4jDispatcher.java | 27 + .../recon/recovery/ReconOmMetadataManagerImpl.java | 8 +- .../hadoop/ozone/recon/scm/ReconNodeManager.java | 6 +- .../scm/ReconStorageContainerManagerFacade.java | 3 +- .../spi/impl/OzoneManagerServiceProviderImpl.java | 204 +- .../impl/ReconContainerMetadataManagerImpl.java | 8 +- .../ozone/recon/spi/impl/ReconDBProvider.java | 3 +- .../recon/spi/impl/ReconRDBSnapshotProvider.java | 245 + .../recon/tasks/ContainerKeyMapperHelper.java | 2 +- .../recon/tasks/DeletedKeysInsightHandler.java | 2 +- .../recon/tasks/MultipartInfoInsightHandler.java | 2 +- .../ozone/recon/tasks/NSSummaryTaskWithLegacy.java | 3 +- .../ozone/recon/tasks/OpenKeysInsightHandler.java | 2 +- .../tasks/util/ParallelTableIteratorOperation.java | 4 +- .../ozone/recon/api/TestTriggerDBSyncEndpoint.java | 1 + .../chatbot/llm/TestLangChain4jDispatcher.java | 57 + .../ozone/recon/chatbot/llm/TestLlmRouting.java | 8 + .../impl/TestOzoneManagerServiceProviderImpl.java | 281 +- .../spi/impl/TestReconRDBSnapshotProvider.java | 223 + .../ozone/recon/tasks/TestFileSizeCountTask.java | 2 +- .../org/apache/hadoop/ozone/audit/S3GAction.java | 3 + .../ozone/s3/commontypes/ObjectKeyNameAdapter.java | 3 +- .../ozone/s3/commontypes/RequestParameters.java | 11 + .../ozone/s3/endpoint/BucketCrudHandler.java | 127 + .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 12 +- .../ozone/s3/endpoint/ListBucketResponse.java | 13 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 35 +- ...utBucketLifecycleConfigurationUnmarshaller.java | 80 + .../hadoop/ozone/s3/endpoint/RootEndpoint.java | 41 +- .../s3/endpoint/S3LifecycleConfiguration.java | 557 ++ .../hadoop/ozone/s3/exception/S3ErrorTable.java | 3 + .../ozone/s3/signature/AWSSignatureProcessor.java | 7 +- .../signature/AccessDeniedResourceException.java} | 11 +- .../s3/signature/AuthorizationV4QueryParser.java | 7 +- .../org/apache/hadoop/ozone/s3/util/S3Consts.java | 4 + .../org/apache/hadoop/ozone/s3/util/S3Utils.java | 11 + .../hadoop/ozone/client/ClientProtocolStub.java | 32 +- .../hadoop/ozone/client/OzoneBucketStub.java | 68 + .../hadoop/ozone/client/OzoneVolumeStub.java | 1 + .../s3/commontypes/TestObjectKeyNameAdapter.java | 2 +- .../hadoop/ozone/s3/endpoint/TestBucketList.java | 38 +- .../hadoop/ozone/s3/endpoint/TestObjectGet.java | 26 + .../hadoop/ozone/s3/endpoint/TestObjectHead.java | 10 + .../hadoop/ozone/s3/endpoint/TestRootList.java | 94 + .../TestS3LifecycleConfigurationDelete.java | 100 + .../endpoint/TestS3LifecycleConfigurationGet.java | 173 + .../endpoint/TestS3LifecycleConfigurationPut.java | 605 ++ .../s3/signature/TestAWSSignatureProcessor.java | 37 + .../signature/TestAuthorizationV4QueryParser.java | 16 +- hadoop-ozone/tools/pom.xml | 4 + .../hadoop/ozone/local/LocalOzoneCluster.java | 228 +- .../hadoop/ozone/local/TestLocalOzoneCluster.java | 109 + pom.xml | 40 +- 465 files changed, 36494 insertions(+), 9061 deletions(-) diff --cc hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java index d49536f822c,6882595928d..bf8374f5e3a --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java @@@ -91,8 -90,7 +90,7 @@@ import org.slf4j.LoggerFactory * how it works, and how it is integrated with the Ozone client. */ public class XceiverClientGrpc extends XceiverClientSpi { - private static final Logger LOG = LoggerFactory.getLogger(XceiverClientGrpc.class); + public static final Logger LOG = LoggerFactory.getLogger(XceiverClientGrpc.class); - private static final int SHUTDOWN_WAIT_INTERVAL_MILLIS = 100; private static final int SHUTDOWN_WAIT_MAX_SECONDS = 5; private final Pipeline pipeline; private final ConfigurationSource config; diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java index 70185f7f0f8,6fe253ffbc7..d6828423a36 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Handler.java @@@ -45,9 -44,9 +45,10 @@@ import org.apache.hadoop.ozone.containe import org.apache.hadoop.ozone.container.common.volume.VolumeSet; import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler; import org.apache.hadoop.ozone.container.keyvalue.TarContainerPacker; +import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; import org.apache.ratis.statemachine.StateMachine; import org.apache.ratis.thirdparty.io.grpc.stub.StreamObserver; + import org.apache.ratis.util.function.CheckedConsumer; /** * Dispatcher sends ContainerCommandRequests to Handler. Each Container Type diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java index 11806813aaf,01000883f17..ecaf42a2059 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/FilePerBlockStrategy.java @@@ -58,8 -57,8 +58,9 @@@ import org.apache.hadoop.ozone.containe import org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils; import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager; import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager; +import org.apache.hadoop.util.Shell; import org.apache.ratis.statemachine.StateMachine; + import org.apache.ratis.util.function.CheckedConsumer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --cc hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh index 2edab9725b2,1609e46d4dd..3260f677807 --- a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh +++ b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh @@@ -1334,10 -1334,7 +1334,8 @@@ function ozone_add_to_classpath_userpat ## @return may exit on failure conditions function ozone_os_tricks { - local bindv6only - OZONE_IS_CYGWIN=false + ARCH_TYPE=$(uname -m) case ${OZONE_OS_TYPE} in Darwin) if [[ -z "${JAVA_HOME}" ]]; then @@@ -1376,47 -1354,6 +1374,28 @@@ # with the many threads that we use in Hadoop. Tune the variable # down to prevent vmem explosion. export MALLOC_ARENA_MAX=${MALLOC_ARENA_MAX:-4} - # we put this in QA test mode off so that non-Linux can test - if [[ "${QATESTMODE}" = true ]]; then - return - fi - - # NOTE! OZONE_ALLOW_IPV6 is a developer hook. We leave it - # undocumented in ozone-env.sh because we don't want users to - # shoot themselves in the foot while devs make IPv6 work. - - bindv6only=$(/sbin/sysctl -n net.ipv6.bindv6only 2> /dev/null) - - if [[ -n "${bindv6only}" ]] && - [[ "${bindv6only}" -eq "1" ]] && - [[ "${OZONE_ALLOW_IPV6}" != "yes" ]]; then - ozone_error "ERROR: \"net.ipv6.bindv6only\" is set to 1 " - ozone_error "ERROR: Hadoop networking could be broken. Aborting." - ozone_error "ERROR: For more info: http://wiki.apache.org/hadoop/HadoopIPv6"; - exit 1 - fi + + if [ "$ARCH_TYPE" = "aarch64" ]; then + TARGET_FILE="libhadoop_linux_aarch_64.so" + elif [ "$ARCH_TYPE" = "x86_64" ]; then + TARGET_FILE="libhadoop_linux_x86_64.so" + fi + + pushd . > /dev/null && cd lib/native + # If no matching file variant was found for the current environment + if [ -z "$TARGET_FILE" ]; then + echo "Error: libhadoop doesn't support platform combination ($OS_TYPE / $ARCH_TYPE)." >&2 + else + LINK_FILE="libhadoop.so" + + # Check if it already exists but points to the wrong target + if [ -L "$LINK_FILE" ] && [ "$(readlink "$LINK_FILE")" != "$TARGET_FILE" ]; then + # Forcefully recreate it so it points to the correct target file + ln -sf "$TARGET_FILE" "$LINK_FILE" > /dev/null 2>&1 + fi + export LD_LIBRARY_PATH=$OZONE_HOME/lib/native:$LD_LIBRARY_PATH + fi + popd > /dev/null ;; CYGWIN*) # Flag that we're running on Cygwin to trigger path translation later. diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java index 09d0b1a6adf,8c7daace078..7caf91294d7 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestDNDataDistributionFinalization.java @@@ -178,8 -180,11 +178,8 @@@ public class TestDNDataDistributionFina // Wait for finalization to complete finalizationFuture.get(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); + HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); - - // Verify finalization completed - assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); + assertTrue(VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION)); // Create more data and deletions to test post-finalization behavior String keyName3 = "testKey3"; @@@ -221,8 -226,10 +221,8 @@@ }); // Wait for finalization finalizationFuture.get(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); + HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); - - assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); + assertTrue(VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION)); // Verify the system can handle scenarios where pendingDeleteBlockCount // might be missing and needs recalculation diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java index 8ffad9279bf,3fb1bc121af..d0820197fb9 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestScmDataDistributionFinalization.java @@@ -197,14 -195,15 +197,14 @@@ public class TestScmDataDistributionFin assertEquals(EMPTY_SUMMARY, cluster.getStorageContainerLocationClient().getDeletedBlockSummary()); finalizationFuture.get(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); + HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); // Make sure old leader has caught up and all SCMs have finalized. waitForScmsToFinalize(cluster.getStorageContainerManagersList()); - assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); + assertTrue(VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION)); - TestHddsUpgradeUtils.testPostUpgradeConditionsSCM( + HddsUpgradeTestUtils.testPostUpgradeConditionsSCM( cluster.getStorageContainerManagersList(), 0, NUM_DATANODES); - TestHddsUpgradeUtils.testPostUpgradeConditionsDataNodes( + HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes( cluster.getHddsDatanodes(), 0, CLOSED); assertNotNull(cluster.getStorageContainerLocationClient().getDeletedBlockSummary()); @@@ -310,14 -309,15 +310,14 @@@ } }); finalizationFuture.get(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); + HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); // Make sure old leader has caught up and all SCMs have finalized. waitForScmsToFinalize(cluster.getStorageContainerManagersList()); - assertEquals(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION.layoutVersion(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); + assertTrue(VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION)); - TestHddsUpgradeUtils.testPostUpgradeConditionsSCM( + HddsUpgradeTestUtils.testPostUpgradeConditionsSCM( cluster.getStorageContainerManagersList(), 0, NUM_DATANODES); - TestHddsUpgradeUtils.testPostUpgradeConditionsDataNodes( + HddsUpgradeTestUtils.testPostUpgradeConditionsDataNodes( cluster.getHddsDatanodes(), 0, CLOSED); assertNotNull(cluster.getStorageContainerLocationClient().getDeletedBlockSummary()); diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestChunkInputStream.java index b2753c0da60,4dce1e8b433..4db70817f7e --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestChunkInputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestChunkInputStream.java @@@ -61,10 -61,10 +61,10 @@@ class TestChunkInputStream extends Inpu * Test to verify that data read from chunks is stored in a list of buffers * with max capacity equal to the bytes per checksum. */ - protected void testChunkReadBuffers(TestBucket bucket) throws Exception { - private void testChunkReadBuffers(BucketForTesting bucket) throws Exception { ++ protected void testChunkReadBuffers(BucketForTesting bucket) throws Exception { String keyName = getNewKeyName(); int dataLength = (2 * BLOCK_SIZE) + (CHUNK_SIZE); - byte[] inputData = bucket.writeRandomBytes(keyName, dataLength); + byte[] inputData = bucket.writeRandomBytes(keyName, getRepConfig(), dataLength); try (KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName)) { @@@ -123,9 -123,9 +123,9 @@@ } } - protected void testCloseReleasesBuffers(TestBucket bucket) throws Exception { - private void testCloseReleasesBuffers(BucketForTesting bucket) throws Exception { ++ protected void testCloseReleasesBuffers(BucketForTesting bucket) throws Exception { String keyName = getNewKeyName(); - bucket.writeRandomBytes(keyName, CHUNK_SIZE); + bucket.writeRandomBytes(keyName, getRepConfig(), CHUNK_SIZE); try (KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName)) { BlockInputStream block0Stream = @@@ -146,9 -146,9 +146,9 @@@ * Test that ChunkInputStream buffers are released as soon as the last byte * of the buffer is read. */ - protected void testBufferRelease(TestBucket bucket) throws Exception { - private void testBufferRelease(BucketForTesting bucket) throws Exception { ++ protected void testBufferRelease(BucketForTesting bucket) throws Exception { String keyName = getNewKeyName(); - byte[] inputData = bucket.writeRandomBytes(keyName, CHUNK_SIZE); + byte[] inputData = bucket.writeRandomBytes(keyName, getRepConfig(), CHUNK_SIZE); try (KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName)) { diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestLocalChunkInputStream.java index f72ddfcae2f,00000000000..ccf366beaaa mode 100644,000000..100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestLocalChunkInputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestLocalChunkInputStream.java @@@ -1,163 -1,0 +1,163 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.client.rpc.read; + +import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import java.io.File; +import java.io.IOException; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.scm.OzoneClientConfig; +import org.apache.hadoop.hdds.scm.XceiverClientGrpc; +import org.apache.hadoop.hdds.scm.XceiverClientShortCircuit; +import org.apache.hadoop.hdds.scm.storage.BlockInputStream; +import org.apache.hadoop.hdds.scm.storage.DomainSocketFactory; +import org.apache.hadoop.hdds.scm.storage.LocalChunkInputStream; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.io.KeyInputStream; +import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion; +import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerSpi; +import org.apache.hadoop.ozone.container.keyvalue.ContainerLayoutTestInfo; - import org.apache.hadoop.ozone.om.TestBucket; ++import org.apache.hadoop.ozone.om.BucketForTesting; +import org.apache.ozone.test.GenericTestUtils; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.io.TempDir; +import org.slf4j.event.Level; + +/** + * Tests {@link LocalChunkInputStream}. + * For local intellij run, please follow the steps below: + * Add Environment variables + * LD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib + * DYLD_LIBRARY_PATH=$PROJECT_DIR$/target/native-lib + * to intellij run configuration. + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +public class TestLocalChunkInputStream extends TestChunkInputStream { + + @TempDir + private File dir; + + @Override + int getDatanodeCount() { + return 1; + } + + @Override + void setCustomizedProperties(OzoneConfiguration configuration) { + OzoneClientConfig clientConfig = configuration.getObject(OzoneClientConfig.class); + clientConfig.setShortCircuit(true); + configuration.setFromObject(clientConfig); + configuration.set(OzoneClientConfig.OZONE_DOMAIN_SOCKET_PATH, + new File(dir, "ozone-socket").getAbsolutePath()); + GenericTestUtils.setLogLevel(XceiverClientShortCircuit.LOG, Level.DEBUG); + GenericTestUtils.setLogLevel(XceiverClientGrpc.LOG, Level.DEBUG); + GenericTestUtils.setLogLevel(LocalChunkInputStream.LOG, Level.DEBUG); + GenericTestUtils.setLogLevel(BlockInputStream.LOG, Level.DEBUG); + } + + @Override + ReplicationConfig getRepConfig() { + return RatisReplicationConfig.getInstance(ONE); + } + + + /** + * Run the tests as a single test method to avoid needing a new mini-cluster + * for each test. + */ + @ContainerLayoutTestInfo.ContainerTest + @Override + void testAll(ContainerLayoutVersion layout) throws Exception { + try (OzoneClient client = getCluster().newClient()) { + updateConfig(layout); + assumeTrue(DomainSocketFactory.getInstance(getCluster().getConf()).isServiceReady()); + - TestBucket bucket = TestBucket.newBuilder(client).build(); ++ BucketForTesting bucket = BucketForTesting.newBuilder(client).build(); + GenericTestUtils.LogCapturer logCapturer1 = + GenericTestUtils.LogCapturer.captureLogs(LocalChunkInputStream.LOG); + GenericTestUtils.LogCapturer logCapturer2 = + GenericTestUtils.LogCapturer.captureLogs(XceiverClientShortCircuit.LOG); + GenericTestUtils.LogCapturer logCapturer3 = + GenericTestUtils.LogCapturer.captureLogs(BlockInputStream.LOG); + GenericTestUtils.LogCapturer logCapturer4 = + GenericTestUtils.LogCapturer.captureLogs(XceiverClientGrpc.LOG); + testChunkReadBuffers(bucket); + testBufferRelease(bucket); + testCloseReleasesBuffers(bucket); + assertTrue(logCapturer1.getOutput().contains("LocalChunkInputStream is created")); + assertTrue(logCapturer2.getOutput().contains("XceiverClientShortCircuit is created")); + assertTrue((logCapturer3.getOutput().contains("Get the FileInputStream of block"))); + assertFalse(logCapturer4.getOutput().contains("XceiverClientGrpc is created")); + } + } + + @Test + void testFallbackToGrpc() throws Exception { + try (OzoneClient client = getCluster().newClient()) { + assumeTrue(DomainSocketFactory.getInstance(getCluster().getConf()).isServiceReady()); + - TestBucket bucket = TestBucket.newBuilder(client).build(); ++ BucketForTesting bucket = BucketForTesting.newBuilder(client).build(); + GenericTestUtils.LogCapturer logCapturer1 = + GenericTestUtils.LogCapturer.captureLogs(XceiverClientShortCircuit.LOG); + GenericTestUtils.LogCapturer logCapturer2 = + GenericTestUtils.LogCapturer.captureLogs(XceiverClientGrpc.LOG); + + // create key + String keyName = getNewKeyName(); + int dataLength = (2 * BLOCK_SIZE) + (CHUNK_SIZE); + byte[] inputData = bucket.writeRandomBytes(keyName, getRepConfig(), dataLength); + try (KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName)) { + BlockInputStream block0Stream = + (BlockInputStream)keyInputStream.getPartStreams().get(0); + block0Stream.initialize(); + assertNotNull(block0Stream.getBlockFileInputStream()); + assertTrue(logCapturer1.getOutput().contains("XceiverClientShortCircuit is created")); + + // stop XceiverServerDomainSocket server before client sends the second getBlockRequest to server + XceiverServerSpi server = getCluster().getHddsDatanodes().get(0) + .getDatanodeStateMachine().getContainer().getReadDomainSocketChannel(); + server.stop(); + BlockInputStream block1Stream = (BlockInputStream)keyInputStream.getPartStreams().get(1); + try { + block1Stream.initialize(); + } catch (IOException e) { + assertTrue(e.getMessage().contains("DomainSocket stream is not open")); + assertTrue(logCapturer1.getOutput().contains("ReceiveResponseTask is closed due to java.io.EOFException")); + } + assertNull(block1Stream.getBlockFileInputStream()); + // read whole key through Grpc channel + byte[] data = new byte[dataLength]; + int readLen = keyInputStream.read(data); + assertEquals(dataLength, readLen); + assertArrayEquals(inputData, data); + assertTrue(logCapturer2.getOutput().contains("XceiverClientGrpc is created")); + } + } + } +} diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java index 7f747ee034f,3c1d3ed6faf..d3364c1f742 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/service/TestBlockDeletionService.java @@@ -49,8 -49,7 +49,8 @@@ import org.apache.hadoop.hdds.scm.serve import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.scm.server.StorageContainerManager; import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizationContext; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature; - import org.apache.hadoop.hdds.upgrade.TestHddsUpgradeUtils; + import org.apache.hadoop.hdds.upgrade.HddsUpgradeTestUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.UniformDatanodesFactory; import org.apache.hadoop.ozone.client.OzoneBucket; @@@ -162,8 -160,9 +162,8 @@@ public class TestBlockDeletionService } }); finalizationFuture.get(); - TestHddsUpgradeUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); + HddsUpgradeTestUtils.waitForFinalizationFromClient(scmClient, CLIENT_ID); - assertEquals(STORAGE_SPACE_DISTRIBUTION.ordinal(), - cluster.getStorageContainerManager().getLayoutVersionManager().getMetadataLayoutVersion()); + assertTrue(VersionedDatanodeFeatures.isFinalized(HDDSLayoutFeature.STORAGE_SPACE_DISTRIBUTION)); // POST-UPGRADE //Step 6: Repeat the same steps in pre-upgrade diff --cc pom.xml index 524e506a144,f06d57e84a8..65c8f23c2f8 --- a/pom.xml +++ b/pom.xml @@@ -2152,6 -2159,14 +2159,20 @@@ <bannedImport>io.netty.util.internal.**</bannedImport> </bannedImports> </restrictImports> + <restrictImports> + <includeTestCode>true</includeTestCode> + <reason>Classes (e.g. FileInputStream, FileOutputStream) overriding the Object.finalize() method will cause gc to run a much longer time</reason> + <bannedImports> + <bannedImport>java.io.FileInputStream</bannedImport> + <bannedImport>java.io.FileOutputStream</bannedImport> + </bannedImports> ++ <exclusions> ++ <exclusion>org.apache.hadoop.hdds.scm.storage.LocalChunkInputStream</exclusion> ++ <exclusion>org.apache.hadoop.hdds.scm.XceiverClientShortCircuit</exclusion> ++ <exclusion>org.apache.hadoop.hdds.scm.storage.BlockInputStream</exclusion> ++ <exclusion>org.apache.hadoop.hdds.TestXceiverServerDomainSocket</exclusion> ++ </exclusions> + </restrictImports> </rules> </configuration> </execution> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
