This is an automated email from the ASF dual-hosted git repository.
ChenSammi pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
from 68c4231f512 Revert "Merge branch 'HDDS-8342' into
master-merge-object-ttl"
add 738b9c60d33 HDDS-8342. S3 Object LifeCycle Management design doc.
(#8240)
add 2b5f88aaf0b HDDS-12808. Implement OM entities of lifecycle
configurations (#8275)
add f81b36593cc HDDS-12781. Implement LifecycleConfiguration proto (#8276)
add 9438be5596c HDDS-12974. Docker could not parse extra host IP (#8397)
(#8545)
add 59a2b7e77d3 HDDS-12783. Implement S3Gateway api for
LifeCycleConfiguration (#8557)
add c9ef5e66539 HDDS-12780. Implement the Lifecycle Service. (#8681)
add ce01a02542c HDDS-12782. Implement create, delete, and get for bucket
lifecycle configurations in OM (#8875)
add fa33739da3e HDDS-12786. Refine keys lifecycle deletion batch size
(#8909)
add 1fe66180880 HDDS-13548. Prevent OM startup failure from past lifecycle
expiration dates (#8953)
add d7077ea0ce6 HDDS-12789. Support move expired key to trash if trash is
enabled (#8974)
add 6542b31ff8a HDDS-12793. Recursive iterate FSO directory. (#9042)
add 3b5cb65c82a HDDS-12784. Add bucket object ID to lifecycle
configuration proto definition (#9255)
add 99ebb223793 HDDS-12790. Mark a delete operation is performed by
Lifecycle Service (#9403)
add 724803f21a0 HDDS-14184. Reject lifecycle configuration if 'Status' is
missing (#9513)
add b16ea44f588 HDDS-12791. Make Retention Service suspendable (#9426)
add dd04f4b7d83 Merge remote-tracking branch 'origin/master' into
HDDS-8342-ttl-branch
add cfa9cfee5a7 HDDS-12787. Make expired key move to trash configurable
(#9594)
add 8ee502aa31b HDDS-12794. Improve Lifecycle rule validity checking
(#9593)
add 949747727b3 HDDS-14377. Refactor OmLifecycleConfiguration#Builder
(#9688)
add ff9446d6538 HDDS-14900. Document S3 Lifecycle Configurations - Object
Expiration (#9979)
add c0706c6b99a HDDS-14347. Handle upgrades to support S3 lifecycle (#9614)
add 06ebc316b89 Merge remote-tracking branch 'origin/master' into
HDDS-8342-ttl-branch
add f3c6ba4afae HDDS-13474. Support abort incomplete multipart upload
action (#10052)
add f364114d112 Merge remote-tracking branch 'origin/master' into HDDS-8342
add fc4416274b3 HDDS-15227. Add robot test for Lifecycle (#10242)
add daef2f19c29 HDDS-15447. Persist bucket scanned key pointer
periodically (#10412)
add 3d6aa1bd0fc Merge remote-tracking branch 'origin/master' into
HDDS-8342-object-ttl
add 2ebf64de05c Merge remote-tracking branch 'origin/master' into
HDDS-8342-object-ttl
add aa6377721c2 HDDS-15853. Fix Ranger ACL validation for lifecycle
requests (#10753)
new cca31ff7206 Merge remote-tracking branch 'origin/HDDS-8342' into master
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../common/src/main/resources/ozone-default.xml | 75 +
.../docs/content/design/lifecycle-task-resume.md | 80 +
...ocksdb.md => s3-object-lifecycle-management.md} | 18 +-
hadoop-hdds/docs/content/feature/Lifecycle.md | 397 +++
hadoop-hdds/docs/content/feature/Lifecycle.zh.md | 396 +++
hadoop-ozone/cli-admin/pom.xml | 4 +
.../ozone/admin/om/LifecycleResumeSubCommand.java} | 53 +-
.../ozone/admin/om/LifecycleStatusSubCommand.java | 89 +
.../admin/om/LifecycleSubCommand.java} | 30 +-
.../ozone/admin/om/LifecycleSuspendSubCommand.java | 76 +
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 3 +-
.../apache/hadoop/ozone/client/OzoneBucket.java | 34 +
.../ozone/client/OzoneLifecycleConfiguration.java | 230 ++
.../ozone/client/protocol/ClientProtocol.java | 31 +
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 34 +
.../apache/hadoop/fs/ozone/OzoneTrashPolicy.java | 2 +-
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 12 +
.../org/apache/hadoop/ozone/om/OMConfigKeys.java | 41 +
.../hadoop/ozone/om/exceptions/OMException.java | 3 +
.../OmLCAbortIncompleteMultipartUpload.java | 139 +
.../apache/hadoop/ozone/om/helpers/OmLCAction.java | 59 +
.../hadoop/ozone/om/helpers/OmLCExpiration.java | 215 ++
.../apache/hadoop/ozone/om/helpers/OmLCFilter.java | 235 ++
.../apache/hadoop/ozone/om/helpers/OmLCRule.java | 411 +++
.../ozone/om/helpers/OmLifecycleConfiguration.java | 334 ++
.../om/helpers/OmLifecycleRuleAndOperator.java | 214 ++
.../ozone/om/helpers/OmLifecycleScanState.java | 255 ++
.../hadoop/ozone/om/helpers/OmLifecycleUtils.java | 131 +
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 72 +
.../ozone/om/protocol/OzoneManagerProtocol.java | 65 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 102 +
.../apache/hadoop/ozone/om/helpers/OMLCUtils.java | 131 +
.../TestOmLCAbortIncompleteMultipartUpload.java | 171 +
.../ozone/om/helpers/TestOmLCExpiration.java | 198 ++
.../hadoop/ozone/om/helpers/TestOmLCFilter.java | 159 +
.../hadoop/ozone/om/helpers/TestOmLCRule.java | 459 +++
.../om/helpers/TestOmLifeCycleConfiguration.java | 266 ++
.../om/helpers/TestOmLifecycleRuleAndOperator.java | 139 +
.../ozone/om/helpers/TestOmLifecycleScanState.java | 79 +
.../src/main/compose/ozonesecure-ha/docker-config | 6 +
...st-s3g-virtual-host.sh => test-om-lifecycle.sh} | 10 +-
.../main/smoketest/lifecycle/om-lifecycle.robot | 119 +
.../src/main/smoketest/s3/bucketlifecycle.robot | 57 +
hadoop-ozone/integration-test-s3/pom.xml | 10 +
.../hadoop/ozone/s3/awssdk/OzoneS3SDKTests.java | 11 +
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 330 ++
.../ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java | 205 ++
.../AbstractRootedOzoneFileSystemTestWithFSO.java | 3 +-
.../ozone/client/rpc/OzoneRpcClientTests.java | 197 ++
.../org/apache/hadoop/ozone/om/TestOmMetrics.java | 14 +-
.../ozone/om/service/TestRootedDDSWithFSO.java | 9 +-
.../hadoop/ozone/shell/TestOzoneShellHA.java | 76 +
.../src/main/proto/OmClientProtocol.proto | 177 +-
.../apache/hadoop/ozone/om/OMMetadataManager.java | 27 +
.../org/apache/hadoop/ozone/audit/OMAction.java | 8 +-
.../org/apache/hadoop/ozone/om/KeyManager.java | 7 +
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 36 +-
.../java/org/apache/hadoop/ozone/om/OMMetrics.java | 56 +
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 77 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 71 +-
.../hadoop/ozone/om/TrashOzoneFileSystem.java | 1 +
.../hadoop/ozone/om/codec/OMDBDefinition.java | 47 +-
.../hadoop/ozone/om/helpers/OMAuditLogger.java | 1 +
.../om/ratis/utils/OzoneManagerRatisUtils.java | 12 +
.../om/request/bucket/OMBucketDeleteRequest.java | 12 +-
.../file/OMDirectoryCreateRequestWithFSO.java | 4 +-
.../key/OMDirectoriesPurgeRequestWithFSO.java | 2 +
.../om/request/key/OMKeyRenameRequestWithFSO.java | 8 +
.../ozone/om/request/key/OMKeysDeleteRequest.java | 109 +-
.../om/request/key/OmKeysDeleteRequestWithFSO.java | 6 +-
.../OMLifecycleConfigurationDeleteRequest.java | 209 ++
.../OMLifecycleConfigurationSetRequest.java | 238 ++
.../lifecycle/OMLifecycleSaveScanStateRequest.java | 74 +
.../OMLifecycleSetServiceStatusRequest.java | 125 +
.../ozone/om/request/lifecycle}/package-info.java | 4 +-
.../om/response/bucket/OMBucketDeleteResponse.java | 11 +-
.../om/response/key/OMKeysDeleteResponse.java | 17 +-
.../response/key/OMKeysDeleteResponseWithFSO.java | 15 +-
.../OMLifecycleConfigurationDeleteResponse.java} | 42 +-
.../OMLifecycleConfigurationSetResponse.java} | 43 +-
.../OMLifecycleSaveScanStateResponse.java} | 29 +-
.../OMLifecycleSetServiceStatusResponse.java} | 16 +-
.../ozone/om/response/lifecycle}/package-info.java | 4 +-
.../ozone/om/service/KeyLifecycleService.java | 2131 +++++++++++++
.../om/service/KeyLifecycleServiceMetrics.java | 196 ++
.../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 3 +-
.../protocolPB/OzoneManagerRequestHandler.java | 34 +
.../apache/hadoop/ozone/om}/FaultInjectorImpl.java | 2 +-
.../hadoop/ozone/om/TestOmMetadataManager.java | 6 +-
.../om/request/key/TestOMKeysDeleteRequest.java | 158 +-
.../key/TestOMKeysDeleteRequestWithFSO.java | 77 +-
.../TestOMLifecycleConfigurationDeleteRequest.java | 300 ++
.../TestOMLifecycleConfigurationRequest.java | 153 +
.../TestOMLifecycleConfigurationSetRequest.java | 390 +++
.../TestOMLifecycleSaveScanStateRequest.java | 118 +
.../ozone/om/response/TestCleanupTableInfo.java | 2 +
.../om/response/key/TestOMKeysDeleteResponse.java | 2 +-
.../key/TestOMKeysDeleteResponseWithFSO.java | 2 +-
...TestOMLifecycleConfigurationDeleteResponse.java | 117 +
.../TestOMLifecycleConfigurationSetResponse.java | 124 +
.../TestOMLifecycleSaveScanStateResponse.java | 61 +
.../ozone/om/service/TestKeyLifecycleService.java | 3343 ++++++++++++++++++++
.../org/apache/hadoop/ozone/audit/S3GAction.java | 3 +
.../ozone/s3/endpoint/BucketCrudHandler.java | 127 +
...tBucketLifecycleConfigurationUnmarshaller.java} | 56 +-
.../s3/endpoint/S3LifecycleConfiguration.java | 557 ++++
.../hadoop/ozone/s3/exception/S3ErrorTable.java | 3 +
.../org/apache/hadoop/ozone/s3/util/S3Consts.java | 1 +
.../hadoop/ozone/client/ClientProtocolStub.java | 18 +
.../hadoop/ozone/client/OzoneBucketStub.java | 68 +
.../hadoop/ozone/client/OzoneVolumeStub.java | 1 +
.../TestS3LifecycleConfigurationDelete.java | 100 +
.../endpoint/TestS3LifecycleConfigurationGet.java | 173 +
.../endpoint/TestS3LifecycleConfigurationPut.java | 605 ++++
115 files changed, 16648 insertions(+), 261 deletions(-)
create mode 100644 hadoop-hdds/docs/content/design/lifecycle-task-resume.md
copy hadoop-hdds/docs/content/design/{dn-merge-rocksdb.md =>
s3-object-lifecycle-management.md} (59%)
create mode 100644 hadoop-hdds/docs/content/feature/Lifecycle.md
create mode 100644 hadoop-hdds/docs/content/feature/Lifecycle.zh.md
copy
hadoop-ozone/{cli-repair/src/main/java/org/apache/hadoop/ozone/repair/om/quota/QuotaStatus.java
=>
cli-admin/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleResumeSubCommand.java}
(53%)
create mode 100644
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleStatusSubCommand.java
copy
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/{hdds/scm/cli/datanode/DatanodeCommands.java
=> ozone/admin/om/LifecycleSubCommand.java} (66%)
create mode 100644
hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/om/LifecycleSuspendSubCommand.java
create mode 100644
hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneLifecycleConfiguration.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCAbortIncompleteMultipartUpload.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCAction.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCExpiration.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCFilter.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLCRule.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleConfiguration.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleRuleAndOperator.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleScanState.java
create mode 100644
hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmLifecycleUtils.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/OMLCUtils.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCAbortIncompleteMultipartUpload.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCExpiration.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCFilter.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLCRule.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifeCycleConfiguration.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleRuleAndOperator.java
create mode 100644
hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmLifecycleScanState.java
copy
hadoop-ozone/dist/src/main/compose/ozonesecure-ha/{test-s3g-virtual-host.sh =>
test-om-lifecycle.sh} (86%)
mode change 100755 => 100644
create mode 100644
hadoop-ozone/dist/src/main/smoketest/lifecycle/om-lifecycle.robot
create mode 100644
hadoop-ozone/dist/src/main/smoketest/s3/bucketlifecycle.robot
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleConfigurationDeleteRequest.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleConfigurationSetRequest.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSaveScanStateRequest.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle/OMLifecycleSetServiceStatusRequest.java
copy {hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage =>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/lifecycle}/package-info.java
(86%)
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/{volume/OMVolumeSetQuotaResponse.java
=> lifecycle/OMLifecycleConfigurationDeleteResponse.java} (59%)
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/{volume/OMVolumeSetQuotaResponse.java
=> lifecycle/OMLifecycleConfigurationSetResponse.java} (54%)
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/{util/OMEchoRPCWriteResponse.java
=> lifecycle/OMLifecycleSaveScanStateResponse.java} (56%)
copy
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/{util/OMEchoRPCWriteResponse.java
=> lifecycle/OMLifecycleSetServiceStatusResponse.java} (75%)
copy {hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage =>
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/lifecycle}/package-info.java
(86%)
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java
create mode 100644
hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleServiceMetrics.java
copy hadoop-ozone/{integration-test/src/test/java/org/apache/hadoop/utils =>
ozone-manager/src/test/java/org/apache/hadoop/ozone/om}/FaultInjectorImpl.java
(98%)
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationDeleteRequest.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationRequest.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleConfigurationSetRequest.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/lifecycle/TestOMLifecycleSaveScanStateRequest.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/lifecycle/TestOMLifecycleConfigurationDeleteResponse.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/lifecycle/TestOMLifecycleConfigurationSetResponse.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/lifecycle/TestOMLifecycleSaveScanStateResponse.java
create mode 100644
hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestKeyLifecycleService.java
copy
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/{MessageUnmarshaller.java
=> PutBucketLifecycleConfigurationUnmarshaller.java} (52%)
create mode 100644
hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3LifecycleConfiguration.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationDelete.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationGet.java
create mode 100644
hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
