This is an automated email from the ASF dual-hosted git repository.
yuqi1129 pushed a change to branch cherry-pick-37f9d23c-to-branch-1.3
in repository https://gitbox.apache.org/repos/asf/gravitino.git
from cc585efe9e [#12897] resolve branch-1.3 cherry-pick conflicts
add 1632135815 [Cherry-pick to branch-1.3] [#12552] feat(catalog): support
testing existing catalog connections (#12553) (#12890)
add e1866dd3b9 [Cherry-pick to branch-1.3] [#12887] fix(server): Validate
statistic name length (#12889) (#12894)
add 1a0a5e9c2e [Cherry-pick to branch-1.3] [#12794] feat(catalog): Support
connection tests with proposed changes (#12798) (#12884)
add 8c1e08cebb [Cherry-pick to branch-1.3] [#12885] fix(audit): Include
role names in role assignment logs (#12886) (#12898)
add feb75315cb [Cherry-pick to branch-1.3] [MINOR] fix(ci): Pin
pnpm/action-setup to an ASF-allowlisted SHA (#12899) (#12905)
add ab07a0a978 [Cherry-pick to branch-1.3] [#11985] fix(openapi): restore
the ExternalType variant to DataType (#11991) (#12907)
add 7bec3c215b [Cherry-pick to branch-1.3] [#12783] fix(server): return
JSON errors for the whole pre-resource-method WebApplicationException family
(#12878) (#12910)
add 3a1fd2f6ef [Cherry-pick to branch-1.3] [#12875] improve(server): Reuse
the shared table authorization expressions (#12876) (#12912)
add 7f59be2615 [Cherry-pick to branch-1.3] [#12501][#12500] feat(core):
Support FUNCTION and VIEW metadata object policies (#12503) (#12904)
add 0b99555a41 [Cherry-pick to branch-1.3] [#12709]
improvement(spark-connector): Route lakehouse-iceberg catalogs through the
Iceberg REST server (#12710) (#12864)
add 0de9fe57dd [Cherry-pick to branch-1.3] [#12546]
improvement(trino-connector): Report catalog registration status through system
tables (#12547) (#12896)
add 7c77716d32 [Cherry-pick to branch-1.3] [#12712] fix(trino-connector):
Add integration test for TLS + OAuth2 credential vending (#12713) (#12908)
add 9c4fd069af [Cherry-pick to branch-1.3] [#12851] fix(iceberg-rest):
Share the managed memory catalog in auxiliary mode (#12852) (#12923)
add a70bf90d02 [Cherry-pick to branch-1.3] [#12892] fix(audit): Suppress
internal operation audit events (#12893) (#12909)
add e1b7e4dd3b [#12897] Merge branch-1.3 and resolve conflicts
No new revisions were added by this update.
Summary of changes:
.../workflows/spark-integration-test-action.yml | 2 +-
.github/workflows/web-ui-tests.yml | 2 +-
.../org/apache/gravitino/SupportsCatalogs.java | 34 +
.../java/org/apache/gravitino/stats/Statistic.java | 3 +
bundles/aliyun/build.gradle.kts | 4 +
.../gravitino/oss/fs/TestOSSCredentialVending.java | 86 ++
bundles/aws/build.gradle.kts | 3 +
.../gravitino/s3/fs/TestS3CredentialVending.java | 101 ++
bundles/azure/build.gradle.kts | 4 +
.../abs/fs/TestAzureCredentialVending.java | 86 ++
bundles/gcp/build.gradle.kts | 4 +
.../gravitino/gcs/fs/TestGCSCredentialVending.java | 64 ++
.../lakehouse/iceberg/IcebergConstants.java | 3 +
.../catalog/fileset/FilesetCatalogImpl.java | 2 +-
.../catalog/fileset/FilesetCatalogOperations.java | 366 ++++++-
.../fileset/TestFilesetCatalogOperations.java | 377 ++++++-
.../fileset/TestFilesetCredentialProvider.java | 79 ++
.../TestFilesetSecondCredentialProvider.java | 11 +-
.../integration/test/FilesetABSCatalogIT.java | 1 +
.../fileset/integration/test/FilesetCatalogIT.java | 1 +
.../integration/test/FilesetGCSCatalogIT.java | 1 +
.../integration/test/FilesetOSSCatalogIT.java | 1 +
.../integration/test/FilesetS3CatalogIT.java | 1 +
....apache.gravitino.credential.CredentialProvider | 4 +-
.../catalog/glue/GlueCatalogOperations.java | 12 +-
.../glue/TestGlueCatalogSchemaOperations.java | 16 +
.../integration/test/AbstractGlueCatalogIT.java | 6 +
.../catalog/hive/HiveCatalogOperations.java | 12 +-
.../hive/integration/test/CatalogHive2IT.java | 5 +
.../catalog/jdbc/JdbcCatalogOperations.java | 29 +-
.../catalog/jdbc/TestJdbcCatalogOperations.java | 31 +-
.../doris/integration/test/CatalogDorisIT.java | 5 +
.../mysql/integration/test/CatalogMysqlIT.java | 30 +
.../integration/test/CatalogPostgreSqlIT.java | 5 +
.../integration/test/CatalogStarRocksIT.java | 5 +
.../catalog/kafka/KafkaCatalogOperations.java | 12 +-
.../catalog/kafka/TestKafkaCatalogOperations.java | 29 +
.../kafka/integration/test/CatalogKafkaIT.java | 1 +
.../generic/GenericCatalogOperations.java | 10 +-
.../test/CatalogGenericCatalogDeltaIT.java | 6 +
.../generic/TestGenericCatalogOperations.java | 7 +
.../test/CatalogGenericCatalogLanceIT.java | 6 +
.../lakehouse/hudi/HudiCatalogOperations.java | 14 +-
.../hudi/integration/test/HudiCatalogHMSIT.java | 1 +
.../iceberg/IcebergCatalogOperations.java | 28 +-
.../iceberg/TestIcebergCatalogOperations.java | 24 +
.../lakehouse/iceberg/TestIcebergSchema.java | 3 +-
.../integration/test/CatalogIcebergBaseIT.java | 5 +
.../lakehouse/paimon/PaimonCatalogOperations.java | 5 +
.../integration/test/CatalogPaimonBaseIT.java | 5 +
.../catalog/model/ModelCatalogOperations.java | 11 +-
.../catalog/model/TestModelCatalogOperations.java | 7 +
.../integration/test/ModelCatalogOperationsIT.java | 6 +
.../fs/InMemoryFileSystemCredentialsProvider.java | 86 ++
.../TestInMemoryFileSystemCredentialsProvider.java | 36 +-
.../org/apache/gravitino/client/ErrorHandlers.java | 3 +
.../apache/gravitino/client/GravitinoClient.java | 17 +
.../apache/gravitino/client/GravitinoMetalake.java | 60 ++
.../gravitino/client/TestGravitinoAdminClient.java | 49 +
.../client/integration/test/CatalogIT.java | 14 +
.../integration/test/JsonErrorHandlerIT.java | 64 +-
.../client/integration/test/PolicyIT.java | 742 +++++++++++++-
.../gravitino/client/gravitino_client.py | 4 +
.../gravitino/client/gravitino_metalake.py | 40 +
.../gravitino/dto/responses/error_response.py | 8 +-
.../client-python/tests/unittests/test_metalake.py | 72 ++
.../java/org/apache/gravitino/GravitinoEnv.java | 88 +-
.../authorization/AuthorizationUtils.java | 6 +-
.../apache/gravitino/catalog/CatalogManager.java | 131 ++-
.../catalog/CatalogNormalizeDispatcher.java | 19 +
.../apache/gravitino/catalog/SupportsCatalogs.java | 19 +
.../catalog/TableOperationDispatcher.java | 6 +-
.../gravitino/catalog/ViewOperationDispatcher.java | 6 +-
.../gravitino/connector/CatalogDropAware.java | 15 +-
.../gravitino/connector/CatalogOperations.java | 17 +-
.../hook/AccessControlHookDispatcher.java | 2 +-
.../gravitino/hook/CatalogHookDispatcher.java | 12 +-
.../gravitino/hook/FilesetHookDispatcher.java | 2 +-
.../apache/gravitino/hook/JobHookDispatcher.java | 4 +-
.../gravitino/hook/MetalakeHookDispatcher.java | 4 +-
.../apache/gravitino/hook/ModelHookDispatcher.java | 4 +-
.../gravitino/hook/PolicyHookDispatcher.java | 2 +-
.../gravitino/hook/SchemaHookDispatcher.java | 2 +-
.../apache/gravitino/hook/TagHookDispatcher.java | 2 +-
.../apache/gravitino/hook/TopicHookDispatcher.java | 2 +-
.../gravitino/listener/CatalogEventDispatcher.java | 14 +-
.../listener/api/event/GrantGroupRolesEvent.java | 7 +
.../api/event/GrantGroupRolesFailureEvent.java | 7 +
.../listener/api/event/GrantUserRolesEvent.java | 7 +
.../api/event/GrantUserRolesFailureEvent.java | 7 +
.../listener/api/event/RevokeGroupRolesEvent.java | 7 +
.../api/event/RevokeGroupRolesFailureEvent.java | 7 +
.../listener/api/event/RevokeUserRolesEvent.java | 7 +
.../api/event/RevokeUserRolesFailureEvent.java | 7 +
...PreEvent.java => RoleAssignmentAuditInfos.java} | 32 +-
.../org/apache/gravitino/policy/PolicyManager.java | 4 +-
.../PolicyMetadataObjectRelBaseSQLProvider.java | 22 +
.../PolicyMetadataObjectRelPostgreSQLProvider.java | 23 +
.../relational/service/FunctionMetaService.java | 6 +
.../relational/service/ViewMetaService.java | 6 +
.../apache/gravitino/utils/MetadataObjectUtil.java | 32 +-
.../gravitino/audit/v2/TestSimpleAuditLogV2.java | 16 +
.../authorization/TestAuthorizationUtils.java | 18 +-
.../gravitino/catalog/TestCatalogManager.java | 158 ++-
.../catalog/TestCatalogNormalizeDispatcher.java | 14 +
.../catalog/TestPartitionOperationDispatcher.java | 2 +-
.../hook/TestAccessControlHookDispatcher.java | 7 +-
.../gravitino/hook/TestCatalogHookDispatcher.java | 14 +-
.../gravitino/hook/TestFilesetHookDispatcher.java | 14 +-
.../gravitino/hook/TestJobHookDispatcher.java | 7 +-
.../gravitino/hook/TestMetalakeHookDispatcher.java | 19 +-
.../gravitino/hook/TestModelHookDispatcher.java | 7 +-
.../gravitino/hook/TestPolicyHookDispatcher.java | 7 +-
.../gravitino/hook/TestSchemaHookDispatcher.java | 7 +-
.../gravitino/hook/TestTagHookDispatcher.java | 7 +-
.../gravitino/hook/TestTopicHookDispatcher.java | 14 +-
.../listener/api/event/TestGroupEvent.java | 8 +
.../listener/api/event/TestUserEvent.java | 8 +
.../apache/gravitino/policy/TestPolicyManager.java | 91 +-
.../org/apache/gravitino/tag/TestTagManager.java | 16 +-
.../gravitino/utils/TestMetadataObjectUtil.java | 125 +++
docs/manage-policies-in-gravitino.md | 2 +-
docs/open-api/catalogs.yaml | 86 +-
docs/open-api/datatype.yaml | 35 +-
docs/open-api/openapi.yaml | 3 +
docs/open-api/policies.yaml | 6 +-
docs/policies.md | 10 +-
docs/spark-connector/spark-catalog-iceberg.md | 99 +-
docs/spark-connector/spark-connector.md | 20 +-
docs/trino-connector/supported-catalog.md | 80 ++
.../iceberg/common/ops/IcebergCatalogWrapper.java | 6 +-
.../iceberg/common/utils/IcebergCatalogUtil.java | 35 +
.../common/ops/TestIcebergCatalogWrapper.java | 27 +
.../common/utils/TestIcebergCatalogUtil.java | 86 ++
.../service/IcebergCatalogWrapperManager.java | 4 +
.../provider/DynamicIcebergConfigProvider.java | 4 +-
.../service/rest/IcebergTableOperations.java | 24 +-
.../service/rest/IcebergTableRenameOperations.java | 7 +-
.../server/web/filter/RenameTableAuthzHandler.java | 9 +-
.../TestIcebergCatalogWrapperManagerForREST.java | 46 +-
.../provider/TestDynamicIcebergConfigProvider.java | 32 +
.../integration/test/container/TrinoContainer.java | 95 +-
.../server/authorization/MetadataAuthzHelper.java | 4 +-
.../AuthorizationExpressionConstants.java | 12 +
.../authorization/jcasbin/JcasbinAuthorizer.java | 2 +-
.../authorization/TestMetadataAuthzHelper.java | 33 +-
.../jcasbin/TestJcasbinAuthorizer.java | 12 +
.../apache/gravitino/server/GravitinoServer.java | 2 +
.../gravitino/server/web/VersioningFilter.java | 12 +-
.../web/mapper/WebApplicationExceptionMapper.java | 77 ++
.../server/web/rest/CatalogOperations.java | 42 +
.../server/web/rest/ExceptionHandlers.java | 65 +-
.../gravitino/server/web/rest/GroupOperations.java | 2 +-
.../server/web/rest/StatisticOperations.java | 78 +-
.../gravitino/server/web/rest/TableOperations.java | 8 +-
.../gravitino/server/web/rest/UserOperations.java | 2 +-
.../gravitino/server/web/TestVersioningFilter.java | 38 +-
.../mapper/TestWebApplicationExceptionMapper.java | 67 ++
.../server/web/rest/TestCatalogOperations.java | 158 +++
.../server/web/rest/TestGroupOperations.java | 3 +-
.../server/web/rest/TestOwnerOperations.java | 7 +-
.../server/web/rest/TestPermissionOperations.java | 5 +-
.../server/web/rest/TestRoleOperations.java | 19 +-
.../server/web/rest/TestStatisticOperations.java | 61 +-
.../server/web/rest/TestUserOperations.java | 3 +-
.../spark/connector/GravitinoSparkConfig.java | 24 +
.../connector/catalog/GravitinoCatalogManager.java | 32 +-
.../connector/iceberg/GravitinoIcebergCatalog.java | 154 ++-
.../iceberg/IcebergPropertiesConstants.java | 18 +
.../iceberg/IcebergPropertiesConverter.java | 205 ++++
.../connector/iceberg/IcebergRestOAuthConfig.java | 135 +++
.../connector/plugin/GravitinoDriverPlugin.java | 1 +
.../catalog/TestGravitinoCatalogManager.java | 107 ++
.../connector/glue/TestGravitinoGlueCatalog.java | 2 +-
.../TestGravitinoIcebergCatalogRestRouting.java | 246 +++++
.../iceberg/TestIcebergPropertiesConverter.java | 190 ++++
.../iceberg/TestIcebergRestOAuthConfig.java | 205 ++++
.../connector/integration/test/SparkEnvIT.java | 71 +-
.../iceberg/SparkIcebergCatalogHiveBackendIT.java | 10 +
.../iceberg/SparkIcebergCatalogRestRoutingIT.java | 76 ++
...ava => SparkIcebergCatalogRestRoutingIT33.java} | 4 +-
...ava => SparkIcebergCatalogRestRoutingIT34.java} | 2 +-
spark-connector/v3.5/spark/build.gradle.kts | 2 +
.../SparkIcebergCatalogRestRoutingIT35.java} | 2 +-
...kIcebergCatalogRestS3CredentialVendingIT35.java | 239 +++++
trino-connector/integration-test/build.gradle.kts | 3 +
.../test/TrinoTlsOAuthCredentialVendingIT.java | 624 +++++++++++
.../testsets/jdbc-mysql/00015_catalog_status.sql | 3 +
.../testsets/jdbc-mysql/00015_catalog_status.txt | 3 +
.../connector/GravitinoConnectorFactory435.java | 6 +-
.../connector/GravitinoSystemConnector435.java | 21 +-
.../connector/GravitinoConnectorFactory440.java | 6 +-
.../connector/GravitinoSystemConnector440.java | 21 +-
.../connector/GravitinoConnectorFactory446.java | 6 +-
.../connector/GravitinoSystemConnector446.java | 21 +-
.../connector/GravitinoConnectorFactory452.java | 6 +-
.../connector/GravitinoSystemConnector452.java | 21 +-
.../connector/GravitinoConnectorFactory469.java | 6 +-
.../connector/GravitinoSystemConnector469.java | 21 +-
.../connector/GravitinoConnectorFactory478.java | 6 +-
.../connector/GravitinoSystemConnector478.java | 21 +-
.../gravitino/trino/connector/GravitinoConfig.java | 5 +-
.../trino/connector/GravitinoConnectorFactory.java | 41 +-
.../connector/catalog/CatalogConnectorManager.java | 685 ++++++++++++-
.../trino/connector/catalog/CatalogRegister.java | 87 +-
.../catalog/CatalogRegistrationState.java | 359 +++++++
.../connector/system/GravitinoSystemConnector.java | 94 +-
.../system/GravitinoSystemConnectorMetadata.java | 25 +-
.../AlterCatalogStoredProcedure.java | 8 +-
.../CreateCatalogStoredProcedure.java | 15 +-
.../DropCatalogStoredProcedure.java | 13 +
.../system/table/GravitinoSystemTable.java | 35 +
.../system/table/GravitinoSystemTableCatalog.java | 23 +-
.../table/GravitinoSystemTableCatalogStatus.java | 123 +++
.../system/table/GravitinoSystemTableFactory.java | 56 +-
.../table/GravitinoSystemTableLoadStatus.java | 135 +++
.../trino/connector/TestGravitinoConfig.java | 10 +
.../trino/connector/TestGravitinoConnector.java | 93 +-
.../TestGravitinoConnectorFactoryInit.java | 87 ++
.../TestGravitinoConnectorFactoryStart.java | 57 +-
.../catalog/TestCatalogConnectorManager.java | 1078 +++++++++++++++++++-
.../connector/catalog/TestCatalogRegister.java | 123 ++-
.../system/TestGravitinoSystemConnector.java | 37 +
.../table/TestGravitinoSystemStatusTables.java | 319 ++++++
224 files changed, 10589 insertions(+), 735 deletions(-)
create mode 100644
bundles/aliyun/src/test/java/org/apache/gravitino/oss/fs/TestOSSCredentialVending.java
create mode 100644
bundles/aws/src/test/java/org/apache/gravitino/s3/fs/TestS3CredentialVending.java
create mode 100644
bundles/azure/src/test/java/org/apache/gravitino/abs/fs/TestAzureCredentialVending.java
create mode 100644
bundles/gcp/src/test/java/org/apache/gravitino/gcs/fs/TestGCSCredentialVending.java
create mode 100644
catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/TestFilesetCredentialProvider.java
copy
maintenance/updaters/src/main/java/org/apache/gravitino/maintenance/optimizer/updater/metrics/storage/jdbc/H2MetricsDialect.java
=>
catalogs/catalog-fileset/src/test/java/org/apache/gravitino/catalog/fileset/TestFilesetSecondCredentialProvider.java
(71%)
copy {bundles/azure/src/main =>
catalogs/catalog-fileset/src/test}/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
(84%)
create mode 100644
catalogs/hadoop-common/src/main/java/org/apache/gravitino/catalog/hadoop/fs/InMemoryFileSystemCredentialsProvider.java
copy
flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/iceberg/TestGravitinoIcebergCatalog.java
=>
catalogs/hadoop-common/src/test/java/org/apache/gravitino/catalog/hadoop/fs/TestInMemoryFileSystemCredentialsProvider.java
(50%)
copy api/src/main/java/org/apache/gravitino/Auditable.java =>
core/src/main/java/org/apache/gravitino/connector/CatalogDropAware.java (74%)
copy
core/src/main/java/org/apache/gravitino/listener/api/event/{LoadCatalogPreEvent.java
=> RoleAssignmentAuditInfos.java} (58%)
create mode 100644
server/src/main/java/org/apache/gravitino/server/web/mapper/WebApplicationExceptionMapper.java
create mode 100644
server/src/test/java/org/apache/gravitino/server/web/mapper/TestWebApplicationExceptionMapper.java
create mode 100644
spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/iceberg/IcebergRestOAuthConfig.java
create mode 100644
spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/catalog/TestGravitinoCatalogManager.java
create mode 100644
spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/iceberg/TestGravitinoIcebergCatalogRestRouting.java
create mode 100644
spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/iceberg/TestIcebergRestOAuthConfig.java
create mode 100644
spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/integration/test/iceberg/SparkIcebergCatalogRestRoutingIT.java
copy
spark-connector/v3.3/spark/src/test/java/org/apache/gravitino/spark/connector/integration/test/iceberg/{SparkIcebergCatalogRestBackendIT33.java
=> SparkIcebergCatalogRestRoutingIT33.java} (94%)
copy
spark-connector/v3.4/spark/src/test/java/org/apache/gravitino/spark/connector/integration/test/iceberg/{SparkIcebergCatalogRestBackendIT34.java
=> SparkIcebergCatalogRestRoutingIT34.java} (91%)
copy
spark-connector/{v3.4/spark/src/test/java/org/apache/gravitino/spark/connector/integration/test/iceberg/SparkIcebergCatalogRestBackendIT34.java
=>
v3.5/spark/src/test/java/org/apache/gravitino/spark/connector/integration/test/iceberg/SparkIcebergCatalogRestRoutingIT35.java}
(91%)
create mode 100644
spark-connector/v3.5/spark/src/test/java/org/apache/gravitino/spark/connector/integration/test/iceberg/SparkIcebergCatalogRestS3CredentialVendingIT35.java
create mode 100644
trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoTlsOAuthCredentialVendingIT.java
create mode 100644
trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00015_catalog_status.sql
create mode 100644
trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/jdbc-mysql/00015_catalog_status.txt
create mode 100644
trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/CatalogRegistrationState.java
create mode 100644
trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/table/GravitinoSystemTableCatalogStatus.java
create mode 100644
trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/system/table/GravitinoSystemTableLoadStatus.java
create mode 100644
trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/TestGravitinoConnectorFactoryInit.java
create mode 100644
trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/system/table/TestGravitinoSystemStatusTables.java
