This is an automated email from the ASF dual-hosted git repository.
jerryshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new a5c13c9343 [#14435] fix(build): Upgrade Ranger to 2.8.0 and ZooKeeper
to 3.8.8, and strip the outdated Avro shaded in hadoop-client-runtime (#14532)
a5c13c9343 is described below
commit a5c13c93435d8afdd108b9b1beca2f69df45e30f
Author: geyanggang <[email protected]>
AuthorDate: Tue Nov 3 22:24:47 2026 +0800
[#14435] fix(build): Upgrade Ranger to 2.8.0 and ZooKeeper to 3.8.8, and
strip the outdated Avro shaded in hadoop-client-runtime (#14532)
---
.../authorization-chain/build.gradle.kts | 19 +-
.../authorization-ranger/build.gradle.kts | 30 ++-
.../ranger/RangerAuthorizationHDFSPlugin.java | 12 +-
.../ranger/RangerAuthorizationHadoopSQLPlugin.java | 10 +-
.../ranger/RangerAuthorizationPlugin.java | 22 +-
.../ranger/RangerClientExtension.java | 39 ++--
.../authorization/ranger/RangerHelper.java | 51 ++---
.../ranger/reference/JsonDateSerializer.java | 6 +-
.../ranger/reference/VXDataObject.java | 11 +-
.../authorization/ranger/reference/VXGroup.java | 8 +-
.../ranger/reference/VXGroupList.java | 8 +-
.../authorization/ranger/reference/VXUser.java | 8 +-
.../authorization/ranger/reference/VXUserList.java | 8 +-
.../ranger/TestRangerClientExtension.java | 221 +++++++++++++++++++++
.../authorization/ranger/TestRangerHelper.java | 12 +-
.../ranger/TestRangerImmutableModelDefaults.java | 154 ++++++++++++++
.../ranger/integration/test/RangerHiveIT.java | 2 +-
.../ranger/integration/test/RangerITEnv.java | 12 +-
.../ranger/reference/TestRangerReferenceJson.java | 146 ++++++++++++++
build.gradle.kts | 113 +++++++++++
catalogs/hive-metastore2-libs/build.gradle.kts | 7 +
catalogs/hive-metastore3-libs/build.gradle.kts | 6 +
gradle/libs.versions.toml | 6 +-
integration-test-common/build.gradle.kts | 9 +-
24 files changed, 820 insertions(+), 100 deletions(-)
diff --git a/authorizations/authorization-chain/build.gradle.kts
b/authorizations/authorization-chain/build.gradle.kts
index 8ab42a9e4d..042d9a5d7b 100644
--- a/authorizations/authorization-chain/build.gradle.kts
+++ b/authorizations/authorization-chain/build.gradle.kts
@@ -67,8 +67,8 @@ dependencies {
testRuntimeOnly(libs.junit.jupiter.engine)
testImplementation(libs.mysql.driver)
testImplementation(libs.postgresql.driver)
+ // Keep in sync with the ranger-intg excludes in
authorization-ranger/build.gradle.kts.
testImplementation(libs.ranger.intg) {
- exclude("org.apache.hadoop", "hadoop-common")
exclude("org.apache.hive", "hive-storage-api")
exclude("org.apache.lucene")
exclude("org.apache.solr")
@@ -76,13 +76,23 @@ dependencies {
exclude("org.elasticsearch")
exclude("org.elasticsearch.client")
exclude("org.elasticsearch.plugin")
+ exclude("org.apache.ranger", "ranger-audit-core")
exclude("org.apache.ranger", "ranger-plugins-audit")
exclude("org.apache.ranger", "ranger-plugins-cred")
exclude("org.apache.ranger", "ranger-plugin-classloader")
+ exclude("org.apache.ranger", "ugsync-util")
+ exclude("org.graalvm.js")
+ exclude("com.nimbusds")
+ exclude("net.minidev")
exclude("net.java.dev.jna")
+ exclude("javax.servlet")
exclude("javax.ws.rs")
exclude("org.eclipse.jetty")
- exclude("org.apache.hadoop", "hadoop-common")
+ exclude("org.apache.hadoop")
+ exclude("com.fasterxml.jackson.core")
+ exclude("com.fasterxml.jackson.jaxrs")
+ exclude("com.fasterxml.jackson.module")
+ exclude("org.apache.commons", "commons-lang3")
}
testImplementation("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion")
testImplementation("org.apache.spark:spark-sql_$scalaVersion:$sparkVersion")
{
@@ -95,6 +105,11 @@ dependencies {
testImplementation("org.apache.kyuubi:kyuubi-spark-authz-shaded_$scalaVersion:$kyuubiVersion")
{
exclude("com.sun.jersey")
}
+ // The Kyuubi Spark authz plugin in the tests runs the Ranger policy engine,
which needs
+ // ugsync-util since Ranger 2.8. Gravitino itself only uses the Ranger Admin
client.
+ testRuntimeOnly(libs.ranger.ugsync.util) {
+ isTransitive = false
+ }
testImplementation(libs.hadoop3.client.api)
testImplementation(libs.hadoop3.client.runtime)
diff --git a/authorizations/authorization-ranger/build.gradle.kts
b/authorizations/authorization-ranger/build.gradle.kts
index 506466fa38..7af1df3f3b 100644
--- a/authorizations/authorization-ranger/build.gradle.kts
+++ b/authorizations/authorization-ranger/build.gradle.kts
@@ -53,6 +53,9 @@ dependencies {
implementation(libs.javax.ws.rs.api)
implementation(libs.jettison)
implementation(libs.mail)
+ // Gravitino only uses the Ranger Admin REST client (RangerClient) and the
model classes. The
+ // excluded modules belong to the in-process plugin side (policy engine,
audit, credential store,
+ // script engines, JWT, user sync) that Gravitino never runs.
implementation(libs.ranger.intg) {
exclude("org.apache.hive", "hive-storage-api")
exclude("org.apache.lucene")
@@ -61,15 +64,31 @@ dependencies {
exclude("org.elasticsearch")
exclude("org.elasticsearch.client")
exclude("org.elasticsearch.plugin")
+ exclude("org.apache.ranger", "ranger-audit-core")
exclude("org.apache.ranger", "ranger-plugins-audit")
exclude("org.apache.ranger", "ranger-plugins-cred")
exclude("org.apache.ranger", "ranger-plugin-classloader")
+ exclude("org.apache.ranger", "ugsync-util")
+ exclude("org.graalvm.js")
+ exclude("com.nimbusds")
+ exclude("net.minidev")
exclude("net.java.dev.jna")
+ exclude("javax.servlet")
exclude("javax.ws.rs")
exclude("org.eclipse.jetty")
- // Conflicts with hadoop-client-api used in hadoop-catalog.
- exclude("org.apache.hadoop", "hadoop-common")
+ // Ranger 2.8 depends on hadoop-client-api/runtime 3.4.x. Keep the Hadoop
version shared with
+ // the catalogs, which load the authorization libs in the same class
loader.
+ exclude("org.apache.hadoop")
+ // Use the versions Gravitino ships with the server, which the isolated
class loader shares.
+ exclude("com.fasterxml.jackson.core")
+ exclude("com.fasterxml.jackson.jaxrs")
+ exclude("com.fasterxml.jackson.module")
+ exclude("org.apache.commons", "commons-lang3")
}
+ // Jackson (including the JAX-RS JSON provider used by RangerRESTClient) is
loaded from the
+ // Gravitino server classpath at runtime, so it is not shipped in the plugin
libs.
+ compileOnly(libs.jackson.annotations)
+ compileOnly(libs.jackson.databind)
implementation(libs.hadoop3.client.api)
implementation(libs.hadoop3.client.runtime)
@@ -81,7 +100,9 @@ dependencies {
testImplementation(project(":server"))
testImplementation(project(":integration-test-common", "testArtifacts"))
testImplementation(libs.awaitility)
+ testImplementation(libs.gson)
testImplementation(libs.junit.jupiter.api)
+ testImplementation(libs.junit.jupiter.params)
testImplementation(libs.mockito.core)
testImplementation(libs.mockito.inline)
testImplementation(libs.testcontainers)
@@ -99,6 +120,11 @@ dependencies {
testImplementation("org.apache.kyuubi:kyuubi-spark-authz-shaded_$scalaVersion:$kyuubiVersion")
{
exclude("com.sun.jersey")
}
+ // The Kyuubi Spark authz plugin in the tests runs the Ranger policy engine,
which needs
+ // ugsync-util since Ranger 2.8. Gravitino itself only uses the Ranger Admin
client.
+ testRuntimeOnly(libs.ranger.ugsync.util) {
+ isTransitive = false
+ }
testImplementation(libs.hadoop3.hdfs) {
exclude("com.sun.jersey")
diff --git
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java
index 82f731f526..05daa1718c 100644
---
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java
+++
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHDFSPlugin.java
@@ -33,7 +33,7 @@ import java.util.Objects;
import java.util.Set;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.gravitino.Entity;
import org.apache.gravitino.GravitinoEnv;
import org.apache.gravitino.MetadataObject;
@@ -255,11 +255,9 @@ public class RangerAuthorizationHDFSPlugin extends
RangerAuthorizationPlugin {
policy.setName(newResourcePath);
// Update the policy resource name to new name
- policy
- .getResources()
- .put(
- rangerHelper.policyResourceDefines.get(0),
- new RangerPolicy.RangerPolicyResource(newResourcePath));
+ policy.setResource(
+ rangerHelper.policyResourceDefines.get(0),
+ new RangerPolicy.RangerPolicyResource(newResourcePath));
boolean alreadyExist =
existNewPolicies.stream()
@@ -354,7 +352,7 @@ public class RangerAuthorizationHDFSPlugin extends
RangerAuthorizationPlugin {
getAuthorizationPath(pathBasedMetadataObject),
false,
pathBasedMetadataObject.recursive());
- policy.getResources().put(RangerDefines.PolicyResource.PATH.getName(),
policyResource);
+ policy.setResource(RangerDefines.PolicyResource.PATH.getName(),
policyResource);
return policy;
}
diff --git
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java
index 35296e4611..a7e9457a98 100644
---
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java
+++
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationHadoopSQLPlugin.java
@@ -250,11 +250,9 @@ public class RangerAuthorizationHadoopSQLPlugin extends
RangerAuthorizationPlugi
}
// Update the policy resource names, up to the renamed level,
to the new names
for (int i = 0; i <= index; i++) {
- policy
- .getResources()
- .put(
- policyResourceDefinesRule().get(i),
- new
RangerPolicy.RangerPolicyResource(newAuthzMetaObject.names().get(i)));
+ policy.setResource(
+ policyResourceDefinesRule().get(i),
+ new
RangerPolicy.RangerPolicyResource(newAuthzMetaObject.names().get(i)));
}
boolean alreadyExist =
@@ -402,7 +400,7 @@ public class RangerAuthorizationHadoopSQLPlugin extends
RangerAuthorizationPlugi
for (int i = 0; i < nsMetadataObject.size(); i++) {
RangerPolicy.RangerPolicyResource policyResource =
new RangerPolicy.RangerPolicyResource(nsMetadataObject.get(i));
- policy.getResources().put(policyResourceDefinesRule().get(i),
policyResource);
+ policy.setResource(policyResourceDefinesRule().get(i), policyResource);
}
return policy;
}
diff --git
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java
index bdb63ba7de..0cf4366573 100644
---
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java
+++
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerAuthorizationPlugin.java
@@ -252,19 +252,18 @@ public abstract class RangerAuthorizationPlugin
ownerPrivilege -> {
// Each owner's privilege will create one RangerPolicyItemAccess
in the policy
RangerPolicy.RangerPolicyItem policyItem = new
RangerPolicy.RangerPolicyItem();
- policyItem
- .getAccesses()
- .add(new
RangerPolicy.RangerPolicyItemAccess(ownerPrivilege.getName()));
+ policyItem.addAccess(
+ new
RangerPolicy.RangerPolicyItemAccess(ownerPrivilege.getName()));
if (newOwner != null) {
if (newOwner.type() == Owner.Type.USER) {
- policyItem.getUsers().add(newOwner.name());
+ policyItem.addUser(newOwner.name());
} else {
- policyItem.getGroups().add(newOwner.name());
+ policyItem.addGroup(newOwner.name());
}
// mark the policy item is created by Gravitino
- policyItem.getRoles().add(RangerHelper.GRAVITINO_OWNER_ROLE);
+ policyItem.addRole(RangerHelper.GRAVITINO_OWNER_ROLE);
}
- policy.getPolicyItems().add(policyItem);
+ policy.addPolicyItem(policyItem);
});
return policy;
}
@@ -278,11 +277,10 @@ public abstract class RangerAuthorizationPlugin
ownerPrivilege -> {
// Each owner's privilege will create one RangerPolicyItemAccess
in the policy
RangerPolicy.RangerPolicyItem policyItem = new
RangerPolicy.RangerPolicyItem();
- policyItem
- .getAccesses()
- .add(new
RangerPolicy.RangerPolicyItemAccess(ownerPrivilege.getName()));
-
policyItem.getRoles().add(rangerHelper.generateGravitinoRoleName(ownerRoleName));
- policy.getPolicyItems().add(policyItem);
+ policyItem.addAccess(
+ new
RangerPolicy.RangerPolicyItemAccess(ownerPrivilege.getName()));
+
policyItem.addRole(rangerHelper.generateGravitinoRoleName(ownerRoleName));
+ policy.addPolicyItem(policyItem);
});
return policy;
}
diff --git
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerClientExtension.java
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerClientExtension.java
index 65832de545..383b6cc6b7 100644
---
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerClientExtension.java
+++
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerClientExtension.java
@@ -20,10 +20,10 @@ package org.apache.gravitino.authorization.ranger;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
-import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.UniformInterfaceException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
+import java.util.Date;
import java.util.Map;
import javax.ws.rs.HttpMethod;
import javax.ws.rs.core.Response;
@@ -67,10 +67,6 @@ public class RangerClientExtension extends RangerClient {
// apache/ranger/intg/src/main/java/org/apache/ranger/RangerClient.java
// The private method callAPI of Ranger is called by reflection
- // private <T> T callAPI(API api, Map<String, String> params, Object
request, GenericType<T>
- // responseType) throws RangerServiceException
- private Method callAPIMethodGenericResponseType;
-
// private <T> T callAPI(API api, Map<String, String> params, Object
request, Class<T>
// responseType) throws RangerServiceException
private Method callAPIMethodClassResponseType;
@@ -82,11 +78,6 @@ public class RangerClientExtension extends RangerClient {
// initialize callAPI method
try {
- callAPIMethodGenericResponseType =
- RangerClient.class.getDeclaredMethod(
- "callAPI", API.class, Map.class, Object.class,
GenericType.class);
- callAPIMethodGenericResponseType.setAccessible(true);
-
callAPIMethodNonResponse =
RangerClient.class.getDeclaredMethod("callAPI", API.class,
Map.class);
callAPIMethodNonResponse.setAccessible(true);
@@ -104,7 +95,7 @@ public class RangerClientExtension extends RangerClient {
public RangerPolicy createPolicy(RangerPolicy policy) throws
RangerServiceException {
Preconditions.checkArgument(
policy.getResources().size() > 0, "Ranger policy resources cannot be
empty!");
- return super.createPolicy(policy);
+ return withoutServerManagedTimes(policy, () -> super.createPolicy(policy));
}
@Override
@@ -112,7 +103,7 @@ public class RangerClientExtension extends RangerClient {
throws RangerServiceException {
Preconditions.checkArgument(
policy.getResources().size() > 0, "Ranger policy resources cannot be
empty!");
- return super.updatePolicy(policyId, policy);
+ return withoutServerManagedTimes(policy, () ->
super.updatePolicy(policyId, policy));
}
public Boolean createUser(VXUser user) throws RuntimeException {
@@ -196,4 +187,28 @@ public class RangerClientExtension extends RangerClient {
}
return true;
}
+
+ /**
+ * Sends the policy without createTime and updateTime, which Ranger Admin
sets itself. Since
+ * Ranger 2.8 the client writes dates as "yyyyMMdd-HH:mm:ss.SSS-Z", which
Ranger Admin 2.4 rejects
+ * with HTTP 400. The caller's policy is left unchanged.
+ */
+ private RangerPolicy withoutServerManagedTimes(RangerPolicy policy,
PolicyCall call)
+ throws RangerServiceException {
+ Date createTime = policy.getCreateTime();
+ Date updateTime = policy.getUpdateTime();
+ policy.setCreateTime(null);
+ policy.setUpdateTime(null);
+ try {
+ return call.run();
+ } finally {
+ policy.setCreateTime(createTime);
+ policy.setUpdateTime(updateTime);
+ }
+ }
+
+ @FunctionalInterface
+ private interface PolicyCall {
+ RangerPolicy run() throws RangerServiceException;
+ }
}
diff --git
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java
index 453f3dc489..05caf64039 100644
---
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java
+++
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerHelper.java
@@ -23,7 +23,8 @@ import com.google.common.collect.Sets;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
-import org.apache.commons.lang.StringUtils;
+import java.util.stream.Stream;
+import org.apache.commons.lang3.StringUtils;
import org.apache.gravitino.authorization.AuthorizationPrivilege;
import org.apache.gravitino.authorization.AuthorizationSecurableObject;
import org.apache.gravitino.authorization.Owner;
@@ -142,12 +143,12 @@ public class RangerHelper {
RangerPolicy.RangerPolicyItemAccess access =
new RangerPolicy.RangerPolicyItemAccess();
access.setType(rangerPrivilege.getName());
- policyItem.getAccesses().add(access);
- policyItem.getRoles().add(generateGravitinoRoleName(roleName));
+ policyItem.addAccess(access);
+ policyItem.addRole(generateGravitinoRoleName(roleName));
if (Privilege.Condition.ALLOW == rangerPrivilege.condition()) {
- policy.getPolicyItems().add(policyItem);
+ policy.addPolicyItem(policyItem);
} else {
- policy.getDenyPolicyItems().add(policyItem);
+ policy.addDenyPolicyItem(policyItem);
}
} else {
// If the policy item exists, then add the role to the policy
item
@@ -158,7 +159,7 @@ public class RangerHelper {
if (!policyItem
.getRoles()
.contains(generateGravitinoRoleName(roleName))) {
-
policyItem.getRoles().add(generateGravitinoRoleName(roleName));
+
policyItem.addRole(generateGravitinoRoleName(roleName));
}
});
}
@@ -171,11 +172,15 @@ public class RangerHelper {
}
public static boolean hasGravitinoManagedPolicyItem(RangerPolicy policy) {
- List<RangerPolicy.RangerPolicyItem> policyItems = policy.getPolicyItems();
- policyItems.addAll(policy.getDenyPolicyItems());
- policyItems.addAll(policy.getRowFilterPolicyItems());
- policyItems.addAll(policy.getDataMaskPolicyItems());
- return
policyItems.stream().anyMatch(RangerHelper::isGravitinoManagedPolicyItemAccess);
+ // Do not add the items to policy.getPolicyItems(): that list belongs to
the policy, which is
+ // sent back to Ranger later, and it can be immutable in Ranger 2.8.
+ return Stream.<List<? extends RangerPolicy.RangerPolicyItem>>of(
+ policy.getPolicyItems(),
+ policy.getDenyPolicyItems(),
+ policy.getRowFilterPolicyItems(),
+ policy.getDataMaskPolicyItems())
+ .flatMap(List::stream)
+ .anyMatch(RangerHelper::isGravitinoManagedPolicyItemAccess);
}
public void removeAllGravitinoManagedPolicyItem(RangerPolicy policy) {
@@ -360,11 +365,11 @@ public class RangerHelper {
if (newOwner != null) {
if (newOwner.type() == Owner.Type.USER) {
if (!policyItem.getUsers().contains(newOwner.name())) {
- policyItem.getUsers().add(newOwner.name());
+ policyItem.addUser(newOwner.name());
}
} else {
if (!policyItem.getGroups().contains(newOwner.name())) {
- policyItem.getGroups().add(newOwner.name());
+ policyItem.addGroup(newOwner.name());
}
}
}
@@ -388,19 +393,18 @@ public class RangerHelper {
// Add lost owner's privilege to the policy
ownerPrivilege -> {
RangerPolicy.RangerPolicyItem policyItem = new
RangerPolicy.RangerPolicyItem();
- policyItem
- .getAccesses()
- .add(new
RangerPolicy.RangerPolicyItemAccess(ownerPrivilege.getName()));
+ policyItem.addAccess(
+ new
RangerPolicy.RangerPolicyItemAccess(ownerPrivilege.getName()));
if (newOwner != null) {
if (newOwner.type() == Owner.Type.USER) {
- policyItem.getUsers().add(newOwner.name());
+ policyItem.addUser(newOwner.name());
} else {
- policyItem.getGroups().add(newOwner.name());
+ policyItem.addGroup(newOwner.name());
}
// mark the policy item is created by Gravitino
addRoleToPolicyItemIfNoExists(policyItem,
GRAVITINO_OWNER_ROLE);
}
- policy.getPolicyItems().add(policyItem);
+ policy.addPolicyItem(policyItem);
});
}
@@ -447,11 +451,10 @@ public class RangerHelper {
// Add lost owner's privilege to the policy
ownerPrivilege -> {
RangerPolicy.RangerPolicyItem policyItem = new
RangerPolicy.RangerPolicyItem();
- policyItem
- .getAccesses()
- .add(new
RangerPolicy.RangerPolicyItemAccess(ownerPrivilege.getName()));
+ policyItem.addAccess(
+ new
RangerPolicy.RangerPolicyItemAccess(ownerPrivilege.getName()));
addRoleToPolicyItemIfNoExists(policyItem, ownerRoleName);
- policy.getPolicyItems().add(policyItem);
+ policy.addPolicyItem(policyItem);
});
}
@@ -459,7 +462,7 @@ public class RangerHelper {
RangerPolicy.RangerPolicyItem policyItem, String roleName) {
String gravitinoRoleName = generateGravitinoRoleName(roleName);
if (!policyItem.getRoles().contains(gravitinoRoleName)) {
- policyItem.getRoles().add(gravitinoRoleName);
+ policyItem.addRole(gravitinoRoleName);
}
}
diff --git
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/JsonDateSerializer.java
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/JsonDateSerializer.java
index 51dfe6fcd8..f4e5e5b96d 100644
---
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/JsonDateSerializer.java
+++
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/JsonDateSerializer.java
@@ -18,12 +18,12 @@
*/
package org.apache.gravitino.authorization.ranger.reference;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.JsonSerializer;
+import com.fasterxml.jackson.databind.SerializerProvider;
import java.io.IOException;
import java.text.SimpleDateFormat;
import java.util.Date;
-import org.codehaus.jackson.JsonGenerator;
-import org.codehaus.jackson.map.JsonSerializer;
-import org.codehaus.jackson.map.SerializerProvider;
/**
* Used to serialize Java.util.Date, which is not a common JSON type, so we
have to create a custom
diff --git
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXDataObject.java
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXDataObject.java
index 8a9af0bdf8..2caa65521b 100644
---
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXDataObject.java
+++
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXDataObject.java
@@ -18,18 +18,19 @@
*/
package org.apache.gravitino.authorization.ranger.reference;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import java.util.Date;
-import org.codehaus.jackson.annotate.JsonAutoDetect;
-import org.codehaus.jackson.annotate.JsonAutoDetect.Visibility;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
//
apache/ranger/security-admin/src/main/java/org/apache/ranger/view/VXDataObject.java
@JsonAutoDetect(
getterVisibility = Visibility.NONE,
setterVisibility = Visibility.NONE,
fieldVisibility = Visibility.ANY)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
public class VXDataObject implements java.io.Serializable {
private static final long serialVersionUID = 1L;
diff --git
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXGroup.java
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXGroup.java
index 611127ec3f..b5b59b0d15 100644
---
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXGroup.java
+++
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXGroup.java
@@ -18,16 +18,16 @@
*/
package org.apache.gravitino.authorization.ranger.reference;
-import org.codehaus.jackson.annotate.JsonAutoDetect;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
//
apache/ranger/security-admin/src/main/java/org/apache/ranger/view/VXGroup.java
@JsonAutoDetect(
getterVisibility = JsonAutoDetect.Visibility.NONE,
setterVisibility = JsonAutoDetect.Visibility.NONE,
fieldVisibility = JsonAutoDetect.Visibility.ANY)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
public class VXGroup extends VXDataObject implements java.io.Serializable {
private static final long serialVersionUID = 1L;
diff --git
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXGroupList.java
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXGroupList.java
index f600142f6c..60b7c14e2b 100644
---
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXGroupList.java
+++
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXGroupList.java
@@ -18,17 +18,19 @@
*/
package org.apache.gravitino.authorization.ranger.reference;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.ArrayList;
import java.util.List;
-import org.codehaus.jackson.annotate.JsonAutoDetect;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
//
apache/ranger/security-admin/src/main/java/org/apache/ranger/view/VXGroupList.java
@JsonAutoDetect(
getterVisibility = JsonAutoDetect.Visibility.NONE,
setterVisibility = JsonAutoDetect.Visibility.NONE,
fieldVisibility = JsonAutoDetect.Visibility.ANY)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
public class VXGroupList extends VList {
private static final long serialVersionUID = 1L;
diff --git
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXUser.java
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXUser.java
index 3dbc2b0236..bd8bd33b6b 100644
---
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXUser.java
+++
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXUser.java
@@ -18,17 +18,17 @@
*/
package org.apache.gravitino.authorization.ranger.reference;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.Collection;
-import org.codehaus.jackson.annotate.JsonAutoDetect;
-import org.codehaus.jackson.annotate.JsonIgnoreProperties;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
//
apache/ranger/security-admin/src/main/java/org/apache/ranger/view/VXUser.java
@JsonAutoDetect(
getterVisibility = JsonAutoDetect.Visibility.NONE,
setterVisibility = JsonAutoDetect.Visibility.NONE,
fieldVisibility = JsonAutoDetect.Visibility.ANY)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
public class VXUser extends VXDataObject implements java.io.Serializable {
private static final long serialVersionUID = 1L;
diff --git
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXUserList.java
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXUserList.java
index cef6e3a5c7..90e3bdaa2b 100644
---
a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXUserList.java
+++
b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXUserList.java
@@ -18,17 +18,19 @@
*/
package org.apache.gravitino.authorization.ranger.reference;
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.fasterxml.jackson.annotation.JsonInclude;
import java.util.ArrayList;
import java.util.List;
-import org.codehaus.jackson.annotate.JsonAutoDetect;
-import org.codehaus.jackson.map.annotate.JsonSerialize;
//
apache/ranger/security-admin/src/main/java/org/apache/ranger/view/VXUserList.java
@JsonAutoDetect(
getterVisibility = JsonAutoDetect.Visibility.NONE,
setterVisibility = JsonAutoDetect.Visibility.NONE,
fieldVisibility = JsonAutoDetect.Visibility.ANY)
-@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+@JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
public class VXUserList extends VList {
private static final long serialVersionUID = 1L;
diff --git
a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerClientExtension.java
b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerClientExtension.java
new file mode 100644
index 0000000000..1d9971a657
--- /dev/null
+++
b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerClientExtension.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.authorization.ranger;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpServer;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.nio.charset.StandardCharsets;
+import java.time.Instant;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArrayList;
+import org.apache.gravitino.authorization.ranger.reference.VXGroup;
+import org.apache.gravitino.authorization.ranger.reference.VXGroupList;
+import org.apache.gravitino.authorization.ranger.reference.VXUser;
+import org.apache.gravitino.authorization.ranger.reference.VXUserList;
+import org.apache.ranger.RangerServiceException;
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/** Exercises {@link RangerClientExtension} over HTTP against a stub Ranger
Admin. */
+public class TestRangerClientExtension {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+ // The stub server and the client use the same IPv4 address, also when IPv6
is preferred.
+ private static final String LOOPBACK = "127.0.0.1";
+
+ private final List<String> requests = new CopyOnWriteArrayList<>();
+ private final Map<String, String> requestBodies = new ConcurrentHashMap<>();
+ private HttpServer server;
+ private RangerClientExtension client;
+
+ @BeforeEach
+ public void setUp() throws IOException {
+ server = HttpServer.create(new InetSocketAddress(LOOPBACK, 0), 0);
+ server.createContext(
+ "/service/xusers/users/external",
+ exchange -> respond(exchange, 200, userJson(7L, readBody(exchange))));
+ server.createContext(
+ "/service/xusers/users",
+ exchange -> {
+ if ("DELETE".equals(exchange.getRequestMethod())) {
+ respond(exchange, 204, null);
+ } else {
+ respond(
+ exchange,
+ 200,
+
"{\"startIndex\":0,\"pageSize\":200,\"totalCount\":1,\"resultSize\":1,"
+ + "\"sortType\":\"asc\",\"sortBy\":\"id\",\"vXUsers\":["
+ + userJson(7L, "{\"name\":\"user1\"}")
+ + "]}");
+ }
+ });
+ server.createContext(
+ "/service/xusers/groups",
+ exchange -> {
+ String method = exchange.getRequestMethod();
+ if ("DELETE".equals(method)) {
+ respond(exchange, 204, null);
+ } else if ("POST".equals(method)) {
+ readBody(exchange);
+ respond(exchange, 200,
"{\"id\":3,\"name\":\"group1\",\"groupSource\":0,\"x\":1}");
+ } else {
+ respond(
+ exchange,
+ 200,
+
"{\"totalCount\":1,\"vXGroups\":[{\"id\":3,\"name\":\"group1\",\"x\":1}]}");
+ }
+ });
+ server.createContext(
+ "/public/v2/api/service/name/",
+ exchange -> respond(exchange, 404,
"{\"statusCode\":1,\"msgDesc\":\"not found\"}"));
+ server.createContext(
+ "/service/public/v2/api/policy",
+ exchange -> {
+ readBody(exchange);
+ respond(exchange, 200,
"{\"id\":11,\"name\":\"p1\",\"createTime\":1714557600000}");
+ });
+ server.start();
+
+ String url = "http://"; + LOOPBACK + ":" + server.getAddress().getPort();
+ client = new RangerClientExtension(url, "simple", "admin", "password");
+ }
+
+ @AfterEach
+ public void tearDown() {
+ if (server != null) {
+ server.stop(0);
+ }
+ }
+
+ @Test
+ public void testUserApis() throws Exception {
+ Assertions.assertTrue(
+
client.createUser(VXUser.builder().withName("user1").withDescription("d").build()));
+ JsonNode body = MAPPER.readTree(requestBodies.get("POST
/service/xusers/users/external"));
+ Assertions.assertEquals("user1", body.get("name").asText());
+ Assertions.assertEquals("d", body.get("description").asText());
+ Assertions.assertTrue(body.has("password"));
+ Assertions.assertFalse(body.has("id"));
+
+ VXUserList users = client.searchUser(ImmutableMap.of("name", "user1"));
+ Assertions.assertEquals(1, users.getListSize());
+ Assertions.assertEquals(7L, users.getList().get(0).getId());
+ Assertions.assertTrue(requests.contains("GET
/service/xusers/users?name=user1"));
+
+ Assertions.assertTrue(client.deleteUser(7L));
+ Assertions.assertTrue(requests.contains("DELETE
/service/xusers/users/7?forceDelete=true"));
+ }
+
+ @Test
+ public void testGroupApis() {
+
Assertions.assertTrue(client.createGroup(VXGroup.builder().withName("group1").build()));
+
+ VXGroupList groups = client.searchGroup(ImmutableMap.of("name", "group1"));
+ Assertions.assertEquals(1, groups.getListSize());
+ Assertions.assertEquals(3L, groups.getList().get(0).getId());
+
+ Assertions.assertTrue(client.deleteGroup(3L));
+ Assertions.assertTrue(requests.contains("DELETE
/service/xusers/groups/3?forceDelete=true"));
+ }
+
+ @Test
+ public void testPolicyRequestsOmitServerManagedTimes() throws Exception {
+ Date createTime = Date.from(Instant.ofEpochMilli(1714557600000L));
+ Date updateTime = Date.from(Instant.ofEpochMilli(1714557601000L));
+ RangerPolicy policy = new RangerPolicy();
+ policy.setId(11L);
+ policy.setName("p1");
+ policy.setResource("database", new
RangerPolicy.RangerPolicyResource("db1"));
+ policy.setCreateTime(createTime);
+ policy.setUpdateTime(updateTime);
+
+ RangerPolicy updated = client.updatePolicy(11L, policy);
+ RangerPolicy created = client.createPolicy(policy);
+
+ for (String request :
+ ImmutableList.of(
+ "PUT /service/public/v2/api/policy/11", "POST
/service/public/v2/api/policy")) {
+ JsonNode body = MAPPER.readTree(requestBodies.get(request));
+ Assertions.assertEquals("p1", body.get("name").asText(), request);
+ Assertions.assertTrue(body.has("resources"), request);
+ Assertions.assertFalse(body.has("createTime"), request);
+ Assertions.assertFalse(body.has("updateTime"), request);
+ }
+ // The caller's policy keeps its values, and responses are parsed as
before.
+ Assertions.assertEquals(createTime, policy.getCreateTime());
+ Assertions.assertEquals(updateTime, policy.getUpdateTime());
+ Assertions.assertEquals(createTime, updated.getCreateTime());
+ Assertions.assertEquals(11L, created.getId());
+ }
+
+ @Test
+ public void testServiceNotFoundStatus() {
+ RangerServiceException e =
+ Assertions.assertThrows(RangerServiceException.class, () ->
client.getService("absent"));
+ Assertions.assertEquals(ClientResponse.Status.NOT_FOUND, e.getStatus());
+ }
+
+ private static String userJson(long id, String requestBody) throws
IOException {
+ JsonNode request = MAPPER.readTree(requestBody);
+ return "{\"id\":"
+ + id
+ + ",\"createDate\":\"2024-05-01T10:00:00Z\",\"name\":\""
+ + request.get("name").asText()
+ +
"\",\"status\":1,\"isVisible\":1,\"userSource\":1,\"groupIdList\":[],"
+ +
"\"userRoleList\":[\"ROLE_USER\"],\"fieldAddedByNewerRangerAdmin\":true}";
+ }
+
+ private String readBody(HttpExchange exchange) throws IOException {
+ try (InputStream in = exchange.getRequestBody()) {
+ String body = new String(in.readAllBytes(), StandardCharsets.UTF_8);
+ requestBodies.put(exchange.getRequestMethod() + " " +
exchange.getRequestURI(), body);
+ return body;
+ }
+ }
+
+ private void respond(HttpExchange exchange, int status, String body) throws
IOException {
+ requests.add(exchange.getRequestMethod() + " " + exchange.getRequestURI());
+ if (body == null) {
+ exchange.sendResponseHeaders(status, -1);
+ exchange.close();
+ return;
+ }
+ byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
+ exchange.getResponseHeaders().add("Content-Type", "application/json");
+ exchange.sendResponseHeaders(status, bytes.length);
+ try (OutputStream out = exchange.getResponseBody()) {
+ out.write(bytes);
+ }
+ }
+}
diff --git
a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerHelper.java
b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerHelper.java
index 09c985d3af..6f095eb73e 100644
---
a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerHelper.java
+++
b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerHelper.java
@@ -104,13 +104,11 @@ public class TestRangerHelper {
private RangerPolicy createLegacyCatalogOwnerPolicy() {
RangerPolicy policy = new RangerPolicy();
RangerPolicy.RangerPolicyItem policyItem = new
RangerPolicy.RangerPolicyItem();
- policyItem
- .getAccesses()
- .add(
- new RangerPolicy.RangerPolicyItemAccess(
- RangerPrivileges.RangerHadoopSQLPrivilege.SELECT.getName()));
- policyItem.getRoles().add(RangerHelper.GRAVITINO_CATALOG_OWNER_ROLE);
- policy.getPolicyItems().add(policyItem);
+ policyItem.addAccess(
+ new RangerPolicy.RangerPolicyItemAccess(
+ RangerPrivileges.RangerHadoopSQLPrivilege.SELECT.getName()));
+ policyItem.addRole(RangerHelper.GRAVITINO_CATALOG_OWNER_ROLE);
+ policy.addPolicyItem(policyItem);
return policy;
}
}
diff --git
a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerImmutableModelDefaults.java
b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerImmutableModelDefaults.java
new file mode 100644
index 0000000000..ae052ed06e
--- /dev/null
+++
b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/TestRangerImmutableModelDefaults.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.authorization.ranger;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import java.util.Collections;
+import org.apache.gravitino.authorization.Owner;
+import org.apache.ranger.RangerClient;
+import org.apache.ranger.plugin.model.RangerBaseModelObject;
+import org.apache.ranger.plugin.model.RangerPolicy;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.Mockito;
+
+/**
+ * Builds Ranger policies with both null-safe collection modes of the Ranger
model classes.
+ *
+ * <p>Ranger 2.8 keeps this mode in a JVM-wide static. The default ("v1")
gives every model object
+ * mutable empty collections. Initializing a {@link
+ * org.apache.ranger.plugin.service.RangerBasePlugin} in the same JVM, as the
Spark authz plugin
+ * used by other tests and by some deployments does, switches it to "v2",
where empty collections
+ * are immutable. The tests restore the exact previous mode so that they leave
the JVM as they found
+ * it.
+ */
+public class TestRangerImmutableModelDefaults {
+
+ private RangerBaseModelObject.NullSafeSupplier previousSupplier;
+ private RangerHelper rangerHelper;
+
+ @BeforeEach
+ public void setUp() {
+ previousSupplier = SupplierAccess.current();
+ rangerHelper =
+ new RangerHelper(
+ Mockito.mock(RangerClient.class),
+ "admin",
+ "service",
+ ImmutableSet.of(RangerPrivileges.RangerHadoopSQLPrivilege.ALL),
+ ImmutableList.of());
+ }
+
+ @AfterEach
+ public void tearDown() {
+ RangerBaseModelObject.setNullSafeSupplier(previousSupplier);
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"v1", RangerBaseModelObject.NULL_SAFE_SUPPLIER_V2})
+ public void testUpdatePolicyOwnerOnNewPolicy(String supplier) {
+ RangerBaseModelObject.setNullSafeSupplier(supplier);
+ Owner owner = Mockito.mock(Owner.class);
+ Mockito.when(owner.type()).thenReturn(Owner.Type.USER);
+ Mockito.when(owner.name()).thenReturn("user1");
+ RangerPolicy policy = new RangerPolicy();
+
+ rangerHelper.updatePolicyOwner(policy, null, owner);
+
+ Assertions.assertEquals(1, policy.getPolicyItems().size());
+ RangerPolicy.RangerPolicyItem policyItem = policy.getPolicyItems().get(0);
+ Assertions.assertEquals(Collections.singletonList("user1"),
policyItem.getUsers());
+ Assertions.assertEquals(
+ Collections.singletonList(RangerHelper.GRAVITINO_OWNER_ROLE),
policyItem.getRoles());
+ Assertions.assertEquals(
+ RangerPrivileges.RangerHadoopSQLPrivilege.ALL.getName(),
+ policyItem.getAccesses().get(0).getType());
+
+ // Changing the owner removes the previous one from the same item.
+ Owner newOwner = Mockito.mock(Owner.class);
+ Mockito.when(newOwner.type()).thenReturn(Owner.Type.USER);
+ Mockito.when(newOwner.name()).thenReturn("user2");
+ rangerHelper.updatePolicyOwner(policy, owner, newOwner);
+ Assertions.assertEquals(
+ Collections.singletonList("user2"),
policy.getPolicyItems().get(0).getUsers());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"v1", RangerBaseModelObject.NULL_SAFE_SUPPLIER_V2})
+ public void testUpdatePolicyOwnerRoleOnNewPolicy(String supplier) {
+ RangerBaseModelObject.setNullSafeSupplier(supplier);
+ RangerPolicy policy = new RangerPolicy();
+ String ownerRole = RangerHelper.generateCatalogOwnerRoleName("1001");
+
+ rangerHelper.updatePolicyOwnerRole(
+ policy, ownerRole, RangerHelper.GRAVITINO_CATALOG_OWNER_ROLE);
+
+ Assertions.assertEquals(1, policy.getPolicyItems().size());
+ Assertions.assertEquals(
+ Collections.singletonList(ownerRole),
policy.getPolicyItems().get(0).getRoles());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"v1", RangerBaseModelObject.NULL_SAFE_SUPPLIER_V2})
+ public void testHasGravitinoManagedPolicyItemDoesNotChangePolicy(String
supplier) {
+ RangerBaseModelObject.setNullSafeSupplier(supplier);
+ RangerPolicy policy = new RangerPolicy();
+ RangerPolicy.RangerPolicyItem denyItem = new
RangerPolicy.RangerPolicyItem();
+ denyItem.addRole(RangerHelper.GRAVITINO_OWNER_ROLE);
+ policy.addDenyPolicyItem(denyItem);
+
+ Assertions.assertTrue(RangerHelper.hasGravitinoManagedPolicyItem(policy));
+ Assertions.assertTrue(policy.getPolicyItems().isEmpty());
+ Assertions.assertEquals(1, policy.getDenyPolicyItems().size());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"v1", RangerBaseModelObject.NULL_SAFE_SUPPLIER_V2})
+ public void testCreatePolicyAddResourcesOnNewPolicy(String supplier) {
+ RangerBaseModelObject.setNullSafeSupplier(supplier);
+ RangerAuthorizationHadoopSQLPlugin plugin =
+ Mockito.mock(RangerAuthorizationHadoopSQLPlugin.class,
Mockito.CALLS_REAL_METHODS);
+
+ RangerPolicy policy =
+ plugin.createPolicyAddResources(
+ new RangerHadoopSQLMetadataObject(
+ "db1", "table1", RangerHadoopSQLMetadataObject.Type.TABLE));
+
+ Assertions.assertEquals(2, policy.getResources().size());
+ Assertions.assertEquals(
+ Collections.singletonList("db1"),
+
policy.getResources().get(plugin.policyResourceDefinesRule().get(0)).getValues());
+ Assertions.assertEquals(
+ Collections.singletonList("table1"),
+
policy.getResources().get(plugin.policyResourceDefinesRule().get(1)).getValues());
+ }
+
+ /** Reads the current JVM-wide supplier, which Ranger only exposes to
subclasses. */
+ private static class SupplierAccess extends RangerBaseModelObject {
+ private static final long serialVersionUID = 1L;
+
+ static NullSafeSupplier current() {
+ return new SupplierAccess().getNullSafeSupplier();
+ }
+ }
+}
diff --git
a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java
b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java
index 5828d6cbbc..869729c1e6 100644
---
a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java
+++
b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerHiveIT.java
@@ -2192,7 +2192,7 @@ public class RangerHiveIT {
for (RangerPolicy.RangerPolicyItem policyItem : policy.getPolicyItems())
{
if (policyItem.getRoles().remove(currentOwnerRole)) {
if (!policyItem.getRoles().contains(legacyOwnerRole)) {
- policyItem.getRoles().add(legacyOwnerRole);
+ policyItem.addRole(legacyOwnerRole);
}
replaced = true;
}
diff --git
a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java
b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java
index 997ba63aca..c30829fe67 100644
---
a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java
+++
b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/integration/test/RangerITEnv.java
@@ -36,6 +36,7 @@ import
org.apache.gravitino.authorization.common.RangerAuthorizationProperties;
import org.apache.gravitino.authorization.ranger.RangerAuthorizationHDFSPlugin;
import
org.apache.gravitino.authorization.ranger.RangerAuthorizationHadoopSQLPlugin;
import org.apache.gravitino.authorization.ranger.RangerAuthorizationPlugin;
+import org.apache.gravitino.authorization.ranger.RangerClientExtension;
import org.apache.gravitino.authorization.ranger.RangerHelper;
import org.apache.gravitino.authorization.ranger.RangerPrivileges;
import org.apache.gravitino.authorization.ranger.reference.RangerDefines;
@@ -99,7 +100,14 @@ public class RangerITEnv {
String.format(
"http://%s:%d";,
containerSuite.getRangerContainer().getContainerIpAddress(),
RANGER_SERVER_PORT);
- rangerClient = containerSuite.getRangerContainer().rangerClient;
+ // Use the client of the plugin, so that test updates of policies fetched
from Ranger Admin
+ // are sent in a format that every supported Ranger Admin version accepts.
+ rangerClient =
+ new RangerClientExtension(
+ RANGER_ADMIN_URL,
+ RangerContainer.authType,
+ RangerContainer.rangerUserName,
+ RangerContainer.rangerPassword);
rangerAuthHivePlugin =
new RangerAuthorizationHadoopSQLPlugin(
@@ -517,7 +525,7 @@ public class RangerITEnv {
Assertions.assertTrue(policies.size() <= 1);
if (!policies.isEmpty()) {
RangerPolicy policy = policies.get(0);
- policy.getPolicyItems().addAll(policyItems);
+ policyItems.forEach(policy::addPolicyItem);
rangerClient.updatePolicy(policy.getId(), policy);
} else {
RangerPolicy policy = new RangerPolicy();
diff --git
a/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/reference/TestRangerReferenceJson.java
b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/reference/TestRangerReferenceJson.java
new file mode 100644
index 0000000000..6c68f3bd96
--- /dev/null
+++
b/authorizations/authorization-ranger/src/test/java/org/apache/gravitino/authorization/ranger/reference/TestRangerReferenceJson.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.authorization.ranger.reference;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import java.text.SimpleDateFormat;
+import java.time.Instant;
+import java.util.Date;
+import java.util.TimeZone;
+import org.apache.ranger.authorization.utils.JsonUtils;
+import org.apache.ranger.plugin.util.JsonUtilsV2;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Pins the JSON exchanged with Ranger Admin for the user and group APIs.
Since Ranger 2.8 the
+ * client serializes requests with {@link JsonUtils} (Jackson 2) and parses
responses with {@link
+ * JsonUtilsV2} (Jackson 2), whereas Ranger 2.4 used Gson for requests and
Jackson 1 for responses.
+ */
+public class TestRangerReferenceJson {
+
+ // The Gson configuration of RangerRESTClient in Ranger 2.4, used to
serialize request bodies.
+ private static final Gson RANGER_24_REQUEST_GSON =
+ new GsonBuilder().setDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ").create();
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ @Test
+ public void testUserRequestMatchesRanger24Format() throws Exception {
+ VXUser user =
VXUser.builder().withName("user1").withDescription("desc").build();
+
+ JsonNode actual = MAPPER.readTree(JsonUtils.objectToJson(user));
+ JsonNode expected = MAPPER.readTree(RANGER_24_REQUEST_GSON.toJson(user));
+
+ Assertions.assertEquals(expected, actual);
+ Assertions.assertEquals("user1", actual.get("name").asText());
+ Assertions.assertEquals("rangerR0cks!", actual.get("password").asText());
+ Assertions.assertEquals(RangerCommonEnums.STATUS_ENABLED,
actual.get("status").asInt());
+ Assertions.assertEquals(RangerCommonEnums.IS_VISIBLE,
actual.get("isVisible").asInt());
+ Assertions.assertFalse(actual.has("id"));
+ Assertions.assertFalse(actual.has("createDate"));
+ }
+
+ @Test
+ public void testGroupRequestMatchesRanger24Format() throws Exception {
+ VXGroup group =
VXGroup.builder().withName("group1").withDescription("desc").build();
+
+ JsonNode actual = MAPPER.readTree(JsonUtils.objectToJson(group));
+ JsonNode expected = MAPPER.readTree(RANGER_24_REQUEST_GSON.toJson(group));
+
+ Assertions.assertEquals(expected, actual);
+ Assertions.assertEquals("group1", actual.get("name").asText());
+ Assertions.assertEquals(RangerCommonEnums.GROUP_INTERNAL,
actual.get("groupSource").asInt());
+ Assertions.assertFalse(actual.has("id"));
+ }
+
+ @Test
+ public void testDateUsesRangerAdminFormat() throws Exception {
+ VXUser user = VXUser.builder().withName("user1").build();
+ Date epoch = Date.from(Instant.EPOCH);
+ user.createDate = epoch;
+
+ JsonNode actual = MAPPER.readTree(JsonUtils.objectToJson(user));
+
+ // JsonDateSerializer, copied from Ranger Admin, formats in the JVM
default time zone, so the
+ // expected value does too. Pinning it to UTC would fail in other time
zones.
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
+ Assertions.assertEquals(format.format(epoch),
actual.get("createDate").asText());
+ }
+
+ @Test
+ public void testParseUserResponses() throws Exception {
+ String userJson =
+
"{\"id\":7,\"createDate\":\"2024-05-01T10:00:00Z\",\"updateDate\":\"2024-05-01T10:00:00Z\","
+ +
"\"owner\":\"Admin\",\"updatedBy\":\"Admin\",\"name\":\"user1\",\"firstName\":\"user1\","
+ +
"\"password\":\"*****\",\"description\":\"desc\",\"groupIdList\":[1,2],"
+ +
"\"groupNameList\":[\"g1\",\"g2\"],\"status\":1,\"isVisible\":1,\"userSource\":1,"
+ +
"\"userRoleList\":[\"ROLE_USER\"],\"otherAttributes\":\"{}\",\"syncSource\":\"ldap\","
+ + "\"fieldAddedByNewerRangerAdmin\":{\"nested\":true}}";
+ String userListJson =
+
"{\"startIndex\":0,\"pageSize\":200,\"totalCount\":1,\"resultSize\":1,\"sortType\":\"asc\","
+ +
"\"sortBy\":\"id\",\"queryTimeMS\":1714557600000,\"listSize\":1,\"vXUsers\":["
+ + userJson
+ + "]}";
+
+ VXUser user = JsonUtilsV2.jsonToObj(userJson, VXUser.class);
+ Assertions.assertEquals(7L, user.getId());
+ Assertions.assertEquals("user1", user.getName());
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
+ format.setTimeZone(TimeZone.getTimeZone("UTC"));
+ Assertions.assertEquals(format.parse("2024-05-01T10:00:00Z"),
user.createDate);
+
+ VXUserList userList = JsonUtilsV2.jsonToObj(userListJson,
VXUserList.class);
+ Assertions.assertEquals(1, userList.getListSize());
+ Assertions.assertEquals(200, userList.getPageSize());
+ Assertions.assertEquals(7L, userList.getList().get(0).getId());
+ Assertions.assertEquals("user1", userList.getList().get(0).getName());
+ }
+
+ @Test
+ public void testParseGroupResponses() throws Exception {
+ String groupJson =
+
"{\"id\":3,\"createDate\":\"2024-05-01T10:00:00Z\",\"name\":\"group1\","
+ +
"\"description\":\"desc\",\"groupType\":1,\"groupSource\":0,\"isVisible\":1,"
+ + "\"fieldAddedByNewerRangerAdmin\":\"x\"}";
+ String groupListJson =
+ "{\"startIndex\":0,\"pageSize\":200,\"totalCount\":1,\"resultSize\":1,"
+ + "\"vXGroups\":["
+ + groupJson
+ + "],\"fieldAddedByNewerRangerAdmin\":[]}";
+
+ VXGroup group = JsonUtilsV2.jsonToObj(groupJson, VXGroup.class);
+ Assertions.assertEquals(3L, group.getId());
+ Assertions.assertTrue(group.toString().contains("name={group1}"));
+
+ VXGroupList groupList = JsonUtilsV2.jsonToObj(groupListJson,
VXGroupList.class);
+ Assertions.assertEquals(1, groupList.getListSize());
+ Assertions.assertEquals(3L, groupList.getList().get(0).getId());
+ }
+
+ @Test
+ public void testParseEmptyListResponse() throws Exception {
+ VXUserList userList =
+ JsonUtilsV2.jsonToObj("{\"startIndex\":0,\"totalCount\":0}",
VXUserList.class);
+ Assertions.assertEquals(0, userList.getListSize());
+ }
+}
diff --git a/build.gradle.kts b/build.gradle.kts
index 3360a2e5d7..b52960826b 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -40,6 +40,7 @@ import org.gradle.kotlin.dsl.support.serviceOf
import java.io.IOException
import java.nio.ByteBuffer
import java.util.Locale
+import java.util.Properties
import java.util.zip.ZipFile
Locale.setDefault(Locale.US)
@@ -305,6 +306,95 @@ class LegalFilesTransformer(
}
}
+/**
+ * Removes the Apache Avro copy that hadoop-client-runtime shades under
+ * org/apache/hadoop/shaded/org/apache/avro when it is older than 1.11.4.
+ *
+ * Hadoop 3.3.6 shades Avro 1.7.7, and a dependency constraint cannot replace
classes that live inside
+ * another jar. Within Hadoop the shaded Avro only backs Avro serialization
(SequenceFile, MapReduce job
+ * history, `hadoop fs -text`), which Gravitino does not use; the FileSystem,
HDFS and Kerberos code
+ * paths do not load it. Other jars pass through unchanged. Remove this
transform once Hadoop is
+ * upgraded to a release that shades Avro 1.11.4 or later (Hadoop 3.4.2+).
+ */
+@CacheableTransform
+abstract class StripShadedAvroTransform :
TransformAction<TransformParameters.None> {
+ companion object {
+ val STRIPPED: Attribute<Boolean> =
+ Attribute.of("org.apache.gravitino.shaded-avro-stripped",
Boolean::class.javaObjectType)
+ const val SHADED_AVRO_PREFIX = "org/apache/hadoop/shaded/org/apache/avro/"
+ const val AVRO_METADATA_PREFIX = "META-INF/maven/org.apache.avro/"
+ const val AVRO_POM_PROPERTIES =
"META-INF/maven/org.apache.avro/avro/pom.properties"
+ private val MIN_AVRO_VERSION = listOf(1, 11, 4)
+
+ /** Returns true if the dotted [version] is lower than 1.11.4. */
+ fun isOutdatedAvro(version: String): Boolean {
+ val parts = version.split('.', '-').map { it.toIntOrNull() ?: 0 }
+ for (i in MIN_AVRO_VERSION.indices) {
+ val part = parts.getOrElse(i) { 0 }
+ if (part != MIN_AVRO_VERSION[i]) {
+ return part < MIN_AVRO_VERSION[i]
+ }
+ }
+ return false
+ }
+ }
+
+ @get:InputArtifact
+ @get:PathSensitive(PathSensitivity.NAME_ONLY)
+ abstract val inputArtifact: Provider<FileSystemLocation>
+
+ override fun transform(outputs: TransformOutputs) {
+ val input = inputArtifact.get().asFile
+ if (!input.isFile || !input.name.startsWith("hadoop-client-runtime-") ||
!input.name.endsWith(".jar")) {
+ outputs.file(inputArtifact)
+ return
+ }
+
+ val avroVersion = ZipFile(input).use { jar ->
+ jar.getEntry(AVRO_POM_PROPERTIES)?.let { entry ->
+ Properties().apply { jar.getInputStream(entry).use { load(it) }
}.getProperty("version")
+ }
+ }
+ if (avroVersion == null || !isOutdatedAvro(avroVersion)) {
+ outputs.file(inputArtifact)
+ return
+ }
+
+ // Keep the original file name so that the distribution layout and license
reports do not change.
+ val output = outputs.file(input.name)
+ var removed = 0
+ ZipFile(input).use { jar ->
+ ZipOutputStream(output).use { out ->
+ jar.entries().asSequence().forEach { source ->
+ if (source.name.startsWith(SHADED_AVRO_PREFIX) ||
source.name.startsWith(AVRO_METADATA_PREFIX)) {
+ removed++
+ return@forEach
+ }
+ val entry = ZipEntry(source.name)
+ entry.time = source.time
+ out.putNextEntry(entry)
+ if (!source.isDirectory) {
+ jar.getInputStream(source).use { it.copyTo(out) }
+ }
+ out.closeEntry()
+ }
+ }
+ }
+ if (removed == 0) {
+ throw GradleException("Expected shaded Avro $avroVersion entries in
${input.name}, found none")
+ }
+ }
+}
+
+// Projects whose runtime libraries ship hadoop-client-runtime in the
distribution.
+val shadedAvroStrippedProjects = setOf(
+ ":authorizations:authorization-ranger",
+ ":catalogs:catalog-fileset",
+ ":catalogs:catalog-lakehouse-iceberg",
+ ":iceberg:iceberg-rest-server",
+ ":iceberg:iceberg-rest-server-experiment"
+)
+
val snappyJavaVersion: String = libs.versions.snappy.java.get()
val scalaVersion: String = project.properties["scalaVersion"] as? String ?:
extra["defaultScalaVersion"].toString()
@@ -616,6 +706,29 @@ subprojects {
// The safe log4j-1.2-api bridge from Log4j 2.x is already included in the
log4j bundle.
exclude(group = "log4j", module = "log4j")
}
+ // Strip the outdated Avro shaded into hadoop-client-runtime, see
StripShadedAvroTransform.
+ // The test classpaths use the same jar so that tests run against what the
distribution ships.
+ if (project.path in shadedAvroStrippedProjects) {
+ dependencies {
+ attributesSchema {
+ attribute(StripShadedAvroTransform.STRIPPED)
+ }
+ artifactTypes.getByName(ArtifactTypeDefinition.JAR_TYPE) {
+ attributes.attribute(StripShadedAvroTransform.STRIPPED, false)
+ }
+ // The source is not restricted to the "jar" artifact type: with that
restriction, artifacts
+ // such as hadoop-yarn-server-tests on the test classpaths had no
matching variant. The
+ // transform passes every artifact other than hadoop-client-runtime
through unchanged.
+ registerTransform(StripShadedAvroTransform::class) {
+ from.attribute(StripShadedAvroTransform.STRIPPED, false)
+ to.attribute(StripShadedAvroTransform.STRIPPED, true)
+ }
+ }
+ configurations.matching { it.name == "runtimeClasspath" || it.name ==
"testRuntimeClasspath" }
+ .configureEach {
+ attributes.attribute(StripShadedAvroTransform.STRIPPED, true)
+ }
+ }
repositories {
mavenCentral()
diff --git a/catalogs/hive-metastore2-libs/build.gradle.kts
b/catalogs/hive-metastore2-libs/build.gradle.kts
index 0c151bef26..19b19aa851 100644
--- a/catalogs/hive-metastore2-libs/build.gradle.kts
+++ b/catalogs/hive-metastore2-libs/build.gradle.kts
@@ -39,6 +39,12 @@ dependencies {
implementation(libs.snappy.java)
implementation(libs.commons.beanutils)
implementation(libs.woodstox.core)
+ // Not a new dependency: hadoop-common, hadoop-auth, Curator and the Hive
shims already bring
+ // ZooKeeper transitively, resolved to 3.6.3 (end of life). Gravitino does
not use ZooKeeper; only
+ // Hadoop/Hive code paths that Gravitino does not call (HA failover,
ZooKeeper token stores and
+ // discovery) reference it. This constraint only upgrades the version that
is shipped.
+ implementation(libs.zookeeper)
+ implementation(libs.zookeeper.jute)
}
implementation(libs.hadoop3.common) {
@@ -60,6 +66,7 @@ dependencies {
exclude(group = "org.slf4j")
}
implementation(libs.hadoop3.mapreduce.client.core) {
+ exclude(group = "ch.qos.logback")
exclude(group = "ch.qos.reload4j")
exclude(group = "com.github.pjfanning", module = "jersey-json")
exclude(group = "com.github.spotbugs")
diff --git a/catalogs/hive-metastore3-libs/build.gradle.kts
b/catalogs/hive-metastore3-libs/build.gradle.kts
index 6e7f9c79af..0727796394 100644
--- a/catalogs/hive-metastore3-libs/build.gradle.kts
+++ b/catalogs/hive-metastore3-libs/build.gradle.kts
@@ -44,6 +44,12 @@ dependencies {
implementation(libs.snappy.java)
implementation(libs.commons.beanutils)
implementation(libs.woodstox.core)
+ // Not a new dependency: hadoop-common, hadoop-auth, Curator and the Hive
shims already bring
+ // ZooKeeper transitively, resolved to 3.6.3 (end of life). Gravitino does
not use ZooKeeper; only
+ // Hadoop/Hive code paths that Gravitino does not call (HA failover,
ZooKeeper token stores and
+ // discovery) reference it. This constraint only upgrades the version that
is shipped.
+ implementation(libs.zookeeper)
+ implementation(libs.zookeeper.jute)
}
implementation(libs.hadoop3.common) {
diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml
index 80fabd5aaa..d274a17618 100644
--- a/gradle/libs.versions.toml
+++ b/gradle/libs.versions.toml
@@ -154,7 +154,8 @@ paimon4flink22 = "1.4.1"
# Explicit ASM dependency (9.8) is required due to cglib's internal use and
Java 17 module system.
cglib = "3.3.0"
asm = "9.8"
-ranger = "2.4.0"
+ranger = "2.8.0"
+zookeeper = "3.8.8"
javax-jaxb-api = "2.3.1"
javax-ws-rs-api = "2.1.1"
spotless-plugin = '6.11.0'
@@ -374,6 +375,9 @@ datanucleus-rdbms = { group = "org.datanucleus", name =
"datanucleus-rdbms", ver
datanucleus-jdo = { group = "org.datanucleus", name = "javax.jdo", version.ref
= "datanucleus-jdo" }
ranger-intg = { group = "org.apache.ranger", name = "ranger-intg", version.ref
= "ranger" }
+ranger-ugsync-util = { group = "org.apache.ranger", name = "ugsync-util",
version.ref = "ranger" }
+zookeeper = { group = "org.apache.zookeeper", name = "zookeeper", version.ref
= "zookeeper" }
+zookeeper-jute = { group = "org.apache.zookeeper", name = "zookeeper-jute",
version.ref = "zookeeper" }
javax-jaxb-api = { group = "javax.xml.bind", name = "jaxb-api", version.ref =
"javax-jaxb-api" }
javax-ws-rs-api = { group = "javax.ws.rs", name = "javax.ws.rs-api",
version.ref = "javax-ws-rs-api" }
selenium = { group = "org.seleniumhq.selenium", name = "selenium-java",
version.ref = "selenium" }
diff --git a/integration-test-common/build.gradle.kts
b/integration-test-common/build.gradle.kts
index a14fed40cb..b7ca544251 100644
--- a/integration-test-common/build.gradle.kts
+++ b/integration-test-common/build.gradle.kts
@@ -52,7 +52,6 @@ dependencies {
testImplementation(libs.testcontainers.mysql)
testImplementation(libs.testcontainers.postgresql)
testImplementation(libs.ranger.intg) {
- exclude("org.apache.hadoop", "hadoop-common")
exclude("org.apache.hive", "hive-storage-api")
exclude("org.apache.lucene")
exclude("org.apache.solr")
@@ -60,7 +59,13 @@ dependencies {
exclude("org.elasticsearch")
exclude("org.elasticsearch.client")
exclude("org.elasticsearch.plugin")
- exclude("org.apache.hadoop", "hadoop-common")
+ // Keep the Hadoop, Jackson and commons-lang3 versions used by the rest of
the build instead of
+ // the newer ones Ranger 2.8 asks for.
+ exclude("org.apache.hadoop")
+ exclude("com.fasterxml.jackson.core")
+ exclude("com.fasterxml.jackson.jaxrs")
+ exclude("com.fasterxml.jackson.module")
+ exclude("org.apache.commons", "commons-lang3")
}
testImplementation(libs.hadoop3.client.api)
testImplementation(libs.hadoop3.client.runtime)
