Repository: sentry Updated Branches: refs/heads/master 2c9a927a9 -> 74676535b
SENTRY-2423: Increase the allocation size for auto-increment of id's for Snapshot tables. (Kalyan Kumar Kalvagadda reviewed by Arjun Mishra and Sergio Pena) Project: http://git-wip-us.apache.org/repos/asf/sentry/repo Commit: http://git-wip-us.apache.org/repos/asf/sentry/commit/74676535 Tree: http://git-wip-us.apache.org/repos/asf/sentry/tree/74676535 Diff: http://git-wip-us.apache.org/repos/asf/sentry/diff/74676535 Branch: refs/heads/master Commit: 74676535b066b48939ae9ad9c1a1061986450f3d Parents: 2c9a927 Author: Kalyan Kumar Kalvagadda <[email protected]> Authored: Tue Oct 9 09:32:49 2018 -0500 Committer: Kalyan Kumar Kalvagadda <[email protected]> Committed: Tue Oct 9 09:32:49 2018 -0500 ---------------------------------------------------------------------- .../sentry/service/common/ServiceConstants.java | 8 ++++++ .../provider/db/service/model/package.jdo | 4 +-- .../db/service/persistent/SentryStore.java | 3 ++ .../db/service/persistent/TestSentryStore.java | 30 ++++++++++++++++++++ 4 files changed, 43 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sentry/blob/74676535/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/service/common/ServiceConstants.java ---------------------------------------------------------------------- diff --git a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/service/common/ServiceConstants.java b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/service/common/ServiceConstants.java index e90fe2d..092060c 100644 --- a/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/service/common/ServiceConstants.java +++ b/sentry-core/sentry-core-common/src/main/java/org/apache/sentry/service/common/ServiceConstants.java @@ -256,6 +256,14 @@ public class ServiceConstants { */ public static final String SENTRY_DB_EXPLICIT_GRANTS_PERMITTED = "sentry.db.explicit.grants.permitted"; public static final String SENTRY_DB_EXPLICIT_GRANTS_PERMITTED_DEFAULT = ""; + + /** + * This value sets the allocation size used by datanucleus for the values it auto generates. + * This is used when the strategy is explicitly mentioned in JDO. With native(default) this configuration + * is not used. + */ + public static final String SENTRY_DB_VALUE_GENERATION_ALLOCATION_SIZE = "sentry.db.valuegeneration.allocation.size"; + public static final int SENTRY_DB_VALUE_GENERATION_ALLOCATION_SIZE_DEFAULT = 100; } public static final String SENTRY_ZK_JAAS_NAME = "Sentry"; http://git-wip-us.apache.org/repos/asf/sentry/blob/74676535/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo ---------------------------------------------------------------------- diff --git a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo index 6539e33..20ec0de 100644 --- a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo +++ b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/model/package.jdo @@ -256,7 +256,7 @@ </class> <class name="MAuthzPathsMapping" identity-type="datastore" table="AUTHZ_PATHS_MAPPING" detachable="true"> - <datastore-identity> + <datastore-identity strategy="increment"> <column name="AUTHZ_OBJ_ID"/> </datastore-identity> <index name="AUTHZ_SNAPSHOT_ID_INDEX" unique="false"> @@ -288,7 +288,7 @@ </class> <class name="MPath" identity-type="datastore" table="AUTHZ_PATH" detachable="true"> - <datastore-identity> + <datastore-identity strategy="increment"> <column name="PATH_ID"/> </datastore-identity> <field name="path"> http://git-wip-us.apache.org/repos/asf/sentry/blob/74676535/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java index 1722109..7a736ca 100644 --- a/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java +++ b/sentry-service/sentry-service-server/src/main/java/org/apache/sentry/provider/db/service/persistent/SentryStore.java @@ -249,6 +249,9 @@ public class SentryStore implements SentryStoreInterface { // Disallow operations outside of transactions prop.setProperty("datanucleus.NontransactionalRead", "false"); prop.setProperty("datanucleus.NontransactionalWrite", "false"); + int allocationSize = conf.getInt(ServerConfig.SENTRY_DB_VALUE_GENERATION_ALLOCATION_SIZE, ServerConfig. + SENTRY_DB_VALUE_GENERATION_ALLOCATION_SIZE_DEFAULT); + prop.setProperty("datanucleus.valuegeneration.increment.allocationSize", Integer.toString(allocationSize)); return prop; } http://git-wip-us.apache.org/repos/asf/sentry/blob/74676535/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java ---------------------------------------------------------------------- diff --git a/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java b/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java index a299e00..4a9afe3 100644 --- a/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java +++ b/sentry-service/sentry-service-server/src/test/java/org/apache/sentry/provider/db/service/persistent/TestSentryStore.java @@ -4571,6 +4571,36 @@ public class TestSentryStore extends org.junit.Assert { assertEquals(0, allPrivileges.get(USER3).size()); } + @Test + public void testPersistFullPathsImageWithHugeData() throws Exception { + Map<String, Collection<String>> authzPaths = new HashMap<>(); + String[] prefixes = {"/user/hive/warehouse"}; + // Makes sure that authorizable object could be associated + // with different paths and can be properly persisted into database. + for(int db_index = 1 ; db_index <= 10; db_index++) { + String db_name = "db" + db_index; + for( int table_index = 1; table_index <= 15; table_index++) { + Set<String> paths = Sets.newHashSet(); + String table_name = "tb" + table_index; + String location = "/u/h/w/" + db_name + "/" + table_name; + for (int part_index = 1; part_index <= 30; part_index++) { + paths.add(location + "/" + part_index); + } + authzPaths.put(db_name+table_name, paths); + } + } + long notificationID = 110000; + sentryStore.persistFullPathsImage(authzPaths, notificationID); + PathsUpdate pathsUpdate = sentryStore.retrieveFullPathsImageUpdate(prefixes); + long savedNotificationID = sentryStore.getLastProcessedNotificationID(); + assertEquals(1, pathsUpdate.getImgNum()); + TPathsDump pathDump = pathsUpdate.toThrift().getPathsDump(); + assertNotNull(pathDump); + Map<Integer, TPathEntry> nodeMap = pathDump.getNodeMap(); + assertTrue(nodeMap.size() > 0); + assertEquals(notificationID, savedNotificationID); + } + private TSentryPrivilege toTSentryPrivilege(String action, String scope, String server, String dbName, String tableName) { TSentryPrivilege privilege = new TSentryPrivilege();
