This is an automated email from the ASF dual-hosted git repository. madhan pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push: new e6b0f880a RANGER-4303: plugin memory sizing tool e6b0f880a is described below commit e6b0f880a30be0767deb78c4136c4c8825a1d732 Author: Madhan Neethiraj <mad...@apache.org> AuthorDate: Sun Jun 25 22:20:46 2023 -0700 RANGER-4303: plugin memory sizing tool --- distro/src/main/assembly/ranger-tools.xml | 6 + ranger-tools/conf/logback-mem-sizing.xml | 45 ++ ranger-tools/scripts/ranger-mem-sizing.sh | 46 ++ .../apache/ranger/sizing/PerfMemTimeTracker.java | 140 ++++++ .../org/apache/ranger/sizing/RangerMemSizing.java | 470 +++++++++++++++++++++ ranger-tools/testdata/test_roles_hive.json | 235 +++++++++++ ranger-tools/testdata/test_userstore_hive.json | 69 +++ 7 files changed, 1011 insertions(+) diff --git a/distro/src/main/assembly/ranger-tools.xml b/distro/src/main/assembly/ranger-tools.xml index b8713d806..794bddb51 100644 --- a/distro/src/main/assembly/ranger-tools.xml +++ b/distro/src/main/assembly/ranger-tools.xml @@ -134,6 +134,12 @@ <destName>ranger-plugin-perftester.sh</destName> <fileMode>755</fileMode> </file> + <file> + <source>${project.parent.basedir}/ranger-tools/scripts/ranger-mem-sizing.sh</source> + <outputDirectory></outputDirectory> + <destName>ranger-mem-sizing.sh</destName> + <fileMode>755</fileMode> + </file> <file> <source>${project.parent.basedir}/ranger-tools/scripts/README.txt</source> <outputDirectory></outputDirectory> diff --git a/ranger-tools/conf/logback-mem-sizing.xml b/ranger-tools/conf/logback-mem-sizing.xml new file mode 100644 index 000000000..c3b601451 --- /dev/null +++ b/ranger-tools/conf/logback-mem-sizing.xml @@ -0,0 +1,45 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<configuration> + <appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender"> + <file>./ranger-mem-sizing-${hostname}-${user}.log</file> + <append>true</append> + <encoder> + <pattern>%date [%thread] %level{5} [%file:%line] %msg%n</pattern> + </encoder> + <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy"> + <fileNamePattern>./ranger-mem-sizing-${hostname}-${user}.log.%d{yyyy-MM-dd}</fileNamePattern> + <maxHistory>15</maxHistory> + <cleanHistoryOnStart>true</cleanHistoryOnStart> + </rollingPolicy> + </appender> + + <appender name="console" class="ch.qos.logback.core.ConsoleAppender"> + <Target>System.out</Target> + <encoder> + <pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p - %m%n</pattern> + </encoder> + <filter class="ch.qos.logback.classic.filter.ThresholdFilter"> + <level>INFO</level> + </filter> + </appender> + <root level="INFO"> + <appender-ref ref="file"/> + </root> +</configuration> diff --git a/ranger-tools/scripts/ranger-mem-sizing.sh b/ranger-tools/scripts/ranger-mem-sizing.sh new file mode 100755 index 000000000..c147c96f3 --- /dev/null +++ b/ranger-tools/scripts/ranger-mem-sizing.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Review and update following variables as needed +# +RANGER_ADMIN_HOME=${RANGER_ADMIN_HOME:-/opt/ranger/admin} +MEMORY=1g + +# +# Usage: +# ranger-mem-sizing.sh -p policies.json -t tags.json -u userstore.json -r roles.json +# + +# +# +# +cdir=$(cd "$(dirname "$0")"; pwd) +cp="${cdir}/dist/*:${cdir}/lib/commons-cli-1.3.1.jar:${RANGER_ADMIN_HOME}/ews/lib/*:${RANGER_ADMIN_HOME}/ews/webapp/WEB-INF/lib/*" + +if [ "${JAVA_HOME}" != "" ] +then + export JAVA_HOME + PATH="${JAVA_HOME}/bin:${PATH}" + export PATH +fi + +JAVA_CMD="java -Xms${MEMORY} -Xmx${MEMORY} -Xloggc:./gc.log -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=. -Dlogback.configurationFile=file:${cdir}/conf/logback-mem-sizing.xml -cp ${cp} org.apache.ranger.sizing.RangerMemSizing" + +cd ${cdir} + +echo "JAVA command = $JAVA_CMD " "$@" +$JAVA_CMD "$@" diff --git a/ranger-tools/src/main/java/org/apache/ranger/sizing/PerfMemTimeTracker.java b/ranger-tools/src/main/java/org/apache/ranger/sizing/PerfMemTimeTracker.java new file mode 100644 index 000000000..917c0da9c --- /dev/null +++ b/ranger-tools/src/main/java/org/apache/ranger/sizing/PerfMemTimeTracker.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.sizing; + +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.List; + +public class PerfMemTimeTracker { + public static final String STR_INTENT = " "; + public static final String STR_FIELD_SEP = "|"; + + private final String tag; + private final long startTime; + private long endTime; + private long gcTime; + private final long startMemory; + private long endMemory; + private List<PerfMemTimeTracker> children = null; + + public PerfMemTimeTracker(String tag) { + this.tag = tag; + this.startTime = System.currentTimeMillis(); + + Runtime rt = Runtime.getRuntime(); + + rt.gc(); + + this.gcTime = System.currentTimeMillis() - startTime; + this.startMemory = rt.totalMemory() - rt.freeMemory(); + } + + public void stop() { + long gcStartTime = System.currentTimeMillis(); + + Runtime rt = Runtime.getRuntime(); + + rt.gc(); + + this.endTime = System.currentTimeMillis(); + this.gcTime += endTime - gcStartTime; + this.endMemory = rt.totalMemory() - rt.freeMemory(); + } + + public void addChild(PerfMemTimeTracker tracker) { + if (children == null) { + children = new ArrayList<>(); + } + + children.add(tracker); + } + + public String getTag() { + return tag; + } + + public long getStartTime() { + return startTime; + } + + public long getEndTime() { + return endTime; + } + + public long getGcTime() { + return gcTime; + } + + public long getStartMemory() { + return startMemory; + } + + public long getEndMemory() { + return endMemory; + } + + public List<PerfMemTimeTracker> getChildren() { + return children; + } + + public long getTimeTaken() { + return endTime - startTime - getTotalGcTime(); + } + + public long getTotalGcTime() { + long ret = gcTime; + + if (children != null) { + for (PerfMemTimeTracker child : children) { + ret += child.getTotalGcTime(); + } + } + + return ret; + } + + public long getMemoryDelta() { + return endMemory - startMemory; + } + + public void print(PrintStream out, boolean printHeader) { + if (printHeader) { + out.println("Task|Time (ms)|Memory (bytes)"); + } + + print("", out); + } + + @Override + public String toString() { + return tag + ", Memory: (start: " + startMemory + ", end: " + endMemory + ", delta: " + getMemoryDelta() + ") bytes, TimeTaken: " + getTimeTaken() + "ms"; + } + + private void print(String intentString, PrintStream out) { + out.println(intentString + tag + STR_FIELD_SEP + getTimeTaken() + STR_FIELD_SEP + getMemoryDelta()); + + if (children != null) { + for (PerfMemTimeTracker child : children) { + child.print(intentString + STR_INTENT, out); + } + } + } +} diff --git a/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java b/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java new file mode 100644 index 000000000..5b6f51339 --- /dev/null +++ b/ranger-tools/src/main/java/org/apache/ranger/sizing/RangerMemSizing.java @@ -0,0 +1,470 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.sizing; + +import java.io.*; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Set; + +import org.apache.commons.cli.*; +import org.apache.commons.lang.StringUtils; +import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig; +import org.apache.ranger.plugin.policyengine.RangerPolicyEngineOptions; +import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.apache.ranger.plugin.util.RangerRoles; +import org.apache.ranger.plugin.util.ServicePolicies; +import org.apache.ranger.plugin.util.ServicePolicies.SecurityZoneInfo; +import org.apache.ranger.plugin.util.ServiceTags; +import org.apache.ranger.plugin.util.RangerUserStore; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; + + +public class RangerMemSizing { + private static final String OPT_MODE_SPACE = "space"; + private static final String OPT_MODEL_RETRIEVAL = "retrieval"; + + private final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S"); + + private final Gson gson; + private final String policyFile; + private final String tagFile; + private final String rolesFile; + private final String userStoreFile; + private final boolean deDup; + private final boolean deDupStrings; + private final String optimizationMode; + private final PrintStream out; + + public RangerMemSizing(CommandLine cmdLine) { + this.out = System.out; + this.gson = createGson(); + this.policyFile = cmdLine.getOptionValue('p'); + this.tagFile = cmdLine.getOptionValue('t'); + this.rolesFile = cmdLine.getOptionValue('r'); + this.userStoreFile = cmdLine.getOptionValue('u'); + this.deDup = Boolean.parseBoolean(cmdLine.getOptionValue("d", "true")); + this.deDupStrings = this.deDup; + this.optimizationMode = StringUtils.startsWithIgnoreCase(cmdLine.getOptionValue('o', "space"), "s") ? OPT_MODE_SPACE : OPT_MODEL_RETRIEVAL; + } + + public void run() { + PerfMemTimeTracker tracker = new PerfMemTimeTracker("RangerMemSizing"); + + ServicePolicies policies = loadPolicies(policyFile, tracker); + ServiceTags tags = loadTags(tagFile, tracker); + RangerRoles roles = loadRoles(rolesFile, tracker); + RangerUserStore userStore = loadUserStore(userStoreFile, tracker); + RangerBasePlugin plugin = createRangerPlugin(policies, tags, roles, userStore, tracker); + + tracker.stop(); + + out.println(); + out.println("Parameters:"); + if (policies != null) { + out.println(" Policies: file=" + policyFile + ", size=" + new File(policyFile).length() + ", " + toSummaryStr(policies)); + } + + if (tags != null) { + out.println(" Tags: file=" + tagFile + ", size=" + new File(tagFile).length() + ", " + toSummaryStr(tags)); + } + + if (roles != null) { + out.println(" Roles: file=" + rolesFile + ", size=" + new File(rolesFile).length() + ", " + toSummaryStr(roles)); + } + + if (userStore != null) { + out.println(" UserStore: file=" + userStoreFile + ", size=" + new File(userStoreFile).length() + ", " + toSummaryStr(userStore)); + } + + out.println(" DeDup: " + deDup); + out.println(" OptMode: " + optimizationMode); + out.println(); + + out.println("Results:"); + out.println("*****************************"); + tracker.print(out, true); + out.println("*****************************"); + } + + public static void main(String[] args) { + CommandLine cmdLine = parseArgs(args); + + if (cmdLine != null) { + RangerMemSizing memSizing = new RangerMemSizing(cmdLine); + + memSizing.run(); + } + } + + private ServicePolicies loadPolicies(String fileName, PerfMemTimeTracker parent) { + if (fileName == null) { + return null; + } + + ServicePolicies ret = null; + + try { + File file = new File(fileName); + PerfMemTimeTracker loadTracker = new PerfMemTimeTracker("Load policies"); + + log("loading policies(file=" + fileName + ")"); + + { + PerfMemTimeTracker tracker = new PerfMemTimeTracker("Read policies"); + + try (FileReader reader = new FileReader(file)) { + ret = gson.fromJson(reader, ServicePolicies.class); + } + + tracker.stop(); + loadTracker.addChild(tracker); + } + + if (deDupStrings) { + PerfMemTimeTracker tracker = new PerfMemTimeTracker("DeDupStrings"); + + ret.dedupStrings(); + + tracker.stop(); + loadTracker.addChild(tracker); + } + + loadTracker.stop(); + parent.addChild(loadTracker); + + log("loaded policies(file=" + fileName + ", size=" + file.length() + "): " + toSummaryStr(ret)); + } catch (FileNotFoundException excp) { + log(fileName + ": file does not exist!"); + } catch (IOException excp) { + log(fileName, excp); + } + + return ret; + } + + private ServiceTags loadTags(String fileName, PerfMemTimeTracker parent) { + if (fileName == null) { + return null; + } + + ServiceTags ret = null; + + try { + File file = new File(fileName); + PerfMemTimeTracker loadTracker = new PerfMemTimeTracker("Load tags"); + + log("loading tags(file=" + fileName + ")"); + + { + PerfMemTimeTracker tracker = new PerfMemTimeTracker("Read tags"); + + try (FileReader reader = new FileReader(file)) { + ret = gson.fromJson(reader, ServiceTags.class); + } + + tracker.stop(); + loadTracker.addChild(tracker); + } + + if (deDup) { + PerfMemTimeTracker tracker = new PerfMemTimeTracker("DeDupTags"); + + int countOfDuplicateTags = ret.dedupTags(); + + tracker.stop(); + loadTracker.addChild(tracker); + log("DeDupTags(duplicateTags=" + countOfDuplicateTags + ")"); + } + + if (deDupStrings) { + PerfMemTimeTracker tracker = new PerfMemTimeTracker("DeDupStrings"); + + ret.dedupStrings(); + + tracker.stop(); + loadTracker.addChild(tracker); + } + + loadTracker.stop(); + parent.addChild(loadTracker); + + log("loaded tags(file=" + fileName + ", size=" + file.length() + "): " + toSummaryStr(ret)); + } catch (FileNotFoundException excp) { + log(fileName + ": file does not exist!"); + } catch (IOException excp) { + log(fileName, excp); + } + + return ret; + } + + private RangerRoles loadRoles(String fileName, PerfMemTimeTracker parent) { + if (fileName == null) { + return null; + } + + RangerRoles ret = null; + + try { + File file = new File(fileName); + PerfMemTimeTracker loadTracker = new PerfMemTimeTracker("Load roles"); + + log("loading roles(file=" + fileName + ")"); + + try (FileReader reader = new FileReader(file)) { + ret = gson.fromJson(reader, RangerRoles.class); + } + + loadTracker.stop(); + parent.addChild(loadTracker); + + log("loaded roles(file=" + fileName + ", size=" + file.length() + "): " + toSummaryStr(ret)); + } catch (FileNotFoundException excp) { + log(fileName + ": file does not exist!"); + } catch (IOException excp) { + log(fileName, excp); + } + + return ret; + } + + private RangerUserStore loadUserStore(String fileName, PerfMemTimeTracker parent) { + if (fileName == null) { + return null; + } + + RangerUserStore ret = null; + + try { + File file = new File(fileName); + PerfMemTimeTracker loadTracker = new PerfMemTimeTracker("Load userStore"); + + log("loading userStore(file=" + fileName + ")"); + + { + PerfMemTimeTracker tracker = new PerfMemTimeTracker("Read userStore"); + + try (FileReader reader = new FileReader(file)) { + ret = gson.fromJson(reader, RangerUserStore.class); + } + + tracker.stop(); + loadTracker.addChild(tracker); + } + + if (deDupStrings) { + PerfMemTimeTracker tracker = new PerfMemTimeTracker("DeDupStrings"); + + ret.dedupStrings(); + + tracker.stop(); + loadTracker.addChild(tracker); + } + + loadTracker.stop(); + parent.addChild(loadTracker); + + log("loaded userStore(file=" + fileName + ", size=" + file.length() + "): " + toSummaryStr(ret) + ")"); + } catch (FileNotFoundException excp) { + log(fileName + ": file does not exist!"); + } catch (IOException excp) { + log(fileName, excp); + } + + return ret; + } + + private RangerBasePlugin createRangerPlugin(ServicePolicies policies, ServiceTags tags, RangerRoles roles, RangerUserStore userStore, PerfMemTimeTracker parent) { + RangerBasePlugin ret = null; + + if (policies != null) { + String serviceType = policies.getServiceDef().getName(); + String serviceName = policies.getServiceName(); + RangerPluginConfig pluginConfig = new RangerPluginConfig(serviceType, serviceName, serviceType, null, null, getPolicyEngineOptions()); + + PerfMemTimeTracker tracker = new PerfMemTimeTracker("RangerBasePlugin initialization"); + + log("Initializing RangerBasePlugin..."); + + ret = new RangerBasePlugin(pluginConfig, policies, tags, roles, userStore); + + tracker.stop(); + parent.addChild(tracker); + log("Initialized RangerBasePlugin."); + } + + return ret; + } + + private static CommandLine parseArgs(String[] args) { + Option help = new Option("h", "help", false, "show help"); + Option deDup = new Option("d", "deDup", true, "deDup string/tags"); + Option policies = new Option("p", "policies", true, "policies file"); + Option tags = new Option("t", "tags", true, "tags file"); + Option roles = new Option("r", "roles", true, "roles file"); + Option userStore = new Option("u", "userStore", true, "userStore file"); + Option optimizeMode = new Option("o", "optMode", true, "optimization mode: space|retrieval"); + + Options options = new Options(); + + options.addOption(help); + options.addOption(policies); + options.addOption(tags); + options.addOption(roles); + options.addOption(userStore); + options.addOption(deDup); + options.addOption(optimizeMode); + + try { + CommandLine cmdLine = new DefaultParser().parse(options, args); + + if (! cmdLine.hasOption("h")) { + return cmdLine; + } + + new HelpFormatter().printHelp("RangerMemSizing", options); + } catch (ParseException excp) { + System.out.println("Failed to parse arguments"); + excp.printStackTrace(System.out); + } + + return null; + } + + private Gson createGson() { + Gson gson = null; + + try { + gson = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z").create(); + } catch(Throwable excp) { + log("failed to create GsonBuilder object", excp); + } + + return gson; + } + + private void log(String msg) { + out.println(DATE_FORMAT.format(new Date()) + ": " +msg); + } + + private void log(String msg, Throwable excp) { + out.println(DATE_FORMAT.format(new Date()) + ": " +msg); + + excp.printStackTrace(out); + } + + private RangerPolicyEngineOptions getPolicyEngineOptions() { + RangerPolicyEngineOptions ret = new RangerPolicyEngineOptions(); + + ret.disablePolicyRefresher = true; + ret.disableTagRetriever = true; + ret.disableUserStoreRetriever = true; + ret.optimizeTrieForSpace = optimizationMode.equals(OPT_MODE_SPACE); + ret.optimizeTrieForRetrieval = !ret.optimizeTrieForSpace; + ret.optimizeTagTrieForSpace = ret.optimizeTrieForSpace; + ret.optimizeTagTrieForRetrieval = ret.optimizeTrieForRetrieval; + + return ret; + } + + private static String toSummaryStr(ServicePolicies policies) { + int policyCount = 0; + + if (policies != null) { + if (policies.getPolicies() != null) { + policyCount += policies.getPolicies().size(); + } + + if (policies.getTagPolicies() != null && policies.getTagPolicies().getPolicies() != null) { + policyCount += policies.getTagPolicies().getPolicies().size(); + } + + if (policies.getSecurityZones() != null) { + for (SecurityZoneInfo zoneInfo : policies.getSecurityZones().values()) { + if (zoneInfo.getPolicies() != null) { + policyCount += zoneInfo.getPolicies().size(); + } + } + } + } + + return "policyCount=" + policyCount; + } + + private static String toSummaryStr(ServiceTags tags) { + int tagDefCount = 0; + int tagCount = 0; + int resourceCount = 0; + + if (tags != null) { + if (tags.getTagDefinitions() != null) { + tagDefCount = tags.getTagDefinitions().size(); + } + + if (tags.getTags() != null) { + tagCount = tags.getTags().size(); + } + + if (tags.getServiceResources() != null) { + resourceCount = tags.getServiceResources().size(); + } + } + + return "tagDefCount=" + tagDefCount + ", tagCount" + tagCount + ", resourceCount=" + resourceCount; + } + + private static String toSummaryStr(RangerRoles roles) { + int roleCount = 0; + + if (roles != null) { + if (roles.getRangerRoles() != null) { + roleCount = roles.getRangerRoles().size(); + } + } + + return "roleCount=" + roleCount; + } + + private static String toSummaryStr(RangerUserStore userStore) { + int userCount = 0; + int groupCount = 0; + int userGroupCount = 0; + + if (userStore != null) { + if (userStore.getUserAttrMapping() != null) { + userCount = userStore.getUserAttrMapping().size(); + } + + if (userStore.getGroupAttrMapping() != null) { + groupCount = userStore.getGroupAttrMapping().size(); + } + + if (userStore.getUserGroupMapping() != null) { + for (Set<String> userGroups : userStore.getUserGroupMapping().values()) { + userGroupCount += userGroups.size(); + } + } + } + + return "users=" + userCount + ", groups=" + groupCount + ", userGroupMappings=" + userGroupCount; + } +} diff --git a/ranger-tools/testdata/test_roles_hive.json b/ranger-tools/testdata/test_roles_hive.json new file mode 100644 index 000000000..306638f75 --- /dev/null +++ b/ranger-tools/testdata/test_roles_hive.json @@ -0,0 +1,235 @@ +{ + "serviceName": "cl1_hive", + "roleVersion": 1, + "rangerRoles": [ + { + "name": "r_hrt-1", + "description": "r_hrt-1", + "createdByUser": "admin", + "users": [ + { "name": "hrt_1", "isAdmin": false } + ], + "groups": [ + { "name": "g_hrt_1", "isAdmin": false } + ], + "roles": [ + ] + }, + { + "name": "r_hrt-2", + "description": "r_hrt-2", + "createdByUser": "admin", + "users": [ + { "name": "hrt_2", "isAdmin": false }, + { "name": "hrt_3", "isAdmin": false } + ], + "groups": [ + { "name": "g_hrt_2", "isAdmin": false }, + { "name": "g_hrt_3", "isAdmin": false } + ], + "roles": [ + { "name": "r_hrt_1", "isAdmin": false } + ] + }, + { + "name": "r_hrt-3", + "description": "r_hrt-3", + "createdByUser": "admin", + "users": [ + { "name": "hrt_4", "isAdmin": false }, + { "name": "hrt_5", "isAdmin": false }, + { "name": "hrt_6", "isAdmin": false } + ], + "groups": [ + { "name": "g_hrt_4", "isAdmin": false }, + { "name": "g_hrt_5", "isAdmin": false }, + { "name": "g_hrt_6", "isAdmin": false } + ], + "roles": [ + { "name": "r_hrt_2", "isAdmin": false } + ] + }, + { + "name": "r_hrt-4", + "description": "r_hrt-4", + "createdByUser": "admin", + "users": [ + { "name": "hrt_7", "isAdmin": false }, + { "name": "hrt_8", "isAdmin": false }, + { "name": "hrt_9", "isAdmin": false }, + { "name": "hrt_10", "isAdmin": false } + ], + "groups": [ + { "name": "g_hrt_7", "isAdmin": false }, + { "name": "g_hrt_8", "isAdmin": false }, + { "name": "g_hrt_9", "isAdmin": false }, + { "name": "g_hrt_10", "isAdmin": false } + ], + "roles": [ + { "name": "r_hrt_3", "isAdmin": false } + ] + }, + { + "name": "r_hrt-5", + "description": "r_hrt-5", + "createdByUser": "admin", + "users": [ + { "name": "hrt_11", "isAdmin": false }, + { "name": "hrt_12", "isAdmin": false }, + { "name": "hrt_13", "isAdmin": false }, + { "name": "hrt_14", "isAdmin": false }, + { "name": "hrt_15", "isAdmin": false } + ], + "groups": [ + { "name": "g_hrt_11", "isAdmin": false }, + { "name": "g_hrt_12", "isAdmin": false }, + { "name": "g_hrt_13", "isAdmin": false }, + { "name": "g_hrt_14", "isAdmin": false }, + { "name": "g_hrt_15", "isAdmin": false } + ], + "roles": [ + { "name": "r_hrt_4", "isAdmin": false } + ] + }, + { + "name": "r_hrt-6", + "description": "r_hrt-6", + "createdByUser": "admin", + "users": [ + { "name": "hrt_16", "isAdmin": false }, + { "name": "hrt_17", "isAdmin": false }, + { "name": "hrt_18", "isAdmin": false }, + { "name": "hrt_19", "isAdmin": false }, + { "name": "hrt_20", "isAdmin": false }, + { "name": "hrt_21", "isAdmin": false } + ], + "groups": [ + { "name": "g_hrt_16", "isAdmin": false }, + { "name": "g_hrt_17", "isAdmin": false }, + { "name": "g_hrt_18", "isAdmin": false }, + { "name": "g_hrt_19", "isAdmin": false }, + { "name": "g_hrt_20", "isAdmin": false }, + { "name": "g_hrt_21", "isAdmin": false } + ], + "roles": [ + { "name": "r_hrt_5", "isAdmin": false } + ] + }, + { + "name": "r_hrt-7", + "description": "r_hrt-7", + "createdByUser": "admin", + "users": [ + { "name": "hrt_22", "isAdmin": false }, + { "name": "hrt_23", "isAdmin": false }, + { "name": "hrt_24", "isAdmin": false }, + { "name": "hrt_25", "isAdmin": false }, + { "name": "hrt_26", "isAdmin": false }, + { "name": "hrt_27", "isAdmin": false }, + { "name": "hrt_28", "isAdmin": false } + ], + "groups": [ + { "name": "g_hrt_22", "isAdmin": false }, + { "name": "g_hrt_23", "isAdmin": false }, + { "name": "g_hrt_24", "isAdmin": false }, + { "name": "g_hrt_25", "isAdmin": false }, + { "name": "g_hrt_26", "isAdmin": false }, + { "name": "g_hrt_27", "isAdmin": false }, + { "name": "g_hrt_28", "isAdmin": false } + ], + "roles": [ + { "name": "r_hrt_6", "isAdmin": false } + ] + }, + { + "name": "r_hrt-8", + "description": "r_hrt-8", + "createdByUser": "admin", + "users": [ + { "name": "hrt_29", "isAdmin": false }, + { "name": "hrt_30", "isAdmin": false }, + { "name": "hrt_31", "isAdmin": false }, + { "name": "hrt_32", "isAdmin": false }, + { "name": "hrt_33", "isAdmin": false }, + { "name": "hrt_34", "isAdmin": false }, + { "name": "hrt_35", "isAdmin": false }, + { "name": "hrt_36", "isAdmin": false } + ], + "groups": [ + { "name": "g_hrt_29", "isAdmin": false }, + { "name": "g_hrt_30", "isAdmin": false }, + { "name": "g_hrt_31", "isAdmin": false }, + { "name": "g_hrt_32", "isAdmin": false }, + { "name": "g_hrt_33", "isAdmin": false }, + { "name": "g_hrt_34", "isAdmin": false }, + { "name": "g_hrt_35", "isAdmin": false }, + { "name": "g_hrt_36", "isAdmin": false } + ], + "roles": [ + { "name": "r_hrt_7", "isAdmin": false } + ] + }, + { + "name": "r_hrt-9", + "description": "r_hrt-9", + "createdByUser": "admin", + "users": [ + { "name": "hrt_37", "isAdmin": false }, + { "name": "hrt_38", "isAdmin": false }, + { "name": "hrt_39", "isAdmin": false }, + { "name": "hrt_40", "isAdmin": false }, + { "name": "hrt_41", "isAdmin": false }, + { "name": "hrt_42", "isAdmin": false }, + { "name": "hrt_43", "isAdmin": false }, + { "name": "hrt_44", "isAdmin": false }, + { "name": "hrt_45", "isAdmin": false } + ], + "groups": [ + { "name": "g_hrt_37", "isAdmin": false }, + { "name": "g_hrt_38", "isAdmin": false }, + { "name": "g_hrt_39", "isAdmin": false }, + { "name": "g_hrt_40", "isAdmin": false }, + { "name": "g_hrt_41", "isAdmin": false }, + { "name": "g_hrt_42", "isAdmin": false }, + { "name": "g_hrt_43", "isAdmin": false }, + { "name": "g_hrt_44", "isAdmin": false }, + { "name": "g_hrt_45", "isAdmin": false } + ], + "roles": [ + { "name": "r_hrt_8", "isAdmin": false } + ] + }, + { + "name": "r_hrt-10", + "description": "r_hrt-10", + "createdByUser": "admin", + "users": [ + { "name": "hrt_46", "isAdmin": false }, + { "name": "hrt_47", "isAdmin": false }, + { "name": "hrt_48", "isAdmin": false }, + { "name": "hrt_49", "isAdmin": false }, + { "name": "hrt_50", "isAdmin": false }, + { "name": "hrt_51", "isAdmin": false }, + { "name": "hrt_52", "isAdmin": false }, + { "name": "hrt_53", "isAdmin": false }, + { "name": "hrt_54", "isAdmin": false }, + { "name": "hrt_55", "isAdmin": false } + ], + "groups": [ + { "name": "g_hrt_46", "isAdmin": false }, + { "name": "g_hrt_47", "isAdmin": false }, + { "name": "g_hrt_48", "isAdmin": false }, + { "name": "g_hrt_49", "isAdmin": false }, + { "name": "g_hrt_50", "isAdmin": false }, + { "name": "g_hrt_51", "isAdmin": false }, + { "name": "g_hrt_52", "isAdmin": false }, + { "name": "g_hrt_53", "isAdmin": false }, + { "name": "g_hrt_54", "isAdmin": false }, + { "name": "g_hrt_55", "isAdmin": false } + ], + "roles": [ + { "name": "r_hrt_9", "isAdmin": false } + ] + } + ] +} diff --git a/ranger-tools/testdata/test_userstore_hive.json b/ranger-tools/testdata/test_userstore_hive.json new file mode 100644 index 000000000..931cda861 --- /dev/null +++ b/ranger-tools/testdata/test_userstore_hive.json @@ -0,0 +1,69 @@ +{ + "userStoreVersion": 1, + "userAttrMapping": { + "hrt-1": { "dept": "hr", "email": "hr...@mycompany.org" }, + "hrt-2": { "dept": "hr", "email": "hr...@mycompany.org" }, + "hrt-3": { "dept": "hr", "email": "hr...@mycompany.org" }, + "hrt-4": { "dept": "hr", "email": "hr...@mycompany.org" }, + "hrt-5": { "dept": "hr", "email": "hr...@mycompany.org" }, + "hrt-6": { "dept": "hr", "email": "hr...@mycompany.org" }, + "hrt-7": { "dept": "hr", "email": "hr...@mycompany.org" }, + "hrt-8": { "dept": "hr", "email": "hr...@mycompany.org" }, + "hrt-9": { "dept": "hr", "email": "hr...@mycompany.org" }, + "hrt-10": { "dept": "hr", "email": "hrt...@mycompany.org" }, + "hrt-11": { "dept": "hr", "email": "hrt...@mycompany.org" }, + "hrt-12": { "dept": "hr", "email": "hrt...@mycompany.org" }, + "hrt-13": { "dept": "hr", "email": "hrt...@mycompany.org" }, + "hrt-14": { "dept": "hr", "email": "hrt...@mycompany.org" }, + "hrt-15": { "dept": "hr", "email": "hrt...@mycompany.org" }, + "hrt-16": { "dept": "hr", "email": "hrt...@mycompany.org" }, + "hrt-17": { "dept": "hr", "email": "hrt...@mycompany.org" }, + "hrt-18": { "dept": "hr", "email": "hrt...@mycompany.org" }, + "hrt-19": { "dept": "hr", "email": "hrt...@mycompany.org" }, + "hrt-20": { "dept": "hr", "email": "hrt...@mycompany.org" } + }, + "groupAttrMapping": { + "g_hrt-1": { "id": "1", "email": "g_hr...@mycompany.org" }, + "g_hrt-2": { "id": "2", "email": "g_hr...@mycompany.org" }, + "g_hrt-3": { "id": "3", "email": "g_hr...@mycompany.org" }, + "g_hrt-4": { "id": "4", "email": "g_hr...@mycompany.org" }, + "g_hrt-5": { "id": "5", "email": "g_hr...@mycompany.org" }, + "g_hrt-6": { "id": "6", "email": "g_hr...@mycompany.org" }, + "g_hrt-7": { "id": "7", "email": "g_hr...@mycompany.org" }, + "g_hrt-8": { "id": "8", "email": "g_hr...@mycompany.org" }, + "g_hrt-9": { "id": "9", "email": "g_hr...@mycompany.org" }, + "g_hrt-10": { "id": "10", "email": "g_hrt...@mycompany.org" }, + "g_hrt-11": { "id": "11", "email": "g_hrt...@mycompany.org" }, + "g_hrt-12": { "id": "12", "email": "g_hrt...@mycompany.org" }, + "g_hrt-13": { "id": "13", "email": "g_hrt...@mycompany.org" }, + "g_hrt-14": { "id": "14", "email": "g_hrt...@mycompany.org" }, + "g_hrt-15": { "id": "15", "email": "g_hrt...@mycompany.org" }, + "g_hrt-16": { "id": "16", "email": "g_hrt...@mycompany.org" }, + "g_hrt-17": { "id": "17", "email": "g_hrt...@mycompany.org" }, + "g_hrt-18": { "id": "18", "email": "g_hrt...@mycompany.org" }, + "g_hrt-19": { "id": "19", "email": "g_hrt...@mycompany.org" }, + "g_hrt-20": { "id": "20", "email": "g_hrt...@mycompany.org" } + }, + "userGroupMapping": { + "hrt-1": [ "g_hrt-1" ], + "hrt-2": [ "g_hrt-2" ], + "hrt-3": [ "g_hrt-3" ], + "hrt-4": [ "g_hrt-4" ], + "hrt-5": [ "g_hrt-5" ], + "hrt-6": [ "g_hrt-6" ], + "hrt-7": [ "g_hrt-7" ], + "hrt-8": [ "g_hrt-8" ], + "hrt-9": [ "g_hrt-9" ], + "hrt-10": [ "g_hrt-10" ], + "hrt-11": [ "g_hrt-11" ], + "hrt-12": [ "g_hrt-12" ], + "hrt-13": [ "g_hrt-13" ], + "hrt-14": [ "g_hrt-14" ], + "hrt-15": [ "g_hrt-15" ], + "hrt-16": [ "g_hrt-16" ], + "hrt-17": [ "g_hrt-17" ], + "hrt-18": [ "g_hrt-18" ], + "hrt-19": [ "g_hrt-19" ], + "hrt-20": [ "g_hrt-20" ] + } +}