RANGER-1810:Ranger supports plugin to enable, monitor and manage apache Sqoop2
Signed-off-by: peng.jianhua <peng.jian...@zte.com.cn> Project: http://git-wip-us.apache.org/repos/asf/ranger/repo Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/6e6e42b0 Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/6e6e42b0 Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/6e6e42b0 Branch: refs/heads/master Commit: 6e6e42b0bffb67ad5613b99bf8f6dadbfeeca90b Parents: 79e91fa Author: zhangqiang2 <zhangqia...@zte.com.cn> Authored: Tue Oct 24 16:57:34 2017 +0800 Committer: peng.jianhua <peng.jian...@zte.com.cn> Committed: Tue Oct 24 17:08:39 2017 +0800 ---------------------------------------------------------------------- agents-common/scripts/enable-agent.sh | 57 +++- .../plugin/store/EmbeddedServiceDefsUtil.java | 10 +- .../service-defs/ranger-servicedef-sqoop.json | 125 +++++++++ plugin-sqoop/.gitignore | 1 + .../conf/ranger-policymgr-ssl-changes.cfg | 21 ++ plugin-sqoop/conf/ranger-policymgr-ssl.xml | 49 ++++ .../conf/ranger-sqoop-audit-changes.cfg | 65 +++++ plugin-sqoop/conf/ranger-sqoop-audit.xml | 271 ++++++++++++++++++ .../conf/ranger-sqoop-security-changes.cfg | 28 ++ plugin-sqoop/conf/ranger-sqoop-security.xml | 83 ++++++ plugin-sqoop/pom.xml | 67 +++++ plugin-sqoop/scripts/install.properties | 140 ++++++++++ .../sqoop/authorizer/RangerSqoopAuthorizer.java | 189 +++++++++++++ .../services/sqoop/RangerServiceSqoop.java | 89 ++++++ .../services/sqoop/client/SqoopClient.java | 280 +++++++++++++++++++ .../services/sqoop/client/SqoopResourceMgr.java | 111 ++++++++ .../json/model/SqoopConnectorResponse.java | 56 ++++ .../json/model/SqoopConnectorsResponse.java | 37 +++ pom.xml | 4 + ranger-sqoop-plugin-shim/.gitignore | 1 + ranger-sqoop-plugin-shim/pom.xml | 62 ++++ .../sqoop/authorizer/RangerSqoopAuthorizer.java | 110 ++++++++ .../scripts/models/BackboneFormDataType.js | 3 + .../scripts/modules/globalize/message/en.js | 1 + src/main/assembly/admin-web.xml | 13 + src/main/assembly/plugin-sqoop.xml | 159 +++++++++++ 26 files changed, 2029 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/agents-common/scripts/enable-agent.sh ---------------------------------------------------------------------- diff --git a/agents-common/scripts/enable-agent.sh b/agents-common/scripts/enable-agent.sh index d31a264..f105f54 100755 --- a/agents-common/scripts/enable-agent.sh +++ b/agents-common/scripts/enable-agent.sh @@ -207,6 +207,8 @@ elif [ "${HCOMPONENT_NAME}" = "atlas" ]; then elif [ "${HCOMPONENT_NAME}" = "hadoop" ] || [ "${HCOMPONENT_NAME}" = "yarn" ]; then HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/share/hadoop/hdfs/lib +elif [ "${HCOMPONENT_NAME}" = "sqoop" ]; then + HCOMPONENT_LIB_DIR=${HCOMPONENT_INSTALL_DIR}/server/lib fi HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/conf @@ -223,6 +225,8 @@ elif [ "${HCOMPONENT_NAME}" = "kafka" ]; then HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/config elif [ "${HCOMPONENT_NAME}" = "hadoop" ]; then HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/etc/hadoop +elif [ "${HCOMPONENT_NAME}" = "sqoop" ]; then + HCOMPONENT_CONF_DIR=${HCOMPONENT_INSTALL_DIR}/conf fi HCOMPONENT_ARCHIVE_CONF_DIR=${HCOMPONENT_CONF_DIR}/.archive @@ -650,8 +654,24 @@ then fi fi +#Check Properties whether in File, return code 1 if not exist +#$1 -> propertyName; $2 -> fileName +checkPropertyInFile(){ + validate=$(sed '/^\#/d' $2 | grep "^$1" | tail -n 1 | cut -d "=" -f1-) # for validation + if test -z "$validate" ; then return 1; fi +} + +#Add Properties to File +#$1 -> propertyName; $2 -> newPropertyValue; $3 -> fileName +addPropertyToFile(){ + echo "$1=$2">>$3 + validate=$(sed '/^\#/d' $3 | grep "^$1" | tail -n 1 | cut -d "=" -f2-) # for validation + if test -z "$validate" ; then log "[E] Failed to add properties '$1' to $3 file!"; exit 1; fi + echo "Property $1 added successfully with : '$2'" +} + #Update Properties to File -#$1 -> propertyName $2 -> newPropertyValue $3 -> fileName +#$1 -> propertyName; $2 -> newPropertyValue; $3 -> fileName updatePropertyToFile(){ sed -i 's@^'$1'=[^ ]*$@'$1'='$2'@g' $3 validate=$(sed '/^\#/d' $3 | grep "^$1" | tail -n 1 | cut -d "=" -f2-) # for validation @@ -659,6 +679,18 @@ updatePropertyToFile(){ echo "Property $1 updated successfully with : '$2'" } +#Add or Update Properties to File +#$1 -> propertyName; $2 -> newPropertyValue; $3 -> fileName +addOrUpdatePropertyToFile(){ + checkPropertyInFile $1 $3 + if [ $? -eq 1 ] + then + addPropertyToFile $1 $2 $3 + else + updatePropertyToFile $1 $2 $3 + fi +} + if [ "${HCOMPONENT_NAME}" = "atlas" ] then if [ "${action}" = "enable" ] @@ -685,6 +717,29 @@ fi # Set notice to restart the ${HCOMPONENT_NAME} # +if [ "${HCOMPONENT_NAME}" = "sqoop" ] +then + if [ "${action}" = "enable" ] + then + authName="org.apache.ranger.authorization.sqoop.authorizer.RangerSqoopAuthorizer" + else + authName="" + fi + + dt=`date '+%Y%m%d%H%M%S'` + fn=`ls ${HCOMPONENT_CONF_DIR}/sqoop.properties 2> /dev/null` + if [ -f "${fn}" ] + then + dn=`dirname ${fn}` + bn=`basename ${fn}` + bf=${dn}/.${bn}.${dt} + echo "backup of ${fn} to ${bf} ..." + cp ${fn} ${bf} + echo "Add or Update properties file: [${fn}] ... " + addOrUpdatePropertyToFile org.apache.sqoop.security.authorization.validator $authName ${fn} + fi +fi + echo "Ranger Plugin for ${HCOMPONENT_NAME} has been ${action}d. Please restart ${HCOMPONENT_NAME} to ensure that changes are effective." exit 0 http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java ---------------------------------------------------------------------- diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java index 9463ab8..08f5437 100755 --- a/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java +++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/EmbeddedServiceDefsUtil.java @@ -47,7 +47,7 @@ public class EmbeddedServiceDefsUtil { // following servicedef list should be reviewed/updated whenever a new embedded service-def is added - private static final String DEFAULT_BOOTSTRAP_SERVICEDEF_LIST = "tag,hdfs,hbase,hive,kms,knox,storm,yarn,kafka,solr,atlas,nifi"; + private static final String DEFAULT_BOOTSTRAP_SERVICEDEF_LIST = "tag,hdfs,hbase,hive,kms,knox,storm,yarn,kafka,solr,atlas,nifi,sqoop"; private static final String PROPERTY_SUPPORTED_SERVICE_DEFS = "ranger.supportedcomponents"; private Set<String> supportedServiceDefs; public static final String EMBEDDED_SERVICEDEF_TAG_NAME = "tag"; @@ -63,6 +63,7 @@ public class EmbeddedServiceDefsUtil { public static final String EMBEDDED_SERVICEDEF_NIFI_NAME = "nifi"; public static final String EMBEDDED_SERVICEDEF_ATLAS_NAME = "atlas"; public static final String EMBEDDED_SERVICEDEF_WASB_NAME = "wasb"; + public static final String EMBEDDED_SERVICEDEF_SQOOP_NAME = "sqoop"; public static final String PROPERTY_CREATE_EMBEDDED_SERVICE_DEFS = "ranger.service.store.create.embedded.service-defs"; @@ -93,6 +94,7 @@ public class EmbeddedServiceDefsUtil { private RangerServiceDef nifiServiceDef; private RangerServiceDef atlasServiceDef; private RangerServiceDef wasbServiceDef; + private RangerServiceDef sqoopServiceDef; private RangerServiceDef tagServiceDef; @@ -132,7 +134,7 @@ public class EmbeddedServiceDefsUtil { tagServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_TAG_NAME); wasbServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_WASB_NAME); - + sqoopServiceDef = getOrCreateServiceDef(store, EMBEDDED_SERVICEDEF_SQOOP_NAME); // Ensure that tag service def is updated with access types of all service defs store.updateTagServiceDefForAccessTypes(); @@ -187,6 +189,10 @@ public class EmbeddedServiceDefsUtil { return getId(atlasServiceDef); } + public long getSqoopServiceDefId() { + return getId(sqoopServiceDef); + } + public long getTagServiceDefId() { return getId(tagServiceDef); } public long getWasbServiceDefId() { return getId(wasbServiceDef); } http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/agents-common/src/main/resources/service-defs/ranger-servicedef-sqoop.json ---------------------------------------------------------------------- diff --git a/agents-common/src/main/resources/service-defs/ranger-servicedef-sqoop.json b/agents-common/src/main/resources/service-defs/ranger-servicedef-sqoop.json new file mode 100644 index 0000000..3f269fb --- /dev/null +++ b/agents-common/src/main/resources/service-defs/ranger-servicedef-sqoop.json @@ -0,0 +1,125 @@ +{ + "id":14, + "name": "sqoop", + "implClass": "org.apache.ranger.services.sqoop.RangerServiceSqoop", + "label": "SQOOP", + "description": "SQOOP", + "guid": "6c63d385-5876-4a4c-ac4a-3b99b50ed600", + "resources": + [ + { + "itemId": 1, + "name": "connector", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { "wildCard":true, "ignoreCase":true}, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Connector", + "description": "Sqoop Connector" + }, + + { + "itemId": 2, + "name": "link", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { "wildCard":true, "ignoreCase":true}, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Link", + "description": "Sqoop Link" + }, + + { + "itemId": 3, + "name": "job", + "type": "string", + "level": 10, + "parent": "", + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": false, + "matcher": "org.apache.ranger.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { "wildCard":true, "ignoreCase":true}, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Job", + "description": "Sqoop Job" + } + ], + + "accessTypes": + [ + { + "itemId": 1, + "name": "READ", + "label": "READ" + }, + + { + "itemId": 2, + "name": "WRITE", + "label": "WRITE" + } + ], + + "configs": + [ + { + "itemId": 1, + "name": "username", + "type": "string", + "mandatory": true, + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Username" + }, + + { + "itemId": 2, + "name": "sqoop.url", + "type": "string", + "mandatory": true, + "defaultValue": "", + "validationRegEx":"", + "validationMessage": "", + "uiHint":"", + "label": "Sqoop URL" + } + + ], + "options": { "enableDenyAndExceptionsInPolicies": "false" }, + + "enums": + [ + + ], + + "contextEnrichers": + [ + + ], + + "policyConditions": + [ + + ] +} http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/plugin-sqoop/.gitignore ---------------------------------------------------------------------- diff --git a/plugin-sqoop/.gitignore b/plugin-sqoop/.gitignore new file mode 100644 index 0000000..b83d222 --- /dev/null +++ b/plugin-sqoop/.gitignore @@ -0,0 +1 @@ +/target/ http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/plugin-sqoop/conf/ranger-policymgr-ssl-changes.cfg ---------------------------------------------------------------------- diff --git a/plugin-sqoop/conf/ranger-policymgr-ssl-changes.cfg b/plugin-sqoop/conf/ranger-policymgr-ssl-changes.cfg new file mode 100644 index 0000000..ae347e8 --- /dev/null +++ b/plugin-sqoop/conf/ranger-policymgr-ssl-changes.cfg @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SSL Params +# +xasecure.policymgr.clientssl.keystore %SSL_KEYSTORE_FILE_PATH% mod create-if-not-exists +xasecure.policymgr.clientssl.keystore.credential.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists +xasecure.policymgr.clientssl.truststore %SSL_TRUSTSTORE_FILE_PATH% mod create-if-not-exists +xasecure.policymgr.clientssl.truststore.credential.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/plugin-sqoop/conf/ranger-policymgr-ssl.xml ---------------------------------------------------------------------- diff --git a/plugin-sqoop/conf/ranger-policymgr-ssl.xml b/plugin-sqoop/conf/ranger-policymgr-ssl.xml new file mode 100644 index 0000000..d4f791d --- /dev/null +++ b/plugin-sqoop/conf/ranger-policymgr-ssl.xml @@ -0,0 +1,49 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> + <!-- The following properties are used for 2-way SSL client server validation --> + <property> + <name>xasecure.policymgr.clientssl.keystore</name> + <value>sqoopdev-clientcert.jks</value> + <description> + Java Keystore files + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.truststore</name> + <value>cacerts-xasecure.jks</value> + <description> + java truststore file + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.keystore.credential.file</name> + <value>jceks://file/tmp/keystore-sqoopdev-ssl.jceks</value> + <description> + java keystore credential file + </description> + </property> + <property> + <name>xasecure.policymgr.clientssl.truststore.credential.file</name> + <value>jceks://file/tmp/truststore-sqoopdev-ssl.jceks</value> + <description> + java truststore credential file + </description> + </property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg ---------------------------------------------------------------------- diff --git a/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg b/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg new file mode 100644 index 0000000..8071e7b --- /dev/null +++ b/plugin-sqoop/conf/ranger-sqoop-audit-changes.cfg @@ -0,0 +1,65 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#xasecure.audit.db.is.enabled %XAAUDIT.DB.IS_ENABLED% mod create-if-not-exists +#xasecure.audit.jpa.javax.persistence.jdbc.url %XAAUDIT_DB_JDBC_URL% mod create-if-not-exists +#xasecure.audit.jpa.javax.persistence.jdbc.user %XAAUDIT.DB.USER_NAME% mod create-if-not-exists +#xasecure.audit.jpa.javax.persistence.jdbc.password crypted mod create-if-not-exists +#xasecure.audit.credential.provider.file jceks://file%CREDENTIAL_PROVIDER_FILE% mod create-if-not-exists +#xasecure.audit.jpa.javax.persistence.jdbc.driver %XAAUDIT_DB_JDBC_DRIVER% mod create-if-not-exists + +xasecure.audit.hdfs.is.enabled %XAAUDIT.HDFS.IS_ENABLED% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.directory %XAAUDIT.HDFS.DESTINATION_DIRECTORY% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.file %XAAUDIT.HDFS.DESTINTATION_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.flush.interval.seconds %XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.rollover.interval.seconds %XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.destination.open.retry.interval.seconds %XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.directory %XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.file %XAAUDIT.HDFS.LOCAL_BUFFER_FILE% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds %XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS% mod create-if-not-exists +xasecure.audit.hdfs.config.local.archive.directory %XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY% mod create-if-not-exists +xasecure.audit.hdfs.config.local.archive.max.file.count %XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT% mod create-if-not-exists + +#xasecure.audit.kafka.is.enabled %XAAUDIT.KAFKA.IS_ENABLED% mod create-if-not-exists +#xasecure.audit.kafka.is.async %XAAUDIT.KAFKA.IS_ASYNC% mod create-if-not-exists +#xasecure.audit.kafka.async.max.queue.size %XAAUDIT.KAFKA.MAX_QUEUE_SIZE% mod create-if-not-exists +#xasecure.audit.kafka.async.max.flush.interval.ms %XAAUDIT.KAFKA.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +#xasecure.audit.kafka.broker_list %XAAUDIT.KAFKA.BROKER_LIST% mod create-if-not-exists +#xasecure.audit.kafka.topic_name %XAAUDIT.KAFKA.TOPIC_NAME% mod create-if-not-exists + +xasecure.audit.solr.is.enabled %XAAUDIT.SOLR.IS_ENABLED% mod create-if-not-exists +xasecure.audit.solr.async.max.queue.size %XAAUDIT.SOLR.MAX_QUEUE_SIZE% mod create-if-not-exists +xasecure.audit.solr.async.max.flush.interval.ms %XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS% mod create-if-not-exists +xasecure.audit.solr.solr_url %XAAUDIT.SOLR.SOLR_URL% mod create-if-not-exists + +#V3 configuration +xasecure.audit.destination.solr %XAAUDIT.SOLR.ENABLE% mod create-if-not-exists +xasecure.audit.destination.solr.urls %XAAUDIT.SOLR.URL% mod create-if-not-exists +xasecure.audit.destination.solr.user %XAAUDIT.SOLR.USER% mod create-if-not-exists +xasecure.audit.destination.solr.password %XAAUDIT.SOLR.PASSWORD% mod create-if-not-exists +xasecure.audit.destination.solr.zookeepers %XAAUDIT.SOLR.ZOOKEEPER% mod create-if-not-exists +xasecure.audit.destination.solr.batch.filespool.dir %XAAUDIT.SOLR.FILE_SPOOL_DIR% mod create-if-not-exists + +xasecure.audit.destination.hdfs %XAAUDIT.HDFS.ENABLE% mod create-if-not-exists +xasecure.audit.destination.hdfs.batch.filespool.dir %XAAUDIT.HDFS.FILE_SPOOL_DIR% mod create-if-not-exists +xasecure.audit.destination.hdfs.dir %XAAUDIT.HDFS.HDFS_DIR% mod create-if-not-exists + +AZURE.ACCOUNTNAME %XAAUDIT.HDFS.AZURE_ACCOUNTNAME% var +xasecure.audit.destination.hdfs.config.fs.azure.shellkeyprovider.script %XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER% mod create-if-not-exists +xasecure.audit.destination.hdfs.config.fs.azure.account.key.%AZURE.ACCOUNTNAME%.blob.core.windows.net %XAAUDIT.HDFS.AZURE_ACCOUNTKEY% mod create-if-not-exists +xasecure.audit.destination.hdfs.config.fs.azure.account.keyprovider.%AZURE.ACCOUNTNAME%.blob.core.windows.net %XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER% mod create-if-not-exists + +#xasecure.audit.destination.file %XAAUDIT.FILE.ENABLE% mod create-if-not-exists +#xasecure.audit.destination.file.dir %XAAUDIT.FILE.DIR% mod create-if-not-exists http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/plugin-sqoop/conf/ranger-sqoop-audit.xml ---------------------------------------------------------------------- diff --git a/plugin-sqoop/conf/ranger-sqoop-audit.xml b/plugin-sqoop/conf/ranger-sqoop-audit.xml new file mode 100644 index 0000000..013a84c --- /dev/null +++ b/plugin-sqoop/conf/ranger-sqoop-audit.xml @@ -0,0 +1,271 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> + <property> + <name>xasecure.audit.is.enabled</name> + <value>true</value> + </property> + + + <!-- DB audit provider configuration --> + <property> + <name>xasecure.audit.db.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.db.is.async</name> + <value>true</value> + </property> + + <property> + <name>xasecure.audit.db.async.max.queue.size</name> + <value>10240</value> + </property> + + <property> + <name>xasecure.audit.db.async.max.flush.interval.ms</name> + <value>30000</value> + </property> + + <property> + <name>xasecure.audit.db.batch.size</name> + <value>100</value> + </property> + + <!-- Properties whose name begin with "xasecure.audit.jpa." are used to configure JPA --> + <property> + <name>xasecure.audit.jpa.javax.persistence.jdbc.url</name> + <value>jdbc:mysql://localhost:3306/ranger_audit</value> + </property> + + <property> + <name>xasecure.audit.jpa.javax.persistence.jdbc.user</name> + <value>rangerlogger</value> + </property> + + <property> + <name>xasecure.audit.jpa.javax.persistence.jdbc.password</name> + <value>none</value> + </property> + + <property> + <name>xasecure.audit.jpa.javax.persistence.jdbc.driver</name> + <value>com.mysql.jdbc.Driver</value> + </property> + + <property> + <name>xasecure.audit.credential.provider.file</name> + <value>jceks://file/etc/ranger/sqoopdev/auditcred.jceks</value> + </property> + + + + <!-- HDFS audit provider configuration --> + <property> + <name>xasecure.audit.hdfs.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.hdfs.is.async</name> + <value>true</value> + </property> + + <property> + <name>xasecure.audit.hdfs.async.max.queue.size</name> + <value>1048576</value> + </property> + + <property> + <name>xasecure.audit.hdfs.async.max.flush.interval.ms</name> + <value>30000</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.encoding</name> + <value></value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.directory</name> + <value>hdfs://NAMENODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd%</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.file</name> + <value>%hostname%-audit.log</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.flush.interval.seconds</name> + <value>900</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.rollover.interval.seconds</name> + <value>86400</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.destination.open.retry.interval.seconds</name> + <value>60</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.directory</name> + <value>/var/log/sqoop/audit</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.file</name> + <value>%time:yyyyMMdd-HHmm.ss%.log</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.file.buffer.size.bytes</name> + <value>8192</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.flush.interval.seconds</name> + <value>60</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.buffer.rollover.interval.seconds</name> + <value>600</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.archive.directory</name> + <value>/var/log/sqoop/audit/archive</value> + </property> + + <property> + <name>xasecure.audit.hdfs.config.local.archive.max.file.count</name> + <value>10</value> + </property> + + <!-- Audit to HDFS on Azure Datastore (WASB) requires v3 style settings. Comment the above and uncomment only the + following to audit to Azure Blob Datastore via hdfs' WASB schema. + + NOTE: If you specify one audit destination in v3 style then other destinations, if any, must also be specified in v3 style + ==== + + <property> + <name>xasecure.audit.destination.hdfs</name> + <value>enabled</value> + </property> + + <property> + <name>xasecure.audit.destination.hdfs.dir</name> + <value>wasb://ranger-aud...@youraccount.blob.core.windows.net</value> + </property> + + the following 3 correspond to the properties with similar name in core-site.xml, i.e. + - fs.azure.account.key.youraccount.blob.core.windows.net => xasecure.audit.destination.hdfs.config.fs.azure.account.key.youraccount.blob.core.windows.net and + - fs.azure.account.keyprovider.youraccount.blob.core.windows.net => xasecure.audit.destination.hdfs.config.fs.azure.account.keyprovider.youraccount.blob.core.windows.net, + - fs.azure.shellkeyprovider.script => xasecure.audit.destination.hdfs.config.fs.azure.shellkeyprovider.script, + + <property> + <name>xasecure.audit.destination.hdfs.config.fs.azure.account.key.youraccount.blob.core.windows.net</name> + <value>YOUR ENCRYPTED ACCESS KEY</value> + </property> + + <property> + <name>xasecure.audit.destination.hdfs.config.fs.azure.account.keyprovider.youraccount.blob.core.windows.net</name> + <value>org.apache.hadoop.fs.azure.ShellDecryptionKeyProvider</value> + </property> + + <property> + <name>xasecure.audit.destination.hdfs.config.fs.azure.shellkeyprovider.script</name> + <value>/usr/lib/python2.7/dist-packages/hdinsight_common/decrypt.sh</value> + </property> + --> + + <!-- Log4j audit provider configuration --> + <property> + <name>xasecure.audit.log4j.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.log4j.is.async</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.log4j.async.max.queue.size</name> + <value>10240</value> + </property> + + <property> + <name>xasecure.audit.log4j.async.max.flush.interval.ms</name> + <value>30000</value> + </property> + + <!-- Kafka audit provider configuration --> + <property> + <name>xasecure.audit.kafka.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.kafka.async.max.queue.size</name> + <value>1</value> + </property> + + <property> + <name>xasecure.audit.kafka.async.max.flush.interval.ms</name> + <value>1000</value> + </property> + + <property> + <name>xasecure.audit.kafka.broker_list</name> + <value>localhost:9092</value> + </property> + + <property> + <name>xasecure.audit.kafka.topic_name</name> + <value>ranger_audits</value> + </property> + + <!-- Ranger audit provider configuration --> + <property> + <name>xasecure.audit.solr.is.enabled</name> + <value>false</value> + </property> + + <property> + <name>xasecure.audit.solr.async.max.queue.size</name> + <value>1</value> + </property> + + <property> + <name>xasecure.audit.solr.async.max.flush.interval.ms</name> + <value>1000</value> + </property> + + <property> + <name>xasecure.audit.solr.solr_url</name> + <value>http://localhost:6083/solr/ranger_audits</value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/plugin-sqoop/conf/ranger-sqoop-security-changes.cfg ---------------------------------------------------------------------- diff --git a/plugin-sqoop/conf/ranger-sqoop-security-changes.cfg b/plugin-sqoop/conf/ranger-sqoop-security-changes.cfg new file mode 100644 index 0000000..fadca54 --- /dev/null +++ b/plugin-sqoop/conf/ranger-sqoop-security-changes.cfg @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Change the original policy parameter to work with policy manager based. +# +# +ranger.plugin.sqoop.service.name %REPOSITORY_NAME% mod create-if-not-exists + +ranger.plugin.sqoop.policy.source.impl org.apache.ranger.admin.client.RangerAdminRESTClient mod create-if-not-exists + +ranger.plugin.sqoop.policy.rest.url %POLICY_MGR_URL% mod create-if-not-exists +ranger.plugin.sqoop.policy.rest.ssl.config.file /etc/hadoop/conf/ranger-policymgr-ssl.xml mod create-if-not-exists +ranger.plugin.sqoop.policy.pollIntervalMs 30000 mod create-if-not-exists +ranger.plugin.sqoop.policy.cache.dir %POLICY_CACHE_FILE_PATH% mod create-if-not-exists +ranger.plugin.sqoop.policy.rest.client.connection.timeoutMs 120000 mod create-if-not-exists +ranger.plugin.sqoop.policy.rest.client.read.timeoutMs 30000 mod create-if-not-exists \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/plugin-sqoop/conf/ranger-sqoop-security.xml ---------------------------------------------------------------------- diff --git a/plugin-sqoop/conf/ranger-sqoop-security.xml b/plugin-sqoop/conf/ranger-sqoop-security.xml new file mode 100644 index 0000000..e54d662 --- /dev/null +++ b/plugin-sqoop/conf/ranger-sqoop-security.xml @@ -0,0 +1,83 @@ +<?xml version="1.0"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> + <property> + <name>ranger.plugin.sqoop.service.name</name> + <value>sqoopdev</value> + <description> + Name of the Ranger service containing policies for this sqoop instance + </description> + </property> + + <property> + <name>ranger.plugin.sqoop.policy.source.impl</name> + <value>org.apache.ranger.admin.client.RangerAdminRESTClient</value> + <description> + Class to retrieve policies from the source + </description> + </property> + + <property> + <name>ranger.plugin.sqoop.policy.rest.url</name> + <value>http://policymanagerhost:port</value> + <description> + URL to Ranger Admin + </description> + </property> + + <property> + <name>ranger.plugin.sqoop.policy.rest.ssl.config.file</name> + <value>/etc/hadoop/conf/ranger-policymgr-ssl.xml</value> + <description> + Path to the file containing SSL details to contact Ranger Admin + </description> + </property> + + <property> + <name>ranger.plugin.sqoop.policy.pollIntervalMs</name> + <value>30000</value> + <description> + How often to poll for changes in policies? + </description> + </property> + + <property> + <name>ranger.plugin.sqoop.policy.cache.dir</name> + <value>/etc/ranger/hadoopdev/policycache</value> + <description> + Directory where Ranger policies are cached after successful retrieval from the source + </description> + </property> + + <property> + <name>ranger.plugin.sqoop.policy.rest.client.connection.timeoutMs</name> + <value>120000</value> + <description> + RangerRestClient Connection Timeout in Milli Seconds + </description> + </property> + + <property> + <name>ranger.plugin.sqoop.policy.rest.client.read.timeoutMs</name> + <value>30000</value> + <description> + RangerRestClient read Timeout in Milli Seconds + </description> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/plugin-sqoop/pom.xml ---------------------------------------------------------------------- diff --git a/plugin-sqoop/pom.xml b/plugin-sqoop/pom.xml new file mode 100644 index 0000000..0c52e66 --- /dev/null +++ b/plugin-sqoop/pom.xml @@ -0,0 +1,67 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <artifactId>ranger-sqoop-plugin</artifactId> + <name>Sqoop Security Plugin</name> + <description>Sqoop Security Plugin</description> + <packaging>jar</packaging> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + </properties> + <parent> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger</artifactId> + <version>1.0.0-SNAPSHOT</version> + <relativePath>..</relativePath> + </parent> + <dependencies> + <dependency> + <groupId>org.apache.sqoop</groupId> + <artifactId>sqoop-core</artifactId> + <version>${sqoop.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger-plugins-common</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger-plugins-audit</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>credentialbuilder</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.solr</groupId> + <artifactId>solr-solrj</artifactId> + <version>${solr.version}</version> + </dependency> + <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + <version>${httpcomponents.httpcore.version}</version> + </dependency> + </dependencies> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/plugin-sqoop/scripts/install.properties ---------------------------------------------------------------------- diff --git a/plugin-sqoop/scripts/install.properties b/plugin-sqoop/scripts/install.properties new file mode 100644 index 0000000..44f16da --- /dev/null +++ b/plugin-sqoop/scripts/install.properties @@ -0,0 +1,140 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Location of Policy Manager URL +# +# Example: +# POLICY_MGR_URL=http://policymanager.xasecure.net:6080 +# +POLICY_MGR_URL= + +# +# This is the repository name created within policy manager +# +# Example: +# REPOSITORY_NAME=sqoopdev +# +REPOSITORY_NAME= + +# +# Name of the directory where the component's lib and conf directory exist. +# This location should be relative to the parent of the directory containing +# the plugin installation files. +# +COMPONENT_INSTALL_DIR_NAME=../sqoop + +# Enable audit logs to Solr +#Example +#XAAUDIT.SOLR.ENABLE=true +#XAAUDIT.SOLR.URL=http://localhost:6083/solr/ranger_audits +#XAAUDIT.SOLR.ZOOKEEPER= +#XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/sqoop/audit/solr/spool + +XAAUDIT.SOLR.ENABLE=false +XAAUDIT.SOLR.URL=NONE +XAAUDIT.SOLR.USER=NONE +XAAUDIT.SOLR.PASSWORD=NONE +XAAUDIT.SOLR.ZOOKEEPER=NONE +XAAUDIT.SOLR.FILE_SPOOL_DIR=/var/log/sqoop/audit/solr/spool + +# Enable audit logs to HDFS +#Example +#XAAUDIT.HDFS.ENABLE=true +#XAAUDIT.HDFS.HDFS_DIR=hdfs://node-1.example.com:8020/ranger/audit +# If using Azure Blob Storage +#XAAUDIT.HDFS.HDFS_DIR=wasb[s]://<containername>@<accountname>.blob.core.windows.net/<path> +#XAAUDIT.HDFS.HDFS_DIR=wasb://ranger_audit_contai...@my-azure-account.blob.core.windows.net/ranger/audit +#XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/sqoop/audit/hdfs/spool + +XAAUDIT.HDFS.ENABLE=false +XAAUDIT.HDFS.HDFS_DIR=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit +XAAUDIT.HDFS.FILE_SPOOL_DIR=/var/log/sqoop/audit/hdfs/spool + +# Following additional propertis are needed When auditing to Azure Blob Storage via HDFS +# Get these values from your /etc/hadoop/conf/core-site.xml +#XAAUDIT.HDFS.HDFS_DIR=wasb[s]://<containername>@<accountname>.blob.core.windows.net/<path> +XAAUDIT.HDFS.AZURE_ACCOUNTNAME=__REPLACE_AZURE_ACCOUNT_NAME +XAAUDIT.HDFS.AZURE_ACCOUNTKEY=__REPLACE_AZURE_ACCOUNT_KEY +XAAUDIT.HDFS.AZURE_SHELL_KEY_PROVIDER=__REPLACE_AZURE_SHELL_KEY_PROVIDER +XAAUDIT.HDFS.AZURE_ACCOUNTKEY_PROVIDER=__REPLACE_AZURE_ACCOUNT_KEY_PROVIDER + +# End of V3 properties + + +# +# Audit to HDFS Configuration +# +# If XAAUDIT.HDFS.IS_ENABLED is set to true, please replace tokens +# that start with __REPLACE__ with appropriate values +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/sqoop/audit +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/sqoop/audit/archive +# +# Example: +# XAAUDIT.HDFS.IS_ENABLED=true +# XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://namenode.example.com:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +# XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=/var/log/sqoop/audit +# XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=/var/log/sqoop/audit/archive +# +XAAUDIT.HDFS.IS_ENABLED=false +XAAUDIT.HDFS.DESTINATION_DIRECTORY=hdfs://__REPLACE__NAME_NODE_HOST:8020/ranger/audit/%app-type%/%time:yyyyMMdd% +XAAUDIT.HDFS.LOCAL_BUFFER_DIRECTORY=__REPLACE__LOG_DIR/sqoop/audit +XAAUDIT.HDFS.LOCAL_ARCHIVE_DIRECTORY=__REPLACE__LOG_DIR/sqoop/audit/archive + +XAAUDIT.HDFS.DESTINTATION_FILE=%hostname%-audit.log +XAAUDIT.HDFS.DESTINTATION_FLUSH_INTERVAL_SECONDS=900 +XAAUDIT.HDFS.DESTINTATION_ROLLOVER_INTERVAL_SECONDS=86400 +XAAUDIT.HDFS.DESTINTATION_OPEN_RETRY_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_FILE=%time:yyyyMMdd-HHmm.ss%.log +XAAUDIT.HDFS.LOCAL_BUFFER_FLUSH_INTERVAL_SECONDS=60 +XAAUDIT.HDFS.LOCAL_BUFFER_ROLLOVER_INTERVAL_SECONDS=600 +XAAUDIT.HDFS.LOCAL_ARCHIVE_MAX_FILE_COUNT=10 + +#Solr Audit Provder +XAAUDIT.SOLR.IS_ENABLED=false +XAAUDIT.SOLR.MAX_QUEUE_SIZE=1 +XAAUDIT.SOLR.MAX_FLUSH_INTERVAL_MS=1000 +XAAUDIT.SOLR.SOLR_URL=http://localhost:6083/solr/ranger_audits + +# +# SSL Client Certificate Information +# +# Example: +# SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks +# SSL_KEYSTORE_PASSWORD=none +# SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks +# SSL_TRUSTSTORE_PASSWORD=none +# +# You do not need use SSL between agent and security admin tool, please leave these sample value as it is. +# +SSL_KEYSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-keystore.jks +SSL_KEYSTORE_PASSWORD=myKeyFilePassword +SSL_TRUSTSTORE_FILE_PATH=/etc/hadoop/conf/ranger-plugin-truststore.jks +SSL_TRUSTSTORE_PASSWORD=changeit + +# +# Custom component user +# CUSTOM_COMPONENT_USER=<custom-user> +# keep blank if component user is default +CUSTOM_USER=sqoop + + +# +# Custom component group +# CUSTOM_COMPONENT_GROUP=<custom-group> +# keep blank if component group is default +CUSTOM_GROUP=hadoop http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/plugin-sqoop/src/main/java/org/apache/ranger/authorization/sqoop/authorizer/RangerSqoopAuthorizer.java ---------------------------------------------------------------------- diff --git a/plugin-sqoop/src/main/java/org/apache/ranger/authorization/sqoop/authorizer/RangerSqoopAuthorizer.java b/plugin-sqoop/src/main/java/org/apache/ranger/authorization/sqoop/authorizer/RangerSqoopAuthorizer.java new file mode 100644 index 0000000..17a7a63 --- /dev/null +++ b/plugin-sqoop/src/main/java/org/apache/ranger/authorization/sqoop/authorizer/RangerSqoopAuthorizer.java @@ -0,0 +1,189 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.authorization.sqoop.authorizer; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.Date; +import java.util.List; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.audit.RangerDefaultAuditHandler; +import org.apache.ranger.plugin.policyengine.RangerAccessRequestImpl; +import org.apache.ranger.plugin.policyengine.RangerAccessResourceImpl; +import org.apache.ranger.plugin.policyengine.RangerAccessResult; +import org.apache.ranger.plugin.service.RangerBasePlugin; +import org.apache.ranger.services.sqoop.client.SqoopResourceMgr; +import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.model.MPrincipal; +import org.apache.sqoop.model.MPrivilege; +import org.apache.sqoop.model.MResource; +import org.apache.sqoop.security.AuthorizationValidator; +import org.apache.sqoop.security.SecurityError; + +import com.google.common.collect.Sets; + +public class RangerSqoopAuthorizer extends AuthorizationValidator { + private static final Log LOG = LogFactory.getLog(RangerSqoopAuthorizer.class); + + private static volatile RangerSqoopPlugin sqoopPlugin = null; + + private static String clientIPAddress = null; + + public RangerSqoopAuthorizer() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerSqoopAuthorizer.RangerSqoopAuthorizer()"); + } + + this.init(); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerSqoopAuthorizer.RangerSqoopAuthorizer()"); + } + } + + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerSqoopAuthorizer.init()"); + } + + RangerSqoopPlugin plugin = sqoopPlugin; + + if (plugin == null) { + synchronized (RangerSqoopAuthorizer.class) { + plugin = sqoopPlugin; + + if (plugin == null) { + plugin = new RangerSqoopPlugin(); + plugin.init(); + sqoopPlugin = plugin; + + clientIPAddress = getClientIPAddress(); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerSqoopAuthorizer.init()"); + } + } + + @Override + public void checkPrivileges(MPrincipal principal, List<MPrivilege> privileges) throws SqoopException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerSqoopAuthorizer.checkPrivileges( principal=" + principal + ", privileges=" + + privileges + ")"); + } + + if (CollectionUtils.isEmpty(privileges)) { + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerSqoopAuthorizer.checkPrivileges() return because privileges is empty."); + } + return; + } + + RangerSqoopPlugin plugin = sqoopPlugin; + String clusterName = sqoopPlugin.getClusterName(); + + if (plugin != null) { + for (MPrivilege privilege : privileges) { + RangerSqoopAccessRequest request = new RangerSqoopAccessRequest(principal, privilege, clusterName, + clientIPAddress); + + RangerAccessResult result = plugin.isAccessAllowed(request); + if (result != null && !result.getIsAllowed()) { + throw new SqoopException(SecurityError.AUTH_0014, "principal=" + principal + + " does not have privileges for : " + privilege); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerSqoopAuthorizer.checkPrivileges() success without exception."); + } + } + + private String getClientIPAddress() { + InetAddress ip = null; + try { + ip = InetAddress.getLocalHost(); + } catch (UnknownHostException e) { + if (LOG.isDebugEnabled()) { + LOG.debug("Failed to get Client IP Address" + e); + } + } + + String ret = null; + if (ip != null) { + ret = ip.getHostAddress(); + } + return ret; + } +} + +class RangerSqoopPlugin extends RangerBasePlugin { + public RangerSqoopPlugin() { + super("sqoop", "sqoop"); + } + + @Override + public void init() { + super.init(); + + RangerDefaultAuditHandler auditHandler = new RangerDefaultAuditHandler(); + + super.setResultProcessor(auditHandler); + } +} + +class RangerSqoopResource extends RangerAccessResourceImpl { + public RangerSqoopResource(MResource resource) { + if (MResource.TYPE.CONNECTOR.name().equals(resource.getType())) { + setValue(SqoopResourceMgr.CONNECTOR, resource.getName()); + } + if (MResource.TYPE.LINK.name().equals(resource.getType())) { + setValue(SqoopResourceMgr.LINK, resource.getName()); + } + if (MResource.TYPE.JOB.name().equals(resource.getType())) { + setValue(SqoopResourceMgr.JOB, resource.getName()); + } + } +} + +class RangerSqoopAccessRequest extends RangerAccessRequestImpl { + public RangerSqoopAccessRequest(MPrincipal principal, MPrivilege privilege, String clusterName, + String clientIPAddress) { + super.setResource(new RangerSqoopResource(privilege.getResource())); + if (MPrincipal.TYPE.USER.name().equals(principal.getType())) { + super.setUser(principal.getName()); + } + if (MPrincipal.TYPE.GROUP.name().equals(principal.getType())) { + super.setUserGroups(Sets.newHashSet(principal.getName())); + } + + String action = privilege.getAction(); + super.setAccessType(action); + super.setAction(action); + + super.setAccessTime(new Date()); + super.setClientIPAddress(clientIPAddress); + super.setClusterName(clusterName); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/RangerServiceSqoop.java ---------------------------------------------------------------------- diff --git a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/RangerServiceSqoop.java b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/RangerServiceSqoop.java new file mode 100644 index 0000000..d962e5f --- /dev/null +++ b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/RangerServiceSqoop.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.services.sqoop; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.model.RangerService; +import org.apache.ranger.plugin.model.RangerServiceDef; +import org.apache.ranger.plugin.service.RangerBaseService; +import org.apache.ranger.plugin.service.ResourceLookupContext; +import org.apache.ranger.services.sqoop.client.SqoopResourceMgr; + +public class RangerServiceSqoop extends RangerBaseService { + + private static final Log LOG = LogFactory.getLog(RangerServiceSqoop.class); + + public RangerServiceSqoop() { + super(); + } + + @Override + public void init(RangerServiceDef serviceDef, RangerService service) { + super.init(serviceDef, service); + } + + @Override + public Map<String, Object> validateConfig() throws Exception { + Map<String, Object> ret = new HashMap<String, Object>(); + String serviceName = getServiceName(); + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceSqoop.validateConfig Service: (" + serviceName + " )"); + } + if (configs != null) { + try { + ret = SqoopResourceMgr.validateConfig(serviceName, configs); + } catch (Exception e) { + LOG.error("<== RangerServiceSqoop.validateConfig Error:" + e); + throw e; + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceSqoop.validateConfig Response : (" + ret + " )"); + } + return ret; + } + + @Override + public List<String> lookupResource(ResourceLookupContext context) throws Exception { + + List<String> ret = new ArrayList<String>(); + String serviceName = getServiceName(); + Map<String, String> configs = getConfigs(); + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerServiceSqoop.lookupResource Context: (" + context + ")"); + } + if (context != null) { + try { + ret = SqoopResourceMgr.getSqoopResources(serviceName, configs, context); + } catch (Exception e) { + LOG.error("<==RangerServiceSqoop.lookupResource Error : " + e); + throw e; + } + } + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerServiceSqoop.lookupResource Response: (" + ret + ")"); + } + return ret; + } +} http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopClient.java ---------------------------------------------------------------------- diff --git a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopClient.java b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopClient.java new file mode 100644 index 0000000..640d5db --- /dev/null +++ b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopClient.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.services.sqoop.client; + +import java.security.PrivilegedAction; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.security.auth.Subject; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.ArrayUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.security.authentication.client.PseudoAuthenticator; +import org.apache.http.HttpStatus; +import org.apache.log4j.Logger; +import org.apache.ranger.plugin.client.BaseClient; +import org.apache.ranger.plugin.client.HadoopException; +import org.apache.ranger.services.sqoop.client.json.model.SqoopConnectorResponse; +import org.apache.ranger.services.sqoop.client.json.model.SqoopConnectorsResponse; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; + +public class SqoopClient extends BaseClient { + + private static final Logger LOG = Logger.getLogger(SqoopClient.class); + + private static final String EXPECTED_MIME_TYPE = "application/json"; + + private static final String SQOOP_CONNECTOR_API_ENDPOINT = "/sqoop/v1/connector/all"; + + private static final String ERROR_MESSAGE = " You can still save the repository and start creating " + + "policies, but you would not be able to use autocomplete for " + + "resource names. Check ranger_admin.log for more info."; + + private String sqoopUrl; + private String userName; + + public SqoopClient(String serviceName, Map<String, String> configs) { + + super(serviceName, configs, "sqoop-client"); + this.sqoopUrl = configs.get("sqoop.url"); + this.userName = configs.get("username"); + + if (StringUtils.isEmpty(this.sqoopUrl)) { + LOG.error("No value found for configuration 'sqoop.url'. Sqoop resource lookup will fail."); + } + if (StringUtils.isEmpty(this.userName)) { + LOG.error("No value found for configuration 'username'. Sqoop resource lookup will fail."); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Sqoop Client is build with url [" + this.sqoopUrl + "], user: [" + this.userName + "]."); + } + } + + public List<String> getConnectorList(final String connectorMatching, final List<String> existingConnectors) { + if (LOG.isDebugEnabled()) { + LOG.debug("Getting sqoop connector list for connectorMatching: " + connectorMatching + + ", existingConnectors: " + existingConnectors); + } + Subject subj = getLoginSubject(); + if (subj == null) { + return null; + } + + List<String> ret = Subject.doAs(subj, new PrivilegedAction<List<String>>() { + + @Override + public List<String> run() { + + ClientResponse response = getClientResponse(sqoopUrl, SQOOP_CONNECTOR_API_ENDPOINT, userName); + + SqoopConnectorsResponse sqoopConnectorsResponse = getSqoopConnectorResponse(response); + if (sqoopConnectorsResponse == null) { + return null; + } + List<SqoopConnectorResponse> connectorResponses = sqoopConnectorsResponse.getConnectors(); + List<String> connectors = null; + if (CollectionUtils.isNotEmpty(connectorResponses)) { + connectors = getConnectorFromResponse(connectorMatching, existingConnectors, connectorResponses); + } + return connectors; + } + }); + + if (LOG.isDebugEnabled()) { + LOG.debug("Getting sqoop project connector result: " + ret); + } + return ret; + } + + public List<String> getLinkList(final String linkMatching, final List<String> existingLinks) { + // TODO + return null; + } + + public List<String> getJobList(final String jobMatching, final List<String> existingJobs) { + // TODO + return null; + } + + private static ClientResponse getClientResponse(String sqoopUrl, String sqoopApi, String userName) { + ClientResponse response = null; + String[] sqoopUrls = sqoopUrl.trim().split("[,;]"); + if (ArrayUtils.isEmpty(sqoopUrls)) { + return null; + } + + Client client = Client.create(); + + for (String currentUrl : sqoopUrls) { + if (StringUtils.isBlank(currentUrl)) { + continue; + } + + String url = currentUrl.trim() + sqoopApi + "?" + PseudoAuthenticator.USER_NAME + "=" + userName; + try { + response = getClientResponse(url, client); + + if (response != null) { + if (response.getStatus() == HttpStatus.SC_OK) { + break; + } else { + response.close(); + } + } + } catch (Throwable t) { + String msgDesc = "Exception while getting sqoop response, sqoopUrl: " + url; + LOG.error(msgDesc, t); + } + } + client.destroy(); + + return response; + } + + private static ClientResponse getClientResponse(String url, Client client) { + if (LOG.isDebugEnabled()) { + LOG.debug("getClientResponse():calling " + url); + } + + WebResource webResource = client.resource(url); + + ClientResponse response = webResource.accept(EXPECTED_MIME_TYPE).get(ClientResponse.class); + + if (response != null) { + if (LOG.isDebugEnabled()) { + LOG.debug("getClientResponse():response.getStatus()= " + response.getStatus()); + } + if (response.getStatus() != HttpStatus.SC_OK) { + LOG.warn("getClientResponse():response.getStatus()= " + response.getStatus() + " for URL " + url + + ", failed to get sqoop resource list."); + String jsonString = response.getEntity(String.class); + LOG.warn(jsonString); + } + } + return response; + } + + private SqoopConnectorsResponse getSqoopConnectorResponse(ClientResponse response) { + SqoopConnectorsResponse sqoopConnectorsResponse = null; + try { + if (response != null && response.getStatus() == HttpStatus.SC_OK) { + String jsonString = response.getEntity(String.class); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + + sqoopConnectorsResponse = gson.fromJson(jsonString, SqoopConnectorsResponse.class); + } else { + String msgDesc = "Unable to get a valid response for " + "expected mime type : [" + EXPECTED_MIME_TYPE + + "], sqoopUrl: " + sqoopUrl + " - got null response."; + LOG.error(msgDesc); + HadoopException hdpException = new HadoopException(msgDesc); + hdpException.generateResponseDataMap(false, msgDesc, msgDesc + ERROR_MESSAGE, null, null); + throw hdpException; + } + } catch (HadoopException he) { + throw he; + } catch (Throwable t) { + String msgDesc = "Exception while getting sqoop project response, sqoopUrl: " + sqoopUrl; + HadoopException hdpException = new HadoopException(msgDesc, t); + + LOG.error(msgDesc, t); + + hdpException.generateResponseDataMap(false, BaseClient.getMessage(t), msgDesc + ERROR_MESSAGE, null, null); + throw hdpException; + + } finally { + if (response != null) { + response.close(); + } + } + return sqoopConnectorsResponse; + } + + private static List<String> getConnectorFromResponse(String connectorMatching, List<String> existingConnectors, + List<SqoopConnectorResponse> connectorResponses) { + List<String> connectors = new ArrayList<String>(); + for (SqoopConnectorResponse connectorResponse : connectorResponses) { + String connectorName = connectorResponse.getName(); + if (CollectionUtils.isNotEmpty(existingConnectors) && existingConnectors.contains(connectorName)) { + continue; + } + if (StringUtils.isEmpty(connectorMatching) || connectorMatching.startsWith("*") + || connectorName.toLowerCase().startsWith(connectorMatching.toLowerCase())) { + if (LOG.isDebugEnabled()) { + LOG.debug("getConnectorFromResponse(): Adding sqoop connector " + connectorName); + } + connectors.add(connectorName); + } + } + return connectors; + } + + public static Map<String, Object> connectionTest(String serviceName, Map<String, String> configs) { + SqoopClient sqoopClient = getSqoopClient(serviceName, configs); + List<String> strList = sqoopClient.getConnectorList(null, null); + + boolean connectivityStatus = false; + if (CollectionUtils.isNotEmpty(strList)) { + if (LOG.isDebugEnabled()) { + LOG.debug("ConnectionTest list size " + strList.size() + " sqoop connectors."); + } + connectivityStatus = true; + } + + Map<String, Object> responseData = new HashMap<String, Object>(); + if (connectivityStatus) { + String successMsg = "ConnectionTest Successful."; + BaseClient.generateResponseDataMap(connectivityStatus, successMsg, successMsg, null, null, responseData); + } else { + String failureMsg = "Unable to retrieve any sqoop connectors using given parameters."; + BaseClient.generateResponseDataMap(connectivityStatus, failureMsg, failureMsg + ERROR_MESSAGE, null, null, + responseData); + } + + return responseData; + } + + public static SqoopClient getSqoopClient(String serviceName, Map<String, String> configs) { + SqoopClient sqoopClient = null; + if (LOG.isDebugEnabled()) { + LOG.debug("Getting SqoopClient for datasource: " + serviceName); + } + if (MapUtils.isEmpty(configs)) { + String msgDesc = "Could not connect sqoop as Connection ConfigMap is empty."; + LOG.error(msgDesc); + HadoopException hdpException = new HadoopException(msgDesc); + hdpException.generateResponseDataMap(false, msgDesc, msgDesc + ERROR_MESSAGE, null, null); + throw hdpException; + } else { + sqoopClient = new SqoopClient(serviceName, configs); + } + return sqoopClient; + } +} http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopResourceMgr.java ---------------------------------------------------------------------- diff --git a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopResourceMgr.java b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopResourceMgr.java new file mode 100644 index 0000000..507c078 --- /dev/null +++ b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/SqoopResourceMgr.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.services.sqoop.client; + +import java.util.List; +import java.util.Map; + +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.log4j.Logger; +import org.apache.ranger.plugin.service.ResourceLookupContext; + +public class SqoopResourceMgr { + + public static final String CONNECTOR = "connector"; + public static final String LINK = "link"; + public static final String JOB = "job"; + + private static final Logger LOG = Logger.getLogger(SqoopResourceMgr.class); + + public static Map<String, Object> validateConfig(String serviceName, Map<String, String> configs) throws Exception { + Map<String, Object> ret = null; + + if (LOG.isDebugEnabled()) { + LOG.debug("==> SqoopResourceMgr.validateConfig ServiceName: " + serviceName + "Configs" + configs); + } + + try { + ret = SqoopClient.connectionTest(serviceName, configs); + } catch (Exception e) { + LOG.error("<== SqoopResourceMgr.validateConfig Error: " + e); + throw e; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== SqoopResourceMgr.validateConfig Result: " + ret); + } + return ret; + } + + public static List<String> getSqoopResources(String serviceName, Map<String, String> configs, + ResourceLookupContext context) { + String userInput = context.getUserInput(); + String resource = context.getResourceName(); + Map<String, List<String>> resourceMap = context.getResources(); + if (LOG.isDebugEnabled()) { + LOG.debug("==> SqoopResourceMgr.getSqoopResources() userInput: " + userInput + ", resource: " + resource + + ", resourceMap: " + resourceMap); + } + + if (MapUtils.isEmpty(configs)) { + LOG.error("Connection Config is empty!"); + return null; + } + + if (StringUtils.isEmpty(userInput)) { + LOG.warn("User input is empty, set default value : *"); + userInput = "*"; + } + + final SqoopClient sqoopClient = SqoopClient.getSqoopClient(serviceName, configs); + if (sqoopClient == null) { + LOG.error("Failed to getSqoopResources!"); + return null; + } + + List<String> resultList = null; + + if (StringUtils.isNotEmpty(resource)) { + switch (resource) { + case CONNECTOR: + List<String> existingConnectors = resourceMap.get(CONNECTOR); + resultList = sqoopClient.getConnectorList(userInput, existingConnectors); + break; + case LINK: + List<String> existingLinks = resourceMap.get(LINK); + resultList = sqoopClient.getLinkList(userInput, existingLinks); + break; + case JOB: + List<String> existingJobs = resourceMap.get(JOB); + resultList = sqoopClient.getJobList(userInput, existingJobs); + break; + default: + break; + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== SqoopResourceMgr.getSqoopResources() result: " + resultList); + } + return resultList; + } + +} http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopConnectorResponse.java ---------------------------------------------------------------------- diff --git a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopConnectorResponse.java b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopConnectorResponse.java new file mode 100644 index 0000000..3dd0541 --- /dev/null +++ b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopConnectorResponse.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.services.sqoop.client.json.model; + + +public class SqoopConnectorResponse { + private Long id; + + private String name; + + private String version; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + @Override + public String toString() { + return "SqoopConnectorResponse [id=" + id + ", name=" + name + ", version=" + version + "]"; + } +} http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopConnectorsResponse.java ---------------------------------------------------------------------- diff --git a/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopConnectorsResponse.java b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopConnectorsResponse.java new file mode 100644 index 0000000..26090b5 --- /dev/null +++ b/plugin-sqoop/src/main/java/org/apache/ranger/services/sqoop/client/json/model/SqoopConnectorsResponse.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ranger.services.sqoop.client.json.model; + +import java.util.List; + +public class SqoopConnectorsResponse { + private List<SqoopConnectorResponse> connectors; + + public List<SqoopConnectorResponse> getConnectors() { + return connectors; + } + + public void setConnectors(List<SqoopConnectorResponse> connectors) { + this.connectors = connectors; + } + + @Override + public String toString() { + return "SqoopConnectorsResponse [connectors=" + connectors + "]"; + } +} http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 3958014..0c968c5 100644 --- a/pom.xml +++ b/pom.xml @@ -108,6 +108,8 @@ <module>ranger-examples</module> <module>ranger-tools</module> <module>plugin-atlas</module> + <module>plugin-sqoop</module> + <module>ranger-sqoop-plugin-shim</module> </modules> <properties> <maven.version.required>3.3.3</maven.version.required> @@ -208,6 +210,7 @@ <springframework.security.version>3.2.10.RELEASE</springframework.security.version> <springframework.test.version>3.2.10.RELEASE</springframework.test.version> <springframework.version>3.2.10.RELEASE</springframework.version> + <sqoop.version>1.99.7</sqoop.version> <storm.version>1.0.2</storm.version> <tomcat.embed.version>7.0.77</tomcat.embed.version> <velocity.version>1.7</velocity.version> @@ -414,6 +417,7 @@ <descriptor>src/main/assembly/ranger-tools.xml</descriptor> <descriptor>src/main/assembly/ranger-src.xml</descriptor> <descriptor>src/main/assembly/plugin-atlas.xml</descriptor> + <descriptor>src/main/assembly/plugin-sqoop.xml</descriptor> </descriptors> </configuration> </plugin> http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/ranger-sqoop-plugin-shim/.gitignore ---------------------------------------------------------------------- diff --git a/ranger-sqoop-plugin-shim/.gitignore b/ranger-sqoop-plugin-shim/.gitignore new file mode 100644 index 0000000..b83d222 --- /dev/null +++ b/ranger-sqoop-plugin-shim/.gitignore @@ -0,0 +1 @@ +/target/ http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/ranger-sqoop-plugin-shim/pom.xml ---------------------------------------------------------------------- diff --git a/ranger-sqoop-plugin-shim/pom.xml b/ranger-sqoop-plugin-shim/pom.xml new file mode 100644 index 0000000..4a8df37 --- /dev/null +++ b/ranger-sqoop-plugin-shim/pom.xml @@ -0,0 +1,62 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <artifactId>ranger-sqoop-plugin-shim</artifactId> + <name>Sqoop Security Plugin Shim</name> + <description>Sqoop Security Plugin Shim</description> + <packaging>jar</packaging> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + </properties> + <parent> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger</artifactId> + <version>1.0.0-SNAPSHOT</version> + <relativePath>..</relativePath> + </parent> + <dependencies> + <dependency> + <groupId>org.apache.sqoop</groupId> + <artifactId>sqoop-core</artifactId> + <version>${sqoop.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger-plugins-common</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger-plugins-audit</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>credentialbuilder</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.ranger</groupId> + <artifactId>ranger-plugin-classloader</artifactId> + <version>${project.version}</version> + </dependency> + </dependencies> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/ranger-sqoop-plugin-shim/src/main/java/org/apache/ranger/authorization/sqoop/authorizer/RangerSqoopAuthorizer.java ---------------------------------------------------------------------- diff --git a/ranger-sqoop-plugin-shim/src/main/java/org/apache/ranger/authorization/sqoop/authorizer/RangerSqoopAuthorizer.java b/ranger-sqoop-plugin-shim/src/main/java/org/apache/ranger/authorization/sqoop/authorizer/RangerSqoopAuthorizer.java new file mode 100644 index 0000000..78b740d --- /dev/null +++ b/ranger-sqoop-plugin-shim/src/main/java/org/apache/ranger/authorization/sqoop/authorizer/RangerSqoopAuthorizer.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ranger.authorization.sqoop.authorizer; + +import java.util.List; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.ranger.plugin.classloader.RangerPluginClassLoader; +import org.apache.sqoop.common.SqoopException; +import org.apache.sqoop.model.MPrincipal; +import org.apache.sqoop.model.MPrivilege; +import org.apache.sqoop.security.AuthorizationValidator; + +public class RangerSqoopAuthorizer extends AuthorizationValidator { + private static final Log LOG = LogFactory.getLog(RangerSqoopAuthorizer.class); + + private static final String RANGER_PLUGIN_TYPE = "sqoop"; + private static final String RANGER_SQOOP_AUTHORIZER_IMPL_CLASSNAME = "org.apache.ranger.authorization.sqoop.authorizer.RangerSqoopAuthorizer"; + + private AuthorizationValidator authorizationValidator = null; + private static RangerPluginClassLoader rangerPluginClassLoader = null; + + public RangerSqoopAuthorizer() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerSqoopAuthorizer.RangerSqoopAuthorizer()"); + } + + this.init(); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerSqoopAuthorizer.RangerSqoopAuthorizer()"); + } + } + + public void init() { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerSqoopAuthorizer.init()"); + } + + try { + + rangerPluginClassLoader = RangerPluginClassLoader.getInstance(RANGER_PLUGIN_TYPE, this.getClass()); + + @SuppressWarnings("unchecked") + Class<AuthorizationValidator> cls = (Class<AuthorizationValidator>) Class.forName( + RANGER_SQOOP_AUTHORIZER_IMPL_CLASSNAME, true, rangerPluginClassLoader); + + activatePluginClassLoader(); + + authorizationValidator = cls.newInstance(); + } catch (Exception e) { + LOG.error("Error Enabling RangerSqoopAuthorizer", e); + } finally { + deactivatePluginClassLoader(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerSqoopAuthorizer.init()"); + } + } + + @Override + public void checkPrivileges(MPrincipal principal, List<MPrivilege> privileges) throws SqoopException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> RangerSqoopAuthorizer.checkPrivileges()"); + } + + try { + activatePluginClassLoader(); + + authorizationValidator.checkPrivileges(principal, privileges); + } finally { + deactivatePluginClassLoader(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== RangerSqoopAuthorizer.checkPrivileges()"); + } + } + + private void activatePluginClassLoader() { + if (rangerPluginClassLoader != null) { + rangerPluginClassLoader.activate(); + } + } + + private void deactivatePluginClassLoader() { + if (rangerPluginClassLoader != null) { + rangerPluginClassLoader.deactivate(); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/security-admin/src/main/webapp/scripts/models/BackboneFormDataType.js ---------------------------------------------------------------------- diff --git a/security-admin/src/main/webapp/scripts/models/BackboneFormDataType.js b/security-admin/src/main/webapp/scripts/models/BackboneFormDataType.js index afcc290..3d62e31 100644 --- a/security-admin/src/main/webapp/scripts/models/BackboneFormDataType.js +++ b/security-admin/src/main/webapp/scripts/models/BackboneFormDataType.js @@ -82,6 +82,9 @@ define(function(require) { if(v.name == 'yarn.url'){ formObj.type = 'TextFiledWithIcon'; formObj.errorMsg = localization.tt("hintMsg.yarnRestUrl"); + }else if(v.name == 'sqoop.url'){ + formObj.type = 'TextFiledWithIcon'; + formObj.errorMsg = localization.tt("hintMsg.sqoopRestUrl"); }else if(v.name == 'jdbc.url'){ formObj.type = 'TextFiledWithIcon'; formObj.errorMsg = localization.tt("hintMsg.hiveJDBCUrl"); http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js ---------------------------------------------------------------------- diff --git a/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js b/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js index 5900c55..b8de5c3 100644 --- a/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js +++ b/security-admin/src/main/webapp/scripts/modules/globalize/message/en.js @@ -446,6 +446,7 @@ define(function(require) { }, hintMsg : { yarnRestUrl :"1.For one url, eg.<br>'http or https://<ipaddr>:8088'<br>2.For multiple urls (use , or ; delimiter), eg.<br>'http://<ipaddr1>:8088,http://<ipaddr2>:8088'", + sqoopRestUrl :"eg. 'http://<ipaddr>:12000'", hiveJDBCUrl :"1.For Remote Mode, eg.<br>jdbc:hive2://<host>:<port><br>2.For Embedded Mode (no host or port), eg.<br>jdbc:hive2:///;initFile=<file><br>3.For HTTP Mode, eg.<br>jdbc:hive2://<host>:<port>/;<br>transportMode=http;httpPath=<httpPath><br>4.For SSL Mode, eg.<br>jdbc:hive2://<host>:<port>/;ssl=true;<br>sslTrustStore=tStore;trustStorePassword=pw<br>5.For ZooKeeper Mode, eg.<br>jdbc:hive2://<host>/;serviceDiscoveryMode=<br>zooKeeper;zooKeeperNamespace=hiveserver2<br>6.For Kerberos Mode, eg.<br>jdbc:hive2://<host>:<port>/;<br>principal=hive/dom...@example.com<br>", hdfsNameNodeUrl :"1.For one Namenode Url, eg.<br>hdfs://<host>:<port><br>2.For HA Namenode Urls(use , delimiter), eg.<br>hdfs://<host>:<port>,hdfs://<host2>:<port2><br>" } http://git-wip-us.apache.org/repos/asf/ranger/blob/6e6e42b0/src/main/assembly/admin-web.xml ---------------------------------------------------------------------- diff --git a/src/main/assembly/admin-web.xml b/src/main/assembly/admin-web.xml index 4dc52fd..aa37426 100644 --- a/src/main/assembly/admin-web.xml +++ b/src/main/assembly/admin-web.xml @@ -328,6 +328,19 @@ </includes> </moduleSet> + <moduleSet> + <binaries> + <includeDependencies>true</includeDependencies> + <outputDirectory>/ews/webapp/WEB-INF/classes/ranger-plugins/sqoop</outputDirectory> + <unpack>false</unpack> + <directoryMode>755</directoryMode> + <fileMode>644</fileMode> + </binaries> + <includes> + <include>org.apache.ranger:ranger-sqoop-plugin</include> + </includes> + </moduleSet> + </moduleSets> <fileSets>
