This is an automated email from the ASF dual-hosted git repository. anandsubbu pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push: new 5eed666 METRON-2221 Notebook import fails through Zeppelin REST API (anandsubbu) closes apache/metron#1489 5eed666 is described below commit 5eed6663f2689e160be0a525c8e423961cd2269f Author: anandsubbu <anandsu...@gmail.com> AuthorDate: Wed Sep 4 22:01:45 2019 +0530 METRON-2221 Notebook import fails through Zeppelin REST API (anandsubbu) closes apache/metron#1489 --- .../CURRENT/package/scripts/indexing_commands.py | 71 +++++++++++----------- .../CURRENT/package/scripts/indexing_master.py | 17 +++--- 2 files changed, 44 insertions(+), 44 deletions(-) diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py index 69e8dff..87397f4 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py @@ -18,7 +18,6 @@ limitations under the License. import os import re -import requests import time import metron_security @@ -28,6 +27,8 @@ from resource_management.core.logger import Logger from resource_management.core.resources.system import Execute from resource_management.libraries.functions import format as ambari_format from resource_management.libraries.functions.format import format +from resource_management.libraries.functions.get_user_call_output import \ + get_user_call_output # Wrap major operations and functionality in this class @@ -435,47 +436,43 @@ class IndexingCommands: Logger.info("Indexing service check completed successfully") - def get_zeppelin_auth_details(self, ses, zeppelin_server_url, env): + def get_zeppelin_auth_details(self, zeppelin_server_url, env): """ - With Ambari 2.5+, Zeppelin server is enabled to work with Shiro authentication, which requires user/password - for authentication (see https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html for details). + With Ambari 2.5+, Zeppelin server is enabled to work with Shiro authentication by default, which requires + user/password for authentication (see https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html + for details). + + This method uses the Shiro authentication credentials on the Zeppelin server to authenticate and returns the + cookie information to be used for importing Zeppelin notebooks. - This method checks if Shiro authentication is enabled on the Zeppelin server. And if enabled, it returns the - session connection details to be used for importing Zeppelin notebooks. - :param ses: Session handle :param zeppelin_server_url: Zeppelin Server URL - :return: ses + :return: session_id - the cookie handle to be used for subsequent interaction """ from params import params env.set_params(params) - # Check if authentication is enabled on the Zeppelin server + session_id = None try: - ses.get(ambari_format('http://{zeppelin_server_url}/api/login')) - - # Establish connection if authentication is enabled - try: - Logger.info("Shiro authentication is found to be enabled on the Zeppelin server.") - # Read the Shiro admin user credentials from Zeppelin config in Ambari - seen_users = False - username = None - password = None - if re.search(r'^\[users\]', params.zeppelin_shiro_ini_content, re.MULTILINE): - seen_users = True - tokens = re.search(r'^admin\ =.*', params.zeppelin_shiro_ini_content, re.MULTILINE).group() - userpassword = tokens.split(',')[0].strip() - username = userpassword.split('=')[0].strip() - password = userpassword.split('=')[1].strip() - else: - Logger.error("ERROR: Admin credentials config was not found in shiro.ini. Notebook import may fail.") - - zeppelin_payload = {'userName': username, 'password' : password} - ses.post(ambari_format('http://{zeppelin_server_url}/api/login'), data=zeppelin_payload) - except: - pass - - # If authentication is not enabled, fall back to default method of imporing notebooks - except requests.exceptions.RequestException: - ses.get(ambari_format('http://{zeppelin_server_url}/api/notebook')) - - return ses + Logger.info("Shiro authentication is found to be enabled on the Zeppelin server.") + # Read the Shiro admin user credentials from Zeppelin config in Ambari + username = None + password = None + if re.search(r'^\[users\]', params.zeppelin_shiro_ini_content, re.MULTILINE): + tokens = re.search(r'^admin\ =.*', params.zeppelin_shiro_ini_content, re.MULTILINE).group() + userpassword = tokens.split(',')[0].strip() + username = userpassword.split('=')[0].strip() + password = userpassword.split('=')[1].strip() + else: + Logger.error("ERROR: Admin credentials config was not found in shiro.ini. Notebook import may fail.") + + zeppelin_creds = "userName=%s&password=%s" % (username, password) + cmd = 'curl -i --data \'{0}\' -X POST \"http://{1}/api/login\" | grep JSESSIONID | grep -v deleteMe | tail -1' + cmd = cmd.format(zeppelin_creds, params.zeppelin_server_url) + return_code, stdout, stderr = get_user_call_output(cmd, user=params.metron_user) + session_id = stdout.replace("Set-Cookie: ",'').strip() + + except Exception as e1: + msg = "Unable to get Shiro authentication details: Error={0}" + Logger.error(msg.format(e1)) + + return session_id diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py index 87064bc..57423ca 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py @@ -17,7 +17,6 @@ limitations under the License. import os import errno -import requests import metron_service from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl @@ -32,6 +31,8 @@ from resource_management.core.source import Template from resource_management.libraries.functions import format as ambari_format from resource_management.libraries.functions.format import format from resource_management.libraries.script import Script +from resource_management.libraries.functions.get_user_call_output import \ + get_user_call_output class Indexing(Script): @@ -228,16 +229,18 @@ class Indexing(Script): Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}')) # Check if authentication is configured on Zeppelin server, and fetch details if enabled. - ses = requests.session() - ses = commands.get_zeppelin_auth_details(ses, params.zeppelin_server_url, env) + session_id = commands.get_zeppelin_auth_details(params.zeppelin_server_url, env) for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path): for fileName in files: if fileName.endswith(".json"): Logger.info("Importing notebook: " + fileName) - zeppelin_import_url = ambari_format('http://{zeppelin_server_url}/api/notebook/import') - zeppelin_notebook = {'file' : open(os.path.join(dirName, fileName), 'rb')} - res = ses.post(zeppelin_import_url, files=zeppelin_notebook) - Logger.info("Result: " + res.text) + zeppelin_notebook = os.path.join(dirName, fileName) + zeppelin_import_url = 'curl -i -b \"{0}\" http://{1}/api/notebook/import -d @\'{2}\'' + zeppelin_import_url = zeppelin_import_url.format(session_id, params.zeppelin_server_url, zeppelin_notebook) + return_code, import_result, stderr = get_user_call_output(zeppelin_import_url, user=params.metron_user) + Logger.info("Status of importing notebook: " + import_result) + if return_code != 0: + Logger.error("Error importing notebook: " + fileName + " Error Message: " + stderr) if __name__ == "__main__": Indexing().execute()