METRON-2221 Notebook import fails through Zeppelin REST API (anandsubbu) closes apache/metron#1489
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index 69e8dff..87397f4 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -18,7 +18,6 @@
import os
import re
-import requests
import time
import metron_security
@@ -28,6 +27,8 @@
from resource_management.core.resources.system import Execute
from resource_management.libraries.functions import format as ambari_format
from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.get_user_call_output import \
+ get_user_call_output
# Wrap major operations and functionality in this class
@@ -435,47 +436,43 @@
Logger.info("Indexing service check completed successfully")
- def get_zeppelin_auth_details(self, ses, zeppelin_server_url, env):
+ def get_zeppelin_auth_details(self, zeppelin_server_url, env):
"""
- With Ambari 2.5+, Zeppelin server is enabled to work with Shiro authentication, which requires user/password
- for authentication (see https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html for details).
+ With Ambari 2.5+, Zeppelin server is enabled to work with Shiro authentication by default, which requires
+ user/password for authentication (see https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html
+ for details).
- This method checks if Shiro authentication is enabled on the Zeppelin server. And if enabled, it returns the
- session connection details to be used for importing Zeppelin notebooks.
- :param ses: Session handle
+ This method uses the Shiro authentication credentials on the Zeppelin server to authenticate and returns the
+ cookie information to be used for importing Zeppelin notebooks.
+
:param zeppelin_server_url: Zeppelin Server URL
- :return: ses
+ :return: session_id - the cookie handle to be used for subsequent interaction
"""
from params import params
env.set_params(params)
- # Check if authentication is enabled on the Zeppelin server
+ session_id = None
try:
- ses.get(ambari_format('http://{zeppelin_server_url}/api/login'))
+ Logger.info("Shiro authentication is found to be enabled on the Zeppelin server.")
+ # Read the Shiro admin user credentials from Zeppelin config in Ambari
+ username = None
+ password = None
+ if re.search(r'^\[users\]', params.zeppelin_shiro_ini_content, re.MULTILINE):
+ tokens = re.search(r'^admin\ =.*', params.zeppelin_shiro_ini_content, re.MULTILINE).group()
+ userpassword = tokens.split(',')[0].strip()
+ username = userpassword.split('=')[0].strip()
+ password = userpassword.split('=')[1].strip()
+ else:
+ Logger.error("ERROR: Admin credentials config was not found in shiro.ini. Notebook import may fail.")
- # Establish connection if authentication is enabled
- try:
- Logger.info("Shiro authentication is found to be enabled on the Zeppelin server.")
- # Read the Shiro admin user credentials from Zeppelin config in Ambari
- seen_users = False
- username = None
- password = None
- if re.search(r'^\[users\]', params.zeppelin_shiro_ini_content, re.MULTILINE):
- seen_users = True
- tokens = re.search(r'^admin\ =.*', params.zeppelin_shiro_ini_content, re.MULTILINE).group()
- userpassword = tokens.split(',')[0].strip()
- username = userpassword.split('=')[0].strip()
- password = userpassword.split('=')[1].strip()
- else:
- Logger.error("ERROR: Admin credentials config was not found in shiro.ini. Notebook import may fail.")
+ zeppelin_creds = "userName=%s&password=%s" % (username, password)
+ cmd = 'curl -i --data \'{0}\' -X POST \"http://{1}/api/login\" | grep JSESSIONID | grep -v deleteMe | tail -1'
+ cmd = cmd.format(zeppelin_creds, params.zeppelin_server_url)
+ return_code, stdout, stderr = get_user_call_output(cmd, user=params.metron_user)
+ session_id = stdout.replace("Set-Cookie: ",'').strip()
- zeppelin_payload = {'userName': username, 'password' : password}
- ses.post(ambari_format('http://{zeppelin_server_url}/api/login'), data=zeppelin_payload)
- except:
- pass
+ except Exception as e1:
+ msg = "Unable to get Shiro authentication details: Error={0}"
+ Logger.error(msg.format(e1))
- # If authentication is not enabled, fall back to default method of imporing notebooks
- except requests.exceptions.RequestException:
- ses.get(ambari_format('http://{zeppelin_server_url}/api/notebook'))
-
- return ses
+ return session_id
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index 87064bc..57423ca 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -17,7 +17,6 @@
import os
import errno
-import requests
import metron_service
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
@@ -32,6 +31,8 @@
from resource_management.libraries.functions import format as ambari_format
from resource_management.libraries.functions.format import format
from resource_management.libraries.script import Script
+from resource_management.libraries.functions.get_user_call_output import \
+ get_user_call_output
class Indexing(Script):
@@ -228,16 +229,18 @@
Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}'))
# Check if authentication is configured on Zeppelin server, and fetch details if enabled.
- ses = requests.session()
- ses = commands.get_zeppelin_auth_details(ses, params.zeppelin_server_url, env)
+ session_id = commands.get_zeppelin_auth_details(params.zeppelin_server_url, env)
for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path):
for fileName in files:
if fileName.endswith(".json"):
Logger.info("Importing notebook: " + fileName)
- zeppelin_import_url = ambari_format('http://{zeppelin_server_url}/api/notebook/import')
- zeppelin_notebook = {'file' : open(os.path.join(dirName, fileName), 'rb')}
- res = ses.post(zeppelin_import_url, files=zeppelin_notebook)
- Logger.info("Result: " + res.text)
+ zeppelin_notebook = os.path.join(dirName, fileName)
+ zeppelin_import_url = 'curl -i -b \"{0}\" http://{1}/api/notebook/import -d @\'{2}\''
+ zeppelin_import_url = zeppelin_import_url.format(session_id, params.zeppelin_server_url, zeppelin_notebook)
+ return_code, import_result, stderr = get_user_call_output(zeppelin_import_url, user=params.metron_user)
+ Logger.info("Status of importing notebook: " + import_result)
+ if return_code != 0:
+ Logger.error("Error importing notebook: " + fileName + " Error Message: " + stderr)
if __name__ == "__main__":
Indexing().execute()