import json import requests import time from requests.auth import HTTPBasicAuth from requests_toolbelt import SSLAdapter from requests_toolbelt import MultipartEncoder from requests.adapters import HTTPAdapter from requests.packages.urllib3.poolmanager import PoolManager import ssl import logging import time from os import listdir from os.path import isfile, join import logging # API specific imports #from scaleiopy.util.tls1adapter import TLS1Adapter from api.im.mapping.im_generic_object import Im_Generic_Object from scaleiopy.api.im.mapping.node import Node_Object from scaleiopy.api.im.mapping.mdm import Mdm_Object from scaleiopy.api.im.mapping.callhome_configuration import Call_Home_Configuration_Object from scaleiopy.api.im.mapping.tb import Tb_Object from scaleiopy.api.im.mapping.syslog_configuration import Remote_Syslog_Configuration_Object from scaleiopy.api.im.mapping.sdc import Sdc_Object from scaleiopy.api.im.mapping.sds import Sds_Object from scaleiopy.api.im.mapping.sds_device import Sds_Device_Object from scaleiopy.api.im.system import System_Object class TLS1Adapter(HTTPAdapter): """ A custom HTTP adapter we mount to the session to force the use of TLSv1, which is the only thing supported by the gateway. Python 2.x tries to establish SSLv2/3 first which failed. """ def init_poolmanager(self, connections, maxsize, block=False): self.poolmanager = PoolManager(num_pools=connections, maxsize=maxsize, block=block, ssl_version=ssl.PROTOCOL_TLSv1) class Im(Im_Generic_Object): """ The IM class provides a pythonic way to interact with and manage a ScaleIO cluster using Installation Manager 'private' API/ """ def __init__(self, api_url, username, password, verify_ssl=False, LiaPassword=None, debugLevel=None): """ Initializes the class :param api_url: Base URL for the API. Often the MDM host. :type api_url: str :param username: Username to login with :type username: str :param password: Password :type password: str :return: A ScaleIO object :rtype: ScaleIO """ self._username = username self._password = password self._im_api_url = api_url self._im_session = requests.Session() #self._im_session.headers.update({'Accept': 'application/json', 'Version': '1.0'}) # Accept only json self._im_session.mount('https://', TLS1Adapter()) self._im_verify_ssl = verify_ssl self._im_logged_in = False requests.packages.urllib3.disable_warnings() # Disable unverified connection warning. self._cluster_config_cached = None self._cache_contains_uncommitted = None logging.basicConfig(format='%(asctime)s: %(levelname)s %(module)s:%(funcName)s | %(message)s', level=self._get_log_level(debugLevel)) self.logger = logging.getLogger(__name__) self.logger.debug("Logger initialized!") self.SIO_Configuration_Object = None # Holds a DICT representation of the ScaleIO System Configuration @staticmethod def _get_log_level(level): """ small static method to get logging level :param str level: string of the level e.g. "INFO" :returns logging.<LEVEL>: appropriate debug level """ # default to DEBUG if level is None or level == "DEBUG": return logging.DEBUG level = level.upper() # Make debugging configurable if level == "INFO": return logging.INFO elif level == "WARNING": return logging.WARNING elif level == "CRITICAL": return logging.CRITICAL elif level == "ERROR": return logging.ERROR elif level == "FATAL": return logging.FATAL else: raise Exception("UnknownLogLevelException: enter a valid log level") def _get_cached_config_json(self): return self._cluster_config_cached def _login(self): """ LOGIN CAN ONLY BE DONE BY POSTING TO A HTTP FORM. A COOKIE IS THEN USED FOR INTERACTING WITH THE API """ self.logger.debug("Logging into " + "{}/{}".format(self._im_api_url, "j_spring_security_check")) self._im_session.headers.update({'Content-Type':'application/x-www-form-urlencoded', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.99 Safari/537.36'}) #self._im_session.mount('https://', TLS1Adapter()) #self._im_verify_ssl = False self.j_username = self._username self.j_password = self._password requests.packages.urllib3.disable_warnings() # Disable unverified connection warning. payload = {'j_username': self.j_username, 'j_password': self.j_password, 'submit':'Login'} # login to ScaleIO IM r = self._im_session.post( "{}/{}".format(self._im_api_url,"j_spring_security_check"), verify=self._im_verify_ssl, #headers = {'Content-Type':'application/x-www-form-urlencoded', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.99 Safari/537.36'}, data=payload) self.logger.debug("Login POST response: " + "{}".format(r.text)) self._im_logged_in = True """ ADD CODE: Check if this is IM have existing configuration. If so populate ScaleIO_configurtion_object """ def _check_login(self): if not self._im_logged_in: self._im_login() else: pass return None # FIX _do_get method, easier to have one place to do error handling than in all other methods that call _do_get() def _do_get(self, uri, **kwargs): """ Convinient method for GET requests Returns http request status value from a POST request """ #TODO: # Add error handling. Check for HTTP status here would be much more conveinent than in each calling method scaleioapi_get_headers = {'Content-type':'application/json','Version':'1.0'} self.logger.debug("_do_get() " + "{}/{}".format(self._api_url,uri)) if kwargs: for key, value in kwargs.iteritems(): if key == 'headers': scaleio_get_headersvalue = value try: #response = self._im_session.get("{}/{}".format(self._api_url, uri), headers = scaleioapi_get_headers, payload = scaleio_payload).json() response = self._im_session.get("{}/{}".format(self._api_url, uri), **kwargs).json() #response = self._session.get(url, headers=scaleioapi_post_headers, **kwargs) if response.status_code == requests.codes.ok: return response else: raise RuntimeError("_do_get() - HTTP response error" + response.status_code) except: raise RuntimeError("_do_get() - Communication error with ScaleIO gateway") return response def _do_put(self, uri, **kwargs): """ Convinient method for POST requests Returns http request status value from a POST request """ #TODO: # Add error handling. Check for HTTP status here would be much more conveinent than in each calling method scaleioapi_put_headers = {'content-type':'application/json'} print "_do_put()" if kwargs: for key, value in kwargs.iteritems(): #if key == 'headers': # scaleio_post_headers = value # print "Adding custom PUT headers" if key == 'json': payload = value try: self.logger.debug("do_put(): " + "{}".format(uri)) #self._session.headers.update({'Content-Type':'application/json'}) response = self._session.put(url, headers=scaleioapi_put_headers, verify_ssl=self._im_verify_ssl, data=json.dumps(payload)) self.logger.debug("_do_put() - Response: " + "{}".format(response.text)) if response.status_code == requests.codes.ok: return response else: self.logger.error("_do_put() - HTTP response error: " + "{}".format(response.status_code)) raise RuntimeError("_do_put() - HTTP response error" + response.status_code) except: raise RuntimeError("_do_put() - Communication error with ScaleIO gateway") return response def _do_post(self, url, **kwargs): """ Convinient method for POST requests Returns http request status value from a POST request """ #TODO: # Add error handling. Check for HTTP status here would be much more conveinent than in each calling method scaleioapi_post_headers = {'Content-type':'application/json','Version':'1.0'} self.logger.debug("_do_post()") if kwargs: for key, value in kwargs.iteritems(): if key == 'headers': scaleio_post_headers = value print "Adding custom POST headers" if key == 'files': upl_files = value print "Adding files to upload" try: response = self._session.post(url, headers=scaleioapi_post_headers, verify_ssl=self._im_verify_ssl, files=upl_files) self.logger.debug("_do_post() - Response: " + "{}".format(response.text)) if response.status_code == requests.codes.ok: return response else: self.logger.error("_do_post() - Response Code: " + "{}".format(response.status_code)) raise RuntimeError("_do_post() - HTTP response error" + response.status_code) except: raise RuntimeError("_do_post() - Communication error with ScaleIO gateway") return response def get_installation_instances(self): self.logger.debug("/types/Installation/instances") #print "/types/Installation/instances/" resp = self._im_session.get("{}/{}".format(self._im_api_url, 'types/Installation/instances')) #print resp.text def get_state(self, count=None): self.logger.debug("/types/State/instances") payload = {'_':'1425822717883'} referer = 'https://192.168.100.12/install.jsp' #resp = self._im_session.get("{}/{}".format(self._im_api_url, 'types/State/instances/'), params = payload) resp = self._im_session.get("{}/{}".format(self._im_api_url, 'types/State/instances/')) return resp.text def set_state(self, state): # state can be: query, upload, install, configure self.logger.debug("set_state(" + "{})".format(state) ) if state == 'query': resp = self._im_session.put("{}/{}".format(self._im_api_url,"types/State/instances/"),data =json.dumps({"state":"query"}), headers={'Content-Type':'application/json'}) #print "PUT Request URL: " + resp.url #print "QUERY Response:" #print resp.text return True if state == 'upload': resp = self._im_session.put("{}/{}".format(self._im_api_url,"types/State/instances/"),data =json.dumps({"state":"upload"}), headers={'Content-Type':'application/json'}) #print "PUT Request URL: " + resp.url #print "UPLOAD Response:" #print resp.text return True if state == 'install': resp = self._im_session.put("{}/{}".format(self._im_api_url,"types/State/instances/"),data =json.dumps({"state":"install"}), headers={'Content-Type':'application/json'}) return True #print "PUT Request URL: " + resp.url #print "INSTALL Response:" #print resp.text if state == 'configure': resp = self._im_session.put("{}/{}".format(self._im_api_url,"types/State/instances/"),data =json.dumps({"state":"configure"}), headers={'Content-Type':'application/json'}) return True #print "PUT Request URL: " + resp.url #print "CONFIGURE Response:" #print resp.text return False def set_abort_pending(self, newstate): """ Method to set Abort state if something goes wrong during provisioning Method also used to finish provisioning process when all is completed Method: POST """ self.logger.debug("set_abort_pending(" + "{})".format(newstate)) # NOT TO BE USED #default_minimal_cluster_config = '{"installationId":null,"mdmIPs":["192.168.102.12","192.168.102.13"],"mdmPassword":"Scaleio123","liaPassword":"Scaleio123","licenseKey":null,"primaryMdm":{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.12"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"managementIPs":null,"mdmIPs":["192.168.102.12"]},"secondaryMdm":{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.13"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"managementIPs":null,"mdmIPs":["192.168.102.13"]},"tb":{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.11"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"tbIPs":["192.168.102.11"]},"sdsList":[{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.11"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"sdsName":"SDS_[192.168.102.11]","protectionDomain":"default","faultSet":null,"allIPs":["192.168.102.11"],"sdsOnlyIPs":null,"sdcOnlyIPs":null,"devices":[{"devicePath":"/home/vagrant/scaleio1","storagePool":null,"deviceName":null}],"optimized":false,"port":7072},{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.12"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"sdsName":"SDS_[192.168.102.12]","protectionDomain":"default","faultSet":null,"allIPs":["192.168.102.12"],"sdsOnlyIPs":null,"sdcOnlyIPs":null,"devices":[{"devicePath":"/home/vagrant/scaleio1","storagePool":null,"deviceName":null}],"optimized":false,"port":7072},{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.13"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"sdsName":"SDS_[192.168.102.13]","protectionDomain":"default","faultSet":null,"allIPs":["192.168.102.13"],"sdsOnlyIPs":null,"sdcOnlyIPs":null,"devices":[{"devicePath":"/home/vagrant/scaleio1","storagePool":null,"deviceName":null}],"optimized":false,"port":7072}],"sdcList":[{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.11"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"splitterRpaIp":null},{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.12"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"splitterRpaIp":null},{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.13"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"splitterRpaIp":null}],"callHomeConfiguration":null,"remoteSyslogConfiguration":null}' r1 = self._im_session.post( "{}/{}".format(self._im_api_url,"types/Command/instances/actions/abortPending"), headers={'Content-type':'application/json','Version':'1.0'}, verify=self._im_verify_ssl, data = newstate, stream=True ) if not r1.ok: # Something went wrong self.logger.error("Error set_abort_pending(" +"{})".format(newstate)) #print "Response after set_abort_pending()" # RESPONSE NEED TO BE WRAPPED IN try/catch. Cannot assume JSON is returned. #print r1.text #pprint (json.loads(r1.text)) return r1.text def set_archive_all(self): """ Last method to be called when provisioning is complete Method: POST """ self.logger.debug("set_archive_all()") r1 = self._im_session.post( "{}/{}".format(self._im_api_url,"types/Command/instances/actions/archiveAll"), headers={'Content-type':'application/json','Version':'1.0'}, verify=self._im_verify_ssl, data = '', stream=True ) if not r1.ok: # Something went wrong self.logger.error("Error code: " + "{}".format(r1.status_code)) #print "Response after set_archive_all()" # RESPONSE NEED TO BE WRAPPED IN tey/catch. Can?t assume JSON is returned. #print r1.text #pprint (json.loads(r1.text)) return r1.text def get_version(self): self.logger.debug("get_version()") payload = {'_':'1425822717883'} # Investigate what this number mean when some IM API calls are done. Its used by IM Webui. Seem to be Unixtime format. referer = 'https://192.168.100.12/status.jsp' resp = self._im_session.get("{}/{}".format(self._im_api_url, 'version/'), params = payload) def get_installation_packages_latest(self): """ In 1.31-256 getting latest or all packages seem not to work. Always same result not matter what value 'onlLatest' have. Same situation in IM WEBUI too. """ self.logger.debug("get_installation_packages_latest()") parameter = {'onlyLatest':'False'} resp = self._im_session.get("{}/{}".format(self._im_api_url, 'types/InstallationPackageWithLatest/instances'), params=parameter) #resp = self._im_session.get('https://192.168.100.52/types/InstallationPackageWithLatest/instances', params=parameter) jresp = json.loads(resp.text) #pprint(jresp) def get_installation_packages(self): """ In 1.31-256 getting latest or all packages seem not to work. Always same result not matter what value 'onlLatest' have. Same situation in IM WEBUI too. """ self.logger.debug("get_installation_packages()") parameter = {'onlyLatest':'False'} resp = self._im_session.get("{}/{}".format(self._im_api_url, 'types/InstallationPackageWithLatest/instances'), params=parameter) #resp = self._im_session.get('https://192.168.100.52/types/InstallationPackageWithLatest/instances', params=parameter) jresp = json.loads(resp.text) #pprint(jresp.text) return jresp def get_command_state(self, count=None): self.logger.debug("get_command_state(" + "{})".format(count)) #print "/types/Command/instances" #payload = {'_':'1425822717883'} # What do this mean? #resp = self._im_session.get("{}/{}".format(self._im_api_url, 'types/Command/instances/'), params = payload) resp = self._im_session.get("{}/{}".format(self._im_api_url, 'types/Command/instances')) #resp = self._im_session.get('https://192.168.100.42/types/Command/instances').json() #print "URL: " + resp.url #pprint (resp.text) return resp.text def get_nodeinfo_instances(self): self.logger.debug("/types/NodeInfo/instances/actions/downloadGetInfo") resp = self._im_session.get("{}/{}".format(self._im_api_url, 'types/NodeInfo/instances/actions/downloadGetInfo')) #resp = self._im_session.get('https://192.168.100.42/types/NodeInfo/instances/actions/downloadGetInfo') #pprint (resp.text) def get_configuration_instances(self, count=None): self.logger.debug("/types/Configuration/instances") payload = {'_':''} resp = self._im_session.get("{}/{}".format(self._im_api_url, 'types/Configuration/instances/')) #resp = self._im_session.get('https://192.168.100.42/types/Configuration/instances') #pprint (resp.text) def get_cluster_topology(self, mdmIP, mdmPassword, liaPassword=None): # Topology is returned as a file. Save it into a string. Parse as JSON. # When adding nodes to existing ScaleIO cluster using IM all node change are driven by changing topology CSV file. #print "Get Topology - Import into ScaleIO_Configuration_Object" #print "IM SESSION OBJECT:" #pprint (self._im_session) self.logger.debug("get_cluster_topology(" + "{},{},{}".format(mdmIP, mdmPassword, liaPassword)) pay1 = {'mdmIps':[mdmIP],'mdmPassword':mdmPassword} #,'liaPassword':liaPassword} #pay1 = {'mdmIp':'192.168.100.42','mdmPassword':'Password1!','liaPassword':'Password1!'} r1 = self._im_session.post( "{}/{}".format(self._im_api_url,"types/Configuration/instances/actions/refreshAndGet"), headers={'Content-type':'application/json','Version':'1.0'}, verify=self._im_verify_ssl, json=pay1, stream=True ) if not r1.ok: # Something went wrong self.logger.error("Could not fetch congiuration from remote IM") self.logger.debug("POST response: " + "{}".format(r1.text)) return r1.text def retrieve_scaleio_cluster_configuration(self, mdmIP, mdmPassword, liaPassword=None): self.logger.debug("retrieve_scaleio_cluster_configuration(" + "{},{},{}".format(mdmIP, mdmPassword, liaPassword)) sysconf_json = self.get_cluster_topology(mdmIP, mdmPassword, liaPassword) confObj = ScaleIO_System_Object.from_dict(json.loads(sysconf_json)) confObj.setMdmPassword(mdmPassword) confObj.setLiaPassword(liaPassword) self._cluster_config_cached = confObj self._cache_contains_uncommitted = False def populate_scaleio_cluster_configuration_cache(self, mdmIP, mdmPassword, liaPassword=None): self.logger.debug("populate_scaleio_cluster_configuration_cache(" + "{},{},{}".format(mdmIP, mdmPassword, liaPassword)) sysconf_json = self.get_cluster_topology(mdmIP, mdmPassword, liaPassword) confObj = ScaleIO_System_Object.from_dict(json.loads(sysconf_json)) confObj.setMdmPassword(mdmPassword) confObj.setLiaPassword(liaPassword) self._cluster_config_cached = confObj self._cache_contains_uncommitted = False def write_cluster_config_to_disk(self): self.logger.debug("write_cluster_config_to_disk()") with open("cache.json", "w") as file: file.write(self._cluster_config_cached.to_JSON()) file.close() def read_cluster_config_from_disk(self, filename = None): self.logger.debug("read_cluster_config_from_disk(" + "{})".format(filename)) if filename: with open(filename, "r") as file: #result = file.read() confObj = ScaleIO_System_Object.from_dict(json.loads(file.read())) file.close() else: with open("cache.json", "r") as file: #result = file.read() confObj = ScaleIO_System_Object.from_dict(json.loads(file.read())) file.close() self._cluster_config_cached = confObj # Read file contents into in-memory cluster configuration cache self._cache_contains_uncommitted = False def push_cached_cluster_configuration(self, mdmPassword, liaPassword, noUpload = False, noInstall= False, noConfigure = False): """ Method push cached ScaleIO cluster configuration to IM (reconfigurations that have been made to cached configuration are committed using IM) Method: POST Attach JSON cluster configuration as request payload (data). Add MDM and LIA passwords) """ self.logger.debug("push_cached_cluster_configuration(" + "{},{},{},{},{}".format(mdmPassword, liaPassword, noUpload, noInstall, noConfigure)) config_params = {'noUpload': noUpload, 'noInstall': noInstall, 'noConfigure':noConfigure} #print "Push cached ScaleIO cluster configuration to IM" self._cluster_config_cached.setMdmPassword(mdmPassword) self._cluster_config_cached.setLiaPassword(liaPassword) self.logger.debug("Push JSON data:") self.logger.debug("{}".format(self._cluster_config_cached.to_JSON())) ####### FINISH METOD - CAN ONLY PUSH - USE CACHE # SDS configured to use /home/scaleio1 #default_minimal_cluster_config = '{"installationId":null,"mdmIPs":["192.168.102.12","192.168.102.13"],"mdmPassword":"Scaleio123","liaPassword":"Scaleio123","licenseKey":null,"primaryMdm":{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.12"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"managementIPs":null,"mdmIPs":["192.168.102.12"]},"secondaryMdm":{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.13"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"managementIPs":null,"mdmIPs":["192.168.102.13"]},"tb":{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.11"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"tbIPs":["192.168.102.11"]},"sdsList":[{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.11"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"sdsName":"SDS_[192.168.102.11]","protectionDomain":"default","faultSet":null,"allIPs":["192.168.102.11"],"sdsOnlyIPs":null,"sdcOnlyIPs":null,"devices":[{"devicePath":"/home/vagrant/scaleio1","storagePool":null,"deviceName":null}],"optimized":false,"port":7072},{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.12"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"sdsName":"SDS_[192.168.102.12]","protectionDomain":"default","faultSet":null,"allIPs":["192.168.102.12"],"sdsOnlyIPs":null,"sdcOnlyIPs":null,"devices":[{"devicePath":"/home/vagrant/scaleio1","storagePool":null,"deviceName":null}],"optimized":false,"port":7072},{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.13"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"sdsName":"SDS_[192.168.102.13]","protectionDomain":"default","faultSet":null,"allIPs":["192.168.102.13"],"sdsOnlyIPs":null,"sdcOnlyIPs":null,"devices":[{"devicePath":"/home/vagrant/scaleio1","storagePool":null,"deviceName":null}],"optimized":false,"port":7072}],"sdcList":[{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.11"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"splitterRpaIp":null},{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.12"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"splitterRpaIp":null},{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.102.13"],"domain":null,"userName":"root","password":"vagrant","liaPassword":null},"nodeInfo":null,"splitterRpaIp":null}],"callHomeConfiguration":null,"remoteSyslogConfiguration":null}' # Generated with scelio_object.py - Progammatically generated JSON using a set of classes that represent different ScaleIO components default_minimal_cluster_config = '{"licenseKey": null, "mdmPassword": "Scaleio123", "mdmIPs": ["192.168.102.12", "192.168.102.13"], "sdsList": [{"node": {"userName": "root", "domain": null, "nodeName": null, "nodeIPs": ["192.168.102.11"], "liaPassword": null, "ostype": "linux", "password": "vagrant"}, "protectionDomain": "default", "nodeInfo": null, "sdsName": "SDS_192.168.102.11", "sdcOnlyIPs": [], "optimized": false, "devices": [{"devicePath": "/home/vagrant/scaleio1", "storagePool": null, "deviceName": null}], "faultSet": null, "port": "7072", "sdsOnlyIPs": [], "allIPs": ["192.168.102.11"]}, {"node": {"userName": "root", "domain": null, "nodeName": null, "nodeIPs": ["192.168.102.12"], "liaPassword": null, "ostype": "linux", "password": "vagrant"}, "protectionDomain": "default", "nodeInfo": null, "sdsName": "SDS_192.168.102.12", "sdcOnlyIPs": [], "optimized": false, "devices": [{"devicePath": "/home/vagrant/scaleio1", "storagePool": null, "deviceName": null}], "faultSet": null, "port": "7072", "sdsOnlyIPs": [], "allIPs": ["192.168.102.12"]}, {"node": {"userName": "root", "domain": null, "nodeName": null, "nodeIPs": ["192.168.102.13"], "liaPassword": null, "ostype": "linux", "password": "vagrant"}, "protectionDomain": "default", "nodeInfo": null, "sdsName": "SDS_192.168.102.13", "sdcOnlyIPs": [], "optimized": false, "devices": [{"devicePath": "/home/vagrant/scaleio1", "storagePool": null, "deviceName": null}], "faultSet": null, "port": "7072", "sdsOnlyIPs": [], "allIPs": ["192.168.102.13"]}], "liaPassword": "Scaleio123", "primaryMdm": {"node": {"userName": "root", "domain": null, "nodeName": null, "nodeIPs": ["192.168.102.12"], "liaPassword": null, "ostype": "linux", "password": "vagrant"}, "nodeInfo": null, "managementIPs": [], "mdmIPs": ["192.168.102.12"]}, "callHomeConfiguration": null, "installationId": null, "secondaryMdm": {"node": {"userName": "root", "domain": null, "nodeName": null, "nodeIPs": ["192.168.102.13"], "liaPassword": null, "ostype": "linux", "password": "vagrant"}, "nodeInfo": null, "managementIPs": [], "mdmIPs": ["192.168.102.13"]}, "sdcList": [{"node": {"userName": "root", "domain": null, "nodeName": null, "nodeIPs": ["192.168.102.11"], "liaPassword": null, "ostype": "linux", "password": "vagrant"}, "nodeInfo": null, "splitterRpaIp": null}, {"node": {"userName": "root", "domain": null, "nodeName": null, "nodeIPs": ["192.168.102.12"], "liaPassword": null, "ostype": "linux", "password": "vagrant"}, "nodeInfo": null, "splitterRpaIp": null}, {"node": {"userName": "root", "domain": null, "nodeName": null, "nodeIPs": ["192.168.102.13"], "liaPassword": null, "ostype": "linux", "password": "vagrant"}, "nodeInfo": null, "splitterRpaIp": null}], "tb": {"node": {"userName": "root", "domain": null, "nodeName": null, "nodeIPs": ["192.168.102.11"], "liaPassword": null, "ostype": "linux", "password": "vagrant"}, "nodeInfo": null, "tbIPs": ["192.168.102.11"]}, "remoteSyslogConfiguration": null}' # #default_minimal_cluster_config = '{"installationId":null,"mdmIPs":["192.168.100.51","192.168.100.52"],"mdmPassword":"Password1!","liaPassword":"Password1!","licenseKey":null,"primaryMdm":{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.100.51"],"domain":null,"userName":"root","password":"password","liaPassword":null},"nodeInfo":null,"managementIPs":null,"mdmIPs":["192.168.100.51"]},"secondaryMdm":{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.100.52"],"domain":null,"userName":"root","password":"password","liaPassword":null},"nodeInfo":null,"managementIPs":null,"mdmIPs":["192.168.100.52"]},"tb":{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.100.53"],"domain":null,"userName":"root","password":"password","liaPassword":null},"nodeInfo":null,"tbIPs":["192.168.100.53"]},"sdsList":[{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.100.51"],"domain":null,"userName":"root","password":"password","liaPassword":null},"nodeInfo":null,"sdsName":"SDS_[192.168.100.51]","protectionDomain":"default","faultSet":null,"allIPs":["192.168.100.51"],"sdsOnlyIPs":null,"sdcOnlyIPs":null,"devices":[{"devicePath":"/dev/sdb","storagePool":null,"deviceName":null}],"optimized":false,"port":7072},{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.100.52"],"domain":null,"userName":"root","password":"password","liaPassword":null},"nodeInfo":null,"sdsName":"SDS_[192.168.100.52]","protectionDomain":"default","faultSet":null,"allIPs":["192.168.100.52"],"sdsOnlyIPs":null,"sdcOnlyIPs":null,"devices":[{"devicePath":"/dev/sdb","storagePool":null,"deviceName":null}],"optimized":false,"port":7072},{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.100.53"],"domain":null,"userName":"root","password":"password","liaPassword":null},"nodeInfo":null,"sdsName":"SDS_[192.168.100.53]","protectionDomain":"default","faultSet":null,"allIPs":["192.168.100.53"],"sdsOnlyIPs":null,"sdcOnlyIPs":null,"devices":[{"devicePath":"/dev/sdb","storagePool":null,"deviceName":null}],"optimized":false,"port":7072}],"sdcList":[{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.100.51"],"domain":null,"userName":"root","password":"password","liaPassword":null},"nodeInfo":null,"splitterRpaIp":null},{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.100.52"],"domain":null,"userName":"root","password":"password","liaPassword":null},"nodeInfo":null,"splitterRpaIp":null},{"node":{"ostype":"linux","nodeName":null,"nodeIPs":["192.168.100.53"],"domain":null,"userName":"root","password":"password","liaPassword":null},"nodeInfo":null,"splitterRpaIp":null}],"callHomeConfiguration":null,"remoteSyslogConfiguration":null}' #print "JSON DUMP OF INSTALL CONFIG:" #pprint (json.loads(default_minimal_cluster_config)) r1 = self._im_session.post( "{}/{}".format(self._im_api_url,"types/Installation/instances/"), headers={'Content-type':'application/json','Version':'1.0'}, params = config_params, verify=self._im_verify_ssl, #json=json.loads(self._cluster_config_cached.to_JSON()), json = json.loads(default_minimal_cluster_config), stream=True ) if not r1.ok: # Something went wrong self.logger.error("Error push_cached_cluster_configuration()") #print "Response after push_cached_cluster_configuration()" # RESPONSE NEED TO BE WRAPPED IN tey/catch. Can?t assume JSON is returned. self.logger.debug("HTTP Response:") self.logger.debug("{}".format(r1.text)) return r1.text def add_sds_to_cluster(self, sdsobject): self.logger.debug("add_sds_to_cluster(" + "{})".format(sdsobject)) self._cluster_config_cached.sdsList.append(sdsobject) self._cache_contains_uncommitted = True def create_minimal_scaleio_cluster(self, mdmPassword, liaPassword): """ Using IM this method create a 3-node ScaleIO cluster with 2xMDM, 1xTB, 3x SDS (using /dev/sdb), 3x SDC """ """ self.read_cluster_config_from_disk("minimal-cluster.json") #self._cluster_config_cached.setMdmPassword(setMdmPassword) #self._cluster_config_cached.setLiaPassword(setLiaPassword) self.push_cached_cluster_configuration(setMdmPassword, setLiaPassword) """ ########################### # Create a ScaleIO System # ########################### # Flow: # Create Nodes # Create basic info. mdmPass, liaPass and some others # Construct MDM and TB and basic info # Create list of SDS # Create list of SDC ################### # Construct nodes # ################### nodeUsername = 'root' nodePassword = 'password' #node1 = ScaleIO_Node_Object(None, None, ['192.168.102.11'], None, 'linux', nodePassword, nodeUsername) #node2 = ScaleIO_Node_Object(None, None, ['192.168.102.12'], None, 'linux', nodePassword, nodeUsername) #node3 = ScaleIO_Node_Object(None, None, ['192.168.102.13'], None, 'linux', nodePassword, nodeUsername) node1 = ScaleIO_Node_Object(None, None, ['192.168.100.101'], None, 'linux', nodePassword, nodeUsername) node2 = ScaleIO_Node_Object(None, None, ['192.168.100.102'], None, 'linux', nodePassword, nodeUsername) node3 = ScaleIO_Node_Object(None, None, ['192.168.100.103'], None, 'linux', nodePassword, nodeUsername) print "Node Object:" pprint (node1.to_JSON()) pprint (node2.to_JSON()) pprint (node2.to_JSON()) print "" ########################################## # Construct basic info for System_Object # ########################################## mdmIPs = ['192.168.100.101','192.168.100.102'] sdcList = [] sdsList = [] #mdmPassword = 'Scaleio123' #liaPassword = 'Scaleio123' licenseKey = None installationId = None ######################################## # Create MDMs and TB for System_Object # ######################################## primaryMdm = Mdm_Object(json.loads(node2.to_JSON()), None, None, node2.nodeIPs) # WHY ISNT ManagementIPs pupulated???? Its not in a working config either. mdmIPs need to be populated though secondaryMdm = Mdm_Object(json.loads(node3.to_JSON()), None, None, node3.nodeIPs) tb = Tb_Object(json.loads(node1.to_JSON()), None, node1.nodeIPs) callHomeConfiguration = None # {'callHomeConfiguration':'None'} remoteSyslogConfiguration = None # {'remoteSysogConfiguration':'None'} ################################################################ #Create SDS objects - To be added to SDS list in System_Object # ################################################################ sds1 = Sds_Object(json.loads(node1.to_JSON()), None, 'SDS_' + str(node1.nodeIPs[0]), 'default', None, node1.nodeIPs, None, None, None, False, '7072') sds1.addDevice("/dev/sdb", None, None) sds2 = Sds_Object(json.loads(node2.to_JSON()), None, 'SDS_' + str(node2.nodeIPs[0]), 'default', None, node2.nodeIPs, None, None, None, False, '7072') sds2.addDevice("/dev/sdb", None, None) sds3 = Sds_Object(json.loads(node3.to_JSON()), None, 'SDS_' + str(node3.nodeIPs[0]), 'default', None, node3.nodeIPs, None, None, None, False, '7072') sds3.addDevice("/dev/sdb", None, None) sdsList.append(json.loads(sds1.to_JSON())) sdsList.append(json.loads(sds2.to_JSON())) sdsList.append(json.loads(sds3.to_JSON())) print "sdsList:" pprint (sdsList) ############################################################# # Create SDC objects - To be added as list to System_Object # ############################################################# """ node=None, nodeInfo=None, splitterRpaIp=None """ sdc1 = Sdc_Object(json.loads(node1.to_JSON()), None, None) sdc2 = Sdc_Object(json.loads(node2.to_JSON()), None, None) sdc3 = Sdc_Object(json.loads(node3.to_JSON()), None, None) sdcList.append(json.loads(sdc1.to_JSON())) sdcList.append(json.loads(sdc2.to_JSON())) sdcList.append(json.loads(sdc3.to_JSON())) ###################################################### # Construct a complete ScaleIO cluster configuration # ###################################################### sioobj = ScaleIO_System_Object(installationId, mdmIPs, mdmPassword, liaPassword, licenseKey, json.loads(primaryMdm.to_JSON()), json.loads(secondaryMdm.to_JSON()), json.loads(tb.to_JSON()), sdsList, sdcList, callHomeConfiguration, remoteSyslogConfiguration ) # Export sioobj to JSON (should upload clean in IM) ########################################################################### # Push System_Object JSON - To be used by IM to install ScaleIO on nodes # ########################################################################### #pprint (sioobj.to_JSON()) self._cluster_config_cached = sioobj.to_JSON() # PUSH CONFIGURATION INTO CONFIGURATION CACHE self._cache_contains_uncommitted= False # New config pushed into cache - Nothing oncommitted self.push_cluster_configuration(self._cluster_config_cached) # sioobj.to_JSON()) def push_cluster_configuration(self, scaleioobj, noUpload = False, noInstall= False, noConfigure = False): """ Method push cached ScaleIO cluster configuration to IM (reconfigurations that have been made to cached configuration are committed using IM) Method: POST Attach JSON cluster configuration as request payload (data). Add MDM and LIA passwords) """ self.logger.debug("push_cluster_configuration(" + "{},{},{},{})".format(scaleioobj, noUpload, noInstall, noConfigure)) #print "JSON DUMP OF CLUSTER CONFIG:" #pprint (json.loads(scaleioobj)) config_params = {'noUpload': noUpload, 'noInstall': noInstall, 'noConfigure':noConfigure} r1 = self._im_session.post( "{}/{}".format(self._im_api_url,"types/Installation/instances/"), headers={'Content-type':'application/json','Version':'1.0'}, params = config_params, verify=self._im_verify_ssl, #json=json.loads(self._cluster_config_cached.to_JSON()), json = json.loads(scaleioobj), stream=True ) if not r1.ok: # Something went wrong self.logger.error("Error push_cluster_configuration() - " + "Errorcode: {}".format(r1.status_code)) #print "Response after push_cluster_configuration()" # RESPONSE NEED TO BE WRAPPED IN try/catch. Cannot assume JSON is returned. #print r1.text #pprint (json.loads(r1.text)) return r1.text # Add API client methods here that interact with IM API @property def system(self): # Change to something that is usable. A Class for Generate CSV for example. pass def uploadPackages(self, directory): """ Not working. Am not able ot figure out how to upload. IT return status 200OK with this code but do not store the files. In tomcat.log (IM) there?s a complaint about character encoding whn uploading file. Not sure how to rectiy it in requests post call though """ files_to_upload_dict = {} files_to_upload_list = [ f for f in listdir(directory) if isfile(join(directory,f)) ] self.logger.debug("uploadPackages(" + "{})".format(directory)) #print "Files to upload:" for index in range(len(files_to_upload_list)): self.logger.info(files_to_upload_list[index]) self.uploadFileToIM (directory, files_to_upload_list[index], files_to_upload_list[index]) #file_tuple = {'files':{str(files_to_upload_list[index]), open(directory + files_to_upload_list[index], 'rb'), 'application/x-rpm'}} #file_tuple = {str(files_to_upload_list[index]), {open(directory + files_to_upload_list[index], 'rb'), 'application/x-rpm'}} #file_tuple = {'files': (str(files_to_upload_list[index]), open(directory + files_to_upload_list[index], 'rb'), 'application/x-rpm')} #file_tuple = (str(files_to_upload_list[index]), open(directory + files_to_upload_list[index], 'rb')) #file_tuple = {str(files_to_upload_list[index]), open(directory + files_to_upload_list[index], 'rb'), 'application/x-rpm'} #files_data_to_upload_list.append(file_tuple) #print "Files to upload Dictionary:" def uploadFileToIM (self, directory, filename, title): """ Parameters as they look in the form for uploading packages to IM """ self.logger.debug("uploadFileToIM(" + "{},{},{})".format(directory, filename, title)) parameters = {'data-filename-placement':'inside', 'title':str(filename), 'filename':str(filename), 'type':'file', 'name':'files', 'id':'fileToUpload', 'multiple':'' } file_dict = {'files':(str(filename), open(directory + filename, 'rb'), 'application/x-rpm')} m = MultipartEncoder(fields=file_dict) temp_username = self._username temp_password = self._password temp_im_api_url = self._im_api_url temp_im_session = requests.Session() temp_im_session.mount('https://', TLS1Adapter()) temp_im_verify_ssl = self._im_verify_ssl resp = temp_im_session.post( "{}/{}".format(temp_im_api_url,"types/InstallationPackage/instances/uploadPackage"), auth=HTTPBasicAuth(temp_username, temp_password), #headers = m.content_type, files = file_dict, verify = False, data = parameters ) self.logger.info("Uploaded: " + "{}".format(filename)) self.logger.debug("HTTP Response: " + "{}".format(resp.status_code)) #print "resp.text = " + resp.text def deleteFileFromIM(self, filename): pass """ Request URL:https://192.168.100.42/instances/InstallationPackage::EMC-ScaleIO-tb-1.31-260.3.el6.x86_64.rpm/ Request Method:DELETE """ self.logger.debug("deleteFileFromIM(" + "{}".format(filename)) ##### NEED TO BE IMPLEMENTED # Get list of installed files. # Reverse engineer how the process works using the IM Webui def uploadCsvConfiguration(self, conf_filename): """ NOT NEEDED. JSON can be POSTed to IM instead of sending a CSV that is locally parsed and converted to JSON. Remote Address:192.168.100.51:443 Request URL:https://192.168.100.51/types/Configuration/instances/actions/parseFromCSV Request Method:POST Status Code:200 OK Request Headersview source Accept:*/* Accept-Encoding:gzip, deflate Accept-Language:en-US,en;q=0.8,sv;q=0.6 Connection:keep-alive Content-Length:433 Content-Type:multipart/form-data; boundary=----WebKitFormBoundaryY1f2eTo1mOvh744k Cookie:JSESSIONID=A0823886072B2CEBA327A9185AC2BFE0 Host:192.168.100.51 Origin:https://192.168.100.51 Referer:https://192.168.100.51/install.jsp User-Agent:Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.111 Safari/537.36 X-Requested-With:XMLHttpRequest Request Payload ------WebKitFormBoundaryY1f2eTo1mOvh744k Content-Disposition: form-data; name="file"; filename="ScaleIO_Minimal_Config_51.csv" Content-Type: text/csv """ parameters = {'selectInstallOrExtend':'install', #'install' or 'extend' 'name':'file', 'id':'fileToUpload', 'filename':'config.csv' } file_dict = {'file':('config.csv', open(conf_filename, 'rb'), 'text/csv')} """ files = {'file': ('report.csv', 'some,data,to,send\nanother,row,to,send\n')} """ temp_username = self._username temp_password = self._password temp_im_api_url = self._im_api_url temp_im_session = requests.Session() #self._im_session.headers.update({'Accept': 'application/json', 'Version': '1.0'}) # Accept only json temp_im_session.mount('https://', TLS1Adapter()) temp_im_verify_ssl = self._im_verify_ssl resp = temp_im_session.post( #resp = self._do_post( "{}/{}".format(temp_im_api_url,"types/Configuration/instances/actions/parseFromCSV"), auth=HTTPBasicAuth('admin', 'Password1!'), #headers = m.content_type, files = file_dict, verify = False, data = parameters ) #pprint (resp) def getInstallerUrl(self): pass ##=============================================== ## IM Integration Implmentation if __name__ == "__main__": pass #logging.basicConfig(format='%(asctime)s: %(levelname)s %(module)s:%(funcName)s | %(message)s', level=logging.WARNING)