Python fabric.api.env.host() Examples

def get_ext_ip_of_node(client):
    node_properties_file = os.path.join(constants.REMOTE_CONF_DIR,
                                        'node.properties')
    with settings(hide('stdout')):
        node_uuid = sudo('sed -n s/^node.id=//p ' + node_properties_file)
    external_ip_row = execute_external_ip_sql(client, node_uuid)
    external_ip = ''
    if len(external_ip_row) > 1:
        warn_more_than_one_ip = 'More than one external ip found for ' + env.host + \
                                '. There could be multiple nodes associated with the same node.id'
        _LOGGER.debug(warn_more_than_one_ip)
        warn(warn_more_than_one_ip)
        return external_ip
    for row in external_ip_row:
        if row:
            external_ip = row[0]
    if not external_ip:
        _LOGGER.debug('Cannot get external IP for ' + env.host)
        external_ip = 'Unknown'
    return external_ip 

def is_port_in_use(host):
    _LOGGER.info("Checking if port used by Prestoserver is already in use..")
    try:
        portnum = lookup_port(host)
    except Exception:
        _LOGGER.info("Cannot find port from config.properties. "
                     "Skipping check for port already being used")
        return 0
    with settings(hide('warnings', 'stdout'), warn_only=True):
        output = run('netstat -ln |grep -E "\<%s\>" |grep LISTEN' % str(portnum))
    if output:
        _LOGGER.info("Presto server port already in use. Skipping "
                     "server start...")
        error('Server failed to start on %s. Port %s already in use'
              % (env.host, str(portnum)))
    return output 

def deploy(clients):
    execute(deployMulti, clients)
#
# @roles("providers")
# def checkHost():
#     print env.host
#
#
# @runs_once
# def takeNamesDb():
#     database = "example.db"
#     db = sqlite3.connect(database)
#     c = db.cursor()
#     c.execute("SELECT * FROM Providers")
#     providers = c.fetchall()
#     for p in providers:
#         print p
#
# 

def deployMultiClient(num):
    local('rm -f testMap.csv')
    for i in range(int(num)):
        dirc = 'client%s' % i
        with cd(dirc):
            with cd('loopix'):
                run("git pull")
                run("git checkout %s" % BRANCH)
            with cd('loopix/loopix'):
                N = hexlify(os.urandom(8))
                providers = getProvidersNames()
                prvName = random.choice(providers)
                port = int(9999 - i)
                print "CLIENT: Client%s" % N
                run("python setup_client.py %d %s Client%s %s" % (port, str(env.host), N, prvName))
                get('publicClient.bin', 'publicClient-%d-%s.bin'%(port, env.host))
                with open('testMap.csv', 'a') as outfile:
                    csvW = csv.writer(outfile)
                    csvW.writerow(['Client%s'%N, dirc]) 

def experiment1collect():        
        # run("ls experiment1/*")
    with cd('/home/ubuntu/projects/rscoin/%s' % env.expname):
        get('issue-times.txt', '%s/%s-issue-times.txt' % (env.expname, env.host))

    with lcd(env.expname):
        local("cat %s-issue-times.txt >> issue-times.txt" % env.host)

    with cd('/home/ubuntu/projects/rscoin/%s' % env.expname):
        get('r1-times.txt', '%s/%s-r1-times.txt' % (env.expname, env.host))
    
    with lcd(env.expname):
        local("cat %s-r1-times.txt >> r1-times.txt" % env.host)

    with cd('/home/ubuntu/projects/rscoin/%s' % env.expname):
        get('r2-times.txt', '%s/%s-r2-times.txt' % (env.expname, env.host))

    with lcd(env.expname):
        local("cat %s-r2-times.txt >> r2-times.txt" % env.host)

        # local("python exp1plot.py experiment1") 

def keys():
    if "rsdir" not in env:
        secret = file("secret.key").read()
        public = rscoin.Key(secret, public=False)
        pid = b64encode(public.id())

        env["rsdir"] = {"special": pid, "directory": []}

    [_, host] = env.host_string.split("@")
    with cd('/home/ubuntu/projects/rscoin'):
        run('touch secret.key')
        run('rm secret.key')
        result = run('python derivekey.py --store')
        [_, key] = result.strip().split()
        
        kid = b64encode(rscoin.Key(b64decode(key)).id())
        env["rsdir"]["directory"] += [ [kid, host, 8080] ]
    

    from json import dumps
    file("directory.conf", "w").write(dumps(env["rsdir"])) 

def get_host_list():
    return [host for host in env.hosts if host not in env.exclude_hosts] 

def test_by_rolename_worker(self):
        callback = Mock()
        callback.side_effect = self.assert_is_worker(self.TEST_ROLEDEFS)
        env.roledefs = self.TEST_ROLEDEFS

        env.host = 'coordinator'
        fabricapi.by_rolename(env.host, 'worker', callback)
        self.assertFalse(callback.called)

        env.host = 'worker0'
        fabricapi.by_rolename(env.host, 'worker', callback)
        self.assertTrue(callback.called) 

def by_rolename(host, rolename, f, *args, **kwargs):
    if rolename is None:
        f(*args, **kwargs)
    else:
        if rolename not in env.roledefs.keys():
            abort("Invalid role name %s. Valid rolenames are %s" %
                  (rolename, env.roledefs.keys()))
        if host in env.roledefs[rolename]:
            return f(*args, **kwargs) 

def by_role_coordinator(host, f, *args, **kwargs):
    if host in get_coordinator_role():
        return f(*args, **kwargs) 

def by_role_worker(host, f, *args, **kwargs):
    if host in get_worker_role() and host not in get_coordinator_role():
        return f(*args, **kwargs) 

def test_uninstall_is_called(self, mock_package_rpm_uninstall, mock_package_is_rpm_installed, mock_version_check):
        env.host = "any_host"
        mock_package_is_rpm_installed.side_effect = [False, True]

        server.uninstall()

        mock_version_check.assert_called_with()
        mock_package_is_rpm_installed.assert_called_with('presto-server')
        mock_package_rpm_uninstall.assert_called_with('presto-server')
        self.assertTrue(mock_package_is_rpm_installed.call_count == 2)
        self.assertTrue(mock_package_rpm_uninstall.call_count == 1) 

def test_server_start_fail(self, mock_port_in_use,
                               mock_version_check, mock_warn,
                               mock_query_for_status, mock_sudo, mock_run, mock_config,
                               mock_presto_config):
        mock_query_for_status.return_value = False
        env.host = "failed_node1"
        mock_version_check.return_value = ''
        mock_port_in_use.return_value = 0
        mock_config.return_value = None
        server.start()
        mock_sudo.assert_called_with('set -m; ' + INIT_SCRIPTS + ' start')
        mock_version_check.assert_called_with()
        mock_warn.assert_called_with(self.SERVER_FAIL_MSG) 

def test_configure_presto(self, deploy_node_mock, deploy_mock):
        env.host = 'localhost'
        conf = {"node.properties": {"key": "value"}, "jvm.config": ["list"]}
        remote_dir = "/my/remote/dir"
        deploy.configure_presto(conf, remote_dir)
        deploy_mock.assert_called_with({"jvm.config": "list"}, remote_dir) 

def test_no_warn_if_port_lookup_fail(self, mock_warn, mock_port):
        e = ConfigurationError()
        mock_port.side_effect = e
        env.host = 'any_host'
        self.assertFalse(server.is_port_in_use(env.host))
        self.assertEqual(False, mock_warn.called) 

def test_worker_is_coordinator(self, env_mock, coord_mock, configure_mock):
        env_mock.host = "my.host"
        coord_mock.return_value = ["my.host"]
        deploy.workers()
        assert not configure_mock.called 

def test_worker_not_coordinator(self,  configure_mock, get_conf_mock):
        env.host = "my.host1"
        env.roledefs["worker"] = ["my.host1"]
        env.roledefs["coordinator"] = ["my.host2"]
        deploy.workers()
        assert configure_mock.called 

def test_coordinator(self, coord_mock, configure_mock):
        env.roledefs['coordinator'] = ['master']
        env.host = 'master'
        deploy.coordinator()
        assert configure_mock.called 

def add_jar(local_path, plugin_name, plugin_dir=REMOTE_PLUGIN_DIR):
    """
    Deploy jar for the specified plugin to the plugin directory.

    Parameters:
        local_path - Local path to the jar to be deployed
        plugin_name - Name of the plugin subdirectory to deploy jars to
        plugin_dir - (Optional) The plugin directory.  If no directory is
                     given, '/usr/lib/presto/lib/plugin' is used by default.
    """
    _LOGGER.info('deploying jars on %s' % env.host)
    write(local_path, os.path.join(plugin_dir, plugin_name)) 

def copy(local_file, remote_dir=REMOTE_COPY_DIR):
    """
    Copy a file to all nodes in the cluster.

    Parameters:
        local_file - The path to the file
        remote_dir - Where to put the file on the cluster.  Default is /tmp.
    """
    _LOGGER.info('copying file to %s' % env.host)
    write(local_file, remote_dir) 

def remove(name):
    """
    Remove a catalog from the cluster.

    Parameters:
        name - Name of the catalog to be removed
    """
    _LOGGER.info('[' + env.host + '] Removing catalog: ' + name)
    ret = remove_file(os.path.join(constants.REMOTE_CATALOG_DIR,
                                   name + '.properties'))
    if ret.succeeded:
        if COULD_NOT_REMOVE in ret:
            fabric.utils.error(ret)
        else:
            print('[%s] Catalog removed. Restart the server for the change '
                  'to take effect' % env.host)
    else:
        fabric.utils.error('Failed to remove catalog ' + name + '.\n\t' +
                           ret)

    local_path = os.path.join(get_catalog_directory(), name + '.properties')
    try:
        os.remove(local_path)
    except OSError as e:
        if e.errno == errno.ENOENT:
            pass
        else:
            raise 

def get_catalog_configs(dest_path):
    remote_catalog_dir = lookup_catalog_directory(env.host)
    _LOGGER.debug('catalogs to be archived on host ' + env.host + ': ' + remote_catalog_dir)
    get_files(remote_catalog_dir, dest_path) 

def system_info():
    """
    Gather system information like nodes in the system, presto
    version, presto-admin version, os version etc.
    """
    if env.host not in fabricapi.get_coordinator_role():
        return
    err_msg = 'Unable to access node information. ' \
              'Please check that server is up with command: server status'
    req = get_request(request_url(NODES_REQUEST_EXT), err_msg)

    downloaded_sys_info_loc = os.path.join(TMP_PRESTO_DEBUG, "sysinfo")
    try:
        os.makedirs(downloaded_sys_info_loc)
    except OSError:
        if not os.path.isdir(downloaded_sys_info_loc):
            raise

    node_info_file_name = os.path.join(downloaded_sys_info_loc, 'node_info.json')
    with open(node_info_file_name, 'w') as out_file:
        out_file.write(json.dumps(req.json(), indent=4))

    _LOGGER.debug('Gathered node information in file: ' + node_info_file_name)

    catalog_file_name = os.path.join(downloaded_sys_info_loc, 'catalog_info.txt')
    client = PrestoClient(env.host, env.user)
    catalog_info = get_catalog_info_from(client)

    with open(catalog_file_name, 'w') as out_file:
        out_file.write(catalog_info + '\n')

    _LOGGER.debug('Gathered catalog information in file: ' + catalog_file_name)

    execute(get_catalog_configs, downloaded_sys_info_loc, roles=env.roles)
    execute(get_system_info, downloaded_sys_info_loc, roles=env.roles)

    make_tarfile(OUTPUT_FILENAME_FOR_SYS_INFO, downloaded_sys_info_loc)
    print 'System info archive created: ' + OUTPUT_FILENAME_FOR_SYS_INFO 

def query_info(query_id):
    """
    Gather information about the query identified by the given
    query_id and store that in a JSON file.

    Parameters:
        query_id - id of the query for which info has to be gathered
    """

    if env.host not in fabricapi.get_coordinator_role():
        return

    err_msg = 'Unable to retrieve information. Please check that the ' \
              'query_id is correct, or check that server is up with ' \
              'command: server status'
    req = get_request(request_url(QUERY_REQUEST_EXT + query_id), err_msg)
    query_info_file_name = os.path.join(TMP_PRESTO_DEBUG, 'query_info_' + query_id + '.json')

    try:
        os.makedirs(TMP_PRESTO_DEBUG)
    except OSError:
        if not os.path.isdir(TMP_PRESTO_DEBUG):
            raise

    with open(query_info_file_name, 'w') as out_file:
        out_file.write(json.dumps(req.json(), indent=4))

    print('Gathered query information in file: ' + query_info_file_name) 

def request_url(url_extension):
    host = env.host
    port = lookup_port(host)
    return 'http://%(host)s:%(port)i/%(url_ext)s' % {'host': host,
                                                     'port': port,
                                                     'url_ext': url_extension} 

def get_files(remote_path, local_path):
    path_with_host_name = os.path.join(local_path, env.host)

    try:
        os.makedirs(path_with_host_name)
    except OSError:
        if not os.path.isdir(path_with_host_name):
            raise

    _LOGGER.debug('local path used ' + path_with_host_name)

    try:
        get(remote_path, path_with_host_name, use_sudo=True)
    except SystemExit:
        warn('remote path ' + remote_path + ' not found on ' + env.host) 

def print_status_header(external_ip, server_status, host):
    print('Server Status:')
    print('\t%s(IP: %s, Roles: %s): %s' % (host, external_ip,
                                           ', '.join(get_roles_for(host)),
                                           is_server_up(server_status))) 

def get_roles_for(host):
    roles = []
    for role in ['coordinator', 'worker']:
        if host in env.roledefs[role]:
            roles.append(role)
    return roles 

def sync_offline_from_control():
    for host in _hosts_in_roles(ROLES_ALL_SRC, exclude_roles=ROLES_DEPLOY):
        run("rsync -rvz --exclude 'commcare-hq/*' {} {}".format(
            env.offline_code_dir,
            '{}@{}:{}'.format(env.user, host, env.offline_releases)
        )) 

def test_server_start_bad_presto_version(self, mock_port_in_use,
                                             mock_version_check, mock_sudo):
        env.host = "good_node"
        mock_version_check.return_value = 'Presto not installed'
        server.start()
        mock_version_check.assert_called_with()
        self.assertEqual(False, mock_sudo.called)