Python fabric.api.get() Examples

def _setup_gradle_authentication(config):
    dse_source_build_artifactory_username = config.get('dse_source_build_artifactory_username')
    dse_source_build_artifactory_password = config.get('dse_source_build_artifactory_password')
    dse_source_build_artifactory_url = config.get('dse_source_build_artifactory_url')

    gradle_settings = """
allprojects {{
    repositories {{
        maven {{
            url = "\\"{url}\\""
            credentials {{
                username '{username}'
                password '{password}'
            }}
        }}
    }}
}}
    """.format(username=dse_source_build_artifactory_username, password=dse_source_build_artifactory_password,
               url=dse_source_build_artifactory_url)

    fab.local('rm -rf ~/.gradle')
    fab.local('mkdir -p ~/.gradle/init.d')
    fab.local('echo "{gradle_settings}" > ~/.gradle/init.d/nexus.gradle'.format(gradle_settings=gradle_settings)) 

def _configure_spark_env(config):
    # Place spark environment file on hosts:
    spark_env = config.get('spark_env', '')

    if isinstance(spark_env, list) or isinstance(spark_env, tuple):
        spark_env = "\n".join(spark_env)
    spark_env += "\n"

    spark_env_script = "spark-{name}.sh".format(name=uuid.uuid1())
    spark_env_file = StringIO(spark_env)
    fab.run('mkdir -p ~/fab/scripts')
    fab.put(spark_env_file, '~/fab/scripts/{spark_env_script}'.format(spark_env_script=spark_env_script))

    fab.puts('spark-env is: {}'.format(spark_env))
    if len(spark_env_script) > 0:
        spark_env_path = os.path.join(get_dse_path(), 'resources', 'spark', 'conf', 'spark-env.sh')
        fab.run('cat ~/fab/scripts/{spark_env_script} >> {spark_env_path}'.format(spark_env_script=spark_env_script,
                                                                                  spark_env_path=spark_env_path)) 

def setup_supervisor():
    # We use supervisord to keep Crestify running in the background
    # Recover from crashes, and to start automatically on bootup
    # Also, using more than 1 gunicorn worker resulted in socket not being released, so only 1 worker will be used
    sudo('apt-get -y install supervisor')
    sudo('mkdir /var/log/crestify/')
    sudo(
        'cd /home/crestify/crestify && ../crestifyenv/bin/honcho export -s /bin/sh -a crestify supervisord /etc/supervisor/conf.d')
    fd = StringIO()
    get('/etc/supervisor/conf.d/crestify.conf', fd)
    content = fd.getvalue().splitlines()
    for n, i in enumerate(content):
        if i.startswith("environment="):
            content[n] = i + ",PATH=/home/crestify/crestifyenv/bin:%(ENV_PATH)s"
        if i.startswith("user="):
            content[n] = "user=crestify"
        if i.startswith("stopsignal="):
            content[n] = "stopsignal=TERM"  # Both Gunicorn and Celery use SIGTERM for graceful shutdown
    content = StringIO("\n".join(content))
    put(content, "/etc/supervisor/conf.d/crestify.conf", use_sudo=True)
    sudo('supervisorctl reread')
    sudo('supervisorctl update') 

def experiment1collect():        
        # run("ls experiment1/*")
    with cd('/home/ubuntu/projects/rscoin/%s' % env.expname):
        get('issue-times.txt', '%s/%s-issue-times.txt' % (env.expname, env.host))

    with lcd(env.expname):
        local("cat %s-issue-times.txt >> issue-times.txt" % env.host)

    with cd('/home/ubuntu/projects/rscoin/%s' % env.expname):
        get('r1-times.txt', '%s/%s-r1-times.txt' % (env.expname, env.host))
    
    with lcd(env.expname):
        local("cat %s-r1-times.txt >> r1-times.txt" % env.host)

    with cd('/home/ubuntu/projects/rscoin/%s' % env.expname):
        get('r2-times.txt', '%s/%s-r2-times.txt' % (env.expname, env.host))

    with lcd(env.expname):
        local("cat %s-r2-times.txt >> r2-times.txt" % env.host)

        # local("python exp1plot.py experiment1") 

def install_java(packages=None):
    # Try to get the os distribution:
    dist = fab.run('lsb_release -is', quiet=True)
    if dist.return_code != 0:
        dist = fab.run('cat /etc/redhat-release', quiet=True)
    if dist.startswith('CentOS'):
        if not packages:
            packages = ['java-1.7.0-openjdk.x86_64',
                        'java-1.7.0-openjdk-devel.x86_64']
        cmd = 'yum -y install {package}'
    elif dist.startswith('Ubuntu'):
        if not packages:
            packages = ['openjdk-7-jdk']
        fab.run('apt-get update')
        cmd = 'apt-get -y install {package}'
    else:
        raise RuntimeError('Unknown distribution: %s' % dist)
    for package in packages:
        fab.run(cmd.format(package=package)) 

def is_enabled(revision_config=None):
    is_compatible = True
    is_enabled = common_module.config.get('flamegraph', False)
    if revision_config:
        if revision_config.get('product', 'cassandra') == 'dse':
            logger.info('Flamegraph is not compatible with DSE yet')
            is_compatible = False
        jvm = revision_config.get('java_home', '')
        try:
            jvm = os.path.basename(jvm)
            jvm = jvm[jvm.index('1'):]
            if v.LooseVersion(jvm) < v.LooseVersion('1.8.0_60'):
                logger.info('Flamegraph is not compatible with java <1.8.0_60')
                is_compatible = False
        except ValueError:
            pass
    return is_enabled and is_compatible 

def update_config(self, content, path):
        old_file = six.BytesIO()
        if files.exists(path, use_sudo=self.sudo):
            fab.get(remote_path=path, local_path=old_file, use_sudo=self.sudo)
        old_content = old_file.getvalue()
        need_update = content != old_content
        if need_update:
            fabricio.move_file(
                path_from=path,
                path_to=path + '.backup',
                sudo=self.sudo,
                ignore_errors=True,
            )
            fab.put(six.BytesIO(content), path, use_sudo=self.sudo, mode='0644')
            fabricio.log('{path} updated'.format(path=path))
        else:
            fabricio.log('{path} not changed'.format(path=path))
        return need_update 

def update(self, tag=None, registry=None, account=None, force=False):
        if not fab.env.parallel:
            fab.abort(
                'Master-slave configuration update requires parallel mode. '
                'Use Fabric\'s `--parallel` option to enable this mode '
                'for a current session.'
            )

        self.instances.put(None)

        try:
            recovery_config_updated = self.update_recovery_config(
                tag=tag,
                registry=registry,
                account=account,
            )

            container_updated = super(
                StreamingReplicatedPostgresqlContainer,
                self,
            ).update(force=force, tag=tag, registry=registry, account=account)

            if not container_updated and recovery_config_updated:
                self.reload()
            self.master_obtained.set()  # one who first comes here is master
            return container_updated or recovery_config_updated
        except Exception as exception:
            self.multiprocessing_data.exception = exception
            raise
        finally:
            try:
                self.master_lock.release()
            except ValueError:  # ignore "released too many times" error
                pass
            self.instances.get()
            self.instances.task_done()
            self.instances.join()  # wait until all instances will be updated

            # reset state at the end to prevent fail of the next Fabric command
            self.master_obtained.clear() 

def _checkout_dse_branch_and_build_tarball_from_source(branch):
    global dse_cache_local, dse_tarball, config

    java_home = config['java_home']
    oauth_token = config.get('dse_source_build_oauth_token')
    bdp_git = '~/fab/bdp.git'

    _setup_maven_authentication(config)
    _setup_gradle_authentication(config)

    fab.local('rm -rf {bdp_git}'.format(bdp_git=bdp_git))
    fab.local('mkdir -p {bdp_git}'.format(bdp_git=bdp_git))
    fab.local('git clone -b {branch} --single-branch https://{oauth_token}@github.com/riptano/bdp.git {bdp_git}'.format(
        branch=branch, oauth_token=oauth_token, bdp_git=bdp_git))

    # build the tarball from source
    env.ok_ret_codes = [0, 1]
    gradle_file_exists = fab.local('[ -e  {bdp_git}/build.gradle ]'.format(bdp_git=bdp_git))
    env.ok_ret_codes = [0]

    if gradle_file_exists.return_code == 0:
        # run the gradle build
        fab.local('export TERM=dumb; cd {bdp_git}; ./gradlew distTar -PbuildType={branch}'.format(bdp_git=bdp_git, branch=branch))
    else:
        # run the ant build
        fab.local(
            'cd {bdp_git}; JAVA_HOME={java_home} ANT_HOME=$HOME/fab/ant/ $HOME/fab/ant/bin/ant -Dversion={branch} -Dcompile.native=true release'.format(
                branch=branch, java_home=java_home, bdp_git=bdp_git))

    # we need to expand the tarball name because the name will look as following: dse-{version}-{branch}-bin.tar.gz
    # example: dse-4.8.3-4.8-dev-bin.tar.gz
    path_name = fab.local('readlink -e {bdp_git}/build/dse-*{branch}-bin.tar.gz'.format(bdp_git=bdp_git, branch=branch),
                          capture=True)
    dse_tarball = os.path.basename(path_name)

    logger.info('Created tarball from source: {tarball}'.format(tarball=dse_tarball))
    fab.local('cp {bdp_git}/build/{tarball} {dse_cache}'.format(bdp_git=bdp_git, dse_cache=dse_cache_local, tarball=dse_tarball))

    # remove the maven & gradle settings after the tarball got created
    fab.local('rm -rf ~/.m2/settings.xml')
    fab.local('rm -rf ~/.gradle') 

def yourkit_is_enabled(revision_config=None):
    is_enabled = common_module.config.get('yourkit_profiler', False)
    if is_enabled and revision_config:
        if not revision_config.get('yourkit_profiler', False):
            is_enabled = False
        elif revision_config.get('product', 'cassandra') == 'dse':
            logger.info('Yourkit profiling is not compatible with DSE yet')
            is_enabled = False

    return is_enabled 

def yourkit_get_config():
    agentpath = common_module.config.get('yourkit_agentpath', None)
    directory = common_module.config.get('yourkit_directory', None)
    options = common_module.config.get('yourkit_options', YOURKIT_DEFAULT_OPTIONS)
    if not (agentpath and directory):
        raise ValueError('Yourkit profiler requires yourkit_agentpath and yourkit_directory in the configuration.')

    return {'agentpath': agentpath, 'directory': directory, 'options': options} 

def copy_yourkit(local_directory, rev_num):
    logger.info("Copying Yourkit data")
    config = yourkit_get_config()
    cfg = common_module.config['hosts'][fab.env.host]
    host_log_dir = os.path.join(local_directory, cfg['hostname'])
    os.makedirs(host_log_dir)
    fab.run("cd {} && tar -cvz --exclude=logs -f yourkit_revision_{}.tar.gz *".format(config['directory'], rev_num))
    fab.get(os.path.join(config['directory'], 'yourkit_revision_{}.tar.gz'.format(rev_num)), host_log_dir) 

def glob(path, remote=True):
    ''' Glob a directory path to get the list of files. '''
    with hide('everything'):
        result = runner.run('ls -1 {}'.format(path), remote=remote)
        return strip_ansi(result).split() 

def read_remote_file(path):
    ''' Read remote file contents. '''
    fd = StringIO()
    get(path, fd)

    return fd.getvalue() 

def get_director_log(region, stack_name):
    """DEBUG: get the Director application log from the launcher instance"""
    ec2_conn = director.create_ec2_connection(region)
    hosts = [director.get_launcher_instance(ec2_conn, stack_name).ip_address]
    execute(
        get, hosts=hosts, local_path='application.log',
        remote_path='/home/ec2-user/.cloudera-director/logs/application.log') 

def start(config):
    _configure_spark_env(config)
    dse_path = get_dse_path()
    dse_node_type = config.get('dse_node_type', 'cassandra')
    fab.puts("Starting DSE - Node Type: {node_type}".format(node_type=dse_node_type))
    dse_home = 'DSE_HOME={dse_path}'.format(dse_path=dse_path)
    cmd = 'JAVA_HOME={java_home} {dse_home} nohup {dse_path}/bin/dse cassandra {node_type}'.format(
        java_home=config['java_home'], dse_home=dse_home, dse_path=dse_path,
        node_type=DSE_NODE_TYPE_TO_STARTUP_PARAM.get(dse_node_type, ''))
    fab.run(cmd) 

def getConfig(section, key):
    config = ConfigParser.ConfigParser()
    path = os.path.split(os.path.realpath(__file__))[0] + '/'+config_file
    config.read(path)
    return config.get(section, key)

# Log path 

def getConfig(section, key):
    config = ConfigParser.ConfigParser()
    path = os.path.split(os.path.realpath(__file__))[0] + '/'+config_file
    config.read(path)
    return config.get(section, key)

# Log path 

def getConfig(section, key):
    config = ConfigParser.ConfigParser()
    path = os.path.split(os.path.realpath(__file__))[0] + '/'+config_file
    config.read(path)
    return config.get(section, key)

# Remote server hosts. 

def auto_gen():

    print green('Auto generate testing reports.')
    
    run('python '+script_dir+'/get_git_version.py > '+script_dir+'/new.log')
    
    #file = open(script_dir+'/old.log', 'r')
    #old=file.read()
    #file.close()
    #newfile = open(script_dir+'/new.log', 'r')
    #new=newfile.read()
    #newfile.close()

    fd = StringIO()
    get(script_dir+'/old.log', fd)
    old=fd.getvalue()

    fd = StringIO()
    get(script_dir+'/new.log', fd)
    new=fd.getvalue()

    if old == new:
        print red('Nothing changed, it won\'t generate testing reports.')
    else:
        run('ant -buildfile '+script_dir+'/build.xml gen-testing-report')
        print green('Auto generate testing reports finished!')

# SCP generated testing reports. 

def getConfig(section, key):
    config = ConfigParser.ConfigParser()
    path = os.path.split(os.path.realpath(__file__))[0] + '/'+config_file
    config.read(path)
    return config.get(section, key)

# Log path 

def getConfig(section, key):
    config = ConfigParser.ConfigParser()
    path = os.path.split(os.path.realpath(__file__))[0] + '/'+config_file
    config.read(path)
    return config.get(section, key)

# Log path 

def deploy_prepare():
    print green('Deploy prepared. Run as root.')
    
    # Install jdk 1.8.25.
    print red('This program require jdk 1.8.25. Make sure jdk and tomcat work out before all of your operations.')

    # Install maven.
    print green('Insall maven.')
    run("wget http://apache.fayea.com/apache-mirror/maven/maven-3/3.2.3/binaries/apache-maven-3.2.3-bin.zip")
    run("unzip -q apache-maven-3.2.3-bin.zip")
    run("mv apache-maven-3.2.3 /usr/local/maven")
    run("echo 'export M2_HOME=/usr/local/maven' >> /etc/profile")
    run("echo 'export PATH=$PATH:$M2_HOME/bin' >> /etc/profile")
    run("source /etc/profile")
    run("rm -rf apache-maven-3.2.3-bin.zip apache-maven-3.2.3")
    run("mvn -version")
    
    log_path='~/logs'
    
    run('mkdir -p '+log_path+' 2>/dev/null >/dev/null')

    # Clear the install_requirement.log
    run('echo "" > '+log_path+'/install_requirement.log')

    # Install Python and fabric on the remote server.
    run("apt-get install dos2unix python python-pip python-dev subversion subversion-tools -y > "+log_path+"/install_requirement.log")
    run("pip install fabric >> "+log_path+"/install_requirement.log")

    print green('Deploy prepared finished.') 

def copy_flamegraph(local_directory, rev_num):
    logger.info("Copying Flamegraph data")
    cfg = common_module.config['hosts'][fab.env.host]
    host_log_dir = os.path.join(local_directory, cfg['hostname'])
    flamegraph_directory = get_flamegraph_directory()
    os.makedirs(host_log_dir)
    with fab.settings(warn_only=True):
        fab.get(os.path.join(flamegraph_directory, 'flamegraph_revision_{}.svg'.format(rev_num)), host_log_dir)
        fab.get(os.path.join(flamegraph_directory, 'perf_revision_{}.data'.format(rev_num)), host_log_dir) 

def install_rabbitmq():
    # RabbitMQ is our message queue
    sudo('echo "deb http://www.rabbitmq.com/debian/ testing main" >> /etc/apt/sources.list')
    sudo('apt-get -y install wget sudo')
    sudo('apt-get -y install ca-certificates')
    sudo('wget --quiet -O - https://www.rabbitmq.com/rabbitmq-release-signing-key.asc | sudo apt-key add -')
    sudo('apt-get update --fix-missing')
    sudo('apt-get -y install rabbitmq-server') 

def install_postgres():
    sudo('echo "deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main" >> /etc/apt/sources.list.d/pgdg.list')
    sudo('wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \
  sudo apt-key add -')
    sudo('apt-get update')
    sudo('apt-get -y install postgresql-9.4 postgresql-server-dev-9.4 postgresql-contrib-9.4') 

def install_redis():
    # Used for temporarily saving incoming tab saves
    sudo('apt-get -y install redis-server') 

def install_phantomjs():
    # The PhantomJS headless browser
    sudo('apt-get -y install fontconfig')
    run('wget https://bitbucket.org/ariya/phantomjs/downloads/phantomjs-2.1.1-linux-x86_64.tar.bz2')
    run('tar xjf phantomjs-2.1.1-linux-x86_64.tar.bz2')
    sudo('mv phantomjs-2.1.1-linux-x86_64/bin/phantomjs /usr/local/bin/phantomjs') 

def config_environment():
    sudo('apt-get -y install git screen')
    sudo('adduser crestify --disabled-password --gecos GECOS')
    sudo('locale-gen en_US.UTF-8')
    with settings(sudo_user='crestify', shell='/bin/bash -c'):
        with cd('/home/crestify'):
            sudo('git clone https://github.com/crestify/crestify.git crestify')
            sudo('virtualenv crestifyenv')
            with prefix('source crestifyenv/bin/activate'):
                sudo('pip install -r crestify/requirements.txt') 

def passcache():
    # Delete old folder and make a new one
    sudo( 'rm -rf /home/ubuntu/projects/rscoin')
    sudo("apt-get install -y sysbench")

    with cd('/home/ubuntu/projects'):
        sudo('pip install petlib --upgrade')
        run("git clone https://github.com/gdanezis/rscoin.git")