Python fabric.api.sudo() Examples

def _clone_code_from_local_path(from_path, to_path, run_as_sudo=True):
    cmd_fn = sudo if run_as_sudo else run
    git_local_submodule_config = [
        'git config {} {}'.format(submodule_config.key, submodule_config.value)
        for submodule_config in _get_local_submodule_urls(from_path)
    ]
    git_remote_submodule_config = [
        'git config {} {}'.format(submodule_config.key, submodule_config.value)
        for submodule_config in _get_remote_submodule_urls(from_path)
    ]

    with cd(from_path):
        cmd_fn('git clone {}/.git {}'.format(
            from_path,
            to_path
        ))

    with cd(to_path):
        cmd_fn('git config receive.denyCurrentBranch updateInstead')
        cmd_fn(' && '.join(git_local_submodule_config))
        cmd_fn('git submodule update --init --recursive')
        cmd_fn(' && '.join(git_remote_submodule_config)) 

def install_required_packages():
    sudo("apt-get update -y")
    sudo("apt-get install -y git")
    sudo("apt-get install -y wget")
    sudo("apt-get install -y python2.7-dev")
    sudo("apt-get install -y python-pip")
    sudo("apt-get install -y ipython")
    sudo("apt-get install -y libffi-dev")
    sudo("apt-get install -y libssl-dev")
    sudo("apt-get install -y libxml2-dev")
    sudo("apt-get install -y libxslt1-dev")
    sudo("apt-get install -y libpng12-dev")
    sudo("apt-get install -y libfreetype6-dev")
    sudo("apt-get install -y python-tk")
    sudo("apt-get install -y libsasl2-dev")
    sudo("apt-get install -y python-pip")
    sudo("apt-get install -y graphviz")
    sudo("pip install --upgrade pip") 

def restart_services(self):
        services = self.config.get('services', [])
        # TODO: validate services
        sudo("rm -rf /etc/supervisor/conf.d && ln -sfT /opt/rorolite/project/.rorolite/supervisor /etc/supervisor/conf.d")
        sudo("supervisorctl update")

        if not services:
            print("Deploy successful. No services found.")
            return

        for s in services:
            sudo("supervisorctl restart {}".format(s['name']))

        host = self.config['host']
        print("Services are live at:")
        for s in services:
            print("  {} -- http://{}:{}/".format(s['name'], host, s['port'])) 

def setup_supervisor():
    # We use supervisord to keep Crestify running in the background
    # Recover from crashes, and to start automatically on bootup
    # Also, using more than 1 gunicorn worker resulted in socket not being released, so only 1 worker will be used
    sudo('apt-get -y install supervisor')
    sudo('mkdir /var/log/crestify/')
    sudo(
        'cd /home/crestify/crestify && ../crestifyenv/bin/honcho export -s /bin/sh -a crestify supervisord /etc/supervisor/conf.d')
    fd = StringIO()
    get('/etc/supervisor/conf.d/crestify.conf', fd)
    content = fd.getvalue().splitlines()
    for n, i in enumerate(content):
        if i.startswith("environment="):
            content[n] = i + ",PATH=/home/crestify/crestifyenv/bin:%(ENV_PATH)s"
        if i.startswith("user="):
            content[n] = "user=crestify"
        if i.startswith("stopsignal="):
            content[n] = "stopsignal=TERM"  # Both Gunicorn and Celery use SIGTERM for graceful shutdown
    content = StringIO("\n".join(content))
    put(content, "/etc/supervisor/conf.d/crestify.conf", use_sudo=True)
    sudo('supervisorctl reread')
    sudo('supervisorctl update') 

def deploy(self):
        self.config = self.read_config(self.directory)
        if "host" not in self.config:
            raise Exception("Missing required field in rorolite.yml: host")

        self.version = self.find_current_version() + 1
        self.deploy_root = "/opt/rorolite/deploys/{}".format(self.version)
        print("Deploying project version {}...".format(self.version))

        remote.sudo("mkdir -p " + self.deploy_root)

        self.push_directory()
        self.setup_virtualenv()

        remote.sudo("ln -sfT {} /opt/rorolite/project".format(self.deploy_root))

        self.restart_services() 

def ensure_checkpoints_safe():
    extras = '--print-only' if env.ignore_kafka_checkpoint_warning else ''
    with cd(env.code_root):
        try:
            sudo('{env.py3_virtualenv_root}/bin/python manage.py validate_kafka_pillow_checkpoints {extras}'.format(
                env=env, extras=extras
            ))
        except Exception as e:
            if not env.ignore_kafka_checkpoint_warning:
                message = (
                    "Deploy failed, likely because kafka checkpoints weren't available.\n"
                    "Scroll up for more detailed information.\n"
                    "You can rerun with --set ignore_kafka_checkpoint_warning=true to prevent this error from blocking the deploy."
                ).format(e)
                raise Exception(message)
            else:
                # if we were forcing and still got an error this is likely a bug so we should raise it
                raise 

def run_command(self, hosts, parallel_pool_size=1):
        from fabric.api import execute, sudo, env, parallel
        if env.ssh_config_path and os.path.isfile(os.path.expanduser(env.ssh_config_path)):
            env.use_ssh_config = True
        env.forward_agent = True
        # pass `-E` to sudo to preserve environment for ssh agent forwarding
        env.sudo_prefix = "sudo -SE -p '%(sudo_prompt)s' "
        env.user = self.user_name
        env.password = self.password
        env.hosts = hosts
        env.warn_only = True

        def _task():
            result = sudo(self.command, user=self.user_as)
            return result

        task = _task
        if parallel_pool_size > 1:
            task = parallel(pool_size=parallel_pool_size)(_task)

        res = execute(task)
        return res 

def rollback_formplayer():
    build_dir = os.path.join(env.code_current, FORMPLAYER_BUILD_DIR)

    builds = _get_old_formplayer_builds(build_dir)
    if not builds:
        utils.abort('No formplayer builds to rollback to.')

    rollback_build = builds[0]
    if not console.confirm('Confirm rollback to "{}"'.format(rollback_build), default=False):
        utils.abort('Action aborted.')

    with cd(build_dir):
        sudo('ln -sfn {build_dir}/{rollback} {build_dir}/current'.format(
            build_dir=build_dir,
            rollback=rollback_build
        )) 

def update_code(full_cluster=True):
    roles_to_use = _get_roles(full_cluster)

    @roles(roles_to_use)
    @parallel
    def update(git_tag, use_current_release=False):
        # If not updating current release,  we are making a new release and thus have to do cloning
        # we should only ever not make a new release when doing a hotfix deploy
        if not use_current_release:
            _update_code_from_previous_release()
        with cd(env.code_root if not use_current_release else env.code_current):
            sudo('git remote prune origin')
            # this can get into a state where running it once fails
            # but primes it to succeed the next time it runs
            sudo('git fetch origin --tags -q || git fetch origin --tags -q')
            sudo('git checkout {}'.format(git_tag))
            sudo('git reset --hard {}'.format(git_tag))
            sudo('git submodule sync')
            sudo('git submodule update --init --recursive -q')
            # remove all untracked files, including submodules
            sudo("git clean -ffd")
            # remove all .pyc files in the project
            sudo("find . -name '*.pyc' -delete")

    return update 

def record_successful_deploy():
    start_time = datetime.strptime(env.deploy_metadata.timestamp, DATE_FMT)
    delta = datetime.utcnow() - start_time
    with cd(env.code_current):
        env.deploy_metadata.tag_commit()
        sudo((
            '%(virtualenv_current)s/bin/python manage.py '
            'record_deploy_success --user "%(user)s" --environment '
            '"%(environment)s" --url %(url)s --minutes %(minutes)s --mail_admins'
        ) % {
            'virtualenv_current': env.py3_virtualenv_current,
            'user': env.user,
            'environment': env.deploy_env,
            'url': env.deploy_metadata.diff_url,
            'minutes': str(int(delta.total_seconds() // 60))
        }) 

def update():
    """Update the svn repo, then reinstall LNT."""
    with cd(LNT_PATH):
        with cd(LNT_PATH + "/docs/"):
            sudo('rm -rf _build')
        with cd(LNT_PATH + "/lnt/server/ui/static"):
            sudo('rm -rf docs')
            sudo('git checkout docs')

        sudo("git pull --rebase")
        run("git log -1 --pretty=%B")
        with cd(LNT_PATH + "/docs/"):
            sudo(in_venv("make"))
        sudo(IN_VENV + "python setup.py install --server")

    put(here + "/blacklist", "/tmp/blacklist")
    sudo("mv /tmp/blacklist /srv/lnt/install/blacklist")
    put(here + "/kill_zombies.py", "/tmp/kill_zombies.py")
    sudo("mv /tmp/kill_zombies.py /etc/cron.hourly/kill_zombies")
    sudo("chmod +x /etc/cron.hourly/kill_zombies")
    rotate_log()
    service_restart() 

def restart_dev():
    npm_install()
    switch_to_dev()
    sudo('supervisorctl status') 

def create_kafka_topics():
    """Create kafka topics if needed.  This is pretty fast."""
    with cd(env.code_root):
        sudo('%(py3_virtualenv_root)s/bin/python manage.py create_kafka_topics' % env) 

def set_in_progress_flag(use_current_release=False):
    venv = env.py3_virtualenv_root if not use_current_release else env.py3_virtualenv_current
    with cd(env.code_root if not use_current_release else env.code_current):
        sudo('{}/bin/python manage.py deploy_in_progress'.format(venv)) 

def copy_node_modules():
    if files.exists('{}/node_modules'.format(env.code_current)):
        sudo('cp -r {}/node_modules {}/node_modules'.format(env.code_current, env.code_root))
    else:
        sudo('mkdir {}/node_modules'.format(env.code_root)) 

def copy_components():
    if files.exists('{}/bower_components'.format(env.code_current)):
        sudo('cp -r {}/bower_components {}/bower_components'.format(env.code_current, env.code_root))
    else:
        sudo('mkdir {}/bower_components'.format(env.code_root)) 

def directory_permissions(build_type='prod'):
    media_root = get_django_setting(build_type, 'MEDIA_ROOT')
    sudo('chown {user}:www-data {media} -R'.format(user=env.deploy_user,
                                                   media=media_root))
    sudo('chmod 777 {media} -R'.format(media=media_root)) 

def copy_formplayer_properties():
    if formplayer_is_running_from_old_release_location():
        sudo(
            'cp -r {} {}'.format(
                os.path.join(env.code_current, FORMPLAYER_BUILD_DIR),
                os.path.join(env.code_root, FORMPLAYER_BUILD_DIR)
            )) 

def copy_localsettings(full_cluster=True):
    roles_to_use = _get_roles(full_cluster)

    @parallel
    @roles(roles_to_use)
    def copy():
        sudo('cp {}/localsettings.py {}/localsettings.py'.format(env.code_current, env.code_root))

    return copy 

def restart_celery():
    sudo('supervisorctl restart celery_worker')
    sudo('supervisorctl restart celerybeat') 

def mark_last_release_unsuccessful():
    # Removes last line from RELEASE_RECORD file
    with cd(env.root):
        sudo("sed -i '$d' {}".format(RELEASE_RECORD)) 

def update_current(release=None):
    """
    Updates the current release to the one specified or to the code_root
    """
    if ((not release and not files.exists(env.code_root)) or
            (release and not files.exists(release))):
        utils.abort('About to update current to non-existant release')

    sudo('ln -nfs {} {}'.format(release or env.code_root, env.code_current)) 

def delete_all_cartocss():
    sudo("rm -r /srv/calthorpe_media/cartocss/*")
    sudo("rm -r /srv/calthorpe_static/cartocss/*") 

def build_sproutcore(build_type='prod', minify=False):
    update_sproutcore_build_number(build_type=build_type)

    # build sproutcore
    with cd('{root}/sproutcore'.format(root=get_django_setting(build_type, 'ROOT_PATH'))):
        # Build main in the build dir
        build_command = 'sproutcore build fp --buildroot=builds'
        if not minify:
            build_command += ' --dont_minify'
        sudo(build_command, user=env.deploy_user)

        # Change ownership on output
        sudo('chown -R {0}.www-data ./builds'.format(env.deploy_user))

        # ln to the builds dir from Django's static dir
        sudo('ln -f -s {root}/sproutcore/builds/static/* {root}/footprint/main/static'.format(
            root=get_django_setting(build_type, 'ROOT_PATH')), user=env.deploy_user)

        # symlink to the sproutcore build directory "build-number"
        sudo('ln -f -s {root}/sproutcore/builds/static/fp/en/$(sproutcore build-number fp)/index.html '
             '{root}/footprint/main/templates/footprint/index.html'.format(
                 root=get_django_setting(build_type, 'ROOT_PATH'),
                 user=env.deploy_user))

    # do a collect static to grab all static files and link them to the right directory
    manage_py('collectstatic -l --noinput', build_type) 

def kill_stale_celery_workers(delay=0):
    with cd(env.code_current):
        sudo(
            'echo "{}/bin/python manage.py '
            'kill_stale_celery_workers" '
            '| at now + {} minutes'.format(env.py3_virtualenv_current, delay)
        ) 

def create_code_dir(full_cluster=True):
    roles_to_use = _get_roles(full_cluster)

    @roles(roles_to_use)
    @parallel
    def create():
        sudo('mkdir -p {}'.format(env.code_root))

    return create 

def pack():
    # Make sure machine and dev tools are up to date
    sudo('sudo yum -y update')
    sudo('sudo yum -y upgrade')
    sudo('yum install -y atlas-devel atlas-sse3-devel blas-devel gcc gcc-c++ lapack-devel python27-devel --enablerepo=epel')
    sudo('pip install -U pip')

    with warn_only():
        run('rm ~/wordnik.zip')

    sudo('dd if=/dev/zero of=/swapfile bs=1024 count=1500000')
    sudo('mkswap /swapfile')
    sudo('chmod 0600 /swapfile')
    sudo('swapon /swapfile')

    run('/usr/bin/virtualenv --python /usr/bin/python build --always-copy --no-site-packages')
    run('source build/bin/activate')

    # Order is important here, so let's make sure we've got these right
    run('pip install -U pip')
    run('pip install --use-wheel numpy')
    run('pip install --use-wheel scipy')
    run('pip install --use-wheel sklearn')
    run('pip install --use-wheel pandas')

    with open('requirements.txt') as f:
        for req in f.read().splitlines():
            if req.split("=")[0].lower() not in ('numpy', 'scipy', 'scikit-learn', 'sklearn', 'pandas'):
                run('pip install --use-wheel {}'.format(req))

    for lib in ('lib', 'lib64'):
        # Strip SO files
        run('find "$VIRTUAL_ENV/{}/python2.7/site-packages/" -name "*.so" | xargs strip'.format(lib))
        with cd('$VIRTUAL_ENV/{}/python2.7/site-packages/'.format(lib)):
            run('zip -r -9 -q ~/wordnik.zip *')

    # Get the file back onto our local machine
    local('scp %s@%s:~/wordnik.zip %s' % (env.user, env.hosts[0], lambdafile))
    update() 

def _clone_virtual_env(virtualenv_current, virtualenv_root):
    print('Cloning virtual env')
    # There's a bug in virtualenv-clone that doesn't allow us to clone envs from symlinks
    current_virtualenv = sudo('readlink -f {}'.format(virtualenv_current))
    sudo("virtualenv-clone {} {}".format(current_virtualenv, virtualenv_root))
    # There was a bug for a while that made new machines set up with commcare-cloud
    # reference current in their virtualenvs instead of their absolute path
    # this line automatically detects and fixes that.
    # It's a noop in steady-state but essentially for fixing the issue.
    sudo('sed -i -e "s~{virtualenv_current}~{virtualenv_root}~g" $(find {virtualenv_root}/bin/ -type f)'
         .format(virtualenv_current=virtualenv_current, virtualenv_root=virtualenv_root)) 

def make_remote_spark_data_dir(nodes, spark_data_dir=os.path.join('/', 'var', 'lib', 'spark'), remove_existing_spark_data=True):
    with fab.settings(fab.show('warnings', 'running', 'stdout', 'stderr'), hosts=nodes):
        if remove_existing_spark_data:
            execute(fab.sudo, 'rm -rf {spark_data}'.format(spark_data=spark_data_dir))
        execute(fab.sudo, 'mkdir -p {spark_data}'.format(spark_data=spark_data_dir)) 

def get_previous_release():
    # Gets second to last line in RELEASES.txt
    with cd(env.root):
        return sudo('tail -2 {} | head -n 1'.format(RELEASE_RECORD))