Python fabric.api.cd() Examples

def delete_old_builds(history):
    ''' Auto delete unnecessary build directories from the filesystem. '''
    build_path = get_release_dir()
    kept_builds = map(lambda x: get_build_name(x['id']), history['builds'])
    found_builds = fs.glob(build_path)
    to_be_deleted_builds = [x for x in found_builds if x not in kept_builds]
    deletion_count = len(to_be_deleted_builds)

    # Skip, if there are no builds to be deleted.
    if deletion_count == 0:
        return

    # Remove directories to be deleted.
    with cd(build_path):
        fs.rm_rf(to_be_deleted_builds)
        remote_info(
            'Deleted {} old build(s) from the remote'.format(deletion_count)
        ) 

def setupServers():
    sudo('yes '' | add-apt-repository ppa:fkrull/deadsnakes-python2.7 -y')
    sudo('apt-get -y update')
    sudo('apt-get -y install python2.7')
    sudo('apt-get -y dist-upgrade')
    sudo('apt-get -y install python-pip python-dev build-essential')
    sudo('apt-get -y install libssl-dev libffi-dev git-all')
    sudo('yes | pip install --upgrade pip')
    sudo('yes | pip install --upgrade virtualenv')
    sudo('yes | pip install --upgrade petlib')
    sudo('yes | pip install twisted==16.6.0')
    sudo('yes | pip install numpy')
    sudo('yes | pip install service_identity')
    sudo('yes | pip install sphinxmix')
    sudo('apt-get -y install htop')
    #sudo('apt-get -y install tshark')
    if fabric.contrib.files.exists("loopix"):
        with cd("loopix"):
            run("git pull")
            run("git checkout %s" % BRANCH)
    else:
        run("git clone https://github.com/UCL-InfoSec/loopix.git") 

def engine_start(http_host, http_port):
    package = env.package

    command = (
        "workon {package}_env &&"
        " (marvin engine-httpserver"
        " -h {http_host}"
        " -p {http_port}"
        " -e {executor}"
        " 1> /var/log/marvin/engines/{package}.out"
        " 2> /var/log/marvin/engines/{package}.err"
        " & echo $! > /var/run/marvin/engines/{package}.pid)"
    ).format(
        package=package,
        http_host=http_host,
        http_port=http_port,
        executor=env.marvin_engine_executor_path
    )

    with cd("/opt/marvin/engines/{package}/current".format(package=package)):
        run(command, pty=False) 

def keys():
    if "rsdir" not in env:
        secret = file("secret.key").read()
        public = rscoin.Key(secret, public=False)
        pid = b64encode(public.id())

        env["rsdir"] = {"special": pid, "directory": []}

    [_, host] = env.host_string.split("@")
    with cd('/home/ubuntu/projects/rscoin'):
        run('touch secret.key')
        run('rm secret.key')
        result = run('python derivekey.py --store')
        [_, key] = result.strip().split()
        
        kid = b64encode(rscoin.Key(b64decode(key)).id())
        env["rsdir"]["directory"] += [ [kid, host, 8080] ]
    

    from json import dumps
    file("directory.conf", "w").write(dumps(env["rsdir"])) 

def setup_supervisor():
    # We use supervisord to keep Crestify running in the background
    # Recover from crashes, and to start automatically on bootup
    # Also, using more than 1 gunicorn worker resulted in socket not being released, so only 1 worker will be used
    sudo('apt-get -y install supervisor')
    sudo('mkdir /var/log/crestify/')
    sudo(
        'cd /home/crestify/crestify && ../crestifyenv/bin/honcho export -s /bin/sh -a crestify supervisord /etc/supervisor/conf.d')
    fd = StringIO()
    get('/etc/supervisor/conf.d/crestify.conf', fd)
    content = fd.getvalue().splitlines()
    for n, i in enumerate(content):
        if i.startswith("environment="):
            content[n] = i + ",PATH=/home/crestify/crestifyenv/bin:%(ENV_PATH)s"
        if i.startswith("user="):
            content[n] = "user=crestify"
        if i.startswith("stopsignal="):
            content[n] = "stopsignal=TERM"  # Both Gunicorn and Celery use SIGTERM for graceful shutdown
    content = StringIO("\n".join(content))
    put(content, "/etc/supervisor/conf.d/crestify.conf", use_sudo=True)
    sudo('supervisorctl reread')
    sudo('supervisorctl update') 

def build_database(build_type='dev', **kwargs):
    """
    Calls footprint_init to load fixture datasets from into the app.

    :param kwargs: all the options for footprint init
    :return:
    """

    # TODO: unify dev+devbuild and prod+init and/or provide a way to
    # override this celery setting. In the mean time this is a hack to
    # make sure we use a build file with CELERY_ALWAYS_EAGER=False
    if build_type == 'dev':
        build_type = 'devbuild'
    elif build_type == 'prod':
        build_type = 'init'
    with cd(get_django_setting(build_type, 'ROOT_PATH')):
        args = ' '.join(["--{}={}".format(key, value) for key, value in kwargs.iteritems()])
        manage_py('footprint_init %s' % args, build_type) 

def deployMultiClient(num):
    local('rm -f testMap.csv')
    for i in range(int(num)):
        dirc = 'client%s' % i
        with cd(dirc):
            with cd('loopix'):
                run("git pull")
                run("git checkout %s" % BRANCH)
            with cd('loopix/loopix'):
                N = hexlify(os.urandom(8))
                providers = getProvidersNames()
                prvName = random.choice(providers)
                port = int(9999 - i)
                print "CLIENT: Client%s" % N
                run("python setup_client.py %d %s Client%s %s" % (port, str(env.host), N, prvName))
                get('publicClient.bin', 'publicClient-%d-%s.bin'%(port, env.host))
                with open('testMap.csv', 'a') as outfile:
                    csvW = csv.writer(outfile)
                    csvW.writerow(['Client%s'%N, dirc]) 

def build(build_type='prod'):
    """
        Drops and recreates the databases for development, then initializes main
        This will raise an error if 127.0.0.1 is not in env.hosts to protect live databases
        Make sure complete migration scripts exist prior to running this
    :return:
    """
    restart_supervisor()
    prepare_build_or_deploy(build_type=build_type)
    start = datetime.datetime.now()
    recreate_db(build_type)
    with cd(get_django_setting(build_type, 'ROOT_PATH')):
        manage_py('syncdb --noinput', build_type)
        manage_py('migrate', build_type)
        manage_py('collectstatic --noinput', build_type)
        setup_tilestache_user()
    build_database(build_type=build_type)
    post_build_database()
    end = datetime.datetime.now()
    elapsed = end - start
    print elapsed
    npm_install() 

def experiment1collect():        
        # run("ls experiment1/*")
    with cd('/home/ubuntu/projects/rscoin/%s' % env.expname):
        get('issue-times.txt', '%s/%s-issue-times.txt' % (env.expname, env.host))

    with lcd(env.expname):
        local("cat %s-issue-times.txt >> issue-times.txt" % env.host)

    with cd('/home/ubuntu/projects/rscoin/%s' % env.expname):
        get('r1-times.txt', '%s/%s-r1-times.txt' % (env.expname, env.host))
    
    with lcd(env.expname):
        local("cat %s-r1-times.txt >> r1-times.txt" % env.host)

    with cd('/home/ubuntu/projects/rscoin/%s' % env.expname):
        get('r2-times.txt', '%s/%s-r2-times.txt' % (env.expname, env.host))

    with lcd(env.expname):
        local("cat %s-r2-times.txt >> r2-times.txt" % env.host)

        # local("python exp1plot.py experiment1") 

def rollback():
    """
    Reverts project state to the last deploy.
    When a deploy is performed, the current state of the project is
    backed up. This includes the last commit checked out, the database,
    and all static files. Calling rollback will revert all of these to
    their state prior to the last deploy.
    """
    with project():
        with update_changed_requirements():
            update = "git checkout" if env.git else "hg up -C"
            run("%s `cat last.commit`" % update)
        with cd(join(static(), "..")):
            run("tar -xf %s" % join(env.proj_path, "last.tar"))
        restore("last.db")
    restart() 

def update():
    """Update the svn repo, then reinstall LNT."""
    with cd(LNT_PATH):
        with cd(LNT_PATH + "/docs/"):
            sudo('rm -rf _build')
        with cd(LNT_PATH + "/lnt/server/ui/static"):
            sudo('rm -rf docs')
            sudo('git checkout docs')

        sudo("git pull --rebase")
        run("git log -1 --pretty=%B")
        with cd(LNT_PATH + "/docs/"):
            sudo(in_venv("make"))
        sudo(IN_VENV + "python setup.py install --server")

    put(here + "/blacklist", "/tmp/blacklist")
    sudo("mv /tmp/blacklist /srv/lnt/install/blacklist")
    put(here + "/kill_zombies.py", "/tmp/kill_zombies.py")
    sudo("mv /tmp/kill_zombies.py /etc/cron.hourly/kill_zombies")
    sudo("chmod +x /etc/cron.hourly/kill_zombies")
    rotate_log()
    service_restart() 

def update_sproutcore_build_number(build_type='prod'):
    with cd('/srv/calthorpe/urbanfootprint/'):
        manage_py('update_builddate', build_type)
    print "build number updated" 

def pull_repository():
    with cd(PROJECT_DIR):
        run('git pull origin master') 

def virtualenv():
    """
    Runs commands within the project's virtualenv.
    """
    with cd(env.venv_path):
        with prefix("source %s/bin/activate" % env.venv_path):
            yield 

def run_provider():
    with cd('loopix/loopix'):
        run("git checkout %s" % BRANCH)
        run("twistd -y run_provider.py")
        pid = run("cat twistd.pid")
        print "Run on %s with PID %s" % (env.host, pid) 

def run_mixnode():
    with cd('loopix/loopix'):
        run("git checkout %s" % BRANCH)
        run("twistd -y run_mixnode.py")
        pid = run("cat twistd.pid")
        print "Run on %s with PID %s" % (env.host, pid) 

def deployProvider():
    with cd("loopix"):
        run("git pull")
        run("git checkout %s" % BRANCH)
        N = hexlify(os.urandom(8))
        with cd("loopix"):
            run("python setup_provider.py 9999 %s Provider%s" % (str(env.host), N))
            get('publicProvider.bin', 'publicProvider-%s.bin'%env.host) 

def project():
    """
    Runs commands within the project's directory.
    """
    with virtualenv():
        with cd(env.proj_dirname):
            yield 

def start_mixnode():
    with cd("loopix/loopix"):
        run("git checkout %s" % BRANCH)
        run("git pull")
        #run("twistd -p profiler_output.dat -y run_mixnode.py")
        run("twistd -y run_mixnode.py")
        pid = run("cat twistd.pid")
        print "Run on %s with PID %s" % (env.host, pid) 

def gitpull():
    with cd('loopix'):
        run('git pull') 

def pack():
    # Make sure machine and dev tools are up to date
    sudo('sudo yum -y update')
    sudo('sudo yum -y upgrade')
    sudo('yum install -y atlas-devel atlas-sse3-devel blas-devel gcc gcc-c++ lapack-devel python27-devel --enablerepo=epel')
    sudo('pip install -U pip')

    with warn_only():
        run('rm ~/wordnik.zip')

    sudo('dd if=/dev/zero of=/swapfile bs=1024 count=1500000')
    sudo('mkswap /swapfile')
    sudo('chmod 0600 /swapfile')
    sudo('swapon /swapfile')

    run('/usr/bin/virtualenv --python /usr/bin/python build --always-copy --no-site-packages')
    run('source build/bin/activate')

    # Order is important here, so let's make sure we've got these right
    run('pip install -U pip')
    run('pip install --use-wheel numpy')
    run('pip install --use-wheel scipy')
    run('pip install --use-wheel sklearn')
    run('pip install --use-wheel pandas')

    with open('requirements.txt') as f:
        for req in f.read().splitlines():
            if req.split("=")[0].lower() not in ('numpy', 'scipy', 'scikit-learn', 'sklearn', 'pandas'):
                run('pip install --use-wheel {}'.format(req))

    for lib in ('lib', 'lib64'):
        # Strip SO files
        run('find "$VIRTUAL_ENV/{}/python2.7/site-packages/" -name "*.so" | xargs strip'.format(lib))
        with cd('$VIRTUAL_ENV/{}/python2.7/site-packages/'.format(lib)):
            run('zip -r -9 -q ~/wordnik.zip *')

    # Get the file back onto our local machine
    local('scp %s@%s:~/wordnik.zip %s' % (env.user, env.hosts[0], lambdafile))
    update() 

def get_commit(build_type='prod'):
    with cd(get_django_setting(build_type, 'ROOT_PATH')):
        with hide('output'):
            commit = run("git rev-list HEAD --max-count=1 --pretty --oneline").stdout
    print str(commit)
    return commit 

def npm_install():
    with cd("/srv/calthorpe/urbanfootprint/websockets"):
        # Symlink issues with Windows+Virtualbox so we add '--no-bin-link'
        # Similar to: https://github.com/npm/npm/issues/7094
        run("/usr/bin/npm install --no-bin-links .") 

def hello(name="world"):
    with cd("."):
        run("echo hello " + name) 

def update():
    u"""Function defining all steps required to properly update application."""

    # Django app refresh:
    with cd('/var/www/volontulo'):
        run('git checkout -f {}'.format(env_vars[env.host_string]['git_branch']))
        run('git pull')

    with contextlib.nested(
        prefix('workon volontulo'),
        cd('/var/www/volontulo/backend'),
    ):
        run('pip install --upgrade -r requirements/base.txt')

    # Django site refresh:
    with contextlib.nested(
        cd('/var/www/volontulo/backend'),
        prefix('workon volontulo')
    ):
        run('python manage.py migrate --traceback')

    # Angular assets refresh:
    with contextlib.nested(
        prefix('nvm use {}'.format(NODE_VERSION)),
        cd('/var/www/volontulo/frontend'),
    ):
        run('npm install .')
        run('$(npm bin)/ng build --prod --env={}'.format(env.host_string))
        run('$(npm bin)/ng build --prod --env={} --app 1 --output-hashing=false'.format(env.host_string))
        run('./node_modules/.bin/webpack --config webpack.server.config.js --progress --colors')

    run('systemctl restart uwsgi.service')
    run('systemctl restart nginx')
    run('systemctl restart pm2-www-data.service') 

def deploy(build_type='prod', skip_git=False):
    """
    Deploy code, pip dependencies and execute migrations
    """
    prepare_build_or_deploy(build_type=build_type)

    if not skip_git:
        with cd('/srv/calthorpe/urbanfootprint/'):
            sudo('git config --global user.name {0}'.format(env.deploy_user))
            sudo('git stash', user=env.deploy_user)
            run_as_user(env.deploy_user, 'git fetch')

            if getattr(env, 'branch', None):
                run_as_user(env.deploy_user, ('git checkout {branch}'.format(branch=env.branch)))

            # There should never be any local commits on hosts that we are deploying to.
            run_as_user(env.deploy_user, 'git pull')
            run_as_user(env.deploy_user, 'git submodule init')
            run_as_user(env.deploy_user, 'git submodule sync')
            run_as_user(env.deploy_user, 'git submodule update --remote')

    commit = get_commit(build_type=build_type)

    with cd(get_django_setting(build_type, 'ROOT_PATH')):
        pip_install(build_type)
        manage_py('migrate --delete-ghost-migrations', build_type)

    build_sproutcore(build_type=build_type)

    with settings(warn_only=True):
        sudo('supervisorctl stop all')
        sudo('/etc/init.d/supervisor stop')
        sudo('/etc/init.d/nginx stop')

    sudo('/etc/init.d/nginx start')
    sudo('/etc/init.d/supervisor start')
    sudo('supervisorctl start all')

    name = getattr(env, 'name', env.host) 

def build_sproutcore(build_type='prod', minify=False):
    update_sproutcore_build_number(build_type=build_type)

    # build sproutcore
    with cd('{root}/sproutcore'.format(root=get_django_setting(build_type, 'ROOT_PATH'))):
        # Build main in the build dir
        build_command = 'sproutcore build fp --buildroot=builds'
        if not minify:
            build_command += ' --dont_minify'
        sudo(build_command, user=env.deploy_user)

        # Change ownership on output
        sudo('chown -R {0}.www-data ./builds'.format(env.deploy_user))

        # ln to the builds dir from Django's static dir
        sudo('ln -f -s {root}/sproutcore/builds/static/* {root}/footprint/main/static'.format(
            root=get_django_setting(build_type, 'ROOT_PATH')), user=env.deploy_user)

        # symlink to the sproutcore build directory "build-number"
        sudo('ln -f -s {root}/sproutcore/builds/static/fp/en/$(sproutcore build-number fp)/index.html '
             '{root}/footprint/main/templates/footprint/index.html'.format(
                 root=get_django_setting(build_type, 'ROOT_PATH'),
                 user=env.deploy_user))

    # do a collect static to grab all static files and link them to the right directory
    manage_py('collectstatic -l --noinput', build_type) 

def run_command(command, workdir=None):
    workdir = workdir or "/opt/rorolite/project"
    command_str = " ".join(command)
    with cd(workdir):
        run(command_str) 

def install(self):
        """Installs the runtime on a remote machine.

        This method must be called only from a fabric task.
        """
        target = "/tmp/rorolite-runtime/" + self.name
        sudo("mkdir -p " + target)

        put(self.root, "/tmp/rorolite-runtime", use_sudo=True)
        if self.before_scripts:
            with cd(target):
                for s in self.before_scripts:
                    print("executing", s)
                    sudo(s)

        if self.apt_packages:
            sudo("apt-get -q update")
            sudo("apt-get -q -y install " + " ".join(self.apt_packages))

        if self.pip_packages:
            sudo("{} -m pip -q install {}".format(
                self.python_binary, " ".join(self.pip_packages)))

        if self.after_scripts:
            with cd(target):
                for s in self.after_scripts:
                    print("executing", s)
                    sudo(s)

        self.setup_system_path() 

def client():
    client = os.environ.get('CLIENT')
    client_path = "footprint/client/configuration/{client}".format(client=client)
    sproutcore_path = "sproutcore/frameworks/footprint/resources/images"
    with cd('/srv/calthorpe/urbanfootprint'):
        run('if [ -e {client_path}/login_page_logo.png ]; then cp -f {client_path}/login_page_logo.png {sproutcore_path}/client_login_page_logo.png; fi'.format(client_path=client_path, sproutcore_path=sproutcore_path))
        run('if [ -e {client_path}/login_page_logo_lower.png ]; then cp -f {client_path}/login_page_logo.png {sproutcore_path}/client_login_page_logo_lower.png; fi'.format(client_path=client_path, sproutcore_path=sproutcore_path))