Python pysftp.Connection() Examples

def get_file_sftp(host, path_to_file, sftp_configuration):
    """
        Copy an existing file from SFTP via sftp://*host*/*path_to_file* link to a home directory.
        The function return the full path to the file that has been downloaded.
    """
    # disable host key checking
    cnopts = pysftp.CnOpts()
    cnopts.hostkeys = None

    # construct SFTP object and get the file on a server
    with pysftp.Connection(host, username = sftp_configuration["user"], password = sftp_configuration["psswd"],
                           cnopts = cnopts) as sftp:
        sftp.get(path_to_file)

    file_location = gc_write_dir + "/" + re.findall("[^/]*$", path_to_file)[0]
    print("File " + path_to_file + " has got successfully.")
    return file_location 

def get_conn(self) -> pysftp.Connection:
        """
        Returns an SFTP connection object
        """
        if self.conn is None:
            cnopts = pysftp.CnOpts()
            if self.no_host_key_check:
                cnopts.hostkeys = None
            cnopts.compression = self.compress
            conn_params = {
                'host': self.remote_host,
                'port': self.port,
                'username': self.username,
                'cnopts': cnopts
            }
            if self.password and self.password.strip():
                conn_params['password'] = self.password
            if self.key_file:
                conn_params['private_key'] = self.key_file
            if self.private_key_pass:
                conn_params['private_key_pass'] = self.private_key_pass

            self.conn = pysftp.Connection(**conn_params)
        return self.conn 

def sftpc(self):

        args_use = self.provider.args
        if len(self.args):
            args_use = self.args

        kwargs_use = {}
        kwargs_use['host'] = self.host
        kwargs_use['port'] = int(self.port) if self.port else 22
        for k, v in self.provider.kwargs.items():
            kwargs_use[k] = v
        for k, v in self.kwargs.items():
            kwargs_use[k] = v

        conn = pysftp.Connection(*args_use, **kwargs_use)
        yield conn
        conn.close() 

def write(self, dump_path, group_key=None, file_name=None):
        import pysftp
        if group_key is None:
            group_key = []

        filebase_path, file_name = self.create_filebase_name(group_key, file_name=file_name)
        destination = (filebase_path + '/' + file_name)
        self.logger.info('Start uploading to {}'.format(dump_path))
        with pysftp.Connection(self.sftp_host, port=self.sftp_port,
                               username=self.sftp_user,
                               password=self.sftp_password) as sftp:
            if not sftp.exists(filebase_path):
                sftp.makedirs(filebase_path)
            progress = SftpUploadProgress(self.logger)
            sftp.put(dump_path, destination, callback=progress)
        self.last_written_file = destination
        self._update_metadata(dump_path, destination)
        self.logger.info('Saved {}'.format(dump_path)) 

def test_no_host_key_check_no_ignore(self, get_connection):
        connection = Connection(
            login='login', host='host',
            extra='{"ignore_hostkey_verification": false}')

        get_connection.return_value = connection
        hook = SFTPHook()
        self.assertEqual(hook.no_host_key_check, False) 

def run(self):
        import vagrant
        import pysftp
        #Figure out how to properly install this
        #Options: pip requires file, or test_requires below
        current_dir = os.path.dirname(os.path.realpath(__file__))
        test_dir = os.path.join(current_dir,"ftp")
        if not os.path.exists(test_dir):
            os.makedirs(test_dir)
        print 'creating dir'
        v = vagrant.Vagrant(test_dir)
        print 'turning on'
        v.up(vm_name='avm')
        config_raw = v.ssh_config(vm_name='avm')
        config = v.conf(vm_name='avm')
        print "Connecting via SSH"
        srv = pysftp.Connection(host=config['HostName'], username=config['User'],
                port=int(config['Port']), private_key=config['IdentityFile'])
        print "Installing prerequisites"
        #This is going to be OS specific, for now assume its debian
        package = srv.execute('export DEBIAN_FRONTEND=noninteractive; sudo apt-get -q -y install git python python-pip python-virtualenv')
        print "reclonening squadron git repo"
        clone = srv.execute('rm -rf squadron; git clone https://github.com/gosquadron/squadron.git')
        print "executing tests"
        out = srv.execute('cd squadron; python setup.py test > /vagrant/test.out')
        fresult = open(os.path.join(test_dir, 'test.out'), 'r')
        tests = fresult.read()
        fresult.close() 

        package = ', '.join(package).replace(', ', '')
        clone = ', '.join(clone).replace(', ', '')
        out = ', '.join(out).replace(', ', '')

        f = open('vmtest.output', 'w')
        f.write("{0}\n{1}\n{2}\n{3}".format(package, clone, tests, out))
        f.close()
        print "Output file: vmtest.output"
        srv.close
        print 'shutting down (disabled)'
       # v.halt(vm_name='avm')
        pass 

def sftp_mock():
    return MagicMock(autospec=pysftp.Connection) 

def _check_write_consistency(self):
        import pysftp
        with pysftp.Connection(self.sftp_host, port=self.sftp_port,
                               username=self.sftp_user,
                               password=self.sftp_password) as sftp:
            for file_info in self.get_metadata('files_written'):
                try:
                    sftp_info = sftp.stat(file_info['filename'])
                except IOError as e:
                    if e.errno == errno.ENOENT:
                        raise InconsistentWriteState(
                            '{} file is not present at destination'.format(file_info['filename']))
                sftp_size = sftp_info.st_size
                if sftp_size != file_info['size']:
                    raise InconsistentWriteState('Wrong size for file {}. Expected: {} - got {}'
                                                 .format(file_info['filename'], file_info['size'],
                                                         sftp_size))
        self.logger.info('Consistency check passed') 

def get_connection():
    """
    Creates a new SFTP connection

    Returns:
        connection(pysftp.Connection):
            the configured connection
    """
    missing_settings = []
    for key in PEARSON_UPLOAD_REQUIRED_SETTINGS:
        if getattr(settings, key) is None:
            missing_settings.append(key)

    if missing_settings:
        raise ImproperlyConfigured(
            "The setting(s) {} are required".format(', '.join(missing_settings))
        )

    cnopts = pysftp.CnOpts()
    cnopts.hostkeys = None  # ignore knownhosts

    try:
        return pysftp.Connection(
            host=str(settings.EXAMS_SFTP_HOST),
            port=int(settings.EXAMS_SFTP_PORT),
            username=str(settings.EXAMS_SFTP_USERNAME),
            password=str(settings.EXAMS_SFTP_PASSWORD),
            cnopts=cnopts,
        )
    except (ConnectionException, SSHException) as ex:
        raise RetryableSFTPException() from ex 

def _sftp_connect(self):
        try:
            self.conn.pwd
            return True
        except:
            try:
                import pysftp
            except ImportError:
                raise ModuleNotFoundError('Please install pysftp to use SFTP.')

            self.conn = pysftp.Connection(self.host, username=self.username, password=self.password,
                                          port=self.port, **self.pysftp_conn_kwargs) 

def test_no_host_key_check_ignore(self, get_connection):
        connection = Connection(
            login='login', host='host',
            extra='{"ignore_hostkey_verification": true}')

        get_connection.return_value = connection
        hook = SFTPHook()
        self.assertEqual(hook.no_host_key_check, True) 

def test_no_host_key_check_disabled(self, get_connection):
        connection = Connection(
            login='login', host='host',
            extra='{"no_host_key_check": false}')

        get_connection.return_value = connection
        hook = SFTPHook()
        self.assertEqual(hook.no_host_key_check, False) 

def test_no_host_key_check_enabled(self, get_connection):
        connection = Connection(
            login='login', host='host',
            extra='{"no_host_key_check": true}')

        get_connection.return_value = connection
        hook = SFTPHook()
        self.assertEqual(hook.no_host_key_check, True) 

def test_no_host_key_check_default(self, get_connection):
        connection = Connection(login='login', host='host')
        get_connection.return_value = connection
        hook = SFTPHook()
        self.assertEqual(hook.no_host_key_check, False) 

def test_get_conn(self):
        output = self.hook.get_conn()
        self.assertEqual(type(output), pysftp.Connection) 

def update_connection(self, login, session=None):
        connection = (session.query(Connection).
                      filter(Connection.conn_id == "sftp_default")
                      .first())
        old_login = connection.login
        connection.login = login
        session.commit()
        return old_login 

def tearDownClass(cls):
        """SFTP (RSE/PROTOCOLS): Removing created directorie s and files """
        # Load local creditentials from file
        credentials = {}
        with open('etc/rse-accounts.cfg') as fil:
            data = json.load(fil)
        credentials = data['LXPLUS']
        lxplus = pysftp.Connection(**credentials)
        with open('etc/rse_repository.json') as fil:
            prefix = json.load(fil)['LXPLUS']['protocols']['supported']['sftp']['prefix']
        lxplus.execute('rm -rf %s' % prefix)
        lxplus.close()
        shutil.rmtree(cls.tmpdir) 

def setupClass(cls):
        """SFTP (RSE/PROTOCOLS): Creating necessary directories and files """
        # Creating local files
        cls.tmpdir = tempfile.mkdtemp()
        cls.user = uuid()

        with open("%s/data.raw" % cls.tmpdir, "wb") as out:
            out.seek((1024 * 1024) - 1)  # 1 MB
            out.write('\0')
        for fil in MgrTestCases.files_local:
            os.symlink('%s/data.raw' % cls.tmpdir, '%s/%s' % (cls.tmpdir, fil))

        # Load local credentials from file
        with open('etc/rse-accounts.cfg') as fil:
            data = json.load(fil)
        credentials = data['LXPLUS']
        lxplus = pysftp.Connection(**credentials)
        with open('etc/rse_repository.json') as fil:
            prefix = json.load(fil)['LXPLUS']['protocols']['supported']['sftp']['prefix']
        lxplus.execute('mkdir %s' % prefix)
        lxplus.execute('dd if=/dev/urandom of=%s/data.raw bs=1024 count=1024' % prefix)
        cls.static_file = 'sftp://lxplus.cern.ch:22%sdata.raw' % prefix
        protocol = mgr.create_protocol(mgr.get_rse_info('LXPLUS'), 'write')
        for fil in MgrTestCases.files_remote:
            tmp = protocol.parse_pfns(protocol.lfns2pfns({'name': fil, 'scope': 'user.%s' % cls.user}).values()[0]).values()[0]
            for cmd in ['mkdir -p %s' % ''.join([tmp['prefix'], tmp['path']]), 'ln -s %sdata.raw %s' % (prefix, ''.join([tmp['prefix'], tmp['path'], tmp['name']]))]:
                lxplus.execute(cmd)
        lxplus.close() 

def connect(self):
        """
            Establishes the actual connection to the referred RSE.

            :param: credentials needed to establish a connection with the stroage.

            :raises RSEAccessDenied: if no connection could be established.
        """
        try:
            self.rse['credentials']['host'] = self.attributes['hostname']
            self.__connection = pysftp.Connection(**self.rse['credentials'])
        except Exception as e:
            raise exception.RSEAccessDenied(e) 

def sftp_copy_subject_files(subject, necessary_files, username, domain, local_subjects_dir, remote_subject_dir,
                            password='', overwrite_files=False, print_traceback=True, port=22):
    import pysftp
    local_subject_dir = op.join(local_subjects_dir, subject)
    if password == '':
        password = ask_for_sftp_password(username)
    try:
        cnopts = pysftp.CnOpts()
        cnopts.hostkeys = None
        sftp_con = pysftp.Connection(domain, username=username, password=password, cnopts=cnopts, port=port)
    except:
        try:
            sftp_con = pysftp.Connection(domain, username=username, password=password, port=port)
        except:
            print("Can't connect via sftp!")
            if print_traceback:
                print(traceback.format_exc())
            return False
    with sftp_con as sftp:
        for fol, files in necessary_files.items():
            fol = fol.replace(':', op.sep)
            if not op.isdir(op.join(local_subject_dir, fol)):
                os.makedirs(op.join(local_subject_dir, fol))
            os.chdir(op.join(local_subject_dir, fol))
            for file_name in files:
                try:
                    file_name = file_name.replace('{subject}', subject)
                    remote_subject_dir = remote_subject_dir.replace('{subject}', subject)
                    local_fname = op.join(local_subject_dir, fol, file_name)
                    if not op.isfile(local_fname) or overwrite_files:
                        # with sftp.cd(op.join(remote_subject_dir, fol)):
                        try:
                            with sftp.cd(remote_subject_dir + '/' + fol):
                                print('sftp: getting {}'.format(file_name))
                                sftp.get(file_name)
                        except FileNotFoundError:
                            print('The file {} does not exist on the remote server! ({})'.format(
                                file_name, remote_subject_dir + '/' + fol))

                    if op.isfile(local_fname) and op.getsize(local_fname) == 0:
                        os.remove(local_fname)
                except:
                    if print_traceback:
                        print(traceback.format_exc())
    return password 

def __init__(self, artifact_uri, client=None):
        self.uri = artifact_uri
        parsed = urllib.parse.urlparse(artifact_uri)
        self.config = {
            'host': parsed.hostname,
            'port': parsed.port,
            'username': parsed.username,
            'password': parsed.password
        }
        self.path = parsed.path

        if client:
            self.sftp = client
        else:
            import pysftp
            import paramiko

            if self.config['host'] is None:
                self.config['host'] = 'localhost'

            ssh_config = paramiko.SSHConfig()
            user_config_file = os.path.expanduser("~/.ssh/config")
            if os.path.exists(user_config_file):
                with open(user_config_file) as f:
                    ssh_config.parse(f)

            user_config = ssh_config.lookup(self.config['host'])

            if 'hostname' in user_config:
                self.config['host'] = user_config['hostname']

            if self.config.get('username', None) is None and 'user' in user_config:
                self.config['username'] = user_config['user']

            if self.config.get('port', None) is None:
                if 'port' in user_config:
                    self.config['port'] = int(user_config['port'])
                else:
                    self.config['port'] = 22

            if 'identityfile' in user_config:
                self.config['private_key'] = user_config['identityfile'][0]

            self.sftp = pysftp.Connection(**self.config)

        super(SFTPArtifactRepository, self).__init__(artifact_uri) 

def sftp_download(self):

        self.sftp_ask = (
            'Please enter your login details for %s\n' % self.host
                if self.sftp_ask is None else
            self.sftp_ask
        )
        self.sftp_passwd_file = (
            os.path.join('cache', '%s.login' % self.sftp_host)
                if self.sftp_passwd_file is None else
            self.sftp_passwd_file
        )
        if self.sftp_user is None:
            self.ask_passwd()
        while True:

            self.sftp_passwd = self.sftp_passwd or None
            cnopts = pysftp.CnOpts()
            cnopts.hostkeys = None

            with pysftp.Connection(
                host = self.sftp_host,
                username = self.sftp_user,
                password = self.sftp_passwd,
                port = self.sftp_port,
                cnopts = cnopts
            ) as con:

                try:

                    con.get(self.sftp_filename, self.cache_file_name)
                    break

                except IOError:

                    msg = 'Failed to get %s from %s\n'\
                        'Try again (1) || Enter new login details (2) '\
                        '|| Cancel (3) ?\n' % (
                            self.sftp_filename, self.sftp_host)
                    whattodo = input(msg)
                    if '1' in whattodo:
                        continue
                    if '2' in whattodo:
                        self.ask_passwd(use_passwd_file = False)
                        continue
                    if '3' in whattodo:
                        return False
        return True 

def pysftp_connect(self, python_version):
        schema="test_pysftp_connect"+python_version
        env=docker_db_environment.DockerDBEnvironment(schema)
        try:
            self.query(udf.fixindent("DROP SCHEMA %s CASCADE"%schema),ignore_errors=True)
            self.query(udf.fixindent("CREATE SCHEMA %s"%schema))
            self.query(udf.fixindent("OPEN SCHEMA %s"%schema))
            self.query(udf.fixindent('''
                CREATE OR REPLACE {python_version} SCALAR SCRIPT connect_container(host varchar(1000), port int,username varchar(1000),password varchar(1000), input_string varchar(1000))  returns VARCHAR(100000) AS
                import socket
                import io
                import traceback
                import sys
                def run(ctx):
                    import pysftp
                    cnopts = pysftp.CnOpts()
                    cnopts.hostkeys = None 
                    try:
                        with pysftp.Connection(ctx.host, username=ctx.username, password=ctx.password,cnopts=cnopts) as sftp:
                            with sftp.cd("tmp"):
                                input_buffer = io.StringIO(ctx.input_string)
                                sftp.putfo(input_buffer,"test_file")
                                output_buffer = io.BytesIO()
                                written=sftp.getfo('test_file',output_buffer)
                                value=output_buffer.getvalue()
                                value_decoded=value.decode("utf-8")
                                return value_decoded
                    except:
                        return traceback.format_exc()
                        
                /
                '''.format(python_version=python_version)))
            env.get_client().images.pull("panubo/sshd",tag="1.1.0")
            container=env.run(name="sshd_sftp",image="panubo/sshd:1.1.0",environment=["SSH_USERS=test_user:1000:1000","SSH_ENABLE_PASSWORD_AUTH=true","SFTP_MODE=true"],
                                tmpfs={'/data': 'size=1M,uid=0'})
            print(container.logs())
            time.sleep(10)
            print(container.logs())
            result=container.exec_run(cmd=''' sh -c "echo 'test_user:test_user' | chpasswd" ''')
            result=container.exec_run(cmd='''mkdir /data/tmp''')
            result=container.exec_run(cmd='''chmod 777 /data/tmp''')
            time.sleep(5)
            print(result)
            host=env.get_ip_address_of_container(container)
            rows=self.query("select connect_container('%s',%s,'test_user','test_user','success')"%(host,22))
            self.assertRowsEqual([("success",)], rows)
            print(container.logs())
        finally:
            try:
                self.query(udf.fixindent("DROP SCHEMA %s CASCADE"%schema))
            except:
                pass
            try:
                env.close()
            except:
                pass 

def sftp():

  cnopts = pysftp.CnOpts()
  cnopts.hostkeys = None
  sftp_configs = project.task['from']['sftp']['connection']
  sftp_configs['cnopts'] = cnopts
  sftp = pysftp.Connection(**sftp_configs)

  file_name = (datetime.datetime.now() + datetime.timedelta(project.task['from']['sftp'].get('day', 0))).strftime(project.task['from']['sftp']['file'])
  input_file_name = '/tmp/%s.csv' % str(uuid.uuid1())
  sftp.get(file_name, localpath=input_file_name)

  compression = project.task['from']['sftp'].get('compression', None)

  if 'table' in project.task['to']:
    input_file = None
    if compression == 'gzip':
      input_file = gzip.open(input_file_name, 'rb')
      uncompressed_file = '/tmp/%s.csv' % str(uuid.uuid1())

      out = open(uncompressed_file, 'wb')
      for line in input_file:
        if len(line) > 1:
          out.write(line)
      out.close()
      input_file.close()

      os.remove(input_file_name)
      input_file_name = uncompressed_file

    input_file = open(input_file_name, 'rb')

    reader = csv.reader(input_file)
    header = next(reader)
    input_file.seek(0)
    schema = make_schema(header)
    output_file_name = '/tmp/%s.csv' % str(uuid.uuid1())
    clean_csv(input_file, output_file_name, len(header), header=True)
    input_file.close()

    output_file = open(output_file_name, 'rb')
    io_to_table(
      project.task['auth'],
      project.id,
      project.task['to'].get('dataset'),
      project.task['to'].get('table'),
      output_file,
      'CSV',
      schema,
      skip_rows=0,
      disposition=project.task['to'].get('write_disposition',
      'WRITE_TRUNCATE')
    )
    output_file.close()

    os.remove(input_file_name)

  os.remove(output_file_name)