Python pysftp.CnOpts() Examples

def get_conn(self) -> pysftp.Connection:
        """
        Returns an SFTP connection object
        """
        if self.conn is None:
            cnopts = pysftp.CnOpts()
            if self.no_host_key_check:
                cnopts.hostkeys = None
            cnopts.compression = self.compress
            conn_params = {
                'host': self.remote_host,
                'port': self.port,
                'username': self.username,
                'cnopts': cnopts
            }
            if self.password and self.password.strip():
                conn_params['password'] = self.password
            if self.key_file:
                conn_params['private_key'] = self.key_file
            if self.private_key_pass:
                conn_params['private_key_pass'] = self.private_key_pass

            self.conn = pysftp.Connection(**conn_params)
        return self.conn 

def get_file_sftp(host, path_to_file, sftp_configuration):
    """
        Copy an existing file from SFTP via sftp://*host*/*path_to_file* link to a home directory.
        The function return the full path to the file that has been downloaded.
    """
    # disable host key checking
    cnopts = pysftp.CnOpts()
    cnopts.hostkeys = None

    # construct SFTP object and get the file on a server
    with pysftp.Connection(host, username = sftp_configuration["user"], password = sftp_configuration["psswd"],
                           cnopts = cnopts) as sftp:
        sftp.get(path_to_file)

    file_location = gc_write_dir + "/" + re.findall("[^/]*$", path_to_file)[0]
    print("File " + path_to_file + " has got successfully.")
    return file_location 

def _connect(self, write=False):
        credentials = self._get_credentials(write)
        if self.settings['protocol'] == 's3':
            from luigi.contrib.s3 import S3Client
            from d6tpipe.luigi.s3 import S3Client as S3ClientToken
            if write:
                if 'aws_session_token' in credentials:
                    cnxn = S3ClientToken(**credentials)
                else:
                    cnxn = S3Client(**credentials)
            else:
                if 'aws_session_token' in credentials:
                    cnxn = S3ClientToken(**credentials)
                else:
                    cnxn = S3Client(**credentials)
        elif self.settings['protocol'] == 'ftp':
            from d6tpipe.luigi.ftp import RemoteFileSystem
            cnxn = RemoteFileSystem(self.settings['location'], credentials['username'], credentials['password'])
        elif self.settings['protocol'] == 'sftp':
            from d6tpipe.luigi.ftp import RemoteFileSystem
            try:
                import pysftp
            except ImportError:
                raise ModuleNotFoundError('Please install pysftp to use SFTP.')
            cnopts = pysftp.CnOpts()
            cnopts.hostkeys = None

            cnxn = RemoteFileSystem(self.settings['location'], credentials['username'], credentials['password'], sftp=True, pysftp_conn_kwargs={'cnopts':cnopts})
        else:
            raise NotImplementedError('only s3 and ftp supported')

        return cnxn 

def _deprovision(self, private_key_path):
        cnopts = pysftp.CnOpts()
        cnopts.hostkeys = None
        if private_key_path:
            with pysftp.\
                    Connection(self.public_ips[0],
                               username=self._provider.vm_default_user_name,
                               cnopts=cnopts,
                               private_key=private_key_path) as sftp:
                sftp.execute('sudo waagent -deprovision -force')
                sftp.close() 

def get_connection():
    """
    Creates a new SFTP connection

    Returns:
        connection(pysftp.Connection):
            the configured connection
    """
    missing_settings = []
    for key in PEARSON_UPLOAD_REQUIRED_SETTINGS:
        if getattr(settings, key) is None:
            missing_settings.append(key)

    if missing_settings:
        raise ImproperlyConfigured(
            "The setting(s) {} are required".format(', '.join(missing_settings))
        )

    cnopts = pysftp.CnOpts()
    cnopts.hostkeys = None  # ignore knownhosts

    try:
        return pysftp.Connection(
            host=str(settings.EXAMS_SFTP_HOST),
            port=int(settings.EXAMS_SFTP_PORT),
            username=str(settings.EXAMS_SFTP_USERNAME),
            password=str(settings.EXAMS_SFTP_PASSWORD),
            cnopts=cnopts,
        )
    except (ConnectionException, SSHException) as ex:
        raise RetryableSFTPException() from ex 

def sftp():

  cnopts = pysftp.CnOpts()
  cnopts.hostkeys = None
  sftp_configs = project.task['from']['sftp']['connection']
  sftp_configs['cnopts'] = cnopts
  sftp = pysftp.Connection(**sftp_configs)

  file_name = (datetime.datetime.now() + datetime.timedelta(project.task['from']['sftp'].get('day', 0))).strftime(project.task['from']['sftp']['file'])
  input_file_name = '/tmp/%s.csv' % str(uuid.uuid1())
  sftp.get(file_name, localpath=input_file_name)

  compression = project.task['from']['sftp'].get('compression', None)

  if 'table' in project.task['to']:
    input_file = None
    if compression == 'gzip':
      input_file = gzip.open(input_file_name, 'rb')
      uncompressed_file = '/tmp/%s.csv' % str(uuid.uuid1())

      out = open(uncompressed_file, 'wb')
      for line in input_file:
        if len(line) > 1:
          out.write(line)
      out.close()
      input_file.close()

      os.remove(input_file_name)
      input_file_name = uncompressed_file

    input_file = open(input_file_name, 'rb')

    reader = csv.reader(input_file)
    header = next(reader)
    input_file.seek(0)
    schema = make_schema(header)
    output_file_name = '/tmp/%s.csv' % str(uuid.uuid1())
    clean_csv(input_file, output_file_name, len(header), header=True)
    input_file.close()

    output_file = open(output_file_name, 'rb')
    io_to_table(
      project.task['auth'],
      project.id,
      project.task['to'].get('dataset'),
      project.task['to'].get('table'),
      output_file,
      'CSV',
      schema,
      skip_rows=0,
      disposition=project.task['to'].get('write_disposition',
      'WRITE_TRUNCATE')
    )
    output_file.close()

    os.remove(input_file_name)

  os.remove(output_file_name) 

def pysftp_connect(self, python_version):
        schema="test_pysftp_connect"+python_version
        env=docker_db_environment.DockerDBEnvironment(schema)
        try:
            self.query(udf.fixindent("DROP SCHEMA %s CASCADE"%schema),ignore_errors=True)
            self.query(udf.fixindent("CREATE SCHEMA %s"%schema))
            self.query(udf.fixindent("OPEN SCHEMA %s"%schema))
            self.query(udf.fixindent('''
                CREATE OR REPLACE {python_version} SCALAR SCRIPT connect_container(host varchar(1000), port int,username varchar(1000),password varchar(1000), input_string varchar(1000))  returns VARCHAR(100000) AS
                import socket
                import io
                import traceback
                import sys
                def run(ctx):
                    import pysftp
                    cnopts = pysftp.CnOpts()
                    cnopts.hostkeys = None 
                    try:
                        with pysftp.Connection(ctx.host, username=ctx.username, password=ctx.password,cnopts=cnopts) as sftp:
                            with sftp.cd("tmp"):
                                input_buffer = io.StringIO(ctx.input_string)
                                sftp.putfo(input_buffer,"test_file")
                                output_buffer = io.BytesIO()
                                written=sftp.getfo('test_file',output_buffer)
                                value=output_buffer.getvalue()
                                value_decoded=value.decode("utf-8")
                                return value_decoded
                    except:
                        return traceback.format_exc()
                        
                /
                '''.format(python_version=python_version)))
            env.get_client().images.pull("panubo/sshd",tag="1.1.0")
            container=env.run(name="sshd_sftp",image="panubo/sshd:1.1.0",environment=["SSH_USERS=test_user:1000:1000","SSH_ENABLE_PASSWORD_AUTH=true","SFTP_MODE=true"],
                                tmpfs={'/data': 'size=1M,uid=0'})
            print(container.logs())
            time.sleep(10)
            print(container.logs())
            result=container.exec_run(cmd=''' sh -c "echo 'test_user:test_user' | chpasswd" ''')
            result=container.exec_run(cmd='''mkdir /data/tmp''')
            result=container.exec_run(cmd='''chmod 777 /data/tmp''')
            time.sleep(5)
            print(result)
            host=env.get_ip_address_of_container(container)
            rows=self.query("select connect_container('%s',%s,'test_user','test_user','success')"%(host,22))
            self.assertRowsEqual([("success",)], rows)
            print(container.logs())
        finally:
            try:
                self.query(udf.fixindent("DROP SCHEMA %s CASCADE"%schema))
            except:
                pass
            try:
                env.close()
            except:
                pass 

def sftp_download(self):

        self.sftp_ask = (
            'Please enter your login details for %s\n' % self.host
                if self.sftp_ask is None else
            self.sftp_ask
        )
        self.sftp_passwd_file = (
            os.path.join('cache', '%s.login' % self.sftp_host)
                if self.sftp_passwd_file is None else
            self.sftp_passwd_file
        )
        if self.sftp_user is None:
            self.ask_passwd()
        while True:

            self.sftp_passwd = self.sftp_passwd or None
            cnopts = pysftp.CnOpts()
            cnopts.hostkeys = None

            with pysftp.Connection(
                host = self.sftp_host,
                username = self.sftp_user,
                password = self.sftp_passwd,
                port = self.sftp_port,
                cnopts = cnopts
            ) as con:

                try:

                    con.get(self.sftp_filename, self.cache_file_name)
                    break

                except IOError:

                    msg = 'Failed to get %s from %s\n'\
                        'Try again (1) || Enter new login details (2) '\
                        '|| Cancel (3) ?\n' % (
                            self.sftp_filename, self.sftp_host)
                    whattodo = input(msg)
                    if '1' in whattodo:
                        continue
                    if '2' in whattodo:
                        self.ask_passwd(use_passwd_file = False)
                        continue
                    if '3' in whattodo:
                        return False
        return True 

def sftp_copy_subject_files(subject, necessary_files, username, domain, local_subjects_dir, remote_subject_dir,
                            password='', overwrite_files=False, print_traceback=True, port=22):
    import pysftp
    local_subject_dir = op.join(local_subjects_dir, subject)
    if password == '':
        password = ask_for_sftp_password(username)
    try:
        cnopts = pysftp.CnOpts()
        cnopts.hostkeys = None
        sftp_con = pysftp.Connection(domain, username=username, password=password, cnopts=cnopts, port=port)
    except:
        try:
            sftp_con = pysftp.Connection(domain, username=username, password=password, port=port)
        except:
            print("Can't connect via sftp!")
            if print_traceback:
                print(traceback.format_exc())
            return False
    with sftp_con as sftp:
        for fol, files in necessary_files.items():
            fol = fol.replace(':', op.sep)
            if not op.isdir(op.join(local_subject_dir, fol)):
                os.makedirs(op.join(local_subject_dir, fol))
            os.chdir(op.join(local_subject_dir, fol))
            for file_name in files:
                try:
                    file_name = file_name.replace('{subject}', subject)
                    remote_subject_dir = remote_subject_dir.replace('{subject}', subject)
                    local_fname = op.join(local_subject_dir, fol, file_name)
                    if not op.isfile(local_fname) or overwrite_files:
                        # with sftp.cd(op.join(remote_subject_dir, fol)):
                        try:
                            with sftp.cd(remote_subject_dir + '/' + fol):
                                print('sftp: getting {}'.format(file_name))
                                sftp.get(file_name)
                        except FileNotFoundError:
                            print('The file {} does not exist on the remote server! ({})'.format(
                                file_name, remote_subject_dir + '/' + fol))

                    if op.isfile(local_fname) and op.getsize(local_fname) == 0:
                        os.remove(local_fname)
                except:
                    if print_traceback:
                        print(traceback.format_exc())
    return password