Python io.open() Examples

The following are 30 code examples for showing how to use io.open(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module io , or try the search function .

Example 1
Project: aegea   Author: kislyuk   File: build_docker_image.py    License: Apache License 2.0 6 votes vote down vote up
def get_dockerfile(args):
    if args.dockerfile:
        return io.open(args.dockerfile, "rb").read()
    else:
        cmd = bash_cmd_preamble + [
            "apt-get update -qq",
            "apt-get install -qqy cloud-init net-tools",
            "echo $CLOUD_CONFIG_B64 | base64 --decode > /etc/cloud/cloud.cfg.d/99_aegea.cfg",
            "cloud-init init",
            "cloud-init modules --mode=config",
            "cloud-init modules --mode=final"
        ]
        return dockerfile.format(base_image=args.base_image,
                                 maintainer=ARN.get_iam_username(),
                                 label=" ".join(args.tags),
                                 cloud_config_b64=base64.b64encode(get_cloud_config(args)).decode(),
                                 run=json.dumps(cmd)).encode() 
Example 2
Project: cherrypy   Author: cherrypy   File: helper.py    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def write_conf(self, extra=''):
        if self.ssl:
            serverpem = os.path.join(thisdir, 'test.pem')
            ssl = """
server.ssl_certificate: r'%s'
server.ssl_private_key: r'%s'
""" % (serverpem, serverpem)
        else:
            ssl = ''

        conf = self.config_template % {
            'host': self.host,
            'port': self.port,
            'error_log': self.error_log,
            'access_log': self.access_log,
            'ssl': ssl,
            'extra': extra,
        }
        with io.open(self.config_file, 'w', encoding='utf-8') as f:
            f.write(str(conf)) 
Example 3
Project: news-corpus-builder   Author: skillachie   File: news_corpus_generator.py    License: MIT License 6 votes vote down vote up
def read_links_file(self,file_path):
        '''
        Read links and associated categories for specified articles 
        in text file seperated by a space

        Args:
            file_path (str): The path to text file with news article links
                             and category

        Returns:
            articles: Array of tuples that contains article link & cateogory
                      ex. [('IPO','www.cs.columbia.edu')]
        '''

        articles = []
        with open(file_path) as f:
            for line in f:
                line = line.strip()
                #Ignore blank lines
                if len(line) != 0:
                    link,category = line.split(' ')
                    articles.append((category.rstrip(),link.strip()))

        return articles 
Example 4
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: text.py    License: Apache License 2.0 6 votes vote down vote up
def _get_data(self):
        archive_file_name, archive_hash = self._archive_file
        data_file_name, data_hash = self._data_file[self._segment]
        path = os.path.join(self._root, data_file_name)
        if not os.path.exists(path) or not check_sha1(path, data_hash):
            namespace = 'gluon/dataset/'+self._namespace
            downloaded_file_path = download(_get_repo_file_url(namespace, archive_file_name),
                                            path=self._root,
                                            sha1_hash=archive_hash)

            with zipfile.ZipFile(downloaded_file_path, 'r') as zf:
                for member in zf.namelist():
                    filename = os.path.basename(member)
                    if filename:
                        dest = os.path.join(self._root, filename)
                        with zf.open(member) as source, \
                             open(dest, "wb") as target:
                            shutil.copyfileobj(source, target)

        data, label = self._read_batch(path)

        self._data = nd.array(data, dtype=data.dtype).reshape((-1, self._seq_len))
        self._label = nd.array(label, dtype=label.dtype).reshape((-1, self._seq_len)) 
Example 5
def _override_epochs(notebook):
    """Overrides the number of epochs in the notebook to 1 epoch. Note this operation is idempotent.

    Args:
        notebook : string
            notebook name in folder/notebook format
    """
    notebook_path = os.path.join(*([NOTEBOOKS_DIR] + notebook.split('/'))) + ".ipynb"

    # Read the notebook and set epochs to num_epochs.
    with io.open(notebook_path, 'r', encoding='utf-8') as f:
        notebook = f.read()

    # Set number of epochs to 1.
    modified_notebook = re.sub(EPOCHS_REGEX, 'epochs = 1', notebook)

    # Replace the original notebook with the modified one.
    with io.open(notebook_path, 'w', encoding='utf-8') as f:
        f.write(modified_notebook) 
Example 6
def _override_relative_paths(notebook):
    """Overrides the relative path for the data and image directories to point
    to the right places. This is required as we run the notebooks in a different
    directory hierarchy more suitable for testing.

    Args:
        notebook : string
            notebook name in folder/notebook format
    """
    notebook_path = os.path.join(*([NOTEBOOKS_DIR] + notebook.split('/'))) + ".ipynb"

    # Read the notebook and set epochs to num_epochs.
    with io.open(notebook_path, 'r', encoding='utf-8') as f:
        notebook = f.read()

    # Update the location for the data directory.
    modified_notebook = re.sub(RELATIVE_PATH_REGEX, NOTEBOOKS_DIR, notebook)

    # Replace the original notebook with the modified one.
    with io.open(notebook_path, 'w', encoding='utf-8') as f:
        f.write(modified_notebook) 
Example 7
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: ipynb2md.py    License: Apache License 2.0 6 votes vote down vote up
def main():
    parser = argparse.ArgumentParser(
        description="Jupyter Notebooks to markdown"
    )

    parser.add_argument("notebook", nargs=1, help="The notebook to be converted.")
    parser.add_argument("-o", "--output", help="output markdown file")
    args = parser.parse_args()

    old_ipynb = args.notebook[0]
    new_ipynb = 'tmp.ipynb'
    md_file = args.output
    print(md_file)
    if not md_file:
        md_file = os.path.splitext(old_ipynb)[0] + '.md'


    clear_notebook(old_ipynb, new_ipynb)
    os.system('jupyter nbconvert ' + new_ipynb + ' --to markdown --output ' + md_file)
    with open(md_file, 'a') as f:
        f.write('<!-- INSERT SOURCE DOWNLOAD BUTTONS -->')
    os.system('rm ' + new_ipynb) 
Example 8
Project: fine-lm   Author: akzaidi   File: cnn_dailymail.py    License: MIT License 6 votes vote down vote up
def write_raw_text_to_files(all_files, urls_path, tmp_dir, is_training):
  """Write text to files."""

  def write_to_file(all_files, urls_path, tmp_dir, filename):
    with io.open(os.path.join(tmp_dir, filename + ".source"), "w") as fstory:
      with io.open(os.path.join(tmp_dir, filename + ".target"),
                   "w") as fsummary:
        for example in example_generator(all_files, urls_path, sum_token=True):
          story, summary = _story_summary_split(example)
          fstory.write(story + "\n")
          fsummary.write(summary + "\n")

  filename = "cnndm.train" if is_training else "cnndm.dev"
  tf.logging.info("Writing %s" % filename)
  write_to_file(all_files, urls_path, tmp_dir, filename)

  if not is_training:
    test_urls_path = generator_utils.maybe_download(tmp_dir, "all_test.txt",
                                                    _TEST_URLS)
    filename = "cnndm.test"
    tf.logging.info("Writing %s" % filename)
    write_to_file(all_files, test_urls_path, tmp_dir, filename) 
Example 9
Project: fine-lm   Author: akzaidi   File: generator_utils_test.py    License: MIT License 6 votes vote down vote up
def testGunzipFile(self):
    tmp_dir = self.get_temp_dir()
    (_, tmp_file_path) = tempfile.mkstemp(dir=tmp_dir)

    # Create a test zip file and unzip it.
    with gzip.open(tmp_file_path + ".gz", "wb") as gz_file:
      gz_file.write(bytes("test line", "utf-8"))
    generator_utils.gunzip_file(tmp_file_path + ".gz", tmp_file_path + ".txt")

    # Check that the unzipped result is as expected.
    lines = []
    for line in io.open(tmp_file_path + ".txt", "rb"):
      lines.append(line.decode("utf-8").strip())
    self.assertEqual(len(lines), 1)
    self.assertEqual(lines[0], "test line")

    # Clean up.
    os.remove(tmp_file_path + ".gz")
    os.remove(tmp_file_path + ".txt")
    os.remove(tmp_file_path) 
Example 10
Project: Kaggler   Author: jeongyoonlee   File: data_io.py    License: MIT License 6 votes vote down vote up
def load_csv(path):
    """Load data from a CSV file.

    Args:
        path (str): A path to the CSV format file containing data.
        dense (boolean): An optional variable indicating if the return matrix
                         should be dense.  By default, it is false.

    Returns:
        Data matrix X and target vector y
    """

    with open(path) as f:
        line = f.readline().strip()

    X = np.loadtxt(path, delimiter=',',
                   skiprows=0 if is_number(line.split(',')[0]) else 1)

    y = np.array(X[:, 0]).flatten()
    X = X[:, 1:]

    return X, y 
Example 11
Project: tldr.py   Author: lord63   File: cli.py    License: MIT License 6 votes vote down vote up
def build_index():
    repo_directory = get_config()['repo_directory']
    index_path = path.join(repo_directory, 'pages', 'index.json')
    page_path = path.join(repo_directory, 'pages')

    tree_generator = os.walk(page_path)
    folders = next(tree_generator)[1]
    commands, new_index = {}, {}
    for folder in folders:
        pages = next(tree_generator)[2]
        for page in pages:
            command_name = path.splitext(page)[0]
            if command_name not in commands:
                commands[command_name] = {'name': command_name,
                                          'platform': [folder]}
            else:
                commands[command_name]['platform'].append(folder)
    command_list = [item[1] for item in
                    sorted(commands.items(), key=itemgetter(0))]
    new_index['commands'] = command_list

    with open(index_path, mode='w') as f:
        json.dump(new_index, f) 
Example 12
Project: tldr.py   Author: lord63   File: parser.py    License: MIT License 6 votes vote down vote up
def parse_page(page):
    """Parse the command man page."""
    colors = get_config()['colors']
    with io.open(page, encoding='utf-8') as f:
        lines = f.readlines()
    output_lines = []
    for line in lines[1:]:
        if is_headline(line):
            continue
        elif is_description(line):
            output_lines.append(click.style(line.replace('>', ' '),
                                            fg=colors['description']))
        elif is_old_usage(line):
            output_lines.append(click.style(line, fg=colors['usage']))
        elif is_code_example(line):
            line = '  ' + line if line.startswith('`') else line[2:]
            output_lines.append(click.style(line.replace('`', ''),
                                            fg=colors['command']))
        elif is_line_break(line):
            output_lines.append(click.style(line))
        else:
            output_lines.append(click.style('- ' + line, fg=colors['usage']))
    return output_lines 
Example 13
Project: toolium   Author: Telefonica   File: driver_utils.py    License: Apache License 2.0 6 votes vote down vote up
def save_webdriver_logs_by_type(self, log_type, test_name):
        """Get webdriver logs of the specified type and write them to a log file

        :param log_type: browser, client, driver, performance, server, syslog, crashlog or logcat
        :param test_name: test that has generated these logs
        """
        try:
            logs = self.driver_wrapper.driver.get_log(log_type)
        except Exception:
            return

        if len(logs) > 0:
            from toolium.driver_wrappers_pool import DriverWrappersPool
            log_file_name = '{}_{}.txt'.format(get_valid_filename(test_name), log_type)
            log_file_name = os.path.join(DriverWrappersPool.logs_directory, log_file_name)
            with open(log_file_name, 'a+', encoding='utf-8') as log_file:
                driver_type = self.driver_wrapper.config.get('Driver', 'type')
                log_file.write(
                    u"\n{} '{}' test logs with driver = {}\n\n".format(datetime.now(), test_name, driver_type))
                for entry in logs:
                    timestamp = datetime.fromtimestamp(float(entry['timestamp']) / 1000.).strftime(
                        '%Y-%m-%d %H:%M:%S.%f')
                    log_file.write(u'{}\t{}\t{}\n'.format(timestamp, entry['level'], entry['message'].rstrip())) 
Example 14
Project: lineflow   Author: tofunlp   File: msr_paraphrase.py    License: MIT License 6 votes vote down vote up
def get_msr_paraphrase() -> Dict[str, List[Dict[str, str]]]:

    url = 'https://raw.githubusercontent.com/wasiahmad/paraphrase_identification/master/dataset/msr-paraphrase-corpus/msr_paraphrase_{}.txt'  # NOQA
    root = download.get_cache_directory(os.path.join('datasets', 'msr_paraphrase'))

    def creator(path):
        dataset = {}
        fieldnames = ('quality', 'id1', 'id2', 'string1', 'string2')
        for split in ('train', 'test'):
            data_path = gdown.cached_download(url.format(split))
            with io.open(data_path, 'r', encoding='utf-8') as f:
                f.readline()  # skip header
                reader = csv.DictReader(f, delimiter='\t', fieldnames=fieldnames)
                dataset[split] = [dict(row) for row in reader]

        with io.open(path, 'wb') as f:
            pickle.dump(dataset, f)
        return dataset

    def loader(path):
        with io.open(path, 'rb') as f:
            return pickle.load(f)

    pkl_path = os.path.join(root, 'msr_paraphrase.pkl')
    return download.cache_or_load_file(pkl_path, creator, loader) 
Example 15
Project: lineflow   Author: tofunlp   File: conll2000.py    License: MIT License 6 votes vote down vote up
def get_conll2000() -> Dict[str, List[str]]:

    url = 'https://www.clips.uantwerpen.be/conll2000/chunking/{}.txt.gz'
    root = download.get_cache_directory(os.path.join('datasets', 'conll2000'))

    def creator(path):
        dataset = {}
        for split in ('train', 'test'):
            data_path = gdown.cached_download(url.format(split))
            with gzip.open(data_path) as f:
                data = f.read().decode('utf-8').split('\n\n')

            dataset[split] = data

        with io.open(path, 'wb') as f:
            pickle.dump(dataset, f)
        return dataset

    def loader(path):
        with io.open(path, 'rb') as f:
            return pickle.load(f)

    pkl_path = os.path.join(root, 'conll2000.pkl')
    return download.cache_or_load_file(pkl_path, creator, loader) 
Example 16
Project: cloudflare-tor-whitelister   Author: DonnchaC   File: setup.py    License: GNU General Public License v2.0 5 votes vote down vote up
def read(*names, **kwargs):
    return io.open(
        os.path.join(os.path.dirname(__file__), *names),
        encoding=kwargs.get("encoding", "utf8")
    ).read() 
Example 17
Project: incubator-spot   Author: apache   File: configurator.py    License: Apache License 2.0 5 votes vote down vote up
def configuration():

    config = ConfigParser.ConfigParser()

    try:
        conf = open("/etc/spot.conf", "r")
    except (OSError, IOError) as e:
        print("Error opening: spot.conf" + " error: " + e.errno)
        raise e

    config.readfp(SecHead(conf))
    return config 
Example 18
Project: incubator-spot   Author: apache   File: configurator.py    License: Apache License 2.0 5 votes vote down vote up
def configuration():

    config = ConfigParser.ConfigParser()

    try:
        conf = open("/etc/spot.conf", "r")
    except (OSError, IOError) as e:
        print("Error opening: spot.conf" + " error: " + e.errno)
        raise e

    config.readfp(SecHead(conf))
    return config 
Example 19
Project: wafw00f   Author: EnableSecurity   File: main.py    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def getheaders(fn):
    headers = {}
    if not os.path.exists(fn):
        logging.getLogger('wafw00f').critical('Headers file "%s" does not exist!' % fn)
        return
    with io.open(fn, 'r', encoding='utf-8') as f:
        for line in f.readlines():
            _t = line.split(':', 2)
            if len(_t) == 2:
                h, v = map(lambda x: x.strip(), _t)
                headers[h] = v
    return headers 
Example 20
Project: aegea   Author: kislyuk   File: __init__.py    License: Apache License 2.0 5 votes vote down vote up
def main(args=None):
    parsed_args = parser.parse_args(args=args)
    logger.setLevel(parsed_args.log_level)
    has_attrs = (getattr(parsed_args, "sort_by", None) and getattr(parsed_args, "columns", None))
    if has_attrs and parsed_args.sort_by not in parsed_args.columns:
        parsed_args.columns.append(parsed_args.sort_by)
    try:
        result = parsed_args.entry_point(parsed_args)
    except Exception as e:
        if isinstance(e, NoRegionError):
            msg = "The AWS CLI is not configured."
            msg += " Please configure it using instructions at"
            msg += " http://docs.aws.amazon.com/cli/latest/userguide/cli-chap-getting-started.html"
            exit(msg)
        elif logger.level < logging.ERROR:
            raise
        else:
            err_msg = traceback.format_exc()
            try:
                err_log_filename = os.path.join(config.user_config_dir, "error.log")
                with open(err_log_filename, "ab") as fh:
                    print(datetime.datetime.now().isoformat(), file=fh)
                    print(err_msg, file=fh)
                exit("{}: {}. See {} for error details.".format(e.__class__.__name__, e, err_log_filename))
            except Exception:
                print(err_msg, file=sys.stderr)
                exit(os.EX_SOFTWARE)
    if isinstance(result, SystemExit):
        raise result
    elif result is not None:
        if isinstance(result, dict) and "ResponseMetadata" in result:
            del result["ResponseMetadata"]
        print(json.dumps(result, indent=2, default=str)) 
Example 21
Project: keras_mixnets   Author: titu1994   File: setup.py    License: MIT License 5 votes vote down vote up
def get_version():
    """Return package version as listed in `__version__` in `init.py`."""
    init_py = open(os.path.join(PACKAGE_NAME, '__init__.py')).read()
    return re.search("__version__ = ['\"]([^'\"]+)['\"]", init_py).group(1) 
Example 22
Project: cherrypy   Author: cherrypy   File: test_static.py    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _check_unicode_filesystem(tmpdir):
    filename = tmpdir / ntou('☃', 'utf-8')
    tmpl = 'File system encoding ({encoding}) cannot support unicode filenames'
    msg = tmpl.format(encoding=sys.getfilesystemencoding())
    try:
        io.open(str(filename), 'w').close()
    except UnicodeEncodeError:
        pytest.skip(msg) 
Example 23
Project: cherrypy   Author: cherrypy   File: test_static.py    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def unicode_file(cls):
        filename = ntou('Слава Україні.html', 'utf-8')
        filepath = curdir / 'static' / filename
        with filepath.open('w', encoding='utf-8')as strm:
            strm.write(ntou('Героям Слава!', 'utf-8'))
        cls.files_to_remove.append(filepath) 
Example 24
Project: cherrypy   Author: cherrypy   File: helper.py    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def get_pid(self):
        if self.daemonize:
            return int(open(self.pid_file, 'rb').read())
        return self._proc.pid 
Example 25
Project: django-click   Author: GaretJax   File: setup.py    License: MIT License 5 votes vote down vote up
def read(fname, fail_silently=False):
        """
        Read the content of the given file. The path is evaluated from the
        directory containing this file.
        """
        try:
            filepath = os.path.join(os.path.dirname(__file__), fname)
            with io.open(filepath, "rt", encoding="utf8") as f:
                return f.read()
        except:
            if not fail_silently:
                raise
            return "" 
Example 26
Project: dustmaps   Author: gregreen   File: setup.py    License: GNU General Public License v2.0 5 votes vote down vote up
def run(self):
        if not self.large_data_dir is None:
            print('Large data directory is set to: {}'.format(self.large_data_dir))
            with open(os.path.expanduser('~/.dustmapsrc'), 'w') as f:
                json.dump({'data_dir': self.large_data_dir}, f, indent=2)

        # install.do_egg_install(self) # Due to bug in setuptools that causes old-style install
        install.run(self) 
Example 27
Project: dustmaps   Author: gregreen   File: setup.py    License: GNU General Public License v2.0 5 votes vote down vote up
def readme():
    with io.open('README.md', mode='r', encoding='utf-8') as f:
        return f.read() 
Example 28
Project: news-corpus-builder   Author: skillachie   File: news_corpus_generator.py    License: MIT License 5 votes vote down vote up
def _save_flat_file(self,clean_article):

        directory = self.corpus_dir + '/' + clean_article['category']

        # create category directory
        if not os.path.exists(directory):
            os.makedirs(directory)

        file_name = directory + '/' + \
        self._remove_punctuations(clean_article['title']).replace(" ","_") + '.json'

        with io.open(file_name, 'w', encoding='utf-8') as f:
              f.write(unicode(json.dumps(clean_article, ensure_ascii=False))) 
Example 29
Project: deep-learning-note   Author: wdxtub   File: 53_machine_translation.py    License: MIT License 5 votes vote down vote up
def read_data(max_seq_len):
    # in和out分别是input和output的缩写
    in_tokens, out_tokens, in_seqs, out_seqs = [], [], [], []
    with io.open('./data/translation/fr-en-small.txt') as f:
        lines = f.readlines()
    for line in lines:
        in_seq, out_seq = line.rstrip().split('\t')
        in_seq_tokens, out_seq_tokens = in_seq.split(' '), out_seq.split(' ')
        if max(len(in_seq_tokens), len(out_seq_tokens)) > max_seq_len - 1:
            continue  # 如果加上EOS后长于max_seq_len,则忽略掉此样本
        process_one_seq(in_seq_tokens, in_tokens, in_seqs, max_seq_len)
        process_one_seq(out_seq_tokens, out_tokens, out_seqs, max_seq_len)
    in_vocab, in_data = build_data(in_tokens, in_seqs)
    out_vocab, out_data = build_data(out_tokens, out_seqs)
    return in_vocab, out_vocab, Data.TensorDataset(in_data, out_data) 
Example 30
Project: NiBetaSeries   Author: HBClab   File: create_long_description.py    License: MIT License 5 votes vote down vote up
def read(*names, **kwargs):
    return io.open(
        join(abspath(*names)),
        encoding=kwargs.get('encoding', 'utf8')
    ).read()