• Search by Module
• Search by Words
• Search Projects

• Java
• Python
• JavaScript
• TypeScript
• C++
• Scala
• Blog

• vecto-master
  • check_ds.py
  • vecto
    • utils
      • data.py
      • fetch_benchmarks.py
      • tqdm_utils.py
      • blas.py
      • metadata.py
      • __init__.py
      • formathelper.py
    • config.py
    • cli.py
    • corpus
      • __init__.py
      • tokenization.py
      • iterators.py
      • corpus.py
      • base.py
    • embeddings
      • utils
        • subword.py
        • word.py
        • __init__.py
      • __init__.py
      • dense.py
      • legacy_w2v.py
      • train_word2vec.py
      • base.py
    • vocabulary
      • __main__.py
      • __init__.py
      • vocabulary.py
    • benchmarks
      • analogy
        • io.py
        • __init__.py
        • analogy.py
        • solvers.py
      • outliers
        • outliers.py
        • __main__.py
        • __init__.py
      • text_classification
        • text_datasets.py
        • nets.py
        • nlp_utils.py
        • text_classification.py
        • __init__.py
      • synonymy_detection
        • synonymy_detection.py
        • __main__.py
        • __init__.py
      • categorization
        • metrics.py
        • categorization.py
        • __init__.py
      • relation_extraction
        • preprocess.py
        • relation_extraction.py
        • __init__.py
      • __init__.py
      • visualize.py
      • sequence_labeling
        • conlleval.pl
        • __init__.py
        • sequence_labeling.py
      • language_modeling
        • language_modeling.py
        • __init__.py
      • base.py
      • similarity
        • similarity.py
        • __init__.py
    • data
      • io.py
      • __init__.py
      • base.py
    • __main__.py
    • __init__.py
    • _version.py
  • examples
    • test_benchmarks.ipynb
    • analogy.ipynb
    • most_similar.ipynb
  • LICENSE
  • test_requirements.txt
  • .readthedocs.yml
  • .appveyor.yml
  • README.rst
  • setup.py
  • .travis.yml
  • setup_boilerplate.py
  • tests
    • test_misc.py
    • test_format.py
    • test_vocab.py
    • test_training.py
    • test_embeddings.py
    • benchmarks
      • test_misc.py
      • test_language_modeling.py
      • test_synonymy_detection.py
      • test_relation_extraction.py
      • test_categorization.py
      • test_outliers.py
      • test_sequence_labeling.py
      • test_similarity.py
      • __init__.py
      • test_text_classification.py
      • test_analogy.py
    • data
      • corpora
        • annotated
        • jap
          • tokenized
            • corpus.txt
          • char2radical
            • char2radical.txt
        • plain
          • sense_small.txt
        • bzipped
          • sense_small.txt.bz2
        • multiple_files
          • emma_small.txt
          • sense_small.txt
        • multiple_small
          • one.txt
          • two.txt
        • gzipped
          • sense_small.txt.gz
      • embeddings
        • npy
          • test.vocab
          • metadata.json
          • test.npy
        • text
          • corrupted
            • metadata.json
            • emb.txt
          • plain_no_file_header
            • metadata.json
            • emb.txt
          • plain_with_file_header
            • metadata.json
            • emb.txt
      • benchmarks_results
        • analogy
          • 2
          • 3
          • 1
          • 4
        • text_classification
          • log
          • vocab.json
          • args.json
          • best_model.npz
        • similarity
          • 2
          • 1
      • benchmarks
        • analogy
          • metadata.json
          • category2
            • subcategory_b.txt
          • category1
            • subcategory_a.txt
        • outliers
          • test.csv
        • text_classification
          • test
          • train
        • synonymy_detection
          • test.csv
        • categorization
          • essli-2008-lite.csv
        • relation_extraction
          • test.txt
          • train.txt
        • sequence_labeling
          • pos
            • valid.txt
            • test.txt
            • train.txt
          • ner
            • valid.txt
            • test.txt
            • train.txt
          • chunk
            • valid.txt
            • test.txt
            • train.txt
        • similarity
          • ws.json
          • ws.txt
          • ws.csv
          • ws
      • vocabs
        • one_column
          • something.vocab
        • plain
          • metadata.json
          • vocab.tsv
        • numbers
          • metadata.json
          • vocab.tsv
    • test_config.py
    • test_cli_misc.py
    • __init__.py
    • test_setup.py
    • test_datasets.py
    • test_corpus.py
  • requirements.txt
  • .gitignore
  • docs
    • Makefile
    • source
      • contribution.rst
      • tutorial
        • visualization.rst
        • images
        • installing.rst
        • evaluating.rst
        • basic.rst
        • roadmap.rst
        • metadata.rst
        • getting_vectors.rst
        • index.rst
        • training_vectors.rst
        • working_with_vectors.rst
      • reference
        • index.rst
      • index.rst
      • conf.py
    • make.bat
    • requirements.txt
  • MANIFEST.in

"""Tests for embeddings module."""

import unittest
import io
import contextlib
import sys
import runpy
import os


def run_module(name: str, args, run_name: str = '__main__') -> None:
    backup_sys_argv = sys.argv
    sys.argv = [name + '.py'] + list(args)
    runpy.run_module(name, run_name=run_name)
    sys.argv = backup_sys_argv


class Tests(unittest.TestCase):

    # def test_train_word2vec(self):
    #     path_corpus = "./tests/data/corpora/plain/"
    #     sio = io.StringIO()
    #     with contextlib.redirect_stderr(sio):
    #         run_module('vecto.embeddings.train_word2vec',
    #                    ['--path_corpus', path_corpus, '--path_out', '/tmp/vecto/embeddings/', '--out_type', 'ns'])
    #         run_module('vecto.embeddings.train_word2vec',
    #                    ['--path_corpus', path_corpus, '--path_out', '/tmp/vecto/embeddings/', '--out_type', 'hsm'])
    #         run_module('vecto.embeddings.train_word2vec',
    #                    ['--path_corpus', path_corpus, '--path_out', '/tmp/vecto/embeddings/', '--out_type', 'original'])
    #         run_module('vecto.embeddings.train_word2vec',
    #                    ['--path_corpus', path_corpus, '--path_out', '/tmp/vecto/embeddings/', '--out_type', 'ns',
    #                     '--model', 'cbow'])
    #         with self.assertRaises(RuntimeError):
    #             run_module('vecto.embeddings.train_word2vec',
    #                        ['--path_corpus', path_corpus + "NONEXISTING", '--path_out', '/tmp/vecto/embeddings/',
    #                         '--out_type', 'ns',
    #                         '--model', 'cbow'])

    # @unittest.skipIf(os.environ.get('APPVEYOR'), 'skipping Appveyor due to memory error')
    # def test_train_word2vec_subword_cnn1d(self):
    #     path_corpus = "./tests/data/corpora/plain/"
    #     run_module('vecto.embeddings.train_word2vec',
    #                ['--path_corpus', path_corpus, '--path_out', '/tmp/vecto/embeddings/', '--dimension', '5',
    #                 '--subword', 'cnn1d'])
    #     with self.assertRaises(RuntimeError):
    #         run_module('vecto.embeddings.train_word2vec',
    #                    ['--path_corpus', path_corpus + "NONEXISTING", '--path_out', '/tmp/vecto/embeddings/',
    #                     '--dimension', '5',
    #                     '--subword', 'cnn1d'])

    def test_train_word2vec_subword(self):
        path_corpus = "./tests/data/corpora/plain/"
        path_vocab = "./tests/data/vocabs/plain/"
        sio = io.StringIO()
        with contextlib.redirect_stderr(sio):
            run_module('vecto.embeddings.train_word2vec',
                       ['--path_corpus', path_corpus, '--path_out', '/tmp/vecto/embeddings/', '--dimension', '5',
                        '--subword', 'cnn1d_small'])

            run_module('vecto.embeddings.train_word2vec',
                       ['--path_corpus', path_corpus, '--path_out', '/tmp/vecto/embeddings/', '--dimension', '5',
                        '--subword', 'bilstm'])
            run_module('vecto.embeddings.train_word2vec',
                       ['--path_corpus', path_corpus, '--path_out', '/tmp/vecto/embeddings/', '--dimension', '5',
                        '--subword', 'sum'])
            run_module('vecto.embeddings.train_word2vec',
                       ['--path_corpus', path_corpus, '--path_out', '/tmp/vecto/embeddings/', '--dimension', '5',
                        '--subword', '_none', '--path_vocab', path_vocab])
            run_module('vecto.embeddings.train_word2vec',
                       ['--path_corpus', path_corpus, '--path_out', '/tmp/vecto/embeddings/', '--dimension', '5',
                        '--subword', 'bilstm_sum'])
            with self.assertRaises(RuntimeError):
                run_module('vecto.embeddings.train_word2vec',
                           ['--path_corpus', path_corpus + "NONEXISTING", '--path_out', '/tmp/vecto/embeddings/',
                            '--dimension', '5',
                            '--subword', 'bilstm_sum'])

    def test_train_word2vec_subword_jap(self):
        path_corpus = "./tests/data/corpora/jap/tokenized/"
        path_word2chars = "./tests/data/corpora/jap/char2radical/char2radical.txt"
        sio = io.StringIO()
        with contextlib.redirect_stderr(sio):
            run_module('vecto.embeddings.train_word2vec',
                       ['--path_corpus', path_corpus, '--path_out', '/tmp/vecto/embeddings/', '--dimension', '5',
                        '--subword', 'sum', '--language', 'jap', '--min_gram', '1', '--max_gram', '1'])
            run_module('vecto.embeddings.train_word2vec',
                       ['--path_corpus', path_corpus, '--path_out', '/tmp/vecto/embeddings/', '--dimension', '5',
                        '--subword', 'sum', '--language', 'jap', '--min_gram', '1', '--max_gram', '1',
                        '--path_word2chars', path_word2chars])

            with self.assertRaises(RuntimeError):
                run_module('vecto.embeddings.train_word2vec',
                           ['--path_corpus', path_corpus + "NONEXISTING", '--path_out', '/tmp/vecto/embeddings/',
                            '--dimension', '5',
                            '--subword', 'sum', '--language', 'jap', '--min_gram', '1', '--max_gram', '1'])