Python codecs.open() Examples

The following are code examples for showing how to use codecs.open(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: malcode   Author: moonsea   File: zeustracker.py    GNU General Public License v3.0 6 votes vote down vote up
def WriteRes(malName, malTitle, malDesc, malAuthor, malDate, malSrc):
    if not os.path.exists('malcode'):
        os.makedirs('malcode')

    malTitle = malTitle.replace('<em>', '').replace('</em>', '').replace('<strong>', '').replace('</strong>', '')
    malDesc = malDesc.replace('<em>', '').replace('</em>', '').replace('<strong>', '').replace('</strong>', '')
    malAuthor = malAuthor.replace('<em>', '').replace('</em>', '').replace('<strong>', '').replace('</strong>', '')
    malDate = malDate.replace('<em>', '').replace('</em>', '').replace('<strong>', '').replace('</strong>', '')

    content = '\n'.join([malName, malTitle, malDesc, malAuthor, malDate, malSrc])

    file_path = os.path.join('malcode', malName)

    with codecs.open(file_path, 'w', 'utf-8') as file:
        file.write(content)
    # file = codecs.open(file_path, 'a+', 'utf-8')
    # file.write(content + '\n')
    # file.close 
Example 2
Project: malcode   Author: moonsea   File: vxheaven.py    GNU General Public License v3.0 6 votes vote down vote up
def WriteRes(malName, malTitle, malDesc, malAuthor, malDate, malSrc):
    if not os.path.exists('malcode'):
        os.makedirs('malcode')

    malTitle = malTitle.replace('<em>', '').replace(
        '</em>', '').replace('<strong>', '').replace('</strong>', '')
    malDesc = malDesc.replace('<em>', '').replace(
        '</em>', '').replace('<strong>', '').replace('</strong>', '')
    malAuthor = malAuthor.replace('<em>', '').replace(
        '</em>', '').replace('<strong>', '').replace('</strong>', '')
    malDate = malDate.replace('<em>', '').replace(
        '</em>', '').replace('<strong>', '').replace('</strong>', '')

    content = '\n'.join([malName, malTitle, malDesc,
                         malAuthor, malDate, malSrc])

    file_path = os.path.join('malcode', malName)

    with codecs.open(file_path, 'w', 'utf-8') as file:
        file.write(content) 
Example 3
Project: explirefit   Author: codogogo   File: data_helper.py    Apache License 2.0 6 votes vote down vote up
def load_text_and_labels(path, lowercase = True, multilingual = False, distinct_labels_index = None):
	"""Loads text instances from files (one text one line), splits the data into words and generates labels (as one-hot vectors).
		Returns split sentences and labels.
	"""
    # Load data from files
	lines = [(s.lower() if lowercase else s).strip().split() for s in list(codecs.open(path,'r',encoding='utf8', errors='replace').readlines())]
	x_instances = [l[1:-1] for l in lines] if multilingual else [l[:-1] for l in lines]

	if multilingual: 
		langs = [l[0] for l in lines]
	labels = [l[-1] for l in lines]
	
	dist_labels = list(set(labels)) if distinct_labels_index is None else distinct_labels_index
	y_instances = [np.zeros(len(dist_labels)) for l in labels]
	for i in range(len(y_instances)):
		y_instances[i][dist_labels.index(labels[i])] = 1

	return [x_instances, y_instances, langs, dist_labels] if multilingual else [x_instances, y_instances, dist_labels] 
Example 4
Project: Att-ChemdNER   Author: lingluodlut   File: utils.py    Apache License 2.0 6 votes vote down vote up
def get_perf(filename):
    ''' run conlleval.pl perl script to obtain
    precision/recall and F1 score '''
    _conlleval = PREFIX + 'conlleval'
    if not isfile(_conlleval):
        #download('http://www-etud.iro.umontreal.ca/~mesnilgr/atis/conlleval.pl') 
        os.system('wget https://www.comp.nus.edu.sg/%7Ekanmy/courses/practicalNLP_2008/packages/conlleval.pl')
        chmod('conlleval.pl', stat.S_IRWXU) # give the execute permissions
    
    out = []
    proc = subprocess.Popen(["perl", _conlleval], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    stdout, _ = proc.communicate(open(filename).read())
    for line in stdout.split('\n'):
        if 'accuracy' in line:
            out = line.split()
            break
    
    # out = ['accuracy:', '16.26%;', 'precision:', '0.00%;', 'recall:', '0.00%;', 'FB1:', '0.00']
    precision = float(out[3][:-2])
    recall    = float(out[5][:-2])
    f1score   = float(out[7])

    return {'p':precision, 'r':recall, 'f1':f1score} 
Example 5
Project: Att-ChemdNER   Author: lingluodlut   File: loader.py    Apache License 2.0 6 votes vote down vote up
def load_sentences(path, lower, zeros):
#{{{
    """
    Load sentences. A line must contain at least a word and its tag.
    Sentences are separated by empty lines.
    """
    sentences = []
    sentence = []
    for line in codecs.open(path, 'r', 'utf8'):
        line = zero_digits(line.rstrip()) if zeros else line.rstrip()
        if not line:
            if len(sentence) > 0:
                if 'DOCSTART' not in sentence[0][0]:
                    sentences.append(sentence)
                sentence = []
        else:
            word = line.split()
            assert len(word) >= 2
            sentence.append(word)
    if len(sentence) > 0:
        if 'DOCSTART' not in sentence[0][0]:
            sentences.append(sentence)
    return sentences
#}}} 
Example 6
Project: prediction-constrained-topic-models   Author: dtak   File: util_io_txt.py    MIT License 6 votes vote down vote up
def load_list_of_unicode_from_txt(txt_path):
    ''' Loads list of unicode_strings from plain-txt file

    Returns
    -------
    list_of_u : list of unicode strings
    '''
    possible_fmts = ['utf-8', 'iso-8859-1', 'ascii']
    for fid, fmt in enumerate(possible_fmts):
        try:
            list_of_u = list()
            with codecs.open(txt_path, 'rU', fmt) as f:
                for line in f.readlines():
                    u = line.strip()
                    u = u.replace(u' ', u'_')
                    u = u.replace(u',', u'+')
                    list_of_u.append(u)
            return list_of_u
        except UnicodeDecodeError as e:
            pass
    raise e 
Example 7
Project: wikilinks   Author: trovdimi   File: redirectscandidatespostioninserter.py    MIT License 6 votes vote down vote up
def manageWork(self):
        #file = open("/home/ddimitrov/20160305_en_wikilinks/tmp/missing_article_ids.p",'r')
        file = open(SSD_HOME+"pickle/redirects_ids.obj",'r')
        object_file = pickle.load(file)
        #print object_file
        #print type(object_file)
        for root, dirs, files in os.walk(STATIC_HTML_DUMP_ARTICLES_DIR+self.path):
            for i, file_name in enumerate(files):
                if file_name.endswith(".zip"):
                    parts = file_name.split('_')
                    if long(parts[1]) in object_file:

                        try:
                            self.parse_article(file_name,root)
                        except  Exception as e:
                            print("FILENAME_FAIL:"+file_name)
                            print(type(e))    # the exception instance
                            print(e)
                            print (e.message) 
Example 8
Project: wikilinks   Author: trovdimi   File: htmlstripper.py    MIT License 6 votes vote down vote up
def plaintext2zip(self, file_name, subdirname, plaintext):

        file_name=file_name.split('.')[0]
        plaintext_file_name = STATIC_PLAINTEXT_ARTICLES_DIR+subdirname+'/'+file_name+'.txt'
        zip_file_name = STATIC_PLAINTEXT_ARTICLES_DIR+subdirname+'/'+file_name+'.zip'

        if not os.path.exists(STATIC_PLAINTEXT_ARTICLES_DIR+subdirname):
            os.makedirs(STATIC_PLAINTEXT_ARTICLES_DIR+subdirname)


        with codecs.open(plaintext_file_name, 'w', encoding='utf8') as outfile:
            outfile.write(plaintext)
            outfile.flush()
            outfile.close()

        zf = zipfile.ZipFile(zip_file_name, mode='w', compression=zipfile.ZIP_DEFLATED)
        try:
            zf.write(plaintext_file_name, os.path.basename(plaintext_file_name))
            os.remove(plaintext_file_name)
        except Exception, e:
            print e
            logging.error('zip error %s ' % plaintext_file_name) 
Example 9
Project: factotum   Author: Denubis   File: newmap.py    GNU General Public License v3.0 5 votes vote down vote up
def newFactorioMap():
	FACTORIOPATH = getFactorioPath()

	mapFileExamplePath="%s/data/map-gen-settings.example.json" % (FACTORIOPATH)
	mapFilePath="%s/config/mapsettings.json" % (FACTORIOPATH)

	if not os.path.isfile(mapFilePath):		
		with codecs.open(mapFileExamplePath, 'r', encoding='utf-8') as map_file:
			mapJson = json.load(map_file)
	
			mapJson['starting_area'] = "very-high"

			for control in mapJson['autoplace_controls']:
				mapJson['autoplace_controls'][control]['size'] = "high"
				mapJson['autoplace_controls'][control]['richness'] = "very-high"
				mapJson['autoplace_controls'][control]['frequency'] = "low"

		with codecs.open(mapFilePath, 'w', encoding='utf-8') as map_file:
			json.dump(mapJson, map_file, indent=4)


	print(subprocess.check_output(
					["%s/bin/x64/factorio" % (FACTORIOPATH), 
					 "--create", "%s/saves/%s" % (FACTORIOPATH, 'Headless-{:%Y%m%d-%H%M%S}'.format(datetime.datetime.now())),
					 "--map-gen-settings", "%s/config/mapsettings.json" % (FACTORIOPATH)	]
					 ).decode("unicode_escape")) 
Example 10
Project: factotum   Author: Denubis   File: settings.py    GNU General Public License v3.0 5 votes vote down vote up
def configAuthenticate(username, password):
	FACTORIOPATH = getFactorioPath()

	url = "https://auth.factorio.com/api-login"
	params = {'username': username, 'password': password, 'apiVersion': 2}


	if not os.path.isfile("%s/bin/x64/factorio" % (FACTORIOPATH) ):
		print("Could not find factorio at %s" % (FACTORIOPATH))
		sys.exit(1)


	print("Fetching token for %s" %  (username))
	myResponse = requests.post(url,data=params, verify=True)
	if(myResponse.ok):

	    jData = json.loads(myResponse.text)
	    print("Writing %s to settings.json" % (jData[0]))
	    
	else:
	  # If response code is not ok (200), print the resulting http error code with description
	    myResponse.raise_for_status()
	    sys.exit(1)
	

	try:
		with codecs.open(getSettingsFile(), 'r', encoding='utf-8') as settings_file:
			settingsJson = json.load(settings_file)
			settingsJson['token'] = jData[0]
			settingsJson['username'] = username
				


		with codecs.open("%s/config/settings.json" % (FACTORIOPATH), 'w', encoding='utf-8') as settings_file:
			json.dump(settingsJson, settings_file, indent=4)
	except Exception as e:
		print(e)
		print("Help! Can't deal with the settings file!") 
Example 11
Project: factotum   Author: Denubis   File: settings.py    GNU General Public License v3.0 5 votes vote down vote up
def getPassword():	
	FACTORIOPATH = getFactorioPath()

	try:

		with codecs.open("%s/config/settings.json" % (FACTORIOPATH), 'r', encoding='utf-8') as settings_file:
			settingsJson = json.load(settings_file)			
			return settingsJson["game_password"]

	except Exception as e:
		return "Unable to read settings.json. Error %s" % (e) 
Example 12
Project: factotum   Author: Denubis   File: factotum.py    GNU General Public License v3.0 5 votes vote down vote up
def runFactorio(stdin=True):

	FACTORIOPATH = getFactorioPath()

	if not stdin:
		print("Interactive mode enabled. Server can only be quit with ctrl-C. /quit will only restart the server.")	
	phrase = generatePhrase(6)
	with open("/tmp/factorioRcon", "w") as phraseFile:
		phraseFile.write(phrase)

	with codecs.open("%s/config/settings.json" % (FACTORIOPATH), 'r+', encoding='utf-8') as settings_file:
		try:
			settingsJson = json.load(settings_file)
		except Exception as e:
			print("Problem with settings file.")
			print(e)
			sys.exit(1)


	if not filter(os.path.isfile, glob.glob('%s/saves/*.zip' % (FACTORIOPATH))):
		print("Cannot find a save file. Exiting.")
		sys.exit(1)

	nohup="nohup "

	if not stdin:
		print(nohup)
		nohup=""
	myprogram = {"cmd": "%s%s/bin/x64/factorio --rcon-port 27015 --rcon-password \"%s\" --start-server-load-latest --server-settings %s/config/settings.json" % (nohup, FACTORIOPATH, phrase, FACTORIOPATH) , "numprocesses": 1, "stop_timeout": 20, "close_child_stdin":stdin}

	arbiter = circus.get_arbiter([myprogram])
	try:
					
		arbiter.start()
	finally:
		os.remove("/tmp/factorioRcon")
		arbiter.stop()
		print(subprocess.check_output(['tail', '-13', '%s/factorio-current.log'%FACTORIOPATH]).decode("unicode_escape")) 
Example 13
Project: mutatest   Author: EvanKepner   File: conf.py    MIT License 5 votes vote down vote up
def read(*parts):
    """
    Build an absolute path from *parts* and and return the contents of the
    resulting file.  Assume UTF-8 encoding.
    """
    with codecs.open(os.path.join(HERE, *parts), "rb", "utf-8") as f:
        return f.read() 
Example 14
Project: mutatest   Author: EvanKepner   File: setup.py    MIT License 5 votes vote down vote up
def read(*parts):
    """
    Build an absolute path from *parts* and and return the contents of the
    resulting file.  Assume UTF-8 encoding.
    """
    with codecs.open(os.path.join(HERE, *parts), "rb", "utf-8") as f:
        return f.read() 
Example 15
Project: perf-script-postprocessor   Author: arcolife   File: setup.py    GNU General Public License v3.0 5 votes vote down vote up
def local_file(file):
  return codecs.open(
    os.path.join(os.path.dirname(__file__), file), 'r', 'utf-8'
  ) 
Example 16
Project: explirefit   Author: codogogo   File: io_helper.py    Apache License 2.0 5 votes vote down vote up
def serialize(item, path):
	pickle.dump(item, open(path, "wb" )) 
Example 17
Project: explirefit   Author: codogogo   File: io_helper.py    Apache License 2.0 5 votes vote down vote up
def deserialize(path):
	return pickle.load(open(path, "rb" )) 
Example 18
Project: explirefit   Author: codogogo   File: io_helper.py    Apache License 2.0 5 votes vote down vote up
def load_file(filepath):
	return (codecs.open(filepath, 'r', encoding = 'utf8', errors = 'replace')).read() 
Example 19
Project: explirefit   Author: codogogo   File: io_helper.py    Apache License 2.0 5 votes vote down vote up
def load_lines(filepath):
	return [l.strip() for l in list(codecs.open(filepath, "r", encoding = 'utf8', errors = 'replace').readlines())]


################################################################################################################################ 
Example 20
Project: explirefit   Author: codogogo   File: io_helper.py    Apache License 2.0 5 votes vote down vote up
def store_embeddings(path, embeddings, language, print_progress = True):
	f = codecs.open(path,'w',encoding='utf8')
	vocab = embeddings.lang_vocabularies[language]
	embs = 	embeddings.lang_embeddings[language]

	cnt = 0
	for word in vocab:
		cnt += 1
		if print_progress and cnt % 1000 == 0:
			print("Storing embeddings " + str(cnt))
		f.write(word + " ")
		for i in range(len(embs[vocab[word]])):
			f.write(str(embs[vocab[word]][i]) + " ")
		f.write("\n")
	f.close() 
Example 21
Project: explirefit   Author: codogogo   File: io_helper.py    Apache License 2.0 5 votes vote down vote up
def load_whitespace_separated_data(filepath):
	lines = list(codecs.open(filepath,'r',encoding='utf8', errors='replace').readlines())
	return [[x.strip() for x in l.strip().split()] for l in lines] 
Example 22
Project: explirefit   Author: codogogo   File: io_helper.py    Apache License 2.0 5 votes vote down vote up
def load_tab_separated_data(filepath):
	lines = list(codecs.open(filepath,'r',encoding='utf8', errors='replace').readlines())
	return [[x.strip() for x in l.strip().split('\t')] for l in lines] 
Example 23
Project: explirefit   Author: codogogo   File: io_helper.py    Apache License 2.0 5 votes vote down vote up
def load_wn_concepts_dict(path):
	lines = list(codecs.open(path,'r',encoding='utf8', errors='replace').readlines())
	lcols = {x[0] : ' '.join((x[1].split('_'))[2:-2]) for x in [l.strip().split() for l in lines]}
	return lcols 
Example 24
Project: explirefit   Author: codogogo   File: io_helper.py    Apache License 2.0 5 votes vote down vote up
def load_bless_dataset(path):
	lines = list(codecs.open(path,'r',encoding='utf8', errors='replace').readlines())
	lcols = [(x[0].split('-')[0], x[3].split('-')[0], "1" if x[2] == "hyper" else "0") for x in [l.strip().split() for l in lines]]
	return lcols 
Example 25
Project: explirefit   Author: codogogo   File: io_helper.py    Apache License 2.0 5 votes vote down vote up
def write_list(path, list):
	f = codecs.open(path,'w',encoding='utf8')
	for l in list:
		f.write(l + "\n")
	f.close() 
Example 26
Project: explirefit   Author: codogogo   File: io_helper.py    Apache License 2.0 5 votes vote down vote up
def load_translation_pairs(filepath):
	lines = list(codecs.open(filepath,'r',encoding='utf8', errors='replace').readlines())
	dataset = []; 
	for line in lines:
		spl = line.split(',')
		srcword = spl[0].strip()
		trgword = spl[1].strip(); 
		if (" " not in srcword.strip()) and  (" " not in trgword.strip()):
			dataset.append((srcword, trgword)); 
	return dataset 
Example 27
Project: explirefit   Author: codogogo   File: io_helper.py    Apache License 2.0 5 votes vote down vote up
def write_list_tuples_separated(path, list, delimiter = '\t'):
	f = codecs.open(path,'w',encoding='utf8')
	for i in range(len(list)):
		for j in range(len(list[i])):
			if j == len(list[i]) - 1: 
				f.write(str(list[i][j]) + '\n')
			else:
				f.write(str(list[i][j]) + delimiter)  
	f.close() 
Example 28
Project: explirefit   Author: codogogo   File: io_helper.py    Apache License 2.0 5 votes vote down vote up
def store_wordnet_rels(dirpath, relname, pos, lang, instances):
	f = codecs.open(dirpath + "/" + lang + "_" + relname + "_" + pos + ".txt",'w',encoding='utf8')
	for i in instances:
		splt = i.split('::')
		f.write(splt[0].replace("_", " ") + "\t" + splt[1].replace("_", " ") + "\t" + str(instances[i]) + "\n")
	f.close() 
Example 29
Project: explirefit   Author: codogogo   File: io_helper.py    Apache License 2.0 5 votes vote down vote up
def load_csv_lines(path, delimiter = ',', indices = None):
	f = codecs.open(path,'r',encoding='utf8')
	lines = [l.strip().split(delimiter) for l in f.readlines()]
	if indices is None:
		return lines
	else:
		return [sublist(l, indices) for l in lines] 
Example 30
Project: explirefit   Author: codogogo   File: io_helper.py    Apache License 2.0 5 votes vote down vote up
def load_csv_lines_line_by_line(path, delimiter = ',', indices = None, limit = None):
	lines = []
	f = codecs.open(path,'r',encoding='utf8')
	line = f.readline().strip()
	cnt = 1
	while line is not None:
		lines.append(sublist(line, indices) if indices is not None else line.split(delimiter))	
		line = f.readline().strip()
		cnt += 1
		if limit is not None and cnt > limit:
			break
	return lines 
Example 31
Project: explirefit   Author: codogogo   File: io_helper.py    Apache License 2.0 5 votes vote down vote up
def load_classification_data(path, delimiter_text_labels = '\t', delimiter_labels = '\t', line_start_skip = None):
	f = codecs.open(path,'r',encoding='utf8')
	lines = [[t.strip() for t in l.split(delimiter_text_labels)] for l in f.readlines()]
	instances = []
	for i in range(len(lines)):
		if line_start_skip is not None and lines[i][0].startswith(line_start_skip):
			continue
		text = data_helper.clean_str(lines[i][0].strip()).split()
		if delimiter_text_labels == delimiter_labels:
			labels = lines[i][1:]
		else:
			labels = lines[i][1].strip().split(delimiter_labels)
		instances.append((text, labels))
	return instances 
Example 32
Project: pyblish-win   Author: pyblish   File: driver.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def parse_file(self, filename, encoding=None, debug=False):
        """Parse a file and return the syntax tree."""
        stream = codecs.open(filename, "r", encoding)
        try:
            return self.parse_stream(stream, debug)
        finally:
            stream.close() 
Example 33
Project: pyblish-win   Author: pyblish   File: refactor.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def _read_python_source(self, filename):
        """
        Do our best to decode a Python source file correctly.
        """
        try:
            f = open(filename, "rb")
        except IOError as err:
            self.log_error("Can't open %s: %s", filename, err)
            return None, None
        try:
            encoding = tokenize.detect_encoding(f.readline)[0]
        finally:
            f.close()
        with _open_with_encoding(filename, "r", encoding=encoding) as f:
            return _from_system_newlines(f.read()), encoding 
Example 34
Project: pyblish-win   Author: pyblish   File: __init__.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def __init__(self, filename, mode='a', encoding=None, delay=0):
        """
        Open the specified file and use it as the stream for logging.
        """
        #keep the absolute path, otherwise derived classes which use this
        #may come a cropper when the current directory changes
        if codecs is None:
            encoding = None
        self.baseFilename = os.path.abspath(filename)
        self.mode = mode
        self.encoding = encoding
        self.delay = delay
        if delay:
            #We don't open the stream, but we still need to call the
            #Handler constructor to set level, formatter, lock etc.
            Handler.__init__(self)
            self.stream = None
        else:
            StreamHandler.__init__(self, self._open()) 
Example 35
Project: pyblish-win   Author: pyblish   File: __init__.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def _open(self):
        """
        Open the current base file with the (original) mode and encoding.
        Return the resulting stream.
        """
        if self.encoding is None:
            stream = open(self.baseFilename, self.mode)
        else:
            stream = codecs.open(self.baseFilename, self.mode, self.encoding)
        return stream 
Example 36
Project: pyblish-win   Author: pyblish   File: __init__.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def emit(self, record):
        """
        Emit a record.

        If the stream was not opened because 'delay' was specified in the
        constructor, open it before calling the superclass's emit.
        """
        if self.stream is None:
            self.stream = self._open()
        StreamHandler.emit(self, record)

#---------------------------------------------------------------------------
#   Manager classes and functions
#--------------------------------------------------------------------------- 
Example 37
Project: pyblish-win   Author: pyblish   File: test_multibytecodec.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_bug1728403(self):
        try:
            open(TESTFN, 'w').write('\xa1')
            f = codecs.open(TESTFN, encoding='cp949')
            self.assertRaises(UnicodeDecodeError, f.read, 2)
        finally:
            try: f.close()
            except: pass
            os.unlink(TESTFN) 
Example 38
Project: pyblish-win   Author: pyblish   File: test_codecs.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_bug691291(self):
        # Files are always opened in binary mode, even if no binary mode was
        # specified.  This means that no automatic conversion of '\n' is done
        # on reading and writing.
        s1 = u'Hello\r\nworld\r\n'

        s = s1.encode(self.encoding)
        self.addCleanup(test_support.unlink, test_support.TESTFN)
        with open(test_support.TESTFN, 'wb') as fp:
            fp.write(s)
        with codecs.open(test_support.TESTFN, 'U', encoding=self.encoding) as reader:
            self.assertEqual(reader.read(), s1) 
Example 39
Project: pyblish-win   Author: pyblish   File: test_argparse.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def create_readonly_file(self, filename):
        file_path = os.path.join(self.temp_dir, filename)
        with open(file_path, 'w') as file:
            file.write(filename)
        os.chmod(file_path, stat.S_IREAD) 
Example 40
Project: pyblish-win   Author: pyblish   File: test_argparse.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def setUp(self):
        super(TestArgumentsFromFile, self).setUp()
        file_texts = [
            ('hello', 'hello world!\n'),
            ('recursive', '-a\n'
                          'A\n'
                          '@hello'),
            ('invalid', '@no-such-path\n'),
        ]
        for path, text in file_texts:
            file = open(path, 'w')
            file.write(text)
            file.close() 
Example 41
Project: pyblish-win   Author: pyblish   File: test_argparse.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def setUp(self):
        super(TestFileTypeR, self).setUp()
        for file_name in ['foo', 'bar']:
            file = open(os.path.join(self.temp_dir, file_name), 'w')
            file.write(file_name)
            file.close()
        self.create_readonly_file('readonly') 
Example 42
Project: pyblish-win   Author: pyblish   File: test_argparse.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def setUp(self):
        super(TestFileTypeDefaults, self).setUp()
        file = open(os.path.join(self.temp_dir, 'good'), 'w')
        file.write('good')
        file.close() 
Example 43
Project: pyblish-win   Author: pyblish   File: test_argparse.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def setUp(self):
        super(TestFileTypeRB, self).setUp()
        for file_name in ['foo', 'bar']:
            file = open(os.path.join(self.temp_dir, file_name), 'w')
            file.write(file_name)
            file.close() 
Example 44
Project: pyblish-win   Author: pyblish   File: test_argparse.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def _test_module_encoding(self, path):
        path, _ = os.path.splitext(path)
        path += ".py"
        with codecs.open(path, 'r', 'utf8') as f:
            f.read() 
Example 45
Project: pyblish-win   Author: pyblish   File: textView.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def view_file(parent, title, filename, encoding=None, modal=True):
    try:
        if encoding:
            import codecs
            textFile = codecs.open(filename, 'r')
        else:
            textFile = open(filename, 'r')
    except IOError:
        tkMessageBox.showerror(title='File Load Error',
                               message='Unable to load file %r .' % filename,
                               parent=parent)
    else:
        return view_text(parent, title, textFile.read(), modal) 
Example 46
Project: DnD4py   Author: bacook17   File: setup.py    MIT License 5 votes vote down vote up
def read(*parts):
    with codecs.open(os.path.join(here, *parts), 'r') as fp:
        return fp.read()


# https://packaging.python.org/guides/single-sourcing-package-version/ 
Example 47
Project: Att-ChemdNER   Author: lingluodlut   File: tagger.py    Apache License 2.0 5 votes vote down vote up
def load_sentences(path):
    sentences = []
    for line in codecs.open(path, 'r', 'utf8'):
        sentence =[];
        line = line.rstrip()
        if line:
            word = line.split()
            for elem in word:
                sentence.append([elem]);
            sentences.append(sentence)
    return sentences 
Example 48
Project: Att-ChemdNER   Author: lingluodlut   File: utils.py    Apache License 2.0 5 votes vote down vote up
def findNotSame(fNameX,fNameY):
#{{{
    """
    verify two file is same or not 
    """
    space='space';
    def loadFile(fName):
        word=[];
        import codecs;
        for line in codecs.open(fName,'r','utf8'):
            line=line.rstrip();
            if len(line)>0:
                word.append(line[0]);
            else:
                word.append(space);
        return word;
    word1=loadFile(fNameX);
    word2=loadFile(fNameY);
    i=0;
    j=0;
    while i<len(word1) and j<len(word2):
        if word1[i]==word2[j]:
            i+=1;
            j+=1;
            continue;
        elif word1[i] ==space:
            i+=1;
        elif word2[j]==space:
            j+=1;
        else:
            print "not same,X:",word1[i],",line:",i,',Y:',word2[j],',line:',j;
            break;
#}}} 
Example 49
Project: Att-ChemdNER   Author: lingluodlut   File: utils.py    Apache License 2.0 5 votes vote down vote up
def evaluate(parameters, f_eval, raw_sentences, parsed_sentences,
             id_to_tag, dictionary_tags,filename,
             useAttend=True):
#{{{
    """
    Evaluate current model using CoNLL script.
    """
    n_tags = len(id_to_tag)
    predictions = []
    count = np.zeros((n_tags, n_tags), dtype=np.int32)

    for raw_sentence, data in zip(raw_sentences, parsed_sentences):
        input = create_input(data, parameters, False,useAttend=useAttend)
        if parameters['crf']:
            y_preds = np.array(f_eval(*input))
        else:
            y_preds = f_eval(*input).argmax(axis=1)
        y_reals = np.array(data['tags']).astype(np.int32)
        assert len(y_preds) == len(y_reals)
        p_tags = [id_to_tag[y_pred] for y_pred in y_preds]
        r_tags = [id_to_tag[y_real] for y_real in y_reals]
        if parameters['tag_scheme'] == 'iobes':
            p_tags = iobes_iob(p_tags)
            r_tags = iobes_iob(r_tags)
        for i, (y_pred, y_real) in enumerate(zip(y_preds, y_reals)):
            new_line = " ".join(raw_sentence[i][:-1] + [r_tags[i], p_tags[i]])
            predictions.append(new_line)
            count[y_real, y_pred] += 1
        predictions.append("")
    #write to file 
    with codecs.open(filename, 'w', 'utf8') as f:
        f.write("\n".join(predictions))
    return get_perf(filename) 
#}}} 
Example 50
Project: Att-ChemdNER   Author: lingluodlut   File: Atten_tagger.py    Apache License 2.0 5 votes vote down vote up
def load_sentences(path):
    sentences = []
    for line in codecs.open(path, 'r', 'utf8'):
        sentence =[];
        line = line.rstrip()
        if line:
            word = line.split()
            for elem in word:
                sentence.append([elem]);
            sentences.append(sentence)
    return sentences