Python codecs.open() Examples

def WriteRes(malName, malTitle, malDesc, malAuthor, malDate, malSrc):
    if not os.path.exists('malcode'):
        os.makedirs('malcode')

    malTitle = malTitle.replace('<em>', '').replace('</em>', '').replace('<strong>', '').replace('</strong>', '')
    malDesc = malDesc.replace('<em>', '').replace('</em>', '').replace('<strong>', '').replace('</strong>', '')
    malAuthor = malAuthor.replace('<em>', '').replace('</em>', '').replace('<strong>', '').replace('</strong>', '')
    malDate = malDate.replace('<em>', '').replace('</em>', '').replace('<strong>', '').replace('</strong>', '')

    content = '\n'.join([malName, malTitle, malDesc, malAuthor, malDate, malSrc])

    file_path = os.path.join('malcode', malName)

    with codecs.open(file_path, 'w', 'utf-8') as file:
        file.write(content)
    # file = codecs.open(file_path, 'a+', 'utf-8')
    # file.write(content + '\n')
    # file.close 

def WriteRes(malName, malTitle, malDesc, malAuthor, malDate, malSrc):
    if not os.path.exists('malcode'):
        os.makedirs('malcode')

    malTitle = malTitle.replace('<em>', '').replace(
        '</em>', '').replace('<strong>', '').replace('</strong>', '')
    malDesc = malDesc.replace('<em>', '').replace(
        '</em>', '').replace('<strong>', '').replace('</strong>', '')
    malAuthor = malAuthor.replace('<em>', '').replace(
        '</em>', '').replace('<strong>', '').replace('</strong>', '')
    malDate = malDate.replace('<em>', '').replace(
        '</em>', '').replace('<strong>', '').replace('</strong>', '')

    content = '\n'.join([malName, malTitle, malDesc,
                         malAuthor, malDate, malSrc])

    file_path = os.path.join('malcode', malName)

    with codecs.open(file_path, 'w', 'utf-8') as file:
        file.write(content) 

def load_text_and_labels(path, lowercase = True, multilingual = False, distinct_labels_index = None):
	"""Loads text instances from files (one text one line), splits the data into words and generates labels (as one-hot vectors).
		Returns split sentences and labels.
	"""
    # Load data from files
	lines = [(s.lower() if lowercase else s).strip().split() for s in list(codecs.open(path,'r',encoding='utf8', errors='replace').readlines())]
	x_instances = [l[1:-1] for l in lines] if multilingual else [l[:-1] for l in lines]

	if multilingual: 
		langs = [l[0] for l in lines]
	labels = [l[-1] for l in lines]
	
	dist_labels = list(set(labels)) if distinct_labels_index is None else distinct_labels_index
	y_instances = [np.zeros(len(dist_labels)) for l in labels]
	for i in range(len(y_instances)):
		y_instances[i][dist_labels.index(labels[i])] = 1

	return [x_instances, y_instances, langs, dist_labels] if multilingual else [x_instances, y_instances, dist_labels] 

def get_perf(filename):
    ''' run conlleval.pl perl script to obtain
    precision/recall and F1 score '''
    _conlleval = PREFIX + 'conlleval'
    if not isfile(_conlleval):
        #download('http://www-etud.iro.umontreal.ca/~mesnilgr/atis/conlleval.pl') 
        os.system('wget https://www.comp.nus.edu.sg/%7Ekanmy/courses/practicalNLP_2008/packages/conlleval.pl')
        chmod('conlleval.pl', stat.S_IRWXU) # give the execute permissions
    
    out = []
    proc = subprocess.Popen(["perl", _conlleval], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
    stdout, _ = proc.communicate(open(filename).read())
    for line in stdout.split('\n'):
        if 'accuracy' in line:
            out = line.split()
            break
    
    # out = ['accuracy:', '16.26%;', 'precision:', '0.00%;', 'recall:', '0.00%;', 'FB1:', '0.00']
    precision = float(out[3][:-2])
    recall    = float(out[5][:-2])
    f1score   = float(out[7])

    return {'p':precision, 'r':recall, 'f1':f1score} 

def load_sentences(path, lower, zeros):
#{{{
    """
    Load sentences. A line must contain at least a word and its tag.
    Sentences are separated by empty lines.
    """
    sentences = []
    sentence = []
    for line in codecs.open(path, 'r', 'utf8'):
        line = zero_digits(line.rstrip()) if zeros else line.rstrip()
        if not line:
            if len(sentence) > 0:
                if 'DOCSTART' not in sentence[0][0]:
                    sentences.append(sentence)
                sentence = []
        else:
            word = line.split()
            assert len(word) >= 2
            sentence.append(word)
    if len(sentence) > 0:
        if 'DOCSTART' not in sentence[0][0]:
            sentences.append(sentence)
    return sentences
#}}} 

def load_list_of_unicode_from_txt(txt_path):
    ''' Loads list of unicode_strings from plain-txt file

    Returns
    -------
    list_of_u : list of unicode strings
    '''
    possible_fmts = ['utf-8', 'iso-8859-1', 'ascii']
    for fid, fmt in enumerate(possible_fmts):
        try:
            list_of_u = list()
            with codecs.open(txt_path, 'rU', fmt) as f:
                for line in f.readlines():
                    u = line.strip()
                    u = u.replace(u' ', u'_')
                    u = u.replace(u',', u'+')
                    list_of_u.append(u)
            return list_of_u
        except UnicodeDecodeError as e:
            pass
    raise e 

def manageWork(self):
        #file = open("/home/ddimitrov/20160305_en_wikilinks/tmp/missing_article_ids.p",'r')
        file = open(SSD_HOME+"pickle/redirects_ids.obj",'r')
        object_file = pickle.load(file)
        #print object_file
        #print type(object_file)
        for root, dirs, files in os.walk(STATIC_HTML_DUMP_ARTICLES_DIR+self.path):
            for i, file_name in enumerate(files):
                if file_name.endswith(".zip"):
                    parts = file_name.split('_')
                    if long(parts[1]) in object_file:

                        try:
                            self.parse_article(file_name,root)
                        except  Exception as e:
                            print("FILENAME_FAIL:"+file_name)
                            print(type(e))    # the exception instance
                            print(e)
                            print (e.message) 

def plaintext2zip(self, file_name, subdirname, plaintext):

        file_name=file_name.split('.')[0]
        plaintext_file_name = STATIC_PLAINTEXT_ARTICLES_DIR+subdirname+'/'+file_name+'.txt'
        zip_file_name = STATIC_PLAINTEXT_ARTICLES_DIR+subdirname+'/'+file_name+'.zip'

        if not os.path.exists(STATIC_PLAINTEXT_ARTICLES_DIR+subdirname):
            os.makedirs(STATIC_PLAINTEXT_ARTICLES_DIR+subdirname)


        with codecs.open(plaintext_file_name, 'w', encoding='utf8') as outfile:
            outfile.write(plaintext)
            outfile.flush()
            outfile.close()

        zf = zipfile.ZipFile(zip_file_name, mode='w', compression=zipfile.ZIP_DEFLATED)
        try:
            zf.write(plaintext_file_name, os.path.basename(plaintext_file_name))
            os.remove(plaintext_file_name)
        except Exception, e:
            print e
            logging.error('zip error %s ' % plaintext_file_name) 

def newFactorioMap():
	FACTORIOPATH = getFactorioPath()

	mapFileExamplePath="%s/data/map-gen-settings.example.json" % (FACTORIOPATH)
	mapFilePath="%s/config/mapsettings.json" % (FACTORIOPATH)

	if not os.path.isfile(mapFilePath):		
		with codecs.open(mapFileExamplePath, 'r', encoding='utf-8') as map_file:
			mapJson = json.load(map_file)
	
			mapJson['starting_area'] = "very-high"

			for control in mapJson['autoplace_controls']:
				mapJson['autoplace_controls'][control]['size'] = "high"
				mapJson['autoplace_controls'][control]['richness'] = "very-high"
				mapJson['autoplace_controls'][control]['frequency'] = "low"

		with codecs.open(mapFilePath, 'w', encoding='utf-8') as map_file:
			json.dump(mapJson, map_file, indent=4)


	print(subprocess.check_output(
					["%s/bin/x64/factorio" % (FACTORIOPATH), 
					 "--create", "%s/saves/%s" % (FACTORIOPATH, 'Headless-{:%Y%m%d-%H%M%S}'.format(datetime.datetime.now())),
					 "--map-gen-settings", "%s/config/mapsettings.json" % (FACTORIOPATH)	]
					 ).decode("unicode_escape")) 

def configAuthenticate(username, password):
	FACTORIOPATH = getFactorioPath()

	url = "https://auth.factorio.com/api-login"
	params = {'username': username, 'password': password, 'apiVersion': 2}


	if not os.path.isfile("%s/bin/x64/factorio" % (FACTORIOPATH) ):
		print("Could not find factorio at %s" % (FACTORIOPATH))
		sys.exit(1)


	print("Fetching token for %s" %  (username))
	myResponse = requests.post(url,data=params, verify=True)
	if(myResponse.ok):

	    jData = json.loads(myResponse.text)
	    print("Writing %s to settings.json" % (jData[0]))
	    
	else:
	  # If response code is not ok (200), print the resulting http error code with description
	    myResponse.raise_for_status()
	    sys.exit(1)
	

	try:
		with codecs.open(getSettingsFile(), 'r', encoding='utf-8') as settings_file:
			settingsJson = json.load(settings_file)
			settingsJson['token'] = jData[0]
			settingsJson['username'] = username
				


		with codecs.open("%s/config/settings.json" % (FACTORIOPATH), 'w', encoding='utf-8') as settings_file:
			json.dump(settingsJson, settings_file, indent=4)
	except Exception as e:
		print(e)
		print("Help! Can't deal with the settings file!") 

def getPassword():	
	FACTORIOPATH = getFactorioPath()

	try:

		with codecs.open("%s/config/settings.json" % (FACTORIOPATH), 'r', encoding='utf-8') as settings_file:
			settingsJson = json.load(settings_file)			
			return settingsJson["game_password"]

	except Exception as e:
		return "Unable to read settings.json. Error %s" % (e) 

def runFactorio(stdin=True):

	FACTORIOPATH = getFactorioPath()

	if not stdin:
		print("Interactive mode enabled. Server can only be quit with ctrl-C. /quit will only restart the server.")	
	phrase = generatePhrase(6)
	with open("/tmp/factorioRcon", "w") as phraseFile:
		phraseFile.write(phrase)

	with codecs.open("%s/config/settings.json" % (FACTORIOPATH), 'r+', encoding='utf-8') as settings_file:
		try:
			settingsJson = json.load(settings_file)
		except Exception as e:
			print("Problem with settings file.")
			print(e)
			sys.exit(1)


	if not filter(os.path.isfile, glob.glob('%s/saves/*.zip' % (FACTORIOPATH))):
		print("Cannot find a save file. Exiting.")
		sys.exit(1)

	nohup="nohup "

	if not stdin:
		print(nohup)
		nohup=""
	myprogram = {"cmd": "%s%s/bin/x64/factorio --rcon-port 27015 --rcon-password \"%s\" --start-server-load-latest --server-settings %s/config/settings.json" % (nohup, FACTORIOPATH, phrase, FACTORIOPATH) , "numprocesses": 1, "stop_timeout": 20, "close_child_stdin":stdin}

	arbiter = circus.get_arbiter([myprogram])
	try:
					
		arbiter.start()
	finally:
		os.remove("/tmp/factorioRcon")
		arbiter.stop()
		print(subprocess.check_output(['tail', '-13', '%s/factorio-current.log'%FACTORIOPATH]).decode("unicode_escape")) 

def read(*parts):
    """
    Build an absolute path from *parts* and and return the contents of the
    resulting file.  Assume UTF-8 encoding.
    """
    with codecs.open(os.path.join(HERE, *parts), "rb", "utf-8") as f:
        return f.read() 

def read(*parts):
    """
    Build an absolute path from *parts* and and return the contents of the
    resulting file.  Assume UTF-8 encoding.
    """
    with codecs.open(os.path.join(HERE, *parts), "rb", "utf-8") as f:
        return f.read() 

def local_file(file):
  return codecs.open(
    os.path.join(os.path.dirname(__file__), file), 'r', 'utf-8'
  ) 

def serialize(item, path):
	pickle.dump(item, open(path, "wb" )) 

def deserialize(path):
	return pickle.load(open(path, "rb" )) 

def load_file(filepath):
	return (codecs.open(filepath, 'r', encoding = 'utf8', errors = 'replace')).read() 

def load_lines(filepath):
	return [l.strip() for l in list(codecs.open(filepath, "r", encoding = 'utf8', errors = 'replace').readlines())]


################################################################################################################################ 

def store_embeddings(path, embeddings, language, print_progress = True):
	f = codecs.open(path,'w',encoding='utf8')
	vocab = embeddings.lang_vocabularies[language]
	embs = 	embeddings.lang_embeddings[language]

	cnt = 0
	for word in vocab:
		cnt += 1
		if print_progress and cnt % 1000 == 0:
			print("Storing embeddings " + str(cnt))
		f.write(word + " ")
		for i in range(len(embs[vocab[word]])):
			f.write(str(embs[vocab[word]][i]) + " ")
		f.write("\n")
	f.close() 

def load_whitespace_separated_data(filepath):
	lines = list(codecs.open(filepath,'r',encoding='utf8', errors='replace').readlines())
	return [[x.strip() for x in l.strip().split()] for l in lines] 

def load_tab_separated_data(filepath):
	lines = list(codecs.open(filepath,'r',encoding='utf8', errors='replace').readlines())
	return [[x.strip() for x in l.strip().split('\t')] for l in lines] 

def load_wn_concepts_dict(path):
	lines = list(codecs.open(path,'r',encoding='utf8', errors='replace').readlines())
	lcols = {x[0] : ' '.join((x[1].split('_'))[2:-2]) for x in [l.strip().split() for l in lines]}
	return lcols 

def load_bless_dataset(path):
	lines = list(codecs.open(path,'r',encoding='utf8', errors='replace').readlines())
	lcols = [(x[0].split('-')[0], x[3].split('-')[0], "1" if x[2] == "hyper" else "0") for x in [l.strip().split() for l in lines]]
	return lcols 

def write_list(path, list):
	f = codecs.open(path,'w',encoding='utf8')
	for l in list:
		f.write(l + "\n")
	f.close() 

def load_translation_pairs(filepath):
	lines = list(codecs.open(filepath,'r',encoding='utf8', errors='replace').readlines())
	dataset = []; 
	for line in lines:
		spl = line.split(',')
		srcword = spl[0].strip()
		trgword = spl[1].strip(); 
		if (" " not in srcword.strip()) and  (" " not in trgword.strip()):
			dataset.append((srcword, trgword)); 
	return dataset 

def write_list_tuples_separated(path, list, delimiter = '\t'):
	f = codecs.open(path,'w',encoding='utf8')
	for i in range(len(list)):
		for j in range(len(list[i])):
			if j == len(list[i]) - 1: 
				f.write(str(list[i][j]) + '\n')
			else:
				f.write(str(list[i][j]) + delimiter)  
	f.close() 

def store_wordnet_rels(dirpath, relname, pos, lang, instances):
	f = codecs.open(dirpath + "/" + lang + "_" + relname + "_" + pos + ".txt",'w',encoding='utf8')
	for i in instances:
		splt = i.split('::')
		f.write(splt[0].replace("_", " ") + "\t" + splt[1].replace("_", " ") + "\t" + str(instances[i]) + "\n")
	f.close() 

def load_csv_lines(path, delimiter = ',', indices = None):
	f = codecs.open(path,'r',encoding='utf8')
	lines = [l.strip().split(delimiter) for l in f.readlines()]
	if indices is None:
		return lines
	else:
		return [sublist(l, indices) for l in lines] 

def load_csv_lines_line_by_line(path, delimiter = ',', indices = None, limit = None):
	lines = []
	f = codecs.open(path,'r',encoding='utf8')
	line = f.readline().strip()
	cnt = 1
	while line is not None:
		lines.append(sublist(line, indices) if indices is not None else line.split(delimiter))	
		line = f.readline().strip()
		cnt += 1
		if limit is not None and cnt > limit:
			break
	return lines 

def load_classification_data(path, delimiter_text_labels = '\t', delimiter_labels = '\t', line_start_skip = None):
	f = codecs.open(path,'r',encoding='utf8')
	lines = [[t.strip() for t in l.split(delimiter_text_labels)] for l in f.readlines()]
	instances = []
	for i in range(len(lines)):
		if line_start_skip is not None and lines[i][0].startswith(line_start_skip):
			continue
		text = data_helper.clean_str(lines[i][0].strip()).split()
		if delimiter_text_labels == delimiter_labels:
			labels = lines[i][1:]
		else:
			labels = lines[i][1].strip().split(delimiter_labels)
		instances.append((text, labels))
	return instances 

def parse_file(self, filename, encoding=None, debug=False):
        """Parse a file and return the syntax tree."""
        stream = codecs.open(filename, "r", encoding)
        try:
            return self.parse_stream(stream, debug)
        finally:
            stream.close() 

def _read_python_source(self, filename):
        """
        Do our best to decode a Python source file correctly.
        """
        try:
            f = open(filename, "rb")
        except IOError as err:
            self.log_error("Can't open %s: %s", filename, err)
            return None, None
        try:
            encoding = tokenize.detect_encoding(f.readline)[0]
        finally:
            f.close()
        with _open_with_encoding(filename, "r", encoding=encoding) as f:
            return _from_system_newlines(f.read()), encoding 

def __init__(self, filename, mode='a', encoding=None, delay=0):
        """
        Open the specified file and use it as the stream for logging.
        """
        #keep the absolute path, otherwise derived classes which use this
        #may come a cropper when the current directory changes
        if codecs is None:
            encoding = None
        self.baseFilename = os.path.abspath(filename)
        self.mode = mode
        self.encoding = encoding
        self.delay = delay
        if delay:
            #We don't open the stream, but we still need to call the
            #Handler constructor to set level, formatter, lock etc.
            Handler.__init__(self)
            self.stream = None
        else:
            StreamHandler.__init__(self, self._open()) 

def _open(self):
        """
        Open the current base file with the (original) mode and encoding.
        Return the resulting stream.
        """
        if self.encoding is None:
            stream = open(self.baseFilename, self.mode)
        else:
            stream = codecs.open(self.baseFilename, self.mode, self.encoding)
        return stream 

def emit(self, record):
        """
        Emit a record.

        If the stream was not opened because 'delay' was specified in the
        constructor, open it before calling the superclass's emit.
        """
        if self.stream is None:
            self.stream = self._open()
        StreamHandler.emit(self, record)

#---------------------------------------------------------------------------
#   Manager classes and functions
#--------------------------------------------------------------------------- 

def test_bug1728403(self):
        try:
            open(TESTFN, 'w').write('\xa1')
            f = codecs.open(TESTFN, encoding='cp949')
            self.assertRaises(UnicodeDecodeError, f.read, 2)
        finally:
            try: f.close()
            except: pass
            os.unlink(TESTFN) 

def test_bug691291(self):
        # Files are always opened in binary mode, even if no binary mode was
        # specified.  This means that no automatic conversion of '\n' is done
        # on reading and writing.
        s1 = u'Hello\r\nworld\r\n'

        s = s1.encode(self.encoding)
        self.addCleanup(test_support.unlink, test_support.TESTFN)
        with open(test_support.TESTFN, 'wb') as fp:
            fp.write(s)
        with codecs.open(test_support.TESTFN, 'U', encoding=self.encoding) as reader:
            self.assertEqual(reader.read(), s1) 

def create_readonly_file(self, filename):
        file_path = os.path.join(self.temp_dir, filename)
        with open(file_path, 'w') as file:
            file.write(filename)
        os.chmod(file_path, stat.S_IREAD) 

def setUp(self):
        super(TestArgumentsFromFile, self).setUp()
        file_texts = [
            ('hello', 'hello world!\n'),
            ('recursive', '-a\n'
                          'A\n'
                          '@hello'),
            ('invalid', '@no-such-path\n'),
        ]
        for path, text in file_texts:
            file = open(path, 'w')
            file.write(text)
            file.close() 

def setUp(self):
        super(TestFileTypeR, self).setUp()
        for file_name in ['foo', 'bar']:
            file = open(os.path.join(self.temp_dir, file_name), 'w')
            file.write(file_name)
            file.close()
        self.create_readonly_file('readonly') 

def setUp(self):
        super(TestFileTypeDefaults, self).setUp()
        file = open(os.path.join(self.temp_dir, 'good'), 'w')
        file.write('good')
        file.close() 

def setUp(self):
        super(TestFileTypeRB, self).setUp()
        for file_name in ['foo', 'bar']:
            file = open(os.path.join(self.temp_dir, file_name), 'w')
            file.write(file_name)
            file.close() 

def _test_module_encoding(self, path):
        path, _ = os.path.splitext(path)
        path += ".py"
        with codecs.open(path, 'r', 'utf8') as f:
            f.read() 

def view_file(parent, title, filename, encoding=None, modal=True):
    try:
        if encoding:
            import codecs
            textFile = codecs.open(filename, 'r')
        else:
            textFile = open(filename, 'r')
    except IOError:
        tkMessageBox.showerror(title='File Load Error',
                               message='Unable to load file %r .' % filename,
                               parent=parent)
    else:
        return view_text(parent, title, textFile.read(), modal) 

def read(*parts):
    with codecs.open(os.path.join(here, *parts), 'r') as fp:
        return fp.read()


# https://packaging.python.org/guides/single-sourcing-package-version/ 

def load_sentences(path):
    sentences = []
    for line in codecs.open(path, 'r', 'utf8'):
        sentence =[];
        line = line.rstrip()
        if line:
            word = line.split()
            for elem in word:
                sentence.append([elem]);
            sentences.append(sentence)
    return sentences 

def findNotSame(fNameX,fNameY):
#{{{
    """
    verify two file is same or not 
    """
    space='space';
    def loadFile(fName):
        word=[];
        import codecs;
        for line in codecs.open(fName,'r','utf8'):
            line=line.rstrip();
            if len(line)>0:
                word.append(line[0]);
            else:
                word.append(space);
        return word;
    word1=loadFile(fNameX);
    word2=loadFile(fNameY);
    i=0;
    j=0;
    while i<len(word1) and j<len(word2):
        if word1[i]==word2[j]:
            i+=1;
            j+=1;
            continue;
        elif word1[i] ==space:
            i+=1;
        elif word2[j]==space:
            j+=1;
        else:
            print "not same,X:",word1[i],",line:",i,',Y:',word2[j],',line:',j;
            break;
#}}} 

def evaluate(parameters, f_eval, raw_sentences, parsed_sentences,
             id_to_tag, dictionary_tags,filename,
             useAttend=True):
#{{{
    """
    Evaluate current model using CoNLL script.
    """
    n_tags = len(id_to_tag)
    predictions = []
    count = np.zeros((n_tags, n_tags), dtype=np.int32)

    for raw_sentence, data in zip(raw_sentences, parsed_sentences):
        input = create_input(data, parameters, False,useAttend=useAttend)
        if parameters['crf']:
            y_preds = np.array(f_eval(*input))
        else:
            y_preds = f_eval(*input).argmax(axis=1)
        y_reals = np.array(data['tags']).astype(np.int32)
        assert len(y_preds) == len(y_reals)
        p_tags = [id_to_tag[y_pred] for y_pred in y_preds]
        r_tags = [id_to_tag[y_real] for y_real in y_reals]
        if parameters['tag_scheme'] == 'iobes':
            p_tags = iobes_iob(p_tags)
            r_tags = iobes_iob(r_tags)
        for i, (y_pred, y_real) in enumerate(zip(y_preds, y_reals)):
            new_line = " ".join(raw_sentence[i][:-1] + [r_tags[i], p_tags[i]])
            predictions.append(new_line)
            count[y_real, y_pred] += 1
        predictions.append("")
    #write to file 
    with codecs.open(filename, 'w', 'utf8') as f:
        f.write("\n".join(predictions))
    return get_perf(filename) 
#}}} 

def load_sentences(path):
    sentences = []
    for line in codecs.open(path, 'r', 'utf8'):
        sentence =[];
        line = line.rstrip()
        if line:
            word = line.split()
            for elem in word:
                sentence.append([elem]);
            sentences.append(sentence)
    return sentences