import logging
import re
from nltk import tokenize
from nltk.stem import WordNetLemmatizer
import safe_IO
from dictionary import *
from safe_IO import *
logger = logging.getLogger('FAIDK.Article')
class Article(object):
def __init__(self, config, file, FLAG):
self.name = file
self.config = config
self.mark = self.config["MARK"]
self.file_path = config['MAIN_PATH'] + config['ARTICLES_PATH'] + file
self.fix_dic = load_lemmatization_list_to_dic(
self.config['LEMMATIZATION_MODE'])
self.known_words = self.read_known_words()
self.article = read_article_from_file(self.file_path)
self.words = self.split_the_article(self.article, self.file_path)
self.new_words = self.read_new_words()
self.num = len(self.new_words)#会被修改，独占，勿用
self.keys = self.load_keys()
if FLAG == '1':
self.learn()
if FLAG=='2':
self.finish()
if self.config['OUT_PUT_MARKED_ARTICLE']:
self.out_put_markded_article()
self.out_put_important_sentences()
self.out_put_vocabulary()
def out_put_markded_article(self):
self.pattern = re.compile(r"(\b"+r"\b|\b".join(self.new_words)+"\b)",flags=re.IGNORECASE)
self.marked_article = re.sub(self.pattern,self.mark+r"\1"+self.mark,self.article)
self.marked_article = re.sub(r"\n", r"\n\n", self.marked_article)
write_marked_article_to_file("./others/",self.name, self.marked_article)
def out_put_important_sentences(self):
pp_m_article = re.sub(r"\n",r".\n",self.marked_article)
sentences = tokenize.sent_tokenize(pp_m_article)
i_sentences = [_ if self.pattern.search(_) else None for _ in sentences]
write_important_sentances_to_file("./others/",self.name, "\n\n".join(list(filter(None,i_sentences))))
def out_put_vocabulary(self):
vocabulary = []
num = len(self.new_words)
for i,_ in enumerate(self.new_words):
vocabulary.append("#### "+_+"\n\n"+google(_)+"\n\n"+eudic(_))
logger.info("looking up "+_+" ("+str(num-i)+"left)")
vocabulary = "\n\n".join(vocabulary)
write_vocabulary_to_file("./others/",self.name,vocabulary)
def load_keys(self):
f = self.config
keys = [f['KEY_FOR_KNOW'], f['KEY_FOR_NOT'], f['KEY_FOR_QUIT']]
logger.debug(keys)
return keys
def read_old_words(self, path):
try:
return read_article_from_file(path)
except:
logger.info('missing ' + path)
return ''
def real_word(self, word, LEMMATIZATION_flag=True):
'''
find the real word
'''
p_forword = re.compile('[a-z,A-Z,\',‘]')
word_s = p_forword.findall(word)
real_word = ''.join(word_s)#.lower()
if LEMMATIZATION_flag and self.config['LEMMATIZATION_MODE'] in ['list', 'both']:
try:
real_word = self.fix_dic[real_word]
except Exception as e:
logger.debug(e)
pass
if LEMMATIZATION_flag and self.config['LEMMATIZATION_MODE'] in ['NLTK', 'both']:
wordnet_lemmatizer = WordNetLemmatizer()
real_word = wordnet_lemmatizer.lemmatize(real_word)
logger.debug(word+'-->'+real_word)
return real_word
def read_known_words(self):
'''
load the word have known
'''
try:
with open(self.config['MAIN_PATH'] + self.config['OLD_WORDS_PATH'])as f:
all_the_words = f.read()
except:
logger.info('\'' + self.config['MAIN_PATH'] +
self.config['OLD_WORDS_PATH'] + '\' missing......')
all_the_words = ""
known_words = self.split_the_article(all_the_words,LEMMATIZATION_flag=False)
logger.info(known_words)
num = len(known_words)
logger.info('There are ' + str(num) + ' words I have known')
return known_words
def split_the_article(self, Article, name=None, LEMMATIZATION_flag=True):
'''
split the article
'''
sep = re.compile('[ \r\n.,'+self.config['SPECIAL_PUNCTUATION']+' ]')
logger.debug(sep)
words = re.split(sep, Article)
filcts = (self.real_word(word,LEMMATIZATION_flag=LEMMATIZATION_flag) for word in words)
set_of_words = set(filcts)
if name == None:
pass
else:
logger.info('there are {} words in {}'.format(
len(set_of_words), name))
logger.debug(set_of_words)
return set_of_words
def read_new_words(self):
'''
read new words from article
'''
new_words = self.words - self.known_words
num = len(new_words)
if num == 0:
logger.info('No new word')
elif num == 1:
logger.info('only 1 new word')
else:
logger.info(str(num) + ' new words')
logger.info(new_words)
return sorted(new_words)
def learn(self):
'''
learn new words & build
'''
logger.info('if you know the word {}, else print {}'.format(self.config['KEY_FOR_KNOW'],self.config['KEY_FOR_NOT']))
for word in self.new_words:
judge = my_input(word+'('+str(self.num)+' Left)',self.keys)
if judge == self.config['KEY_FOR_QUIT']:
self.user_exit()
return
if judge == self.config['KEY_FOR_KNOW']:
self.known_words.add(word)
self.num -= 1
self.new_words = sorted(set(self.new_words) - self.known_words)
if self.new_words:
logger.info('new words ({}):'.format(len(self.new_words)))
logger.info(self.new_words)
self.finish()
def user_exit(self):
write_each_words(
self.config['ARTICLES_PATH'], 'l_'+self.name, list(self.new_words)[-self.num:])
logger.debug('writing left words')
logger.debug(self.new_words[-self.num:])
logger.debug('get new words')
self.new_words = set(self.new_words[:-self.num])-self.known_words
logger.debug(self.new_words)
self.finish()
def finish(self):
CONFIG = self.config
NEW_WORDS_EACH_ARTICLE_PATH = CONFIG['MAIN_PATH'] + \
CONFIG['NEW_WORDS_EACH_ARTICLE_PATH']
safe_IO.mv_file(self.file_path, CONFIG['MAIN_PATH'] +
CONFIG['OLD_ARTICLES_PATH'])
safe_IO.write_each_words(
NEW_WORDS_EACH_ARTICLE_PATH, self.name, self.new_words)
with open(CONFIG['MAIN_PATH'] + CONFIG['OLD_WORDS_PATH'], 'w') as old:
old.write('\n'.join(self.known_words))
logger.debug('write new words to '+CONFIG['MAIN_PATH'] + CONFIG['NEW_WORDS_PATH'])
with open(CONFIG['MAIN_PATH'] + CONFIG['NEW_WORDS_PATH'], 'a') as new:
new.write('\n'.join(self.new_words))
