Copyright 2018 IBM Corporation

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at


   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   See the License for the specific language governing permissions and
   limitations under the License.

import os
import re
import pandas as pd
import numpy as np
from gensim.models import doc2vec
from gensim.utils import simple_preprocess

    Parse through every xlsx file in the specified path. For every sheet in each file, 
    create a pandas data frame and if a column has key words representing undp goals, keep the information in 
    that column.

        path (string)          : Directory to the xlsx documents (Current directory by default).
        exclude (list[string]) : List of files to be excluded from data extraction

        data (list[pandas.DataFrame]) : list of pandas data frames. Each data frame has two columns, one being 
                                        the target undpand the other being text identified as matching the target.
def extract_template_data(path = '.', exclude = []):
    data = []
    found = False
    for file in os.listdir(path):
        if file[-4:] == 'xlsx' and '~' not in file and file not in exclude:
            file_path = os.path.join(path, file)
            xls = pd.ExcelFile(file_path)
            for sheet in range(len(xls.sheet_names)):
                template = pd.read_excel(file_path, header = None, sheetname = sheet)
                template.fillna('', inplace = True)
                for col in range(len(template.columns)):
                        if 'Identify closest' in str(template.iloc[0][col]) or 'Identify closest' in str(template.iloc[1][col]):
                            keep = col
                            found = True
                        elif 'National Development Plan' in str(template.iloc[1][col]) and not found:
                            keep = col
                            found = True
                if found:
                    data.append(template[[template.keys()[1], template.keys()[keep]]])
                found = False

    return data

    Creates a dictionary of target : list[sentences], where each sentence was identified to match the target.

        development_matches (pandas DataFrame): Data frame with 2 columns, one being the undp target and the other 
                                                being the sentence/paragraph identified to match the target

        target_matches (dict): Dictionary of target:list[sentences]
def create_target_dictionary(development_matches):
    sgd_target_pattern = r'[0-9]+\.[0-9]+' # pattern to match target format
    target_matches = {}
    for development_match in development_matches:
        development_match.replace(np.nan, '', regex=True, inplace = True)
        target = None

        for row in development_match.itertuples():
            match = re.search(sgd_target_pattern, str(row[1]), flags=0)
            if match: # If we found a undp target
                target = match.group()
                if target in target_matches: # Add sentence to the set for that target's key
                    target_matches[target] = set({row[1][len(target):]})
            # Continue adding to the current target's key if there is text in the data frame
            if target != None and row[2] != '':
    return target_matches

    Parse through every english text file in the specified path. For every line in each file,
    pre-process the line (convert to lower case, remove punctuation). Only yield doc2Vec 
    TaggedDocuments for every processed line in which there are more than 10 words and over
    half the words have more than 3 characters.

        path (string)          : Directory to the text documents (Current directory by default).
        exclude (list[string]) : List of files to be excluded

        doc2vec.TaggedDocument : Doc2Vec Tagged Document object of the next processed line with a unique id. 
def read_corpus(path = '.', exclude = [], targets = None):
    i= 0
    for file in os.listdir(path):
        if file[-4:] == '.txt' and file not in exclude and 'no_en' not in file: # ensure file is an english txt file
            with open(os.path.join(path, file),  encoding="utf8") as document_text:
                for line in document_text:
                    count = 0
                    words = simple_preprocess(line)
                    for word in words: # count the number of words with <= 3 characters
                        if len(word) <= 3:
                            count += 1
                    if count < len(words)/2 and len(words) > 10: # exclude lines in which 1/2 the words have less 
                        yield(doc2vec.TaggedDocument(words, [i])) # than 3 characters or have less than 10 words
    if targets:
        for key, val in targets.items():
            yield(doc2vec.TaggedDocument(simple_preprocess(val), [i]))