For manipulating text.

from sklearn.feature_extraction.text import TfidfTransformer, CountVectorizer, HashingVectorizer
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import Normalizer
from broca.vectorize import Vectorizer
from broca.tokenize import LemmaTokenizer

class Tokenizer():
    Wrap broca tokenizers
    def __init__(self, tokenizer):
        self.tokenizer = tokenizer
    def __call__(self, doc):
        return self.tokenizer.tokenize([doc])[0]

class BoWVectorizer(Vectorizer):
    def __init__(self, min_df=1, max_df=0.9, tokenizer=LemmaTokenizer, hash=False):
        `min_df` is set to filter out extremely rare words,
        since we don't want those to dominate the distance metric.

        `max_df` is set to filter out extremely common words,
        since they don't convey much information.

        # Wrap the specified tokenizer
        t = Tokenizer(tokenizer())

        if hash:
            vectr = HashingVectorizer(input='content', stop_words='english', lowercase=True, tokenizer=t)
            vectr = CountVectorizer(input='content', stop_words='english', lowercase=True, tokenizer=t, min_df=min_df, max_df=max_df)

        args = [
            ('vectorizer', vectr),
            ('tfidf', TfidfTransformer(norm=None, use_idf=True, smooth_idf=True)),
            ('normalizer', Normalizer(copy=False))

        self.pipeline = Pipeline(args)
        self.trained = False

    def vectorize(self, docs):
        if not self.trained:
            return self.train(docs)
        return self.pipeline.transform(docs)

    def train(self, docs):
        self.trained = True
        return self.pipeline.fit_transform(docs)

    def vocabulary(self):
        return self.pipeline.named_steps['vectorizer'].get_feature_names()