• Search by Module
• Search by Words
• Search Projects

• Java
• Python
• JavaScript
• TypeScript
• C++
• Scala
• Blog

• python-examples-master
  • .circleci
    • config.yml
  • pdfquery-example.py
  • triplot-example.py
  • polyglot-example.py
  • ngrams-example.py
  • merge-pdfs-example.py
  • base64-example.py
  • sentiment_analysis_nltk-example.py
  • scrape_twitter-example.py
  • textract-example.py
  • djvu-pdf-example.py
  • request_post_go_with_hug_post-example.py
  • circles-example.py
  • smash-site-example.py
  • fuzzywuzzy-example.py
  • scatterplot-example.py
  • ryu-example.py
  • json-example.py
  • hug-postgresql-example.py
  • geoname-example.py
  • quandl-example.py
  • spotify-example.py
  • shodan-example.py
  • imaplib-example.py
  • 4sq-example.py
  • tika-example-too.py
  • requests-example.py
  • bitcoin-example-1.py
  • flask-example.py
  • exif_reader-example.py
  • server-example.py
  • mechanize-example.py
  • Pipfile
  • arabic_dict-example.py
  • tuple-example.py
  • tkinter-example.py
  • stem_tor-example.py
  • test_tor-example.py
  • scapy_arp-example.py
  • hug_post_server-example.py
  • tika-get-text-example.py
  • html2plaintext-example.py
  • pyzillow-example.py
  • browser-example.py
  • pickle_load-example.py
  • websockify-example.py
  • bs4_email_regex-example.py
  • setup.py
  • audio_waveform-example.py
  • list.txt
  • parse_divs-example.py
  • cvlib_example.py
  • read_wav_display_audio-example.py
  • read-spreadsheet-example.py
  • get_geo-example.py
  • rethinkdb-example.py
  • Pipfile.lock
  • hug_api_example.py
  • argparse-example.py
  • setup.cfg
  • httpserver-example.py
  • .travis.yml
  • README.md
  • tor-example.py
  • hashing_example.py
  • urllib3_proxymanager-example.py
  • spider.py
  • iptcinfo3-example.py
  • pdf2random_text-example.py
  • http.html
  • requirements.txt
  • amazon_ec2_boto-example.py
  • .gitignore
  • pinboard-example.py
  • LICENSE.txt
  • tika-example.py
  • instagram_geo-example.py
  • speech-example.py
  • all.py
  • multi-categorization-tweets-example.py
  • chrome-headless-example.py
  • assets
    • testimage.jpg~
    • test.xls
    • discordia.json
    • archive_name.tar.gz
    • testtweets.txt
    • .DS_Store
    • iptc-example.jpg~
    • index.html
    • discordia.pkl
    • cnn.txt
  • opencv_facial_recognition-example.py

#!/usr/bin/python3
# Author: James Campbell
# Date: November 11 2016
# What: Example using nltk tokenize and ngrams
from nltk.tokenize import word_tokenize
from nltk.util import ngrams
import re
import string


def get_ngrams(text, n):
    ngramnums = word_tokenize(text)
    ll = [x for x in ngramnums if not re.fullmatch('[' + string.punctuation + ']+', x)]
    ll = ngrams(ll, n)
    return [' '.join(grams) for grams in ll]


ngramer = get_ngrams("This is a sentence to parse out ngrams for it.", 4)
for gram in ngramer:
    print(gram)