#!/usr/bin/env python
"""
CleanScraper.py
This module defines a series of functions to fetch html content from
from any publically-available URL, optionally clean it of any ads,
navigation bars, and other irrelevant boilerplate, and convert it
into a pdf document framed with wide margins, in an easy-to-read
format.
Edit the associated settings.py file to match your environment, or
use a local_settings.py file which is not checked into this repo.
"""
from readability.readability import Document
import pycurl
from cStringIO import StringIO
from string import Template
from datetime import datetime
import subprocess
import codecs
import sys
import os
from settings import ENCODING, UA, OUTPUT_FOLDER, \
PANDOC_PATH, EPUB_XML, ISBN_XML, EPUB_CSS, EPUB_COVER, EPUB_LANG, \
WKHTMLTOX_PATH, PDF_PAGE_SIZE, HTML_FRAME
EPUB_CONVERT_CMD = Template("""$pandoc_path/pandoc -f html -t epub --epub-metadata="$metadata_xml" -o $folder/$article_id.epub """)
PDF_CONVERT_CMD = Template("$wkhtmltox_path/wkhtmltopdf --page-size $page_size $folder/$article_id.html $folder/$article_id.pdf")
def get_url (url, user_agent=UA, referrer=None):
"""Make a GET request of the url using pycurl and return the data
(which is None if unsuccessful)"""
data = None
databuffer = StringIO()
curl = pycurl.Curl()
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.FOLLOWLOCATION, 1)
curl.setopt(pycurl.CONNECTTIMEOUT, 5)
curl.setopt(pycurl.TIMEOUT, 8)
curl.setopt(pycurl.WRITEFUNCTION, databuffer.write)
curl.setopt(pycurl.COOKIEFILE, '')
if user_agent:
curl.setopt(pycurl.USERAGENT, user_agent)
if referrer is not None:
curl.setopt(pycurl.REFERER, referrer)
try:
curl.perform()
data = databuffer.getvalue()
except Exception:
pass
curl.close()
return data
def to_unicode (s, enc=ENCODING):
"""Convert the given string to unicode, using the requested encoding,
(unless it's already in unicode), then return it"""
if isinstance(s, basestring):
if not isinstance(s, unicode):
s = unicode(s, enc)
return s
def datestring (display_format="%a, %d %b %Y %H:%M:%S", datetime_object=None):
"""Convert the datetime.date object (defaults to now, in utc) into a string,
in the given display format"""
if datetime_object is None:
datetime_object = datetime.utcnow()
return datetime.strftime(datetime_object, display_format)
def generate_epub_xml (epub_title, lang, isbn):
"""Return the contents of an xml file to be used in the
'--epub-metadata' parameter in the epub conversion command"""
id_string = ''
if isbn is not None and len(isbn) > 0:
id_string = ISBN_XML.substitute(isbn_number=isbn)
return EPUB_XML.substitute(title=epub_title,
lang=lang,
date=datestring(display_format="%Y-%m-%dT%H:%M:%S"), # iso 8601 format
identifier=id_string)
def write_file (folder, filename, contents):
"""Attempt to write the contents to the filename in the tmp_folder
as a utf-8 file, and return a boolean on success/fail"""
result = False
try:
f = codecs.open(os.path.join(folder, filename), 'w', ENCODING)
f.write(contents)
f.close()
result = True
except (OSError, IOError):
print "Sorry, could not save contents in", os.path.join(folder, filename)
return result
def generate_epub (file_folder, filename, epub_title, html_file, css_file, cover_image, lang, isbn):
"""Use pandoc to convert the html file and associated metadata into an epub file"""
metadata = generate_epub_xml(epub_title, lang, isbn)
if write_file(file_folder, 'metadata.xml', metadata):
pandoc_cmd = EPUB_CONVERT_CMD.substitute(pandoc_path=PANDOC_PATH,
folder=file_folder,
metadata_xml=os.path.join(file_folder, 'metadata.xml'),
article_id=filename)
if cover_image is not None:
pandoc_cmd += '--epub-cover-image="' + cover_image +'" '
pandoc_cmd += '-s --smart --parse-raw '
pandoc_cmd += os.path.join(file_folder, html_file)
proc = subprocess.Popen(pandoc_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_value, stderr_value = proc.communicate()
print u'\n'.join(filter(None, [pandoc_cmd, stdout_value, stderr_value])), '\n'
def generate_pdf (tmp_folder, filename, pdf_page_size):
"""Use wkhtmltopdf to convert the html file at tmp_folder/filename
into a pdf file, and return the stdout and stderr results"""
shell_cmd = PDF_CONVERT_CMD.substitute(wkhtmltox_path=WKHTMLTOX_PATH, folder=tmp_folder, article_id=filename, page_size=pdf_page_size)
proc = subprocess.Popen(shell_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_value, stderr_value = proc.communicate()
print u'\n'.join(filter(None, [shell_cmd, stdout_value, stderr_value]))
def scrape (url, pdf_filename, pdf_page_size=PDF_PAGE_SIZE, folder=OUTPUT_FOLDER, clean_it=True,
css_file=EPUB_CSS, lang=EPUB_LANG, cover_image=EPUB_COVER, isbn=None):
"""Fetch the html content at url and convert it to a pdf file,
cleaned by readability and framed in an easy-to-read format if
clean_it is True"""
raw_html = get_url(url)
if raw_html is None:
print "Sorry, could not read ", url
else:
filename_prefix, file_ext = os.path.splitext(pdf_filename)
if clean_it:
# use readability to get rid of crap
title = Document(raw_html).short_title()
content = Document(raw_html).summary(html_partial=True)
# write the cleaned contents to an html frame for pdf conversion
frame = HTML_FRAME.substitute(content=to_unicode(content),
url=url,
title=title)
# unlike pdf, epub is controlled by css, so save the cleaned html alone
epub_source = write_file(folder, os.extsep.join([filename_prefix+'_epub', 'html']), to_unicode(content))
pdf_source = write_file(folder, os.extsep.join([filename_prefix, 'html']), frame)
else:
title = filename_prefix
# no readability cleaning requested, so use the fetched html as-is
epub_source = write_file(folder, os.extsep.join([filename_prefix+'_epub', 'html']), to_unicode(raw_html))
pdf_source = write_file(folder, os.extsep.join([filename_prefix, 'html']), to_unicode(raw_html))
if epub_source:
generate_epub (folder, filename_prefix, title,
os.path.join(folder, os.extsep.join([filename_prefix+'_epub', 'html'])),
css_file, cover_image, lang, isbn)
if pdf_source:
generate_pdf (folder, filename_prefix, pdf_page_size)
if __name__ == "__main__":
"""Create a command-line main() entry point"""
if len(sys.argv) < 3:
# Define the usage
print sys.argv[0], '[URL]', '[PDF & EPUB filename]', '--noclean (optional: leave the source html found at the URL as-is)'
else:
# Do the deed
clean_html = True
try:
if sys.argv[3] == '--noclean':
clean_html = False
except IndexError:
pass
scrape(sys.argv[1], sys.argv[2], clean_it=clean_html)
