from urlparse import urlparse
from bs4 import BeautifulSoup
import requests
import io
import json
import praw
def scrape_quora(url):
source_code = requests.get(url).text
soup = BeautifulSoup(source_code, "html.parser")
output = dict()
output['question'] = soup.find('div', {'class': 'question_text_edit'}).find('span', {'class': 'rendered_qtext'}).get_text()
output['answers'] = list()
for answer in soup.findAll('div', {'class': 'AnswerBase'}):
ans = dict()
try:
ans['author'] = answer.find('a', {'class': 'user'}).get_text()
except:
ans['author'] = "Hidden"
ans['content'] = answer.find('span', {'class': 'rendered_qtext'}).get_text()
output['answers'].append(ans)
json_object = json.dumps(output, ensure_ascii=False, indent=4)
io.open('quora_output.json', 'w').write(json_object)
def scrape_reddit(url):
reddit = praw.Reddit(user_agent='Comment Extraction (by /u/USERNAME)',
client_id='INSERT CLIENT ID', client_secret="INSERT CLIENT SECRET")
submission = reddit.submission(url=URL2)
output = dict()
output['Submission'] = submission.title
output['Comments'] = list()
submission.comments.replace_more(limit=0)
for top_level_comment in submission.comments:
ans = dict()
try:
ans['Author'] = top_level_comment.author.name
except:
ans['Author'] = "Hidden"
ans['content'] = top_level_comment.body
output['Comments'].append(ans)
json_object = json.dumps(output, ensure_ascii=False, indent=4)
io.open('reddit_output.json', 'w').write(json_object)
scrape_reddit('https://www.reddit.com/r/AskReddit/comments/4lx5a9/serious_what_is_the_creepiest_most_blood_chilling/')
