#!/usr/bin/python
"""
get-lec.py
Maxim Aleksa
maximal@umich.edu
Downloads lectures for a course.
Usage: ./dleccap.py
"""
# command-line arguments
import argparse
# HTTP stuff
# import cookielib
# import urllib
# import urllib2
import requests
# import json
import json
# file system access
import os
# I/O
import sys
# file downloads
import wget
# password input
from getpass import getpass
# parsing HTML
# pip install beautifulsoup4
from bs4 import BeautifulSoup
# globals
session = requests.Session()
"""
Prints message in red.
"""
def print_error(message):
print '\033[31m{}\033[0m'.format(str(message))
"""
Prints message in yellow.
"""
def print_warning(message):
print '\033[33m{}\033[0m'.format(str(message))
"""
Prints message in green.
"""
def print_success(message):
print '\033[32m{}\033[0m'.format(str(message))
"""
Finds string between two substrings.
Thanks to http://stackoverflow.com/questions/3368969/find-string-between-two-substrings
"""
def find_between(s, first, last):
try:
start = s.index(first) + len(first)
end = s.index(last, start)
return s[start:end]
except:
return None
"""
Authenticates user for specified service.
"""
def authenticate_for(service, username, password):
# URL
authentication_url = "https://weblogin.umich.edu/cosign-bin/cosign.cgi"
# POST parameters
data_to_post = {
"service": service,
"required": "",
"login": username,
"password": password
}
# get cookie
p = session.get("https://weblogin.umich.edu")
# authenticate
p = session.post(authentication_url, data_to_post)
"""
Gets leccap cookie via CTools.
Returns site ID.
"""
def get_cookie_and_site_id_from_ctools(url):
# check auth status
if not session.cookies.get("cosign-weblogin"):
print_warning("Authentication required.")
authenticate("cosign-ctools")
try:
# get CTools leccap page
p = session.get(url)
# find url of page that gets cookie
soup = BeautifulSoup(p.text, "html.parser")
iframe = soup.find("iframe")
iframeURL = "https://ctools.umich.edu" + iframe.get("src")
# find form that gets cookie
p = session.get(iframeURL)
soup = BeautifulSoup(p.text, "html.parser")
form = soup.find(id="ltiLaunchForm")
inputs = form.find_all("input")
data_to_post = {}
for input_element in inputs:
data_to_post[input_element.get("name")] = input_element.get("value")
# get cookie
auth_url = form.get("action")
p = session.post(auth_url, data_to_post)
# get site ID
soup = BeautifulSoup(p.text, "html.parser")
recording_url = soup.find("div", {"class": "recording"}).find("div", {"class": "play-link"}).find("a").get("href")
return get_site_id_from_view_url("https://leccap.engin.umich.edu" + recording_url)
except:
return None
"""
Gets leccap cookie via Canvas.
Returns site ID.
"""
def get_cookie_and_site_id_from_canvas(url):
# check auth status
if not session.cookies.get("cosign-weblogin"):
print_warning("Authentication required.")
authenticate("cosign-shibboleth.umich.edu")
try:
# get Canvas leccap page
p = session.get(url)
# print p.text
# find form on the page that will get the cookie
soup = BeautifulSoup(p.text, "html.parser")
form = soup.find("form")
inputs = form.find_all("input")
data_to_post = {}
for input_element in inputs:
data_to_post[input_element.get("name")] = input_element.get("value")
print data_to_post
# get cookie
auth_url = form.get("action")
p = session.post(auth_url, data_to_post)
print p.text
# get site ID
soup = BeautifulSoup(p.text, "html.parser")
recording_url = soup.find("div", {"class": "recording"}).find("div", {"class": "play-link"}).find("a").get("href")
return get_site_id_from_view_url("https://leccap.engin.umich.edu" + recording_url)
except:
return None
"""
Gets username and password from user.
"""
def authenticate(service=None):
# get auth data from user
username = raw_input("Uniqname: ")
password = getpass("Password: ")
print "Authenticating...\n"
if not service:
services = ["cosign-ctools", "cosign-shibboleth.umich.edu", "cosign-leccap.engin"]
for service in services:
authenticate_for(service, username, password)
else:
authenticate_for(service, username, password)
"""
Gets information about a recording from its ID.
"""
def get_recording_from_id(recording_id):
# URL
recording_info_url = "https://leccap.engin.umich.edu/leccap/viewer/api/product/?rk=" + recording_id
# check auth status
if not session.cookies.get("cosign-weblogin"):
print_warning("Authentication required.")
authenticate()
try:
# get recording info
r = session.get(recording_info_url)
recording_info = json.loads(r.text)
return recording_info
except:
print r.text
return None
"""
Returns site ID of a recording viewable at specified URL.
"""
def get_site_id_from_view_url(url):
# followed by recording ID
str3 = "//leccap.engin.umich.edu/leccap/viewer/r/"
recording = get_recording_from_id(url.split(str3, 1)[1])
if not recording:
return None
else:
return recording["sitekey"]
"""
Parses url and retrieves ID of the recordings site.
"""
def get_site_id(url):
# followed by site ID
str1 = "//leccap.engin.umich.edu/leccap/site/"
str2 = "//leccap.engin.umich.edu/leccap/viewer/s/"
# followed by recording ID
str3 = "//leccap.engin.umich.edu/leccap/viewer/r/"
# CTools page
str4 = "//ctools.umich.edu/portal/site/"
# Canvas page
str5 = "//umich.instructure.com/courses/"
# parse
if str1 in url:
return url.split(str1, 1)[1]
elif str2 in url:
return url.split(str2, 1)[1]
elif str3 in url:
# more complicated
return get_site_id_from_view_url(url)
elif str4 in url:
return get_cookie_and_site_id_from_ctools(url)
elif str5 in url:
print_error(":( Cannot download recordings published to Canvas (yet)!")
return
# return get_cookie_and_site_id_from_canvas(url)
else:
return None
"""
Gets recordings for a site with the specified ID.
"""
def get_recordings_for_site(site_id):
# URL
site_contents_url = "https://leccap.engin.umich.edu/leccap/viewer/api/recordings/?sk=" + site_id
# check auth status
if not session.cookies.get("cosign-weblogin"):
print_warning("Authentication required.")
authenticate()
try:
# get recordings data
r = session.get(site_contents_url)
recordings = json.loads(r.text)
return recordings
except:
return None
def download_recoding(recording, dest_folder=None):
# construct url
url = "https:%s%s/%s.%s" % (recording["mediaPrefix"],
recording["sitekey"],
recording["info"]["movie_exported_name"],
recording["info"]["movie_type"])
if dest_folder is None:
dest_folder = os.path.realpath(recording["sitename"])
title = recording["title"].replace("/", "-")
filename = "%s.%s" % (title, recording["info"]["movie_type"])
destination = os.path.join(dest_folder, filename)
# download!
wget.download(url, out=destination)
def main():
# parse command-line arguments
parser = argparse.ArgumentParser()
args = parser.parse_args()
# ask for any URL
ctools_sample_url = "https://ctools.umich.edu/portal/site/123/page/456"
canvas_sample_url = "https://umich.instructure.com/courses/123/external_tools/1262"
leccap_sample_url1 = "https://leccap.engin.umich.edu/leccap/site/123"
leccap_sample_url2 = "https://leccap.engin.umich.edu/leccap/viewer/r/123"
leccap_sample_url3 = "https://leccap.engin.umich.edu/leccap/viewer/s/123"
print "Please enter a URL of any video in the recording series.\n"
# print "If from Canvas, URL should look like\n%s\n" % (canvas_sample_url)
print "If from CTools, URL should look like\n%s\n" % (ctools_sample_url)
print "If neither from Canvas or from CTools, URL should look like\n%s or\n%s or\n%s\n" % (leccap_sample_url1, leccap_sample_url2, leccap_sample_url3)
any_url = raw_input()
print
# parse side ID
site_id = get_site_id(any_url)
if not site_id:
print_error(":( Could not parse URL.")
print "This could happen becuase the URL is invalid, because username or password is invalid, or becuase you do not have access to the recordings."
return
# get recordings for site
print "Getting recordings info...\n"
recordings = get_recordings_for_site(site_id)
if not recordings:
print_error(":( Could not get recordings.")
print "This could happen becuase username or password is invalid, or becuase you do not have access to the recordings."
return
elif len(recordings) == 0:
print_error(":( No recordings available.")
return
print "%i recording%s available:" % (len(recordings), ("s" if (len(recordings) != 1) else ""))
# print lecture info
for index, recording in enumerate(recordings):
print "%i\t(%s)\t%s" % (index + 1, recording["date"], recording["title"])
print
# which recordings to download?
print "Which recording would you like to download?"
input = raw_input("Enter a number between %i and %i, or press Enter to download all: " % (1, len(recordings)))
try:
input_as_int = int(input)
if input_as_int < 1 or input_as_int > len(recordings):
raise Exception("Invalid input")
recordings = recordings[input_as_int - 1 : input_as_int]
except:
if input != "":
# user did not press enter
print_error(":( Invalid input")
return
# create folder for recordings
course = recordings[0]["sitename"]
directory = course
if not os.path.exists(directory):
os.makedirs(directory)
# download files
for index, recording in enumerate(recordings):
print "(%i/%i) Downloading %s ..." % (index + 1, len(recordings), recording["title"])
download_recoding(recording)
print
print_success("Finished!")
if __name__ == '__main__':
main()
