/**
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
**/
/**
This file is part of Save For Offline, an Android app which saves / downloads complete webpages for offine reading.
**/
/**
If you modify, redistribute, or write something based on this or parts of it, you MUST,
I repeat, you MUST comply with the GPLv2+ license. This means that if you use or modify
my code, you MUST release the source code of your modified version, if / when this is
required under the terms of the license.
If you cannot / do not want to do this, DO NOT USE MY CODE. Thanks.
(I've added this message to to the source because it's been used in severeral proprietary
closed source apps, which I don't want, and which is also a violation of the liense.)
**/
/**
Written by Jonas Czech (JonasCz, stackoverflow.com/users/4428462/JonasCz and github.com/JonasCz) originally and partially based on https://github.com/PramodKhare/GetMeThatPage/
with lots of improvements. (4428462jonascz/eafc4d1afq)
**/
package jonas.tool.saveForOffline;
import com.squareup.okhttp.Cache;
import com.squareup.okhttp.OkHttpClient;
import com.squareup.okhttp.Request;
import com.squareup.okhttp.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Entities;
import org.jsoup.select.Elements;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class PageSaver {
private EventCallback eventCallback;
private OkHttpClient client = new OkHttpClient();
private final String HTTP_REQUEST_TAG = "TAG";
private boolean isCancelled = false;
private Options options = new Options();
// filesToGrab - maintains all the links to files (eg images, scripts) which we are going to grab/download
private List<String> filesToGrab = new ArrayList<String>();
//framesToGrab - list of html frame files to download, as we parse these recursively
private List<String> framesToGrab = new ArrayList<String>();
//cssToGrab - list of all css files to download and parse, these need to be parsed to extract urls
private List<String> cssToGrab = new ArrayList<String>();
private String title = "";
private String pageIconUrl = "";
private String indexFileName = "index.html";
private final Pattern fileNameReplacementPattern = Pattern.compile("[^a-zA-Z0-9-_\\.]");
public Options getOptions() {
return this.options;
}
public String getPageTitle () {
return this.title;
}
public PageSaver(EventCallback callback) {
this.eventCallback = callback;
client.setConnectTimeout(20, TimeUnit.SECONDS);
client.setReadTimeout(20, TimeUnit.SECONDS);
client.setWriteTimeout(20, TimeUnit.SECONDS);
client.setFollowRedirects(true);
client.setFollowSslRedirects(true);
}
public void cancel() {
this.isCancelled = true;
client.cancel(HTTP_REQUEST_TAG);
}
public void resetState () {
filesToGrab.clear();
framesToGrab.clear();
cssToGrab.clear();
title = "";
pageIconUrl = "";
isCancelled = false;
}
public boolean isCancelled () {
return this.isCancelled;
}
public boolean getPage(String url, String outputDirPath, String indexFilename) {
this.indexFileName = indexFilename;
File outputDir = new File(outputDirPath);
if (!outputDir.exists() && outputDir.mkdirs() == false) {
eventCallback.onFatalError(new IOException("File " + outputDirPath + "could not be created"), url);
return false;
}
//download main html and parse -- isExtra parameter should be false
boolean success = downloadHtmlAndParseLinks(url, outputDirPath, false);
if (isCancelled || !success) {
return false;
}
//download and parse html frames - use iterator because our list may be modified as frames can contain other frames
for (Iterator<String> i = framesToGrab.iterator(); i.hasNext();) {
downloadHtmlAndParseLinks(i.next(), outputDirPath, true);
if (isCancelled) return true;
}
//download and parse css files
for (Iterator<String> i = cssToGrab.iterator(); i.hasNext();) {
if (isCancelled) return true;
downloadCssAndParse(i.next(), outputDirPath);
}
ThreadPoolExecutor pool = new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors(), Runtime.getRuntime().availableProcessors(), 60, TimeUnit.SECONDS, new BlockingDownloadTaskQueue<Runnable>());
for (Iterator<String> i = filesToGrab.iterator(); i.hasNext();) {
if (isCancelled) {
eventCallback.onProgressMessage("Cancelling...");
shutdownExecutor(pool, 10, TimeUnit.SECONDS);
return success;
}
String urlToDownload = i.next();
eventCallback.onProgressMessage("Saving file: " + getFileName(urlToDownload));
eventCallback.onProgressChanged(filesToGrab.indexOf(urlToDownload), filesToGrab.size(), false);
pool.submit(new DownloadTask(urlToDownload, outputDir));
}
pool.submit(new DownloadTask(pageIconUrl, outputDir, "saveForOffline_icon.png"));
eventCallback.onProgressMessage("Finishing file downloads...");
shutdownExecutor(pool, 60, TimeUnit.SECONDS);
return success;
}
private boolean downloadHtmlAndParseLinks(final String url, final String outputDir, final boolean isExtra) {
//isExtra should be true when saving a html frame file.
String filename;
if (isExtra) {
filename = getFileName(url);
} else {
filename = indexFileName;
}
String baseUrl = url;
if (url.endsWith("/")) {
baseUrl = url + filename;
}
try {
eventCallback.onProgressMessage(isExtra ? "Getting HTML frame file: " + filename : "Getting main HTML file");
String htmlContent = getStringFromUrl(url);
eventCallback.onProgressMessage(isExtra ? "Processing HTML frame file: " + filename: "Processing main HTML file");
htmlContent = parseHtmlForLinks(htmlContent, baseUrl);
eventCallback.onProgressMessage(isExtra ? "Saving HTML frame file: " + filename: "Saving main HTML file");
File outputFile = new File(outputDir, filename);
saveStringToFile(htmlContent, outputFile);
return true;
} catch (IOException | IllegalStateException e) {
if (isExtra) {
eventCallback.onError(e);
} else {
eventCallback.onFatalError(e, url);
}
e.printStackTrace();
return false;
}
}
private void downloadCssAndParse(final String url, final String outputDir) {
String filename = getFileName(url);
File outputFile = new File(outputDir, filename);
try {
eventCallback.onProgressMessage("Getting CSS file: " + filename);
String cssContent = getStringFromUrl(url);
eventCallback.onProgressMessage("Processing CSS file: " + filename);
cssContent = parseCssForLinks(cssContent, url);
eventCallback.onProgressMessage("Saving CSS file: " + filename);
saveStringToFile(cssContent, outputFile);
} catch (IOException e) {
eventCallback.onError(e);
e.printStackTrace();
}
}
private class DownloadTask implements Runnable {
private String url;
private File outputDir;
private String fileName;
public DownloadTask(String url, File toPath) {
this.url = url;
this.outputDir = toPath;
}
public DownloadTask(String url, File toPath, String fileName) {
this.url = url;
this.outputDir = toPath;
this.fileName = fileName;
}
@Override
public void run() {
if (fileName == null) {
fileName = getFileName(url);
}
File outputFile = new File(outputDir, fileName);
Request request = new Request.Builder()
.url(url)
.addHeader("User-Agent", getOptions().getUserAgent())
.tag(HTTP_REQUEST_TAG)
.build();
try {
Response response = client.newCall(request).execute();
InputStream is = response.body().byteStream();
FileOutputStream fos = new FileOutputStream(outputFile);
final byte[] buffer = new byte[1024 * 32]; // read in batches of 32K
int length;
while ((length = is.read(buffer)) != -1) {
fos.write(buffer, 0, length);
}
response.body().close();
fos.flush();
fos.close();
is.close();
} catch (IllegalArgumentException | IOException e) {
IOException ex = new IOException("File download failed, URL: " + url + ", Output file path: " + outputFile.getPath());
if (isCancelled) {
ex.initCause(new IOException("Save was cancelled, isCancelled is true").initCause(e));
eventCallback.onError(ex);
} else {
eventCallback.onError(ex.initCause(e));
}
}
}
}
private String getStringFromUrl(String url) throws IOException, IllegalStateException {
Request request = new Request.Builder()
.url(url)
.addHeader("User-Agent", getOptions().getUserAgent())
.tag(HTTP_REQUEST_TAG)
.build();
Response response = client.newCall(request).execute();
String out = response.body().string();
response.body().close();
return out;
}
private void saveStringToFile(String ToSave, File outputFile) throws IOException {
if (outputFile.exists()) {
return;
}
outputFile.createNewFile();
FileOutputStream fos = new FileOutputStream(outputFile);
fos.write(ToSave.getBytes());
fos.flush();
fos.close();
}
private String parseHtmlForLinks(String htmlToParse, String baseUrl) {
//get all links from this webpage and add them to LinksToVisit ArrayList
Document document = Jsoup.parse(htmlToParse, baseUrl);
document.outputSettings().escapeMode(Entities.EscapeMode.extended);
if (title.isEmpty()) {
title = document.title();
eventCallback.onPageTitleAvailable(title);
}
if (pageIconUrl.isEmpty()) {
eventCallback.onProgressMessage("Getting icon...");
pageIconUrl = FaviconFetcher.getInstance().getFaviconUrl(document);
}
eventCallback.onProgressMessage("Processing HTML...");
String urlToGrab;
Elements links;
if (getOptions().saveFrames()) {
links = document.select("frame[src]");
eventCallback.onLogMessage("Got " + links.size() + " frames");
for (Element link : links) {
urlToGrab = link.attr("abs:src");
addLinkToList(urlToGrab, framesToGrab);
String replacedURL = getFileName(urlToGrab);
link.attr("src", replacedURL);
}
links = document.select("iframe[src]");
eventCallback.onLogMessage("Got " + links.size() + " iframes");
for (Element link : links) {
urlToGrab = link.attr("abs:src");
addLinkToList(urlToGrab, framesToGrab);
String replacedURL = getFileName(urlToGrab);
link.attr("src", replacedURL);
}
}
if (getOptions().saveOther()) {
// Get all the links
links = document.select("link[href]");
eventCallback.onLogMessage("Got " + links.size() + " link elements with a href attribute");
for (Element link : links) {
urlToGrab = link.attr("abs:href");
//if it is css, parse it later to extract urls (images referenced from "background" attributes for example)
if (link.attr("rel").equals("stylesheet")) {
cssToGrab.add(link.attr("abs:href"));
} else {
addLinkToList(urlToGrab, filesToGrab);
}
String replacedURL = getFileName(urlToGrab);
link.attr("href", replacedURL);
}
//get links in embedded css also, and modify the links to point to local files
links = document.select("style[type=text/css]");
eventCallback.onLogMessage("Got " + links.size() + " embedded stylesheets, parsing CSS");
for (Element link : links) {
String cssToParse = link.data();
String parsedCss = parseCssForLinks(cssToParse, baseUrl);
if (link.dataNodes().size() != 0) {
link.dataNodes().get(0).setWholeData(parsedCss);
}
}
//get input types with an image type
links = document.select("input[type=image]");
eventCallback.onLogMessage("Got " + links.size() + " input elements with type = image");
for (Element link : links) {
urlToGrab = link.attr("abs:src");
addLinkToList(urlToGrab, filesToGrab);
String replacedURL = getFileName(urlToGrab);
link.attr("src", replacedURL);
}
//get everything which has a background attribute
links = document.select("[background]");
eventCallback.onLogMessage("Got " + links.size() + " elements with a background attribute");
for (Element link : links) {
urlToGrab = link.attr("abs:src");
addLinkToList(urlToGrab, filesToGrab);
String replacedURL = getFileName(urlToGrab);
link.attr("src", replacedURL);
}
links = document.select("[style]");
eventCallback.onLogMessage("Got " + links.size() + " elements with a style attribute, parsing CSS");
for (Element link : links) {
String cssToParse = link.attr("style");
String parsedCss = parseCssForLinks(cssToParse, baseUrl);
link.attr("style", parsedCss);
}
}
if (getOptions().saveScripts()) {
links = document.select("script[src]");
eventCallback.onLogMessage("Got " + links.size() + " script elements");
for (Element link : links) {
urlToGrab = link.attr("abs:src");
addLinkToList(urlToGrab, filesToGrab);
String replacedURL = getFileName(urlToGrab);
link.attr("src", replacedURL);
}
}
if (getOptions().saveImages()) {
links = document.select("img[src]");
eventCallback.onLogMessage("Got " + links.size() + " image elements");
for (Element link : links) {
urlToGrab = link.attr("abs:src");
addLinkToList(urlToGrab, filesToGrab);
String replacedURL = getFileName(urlToGrab);
link.attr("src", replacedURL);
link.removeAttr("srcset"); //we don't use this for now, so remove it.
}
links = document.select("img[data-canonical-src]");
eventCallback.onLogMessage("Got " + links.size() + " image elements, w. data-canonical-src");
for (Element link : links) {
urlToGrab = link.attr("abs:data-canonical-src");
addLinkToList(urlToGrab, filesToGrab);
String replacedURL = getFileName(urlToGrab);
link.attr("data-canonical-src", replacedURL);
link.removeAttr("srcset"); //we don't use this for now, so remove it.
}
}
if (getOptions().saveVideo()) {
//video src is sometimes in a child element
links = document.select("video:not([src])");
eventCallback.onLogMessage("Got " + links.size() + " video elements without src attribute");
for (Element link : links.select("[src]")) {
urlToGrab = link.attr("abs:src");
addLinkToList(urlToGrab, filesToGrab);
String replacedURL = getFileName(urlToGrab);
link.attr("src", replacedURL);
}
links = document.select("video[src]");
eventCallback.onLogMessage("Got " + links.size() + " video elements");
for (Element link : links) {
urlToGrab = link.attr("abs:src");
addLinkToList(urlToGrab, filesToGrab);
String replacedURL = getFileName(urlToGrab);
link.attr("src", replacedURL);
}
}
if (getOptions().makeLinksAbsolute()) {
//make links absolute, so they are not broken
links = document.select("a[href]");
eventCallback.onLogMessage("Making " + links.size() + " links absolute");
for (Element link : links) {
String absUrl = link.attr("abs:href");
link.attr("href", absUrl);
}
}
return document.outerHtml();
}
private String parseCssForLinks(String cssToParse, String baseUrl) {
String patternString = "url(\\s*\\(\\s*['\"]*\\s*)(.*?)\\s*['\"]*\\s*\\)"; //I hate regexes...
Pattern pattern = Pattern.compile(patternString);
Matcher matcher = pattern.matcher(cssToParse);
eventCallback.onLogMessage("Parsing CSS");
//find everything inside url(" ... ")
while (matcher.find()) {
if (matcher.group().replaceAll(patternString, "$2").contains("/")) {
cssToParse = cssToParse.replace(matcher.group().replaceAll(patternString, "$2"), getFileName(matcher.group().replaceAll(patternString, "$2")));
}
addLinkToList(matcher.group().replaceAll(patternString, "$2").trim(), baseUrl, filesToGrab);
}
// find css linked with @import - needs testing
//todo: test this to see if it actually works
String importString = "@(import\\s*['\"])()([^ '\"]*)";
pattern = Pattern.compile(importString);
matcher = pattern.matcher(cssToParse);
matcher.reset();
while (matcher.find()) {
if (matcher.group().replaceAll(patternString, "$2").contains("/")) {
cssToParse = cssToParse.replace(matcher.group().replaceAll(patternString, "$2"), getFileName(matcher.group().replaceAll(patternString, "$2")));
}
addLinkToList(matcher.group().replaceAll(patternString, "$2").trim(), baseUrl, cssToGrab);
}
return cssToParse;
}
private boolean isLinkValid (String url) {
if (url == null || url.length() == 0) {
return false;
} else if (!url.startsWith("http")) {
return false;
} else {
return true;
}
}
private void addLinkToList(String link, List<String> list) {
if (!isLinkValid(link) || list.contains(link)) {
return;
} else {
list.add(link);
}
}
private void addLinkToList(String link, String baseUrl, List<String> list) {
if (link.startsWith("data:image")) {
return;
}
try {
URL u = new URL(new URL(baseUrl), link);
link = u.toString();
} catch (MalformedURLException e) {
return;
}
if (!isLinkValid(link) || list.contains(link)) {
return;
} else {
list.add(link);
}
}
private String getFileName(String url) {
String filename = url.substring(url.lastIndexOf('/') + 1);
if (filename.trim().length() == 0) {
filename = String.valueOf(url.hashCode());
}
if (filename.contains("?")) {
filename = filename.substring(0, filename.indexOf("?")) + filename.substring(filename.indexOf("?") + 1).hashCode();
}
filename = fileNameReplacementPattern.matcher(filename).replaceAll("_");
filename = filename.substring(0, Math.min(200, filename.length()));
;
return filename;
}
private void shutdownExecutor (ExecutorService e, int waitTime, TimeUnit waitTimeUnit) {
e.shutdown();
try {
if (!e.awaitTermination(waitTime, waitTimeUnit)) {
eventCallback.onError("Executor pool did not termimate after " + waitTime + " " + waitTimeUnit.toString() +", doing shutdownNow()");
e.shutdownNow();
}
} catch (InterruptedException ie) {
eventCallback.onError(ie);
}
}
private class BlockingDownloadTaskQueue<E> extends SynchronousQueue<E> {
public BlockingDownloadTaskQueue () {
super();
}
@Override
public boolean offer (E e) {
try {
put(e);
return true;
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
eventCallback.onError(ie);
return false;
}
}
}
class Options {
private boolean makeLinksAbsolute = true;
private boolean saveImages = true;
private boolean saveFrames = true;
private boolean saveOther = true;
private boolean saveScripts = true;
private boolean saveVideo = false;
private String userAgent = " ";
public void setCache (File cacheDirectory, long maxCacheSize) {
Cache cache = (new Cache(cacheDirectory, maxCacheSize));
client.setCache(cache);
}
public void clearCache() throws IOException {
client.getCache().evictAll();
}
public String getUserAgent() {
return userAgent;
}
public void setUserAgent(final String userAgent) {
this.userAgent = userAgent;
}
public boolean makeLinksAbsolute() {
return makeLinksAbsolute;
}
public void makeLinksAbsolute(final boolean makeLinksAbsolute) {
this.makeLinksAbsolute = makeLinksAbsolute;
}
public boolean saveImages() {
return saveImages;
}
public void saveImages(final boolean saveImages) {
this.saveImages = saveImages;
}
public boolean saveFrames() {
return saveFrames;
}
public void saveFrames(final boolean saveFrames) {
this.saveFrames = saveFrames;
}
public boolean saveScripts() {
return saveScripts;
}
public void saveScripts(final boolean saveScripts) {
this.saveScripts = saveScripts;
}
public boolean saveOther() {
return saveOther;
}
public void saveOther(final boolean saveOther) {
this.saveOther = saveOther;
}
public boolean saveVideo() {
return saveVideo;
}
public void saveVideo(final boolean saveVideo) {
this.saveVideo = saveVideo;
}
}
}
interface EventCallback {
public void onProgressChanged(int progress, int maxProgress, boolean indeterminate);
public void onProgressMessage(String fileName);
public void onPageTitleAvailable (String pageTitle);
public void onLogMessage (String message);
public void onError(Throwable error);
public void onError(String errorMessage);
public void onFatalError (Throwable error, String pageUrl);
}
