/*************************************************************************
*                                                                        *
*  This file is part of the 20n/act project.                             *
*  20n/act enables DNA prediction for synthetic biology/bioengineering.  *
*  Copyright (C) 2017 20n Labs, Inc.                                     *
*                                                                        *
*  Please direct all queries to [email protected].                             *
*                                                                        *
*  This program is free software: you can redistribute it and/or modify  *
*  it under the terms of the GNU General Public License as published by  *
*  the Free Software Foundation, either version 3 of the License, or     *
*  (at your option) any later version.                                   *
*                                                                        *
*  This program is distributed in the hope that it will be useful,       *
*  but WITHOUT ANY WARRANTY; without even the implied warranty of        *
*  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
*  GNU General Public License for more details.                          *
*                                                                        *
*  You should have received a copy of the GNU General Public License     *
*  along with this program.  If not, see <http://www.gnu.org/licenses/>. *
*                                                                        *
*************************************************************************/

package com.twentyn.patentExtractor;

import com.fasterxml.jackson.annotation.JsonProperty;
import org.apache.commons.io.input.ReaderInputStream;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.TextNode;
import org.jsoup.select.NodeTraversor;
import org.jsoup.select.NodeVisitor;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathExpressionException;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;

/**
 * This class represents parts of a USPTO patent document that are relevant to 20's use cases.  It can extract
 * information from the USPTO's XML documents and convert it to a POJO that can then be serialized as JSON.  Use this
 * as the basis for any processing of patent text.
 */
public class PatentDocument {

  public static final Logger LOGGER = LogManager.getLogger(PatentDocument.class);

  // See http://www.uspto.gov/learning-and-resources/xml-resources.
  public static final String DTD2014 = "v4.5 2014-04-03";
  public static final String DTD2013 = "v4.4 2013-05-16";
  public static final String DTD2012 = "v4.3 2012-12-04";
  public static final String DTD2006 = "v4.2 2006-08-23";
  public static final String DTD2005 = "v4.1 2005-08-25";
  public static final String DTD2004 = "v40 2004-12-02";

  public static final String DTD2014_APP = "v4.4 2014-04-03";
  public static final String DTD2012_APP = "v4.3 2012-12-04";
  public static final String DTD2006_APP = "v4.2 2006-08-23";
  public static final String DTD2005_APP = "v4.1 2005-08-25";
  public static final String DTD2004_APP = "v4.0 2004-12-02";

  public static final String PATH_DTD_VERSION = "/us-patent-grant/@dtd-version";
  public static final String PATH_DTD_VERSION_APP = "/us-patent-application/@dtd-version";
  public static final String[] PATHS_TEXT = {
      "//description",
      "//invention-title",
      "//abstract",
  };
  public static final String PATH_CLAIMS = "//claims";

  public static final String
      PATH_KEY_FILE_ID = "fileId",
      PATH_KEY_TITLE = "title",
      PATH_KEY_DATE = "date",
      PATH_KEY_MAIN_CLASSIFICATION = "classification",
      PATH_KEY_FURTHER_CLASSIFICATIONS = "further_classifications",
      PATH_KEY_SEARCHED_CLASSIFICATIONS = "referenced_classifications";

  // TODO: is there a type-safe way of building an object from XPath with a map of functions?
  public static final HashMap<String, String> PATHS_2013 = new HashMap<String, String>() {{
    put(PATH_KEY_FILE_ID, "/us-patent-grant/@file");
    put(PATH_KEY_TITLE, "/us-patent-grant/us-bibliographic-data-grant/invention-title");
    put(PATH_KEY_DATE, "/us-patent-grant/@date-publ");
    put(PATH_KEY_MAIN_CLASSIFICATION,
        "/us-patent-grant/us-bibliographic-data-grant/classification-national/main-classification/text()");
    put(PATH_KEY_FURTHER_CLASSIFICATIONS,
        "/us-patent-grant/us-bibliographic-data-grant/classification-national/further-classification");
    put(PATH_KEY_SEARCHED_CLASSIFICATIONS,
        "/us-patent-grant/us-bibliographic-data-grant/us-field-of-classification-search/classification-national[./country/text()='US']/main-classification");
  }};

  public static final HashMap<String, String> PATHS_2004 = new HashMap<String, String>() {{
    put(PATH_KEY_FILE_ID, "/us-patent-grant/@file");
    put(PATH_KEY_TITLE, "/us-patent-grant/us-bibliographic-data-grant/invention-title");
    put(PATH_KEY_DATE, "/us-patent-grant/@date-publ");
    put(PATH_KEY_MAIN_CLASSIFICATION,
        "/us-patent-grant/us-bibliographic-data-grant/classification-national/main-classification/text()");
    put(PATH_KEY_FURTHER_CLASSIFICATIONS,
        "/us-patent-grant/us-bibliographic-data-grant/classification-national/further-classification");
    put(PATH_KEY_SEARCHED_CLASSIFICATIONS,
        "/us-patent-grant/us-bibliographic-data-grant/field-of-search/classification-national[./country/text()='US']/main-classification");
  }};

  public static final HashMap<String, String> PATHS_2014_APP = new HashMap<String, String>() {{
    put(PATH_KEY_FILE_ID, "/us-patent-application/@file");
    put(PATH_KEY_TITLE, "/us-patent-application/us-bibliographic-data-application/invention-title");
    put(PATH_KEY_DATE, "/us-patent-application/@date-publ");
    put(PATH_KEY_MAIN_CLASSIFICATION,
        "/us-patent-application/us-bibliographic-data-application/classification-national/main-classification/text()");
    put(PATH_KEY_FURTHER_CLASSIFICATIONS,
        "/us-patent-application/us-bibliographic-data-application/classification-national/further-classification");
    put(PATH_KEY_SEARCHED_CLASSIFICATIONS, // Note: doesn't exist, but left for ease of use.
        "/us-patent-application/us-bibliographic-data-application/us-field-of-classification-search/classification-national[./country/text()='US']/main-classification");
  }};

  public static final HashMap<String, HashMap<String, String>> VERSION_MAP =
      new HashMap<String, HashMap<String, String>>() {{
        put(DTD2014, PATHS_2013); // All the 2013 paths work with the 2014 DTD.
        put(DTD2013, PATHS_2013);
        put(DTD2012, PATHS_2013); // All the 2013 paths work with the 2012 DTD.
        put(DTD2006, PATHS_2013); // All the 2013 paths work with the 2006 DTD.
        put(DTD2005, PATHS_2013); // All the 2013 paths work with the 2005 DTD.
        put(DTD2004, PATHS_2004);
        put(DTD2014_APP, PATHS_2014_APP);
        put(DTD2012_APP, PATHS_2014_APP); // All the 2014 app paths work with the 2012 app DTD.
        put(DTD2006_APP, PATHS_2014_APP); // All the 2014 app paths work with the 2006 app DTD.
        put(DTD2005_APP, PATHS_2014_APP); // All the 2014 app paths work with the 2005 app DTD, though the classifications might be different.
        put(DTD2004_APP, PATHS_2014_APP); // All the 2014 app paths work with the 2005 app DTD assuming searched classifications are always empty.
      }};

  private static final Pattern GZIP_PATTERN = Pattern.compile("\\.gz$");

  public static class HtmlVisitor implements NodeVisitor {
    // Based on https://github.com/jhy/jsoup/blob/master/src/main/java/org/jsoup/examples/HtmlToPlainText.java
    private static final HashSet<String> SEGMENTING_NODES = new HashSet<String>() {{
      addAll(Arrays.asList(
          "p", "h1", "h2", "h3", "h4", "h5", "h6", "dt", "dd", "tr", "li", "body", "div", // HTML entities
          "row", "claim" // patent-specific entities
      ));
    }};
    private static final Pattern SPACE_PATTERN = Pattern.compile("^\\s+$");

    private StringBuilder segmentBuilder = new StringBuilder();
    private List<String> textSegments = new LinkedList<>();

    @Override
    public void head(org.jsoup.nodes.Node node, int i) {
      // This borrows a page from HtmlToPlainText's book.
      if (node instanceof TextNode) {
        String text = ((TextNode) node).text();
        if (text != null && text.length() > 0) {
          segmentBuilder.append(((TextNode) node).text());
        }
      }
    }

    @Override
    public void tail(org.jsoup.nodes.Node node, int i) {
      String nodeName = node.nodeName();
      if (nodeName.equals("a")) {
        // Same as Jsoup's HtmlToPlainText.
        segmentBuilder.append(String.format(" <%s>", node.absUrl("href")));
      } else if (SEGMENTING_NODES.contains(nodeName) && segmentBuilder.length() > 0) {
        String segmentText = segmentBuilder.toString();
        // Ignore blank lines, as we'll be tagging each line separately.
        if (!SPACE_PATTERN.matcher(segmentText).matches()) {
          this.textSegments.add(segmentText);
        }
        // TODO: is it better to drop the old one than clear the existing?
        segmentBuilder.setLength(0);
      }
    }

    public List<String> getTextContent() {
      return this.textSegments;
    }
  }

  private static List<String> extractTextFromHTML(DocumentBuilder docBuilder, NodeList textNodes)
      throws ParserConfigurationException, TransformerConfigurationException,
      TransformerException, XPathExpressionException {
    List<String> allTextList = new ArrayList<>(0);
    if (textNodes != null) {
      for (int i = 0; i < textNodes.getLength(); i++) {
        Node n = textNodes.item(i);
                    /* This extremely around-the-horn approach to handling text content is due to the mix of HTML and
                     * XML in the patent body.  We use Jsoup to parse the HTML entities we find in the body, and use
                     * its extremely convenient NodeVisitor API to recursively traverse the document and extract the
                     * text content in reasonable chunks.
                     */
        Document contentsDoc = Util.nodeToDocument(docBuilder, "body", n);
        String docText = Util.documentToString(contentsDoc);
        // With help from http://stackoverflow.com/questions/832620/stripping-html-tags-in-java
        org.jsoup.nodes.Document htmlDoc = Jsoup.parse(docText);
        HtmlVisitor visitor = new HtmlVisitor();
        NodeTraversor traversor = new NodeTraversor(visitor);
        traversor.traverse(htmlDoc);
        List<String> textSegments = visitor.getTextContent();
        allTextList.addAll(textSegments);
      }
    }
    return allTextList;
  }

  /**
   * Extracts the text content from text fields in a patent XML document.
   *
   * @param docBuilder A document builder to use when constructing intermediate XML/HTML documents in the extraction
   *                   process.
   * @param paths      A list of XPath paths from which to exactract text.
   * @param xpath      An XPath instance to use when running XPath queries.
   * @param doc        The XML document from which to extract text.
   * @return A list of strings representing the textual content of the document.  These could be sentences,
   * paragraphs, or larger text units, but should represent some sort of structure in the document's text.
   * @throws ParserConfigurationException
   * @throws TransformerConfigurationException
   * @throws TransformerException
   * @throws XPathExpressionException
   */
  private static List<String> getRelevantDocumentText(DocumentBuilder docBuilder, String[] paths,
                                                      XPath xpath, Document doc)
      throws ParserConfigurationException, TransformerConfigurationException,
      TransformerException, XPathExpressionException {
    List<String> allTextList = new ArrayList<>(0);
    for (String path : paths) {
      XPathExpression exp = xpath.compile(path);
      NodeList textNodes = (NodeList) exp.evaluate(doc, XPathConstants.NODESET);
      allTextList.addAll(extractTextFromHTML(docBuilder, textNodes));
    }

    return allTextList;
  }

  /**
   * Converts an XML file into a patent document object, extracting relevant fields from the patent XML.
   *
   * @param inputPath A path to the file to be read.
   * @return A patent object if the XML can be read, or null otherwise.
   * @throws IOException                  Thrown on file I/O errors.
   * @throws ParserConfigurationException Thrown when the XML parser cannot be configured correctly.
   * @throws SAXException                 Thrown on XML parser errors.
   * @throws XPathExpressionException     Thrown when XPath fails to handle queries against the specified document.
   */
  // TODO: logging?
  // TODO: are @nullable and @non-null annotations still a thing?
  // TODO: prolly belongs in a factory.
  public static PatentDocument patentDocumentFromXMLFile(File inputPath)
      throws IOException, ParserConfigurationException,
      SAXException, TransformerConfigurationException,
      TransformerException, XPathExpressionException {
    InputStream iStream = null;

    iStream = new BufferedInputStream(new FileInputStream(inputPath));
    if (GZIP_PATTERN.matcher(inputPath.getName()).find()) {
      iStream = new GZIPInputStream(iStream);
    }
    return patentDocumentFromXMLStream(iStream);
  }

  /**
   * Converts a string of XML into a patent document object, extracting relevant fields from the patent XML.
   *
   * @param text The XML string to parse and extract.
   * @return A patent object if the XML can be read, or null otherwise.
   * @throws IOException
   * @throws ParserConfigurationException
   * @throws SAXException
   * @throws TransformerConfigurationException
   * @throws TransformerException
   * @throws XPathExpressionException
   */
  public static PatentDocument patentDocumentFromXMLString(String text)
      throws IOException, ParserConfigurationException,
      SAXException, TransformerConfigurationException,
      TransformerException, XPathExpressionException {
    StringReader stringReader = new StringReader(text);
    return patentDocumentFromXMLStream(new ReaderInputStream(stringReader));
  }

  public static PatentDocument patentDocumentFromXMLStream(InputStream iStream)
      throws IOException, ParserConfigurationException,
      SAXException, TransformerConfigurationException,
      TransformerException, XPathExpressionException {

    // Create XPath objects for validating that this document is actually a patent.
    XPath xpath = Util.getXPathFactory().newXPath();
    XPathExpression versionXPath = xpath.compile(PATH_DTD_VERSION);
    XPathExpression versionXPathApp = xpath.compile(PATH_DTD_VERSION_APP);

    DocumentBuilderFactory docFactory = Util.mkDocBuilderFactory();
    DocumentBuilder docBuilder = docFactory.newDocumentBuilder();
    Document doc = docBuilder.parse(iStream);

    Util.DocumentType docType = Util.identifyDocType(doc);
    if (docType != Util.DocumentType.PATENT && docType != Util.DocumentType.APPLICATION) {
      LOGGER.warn("Found unexpected document type: " + docType);
      return null;
    }

    boolean isApplication = docType == Util.DocumentType.APPLICATION;
    // Yes this is in fact the way suggested by the XPath API.
    String version;
    if (!isApplication) {
      version = (String) versionXPath.evaluate(doc, XPathConstants.STRING);
    } else {
      version = (String) versionXPathApp.evaluate(doc, XPathConstants.STRING);
    }

    if (version == null || !VERSION_MAP.containsKey(version)) {
      LOGGER.warn(String.format("Unrecognized patent DTD version: %s", version));
      return null;
    }

    HashMap<String, String> paths = VERSION_MAP.get(version);

    /* Create XPath objects for extracting the fields of interest based on the version information.
     * TODO: extract these into some sharable, thread-safe place, maybe via dependency injection.
     */
    XPathExpression idXPath = xpath.compile(paths.get(PATH_KEY_FILE_ID));
    XPathExpression dateXPath = xpath.compile(paths.get(PATH_KEY_DATE));
    XPathExpression titleXPath = xpath.compile(paths.get(PATH_KEY_TITLE));
    XPathExpression classificationXPath = xpath.compile(paths.get(PATH_KEY_MAIN_CLASSIFICATION));
    XPathExpression furtherClassificationsXPath = xpath.compile(paths.get(PATH_KEY_FURTHER_CLASSIFICATIONS));
    XPathExpression searchedClassificationsXPath = xpath.compile(paths.get(PATH_KEY_SEARCHED_CLASSIFICATIONS));

    String fileId = (String) idXPath.evaluate(doc, XPathConstants.STRING);
    String date = (String) dateXPath.evaluate(doc, XPathConstants.STRING);
    NodeList titleNodes = (NodeList) titleXPath.evaluate(doc, XPathConstants.NODESET);
    String title = StringUtils.join(" ", extractTextFromHTML(docBuilder, titleNodes));
    String classification = (String) classificationXPath.evaluate(doc, XPathConstants.STRING);
    NodeList furtherClassificationNodes =
        (NodeList) furtherClassificationsXPath.evaluate(doc, XPathConstants.NODESET);
    ArrayList<String> furtherClassifications = null;
    if (furtherClassificationNodes != null) {
      furtherClassifications = new ArrayList<>(furtherClassificationNodes.getLength());
      for (int i = 0; i < furtherClassificationNodes.getLength(); i++) {
        Node n = furtherClassificationNodes.item(i);
        String txt = n.getTextContent();
        if (txt != null) {
          furtherClassifications.add(i, txt);
        }
      }
    } else {
      furtherClassifications = new ArrayList<>(0);
    }

    NodeList otherClassificationNodes =
        (NodeList) searchedClassificationsXPath.evaluate(doc, XPathConstants.NODESET);
    ArrayList<String> otherClassifications = null;
    if (otherClassificationNodes != null) {
      otherClassifications = new ArrayList<>(otherClassificationNodes.getLength());
      for (int i = 0; i < otherClassificationNodes.getLength(); i++) {
        Node n = otherClassificationNodes.item(i);
        String txt = n.getTextContent();
        if (txt != null) {
          otherClassifications.add(i, txt);
        }
      }
    } else {
      otherClassifications = new ArrayList<>(0);
    }

    // Extract text content for salient document paths.
    List<String> allTextList = getRelevantDocumentText(docBuilder, PATHS_TEXT, xpath, doc);
    List<String> claimsTextList = getRelevantDocumentText(docBuilder, new String[]{PATH_CLAIMS}, xpath, doc);

    return new PatentDocument(fileId, date, title, classification,
        furtherClassifications, otherClassifications, allTextList, claimsTextList, isApplication);
  }

  @JsonProperty("file_id")
  protected String fileId;
  @JsonProperty("grant_date")
  protected String grantDate;
  @JsonProperty("title")
  protected String title;
  @JsonProperty("primary_classification")
  protected String mainClassification;
  @JsonProperty("further_classification")
  protected List<String> furtherClassifications;
  @JsonProperty("searched_classifications")
  protected List<String> searchedClassifications;
  @JsonProperty("text_content")
  protected List<String> textContent;
  @JsonProperty("claims")
  protected List<String> claimsText;
  @JsonProperty("isApplication")
  protected Boolean isApplication;

  // TODO: this could probably use a builder if it gets more complicated.

  protected PatentDocument(String fileId, String grantDate, String title, String mainClassification,
                           List<String> furtherClassifications, List<String> searchedClassifications,
                           List<String> textContent, List<String> claimsText, Boolean isApplication) {
    this.fileId = fileId;
    this.grantDate = grantDate;
    this.title = title;
    this.mainClassification = mainClassification;
    this.furtherClassifications = furtherClassifications;
    this.searchedClassifications = searchedClassifications;
    this.textContent = textContent;
    this.claimsText = claimsText;
    this.isApplication = isApplication;
  }

  public String getFileId() {
    return fileId;
  }

  public String getGrantDate() {
    return grantDate;
  }

  public String getTitle() {
    return title;
  }

  public String getMainClassification() {
    return mainClassification;
  }

  public List<String> getFurtherClassifications() {
    return furtherClassifications;
  }

  public List<String> getSearchedClassifications() {
    return searchedClassifications;
  }

  public List<String> getTextContent() {
    return textContent;
  }

  public List<String> getClaimsText() {
    return claimsText;
  }

  public Boolean getIsApplication() {
    return isApplication;
  }
}