/*
 *
 * Copyright (c) 2013 - 2020 Lijun Liao
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.xipki.ca.mgmt.db.port;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.math.BigInteger;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.security.cert.CRLException;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;

import org.bouncycastle.asn1.ASN1Integer;
import org.bouncycastle.asn1.x509.Extension;
import org.bouncycastle.cert.X509CRLHolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xipki.datasource.DataAccessException;
import org.xipki.datasource.DataSourceWrapper;
import org.xipki.security.HashAlgo;
import org.xipki.security.util.X509Util;
import org.xipki.util.Args;
import org.xipki.util.Base64;
import org.xipki.util.InvalidConfException;
import org.xipki.util.IoUtil;
import org.xipki.util.LogUtil;
import org.xipki.util.ProcessLog;
import org.xipki.util.StringUtil;

import com.alibaba.fastjson.JSON;

/**
 * Database exporter of CA CertStore.
 *
 * @author Lijun Liao
 * @since 2.0.0
 */

class CaCertstoreDbExporter extends DbPorter {

  private static final Logger LOG = LoggerFactory.getLogger(CaCertstoreDbExporter.class);

  private final int numCertsInBundle;

  private final int numCertsPerSelect;

  private final boolean resume;

  CaCertstoreDbExporter(DataSourceWrapper datasource, String baseDir, int numCertsInBundle,
      int numCertsPerSelect, boolean resume, AtomicBoolean stopMe)
          throws DataAccessException {
    super(datasource, baseDir, stopMe);

    this.numCertsInBundle = Args.positive(numCertsInBundle, "numCertsInBundle");
    this.numCertsPerSelect = Args.positive(numCertsPerSelect, "numCertsPerSelect");
    this.resume = resume;
  } // constructor

  public void export() throws Exception {
    CaCertstore certstore;
    if (resume) {
      try (InputStream is = Files.newInputStream(Paths.get(baseDir, FILENAME_CA_CERTSTORE))) {
        certstore = JSON.parseObject(is, CaCertstore.class);
      }
      certstore.validate();

      if (certstore.getVersion() > VERSION) {
        throw new Exception("could not continue with CertStore greater than "
            + VERSION + ": " + certstore.getVersion());
      }
    } else {
      certstore = new CaCertstore();
      certstore.setVersion(VERSION);
    }

    Exception exception = null;
    System.out.println("exporting CA certstore from database");
    try {
      if (!resume) {
        exportPublishQueue(certstore);
        exportDeltaCrlCache(certstore);
      }

      File processLogFile = new File(baseDir, DbPorter.EXPORT_PROCESS_LOG_FILENAME);

      Long idProcessedInLastProcess = null;
      CaDbEntryType typeProcessedInLastProcess = null;
      if (processLogFile.exists()) {
        byte[] content = IoUtil.read(processLogFile);
        if (content != null && content.length > 0) {
          String str = new String(content);
          int idx = str.indexOf(':');
          String typeName = str.substring(0, idx).trim();
          typeProcessedInLastProcess = CaDbEntryType.valueOf(typeName);
          idProcessedInLastProcess = Long.parseLong(str.substring(idx + 1).trim());
        }
      }

      if (CaDbEntryType.CRL == typeProcessedInLastProcess || typeProcessedInLastProcess == null) {
        exception = exportEntries(CaDbEntryType.CRL, certstore, processLogFile,
            idProcessedInLastProcess);
        typeProcessedInLastProcess = null;
        idProcessedInLastProcess = null;
      }

      CaDbEntryType[] types = {CaDbEntryType.CERT, CaDbEntryType.REQUEST, CaDbEntryType.REQCERT};

      for (CaDbEntryType type : types) {
        if (exception == null
            && (type == typeProcessedInLastProcess || typeProcessedInLastProcess == null)) {
          exception = exportEntries(type, certstore, processLogFile, idProcessedInLastProcess);
          typeProcessedInLastProcess = null;
          idProcessedInLastProcess = null;
        }
      }

      certstore.validate();
      try (OutputStream os = Files.newOutputStream(Paths.get(baseDir, FILENAME_CA_CERTSTORE))) {
        JSON.writeJSONString(os, Charset.forName("UTF-8"), certstore);
      }
    } catch (Exception ex) {
      System.err.println("could not export CA certstore from database");
      exception = ex;
    }

    if (exception == null) {
      System.out.println(" exported CA certstore from database");
    } else {
      throw exception;
    }
  } // method export

  private Exception exportEntries(CaDbEntryType type, CaCertstore certstore,
      File processLogFile, Long idProcessedInLastProcess) {
    String tablesText = "table " + type.getTableName();

    File dir = new File(baseDir, type.getDirName());
    dir.mkdirs();

    OutputStream entriesFileOs = null;
    try {
      entriesFileOs = Files.newOutputStream(Paths.get(baseDir, type.getDirName() + ".mf"),
          StandardOpenOption.CREATE, StandardOpenOption.APPEND);
      exportEntries(type, certstore, processLogFile, entriesFileOs, idProcessedInLastProcess);
      return null;
    } catch (Exception ex) {
      // delete the temporary files
      deleteTmpFiles(baseDir, "tmp-");

      System.err.println("\nexporting " + tablesText + " has been cancelled due to error,\n"
          + "please continue with the option '--resume'");
      LOG.error("Exception", ex);
      return ex;
    } finally {
      IoUtil.closeQuietly(entriesFileOs);
    }
  } // method exportEntries

  private void exportEntries(CaDbEntryType type, CaCertstore certstore, File processLogFile,
      OutputStream filenameListOs, Long idProcessedInLastProcess) throws Exception {
    // CHECKSTYLE:SKIP
    int numEntriesPerSelect = Math.max(1, Math.round(type.getSqlBatchFactor() * numCertsPerSelect));
    int numEntriesPerZip = Math.max(1, Math.round(type.getSqlBatchFactor() * numCertsInBundle));
    File entriesDir = new File(baseDir, type.getDirName());
    String tableName = type.getTableName();

    int numProcessedBefore;
    String coreSql;

    switch (type) {
      case CERT:
        numProcessedBefore = certstore.getCountCerts();
        coreSql = "ID,SN,CA_ID,PID,RID,RTYPE,TID,UID,EE,LUPDATE,REV,RR,RT,RIT,FP_RS,"
            + "REQ_SUBJECT,CRL_SCOPE,CERT FROM CERT WHERE ID>=?";
        break;
      case CRL:
        numProcessedBefore = certstore.getCountCrls();
        coreSql = "ID,CA_ID,CRL_SCOPE,CRL FROM CRL WHERE ID>=?";
        break;
      case REQUEST:
        numProcessedBefore = certstore.getCountRequests();
        coreSql = "ID,LUPDATE,DATA FROM REQUEST WHERE ID>=?";
        break;
      case REQCERT:
        numProcessedBefore = certstore.getCountReqCerts();
        coreSql = "ID,RID,CID FROM REQCERT WHERE ID>=?";
        break;
      default:
        throw new IllegalStateException("unknown CaDbEntryType " + type);
    }

    Long minId = (idProcessedInLastProcess != null) ? idProcessedInLastProcess + 1
        : min(tableName, "ID");

    String tablesText = "table " + type.getTableName();
    System.out.println("exporting " + tablesText + " from ID " + minId);

    final long maxId = max(tableName, "ID");
    long total = count(tableName) - numProcessedBefore;
    if (total < 1) {
      total = 1; // to avoid exception
    }

    String sql = datasource.buildSelectFirstSql(numEntriesPerSelect, "ID ASC", coreSql);

    Object entriesInCurrentFile = createContainer(type);
    PreparedStatement ps = prepareStatement(sql.toString());

    int numEntriesInCurrentFile = 0;

    int sum = 0;
    File currentEntriesZipFile = new File(baseDir,
        "tmp-" + type.getDirName() + "-" + System.currentTimeMillis() + ".zip");
    ZipOutputStream currentEntriesZip = getZipOutputStream(currentEntriesZipFile);

    long minIdOfCurrentFile = -1;
    long maxIdOfCurrentFile = -1;

    ProcessLog processLog = new ProcessLog(total);
    processLog.printHeader();

    try {
      Long id = null;
      boolean interrupted = false;
      long lastMaxId = minId - 1;

      while (true) {
        if (stopMe.get()) {
          interrupted = true;
          break;
        }

        ps.setLong(1, lastMaxId + 1);

        ResultSet rs = ps.executeQuery();

        // no entries anymore
        if (!rs.next()) {
          break;
        }

        do {
          id = rs.getLong("ID");
          if (lastMaxId < id) {
            lastMaxId = id;
          }

          if (minIdOfCurrentFile == -1) {
            minIdOfCurrentFile = id;
          } else if (minIdOfCurrentFile > id) {
            minIdOfCurrentFile = id;
          }

          if (maxIdOfCurrentFile == -1) {
            maxIdOfCurrentFile = id;
          } else if (maxIdOfCurrentFile < id) {
            maxIdOfCurrentFile = id;
          }

          if (CaDbEntryType.CERT == type) {
            byte[] certBytes = Base64.decodeFast(rs.getString("CERT"));

            String sha1 = HashAlgo.SHA1.hexHash(certBytes);
            String certFileName = sha1 + ".der";
            ZipEntry certZipEntry = new ZipEntry(certFileName);
            currentEntriesZip.putNextEntry(certZipEntry);
            try {
              currentEntriesZip.write(certBytes);
            } finally {
              currentEntriesZip.closeEntry();
            }

            CaCertstore.Cert cert = new CaCertstore.Cert();
            cert.setId(id);
            cert.setCaId(rs.getInt("CA_ID"));
            cert.setEe(rs.getBoolean("EE"));
            cert.setFile(certFileName);

            long fpReqSubject = rs.getLong("FP_RS");
            if (fpReqSubject != 0) {
              cert.setFpRs(fpReqSubject);
              cert.setRs(rs.getString("REQ_SUBJECT"));
            }

            cert.setPid(rs.getInt("PID"));
            cert.setReqType(rs.getInt("RTYPE"));
            cert.setRid(rs.getInt("RID"));
            cert.setSn(rs.getString("SN"));

            String str = rs.getString("TID");
            if (StringUtil.isNotBlank(str)) {
              cert.setTid(str);
            }

            int userId = rs.getInt("UID");
            if (userId != 0) {
              cert.setUid(userId);
            }
            cert.setUpdate(rs.getLong("LUPDATE"));

            int revoked = rs.getInt("REV");
            cert.setRev(revoked);

            if (revoked == 1) {
              cert.setRr(rs.getInt("RR"));
              cert.setRt(rs.getLong("RT"));
              long revInvTime = rs.getLong("RIT");
              if (revInvTime != 0) {
                cert.setRit(revInvTime);
              }
            }

            cert.setCrlScope(rs.getInt("CRL_SCOPE"));

            cert.validate();
            ((CaCertstore.Certs) entriesInCurrentFile).add(cert);
          } else if (CaDbEntryType.CRL == type) {
            byte[] crlBytes = Base64.decodeFast(rs.getString("CRL"));

            X509CRLHolder x509Crl = null;
            try {
              x509Crl = X509Util.parseCrl(crlBytes);
            } catch (CRLException ex) {
              LogUtil.error(LOG, ex, "could not parse CRL with id " + id);
              throw ex;
            } catch (Exception ex) {
              LogUtil.error(LOG, ex, "could not parse CRL with id " + id);
              throw new CRLException(ex.getMessage(), ex);
            }

            byte[] extnValue = X509Util.getCoreExtValue(x509Crl.getExtensions(),
                                  Extension.cRLNumber);
            if (extnValue == null) {
              LOG.warn("CRL without CRL number, ignore it");
              continue;
            }
            String sha1 = HashAlgo.SHA1.hexHash(crlBytes);

            final String crlFilename = sha1 + ".crl";
            ZipEntry certZipEntry = new ZipEntry(crlFilename);
            currentEntriesZip.putNextEntry(certZipEntry);
            try {
              currentEntriesZip.write(crlBytes);
            } finally {
              currentEntriesZip.closeEntry();
            }

            CaCertstore.Crl crl = new CaCertstore.Crl();
            crl.setId(id);

            crl.setCaId(rs.getInt("CA_ID"));

            BigInteger crlNumber = ASN1Integer.getInstance(extnValue).getPositiveValue();
            crl.setCrlNo(crlNumber.toString());
            crl.setCrlScope(rs.getInt("CRL_SCOPE"));
            crl.setFile(crlFilename);

            crl.validate();
            ((CaCertstore.Crls) entriesInCurrentFile).add(crl);
          } else if (CaDbEntryType.REQUEST == type) {
            byte[] dataBytes = Base64.decodeFast(rs.getString("DATA"));
            String sha1 = HashAlgo.SHA1.hexHash(dataBytes);
            final String dataFilename = sha1 + ".req";
            ZipEntry certZipEntry = new ZipEntry(dataFilename);
            currentEntriesZip.putNextEntry(certZipEntry);
            try {
              currentEntriesZip.write(dataBytes);
            } finally {
              currentEntriesZip.closeEntry();
            }

            CaCertstore.Request entry = new CaCertstore.Request();
            entry.setId(id);
            entry.setUpdate(rs.getLong("LUPDATE"));
            entry.setFile(dataFilename);

            entry.validate();
            ((CaCertstore.Requests) entriesInCurrentFile).add(entry);
          } else if (CaDbEntryType.REQCERT == type) {
            CaCertstore.ReqCert entry = new CaCertstore.ReqCert();
            entry.setId(id);
            entry.setCid(rs.getLong("CID"));
            entry.setRid(rs.getLong("RID"));

            entry.validate();
            ((CaCertstore.ReqCerts) entriesInCurrentFile).add(entry);
          } else {
            throw new IllegalStateException("unknown CaDbEntryType " + type);
          }

          numEntriesInCurrentFile++;
          sum++;

          if (numEntriesInCurrentFile == numEntriesPerZip) {
            String currentEntriesFilename = buildFilename(type.getDirName() + "_", ".zip",
                minIdOfCurrentFile, maxIdOfCurrentFile, maxId);
            finalizeZip(currentEntriesZip, "overview.json", entriesInCurrentFile);
            currentEntriesZipFile.renameTo(new File(entriesDir, currentEntriesFilename));

            writeLine(filenameListOs, currentEntriesFilename);
            setCount(type, certstore, numProcessedBefore + sum);
            echoToFile(tableName + ":" + Long.toString(id), processLogFile);

            processLog.addNumProcessed(numEntriesInCurrentFile);
            processLog.printStatus();

            // reset
            entriesInCurrentFile = createContainer(type);
            numEntriesInCurrentFile = 0;
            minIdOfCurrentFile = -1;
            maxIdOfCurrentFile = -1;
            currentEntriesZipFile = new File(baseDir, "tmp-" + type.getDirName() + "-"
                + System.currentTimeMillis() + ".zip");
            currentEntriesZip = getZipOutputStream(currentEntriesZipFile);
          }
        } while (rs.next());

        rs.close();
      } // end for

      if (interrupted) {
        currentEntriesZip.close();
        throw new InterruptedException("interrupted by the user");
      }

      if (numEntriesInCurrentFile > 0) {
        finalizeZip(currentEntriesZip, "overview.json", entriesInCurrentFile);

        String currentEntriesFilename = buildFilename(type.getDirName() + "_", ".zip",
            minIdOfCurrentFile, maxIdOfCurrentFile, maxId);
        currentEntriesZipFile.renameTo(new File(entriesDir, currentEntriesFilename));

        writeLine(filenameListOs, currentEntriesFilename);
        setCount(type, certstore, numProcessedBefore + sum);
        if (id != null) {
          echoToFile(Long.toString(id), processLogFile);
        }

        processLog.addNumProcessed(numEntriesInCurrentFile);
      } else {
        currentEntriesZip.close();
        currentEntriesZipFile.delete();
      }

    } catch (SQLException ex) {
      throw translate(null, ex);
    } finally {
      releaseResources(ps, null);
    } // end try

    processLog.printTrailer();
    // all successful, delete the processLogFile
    processLogFile.delete();
    System.out.println(" exported " + sum + " entries from " + tablesText);
  } // method exportEntries

  private void exportPublishQueue(CaCertstore certstore)
      throws DataAccessException, InvalidConfException {
    System.out.println("exporting table PUBLISHQUEUE");

    String sql = "SELECT CID,PID,CA_ID FROM PUBLISHQUEUE WHERE CID>=? AND CID<? ORDER BY CID ASC";
    final int minId = (int) min("PUBLISHQUEUE", "CID");
    final int maxId = (int) max("PUBLISHQUEUE", "CID");

    List<CaCertstore.ToPublish> queue = new LinkedList<>();
    certstore.setPublishQueue(queue);
    if (maxId == 0) {
      System.out.println(" exported table PUBLISHQUEUE");
      return;
    }

    PreparedStatement ps = prepareStatement(sql);
    ResultSet rs = null;

    final int n = 500;

    try {
      for (int i = minId; i <= maxId; i += n) {
        ps.setInt(1, i);
        ps.setInt(2, i + n);

        rs = ps.executeQuery();

        while (rs.next()) {
          CaCertstore.ToPublish toPub = new CaCertstore.ToPublish();
          toPub.setPubId(rs.getInt("PID"));
          toPub.setCertId(rs.getInt("CID"));
          toPub.setCaId(rs.getInt("CA_ID"));

          toPub.validate();
          queue.add(toPub);
        }
      }
    } catch (SQLException ex) {
      throw translate(sql, ex);
    } finally {
      releaseResources(ps, rs);
    }
    System.out.println(" exported table PUBLISHQUEUE");
  } // method exportPublishQueue

  private void exportDeltaCrlCache(CaCertstore certstore)
      throws DataAccessException, InvalidConfException {
    System.out.println("exporting table DELTACRL_CACHE");

    final String sql = "SELECT SN,CA_ID FROM DELTACRL_CACHE";

    List<CaCertstore.DeltaCrlCacheEntry> deltaCache = new LinkedList<>();
    certstore.setDeltaCrlCache(deltaCache);

    PreparedStatement ps = prepareStatement(sql);
    ResultSet rs = null;

    try {
      rs = ps.executeQuery();

      while (rs.next()) {
        CaCertstore.DeltaCrlCacheEntry entry = new CaCertstore.DeltaCrlCacheEntry();
        entry.setCaId(rs.getInt("CA_ID"));
        entry.setSerial(rs.getString("SN"));

        entry.validate();
        deltaCache.add(entry);
      }
    } catch (SQLException ex) {
      throw translate(sql, ex);
    } finally {
      releaseResources(ps, rs);
    }

    System.out.println(" exported table DELTACRL_CACHE");
  } // method exportDeltaCrlCache

  private void finalizeZip(ZipOutputStream zipOutStream, String filename, Object container)
      throws IOException {
    ZipEntry certZipEntry = new ZipEntry(filename);
    zipOutStream.putNextEntry(certZipEntry);
    try {
      JSON.writeJSONString(zipOutStream, Charset.forName("UTF-8"), container);
    } finally {
      zipOutStream.closeEntry();
    }

    zipOutStream.close();
  } // method finalizeZip

  private static Object createContainer(CaDbEntryType type) throws IOException {
    switch (type) {
      case CERT:
        return new CaCertstore.Certs();
      case CRL:
        return new CaCertstore.Crls();
      case REQUEST:
        return new CaCertstore.Requests();
      case REQCERT:
        return new CaCertstore.ReqCerts();
      default:
        throw new IllegalStateException("unknown CaDbEntryType " + type);
    }
  } // method createContainer

  private static void setCount(CaDbEntryType type, CaCertstore certstore, int num) {
    switch (type) {
      case CERT:
        certstore.setCountCerts(num);
        break;
      case CRL:
        certstore.setCountCrls(num);
        break;
      case REQUEST:
        certstore.setCountRequests(num);
        break;
      case REQCERT:
        certstore.setCountReqCerts(num);
        break;
      default:
        throw new IllegalStateException("unknown CaDbEntryType " + type);
    }
  } // method setCount
}