/*
 * Copyright (c) 2010-2015 Pivotal Software, Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You
 * may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License. See accompanying
 * LICENSE file.
 */
package com.pivotal.gemfirexd.internal.tools.dataextractor;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.io.StringReader;
import java.math.BigDecimal;
import java.sql.Connection;
import java.sql.Date;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Time;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;

import com.pivotal.gemfirexd.Attribute;
import com.pivotal.gemfirexd.TestUtil;
import com.pivotal.gemfirexd.internal.impl.jdbc.EmbedConnection;
import com.pivotal.gemfirexd.internal.tools.dataextractor.domain.ServerInfo;
import com.pivotal.gemfirexd.internal.tools.dataextractor.domain.ServerInfo.Builder;
import com.pivotal.gemfirexd.internal.tools.dataextractor.extractor.GemFireXDDataExtractorImpl;
import com.pivotal.gemfirexd.internal.tools.dataextractor.report.ReportGenerator;
import com.pivotal.gemfirexd.internal.tools.dataextractor.snapshot.GFXDSnapshot;
import com.pivotal.gemfirexd.internal.tools.dataextractor.snapshot.GFXDSnapshotExportStat;
import com.pivotal.gemfirexd.internal.tools.dataextractor.utils.ExtractorUtils;
import com.pivotal.gemfirexd.jdbc.JdbcTestBase;

public class GemFireXDDataExtractorJUnit extends JdbcTestBase{

  private Connection connection;
  private Statement statement;
  private File ddCopy = new File("dd-copy");
  private File oplogCopy = new File("oplog-copy");
  private File outputDir = new File("export-dir");
  private String testDiskStoreName = "TESTDISKSTORE";
  
  public GemFireXDDataExtractorJUnit(String name) {
    super(name);
  }
  
  public void setUp() throws Exception {
    super.setUp();
    TestUtil.deletePersistentFiles = true;
    connection = super.getConnection();
    statement = connection.createStatement();
    outputDir.mkdirs();
    ddCopy.mkdirs();
    oplogCopy.mkdirs();
    EmbedConnection embeddedConnection = (EmbedConnection)connection;
    embeddedConnection.getTR().setupContextStack();
    this.deleteDirs = new String[0];
  }
  
  public void tearDown() throws Exception {
    connection.close();
    deleteAllTestOpLogs();
    TestUtil.shutDown();
    super.tearDown();
  }
  
  private void deleteAllTestOpLogs() throws Exception {
    FileUtils.forceDelete(oplogCopy);
    FileUtils.forceDelete(ddCopy);
    FileUtils.forceDelete(outputDir);
  }
  
  public void testDDLExport() throws Exception {
    Map<ServerInfo, List<GFXDSnapshotExportStat>> hostToDdlMap = new HashMap<ServerInfo, List<GFXDSnapshotExportStat>>();
    int numDDL = 5;
    this.createDiskStore(testDiskStoreName);
    this.createPersistentReplicateTable("PERSIST_REPLICATE", testDiskStoreName);
    this.createPersistentPartitionedTable("PERSIST_PARTITIONED", testDiskStoreName);
    this.createIndex("PERSIST_REPLICATE");
    this.createIndex("PERSIST_PARTITIONED");
//    this.createReplicateTable();
//    this.createPartitionedTable();
//    this.createStoredProcedure();
    //create index;
    copyDataDictionary();
    String server1OutputDirPath = FilenameUtils.concat(outputDir.getCanonicalPath(), "s1");
    String server2OutputDirPath = FilenameUtils.concat(outputDir.getCanonicalPath(), "s2");
    File server1OutputDir = new File(server1OutputDirPath);
    File server2OutputDir = new File(server2OutputDirPath);
    
    server1OutputDir.mkdirs();
    server2OutputDir.mkdirs();
    
    
    List<GFXDSnapshotExportStat> stats = GemFireXDDataExtractorImpl.exportOfflineDDL(ddCopy.getCanonicalPath(), server1OutputDir.getCanonicalPath());
    
    GFXDSnapshotExportStat stat = stats.get(0);
    GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
    List<String> ddls = ExtractorUtils.readSqlStatements(stat.getFileName());
    assertEquals(numDDL, ddls.size());
    
    Builder builder = new Builder();
    builder.serverName("s1");
    ServerInfo s1 = builder.build();
    hostToDdlMap.put(s1, stats);
    
    this.dropTable("PERSIST_REPLICATE");
    copyDataDictionary();
    stats = GemFireXDDataExtractorImpl.exportOfflineDDL(ddCopy.getCanonicalPath(), server2OutputDir.getCanonicalPath());
    
    
    builder = new Builder();
    builder.serverName("s2");
    ServerInfo s2 = builder.build();
    hostToDdlMap.put(s2, stats);
    
    
    //Check if the DDL are not grouped together as their content is not the same. 
    //Note that the persistent view is the same here. 
    List<List<GFXDSnapshotExportStat>> rankedDDLs = ReportGenerator.rankAndGroupDdlStats(hostToDdlMap);
    assertFalse(rankedDDLs.isEmpty());
    assertTrue(rankedDDLs.size() == 2);
    
  }
  
  public void testDDLSchemaChangeExport() throws Exception {
    int numDDL = 7;
    this.createSchema("firstSchema");
    this.setCurrentSchema("firstSchema");
    this.createDiskStore(testDiskStoreName);
    this.createPersistentReplicateTable("PERSIST_REPLICATE", testDiskStoreName);
    this.createPersistentPartitionedTable("PERSIST_PARTITIONED", testDiskStoreName);
    this.createIndex("PERSIST_REPLICATE");
    this.createIndex("PERSIST_PARTITIONED");
//    this.createReplicateTable();
//    this.createPartitionedTable();
//    this.createStoredProcedure();
    //create index;
    copyDataDictionary();
    GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
    extractor.createTestConnection();
    extractor.retrieveAllRowFormatters();
    List<GFXDSnapshotExportStat> stats = GemFireXDDataExtractorImpl.exportOfflineDDL(ddCopy.getCanonicalPath(), outputDir.getCanonicalPath());
    GFXDSnapshotExportStat stat = stats.get(0);
    List<String> ddls = ExtractorUtils.readSqlStatements(stat.getFileName());
    assertEquals(numDDL, ddls.size());
  }
  
  public void testNoDDLToExport() throws Exception {
    this.createDiskStore(testDiskStoreName);
    this.createPersistentReplicateTable("PERSIST_REPLICATE", testDiskStoreName);
    this.createPersistentPartitionedTable("PERSIST_PARTITIONED", testDiskStoreName);
    this.createIndex("PERSIST_REPLICATE");
    this.createIndex("PERSIST_PARTITIONED");
    GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
    extractor.createTestConnection();
    extractor.retrieveAllRowFormatters();
    try {
    List<GFXDSnapshotExportStat> stats = GemFireXDDataExtractorImpl.exportOfflineDDL(ddCopy.getCanonicalPath(), outputDir.getCanonicalPath());
    }
    catch (IllegalStateException e) {
      return;
    }
    fail("expected an exception to be thrown due to missing ddl");
  }
  
  public void testCSVExportNoOplogsFound() throws Exception {
    String schemaName = "APP";
    String tableName = "REPLICATED_TABLE";
    String bucketName = null;
    this.createDiskStore(testDiskStoreName);
    this.createPersistentReplicateTable(tableName, testDiskStoreName);
    this.insertData(tableName, 0, 1000);
    this.updateData(tableName, 0, 300);
    this.deleteData(tableName, 300, 600);
    
    copyDataDictionary();
    GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
    extractor.createTestConnection();
    extractor.retrieveAllRowFormatters();
    try {
      List<GFXDSnapshotExportStat> stats = GemFireXDDataExtractorImpl.exportDataOpLog(testDiskStoreName, oplogCopy.getCanonicalPath(), outputDir.getCanonicalPath(), false);
    }
    catch (IllegalStateException e) {
      return;
    }
    fail("should have received an exception due to no oplogs present");
  }
  
  public void testExportOnOnlineNode() throws Exception {
    String schemaName = "APP";
    String tableName = "REPLICATED_TABLE";
    String bucketName = null;
    this.createDiskStore(testDiskStoreName);
    this.createPersistentReplicateTable(tableName, testDiskStoreName);
    this.insertData(tableName, 0, 1000);
    this.updateData(tableName, 0, 300);
    this.deleteData(tableName, 300, 600);
    
    boolean ok = false;
    copyDataDictionary();
    GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
    extractor.createTestConnection();
    extractor.retrieveAllRowFormatters();
    try {
      List<GFXDSnapshotExportStat> stats = GemFireXDDataExtractorImpl.exportDataOpLog(testDiskStoreName, "./" + testDiskStoreName, outputDir.getCanonicalPath(), false);
    }
    catch (Exception e) {
      //e.printStackTrace();
      ok = true;
    }
    if (!ok) {
      fail();
    }
  }

  public void testCSVExportSingleNodeReplicate() throws Exception {
    String schemaName = "APP";
    String tableName = "REPLICATED_TABLE";
    String bucketName = null;
    this.createDiskStore(testDiskStoreName);
    this.createPersistentReplicateTable(tableName, testDiskStoreName);
    this.insertData(tableName, 0, 1000);
    this.updateData(tableName, 0, 300);
    this.deleteData(tableName, 300, 600);
    
    copyDataDictionary();
    GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
    extractor.createTestConnection();
    extractor.retrieveAllRowFormatters();
    copyOplogs();
    List<GFXDSnapshotExportStat> stats = GemFireXDDataExtractorImpl.exportDataOpLog(testDiskStoreName, oplogCopy.getCanonicalPath(), outputDir.getCanonicalPath(), false);
    GFXDSnapshotExportStat expectedStat = new GFXDSnapshotExportStat(schemaName, tableName, bucketName, null);
    expectedStat.setNumValuesDecoded(700);
    GFXDSnapshotExportStat stat = stats.get(0);
    assertEquals(1, stats.size());
    assertEquals(0, stat.getEntryDecodeErrors().size());
    assertTrue(areStatsEqual(expectedStat, stat));
    
    File csvFile = new File(stat.getFileName());
    assertTrue(csvFile.exists());
    assertEquals(expectedStat.getNumValuesDecoded(), numLines(csvFile));
  }
  
  public void testCSVExportSingleNodeReplicateDropTable() throws Exception {
    String schemaName = "APP";
    String tableName = "REPLICATE_TABLE";
    String dropTableName = "REPLICATE_DROP_TABLE";
    String bucketName = null;
    this.createDiskStore(testDiskStoreName);
    this.createPersistentReplicateTable(tableName, testDiskStoreName);
    this.insertData(tableName, 0, 1000);
    this.updateData(tableName, 0, 300);
    this.deleteData(tableName, 300, 600);
    this.createPersistentReplicateTable(dropTableName, testDiskStoreName);
    this.insertData(dropTableName, 0, 1000);
    this.updateData(dropTableName, 0, 300);
    this.deleteData(dropTableName, 300, 600);
    this.dropTable(dropTableName);
    
    copyDataDictionary();
    copyOplogs();
    
    GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
    extractor.createTestConnection();
    extractor.retrieveAllRowFormatters();
    List<GFXDSnapshotExportStat> stats = GemFireXDDataExtractorImpl.exportDataOpLog(testDiskStoreName, oplogCopy.getCanonicalPath(), outputDir.getCanonicalPath(), false);
    GFXDSnapshotExportStat expectedStat = new GFXDSnapshotExportStat(schemaName, tableName, bucketName, null);
    expectedStat.setNumValuesDecoded(700);
    GFXDSnapshotExportStat stat = stats.get(0);
    assertEquals(1, stats.size());
    assertEquals(0, stat.getEntryDecodeErrors().size());
    assertTrue(areStatsEqual(expectedStat, stat));
    
    File csvFile = new File(stat.getFileName());
    assertTrue(csvFile.exists());
    assertEquals(expectedStat.getNumValuesDecoded(), numLines(csvFile));  
  }
    
  //Disabled due to removing krf num values check
  public void testCSVExportSingleNodeReplicateOplogCrfCorrupt() throws Exception {
    String tableName = "REPLICATED_TABLE";
    this.createDiskStore(testDiskStoreName);
    this.createPersistentReplicateTable(tableName, testDiskStoreName);
    this.insertData(tableName, 0, 10000);
    this.updateData(tableName, 0, 300);
    this.deleteData(tableName, 300, 600);
    
    copyDataDictionary();
    copyOplogs();

    Iterator<File> oplogIterator = FileUtils.iterateFiles(oplogCopy, new String[]{"crf"}, true);
    while (oplogIterator.hasNext()) {
      corruptFile(500, (File)oplogIterator.next());
    }
        
    GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
    extractor.createTestConnection();
    extractor.retrieveAllRowFormatters();
    
    List<String> diskStoreList = new ArrayList<String>();
    diskStoreList.add(oplogCopy.getCanonicalPath());
    List<GFXDSnapshotExportStat> stats = extractor.extractDiskStores(testDiskStoreName, diskStoreList, outputDir.getCanonicalPath());
    GFXDSnapshotExportStat stat = stats.get(0);
    assertEquals(1, stats.size());
    assertTrue(stat.isCorrupt());
  }
  
  public void testCSVExportSingleNodeReplicateOplogCrfDelete() throws Exception {
    String tableName = "REPLICATED_TABLE";
    this.createDiskStore(testDiskStoreName);
    this.createPersistentReplicateTable(tableName, testDiskStoreName);
    this.insertData(tableName, 0, 10000);
    this.updateData(tableName, 0, 300);
    this.deleteData(tableName, 300, 600);
    
    copyDataDictionary();
    copyOplogs();
    
    Iterator<File> oplogIterator = FileUtils.iterateFiles(oplogCopy, new String[]{"crf"}, true);
    while (oplogIterator.hasNext()) {
      this.deleteFile((File)oplogIterator.next());
    }
    
    GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
    extractor.createTestConnection();
    extractor.retrieveAllRowFormatters();
    List<String> diskStoreList = new ArrayList<String>();
    diskStoreList.add(oplogCopy.getCanonicalPath());
    List<GFXDSnapshotExportStat> stats = extractor.extractDiskStores(testDiskStoreName, diskStoreList, outputDir.getCanonicalPath());
    assertEquals(0, stats.size());
  }
  
  public void DISABLEtestCSVExportSingleNodeReplicateOplogKrfCorrupt() throws Exception {
    String tableName = "REPLICATED_TABLE";
    this.createDiskStore(testDiskStoreName);
    this.createPersistentReplicateTable(tableName, testDiskStoreName);
    this.insertData(tableName, 0, 10000);
    this.updateData(tableName, 0, 300);
    this.deleteData(tableName, 300, 600);
    
    copyDataDictionary();
    copyOplogs();
    
    Iterator<File> oplogIterator = FileUtils.iterateFiles(oplogCopy, new String[]{"krf"}, true);
    while (oplogIterator.hasNext()) {
      corruptFile(500, (File)oplogIterator.next());
    }
    GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
    extractor.createTestConnection();
    extractor.retrieveAllRowFormatters();
    
    List<String> diskStoreList = new ArrayList<String>();
    diskStoreList.add(oplogCopy.getCanonicalPath());
    List<GFXDSnapshotExportStat> stats = extractor.extractDiskStores(testDiskStoreName, diskStoreList, outputDir.getCanonicalPath());
    GFXDSnapshotExportStat stat = stats.get(0);
    assertEquals(1, stats.size());
    assertTrue(stat.isCorrupt());
  }
  
  public void DISABLEtestCSVExportSingleNodeReplicateOplogKrfMissing() throws Exception {
    String tableName = "REPLICATED_TABLE";
    this.createDiskStore(testDiskStoreName);
    this.createPersistentReplicateTable(tableName, testDiskStoreName);
    this.insertData(tableName, 0, 10000);
    this.updateData(tableName, 0, 300);
    this.deleteData(tableName, 300, 600);
    
    copyDataDictionary();
    copyOplogs();

    Iterator<File> oplogIterator = FileUtils.iterateFiles(oplogCopy, new String[]{"krf"}, true);
    while (oplogIterator.hasNext()) {
      this.deleteFile((File)oplogIterator.next());
    }
    
    GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
    extractor.createTestConnection();
    extractor.retrieveAllRowFormatters();
    
    List<String> diskStoreList = new ArrayList<String>();
    diskStoreList.add(oplogCopy.getCanonicalPath());
    List<GFXDSnapshotExportStat> stats = extractor.extractDiskStores(testDiskStoreName, diskStoreList, outputDir.getCanonicalPath());
    GFXDSnapshotExportStat stat = stats.get(0);
    assertEquals(1, stats.size());
    //we actually can't detect if crf is missing any values if KRF is missing
//    assertTrue(stat.isCorrupt());
  }
  
  public void testCSVExportSingleNodeReplicateOplogDrfCorrupt() throws Exception {
    String tableName = "REPLICATED_TABLE";
    this.createDiskStore(testDiskStoreName);
    this.createPersistentReplicateTable(tableName, testDiskStoreName);
    this.insertData(tableName, 0, 10000);
    this.updateData(tableName, 0, 300);
    this.deleteData(tableName, 300, 600);
    
    copyDataDictionary();
    copyOplogs();

    Iterator<File> oplogIterator = FileUtils.iterateFiles(oplogCopy, new String[]{"drf"}, true);
    while (oplogIterator.hasNext()) {
      corruptFile(21, (File)oplogIterator.next());
    }
    
    GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
    extractor.createTestConnection();
    extractor.retrieveAllRowFormatters();
        
    List<String> diskStoreList = new ArrayList<String>();
    diskStoreList.add(oplogCopy.getCanonicalPath());
    List<GFXDSnapshotExportStat> stats = extractor.extractDiskStores(testDiskStoreName, diskStoreList, outputDir.getCanonicalPath());
    //Corrupt DRF leads to no export and no stat and no recommendation
    assertEquals(0, stats.size());
  }
  
  public void testCSVExportSingleNodeReplicateOplogDrfMissing() throws Exception {
    addExpectedException("Unable to recover the following tables, possibly due to a missing .drf file:");
    String tableName = "REPLICATED_TABLE";
    this.createDiskStore(testDiskStoreName);
    this.createPersistentReplicateTable(tableName, testDiskStoreName);
    this.insertData(tableName, 0, 10000);
    this.updateData(tableName, 0, 300);
    this.deleteData(tableName, 300, 600);
    
    copyDataDictionary();
    copyOplogs();

    Iterator<File> oplogIterator = FileUtils.iterateFiles(oplogCopy, new String[]{"drf"}, true);
    while (oplogIterator.hasNext()) {
      this.deleteFile((File)oplogIterator.next());
    }
    
    GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
    extractor.createTestConnection();
    extractor.retrieveAllRowFormatters();
        
    List<String> diskStoreList = new ArrayList<String>();
    diskStoreList.add(oplogCopy.getCanonicalPath());
    List<GFXDSnapshotExportStat> stats = extractor.extractDiskStores(testDiskStoreName, diskStoreList, outputDir.getCanonicalPath());
    //Corrupt DRF leads to no export and no stat and no recommendation
    assertEquals(0, stats.size());
  }
  
  public void testCSVExportSingleNodePartitioned() throws Exception {
    String schemaName = "APP";
    String tableName = "PARTITIONED_TABLE";
    String bucketName = null;
    this.createDiskStore(testDiskStoreName);
    this.createPersistentPartitionedTable(tableName, testDiskStoreName);
    this.insertData(tableName, 0, 1000);
    this.updateData(tableName, 0, 300);
    this.deleteData(tableName, 300, 600);
    
    copyDataDictionary();
    copyOplogs();
    GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
    extractor.createTestConnection();
    extractor.retrieveAllRowFormatters();
    
    List<GFXDSnapshotExportStat> stats = GemFireXDDataExtractorImpl.exportDataOpLog(testDiskStoreName, oplogCopy.getCanonicalPath(), outputDir.getCanonicalPath(), false);
    GFXDSnapshotExportStat expectedStat = new GFXDSnapshotExportStat(schemaName, tableName, bucketName, null);
    expectedStat.setNumValuesDecoded(700);
    GFXDSnapshotExportStat stat = stats.get(0);
    //113 for number of default buckets
    assertEquals(113, stats.size());
    assertEquals(0, stat.getEntryDecodeErrors().size());
    
    //let's iterate through and total up the stats and make sure the counts are the same
    int total = 0;
    Iterator<GFXDSnapshotExportStat> listIterator = stats.iterator();
    while (listIterator.hasNext()) {
      total += listIterator.next().getNumValuesDecoded();
    }
    assertEquals(700, total);
  }
  
  public void testCSVExportSingleNodePartitionedDropTable() throws Exception {
    String schemaName = "APP";
    String tableName = "PARTITIONED_TABLE";
    String dropTableName = "PARTITIONED_DROP_TABLE";
    String bucketName = null;
    this.createDiskStore(testDiskStoreName);
    this.createPersistentPartitionedTable(tableName, testDiskStoreName);
    this.insertData(tableName, 0, 1000);
    this.updateData(tableName, 0, 300);
    this.deleteData(tableName, 300, 600);
    this.createPersistentPartitionedTable(dropTableName, testDiskStoreName);
    this.insertData(dropTableName, 0, 1000);
    this.updateData(dropTableName, 0, 300);
    this.deleteData(dropTableName, 300, 600);
    this.dropTable(dropTableName);
    
    copyDataDictionary();
    copyOplogs();
    
    GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
    extractor.createTestConnection();
    extractor.retrieveAllRowFormatters();
    
    List<GFXDSnapshotExportStat> stats = GemFireXDDataExtractorImpl.exportDataOpLog(testDiskStoreName, oplogCopy.getCanonicalPath(), outputDir.getCanonicalPath(), false);
    GFXDSnapshotExportStat expectedStat = new GFXDSnapshotExportStat(schemaName, tableName, bucketName, null);
    expectedStat.setNumValuesDecoded(700);
    GFXDSnapshotExportStat stat = stats.get(0);
    //113 for number of default buckets
    assertEquals(113, stats.size());
    assertEquals(0, stat.getEntryDecodeErrors().size());
    
    //let's iterate through and total up the stats and make sure the counts are the same
    int total = 0;
    Iterator<GFXDSnapshotExportStat> listIterator = stats.iterator();
    while (listIterator.hasNext()) {
      total += listIterator.next().getNumValuesDecoded();
    }
    assertEquals(700, total);
  }
  
  public void testCSVExportSingleNodeNoIndexExportTable() throws Exception {
    try {
        String schemaName = "APP";
        String tableName = "REPLICATE_TABLE";
        String bucketName = null;
        System.setProperty( Attribute.PERSIST_INDEXES, "true");
        this.createDiskStore(testDiskStoreName);
        this.createPersistentReplicateTable(tableName, testDiskStoreName);
        this.createIndex(tableName);
        this.insertData(tableName, 0, 1000);
        //this.updateData(tableName, 0, 300);  //Due to an issue comparing xml fields with index
        this.deleteData(tableName, 300, 600);
       
        copyDataDictionary();
        copyOplogs();
        GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
        extractor.createTestConnection();
        extractor.retrieveAllRowFormatters();
        
        List<GFXDSnapshotExportStat> stats = GemFireXDDataExtractorImpl.exportDataOpLog(testDiskStoreName, oplogCopy.getCanonicalPath(), outputDir.getCanonicalPath(), false);
        GFXDSnapshotExportStat expectedStat = new GFXDSnapshotExportStat(schemaName, tableName, bucketName, null);
        expectedStat.setNumValuesDecoded(700);
        GFXDSnapshotExportStat stat = stats.get(0);
        assertEquals(1, stats.size());
        assertEquals(0, stat.getEntryDecodeErrors().size());
        assertTrue(areStatsEqual(expectedStat, stat));
    }
    finally {
      System.setProperty( Attribute.PERSIST_INDEXES, "false");
    }
  }
  
  public void testextractorArgs() {
    GemFireXDDataExtractorImpl extractor = new GemFireXDDataExtractorImpl();
    boolean exceptionThrown = false;
    try {
      extractor.consumeProperties();
      extractor.extract();
    } catch (Exception e) {
      exceptionThrown = true;
      //e.printStackTrace();
    }
    assertTrue(exceptionThrown);
    List<String> argsList = new ArrayList<String>();
    argsList.add(GemFireXDDataExtractorImpl.PROPERTY_FILE_ARG + "=" + "salvage.properties");
    try {
      extractor.processArgs(argsList.toArray(new String[argsList.size()]));
      extractor.consumeProperties();
      extractor.extract();
    } catch (Exception e) {
      assertTrue(e instanceof FileNotFoundException);
    }
    
    argsList.clear();
    String propFilePath = "salvage.properties";
    String logLevelString = "FINE";
    String logFilePath = "extractor.log";
    String delimiter = ",";
    String outputDirPath = "/kuwait1/salvage";
    String numThreads = "3";
    
    argsList.add(GemFireXDDataExtractorImpl.PROPERTY_FILE_ARG + "=" + propFilePath);
    argsList.add(GemFireXDDataExtractorImpl.LOG_LEVEL_OPT + "=" + logLevelString);
    argsList.add(GemFireXDDataExtractorImpl.LOG_FILE_OPT + "=" + logFilePath);
    argsList.add(GemFireXDDataExtractorImpl.STRING_DELIMITER + "=" + ",");
    argsList.add(GemFireXDDataExtractorImpl.EXTRACTOR_OUTPUT_DIR_OPT + "=" + outputDirPath);
    argsList.add(GemFireXDDataExtractorImpl.NUM_THREADS_OPT + "=" + numThreads);

    try {
      extractor.processArgs(argsList.toArray(new String[argsList.size()]));
      extractor.consumeProperties();
      assertEquals(extractor.getLogLevelString(), logLevelString);
      assertEquals(extractor.getStringDelimiter(), delimiter);
      System.out.println(extractor.getLogFilePath());
      assertTrue(extractor.getLogFilePath().contains(logFilePath));
      assertFalse(extractor.isShowHelp());
      assertFalse(extractor.isUseSingleDDL());
      assertFalse(extractor.isSalvageInServerDir());
      assertTrue(extractor.useOverrodeNumThreads());
      assertTrue(extractor.getUserNumThreads() == 3);
    } catch (Exception e) {
      e.printStackTrace();
    }
  }
//  public void testRecoverFromCompactedOplog() throws Exception {
//    String schemaName = "APP";
//    String tableName = "REPLICATED_TABLE";
//    String bucketName = null;
//    this.createDiskStoreAllowForceCompaction(testDiskStoreName);
//    this.createPersistentReplicateTable(tableName, testDiskStoreName);
//    this.insertData(tableName, 0, 20000);
//    this.updateData(tableName, 0, 300);
//    this.deleteData(tableName, 0, 20000);
//
//    copyDataDictionary();
//    copyOplogs();
//    for (DiskStoreImpl diskStore : Misc.getGemFireCache().listDiskStores()) {
//      System.out.println("Forcing compaction on :" + diskStore.getName() + " and returned:" + diskStore.forceCompaction());
//    }
//    
//    LocalRegion region = (LocalRegion)Misc.getRegionForTable(schemaName + "." + tableName, false);
//    assertTrue(region.getDiskStore().getAllowForceCompaction());
//    assertTrue(region.getDiskStore().forceCompaction());
//    
//    List<GFXDSnapshotExportStat> stats = GemFireXDDataextractor.exportDataOpLog(testDiskStoreName, oplogCopy.getCanonicalPath(), outputDir.getCanonicalPath(), false);
//    GFXDSnapshotExportStat expectedStat = new GFXDSnapshotExportStat(schemaName, tableName, bucketName, null);
//    expectedStat.setNumValuesDecoded(700);
//    GFXDSnapshotExportStat stat = stats.get(0);
//    assertEquals(1, stats.size());
//    assertEquals(0, stat.getEntryDecodeErrors().size());
//    assertTrue(areStatsEqual(expectedStat, stat));
//    
//    File csvFile = new File(stat.getFileName());
//    assertTrue(csvFile.exists());
//    assertEquals(expectedStat.getNumValuesDecoded(), numLines(csvFile));
//  }

  
//  /** Tests the report generation comparison with a faked 2 node system with a RR and PR**/
//  public void testReportGeneratorWithSameMockedStats() throws Exception {
//    String schemaName = "APP";
//    String table1Name = "TABLE_1";
//    String table2Name = "TABLE_2";
//    String table2Bucket1 = "_BUCKET_1";
//    
//    Map<String,List<GFXDSnapshotExportStat>> hostToDdlStatsMap = new HashMap<String,List<GFXDSnapshotExportStat>>();
//    Map<String,List<GFXDSnapshotExportStat>> hostToStatsMap = new HashMap<String,List<GFXDSnapshotExportStat>>();
//    
//    List<GFXDSnapshotExportStat> node1Stats = new ArrayList<GFXDSnapshotExportStat>();
//    List<GFXDSnapshotExportStat> node2Stats = new ArrayList<GFXDSnapshotExportStat>();
//    GFXDSnapshotExportStat node1Stat1 = new GFXDSnapshotExportStat(schemaName, table1Name, null, null);
//    node1Stat1.setNumValuesDecoded(100);
//    node1Stat1.setFileName("APP_TABLE_1.csv");
//    node1Stat1.setLastModifiedTime(100);
//    node1Stat1.setServerName("Node1");
//    node1Stat1.setTableType("RR");
//    node1Stats.add(node1Stat1);
//    
//    GFXDSnapshotExportStat node1Stat2 = new GFXDSnapshotExportStat(schemaName, table2Name, table2Bucket1, null);
//    node1Stat2.setNumValuesDecoded(100);
//    node1Stat2.setFileName("APP_TABLE_2_BUCKET_1.csv");
//    node1Stat2.setLastModifiedTime(100);
//    node1Stat2.setServerName("Node2");
//    node1Stat2.setTableType("PR");
//    node1Stats.add(node1Stat2);
//    
//    GFXDSnapshotExportStat node2Stat1 = new GFXDSnapshotExportStat(schemaName, table1Name, null, null);
//    node2Stat1.setNumValuesDecoded(100);
//    node2Stat1.setFileName("APP_TABLE_1.csv");
//    node2Stat1.setLastModifiedTime(100);
//    node2Stat1.setServerName("Node2");
//    node2Stat1.setTableType("RR");
//    node2Stats.add(node2Stat1);
//    
//    GFXDSnapshotExportStat node2Stat2 = new GFXDSnapshotExportStat(schemaName, table2Name, table2Bucket1, null);
//    node2Stat2.setNumValuesDecoded(100);
//    node2Stat2.setFileName("APP_TABLE_2_BUCKET_1.csv");
//    node2Stat2.setLastModifiedTime(100);
//    node2Stat2.setServerName("Node2");
//    node2Stat2.setTableType("PR");
//    node2Stats.add(node2Stat2);
//    
//    hostToStatsMap = new HashMap<String, List<GFXDSnapshotExportStat>>();
//    hostToStatsMap.put("Node1", node1Stats);
//    hostToStatsMap.put("Node2", node2Stats);
//    
//    ReportGenerator reporter = new ReportGenerator(hostToDdlStatsMap, hostToStatsMap);
//    Map<String, ReportGenerator.StatGroup> schemaTableNameToStatGroup =reporter.analyzeHostToStatsMap(hostToStatsMap, new ReportGenerator.TextFileCloseEnoughMatchComparator(), false);
//    assertEquals(2, schemaTableNameToStatGroup.size());
//    
//    //Assert that the stats were considered equal and tallied into one tally
//    Map<GFXDSnapshotExportStat, List<GFXDSnapshotExportStat>> talliedStats = schemaTableNameToStatGroup.get("APP_TABLE_1").getTalliedStats();
//    assertEquals(1, talliedStats.keySet().size());
//    
//    talliedStats = schemaTableNameToStatGroup.get("APP_TABLE_2__BUCKET_1").getTalliedStats();
//    assertEquals(1, talliedStats.keySet().size());
//    
//  }
//  
//  /** Tests the report generation comparison with a faked 2 node system with a RR and PR**/
//  public void testReportGeneratorWithDifferentMockedStats() throws Exception {
//    String schemaName = "APP";
//    String table1Name = "TABLE_1";
//    String table2Name = "TABLE_2";
//    String table2Bucket1 = "_BUCKET_1";
//    
//    Map<String,List<GFXDSnapshotExportStat>> hostToDdlStatsMap = new HashMap<String,List<GFXDSnapshotExportStat>>();
//    Map<String,List<GFXDSnapshotExportStat>> hostToStatsMap = new HashMap<String,List<GFXDSnapshotExportStat>>();
//    
//    List<GFXDSnapshotExportStat> node1Stats = new ArrayList<GFXDSnapshotExportStat>();
//    List<GFXDSnapshotExportStat> node2Stats = new ArrayList<GFXDSnapshotExportStat>();
//    GFXDSnapshotExportStat node1Stat1 = new GFXDSnapshotExportStat(schemaName, table1Name, null, null);
//    node1Stat1.setNumValuesDecoded(100);
//    node1Stat1.setFileName("APP_TABLE_1.csv");
//    node1Stat1.setLastModifiedTime(100);
//    node1Stat1.setServerName("Node1");
//    node1Stat1.setTableType("RR");
//    node1Stats.add(node1Stat1);
//    
//    GFXDSnapshotExportStat node1Stat2 = new GFXDSnapshotExportStat(schemaName, table2Name, table2Bucket1, null);
//    node1Stat2.setNumValuesDecoded(100);
//    node1Stat2.setFileName("APP_TABLE_2_BUCKET_1.csv");
//    node1Stat2.setLastModifiedTime(100);
//    node1Stat2.setServerName("Node2");
//    node1Stat2.setTableType("PR");
//    node1Stats.add(node1Stat2);
//    
//    GFXDSnapshotExportStat node2Stat1 = new GFXDSnapshotExportStat(schemaName, table1Name, null, null);
//    node2Stat1.setNumValuesDecoded(50);
//    node2Stat1.setFileName("APP_TABLE_1.csv");
//    node2Stat1.setLastModifiedTime(100);
//    node2Stat1.setServerName("Node2");
//    node2Stat1.setTableType("RR");
//    node2Stats.add(node2Stat1);
//    
//    GFXDSnapshotExportStat node2Stat2 = new GFXDSnapshotExportStat(schemaName, table2Name, table2Bucket1, null);
//    node2Stat2.setNumValuesDecoded(50);
//    node2Stat2.setFileName("APP_TABLE_2_BUCKET_1.csv");
//    node2Stat2.setLastModifiedTime(100);
//    node2Stat2.setServerName("Node2");
//    node2Stat2.setTableType("PR");
//    node2Stats.add(node2Stat2);
//    
//    hostToStatsMap = new HashMap<String, List<GFXDSnapshotExportStat>>();
//    hostToStatsMap.put("Node1", node1Stats);
//    hostToStatsMap.put("Node2", node2Stats);
//    
//    ReportGenerator reporter = new ReportGenerator(hostToDdlStatsMap, hostToStatsMap);
//    Map<String, ReportGenerator.StatGroup> schemaTableNameToStatGroup =reporter.analyzeHostToStatsMap(hostToStatsMap, new ReportGenerator.TextFileCloseEnoughMatchComparator(), false);
//    assertEquals(2, schemaTableNameToStatGroup.size());
//    
//    //Assert that the stats were considered equal and tallied into one tally
//    Map<GFXDSnapshotExportStat, List<GFXDSnapshotExportStat>> talliedStats = schemaTableNameToStatGroup.get("APP_TABLE_1").getTalliedStats();
//    assertEquals(2, talliedStats.keySet().size());
//    
//    talliedStats = schemaTableNameToStatGroup.get("APP_TABLE_2__BUCKET_1").getTalliedStats();
//    assertEquals(2, talliedStats.keySet().size());
//  }
  
 
  
  private void copyDataDictionary() throws IOException {
    FileUtils.copyDirectory(new File("./datadictionary"), ddCopy);
  }
  
  private void copyOplogs() throws IOException {
    copyOplogs("./" + testDiskStoreName);
  }
  
  private void copyOplogs(String diskStoreLocation) throws IOException {
    FileUtils.copyDirectory(new File(diskStoreLocation), oplogCopy);
  }
  
  private boolean areStatsEqual(GFXDSnapshotExportStat expectedStat, GFXDSnapshotExportStat stat) throws Exception {
    if (expectedStat.getNumValuesDecoded() != stat.getNumValuesDecoded()) {
      throw new Exception("Expected values decoded:" + expectedStat.getNumValuesDecoded() + " but was " + stat.getNumValuesDecoded());
    }
    if (!expectedStat.getSchemaTableName().equals(stat.getSchemaTableName())) {
      throw new Exception("Expected schema/table name:" + expectedStat.getSchemaTableName() + " but was " + stat.getSchemaTableName());
    }
    return true;
  }
  
  static int numLines(File csvFile) throws IOException {
    int numLines = 0;
    BufferedReader reader = new BufferedReader(new FileReader(csvFile));
    String line = reader.readLine();
    while (line != null) {
      numLines++;
      line = reader.readLine();
    }
    return numLines;
  }
  
  public static void corruptFile(int sizeToCopy, File file) throws IOException {
    File parent = file.getParentFile();
    File corruptFile = new File(parent, "corruptOplog");
    RandomAccessFile raf = null;
    FileOutputStream fos = null;
    try {
      raf = new RandomAccessFile(file, "r");  
      byte[] bytesRead = new byte[sizeToCopy];  
      long remainingSize = file.length() - sizeToCopy;
      raf.read(bytesRead);    
      byte[] randomStuff = new byte[(int)remainingSize];
      for (int i = 0; i < remainingSize; i++) {
        randomStuff[i] = 1;
      }
      fos = new FileOutputStream(corruptFile);  
      fos.write(bytesRead);
      fos.write(randomStuff);
    }
    finally {
      fos.close();
      raf.close();
    }
    FileUtils.forceDelete(file);
    corruptFile.renameTo(file);
  }
  
  private void deleteFile(File file) throws IOException {
    FileUtils.forceDelete(file);
  }
  

  
  private void insertData(String tableName, int startIndex, int endIndex) throws SQLException{
    PreparedStatement ps = connection.prepareStatement("INSERT INTO " + tableName + "(bigIntegerField, blobField, charField," +
        "charForBitData, clobField, dateField, decimalField, doubleField, floatField, longVarcharForBitDataField, numericField," +
        "realField, smallIntField, timeField, timestampField, varcharField, varcharForBitData, xmlField) values( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, xmlparse(document cast (? as clob) PRESERVE WHITESPACE))");
 
    for (int i = startIndex; i < endIndex; i++) {
      int lessThan10 = i % 10;

      ps.setLong(1, i); //BIG INT
      ps.setBlob(2,new ByteArrayInputStream(new byte[]{(byte)i,(byte)i,(byte)i,(byte)i}));
      ps.setString(3, ""+lessThan10);
      ps.setBytes(4, ("" + lessThan10).getBytes());
      ps.setClob(5, new StringReader("SOME CLOB " + i));
      ps.setDate(6, new Date(System.currentTimeMillis()));
      ps.setBigDecimal(7, new BigDecimal(lessThan10 + .8));
      ps.setDouble(8, i + .88);
      ps.setFloat(9, i + .9f);
      ps.setBytes(10, ("A" + lessThan10).getBytes());
      ps.setBigDecimal(11, new BigDecimal(i));
      ps.setFloat(12, lessThan10 * 1111);
      ps.setShort(13, (short)i);
      ps.setTime(14, new Time(System.currentTimeMillis()));
      ps.setTimestamp(15, new Timestamp(System.currentTimeMillis()));
      ps.setString(16, "HI" + lessThan10);
      ps.setBytes(17, ("" + lessThan10).getBytes());
      ps.setClob(18, new StringReader("<xml><sometag>SOME XML CLOB " + i + "</sometag></xml>"));
      ps.execute();
    }
  }
  
  private void deleteData(String tableName, int startIndex, int endIndex) throws SQLException {
    PreparedStatement ps = connection.prepareStatement("delete from " + tableName +" where bigIntegerField = ?");
    for (int i = startIndex; i < endIndex; i++) {
      ps.setLong(1, i);
      ps.execute();      
    }
  }
  
  private void updateData(String tableName, int startIndex, int endIndex) throws SQLException {
    PreparedStatement ps = connection.prepareStatement("UPDATE " + tableName + " set blobField=?, charField=?," +
        "charForBitData=?, clobField=?, dateField=?, decimalField=?, doubleField=?, floatField=?, longVarcharForBitDataField=?, numericField=?," +
        "realField=?, smallIntField=?, timeField=?, timestampField=?, varcharField=?, varcharForBitData=?, xmlField=xmlparse(document cast (? as clob) PRESERVE WHITESPACE) where bigIntegerField=?");
 
    for (int i = startIndex; i < endIndex; i++) {
      int lessThan10 = i % 10;

      ps.setBlob(1,new ByteArrayInputStream(new byte[]{(byte)i,(byte)i,(byte)i,(byte)i}));
      ps.setString(2, ""+lessThan10);
      ps.setBytes(3, ("" + lessThan10).getBytes());
      ps.setClob(4, new StringReader("UPDATE CLOB " + i));
      ps.setDate(5, new Date(System.currentTimeMillis()));
      ps.setBigDecimal(6, new BigDecimal(lessThan10 + .8));
      ps.setDouble(7, i + .88);
      ps.setFloat(8, i + .9f);
      ps.setBytes(9, ("B" + lessThan10).getBytes());
      ps.setBigDecimal(10, new BigDecimal(i));
      ps.setFloat(11, lessThan10 * 1111);
      ps.setShort(12, (short)i);
      ps.setTime(13, new Time(System.currentTimeMillis()));
      ps.setTimestamp(14, new Timestamp(System.currentTimeMillis()));
      ps.setString(15, "BY" + lessThan10);
      ps.setBytes(16, ("" + lessThan10).getBytes());
      ps.setClob(17, new StringReader("<xml><sometag>UPDATE XML CLOB " + i + "</sometag></xml>"));
      ps.setLong(18, i);
      ps.execute();
    }
  }
  
  private void createSchema(String schemaName) throws SQLException {
    statement.execute("CREATE SCHEMA " + schemaName);
  }
  
  private void setCurrentSchema(String schemaName) throws SQLException {
    statement.execute("SET CURRENT SCHEMA " + schemaName);
  }
  
  private void createDiskStore(String diskStoreName) throws SQLException {
    String createDiskStore = "CREATE DISKSTORE " + diskStoreName + "('" + diskStoreName + "') MAXLOGSIZE 1";
    String[] newDeleteDirs = new String[this.deleteDirs.length + 1];
    for (int i = 0 ; i < this.deleteDirs.length; i++) {
      newDeleteDirs[i] = this.deleteDirs[i];
    }
    newDeleteDirs[this.deleteDirs.length] = "./" + diskStoreName;
    this.deleteDirs = newDeleteDirs;
    statement.execute(createDiskStore);
  }
  
  private void createDiskStoreAllowForceCompaction(String diskStoreName) throws SQLException {
    String createDiskStore = "CREATE DISKSTORE " + diskStoreName + " ALLOWFORCECOMPACTION TRUE  COMPACTIONTHRESHOLD  1('" + diskStoreName + "') MAXLOGSIZE 1";
    String[] newDeleteDirs = new String[this.deleteDirs.length + 1];
    for (int i = 0 ; i < this.deleteDirs.length; i++) {
      newDeleteDirs[i] = this.deleteDirs[i];
    }
    newDeleteDirs[this.deleteDirs.length] = "./" + diskStoreName;
    this.deleteDirs = newDeleteDirs;
    statement.execute(createDiskStore);
  }
  
  private void createPersistentReplicateTable(String tableName, String diskStoreName) throws SQLException {
    String create = "CREATE TABLE " + tableName + "(bigIntegerField BIGINT, blobField BLOB(1K), charField CHAR(1)," +
        "charForBitData CHAR(1) FOR BIT DATA, clobField CLOB(1K), dateField DATE, decimalField DECIMAL(10,1)," +
        "doubleField DOUBLE, floatField FLOAT(10), longVarcharForBitDataField LONG VARCHAR FOR BIT DATA," +
        "numericField NUMERIC(10,1), realField REAL, smallIntField SMALLINT, timeField TIME, timestampField TIMESTAMP," +
        "varcharField VARCHAR(10), varcharForBitData VARCHAR(1) FOR BIT DATA, xmlField XML) REPLICATE persistent '" + diskStoreName + "'";
    statement.execute(create);
  }
  
  private void createPersistentPartitionedTable(String tableName, String diskStoreName) throws SQLException {
    String create = "CREATE TABLE " + tableName + "(bigIntegerField BIGINT, blobField BLOB(1K), charField CHAR(1)," +
            "charForBitData CHAR(1) FOR BIT DATA, clobField CLOB(1K), dateField DATE, decimalField DECIMAL(10,1)," +
            "doubleField DOUBLE, floatField FLOAT(10), longVarcharForBitDataField LONG VARCHAR FOR BIT DATA," +
            "numericField NUMERIC(10,1), realField REAL, smallIntField SMALLINT, timeField TIME, timestampField TIMESTAMP," +
            "varcharField VARCHAR(10), varcharForBitData VARCHAR(1) FOR BIT DATA, xmlField XML, " +
            "PRIMARY KEY(bigIntegerField)) PARTITION BY PRIMARY KEY persistent '" + diskStoreName +"'";
    statement.execute(create);
  }
  
  
  private void dropTable(String tableName) throws SQLException {
    String drop = "DROP TABLE " + tableName;
    statement.execute(drop);
  }
  
  private void createIndex(String tableName) throws SQLException {
    String index = "CREATE INDEX " + tableName + "_INDEX on " + tableName  + "(bigIntegerField)";
    statement.execute(index);
  }
  
  
//  public static EmbedConnection getextractorConnection() throws SQLException {
//    GemFireXDDataextractor extractor = new GemFireXDDataextractor();
//    extractor.loadDriver("com.pivotal.gemfirexd.jdbc.EmbeddedDriver");
//    Connection conn = extractor.getConnection("jdbc:gemfirexd:", extractor.createTestConnectionProperties());
//    return  (EmbedConnection)conn;
//  }

}