package org.apache.blur.manager.writer;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.SortedSet;
import java.util.Timer;
import java.util.TimerTask;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantReadWriteLock;

import org.apache.blur.log.Log;
import org.apache.blur.log.LogFactory;
import org.apache.blur.lucene.search.IndexSearcherCloseable;
import org.apache.blur.manager.BlurPartitioner;
import org.apache.blur.manager.writer.MergeSortRowIdLookup.Action;
import org.apache.blur.server.ShardContext;
import org.apache.blur.server.TableContext;
import org.apache.blur.server.cache.ThriftCache;
import org.apache.blur.store.hdfs.DirectoryDecorator;
import org.apache.blur.store.hdfs.HdfsDirectory;
import org.apache.blur.store.hdfs_v2.JoinDirectory;
import org.apache.blur.utils.BlurConstants;
import org.apache.blur.utils.ShardUtil;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.Text;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.CompositeReaderContext;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;

public class IndexImporter extends TimerTask implements Closeable {

  private static final String INPROGRESS = ".inprogress";
  private static final String BADROWIDS = ".badrowids";
  private static final String COMMIT = ".commit";
  private static final String INUSE = ".inuse";
  private static final String BADINDEX = ".badindex";
  private static final Lock _globalLock = new ReentrantReadWriteLock().writeLock();

  private final static Log LOG = LogFactory.getLog(IndexImporter.class);

  private final BlurIndex _blurIndex;
  private final ShardContext _shardContext;
  private final String _table;
  private final String _shard;
  private final long _cleanupDelay;
  private final Timer _inindexImporterTimer;
  private final ThriftCache _thriftCache;
  private final HdfsDirectory _directory;
  private final int MAX_ATTEMPTS = 10;

  private long _lastCleanup;
  private Runnable _testError;

  public IndexImporter(Timer indexImporterTimer, BlurIndex blurIndex, ShardContext shardContext, TimeUnit timeUnit,
      long refreshAmount, long initalWaitTime, ThriftCache thriftCache, Directory dir) throws IOException {
    _thriftCache = thriftCache;
    _blurIndex = blurIndex;
    _shardContext = shardContext;
    _directory = getHdfsDirectory(dir);

    long period = timeUnit.toMillis(refreshAmount);
    long delay = timeUnit.toMillis(initalWaitTime);
    indexImporterTimer.schedule(this, delay, period);
    _inindexImporterTimer = indexImporterTimer;
    _table = _shardContext.getTableContext().getTable();
    _shard = _shardContext.getShard();
    _cleanupDelay = TimeUnit.MINUTES.toMillis(10);
  }

  private HdfsDirectory getHdfsDirectory(Directory dir) throws IOException {
    if (dir instanceof HdfsDirectory) {
      return (HdfsDirectory) dir;
    } else if (dir instanceof DirectoryDecorator) {
      DirectoryDecorator decorator = (DirectoryDecorator) dir;
      return getHdfsDirectory(decorator.getOriginalDirectory());
    } else if (dir instanceof JoinDirectory) {
      JoinDirectory directory = (JoinDirectory) dir;
      return directory.getSymlinkDirectory();
    } else {
      throw new IOException("Directory [" + dir + "] is not HdfsDirectory or DirectoryDecorator");
    }
  }

  @Override
  public void close() throws IOException {
    cancel();
    _inindexImporterTimer.purge();
  }

  public long getSegmentImportPendingCount() throws IOException {
    Path path = _shardContext.getHdfsDirPath();
    Configuration configuration = _shardContext.getTableContext().getConfiguration();
    FileSystem fileSystem = path.getFileSystem(configuration);
    for (int i = 0; i < 10; i++) {
      try {
        FileStatus[] listStatus = fileSystem.listStatus(path, new PathFilter() {
          @Override
          public boolean accept(Path path) {
            if (path != null && path.getName().endsWith(COMMIT)) {
              return true;
            }
            return false;
          }
        });
        return listStatus.length;
      } catch (FileNotFoundException e) {
        LOG.warn("File not found error, retrying.");
      }
      try {
        Thread.sleep(100);
      } catch (InterruptedException e) {
        return 0L;
      }
    }
    throw new IOException("Received too many errors. Give up.");
  }

  public long getSegmentImportInProgressCount() throws IOException {
    Path path = _shardContext.getHdfsDirPath();
    Configuration configuration = _shardContext.getTableContext().getConfiguration();
    FileSystem fileSystem = path.getFileSystem(configuration);
    for (int i = 0; i < 10; i++) {
      try {
        FileStatus[] listStatus = fileSystem.listStatus(path, new PathFilter() {
          @Override
          public boolean accept(Path path) {
            if (path != null && path.getName().endsWith(INUSE)) {
              return true;
            }
            return false;
          }
        });
        long count = 0;
        for (FileStatus fileStatus : listStatus) {
          Path p = fileStatus.getPath();
          if (fileSystem.exists(new Path(p, INPROGRESS))) {
            count++;
          }
        }
        return count;
      } catch (FileNotFoundException e) {
        LOG.warn("File not found error, retrying.");
      }
      try {
        Thread.sleep(100);
      } catch (InterruptedException e) {
        return 0L;
      }
    }
    throw new IOException("Received too many errors. Give up.");
  }

  @Override
  public void run() {
    // Only allow one import to occur in the process at a time.
    _globalLock.lock();
    try {
      if (_lastCleanup + _cleanupDelay < System.currentTimeMillis()) {
        try {
          cleanupOldDirs();
        } catch (IOException e) {
          LOG.error("Unknown error while trying to clean old directories on [{1}/{2}].", e, _shard, _table);
        }
        _lastCleanup = System.currentTimeMillis();
      }
      Path path = _shardContext.getHdfsDirPath();
      Configuration configuration = _shardContext.getTableContext().getConfiguration();
      try {
        FileSystem fileSystem = path.getFileSystem(configuration);
        SortedSet<FileStatus> listStatus = null;
        for (int i = 0; i < MAX_ATTEMPTS; i++) {
          try {
            if (!fileSystem.exists(path)) {
              LOG.warn("Path [{0}] no longer exists, exiting.", path);
              return;
            }
            listStatus = sort(fileSystem.listStatus(path, new PathFilter() {
              @Override
              public boolean accept(Path path) {
                if (path != null && path.getName().endsWith(COMMIT)) {
                  return true;
                }
                return false;
              }
            }));
            break;
          } catch (FileNotFoundException e) {
            LOG.warn("File not found error, retrying.");
          }
          try {
            Thread.sleep(100 * (i + 1));
          } catch (InterruptedException e) {
            return;
          }
        }
        if (listStatus == null) {
          LOG.warn("Could not get listing of path [{0}], exiting.", path);
          return;
        }
        for (FileStatus fileStatus : listStatus) {
          Path file = fileStatus.getPath();
          if (fileStatus.isDir() && file.getName().endsWith(COMMIT)) {
            // rename to inuse, if good continue else rename to badindex
            Path inuse = new Path(file.getParent(), rename(file.getName(), INUSE));
            touch(fileSystem, new Path(file, INPROGRESS));
            if (fileSystem.rename(file, inuse)) {
              if (_testError != null) {
                _testError.run();
              }
              HdfsDirectory hdfsDirectory = new HdfsDirectory(configuration, inuse);
              try {
                if (DirectoryReader.indexExists(hdfsDirectory)) {
                  IndexAction indexAction = getIndexAction(hdfsDirectory, fileSystem);
                  _blurIndex.process(indexAction);
                  return;
                } else {
                  Path badindex = new Path(file.getParent(), rename(file.getName(), BADINDEX));
                  if (fileSystem.rename(inuse, badindex)) {
                    LOG.error("Directory found at [{0}] is not a vaild index, renaming to [{1}].", inuse, badindex);
                  } else {
                    LOG.fatal("Directory found at [{0}] is not a vaild index, could not rename to [{1}].", inuse,
                        badindex);
                  }
                }
              } finally {
                hdfsDirectory.close();
              }
            } else {
              LOG.fatal("Could not rename [{0}] to inuse dir.", file);
            }
          }
        }
      } catch (IOException e) {
        LOG.error("Unknown error while trying to refresh imports on [{1}/{2}].", e, _shard, _table);
      }
    } catch (Throwable t) {
      LOG.error("Unknown error while tyring to run index importer.", t);
    } finally {
      _globalLock.unlock();
    }

  }

  private void touch(FileSystem fileSystem, Path path) throws IOException {
    fileSystem.create(path, true).close();
  }

  private String rename(String name, String newSuffix) {
    int lastIndexOf = name.lastIndexOf('.');
    return name.substring(0, lastIndexOf) + newSuffix;
  }

  private IndexAction getIndexAction(final HdfsDirectory directory, final FileSystem fileSystem) {
    return new IndexAction() {

      @Override
      public void performMutate(IndexSearcherCloseable searcher, IndexWriter writer) throws IOException {
        LOG.info("About to import [{0}] into [{1}/{2}]", directory, _shard, _table);
        boolean emitDeletes = searcher.getIndexReader().numDocs() != 0;
        Configuration configuration = _shardContext.getTableContext().getConfiguration();

        applyDeletes(directory, writer, searcher, _shard, emitDeletes, configuration);
        LOG.info("Add index [{0}] [{1}/{2}]", directory, _shard, _table);
        writer.addIndexes(directory);
        LOG.info("Removing delete markers [{0}] on [{1}/{2}]", directory, _shard, _table);
        writer.deleteDocuments(new Term(BlurConstants.DELETE_MARKER, BlurConstants.DELETE_MARKER_VALUE));
        LOG.info("Finishing import [{0}], commiting on [{1}/{2}]", directory, _shard, _table);
      }

      @Override
      public void doPreCommit(IndexSearcherCloseable indexSearcher, IndexWriter writer) throws IOException {

      }

      @Override
      public void doPostCommit(IndexWriter writer) throws IOException {
        Path path = directory.getPath();
        fileSystem.delete(new Path(path, INPROGRESS), false);
        LOG.info("Import complete on [{0}/{1}]", _shard, _table);
        writer.maybeMerge();
      }

      @Override
      public void doPreRollback(IndexWriter writer) throws IOException {
        LOG.info("Starting rollback on [{0}/{1}]", _shard, _table);
      }

      @Override
      public void doPostRollback(IndexWriter writer) throws IOException {
        LOG.info("Finished rollback on [{0}/{1}]", _shard, _table);
        Path path = directory.getPath();
        String name = path.getName();
        fileSystem.rename(path, new Path(path.getParent(), rename(name, BADROWIDS)));
      }
    };
  }

  private SortedSet<FileStatus> sort(FileStatus[] listStatus) {
    SortedSet<FileStatus> result = new TreeSet<FileStatus>();
    for (FileStatus fileStatus : listStatus) {
      result.add(fileStatus);
    }
    return result;
  }

  private void applyDeletes(Directory directory, IndexWriter indexWriter, IndexSearcherCloseable searcher,
      String shard, boolean emitDeletes, Configuration configuration) throws IOException {
    DirectoryReader newReader = DirectoryReader.open(directory);
    try {
      List<AtomicReaderContext> newLeaves = newReader.getContext().leaves();
      BlurPartitioner blurPartitioner = new BlurPartitioner();
      Text key = new Text();
      int numberOfShards = _shardContext.getTableContext().getDescriptor().getShardCount();
      int shardId = ShardUtil.getShardIndex(shard);

      Action action = new Action() {
        @Override
        public void found(AtomicReader reader, Bits liveDocs, TermsEnum termsEnum) throws IOException {
          DocsEnum docsEnum = termsEnum.docs(liveDocs, null);
          if (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
            indexWriter.deleteDocuments(new Term(BlurConstants.ROW_ID, BytesRef.deepCopyOf(termsEnum.term())));
          }
        }
      };

      LOG.info("Applying deletes for table [{0}] shard [{1}] new reader [{2}]", _table, shard, newReader);
      boolean skipCheckRowIds = isInternal(newReader);
      LOG.info("Skip rowid check [{0}] for table [{1}] shard [{2}] new reader [{3}]", skipCheckRowIds, _table, shard,
          newReader);
      for (AtomicReaderContext context : newLeaves) {
        AtomicReader newAtomicReader = context.reader();
        if (isFastRowIdDeleteSupported(newAtomicReader)) {
          runNewRowIdCheckAndDelete(indexWriter, emitDeletes, blurPartitioner, key, numberOfShards, shardId,
              newAtomicReader, skipCheckRowIds);
        } else {
          runOldMergeSortRowIdCheckAndDelete(emitDeletes, searcher.getIndexReader(), blurPartitioner, key,
              numberOfShards, shardId, action, newAtomicReader);
        }
      }
    } finally {
      newReader.close();
    }
  }

  private boolean isInternal(DirectoryReader reader) throws IOException {
    Map<String, String> map = reader.getIndexCommit().getUserData();
    return BlurConstants.INTERNAL.equals(map.get(BlurConstants.INTERNAL));
  }

  private void runNewRowIdCheckAndDelete(IndexWriter indexWriter, boolean emitDeletes, BlurPartitioner blurPartitioner,
      Text key, int numberOfShards, int shardId, AtomicReader atomicReader, boolean skipCheckRowIds) throws IOException {
    Fields fields = atomicReader.fields();
    if (skipCheckRowIds) {
      Terms rowIdTerms = fields.terms(BlurConstants.ROW_ID);
      if (rowIdTerms != null) {
        LOG.info("Checking rowIds for import on table [{0}] shard [{1}]", _table, _shard);
        TermsEnum rowIdTermsEnum = rowIdTerms.iterator(null);
        BytesRef ref = null;
        while ((ref = rowIdTermsEnum.next()) != null) {
          key.set(ref.bytes, ref.offset, ref.length);
          int partition = blurPartitioner.getPartition(key, null, numberOfShards);
          if (shardId != partition) {
            throw new IOException("Index is corrupted, RowIds are found in wrong shard, partition [" + partition
                + "] does not shard [" + shardId + "], this can happen when rows are not hashed correctly.");
          }
        }
      }
    }
    if (emitDeletes) {
      Terms rowIdsToDeleteTerms = fields.terms(BlurConstants.UPDATE_ROW);
      if (rowIdsToDeleteTerms != null) {
        LOG.info("Performing deletes on rowIds for import on table [{0}] shard [{1}]", _table, _shard);
        TermsEnum rowIdsToDeleteTermsEnum = rowIdsToDeleteTerms.iterator(null);
        BytesRef ref = null;
        while ((ref = rowIdsToDeleteTermsEnum.next()) != null) {
          indexWriter.deleteDocuments(new Term(BlurConstants.ROW_ID, BytesRef.deepCopyOf(ref)));
        }
      }
    }
  }

  private void runOldMergeSortRowIdCheckAndDelete(boolean emitDeletes, IndexReader currentIndexReader,
      BlurPartitioner blurPartitioner, Text key, int numberOfShards, int shardId, Action action,
      AtomicReader atomicReader) throws IOException {
    MergeSortRowIdLookup lookup = new MergeSortRowIdLookup(currentIndexReader);
    Fields fields = atomicReader.fields();
    Terms terms = fields.terms(BlurConstants.ROW_ID);
    if (terms != null) {
      TermsEnum termsEnum = terms.iterator(null);
      BytesRef ref = null;
      while ((ref = termsEnum.next()) != null) {
        key.set(ref.bytes, ref.offset, ref.length);
        int partition = blurPartitioner.getPartition(key, null, numberOfShards);
        if (shardId != partition) {
          throw new IOException("Index is corrupted, RowIds are found in wrong shard, partition [" + partition
              + "] does not shard [" + shardId + "], this can happen when rows are not hashed correctly.");
        }
        if (emitDeletes) {
          lookup.lookup(ref, action);
        }
      }
    }
  }

  private boolean isFastRowIdDeleteSupported(AtomicReader atomicReader) throws IOException {
    if (atomicReader.fields().terms(BlurConstants.NEW_ROW) != null) {
      return true;
    }
    if (atomicReader.fields().terms(BlurConstants.UPDATE_ROW) != null) {
      return true;
    }
    return false;
  }

  public void cleanupOldDirs() throws IOException {
    Path hdfsDirPath = _shardContext.getHdfsDirPath();
    TableContext tableContext = _shardContext.getTableContext();
    Configuration configuration = tableContext.getConfiguration();
    FileSystem fileSystem = hdfsDirPath.getFileSystem(configuration);
    FileStatus[] inuseSubDirs = fileSystem.listStatus(hdfsDirPath, new PathFilter() {
      @Override
      public boolean accept(Path path) {
        return path.getName().endsWith(INUSE);
      }
    });
    Set<Path> inuseDirs = toSet(inuseSubDirs);
    Map<Path, Path> inuseFileToDir = toMap(fileSystem, inuseDirs);
    FileStatus[] listStatus = fileSystem.listStatus(hdfsDirPath, new PathFilter() {
      @Override
      public boolean accept(Path path) {
        return path.getName().endsWith(HdfsDirectory.LNK);
      }
    });

    for (FileStatus status : listStatus) {
      String realFileName = HdfsDirectory.getRealFileName(status.getPath().getName());
      Path realPath = _directory.getRealFilePathFromSymlink(realFileName);
      Path inuseDir = inuseFileToDir.get(realPath);
      inuseDirs.remove(inuseDir);
      // if the inuse dir has an inprogress file then remove it because there
      // are files that reference this dir so it had to be committed.
      Path path = new Path(inuseDir, INPROGRESS);
      if (fileSystem.exists(path)) {
        fileSystem.delete(path, false);
        if (_thriftCache != null) {
          _thriftCache.clearTable(_table);
        }
      }
    }

    // Check if any inuse dirs have inprogress files.
    // If they do, rename inuse to commit to retry import.
    for (Path inuse : new HashSet<Path>(inuseDirs)) {
      Path path = new Path(inuse, INPROGRESS);
      if (fileSystem.exists(path)) {
        LOG.info("Path [{0}] is not imported but has inprogress file, retrying import.", path);
        inuseDirs.remove(inuse);
        Path commit = new Path(inuse.getParent(), rename(inuse.getName(), COMMIT));
        fileSystem.rename(inuse, commit);
      }
    }

    for (Path p : inuseDirs) {
      LOG.info("Deleting path [{0}] no longer in use.", p);
      fileSystem.delete(p, true);
    }
  }

  private Map<Path, Path> toMap(FileSystem fileSystem, Set<Path> inuseDirs) throws IOException {
    Map<Path, Path> result = new TreeMap<Path, Path>();
    for (Path p : inuseDirs) {
      if (!fileSystem.isFile(p)) {
        FileStatus[] listStatus = fileSystem.listStatus(p);
        for (FileStatus status : listStatus) {
          result.put(status.getPath(), p);
        }
      }
    }
    return result;
  }

  private Set<Path> toSet(FileStatus[] dirs) {
    Set<Path> result = new TreeSet<Path>();
    for (FileStatus status : dirs) {
      result.add(status.getPath());
    }
    return result;
  }

  public Runnable getTestError() {
    return _testError;
  }

  public void setTestError(Runnable testError) {
    _testError = testError;
  }

}