/**
s * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.blur.manager.writer;

import static org.apache.blur.lucene.LuceneVersionConstant.LUCENE_VERSION;
import static org.apache.blur.utils.BlurConstants.ACL_DISCOVER;
import static org.apache.blur.utils.BlurConstants.ACL_READ;
import static org.apache.blur.utils.BlurConstants.BLUR_RECORD_SECURITY_DEFAULT_READMASK_MESSAGE;
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WRITER_SORT_FACTOR;
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_INDEX_WRITER_SORT_MEMORY;
import static org.apache.blur.utils.BlurConstants.BLUR_SHARD_QUEUE_MAX_INMEMORY_LENGTH;

import java.io.Closeable;
import java.io.IOException;
import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;

import org.apache.blur.BlurConfiguration;
import org.apache.blur.analysis.FieldManager;
import org.apache.blur.index.ExitableReader;
import org.apache.blur.index.IndexDeletionPolicyReader;
import org.apache.blur.log.Log;
import org.apache.blur.log.LogFactory;
import org.apache.blur.lucene.codec.Blur024Codec;
import org.apache.blur.lucene.search.IndexSearcherCloseable;
import org.apache.blur.lucene.search.IndexSearcherCloseableBase;
import org.apache.blur.lucene.search.SuperQuery;
import org.apache.blur.lucene.security.index.AccessControlFactory;
import org.apache.blur.memory.MemoryLeakDetector;
import org.apache.blur.server.IndexSearcherCloseableSecureBase;
import org.apache.blur.server.ShardContext;
import org.apache.blur.server.TableContext;
import org.apache.blur.server.cache.ThriftCache;
import org.apache.blur.store.hdfs_v2.StoreDirection;
import org.apache.blur.thrift.generated.BlurException;
import org.apache.blur.thrift.generated.RowMutation;
import org.apache.blur.thrift.generated.ScoreType;
import org.apache.blur.thrift.generated.TableDescriptor;
import org.apache.blur.trace.Trace;
import org.apache.blur.trace.Tracer;
import org.apache.blur.user.User;
import org.apache.blur.user.UserContext;
import org.apache.blur.utils.BlurConstants;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.SequenceFile.Reader;
import org.apache.hadoop.io.SequenceFile.Sorter;
import org.apache.hadoop.io.SequenceFile.Writer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.io.compress.SnappyCodec;
import org.apache.hadoop.util.Progressable;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.AtomicReader;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.BlurIndexWriter;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;

import com.google.common.base.Splitter;

public class BlurIndexSimpleWriter extends BlurIndex {

  private static final String TRUE = "true";

  private static final Log LOG = LogFactory.getLog(BlurIndexSimpleWriter.class);

  private final BlurIndexCloser _indexCloser;
  private final AtomicReference<DirectoryReader> _indexReader = new AtomicReference<DirectoryReader>();
  private final ExecutorService _searchThreadPool;
  private final Directory _directory;
  private final IndexWriterConfig _conf;
  private final TableContext _tableContext;
  private final FieldManager _fieldManager;
  private final ShardContext _shardContext;
  private final AtomicReference<BlurIndexWriter> _writer = new AtomicReference<BlurIndexWriter>();
  private final boolean _makeReaderExitable = true;
  private final ReentrantReadWriteLock _lock = new ReentrantReadWriteLock();
  private final WriteLock _writeLock = _lock.writeLock();
  private final ReadWriteLock _indexRefreshLock = new ReentrantReadWriteLock();
  private final Lock _indexRefreshWriteLock = _indexRefreshLock.writeLock();
  private final Lock _indexRefreshReadLock = _indexRefreshLock.readLock();
  private final IndexDeletionPolicyReader _policy;
  private final SnapshotIndexDeletionPolicy _snapshotIndexDeletionPolicy;
  private final String _context;
  private final AtomicInteger _writesWaiting = new AtomicInteger();
  private final BlockingQueue<RowMutation> _queue;
  private final MutationQueueProcessor _mutationQueueProcessor;
  private final Timer _indexImporterTimer;
  private final Map<String, BulkEntry> _bulkWriters;
  private final boolean _security;
  private final AccessControlFactory _accessControlFactory;
  private final Set<String> _discoverableFields;
  private final Splitter _commaSplitter;
  private final Timer _bulkIndexingTimer;
  private final TimerTask _watchForIdleBulkWriters;
  private final ThriftCache _thriftCache;
  private final String _defaultReadMaskMessage;
  private final IndexImporter _indexImporter;
  private final Timer _indexWriterTimer;
  private final AtomicLong _lastWrite = new AtomicLong();
  private final long _maxWriterIdle;
  private final TimerTask _watchForIdleWriter;

  private volatile Thread _optimizeThread;

  public BlurIndexSimpleWriter(BlurIndexConfig blurIndexConf) throws IOException {
    super(blurIndexConf);
    _maxWriterIdle = blurIndexConf.getMaxWriterIdle();
    _indexWriterTimer = blurIndexConf.getIndexWriterTimer();
    _thriftCache = blurIndexConf.getThriftCache();
    _commaSplitter = Splitter.on(',');
    _bulkWriters = new ConcurrentHashMap<String, BlurIndexSimpleWriter.BulkEntry>();
    _indexImporterTimer = blurIndexConf.getIndexImporterTimer();
    _bulkIndexingTimer = blurIndexConf.getBulkIndexingTimer();
    _searchThreadPool = blurIndexConf.getSearchExecutor();
    _shardContext = blurIndexConf.getShardContext();
    _tableContext = _shardContext.getTableContext();
    _context = _tableContext.getTable() + "/" + _shardContext.getShard();
    _fieldManager = _tableContext.getFieldManager();
    _discoverableFields = _tableContext.getDiscoverableFields();
    _accessControlFactory = _tableContext.getAccessControlFactory();
    _defaultReadMaskMessage = getDefaultReadMaskMessage(_tableContext);

    TableDescriptor descriptor = _tableContext.getDescriptor();
    Map<String, String> tableProperties = descriptor.getTableProperties();
    if (tableProperties != null) {
      String value = tableProperties.get(BlurConstants.BLUR_RECORD_SECURITY);
      if (value != null && value.equals(TRUE)) {
        LOG.info("Record Level Security has been enabled for table [{0}] shard [{1}]", _tableContext.getTable(),
            _shardContext.getShard());
        _security = true;
      } else {
        _security = false;
      }
    } else {
      _security = false;
    }
    Analyzer analyzer = _fieldManager.getAnalyzerForIndex();
    _conf = new IndexWriterConfig(LUCENE_VERSION, analyzer);
    _conf.setWriteLockTimeout(TimeUnit.MINUTES.toMillis(5));
    _conf.setCodec(new Blur024Codec(_tableContext.getBlurConfiguration()));
    _conf.setSimilarity(_tableContext.getSimilarity());
    _conf.setInfoStream(new LoggingInfoStream(_tableContext.getTable(), _shardContext.getShard()));
    TieredMergePolicy mergePolicy = (TieredMergePolicy) _conf.getMergePolicy();
    mergePolicy.setUseCompoundFile(false);
    _conf.setMergeScheduler(blurIndexConf.getMergeScheduler().getMergeScheduler());
    _snapshotIndexDeletionPolicy = new SnapshotIndexDeletionPolicy(_tableContext.getConfiguration(),
        SnapshotIndexDeletionPolicy.getGenerationsPath(_shardContext.getHdfsDirPath()));
    _policy = new IndexDeletionPolicyReader(_snapshotIndexDeletionPolicy);
    _conf.setIndexDeletionPolicy(_policy);
    BlurConfiguration blurConfiguration = _tableContext.getBlurConfiguration();
    _queue = new ArrayBlockingQueue<RowMutation>(blurConfiguration.getInt(BLUR_SHARD_QUEUE_MAX_INMEMORY_LENGTH, 100));
    _mutationQueueProcessor = new MutationQueueProcessor(_queue, this, _shardContext, _writesWaiting);

    _directory = blurIndexConf.getDirectory();
    if (!DirectoryReader.indexExists(_directory)) {
      new BlurIndexWriter(_directory, _conf).close();
    }

    _indexCloser = blurIndexConf.getIndexCloser();
    DirectoryReader realDirectoryReader = DirectoryReader.open(_directory);
    DirectoryReader wrappped = wrap(realDirectoryReader);
    String message = "BlurIndexSimpleWriter - inital open";
    DirectoryReader directoryReader = checkForMemoryLeaks(wrappped, message);
    _indexReader.set(directoryReader);

    _indexImporter = new IndexImporter(_indexImporterTimer, BlurIndexSimpleWriter.this, _shardContext,
        TimeUnit.SECONDS, 10, 120, _thriftCache, _directory);

    _watchForIdleBulkWriters = new TimerTask() {
      @Override
      public void run() {
        try {
          watchForIdleBulkWriters();
        } catch (Throwable t) {
          LOG.error("Unknown error.", t);
        }
      }

      private void watchForIdleBulkWriters() {
        for (BulkEntry bulkEntry : _bulkWriters.values()) {
          bulkEntry._lock.lock();
          try {
            if (!bulkEntry.isClosed() && bulkEntry.isIdle()) {
              LOG.info("Bulk Entry [{0}] has become idle and now closing.", bulkEntry);
              try {
                bulkEntry.close();
              } catch (IOException e) {
                LOG.error("Unkown error while trying to close bulk writer when it became idle.", e);
              }
            }
          } finally {
            bulkEntry._lock.unlock();
          }
        }
      }
    };
    long delay = TimeUnit.SECONDS.toMillis(30);
    _bulkIndexingTimer.schedule(_watchForIdleBulkWriters, delay, delay);
    _watchForIdleWriter = new TimerTask() {
      @Override
      public void run() {
        try {
          closeWriter();
        } catch (Throwable t) {
          LOG.error("Unknown error while trying to close idle writer.", t);
        }
      }
    };
    _indexWriterTimer.schedule(_watchForIdleWriter, _maxWriterIdle, _maxWriterIdle);
  }

  public int getReaderGenerationCount() {
    return _policy.getReaderGenerationCount();
  }

  private String getDefaultReadMaskMessage(TableContext tableContext) {
    BlurConfiguration blurConfiguration = tableContext.getBlurConfiguration();
    String message = blurConfiguration.get(BLUR_RECORD_SECURITY_DEFAULT_READMASK_MESSAGE);
    if (message == null || message.trim().isEmpty()) {
      return null;
    }
    return message.trim();
  }

  private DirectoryReader checkForMemoryLeaks(DirectoryReader wrappped, String message) {
    DirectoryReader directoryReader = MemoryLeakDetector.record(wrappped, message, _tableContext.getTable(),
        _shardContext.getShard());
    if (directoryReader instanceof ExitableReader) {
      ExitableReader exitableReader = (ExitableReader) directoryReader;
      checkForMemoryLeaks(exitableReader.getIn().leaves(), message);
    } else {
      checkForMemoryLeaks(directoryReader.leaves(), message);
    }
    return directoryReader;
  }

  private void checkForMemoryLeaks(List<AtomicReaderContext> leaves, String message) {
    for (AtomicReaderContext context : leaves) {
      AtomicReader reader = context.reader();
      MemoryLeakDetector.record(reader, message, _tableContext.getTable(), _shardContext.getShard());
    }
  }

  private DirectoryReader wrap(DirectoryReader reader) throws IOException {
    if (_makeReaderExitable) {
      reader = new ExitableReader(reader);
    }
    return _policy.register(reader);
  }

  @Override
  public IndexSearcherCloseable getIndexSearcher() throws IOException {
    return getIndexSearcher(_security);
  }

  public IndexSearcherCloseable getIndexSearcher(boolean security) throws IOException {
    final IndexReader indexReader;
    _indexRefreshReadLock.lock();
    try {
      indexReader = _indexReader.get();
      indexReader.incRef();
    } finally {
      _indexRefreshReadLock.unlock();
    }
    if (indexReader instanceof ExitableReader) {
      ((ExitableReader) indexReader).reset();
    }
    if (security) {
      return getSecureIndexSearcher(indexReader);
    } else {
      return getInsecureIndexSearcher(indexReader);
    }
  }

  private IndexSearcherCloseable getSecureIndexSearcher(final IndexReader indexReader) throws IOException {
    String readStr = null;
    String discoverStr = null;
    User user = UserContext.getUser();
    if (user != null) {
      Map<String, String> attributes = user.getAttributes();
      if (attributes != null) {
        readStr = attributes.get(ACL_READ);
        discoverStr = attributes.get(ACL_DISCOVER);
      }
    }
    Collection<String> readAuthorizations = toCollection(readStr);
    Collection<String> discoverAuthorizations = toCollection(discoverStr);
    return new IndexSearcherCloseableSecureBase(indexReader, _searchThreadPool, _accessControlFactory,
        readAuthorizations, discoverAuthorizations, _discoverableFields, _defaultReadMaskMessage) {
      private boolean _closed;

      @Override
      public Directory getDirectory() {
        return _directory;
      }

      @Override
      public synchronized void close() throws IOException {
        if (!_closed) {
          indexReader.decRef();
          _closed = true;
        } else {
          // Not really sure why some indexes get closed called twice on them.
          // This is in place to log it.
          if (LOG.isDebugEnabled()) {
            LOG.debug("Searcher already closed [{0}].", new Throwable(), this);
          }
        }
      }
    };
  }

  @SuppressWarnings("unchecked")
  private Collection<String> toCollection(String aclStr) {
    if (aclStr == null) {
      return Collections.EMPTY_LIST;
    }
    Set<String> result = new HashSet<String>();
    for (String s : _commaSplitter.split(aclStr)) {
      result.add(s);
    }
    return result;
  }

  private IndexSearcherCloseable getInsecureIndexSearcher(final IndexReader indexReader) {
    return new IndexSearcherCloseableBase(indexReader, _searchThreadPool) {
      private boolean _closed;

      @Override
      public Directory getDirectory() {
        return _directory;
      }

      @Override
      public synchronized void close() throws IOException {
        if (!_closed) {
          indexReader.decRef();
          _closed = true;
        } else {
          // Not really sure why some indexes get closed called twice on them.
          // This is in place to log it.
          if (LOG.isDebugEnabled()) {
            LOG.debug("Searcher already closed [{0}].", new Throwable(), this);
          }
        }
      }
    };
  }

  @Override
  public void close() throws IOException {
    _isClosed.set(true);
    IOUtils.cleanup(LOG, makeCloseable(_bulkIndexingTimer, _watchForIdleBulkWriters),
        makeCloseable(_indexWriterTimer, _watchForIdleWriter), _indexImporter, _mutationQueueProcessor,
        makeCloseable(_writer.get()), _indexReader.get(), _directory);
  }

  private Closeable makeCloseable(final BlurIndexWriter blurIndexWriter) {
    return new Closeable() {
      @Override
      public void close() throws IOException {
        if (blurIndexWriter != null) {
          blurIndexWriter.close(false);
        }
      }
    };
  }

  private Closeable makeCloseable(Timer timer, final TimerTask timerTask) {
    return new Closeable() {
      @Override
      public void close() throws IOException {
        timerTask.cancel();
        timer.purge();
      }
    };
  }

  private void closeWriter() {
    _writeLock.lock();
    try {
      if (_lastWrite.get() + _maxWriterIdle < System.currentTimeMillis()) {
        BlurIndexWriter writer = _writer.getAndSet(null);
        if (writer != null) {
          LOG.info("Closing idle writer for table [{0}] shard [{1}]", _tableContext.getTable(),
              _shardContext.getShard());
          IOUtils.cleanup(LOG, writer);
        }
      }
    } finally {
      _writeLock.unlock();
    }
  }

  /**
   * Testing only.
   */
  protected boolean isWriterClosed() {
    return _writer.get() == null;
  }

  private BlurIndexWriter getBlurIndexWriter() throws IOException {
    _writeLock.lock();
    try {
      BlurIndexWriter blurIndexWriter = _writer.get();
      if (blurIndexWriter == null) {
        blurIndexWriter = new BlurIndexWriter(_directory, _conf.clone());
        _writer.set(blurIndexWriter);
        _lastWrite.set(System.currentTimeMillis());
      }
      return blurIndexWriter;
    } finally {
      _writeLock.unlock();
    }
  }

  private void resetBlurIndexWriter() {
    _writeLock.lock();
    try {
      _writer.set(null);
    } finally {
      _writeLock.unlock();
    }
  }

  @Override
  public synchronized void optimize(final int numberOfSegmentsPerShard) throws IOException {
    final String table = _tableContext.getTable();
    final String shard = _shardContext.getShard();
    if (_optimizeThread != null && _optimizeThread.isAlive()) {
      LOG.info("Already running an optimize on table [{0}] shard [{1}]", table, shard);
      return;
    }
    _optimizeThread = new Thread(new Runnable() {
      @Override
      public void run() {
        try {
          BlurIndexWriter writer = getBlurIndexWriter();
          writer.forceMerge(numberOfSegmentsPerShard, true);
          _writeLock.lock();
          try {
            commit();
          } finally {
            _writeLock.unlock();
          }
        } catch (Exception e) {
          LOG.error("Unknown error during optimize on table [{0}] shard [{1}]", e, table, shard);
        }
      }
    });
    _optimizeThread.setDaemon(true);
    _optimizeThread.setName("Optimize table [" + table + "] shard [" + shard + "]");
    _optimizeThread.start();
  }

  @Override
  public void createSnapshot(String name) throws IOException {
    _snapshotIndexDeletionPolicy.createSnapshot(name, _indexReader.get(), _context);
  }

  @Override
  public void removeSnapshot(String name) throws IOException {
    _snapshotIndexDeletionPolicy.removeSnapshot(name, _context);
  }

  @Override
  public List<String> getSnapshots() throws IOException {
    return new ArrayList<String>(_snapshotIndexDeletionPolicy.getSnapshots());
  }

  private void commit() throws IOException {
    Tracer trace1 = Trace.trace("prepareCommit");
    BlurIndexWriter writer = getBlurIndexWriter();
    writer.prepareCommit();
    trace1.done();

    Tracer trace2 = Trace.trace("commit");
    writer.commit();
    trace2.done();

    Tracer trace3 = Trace.trace("index refresh");
    DirectoryReader currentReader = _indexReader.get();
    DirectoryReader newReader = DirectoryReader.openIfChanged(currentReader);
    if (newReader == null) {
      LOG.debug("Reader should be new after commit for table [{0}] shard [{1}].", _tableContext.getTable(),
          _shardContext.getShard());
    } else {
      DirectoryReader reader = wrap(newReader);
      checkForMemoryLeaks(reader, "BlurIndexSimpleWriter - reopen table [{0}] shard [{1}]");
      _indexRefreshWriteLock.lock();
      try {
        _indexReader.set(reader);
      } finally {
        _indexRefreshWriteLock.unlock();
      }
      _indexCloser.close(currentReader);
    }
    trace3.done();
  }

  @Override
  public void process(IndexAction indexAction) throws IOException {
    _writesWaiting.incrementAndGet();
    _writeLock.lock();
    _writesWaiting.decrementAndGet();
    indexAction.setWritesWaiting(_writesWaiting);
    BlurIndexWriter writer = getBlurIndexWriter();
    IndexSearcherCloseable indexSearcher = null;
    try {
      indexSearcher = getIndexSearcher(false);
      indexAction.performMutate(indexSearcher, writer);
      indexAction.doPreCommit(indexSearcher, writer);
      commit();
      indexAction.doPostCommit(writer);
    } catch (Exception e) {
      indexAction.doPreRollback(writer);
      writer.rollback();
      resetBlurIndexWriter();
      indexAction.doPostRollback(writer);
      throw new IOException("Unknown error during mutation", e);
    } finally {
      if (_thriftCache != null) {
        _thriftCache.clearTable(_tableContext.getTable());
      }
      if (indexSearcher != null) {
        indexSearcher.close();
      }
      _lastWrite.set(System.currentTimeMillis());
      _writeLock.unlock();
    }
  }

  public Path getSnapshotsDirectoryPath() {
    return _snapshotIndexDeletionPolicy.getSnapshotsDirectoryPath();
  }

  @Override
  public void enqueue(List<RowMutation> mutations) throws IOException {
    startQueueIfNeeded();
    try {
      for (RowMutation mutation : mutations) {
        _queue.put(mutation);
      }
      synchronized (_queue) {
        _queue.notifyAll();
      }
    } catch (InterruptedException e) {
      throw new IOException(e);
    }
  }

  private void startQueueIfNeeded() {
    _mutationQueueProcessor.startIfNotRunning();
  }

  static class BulkEntry {

    private final long _idleTime = TimeUnit.SECONDS.toNanos(30);
    private final Path _parentPath;
    private final String _bulkId;
    private final TableContext _tableContext;
    private final ShardContext _shardContext;
    private final Configuration _configuration;
    private final FileSystem _fileSystem;
    private final String _table;
    private final String _shard;
    private final Lock _lock = new ReentrantReadWriteLock().writeLock();

    private volatile SequenceFile.Writer _writer;
    private volatile long _lastWrite;
    private volatile int _count = 0;

    public BulkEntry(String bulkId, Path parentPath, ShardContext shardContext) throws IOException {
      _bulkId = bulkId;
      _parentPath = parentPath;
      _shardContext = shardContext;
      _tableContext = shardContext.getTableContext();
      _configuration = _tableContext.getConfiguration();
      _fileSystem = _parentPath.getFileSystem(_configuration);
      _shard = _shardContext.getShard();
      _table = _tableContext.getTable();
    }

    public boolean isClosed() {
      return _writer == null;
    }

    private Writer openSeqWriter() throws IOException {
      Progressable progress = new Progressable() {
        @Override
        public void progress() {

        }
      };
      final CompressionCodec codec;
      final CompressionType type;

      if (isSnappyCodecLoaded(_configuration)) {
        codec = new SnappyCodec();
        type = CompressionType.BLOCK;
      } else {
        codec = new DefaultCodec();
        type = CompressionType.NONE;
      }

      Path path = new Path(_parentPath, _shard + "." + _count + ".unsorted.seq");

      _count++;

      return SequenceFile.createWriter(_fileSystem, _configuration, path, Text.class, RowMutationWritable.class, type,
          codec, progress);
    }

    public void close() throws IOException {
      _lock.lock();
      try {
        if (_writer != null) {
          _writer.close();
          _writer = null;
        }
      } finally {
        _lock.unlock();
      }
    }

    public void append(Text key, RowMutationWritable rowMutationWritable) throws IOException {
      _lock.lock();
      try {
        getWriter().append(key, rowMutationWritable);
        _lastWrite = System.nanoTime();
      } finally {
        _lock.unlock();
      }
    }

    private SequenceFile.Writer getWriter() throws IOException {
      if (_writer == null) {
        _writer = openSeqWriter();
        _lastWrite = System.nanoTime();
      }
      return _writer;
    }

    public boolean isIdle() {
      if (_lastWrite + _idleTime < System.nanoTime()) {
        return true;
      }
      return false;
    }

    public List<Path> getUnsortedFiles() throws IOException {
      FileStatus[] listStatus = _fileSystem.listStatus(_parentPath, new PathFilter() {
        @Override
        public boolean accept(Path path) {
          return path.getName().matches(_shard + "\\.[0-9].*\\.unsorted\\.seq");
        }
      });

      List<Path> unsortedPaths = new ArrayList<Path>();
      for (FileStatus fileStatus : listStatus) {
        unsortedPaths.add(fileStatus.getPath());
      }
      return unsortedPaths;
    }

    public void cleanupFiles(List<Path> unsortedPaths, Path sorted) throws IOException {
      if (unsortedPaths != null) {
        for (Path p : unsortedPaths) {
          _fileSystem.delete(p, false);
        }
      }
      if (sorted != null) {
        _fileSystem.delete(sorted, false);
      }
      removeParentIfLastFile(_fileSystem, _parentPath);
    }

    public IndexAction getIndexAction() throws IOException {
      return new IndexAction() {
        private Path _sorted;
        private List<Path> _unsortedPaths;

        @Override
        public void performMutate(IndexSearcherCloseable searcher, IndexWriter writer) throws IOException {
          Configuration configuration = _tableContext.getConfiguration();

          BlurConfiguration blurConfiguration = _tableContext.getBlurConfiguration();

          SequenceFile.Sorter sorter = new Sorter(_fileSystem, Text.class, RowMutationWritable.class, configuration);
          // This should support up to ~100 GB per shard, probably have
          // incremental updates in that batch size.
          sorter.setFactor(blurConfiguration.getInt(BLUR_SHARD_INDEX_WRITER_SORT_FACTOR, 10000));
          sorter.setMemory(blurConfiguration.getInt(BLUR_SHARD_INDEX_WRITER_SORT_MEMORY, 10 * 1024 * 1024));

          _unsortedPaths = getUnsortedFiles();

          _sorted = new Path(_parentPath, _shard + ".sorted.seq");

          LOG.info("Shard [{2}/{3}] Id [{4}] Sorting mutates paths [{0}] sorted path [{1}]", _unsortedPaths, _sorted,
              _table, _shard, _bulkId);
          sorter.sort(_unsortedPaths.toArray(new Path[_unsortedPaths.size()]), _sorted, true);

          LOG.info("Shard [{1}/{2}] Id [{3}] Applying mutates sorted path [{0}]", _sorted, _table, _shard, _bulkId);
          Reader reader = new SequenceFile.Reader(_fileSystem, _sorted, configuration);

          Text key = new Text();
          RowMutationWritable value = new RowMutationWritable();

          Text last = null;
          List<RowMutation> list = new ArrayList<RowMutation>();
          while (reader.next(key, value)) {
            if (!key.equals(last)) {
              flushMutates(searcher, writer, list);
              last = new Text(key);
              list.clear();
            }
            list.add(value.getRowMutation().deepCopy());
          }
          flushMutates(searcher, writer, list);
          reader.close();
          LOG.info("Shard [{0}/{1}] Id [{2}] Finished applying mutates starting commit.", _table, _shard, _bulkId);
        }

        private void flushMutates(IndexSearcherCloseable searcher, IndexWriter writer, List<RowMutation> list)
            throws IOException {
          if (!list.isEmpty()) {
            List<RowMutation> reduceMutates;
            try {
              reduceMutates = MutatableAction.reduceMutates(list);
            } catch (BlurException e) {
              throw new IOException(e);
            }
            for (RowMutation mutation : reduceMutates) {
              MutatableAction mutatableAction = new MutatableAction(_shardContext);
              mutatableAction.mutate(mutation);
              mutatableAction.performMutate(searcher, writer);
            }
          }
        }

        @Override
        public void doPreRollback(IndexWriter writer) throws IOException {

        }

        @Override
        public void doPreCommit(IndexSearcherCloseable indexSearcher, IndexWriter writer) throws IOException {

        }

        @Override
        public void doPostRollback(IndexWriter writer) throws IOException {
          cleanupFiles(_unsortedPaths, _sorted);
        }

        @Override
        public void doPostCommit(IndexWriter writer) throws IOException {
          cleanupFiles(_unsortedPaths, _sorted);
        }
      };
    }

    @Override
    public String toString() {
      return "BulkEntry [_bulkId=" + _bulkId + ", _table=" + _table + ", _shard=" + _shard + ", _idleTime=" + _idleTime
          + ", _lastWrite=" + _lastWrite + ", _count=" + _count + "]";
    }

  }

  public synchronized BulkEntry startBulkMutate(String bulkId) throws IOException {
    BulkEntry bulkEntry = _bulkWriters.get(bulkId);
    if (bulkEntry == null) {
      Path tablePath = _tableContext.getTablePath();
      Path bulk = new Path(tablePath, "bulk");
      Path bulkInstance = new Path(bulk, bulkId);
      Path path = new Path(bulkInstance, _shardContext.getShard() + ".notsorted.seq");

      bulkEntry = new BulkEntry(bulkId, path, _shardContext);
      _bulkWriters.put(bulkId, bulkEntry);
    } else {
      LOG.info("Bulk [{0}] mutate already started on shard [{1}] in table [{2}].", bulkId, _shardContext.getShard(),
          _tableContext.getTable());
    }
    return bulkEntry;
  }

  @Override
  public void finishBulkMutate(final String bulkId, boolean apply, boolean blockUntilComplete) throws IOException {
    final String table = _tableContext.getTable();
    final String shard = _shardContext.getShard();

    final BulkEntry bulkEntry = _bulkWriters.get(bulkId);
    if (bulkEntry == null) {
      LOG.info("Shard [{2}/{3}] Id [{0}] Nothing to apply.", bulkId, apply, table, shard);
      return;
    }
    LOG.info("Shard [{2}/{3}] Id [{0}] Finishing bulk mutate apply [{1}]", bulkId, apply, table, shard);
    bulkEntry.close();

    if (!apply) {
      bulkEntry.cleanupFiles(bulkEntry.getUnsortedFiles(), null);
    } else {
      final IndexAction indexAction = bulkEntry.getIndexAction();
      if (blockUntilComplete) {
        StoreDirection.LONG_TERM.set(true);
        try {
          process(indexAction);
        } finally {
          StoreDirection.LONG_TERM.set(false);
        }
      } else {
        Thread thread = new Thread(new Runnable() {
          @Override
          public void run() {
            try {
              StoreDirection.LONG_TERM.set(true);
              process(indexAction);
            } catch (IOException e) {
              LOG.error("Shard [{0}/{1}] Id [{2}] Unknown error while trying to finish the bulk updates.", e, table,
                  shard, bulkId);
            } finally {
              StoreDirection.LONG_TERM.set(false);
            }
          }
        });
        thread.setName("Bulk Finishing Thread Table [" + table + "] Shard [" + shard + "] BulkId [" + bulkId + "]");
        thread.start();
      }
    }
  }

  @Override
  public void addBulkMutate(String bulkId, RowMutation mutation) throws IOException {
    BulkEntry bulkEntry = _bulkWriters.get(bulkId);
    if (bulkEntry == null) {
      bulkEntry = startBulkMutate(bulkId);
    }
    RowMutationWritable rowMutationWritable = new RowMutationWritable();
    rowMutationWritable.setRowMutation(mutation);
    bulkEntry.append(getKey(mutation), rowMutationWritable);
  }

  private Text getKey(RowMutation mutation) {
    return new Text(mutation.getRowId());
  }

  private static void removeParentIfLastFile(final FileSystem fileSystem, Path parent) throws IOException {
    FileStatus[] listStatus = fileSystem.listStatus(parent);
    if (listStatus != null) {
      if (listStatus.length == 0) {
        if (!fileSystem.delete(parent, false)) {
          if (fileSystem.exists(parent)) {
            LOG.error("Could not remove parent directory [{0}]", parent);
          }
        }
      }
    }
  }

  @Override
  public long getRecordCount() throws IOException {
    IndexSearcherCloseable searcher = getIndexSearcher(false);
    try {
      return searcher.getIndexReader().numDocs();
    } finally {
      if (searcher != null) {
        searcher.close();
      }
    }
  }

  @Override
  public long getRowCount() throws IOException {
    IndexSearcherCloseable searcher = getIndexSearcher(false);
    try {
      return getRowCount(searcher);
    } finally {
      if (searcher != null) {
        searcher.close();
      }
    }
  }

  protected long getRowCount(IndexSearcherCloseable searcher) throws IOException {
    TopDocs topDocs = searcher.search(
        new SuperQuery(new MatchAllDocsQuery(), ScoreType.CONSTANT, _tableContext.getDefaultPrimeDocTerm()), 1);
    return topDocs.totalHits;
  }

  @Override
  public long getIndexMemoryUsage() throws IOException {
    return 0;
  }

  @Override
  public long getSegmentCount() throws IOException {
    IndexSearcherCloseable indexSearcherClosable = getIndexSearcher(false);
    try {
      IndexReader indexReader = indexSearcherClosable.getIndexReader();
      IndexReaderContext context = indexReader.getContext();
      return context.leaves().size();
    } finally {
      indexSearcherClosable.close();
    }
  }

  private static boolean isSnappyCodecLoaded(Configuration configuration) {
    try {
      Method methodHadoop1 = SnappyCodec.class.getMethod("isNativeSnappyLoaded", new Class[] { Configuration.class });
      Boolean loaded = (Boolean) methodHadoop1.invoke(null, new Object[] { configuration });
      if (loaded != null && loaded) {
        LOG.info("Using SnappyCodec");
        return true;
      } else {
        LOG.info("Not using SnappyCodec");
        return false;
      }
    } catch (NoSuchMethodException e) {
      Method methodHadoop2;
      try {
        methodHadoop2 = SnappyCodec.class.getMethod("isNativeCodeLoaded", new Class[] {});
      } catch (NoSuchMethodException ex) {
        LOG.info("Can not determine if SnappyCodec is loaded.");
        return false;
      } catch (SecurityException ex) {
        LOG.error("Not allowed.", ex);
        return false;
      }
      Boolean loaded;
      try {
        loaded = (Boolean) methodHadoop2.invoke(null);
        if (loaded != null && loaded) {
          LOG.info("Using SnappyCodec");
          return true;
        } else {
          LOG.info("Not using SnappyCodec");
          return false;
        }
      } catch (Exception ex) {
        LOG.info("Unknown error while trying to determine if SnappyCodec is loaded.", ex);
        return false;
      }
    } catch (SecurityException e) {
      LOG.error("Not allowed.", e);
      return false;
    } catch (Exception e) {
      LOG.info("Unknown error while trying to determine if SnappyCodec is loaded.", e);
      return false;
    }
  }

  @Override
  public long getSegmentImportPendingCount() throws IOException {
    if (_indexImporter != null) {
      return _indexImporter.getSegmentImportPendingCount();
    }
    return 0l;
  }

  @Override
  public long getSegmentImportInProgressCount() throws IOException {
    if (_indexImporter != null) {
      return _indexImporter.getSegmentImportInProgressCount();
    }
    return 0l;
  }

  @Override
  public long getOnDiskSize() throws IOException {
    Path hdfsDirPath = _shardContext.getHdfsDirPath();
    Configuration configuration = _tableContext.getConfiguration();
    FileSystem fileSystem = hdfsDirPath.getFileSystem(configuration);
    ContentSummary contentSummary = fileSystem.getContentSummary(hdfsDirPath);
    return contentSummary.getLength();
  }
}