/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.lucene.index;

import java.io.Closeable;
import java.io.IOException;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import java.util.concurrent.CopyOnWriteArrayList;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.search.Query;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.NullInfoStream;
import org.apache.lucene.util.TestUtil;

/** Silly class that randomizes the indexing experience.  EG
 *  it may swap in a different merge policy/scheduler; may
 *  commit periodically; may or may not forceMerge in the end,
 *  may flush by doc count instead of RAM, etc. 
 */

public class RandomIndexWriter implements Closeable {

  public final IndexWriter w;
  private final Random r;
  int docCount;
  int flushAt;
  private double flushAtFactor = 1.0;
  private boolean getReaderCalled;
  private final Analyzer analyzer; // only if WE created it (then we close it)
  private final double softDeletesRatio;
  private final LiveIndexWriterConfig config;

  /** Returns an indexwriter that randomly mixes up thread scheduling (by yielding at test points) */
  public static IndexWriter mockIndexWriter(Directory dir, IndexWriterConfig conf, Random r) throws IOException {
    // Randomly calls Thread.yield so we mixup thread scheduling
    final Random random = new Random(r.nextLong());
    return mockIndexWriter(r, dir, conf, new TestPoint() {
      @Override
      public void apply(String message) {
        if (random.nextInt(4) == 2)
          Thread.yield();
      }
    });
  }
  
  /** Returns an indexwriter that enables the specified test point */
  public static IndexWriter mockIndexWriter(Random r, Directory dir, IndexWriterConfig conf, TestPoint testPoint) throws IOException {
    conf.setInfoStream(new TestPointInfoStream(conf.getInfoStream(), testPoint));
    DirectoryReader reader = null;
    if (r.nextBoolean() && DirectoryReader.indexExists(dir) && conf.getOpenMode() != IndexWriterConfig.OpenMode.CREATE) {
      if (LuceneTestCase.VERBOSE) {
        System.out.println("RIW: open writer from reader");
      }
      reader = DirectoryReader.open(dir);
      conf.setIndexCommit(reader.getIndexCommit());
    }

    IndexWriter iw;
    boolean success = false;
    try {
      iw = new IndexWriter(dir, conf) {
        @Override
        protected boolean isEnableTestPoints() {
          return true;
        }
      };
      success = true;
    } finally {
      if (reader != null) {
        if (success) {
          IOUtils.close(reader);
        } else {
          IOUtils.closeWhileHandlingException(reader);
        }
      }
    }
    return iw;
  }

  /** create a RandomIndexWriter with a random config: Uses MockAnalyzer */
  public RandomIndexWriter(Random r, Directory dir) throws IOException {
    this(r, dir, LuceneTestCase.newIndexWriterConfig(r, new MockAnalyzer(r)), true, r.nextBoolean());
  }
  
  /** create a RandomIndexWriter with a random config */
  public RandomIndexWriter(Random r, Directory dir, Analyzer a) throws IOException {
    this(r, dir, LuceneTestCase.newIndexWriterConfig(r, a));
  }
  
  /** create a RandomIndexWriter with the provided config */
  public RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c) throws IOException {
    this(r, dir, c, false, r.nextBoolean());
  }

  /** create a RandomIndexWriter with the provided config */
  public RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c, boolean useSoftDeletes) throws IOException {
    this(r, dir, c, false, useSoftDeletes);
  }
      
  private RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c, boolean closeAnalyzer, boolean useSoftDeletes) throws IOException {
    // TODO: this should be solved in a different way; Random should not be shared (!).
    this.r = new Random(r.nextLong());
    if (useSoftDeletes) {
      c.setSoftDeletesField("___soft_deletes");
      softDeletesRatio = 1.d / (double)1 + r.nextInt(10);
    } else {
      softDeletesRatio = 0d;
    }
    w = mockIndexWriter(dir, c, r);
    config = w.getConfig();
    flushAt = TestUtil.nextInt(r, 10, 1000);
    if (closeAnalyzer) {
      analyzer = w.getAnalyzer();
    } else {
      analyzer = null;
    }
    if (LuceneTestCase.VERBOSE) {
      System.out.println("RIW dir=" + dir);
    }

    // Make sure we sometimes test indices that don't get
    // any forced merges:
    doRandomForceMerge = !(c.getMergePolicy() instanceof NoMergePolicy) && r.nextBoolean();
  } 
  
  /**
   * Adds a Document.
   * @see IndexWriter#addDocument(Iterable)
   */
  public <T extends IndexableField> long addDocument(final Iterable<T> doc) throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    long seqNo;
    if (r.nextInt(5) == 3) {
      // TODO: maybe, we should simply buffer up added docs
      // (but we need to clone them), and only when
      // getReader, commit, etc. are called, we do an
      // addDocuments?  Would be better testing.
      seqNo = w.addDocuments(new Iterable<Iterable<T>>() {

        @Override
        public Iterator<Iterable<T>> iterator() {
          return new Iterator<Iterable<T>>() {

            boolean done;
            
            @Override
            public boolean hasNext() {
              return !done;
            }

            @Override
            public void remove() {
              throw new UnsupportedOperationException();
            }

            @Override
            public Iterable<T> next() {
              if (done) {
                throw new IllegalStateException();
              }
              done = true;
              return doc;
            }
          };
        }
        });
    } else {
      seqNo = w.addDocument(doc);
    }
    
    maybeFlushOrCommit();

    return seqNo;
  }

  private void maybeFlushOrCommit() throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    if (docCount++ == flushAt) {
      if (r.nextBoolean()) {
        flushAllBuffersSequentially();
      } else if (r.nextBoolean()) {
        if (LuceneTestCase.VERBOSE) {
          System.out.println("RIW.add/updateDocument: now doing a flush at docCount=" + docCount);
        }
        w.flush();
      } else {
        if (LuceneTestCase.VERBOSE) {
          System.out.println("RIW.add/updateDocument: now doing a commit at docCount=" + docCount);
        }
        w.commit();
      }
      flushAt += TestUtil.nextInt(r, (int) (flushAtFactor * 10), (int) (flushAtFactor * 1000));
      if (flushAtFactor < 2e6) {
        // gradually but exponentially increase time b/w flushes
        flushAtFactor *= 1.05;
      }
    }
  }

  private void flushAllBuffersSequentially() throws IOException {
    if (LuceneTestCase.VERBOSE) {
      System.out.println("RIW.add/updateDocument: now flushing the largest writer at docCount=" + docCount);
    }
    int threadPoolSize = w.docWriter.perThreadPool.size();
    int numFlushes = Math.min(1, r.nextInt(threadPoolSize+1));
    for (int i = 0; i < numFlushes; i++) {
      if (w.flushNextBuffer() == false) {
        break; // stop once we didn't flush anything
      }
    }
  }
  
  public long addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    long seqNo = w.addDocuments(docs);
    maybeFlushOrCommit();
    return seqNo;
  }

  public long updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    long seqNo;
    if (useSoftDeletes()) {
      seqNo = w.softUpdateDocuments(delTerm, docs, new NumericDocValuesField(config.getSoftDeletesField(), 1));
    } else {
      seqNo = w.updateDocuments(delTerm, docs);
    }
    maybeFlushOrCommit();
    return seqNo;
  }

  private boolean useSoftDeletes() {
    return r.nextDouble() < softDeletesRatio;
  }

  /**
   * Updates a document.
   * @see IndexWriter#updateDocument(Term, Iterable)
   */
  public <T extends IndexableField> long updateDocument(Term t, final Iterable<T> doc) throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    final long seqNo;
    if (useSoftDeletes()) {
      if (r.nextInt(5) == 3) {
        seqNo = w.softUpdateDocuments(t, Arrays.asList(doc), new NumericDocValuesField(config.getSoftDeletesField(), 1));
      } else {
        seqNo = w.softUpdateDocument(t, doc, new NumericDocValuesField(config.getSoftDeletesField(), 1));
      }
    } else {
      if (r.nextInt(5) == 3) {
        seqNo = w.updateDocuments(t, Arrays.asList(doc));
      } else {
        seqNo = w.updateDocument(t, doc);
      }
    }
    maybeFlushOrCommit();

    return seqNo;
  }
  
  public long addIndexes(Directory... dirs) throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    return w.addIndexes(dirs);
  }

  public long addIndexes(CodecReader... readers) throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    return w.addIndexes(readers);
  }
  
  public long updateNumericDocValue(Term term, String field, Long value) throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    return w.updateNumericDocValue(term, field, value);
  }
  
  public long updateBinaryDocValue(Term term, String field, BytesRef value) throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    return w.updateBinaryDocValue(term, field, value);
  }
  
  public long updateDocValues(Term term, Field... updates) throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    return w.updateDocValues(term, updates);
  }
  
  public long deleteDocuments(Term term) throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    return w.deleteDocuments(term);
  }

  public long deleteDocuments(Query q) throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    return w.deleteDocuments(q);
  }

  public long commit() throws IOException {
    return commit(r.nextInt(10) == 0);
  }
  
  public long commit(boolean flushConcurrently) throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    if (flushConcurrently) {
      List<Throwable> throwableList = new CopyOnWriteArrayList<>();
      Thread thread = new Thread(() -> {
        try {
          flushAllBuffersSequentially();
        } catch (Throwable e) {
          throwableList.add(e);
        }
      });
      thread.start();
      try {
        return w.commit();
      } catch (Throwable t) {
        throwableList.add(t);
      } finally {
        try {
          // make sure we wait for the thread to join otherwise it might still be processing events
          // and the IW won't be fully closed in the case of a fatal exception
          thread.join();
        } catch (InterruptedException e) {
          throwableList.add(e);
        }
      }
      if (throwableList.size() != 0) {
        Throwable primary = throwableList.get(0);
        for (int i = 1; i < throwableList.size(); i++) {
          primary.addSuppressed(throwableList.get(i));
        }
        if (primary instanceof IOException) {
          throw (IOException)primary;
        } else if (primary instanceof RuntimeException) {
          throw (RuntimeException)primary;
        } else {
          throw new AssertionError(primary);
        }
      }

    }
    return w.commit();
  }

  public IndexWriter.DocStats getDocStats() {
    return w.getDocStats();
  }

  public long deleteAll() throws IOException {
    return w.deleteAll();
  }

  public DirectoryReader getReader() throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    return getReader(true, false);
  }

  private boolean doRandomForceMerge;
  private boolean doRandomForceMergeAssert;

  public void forceMergeDeletes(boolean doWait) throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    w.forceMergeDeletes(doWait);
  }

  public void forceMergeDeletes() throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    w.forceMergeDeletes();
  }

  public void setDoRandomForceMerge(boolean v) {
    doRandomForceMerge = v;
  }

  public void setDoRandomForceMergeAssert(boolean v) {
    doRandomForceMergeAssert = v;
  }

  private void doRandomForceMerge() throws IOException {
    if (doRandomForceMerge) {
      final int segCount = w.getSegmentCount();
      if (r.nextBoolean() || segCount == 0) {
        // full forceMerge
        if (LuceneTestCase.VERBOSE) {
          System.out.println("RIW: doRandomForceMerge(1)");
        }
        w.forceMerge(1);
      } else if (r.nextBoolean()) {
        // partial forceMerge
        final int limit = TestUtil.nextInt(r, 1, segCount);
        if (LuceneTestCase.VERBOSE) {
          System.out.println("RIW: doRandomForceMerge(" + limit + ")");
        }
        w.forceMerge(limit);
        if (limit == 1 || (config.getMergePolicy() instanceof TieredMergePolicy) == false) {
          assert !doRandomForceMergeAssert || w.getSegmentCount() <= limit : "limit=" + limit + " actual=" + w.getSegmentCount();
        }
      } else {
        if (LuceneTestCase.VERBOSE) {
          System.out.println("RIW: do random forceMergeDeletes()");
        }
        w.forceMergeDeletes();
      }
    }
  }

  public DirectoryReader getReader(boolean applyDeletions, boolean writeAllDeletes) throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    getReaderCalled = true;
    if (r.nextInt(20) == 2) {
      doRandomForceMerge();
    }
    if (!applyDeletions || r.nextBoolean()) {
      // if we have soft deletes we can't open from a directory
      if (LuceneTestCase.VERBOSE) {
        System.out.println("RIW.getReader: use NRT reader");
      }
      if (r.nextInt(5) == 1) {
        w.commit();
      }
      return w.getReader(applyDeletions, writeAllDeletes);
    } else {
      if (LuceneTestCase.VERBOSE) {
        System.out.println("RIW.getReader: open new reader");
      }
      w.commit();
      if (r.nextBoolean()) {
        DirectoryReader reader = DirectoryReader.open(w.getDirectory());
        if (config.getSoftDeletesField() != null) {
          return new SoftDeletesDirectoryReaderWrapper(reader, config.getSoftDeletesField());
        } else {
          return reader;
        }
      } else {
        return w.getReader(applyDeletions, writeAllDeletes);
      }
    }
  }

  /**
   * Close this writer.
   * @see IndexWriter#close()
   */
  @Override
  public void close() throws IOException {
    boolean success = false;
    try {
      if (w.isClosed() == false) {
        LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
      }
      // if someone isn't using getReader() API, we want to be sure to
      // forceMerge since presumably they might open a reader on the dir.
      if (getReaderCalled == false && r.nextInt(8) == 2 && w.isClosed() == false) {
        doRandomForceMerge();
        if (config.getCommitOnClose() == false) {
          // index may have changed, must commit the changes, or otherwise they are discarded by the call to close()
          w.commit();
        }
      }
      success = true;
    } finally {
      if (success) {
        IOUtils.close(w, analyzer);
      } else {
        IOUtils.closeWhileHandlingException(w, analyzer);
      }
    }
  }

  /**
   * Forces a forceMerge.
   * <p>
   * NOTE: this should be avoided in tests unless absolutely necessary,
   * as it will result in less test coverage.
   * @see IndexWriter#forceMerge(int)
   */
  public void forceMerge(int maxSegmentCount) throws IOException {
    LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    w.forceMerge(maxSegmentCount);
  }
  
  static final class TestPointInfoStream extends InfoStream {
    private final InfoStream delegate;
    private final TestPoint testPoint;
    
    public TestPointInfoStream(InfoStream delegate, TestPoint testPoint) {
      this.delegate = delegate == null ? new NullInfoStream(): delegate;
      this.testPoint = testPoint;
    }

    @Override
    public void close() throws IOException {
      delegate.close();
    }

    @Override
    public void message(String component, String message) {
      if ("TP".equals(component)) {
        testPoint.apply(message);
      }
      if (delegate.isEnabled(component)) {
        delegate.message(component, message);
      }
    }
    
    @Override
    public boolean isEnabled(String component) {
      return "TP".equals(component) || delegate.isEnabled(component);
    }
  }
  
  /** Writes all in-memory segments to the {@link Directory}. */
  public final void flush() throws IOException {
    w.flush();
  }

  /**
   * Simple interface that is executed for each <code>TP</code> {@link InfoStream} component
   * message. See also {@link RandomIndexWriter#mockIndexWriter(Random, Directory, IndexWriterConfig, TestPoint)}
   */
  public interface TestPoint {
    void apply(String message);
  }
}