Java Code Examples for org.apache.lucene.store.Directory#listAll()

@Override
public boolean exists(String path) throws IOException {
  String fullPath = normalize(path);
  synchronized (this) {
    final CacheValue cacheValue = byPathCache.get(fullPath);
    if (null == cacheValue) {
      return false;
    }
    final Directory directory = cacheValue.directory;
    if (null == directory) {
      return false;
    }
    if (0 < directory.listAll().length) {
      return true;
    } 
    return false;
  }
}
 

/**
 * This method removes all lucene files from the given directory. It will first try to delete all commit points / segments
 * files to ensure broken commits or corrupted indices will not be opened in the future. If any of the segment files can't be deleted
 * this operation fails.
 */
public static void cleanLuceneIndex(Directory directory) throws IOException {
    try (Lock writeLock = directory.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
        for (final String file : directory.listAll()) {
            if (file.startsWith(IndexFileNames.SEGMENTS) || file.equals(IndexFileNames.OLD_SEGMENTS_GEN)) {
                directory.deleteFile(file); // remove all segment_N files
            }
        }
    }
    try (IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)
            .setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
            .setMergePolicy(NoMergePolicy.INSTANCE) // no merges
            .setCommitOnClose(false) // no commits
            .setOpenMode(IndexWriterConfig.OpenMode.CREATE))) { // force creation - don't append...
        // do nothing and close this will kick of IndexFileDeleter which will remove all pending files
    }
}
 

/**
 * Returns <code>true</code> if an index likely exists at
 * the specified directory.  Note that if a corrupt index
 * exists, or if an index in the process of committing 
 * @param  directory the directory to check for an index
 * @return <code>true</code> if an index exists; <code>false</code> otherwise
 */
public static boolean indexExists(Directory directory) throws IOException {
  // LUCENE-2812, LUCENE-2727, LUCENE-4738: this logic will
  // return true in cases that should arguably be false,
  // such as only IW.prepareCommit has been called, or a
  // corrupt first commit, but it's too deadly to make
  // this logic "smarter" and risk accidentally returning
  // false due to various cases like file description
  // exhaustion, access denied, etc., because in that
  // case IndexWriter may delete the entire index.  It's
  // safer to err towards "index exists" than try to be
  // smart about detecting not-yet-fully-committed or
  // corrupt indices.  This means that IndexWriter will
  // throw an exception on such indices and the app must
  // resolve the situation manually:
  String[] files = directory.listAll();

  String prefix = IndexFileNames.SEGMENTS + "_";
  for(String file : files) {
    if (file.startsWith(prefix)) {
      return true;
    }
  }
  return false;
}
 

public void testDeleteUnusedUpdatesFiles() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("id", "d0", Store.NO));
  doc.add(new NumericDocValuesField("f1", 1L));
  doc.add(new NumericDocValuesField("f2", 1L));
  writer.addDocument(doc);

  // update each field twice to make sure all unneeded files are deleted
  for (String f : new String[] { "f1", "f2" }) {
    writer.updateNumericDocValue(new Term("id", "d0"), f, 2L);
    writer.commit();
    int numFiles = dir.listAll().length;
    
    // update again, number of files shouldn't change (old field's gen is
    // removed) 
    writer.updateNumericDocValue(new Term("id", "d0"), f, 3L);
    writer.commit();
    
    assertEquals(numFiles, dir.listAll().length);
  }
  
  writer.close();
  dir.close();
}
 

public static void assertNoUnreferencedFiles(Directory dir, String message) throws IOException {
  String[] startFiles = dir.listAll();
  new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))).rollback();
  String[] endFiles = dir.listAll();

  Arrays.sort(startFiles);
  Arrays.sort(endFiles);

  if (!Arrays.equals(startFiles, endFiles)) {
    fail(message + ": before delete:\n    " + arrayToString(startFiles) + "\n  after delete:\n    " + arrayToString(endFiles));
  }
}
 

private void assertNoPrx(Directory dir) throws Throwable {
  final String[] files = dir.listAll();
  for(int i=0;i<files.length;i++) {
    assertFalse(files[i].endsWith(".prx"));
    assertFalse(files[i].endsWith(".pos"));
  }
}
 

private void checkBitFlips(Directory dir) throws IOException {
  for(String name : dir.listAll()) {
    if (name.equals(IndexWriter.WRITE_LOCK_NAME) == false) {
      corruptFile(dir, name);
    }
  }
}
 

private boolean hasUnusedFiles(Directory indexDir, IndexCommit commit) throws IOException {
  String segmentsFileName = commit.getSegmentsFileName();
  SegmentInfos infos = SegmentInfos.readCommit(indexDir, segmentsFileName);
  Set<String> currentFiles = new HashSet<>(infos.files(true));
  String[] allFiles = indexDir.listAll();
  for (String file : allFiles) {
    if (!file.equals(segmentsFileName) && !currentFiles.contains(file) && !file.endsWith(".lock")) {
      log.info("Found unused file: {}", file);
      return true;
    }
  }
  return false;
}
 

/**
 * Returns a new Directory instance, using the specified random
 * with contents copied from the provided directory. See 
 * {@link #newDirectory()} for more information.
 */
public static BaseDirectoryWrapper newDirectory(Random r, Directory d) throws IOException {
  Directory impl = newDirectoryImpl(r, TEST_DIRECTORY);
  for (String file : d.listAll()) {
    if (file.startsWith(IndexFileNames.SEGMENTS) || IndexFileNames.CODEC_FILE_PATTERN.matcher(file).matches()) {
      impl.copyFrom(d, file, file, newIOContext(r));
    }
  }
  return wrapDirectory(r, impl, rarely(r), false);
}
 

private void swapFiles(Directory dir1, Directory dir2) throws IOException {
  if (VERBOSE) {
    System.out.println("TEST: dir1 files: " + Arrays.toString(dir1.listAll()));
    System.out.println("TEST: dir2 files: " + Arrays.toString(dir2.listAll()));
  }
  for(String name : dir1.listAll()) {
    if (name.equals(IndexWriter.WRITE_LOCK_NAME)) {
      continue;
    }
    swapOneFile(dir1, dir2, name);
  }
}
 

public static boolean anyFilesExceptWriteLock(Directory dir) throws IOException {
  String[] files = dir.listAll();
  if (files.length > 1 || (files.length == 1 && !files[0].equals("write.lock"))) {
    return true;
  } else {
    return false;
  }
}
 

static long dirSize(Directory d) throws IOException {
  long size = 0;
  for (String file : d.listAll()) {
    size += d.fileLength(file);
  }
  return size;
}
 

/**
 * Cleans up the index directory from old index files. This method uses the
 * last commit found by {@link #getLastCommit(Directory)}. If it matches the
 * expected segmentsFile, then all files not referenced by this commit point
 * are deleted.
 * <p>
 * <b>NOTE:</b> this method does a best effort attempt to clean the index
 * directory. It suppresses any exceptions that occur, as this can be retried
 * the next time.
 */
public static void cleanupOldIndexFiles(Directory dir, String segmentsFile, InfoStream infoStream) {
  try {
    IndexCommit commit = getLastCommit(dir);
    // commit == null means weird IO errors occurred, ignore them
    // if there were any IO errors reading the expected commit point (i.e.
    // segments files mismatch), then ignore that commit either.
    if (commit != null && commit.getSegmentsFileName().equals(segmentsFile)) {
      Set<String> commitFiles = new HashSet<>(commit.getFileNames());
      Matcher matcher = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
      for (String file : dir.listAll()) {
        if (!commitFiles.contains(file)
            && (matcher.reset(file).matches() || file.startsWith(IndexFileNames.SEGMENTS))) {
          // suppress exceptions, it's just a best effort
          IOUtils.deleteFilesIgnoringExceptions(dir, file);
        }
      }
    }
  } catch (Throwable t) {
    // ignore any errors that happen during this state and only log it. this
    // cleanup will have a chance to succeed the next time we get a new
    // revision.
    if (infoStream.isEnabled(INFO_STREAM_COMPONENT)) {
      infoStream.message(INFO_STREAM_COMPONENT, "cleanupOldIndexFiles(): failed on error " + t.getMessage());
    }
  }
}
 

/**
 * Get the generation (N) of the current segments_N file in the directory.
 * 
 * @param directory -- directory to search for the latest segments_N file
 */
public static long getCurrentSegmentGeneration(Directory directory) throws IOException {
  String[] files = directory.listAll();
  if (files == null) throw new IOException("cannot read directory " + directory
      + ": list() returned null");
  return getCurrentSegmentGeneration(files);
}
 

private void checkTruncation(Directory dir) throws IOException {
  for(String name : dir.listAll()) {
    if (name.equals(IndexWriter.WRITE_LOCK_NAME) == false) {
      truncateOneFile(dir, name);
    }
  }
}
 

/** Returns all commit points that exist in the Directory.
 *  Normally, because the default is {@link
 *  KeepOnlyLastCommitDeletionPolicy}, there would be only
 *  one commit point.  But if you're using a custom {@link
 *  IndexDeletionPolicy} then there could be many commits.
 *  Once you have a given commit, you can open a reader on
 *  it by calling {@link DirectoryReader#open(IndexCommit)}
 *  There must be at least one commit in
 *  the Directory, else this method throws {@link
 *  IndexNotFoundException}.  Note that if a commit is in
 *  progress while this method is running, that commit
 *  may or may not be returned.
 *  
 *  @return a sorted list of {@link IndexCommit}s, from oldest 
 *  to latest. */
public static List<IndexCommit> listCommits(Directory dir) throws IOException {
  final String[] files = dir.listAll();

  List<IndexCommit> commits = new ArrayList<>();

  SegmentInfos latest = SegmentInfos.readLatestCommit(dir);
  final long currentGen = latest.getGeneration();

  commits.add(new StandardDirectoryReader.ReaderCommit(null, latest, dir));

  for(int i=0;i<files.length;i++) {

    final String fileName = files[i];

    if (fileName.startsWith(IndexFileNames.SEGMENTS) &&
        SegmentInfos.generationFromSegmentsFileName(fileName) < currentGen) {

      SegmentInfos sis = null;
      try {
        // IOException allowed to throw there, in case
        // segments_N is corrupt
        sis = SegmentInfos.readCommit(dir, fileName);
      } catch (FileNotFoundException | NoSuchFileException fnfe) {
        // LUCENE-948: on NFS (and maybe others), if
        // you have writers switching back and forth
        // between machines, it's very likely that the
        // dir listing will be stale and will claim a
        // file segments_X exists when in fact it
        // doesn't.  So, we catch this and handle it
        // as if the file does not exist
      }

      if (sis != null) {
        commits.add(new StandardDirectoryReader.ReaderCommit(null, sis, dir));
      }
    }
  }

  // Ensure that the commit points are sorted in ascending order.
  Collections.sort(commits);

  return commits;
}
 

@Test
public void testPrepareCommit() throws Exception {
  assertU(adoc("id", "999"));
  assertU(optimize("maxSegments", "1"));     // make sure there's just one segment
  assertU(commit());       // commit a second time to make sure index files aren't still referenced by the old searcher

  SolrQueryRequest sr = req();
  DirectoryReader r = sr.getSearcher().getIndexReader();
  Directory d = r.directory();

  if (log.isInfoEnabled()) {
    log.info("FILES before addDoc={}", Arrays.asList(d.listAll()));
  }
  assertU(adoc("id", "1"));

  int nFiles = d.listAll().length;
  if (log.isInfoEnabled()) {
    log.info("FILES before prepareCommit={}", Arrays.asList(d.listAll()));
  }

  updateJ("", params("prepareCommit", "true"));

  if (log.isInfoEnabled()) {
    log.info("FILES after prepareCommit={}", Arrays.asList(d.listAll()));
  }
  assertTrue( d.listAll().length > nFiles);  // make sure new index files were actually written
  
  assertJQ(req("q", "id:1")
      , "/response/numFound==0"
  );

  updateJ("", params("rollback","true"));
  assertU(commit());

  assertJQ(req("q", "id:1")
      , "/response/numFound==0"
  );

  assertU(adoc("id","1"));
  updateJ("", params("prepareCommit","true"));

  assertJQ(req("q", "id:1")
      , "/response/numFound==0"
  );

  assertU(commit());

  assertJQ(req("q", "id:1")
      , "/response/numFound==1"
  );

  sr.close();
}
 

@Test
public void testRateLimitedReplication() throws Exception {

  //clean index
  masterClient.deleteByQuery("*:*");
  slaveClient.deleteByQuery("*:*");
  masterClient.commit();
  slaveClient.commit();

  masterJetty.stop();
  slaveJetty.stop();

  //Start master with the new solrconfig
  master.copyConfigFile(CONF_DIR + "solrconfig-master-throttled.xml", "solrconfig.xml");
  useFactory(null);
  masterJetty = createAndStartJetty(master);
  masterClient.close();
  masterClient = createNewSolrClient(masterJetty.getLocalPort());

  //index docs
  final int totalDocs = TestUtil.nextInt(random(), 17, 53);
  for (int i = 0; i < totalDocs; i++)
    index(masterClient, "id", i, "name", TestUtil.randomSimpleString(random(), 1000 , 5000));

  masterClient.commit();

  //Check Index Size
  String dataDir = master.getDataDir();
  masterClient.close();
  masterJetty.stop();

  Directory dir = FSDirectory.open(Paths.get(dataDir).resolve("index"));
  String[] files = dir.listAll();
  long totalBytes = 0;
  for(String file : files) {
    totalBytes += dir.fileLength(file);
  }

  float approximateTimeInSeconds = Math.round( totalBytes/1024/1024/0.1 ); // maxWriteMBPerSec=0.1 in solrconfig

  //Start again and replicate the data
  useFactory(null);
  masterJetty = createAndStartJetty(master);
  masterClient = createNewSolrClient(masterJetty.getLocalPort());

  //start slave
  slave.setTestPort(masterJetty.getLocalPort());
  slave.copyConfigFile(CONF_DIR + "solrconfig-slave1.xml", "solrconfig.xml");
  slaveJetty = createAndStartJetty(slave);
  slaveClient.close();
  slaveClient = createNewSolrClient(slaveJetty.getLocalPort());

  long startTime = System.nanoTime();

  pullFromMasterToSlave();

  //Add a few more docs in the master. Just to make sure that we are replicating the correct index point
  //These extra docs should not get replicated
  new Thread(new AddExtraDocs(masterClient, totalDocs)).start();

  //Wait and make sure that it actually replicated correctly.
  @SuppressWarnings({"rawtypes"})
  NamedList slaveQueryRsp = rQuery(totalDocs, "*:*", slaveClient);
  SolrDocumentList slaveQueryResult = (SolrDocumentList) slaveQueryRsp.get("response");
  assertEquals(totalDocs, slaveQueryResult.getNumFound());

  long timeTaken = System.nanoTime() - startTime;

  long timeTakenInSeconds = TimeUnit.SECONDS.convert(timeTaken, TimeUnit.NANOSECONDS);

  //Let's make sure it took more than approximateTimeInSeconds to make sure that it was throttled
  log.info("approximateTimeInSeconds = {} timeTakenInSeconds = {}"
      , approximateTimeInSeconds, timeTakenInSeconds);
  assertTrue(timeTakenInSeconds - approximateTimeInSeconds > 0);
}
 

private void corruptFile(Directory dir, String victim) throws IOException {
  try (BaseDirectoryWrapper dirCopy = newDirectory()) {
    dirCopy.setCheckIndexOnClose(false);

    long victimLength = dir.fileLength(victim);
    long flipOffset = TestUtil.nextLong(random(), 0, victimLength - 1);

    if (VERBOSE) {
      System.out.println("TEST: now corrupt file " + victim + " by changing byte at offset " + flipOffset + " (length= " + victimLength + ")");
    }

    for(String name : dir.listAll()) {
      if (name.equals(victim) == false) {
        dirCopy.copyFrom(dir, name, name, IOContext.DEFAULT);
      } else {
        try (IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT);
            IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {
            out.copyBytes(in, flipOffset);
            out.writeByte((byte) (in.readByte() + TestUtil.nextInt(random(), 0x01, 0xFF)));
            out.copyBytes(in, victimLength - flipOffset - 1);
        }
        try (IndexInput in = dirCopy.openInput(name, IOContext.DEFAULT)) {
          try {
            CodecUtil.checksumEntireFile(in);
            System.out.println("TEST: changing a byte in " + victim + " did not update the checksum)");
            return;
          } catch (CorruptIndexException e) {
            // ok
          }
        }
      }
      dirCopy.sync(Collections.singleton(name));
    }

    // corruption must be detected
    expectThrowsAnyOf(Arrays.asList(CorruptIndexException.class, IndexFormatTooOldException.class, IndexFormatTooNewException.class),
        () -> {
          try (IndexReader reader = DirectoryReader.open(dirCopy)) {
            for (LeafReaderContext context : reader.leaves()) {
              context.reader().checkIntegrity();
            }
          }
        }
    );
  }
}
 

public static void main(String[] args) throws Exception{
    File samplesFilesDir = new File("build/classes/ensemble/");
    File indexDir = new File("build/classes/ensemble/search/index");
    File docDir = new File("../../../artifacts/sdk/docs/api");
    File samplesDir = new File("src/ensemble/samples");
    // create index
    ///System.out.println("Indexing to directory '" + indexDir + "'...");
    long start = System.currentTimeMillis();
    Directory dir = FSDirectory.open(indexDir);
    Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_31);
    IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_31, analyzer);
    iwc.setOpenMode(OpenMode.CREATE);
    // generate and write index of all java doc and samples
    IndexWriter writer = new IndexWriter(dir, iwc);

    List<String> samplesFileList = new ArrayList<String>();

    indexSamples(writer, samplesDir, samplesFileList);
    try {
        indexJavaDocAllClasses(writer, docDir);
    } catch (Exception e) {
        System.out.println("\nWarning: We were not able to locate the JavaFX API documentation for your build environment.\n"
                + "Ensemble search will not include the API documentation.\n"); 
    }
    writer.close();
    // create a listAll.txt file that is used
    FileWriter listAllOut = new FileWriter(new File(indexDir,"listAll.txt"));
    for (String fileName: dir.listAll()) {
        if (!"listAll.txt".equals(fileName)) { // don't include the "listAll.txt" file
            Long length = dir.fileLength(fileName);
            listAllOut.write(fileName);
            listAllOut.write(':');
            listAllOut.write(length.toString());
            listAllOut.write('\n');
        }
    }
    listAllOut.flush();
    listAllOut.close();

    FileWriter sampleFilesCache = new FileWriter(new File(samplesFilesDir,"samplesAll.txt"));
    for (String oneSample: samplesFileList) {
            sampleFilesCache.write(oneSample);
            sampleFilesCache.write('\n');
    }
    sampleFilesCache.flush();
    sampleFilesCache.close();

    // print time taken
    ///System.out.println(System.currentTimeMillis() - start + " total milliseconds");
}