Java Code Examples for org.apache.lucene.util.TestUtil#unzip()

public void testSoftDeletes() throws Exception {
  Path oldIndexDir = createTempDir("dvupdates");
  TestUtil.unzip(getDataInputStream(dvUpdatesIndex), oldIndexDir);
  Directory dir = newFSDirectory(oldIndexDir);
  verifyUsesDefaultCodec(dir, dvUpdatesIndex);
  IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random())).setSoftDeletesField("__soft_delete");
  IndexWriter writer = new IndexWriter(dir, conf);
  int maxDoc = writer.getDocStats().maxDoc;
  writer.updateDocValues(new Term("id", "1"),new NumericDocValuesField("__soft_delete", 1));

  if (random().nextBoolean()) {
    writer.commit();
  }
  writer.forceMerge(1);
  writer.commit();
  assertEquals(maxDoc-1, writer.getDocStats().maxDoc);
  writer.close();
  dir.close();
}
 

public void testSortedIndex() throws Exception {
  for(String name : oldSortedNames) {
    Path path = createTempDir("sorted");
    InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream(name + ".zip");
    assertNotNull("Sorted index index " + name + " not found", resource);
    TestUtil.unzip(resource, path);

    // TODO: more tests
    Directory dir = newFSDirectory(path);

    DirectoryReader reader = DirectoryReader.open(dir);
    assertEquals(1, reader.leaves().size());
    Sort sort = reader.leaves().get(0).reader().getMetaData().getSort();
    assertNotNull(sort);
    assertEquals("<long: \"dateDV\">!", sort.toString());
    reader.close();

    // this will confirm the docs really are sorted:
    TestUtil.checkIndex(dir);
    dir.close();
  }
}
 

public void testOpenModeAndCreatedVersion() throws IOException {
  assumeTrue("Reenable when 8.0 is released", false);
  InputStream resource = getClass().getResourceAsStream("index.single-empty-doc.8.0.0.zip");
  assertNotNull(resource);
  Path path = createTempDir();
  TestUtil.unzip(resource, path);
  Directory dir = newFSDirectory(path);
  for (OpenMode openMode : OpenMode.values()) {
    Directory tmpDir = newDirectory(dir);
    assertEquals(7 /** 7.0.0 */, SegmentInfos.readLatestCommit(tmpDir).getIndexCreatedVersionMajor());
    IndexWriter w = new IndexWriter(tmpDir, newIndexWriterConfig().setOpenMode(openMode));
    w.commit();
    w.close();
    switch (openMode) {
      case CREATE:
        assertEquals(Version.LATEST.major, SegmentInfos.readLatestCommit(tmpDir).getIndexCreatedVersionMajor());
        break;
      default:
        assertEquals(7 /** 7.0.0 */, SegmentInfos.readLatestCommit(tmpDir).getIndexCreatedVersionMajor());
    }
    tmpDir.close();
  }
  dir.close();
}
 

/** Run a vocabulary test against two data files inside a zip file */
public static void assertVocabulary(Analyzer a, Path zipFile, String voc, String out) throws IOException {
  Path tmp = LuceneTestCase.createTempDir();
  try (InputStream in = Files.newInputStream(zipFile)) {
    TestUtil.unzip(in, tmp);
  }
  try (InputStream v = Files.newInputStream(tmp.resolve(voc)); 
       InputStream o = Files.newInputStream(tmp.resolve(out))) {
    assertVocabulary(a, v, o);
  }
}
 

/** Run a vocabulary test against a tab-separated data file inside a zip file */
public static void assertVocabulary(Analyzer a, Path zipFile, String vocOut) throws IOException {
  Path tmp = LuceneTestCase.createTempDir();
  try (InputStream in = Files.newInputStream(zipFile)) {
    TestUtil.unzip(in, tmp);
  }
  try (InputStream in = Files.newInputStream(tmp.resolve(vocOut))) {
    assertVocabulary(a, in);
  }
}
 

public void testCheckOldIndex() throws IOException {
  assumeTrue("Reenable when 7.0 is released", false);
  Path path = createTempDir("manypointsindex");
  InputStream resource = getClass().getResourceAsStream("manypointsindex.zip");
  assertNotNull("manypointsindex not found", resource);
  TestUtil.unzip(resource, path);
  BaseDirectoryWrapper dir = newFSDirectory(path);
  // disable default checking...
  dir.setCheckIndexOnClose(false);

  // ... because we check ourselves here:
  TestUtil.checkIndex(dir, false, true, null);
  dir.close();
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  List<String> names = new ArrayList<>(oldNames.length + oldSingleSegmentNames.length);
  names.addAll(Arrays.asList(oldNames));
  names.addAll(Arrays.asList(oldSingleSegmentNames));
  oldIndexDirs = new HashMap<>();
  for (String name : names) {
    Path dir = createTempDir(name);
    InputStream resource = TestBackwardsCompatibility.class.getResourceAsStream("index." + name + ".zip");
    assertNotNull("Index name " + name + " not found", resource);
    TestUtil.unzip(resource, dir);
    oldIndexDirs.put(name, newFSDirectory(dir));
  }
}
 

public void testUpgradeEmptyOldIndex() throws Exception {
  Path oldIndexDir = createTempDir("emptyIndex");
  TestUtil.unzip(getDataInputStream(emptyIndex), oldIndexDir);
  Directory dir = newFSDirectory(oldIndexDir);

  newIndexUpgrader(dir).upgrade();

  checkAllSegmentsUpgraded(dir, 8);
  
  dir.close();
}
 

public void testMoreTerms() throws Exception {
  Path oldIndexDir = createTempDir("moreterms");
  TestUtil.unzip(getDataInputStream(moreTermsIndex), oldIndexDir);
  Directory dir = newFSDirectory(oldIndexDir);
  verifyUsesDefaultCodec(dir, moreTermsIndex);
  // TODO: more tests
  TestUtil.checkIndex(dir);
  dir.close();
}
 

public void testDocValuesUpdates() throws Exception {
  Path oldIndexDir = createTempDir("dvupdates");
  TestUtil.unzip(getDataInputStream(dvUpdatesIndex), oldIndexDir);
  Directory dir = newFSDirectory(oldIndexDir);
  verifyUsesDefaultCodec(dir, dvUpdatesIndex);
  
  verifyDocValues(dir);
  
  // update fields and verify index
  IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  updateNumeric(writer, "1", "ndv1", "ndv1_c", 300L);
  updateNumeric(writer, "1", "ndv2", "ndv2_c", 300L);
  updateBinary(writer, "1", "bdv1", "bdv1_c", 300L);
  updateBinary(writer, "1", "bdv2", "bdv2_c", 300L);

  writer.commit();
  verifyDocValues(dir);
  
  // merge all segments
  writer.forceMerge(1);
  writer.commit();
  verifyDocValues(dir);
  
  writer.close();
  dir.close();
}
 

private void startUpNodeWithDataDir(String dataPath) throws Exception {
    Path indexDir = createTempDir();
    try (InputStream stream = Files.newInputStream(getDataPath(dataPath))) {
        TestUtil.unzip(stream, indexDir);
    }
    Settings.Builder builder = Settings.builder().put(Environment.PATH_DATA_SETTING.getKey(), indexDir.toAbsolutePath());
    internalCluster().startNode(builder.build());
    ensureGreen();
}
 

/** 
 * Open a trec content source over a directory with files of all trec path types and all
 * supported formats - bzip, gzip, txt. 
 */
public void testTrecFeedDirAllTypes() throws Exception {
  Path dataDir =  createTempDir("trecFeedAllTypes");
  TestUtil.unzip(getDataInputStream("trecdocs.zip"), dataDir);
  TrecContentSource tcs = new TrecContentSource();
  Properties props = new Properties();
  props.setProperty("print.props", "false");
  props.setProperty("content.source.verbose", "false");
  props.setProperty("content.source.excludeIteration", "true");
  props.setProperty("docs.dir", dataDir.toRealPath().toString().replace('\\','/'));
  props.setProperty("trec.doc.parser", TrecParserByPath.class.getName());
  props.setProperty("content.source.forever", "false");
  tcs.setConfig(new Config(props));
  tcs.resetInputs();
  DocData dd = new DocData();
  int n = 0;
  boolean gotExpectedException = false;
  HashSet<ParsePathType> unseenTypes = new HashSet<>(Arrays.asList(ParsePathType.values()));
  try {
    while (n<100) { // arbiterary limit to prevent looping forever in case of test failure
      dd = tcs.getNextDocData(dd);
      ++n;
      assertNotNull("doc data "+n+" should not be null!", dd);
      unseenTypes.remove(tcs.currPathType);
      switch(tcs.currPathType) {
        case GOV2:
          assertDocData(dd, "TEST-000", "TEST-000 title", "TEST-000 text", tcs.parseDate("Sun, 11 Jan 2009 08:00:00 GMT"));
          break;
        case FBIS:
          assertDocData(dd, "TEST-001", "TEST-001 Title", "TEST-001 text", tcs.parseDate("1 January 1991"));
          break;
        case FR94:
          // no title extraction in this source for now
          assertDocData(dd, "TEST-002", null, "DEPARTMENT OF SOMETHING", tcs.parseDate("February 3, 1994"));
          break;
        case FT:
          assertDocData(dd, "TEST-003", "Test-003 title", "Some pub text", tcs.parseDate("980424"));
          break;
        case LATIMES:
          assertDocData(dd, "TEST-004", "Test-004 Title", "Some paragraph", tcs.parseDate("January 17, 1997, Sunday"));
          break;
        default:
          assertTrue("Should never get here!", false);
      }
    }
  } catch (NoMoreDataException e) {
    gotExpectedException = true;
  }
  assertTrue("Should have gotten NoMoreDataException!", gotExpectedException);
  assertEquals("Wrong number of documents created by source!",5,n);
  assertTrue("Did not see all types!",unseenTypes.isEmpty());
}
 

public void testCommandLineArgs() throws Exception {

    PrintStream savedSystemOut = System.out;
    System.setOut(new PrintStream(new ByteArrayOutputStream(), false, "UTF-8"));
    try {
      for (Map.Entry<String,Directory> entry : oldIndexDirs.entrySet()) {
        String name = entry.getKey();
        int indexCreatedVersion = SegmentInfos.readLatestCommit(entry.getValue()).getIndexCreatedVersionMajor();
        Path dir = createTempDir(name);
        TestUtil.unzip(getDataInputStream("index." + name + ".zip"), dir);
        
        String path = dir.toAbsolutePath().toString();
        
        List<String> args = new ArrayList<>();
        if (random().nextBoolean()) {
          args.add("-verbose");
        }
        if (random().nextBoolean()) {
          args.add("-delete-prior-commits");
        }
        if (random().nextBoolean()) {
          // TODO: need to better randomize this, but ...
          //  - LuceneTestCase.FS_DIRECTORIES is private
          //  - newFSDirectory returns BaseDirectoryWrapper
          //  - BaseDirectoryWrapper doesn't expose delegate
          Class<? extends FSDirectory> dirImpl = NIOFSDirectory.class;
          
          args.add("-dir-impl");
          args.add(dirImpl.getName());
        }
        args.add(path);
        
        IndexUpgrader upgrader = null;
        try {
          upgrader = IndexUpgrader.parseArgs(args.toArray(new String[0]));
        } catch (Exception e) {
          throw new AssertionError("unable to parse args: " + args, e);
        }
        upgrader.upgrade();
        
        Directory upgradedDir = newFSDirectory(dir);
        try {
          checkAllSegmentsUpgraded(upgradedDir, indexCreatedVersion);
        } finally {
          upgradedDir.close();
        }
      }
    } finally {
      System.setOut(savedSystemOut);
    }
  }
 

public void testDocValuesUpdatesWithNewField() throws Exception {
  Path oldIndexDir = createTempDir("dvupdates");
  TestUtil.unzip(getDataInputStream(dvUpdatesIndex), oldIndexDir);
  Directory dir = newFSDirectory(oldIndexDir);
  verifyUsesDefaultCodec(dir, dvUpdatesIndex);

  // update fields and verify index
  IndexWriterConfig conf = new IndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  // introduce a new field that we later update
  writer.addDocument(Arrays.asList(new StringField("id", "" + Integer.MAX_VALUE, Field.Store.NO),
      new NumericDocValuesField("new_numeric", 1),
      new BinaryDocValuesField("new_binary", toBytes(1))));
  writer.updateNumericDocValue(new Term("id", "1"), "new_numeric", 1);
  writer.updateBinaryDocValue(new Term("id", "1"), "new_binary", toBytes(1));

  writer.commit();
  Runnable assertDV = () -> {
    boolean found = false;
    try (DirectoryReader reader = DirectoryReader.open(dir)) {
      for (LeafReaderContext ctx : reader.leaves()) {
        LeafReader leafReader = ctx.reader();
        TermsEnum id = leafReader.terms("id").iterator();
        if (id.seekExact(new BytesRef("1"))) {
          PostingsEnum postings = id.postings(null, PostingsEnum.NONE);
          NumericDocValues numericDocValues = leafReader.getNumericDocValues("new_numeric");
          BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("new_binary");
          int doc;
          while ((doc = postings.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
            found = true;
            assertTrue(binaryDocValues.advanceExact(doc));
            assertTrue(numericDocValues.advanceExact(doc));
            assertEquals(1, numericDocValues.longValue());
            assertEquals(toBytes(1), binaryDocValues.binaryValue());
          }
        }
      }
    } catch (IOException e) {
      throw new AssertionError(e);
    }
    assertTrue(found);
  };
  assertDV.run();
  // merge all segments
  writer.forceMerge(1);
  writer.commit();
  assertDV.run();
  writer.close();
  dir.close();
}