org.apache.lucene.util.LuceneTestCase Java Examples

The following examples show how to use org.apache.lucene.util.LuceneTestCase. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MockFSDirectoryFactory.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException {
  // we pass NoLockFactory, because the real lock factory is set later by injectLockFactory:
  Directory dir = LuceneTestCase.newFSDirectory(new File(path).toPath(), lockFactory);
  // we can't currently do this check because of how
  // Solr has to reboot a new Directory sometimes when replicating
  // or rolling back - the old directory is closed and the following
  // test assumes it can open an IndexWriter when that happens - we
  // have a new Directory for the same dir and still an open IW at 
  // this point
  
  Directory cdir = reduce(dir);
  cdir = reduce(cdir);
  cdir = reduce(cdir);
  
  if (cdir instanceof MockDirectoryWrapper) {
    ((MockDirectoryWrapper)cdir).setAssertNoUnrefencedFilesOnClose(false);
  }
  return dir;
}
 
Example #2
Source File: RandomPostingsTester.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testTerms(final Fields fieldsSource, final EnumSet<Option> options,
                      final IndexOptions maxTestOptions,
                      final IndexOptions maxIndexOptions,
                      final boolean alwaysTestMax) throws Exception {

  if (options.contains(Option.THREADS)) {
    int numThreads = LuceneTestCase.TEST_NIGHTLY ? TestUtil.nextInt(random, 2, 5) : 2;
    Thread[] threads = new Thread[numThreads];
    for(int threadUpto=0;threadUpto<numThreads;threadUpto++) {
      threads[threadUpto] = new TestThread(new Random(random.nextLong()), this, fieldsSource, options, maxTestOptions, maxIndexOptions, alwaysTestMax);
      threads[threadUpto].start();
    }
    for(int threadUpto=0;threadUpto<numThreads;threadUpto++) {
      threads[threadUpto].join();
    }
  } else {
    testTermsOneThread(random, fieldsSource, options, maxTestOptions, maxIndexOptions, alwaysTestMax);
  }
}
 
Example #3
Source File: QueryUtils.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Given an IndexSearcher, returns a new IndexSearcher whose IndexReader
 * is a MultiReader containing the Reader of the original IndexSearcher,
 * as well as several "empty" IndexReaders -- some of which will have
 * deleted documents in them.  This new IndexSearcher should
 * behave exactly the same as the original IndexSearcher.
 * @param s the searcher to wrap
 * @param edge if negative, s will be the first sub; if 0, s will be in the middle, if positive s will be the last sub
 */
public static IndexSearcher wrapUnderlyingReader(Random random, final IndexSearcher s, final int edge)
  throws IOException {

  IndexReader r = s.getIndexReader();

  // we can't put deleted docs before the nested reader, because
  // it will throw off the docIds
  IndexReader[] readers = new IndexReader[] {
    edge < 0 ? r : new MultiReader(),
    new MultiReader(),
    new MultiReader(edge < 0 ? emptyReader(4) : new MultiReader(),
        new MultiReader(),
        0 == edge ? r : new MultiReader()),
    0 < edge ? new MultiReader() : emptyReader(7),
    new MultiReader(),
    new MultiReader(0 < edge ? new MultiReader() : emptyReader(5),
        new MultiReader(),
        0 < edge ? r : new MultiReader())
  };

  IndexSearcher out = LuceneTestCase.newSearcher(new MultiReader(readers));
  out.setSimilarity(s.getSimilarity());
  return out;
}
 
Example #4
Source File: RandomPostingsTester.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testFields(Fields fields) throws Exception {
  Iterator<String> iterator = fields.iterator();
  while (iterator.hasNext()) {
    iterator.next();
    try {
      iterator.remove();
      throw new AssertionError("Fields.iterator() allows for removal");
    } catch (UnsupportedOperationException expected) {
      // expected;
    }
  }
  assertFalse(iterator.hasNext());
  LuceneTestCase.expectThrows(NoSuchElementException.class, () -> {
    iterator.next();
  });
}
 
Example #5
Source File: CheckHits.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that a query matches the an expected set of documents using Hits.
 *
 * <p>
 * Note that when using the Hits API, documents will only be returned
 * if they have a positive normalized score.
 * </p>
 * @param query the query to test
 * @param searcher the searcher to test the query against
 * @param defaultFieldName used for displaing the query in assertion messages
 * @param results a list of documentIds that must match the query
 * @see #checkHitCollector
 */
public static void checkHits(
      Random random,
      Query query,
      String defaultFieldName,
      IndexSearcher searcher,
      int[] results)
        throws IOException {

  ScoreDoc[] hits = searcher.search(query, 1000).scoreDocs;

  Set<Integer> correct = new TreeSet<>();
  for (int i = 0; i < results.length; i++) {
    correct.add(Integer.valueOf(results[i]));
  }

  Set<Integer> actual = new TreeSet<>();
  for (int i = 0; i < hits.length; i++) {
    actual.add(Integer.valueOf(hits[i].doc));
  }

  assertEquals(query.toString(defaultFieldName), correct, actual);

  QueryUtils.check(random, query,searcher, LuceneTestCase.rarely(random));
}
 
Example #6
Source File: VirusCheckingFS.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void delete(Path path) throws IOException {

  // Fake but deterministic and hopefully portable like-randomness:
  long hash = state.incrementAndGet() * path.getFileName().hashCode();
  
  if (enabled // test infra disables when it's "really" time to delete after test is done, so it can reclaim temp dirs
      && Files.exists(path) // important that we NOT delay a NoSuchFileException until later
      && path.getFileName().toString().equals(IndexWriter.WRITE_LOCK_NAME) == false // life is particularly difficult if the virus checker hits our lock file
      && (hash % 5) == 1) {
    if (LuceneTestCase.VERBOSE) {
      System.out.println("NOTE: VirusCheckingFS now refusing to delete " + path);
    }
    throw new AccessDeniedException("VirusCheckingFS is randomly refusing to delete file \"" + path + "\"");
  }
  super.delete(path);
}
 
Example #7
Source File: RandomPostingsTester.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** Indexes all fields/terms at the specified
 *  IndexOptions, and fully tests at that IndexOptions. */
public void testFull(Codec codec, Path path, IndexOptions options, boolean withPayloads) throws Exception {
  Directory dir = LuceneTestCase.newFSDirectory(path);

  // TODO test thread safety of buildIndex too
  FieldsProducer fieldsProducer = buildIndex(codec, dir, options, withPayloads, true);

  testFields(fieldsProducer);

  IndexOptions[] allOptions = IndexOptions.values();
  int maxIndexOption = Arrays.asList(allOptions).indexOf(options);

  for(int i=0;i<=maxIndexOption;i++) {
    testTerms(fieldsProducer, EnumSet.allOf(Option.class), allOptions[i], options, true);
    if (withPayloads) {
      // If we indexed w/ payloads, also test enums w/o accessing payloads:
      testTerms(fieldsProducer, EnumSet.complementOf(EnumSet.of(Option.PAYLOADS)), allOptions[i], options, true);
    }
  }

  fieldsProducer.close();
  dir.close();
}
 
Example #8
Source File: EmbeddedSolrServerTestBase.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public static String legacyExampleCollection1SolrHome() throws IOException {
  final String sourceHome = ExternalPaths.SOURCE_HOME;
  if (sourceHome == null)
    throw new IllegalStateException("No source home! Cannot create the legacy example solr home directory.");

  final File tempSolrHome = LuceneTestCase.createTempDir().toFile();
  FileUtils.copyFileToDirectory(new File(sourceHome, "server/solr/solr.xml"), tempSolrHome);
  final File collectionDir = new File(tempSolrHome, DEFAULT_CORE_NAME);
  FileUtils.forceMkdir(collectionDir);
  final File configSetDir = new File(sourceHome, "server/solr/configsets/sample_techproducts_configs/conf");
  FileUtils.copyDirectoryToDirectory(configSetDir, collectionDir);

  final Properties props = new Properties();
  props.setProperty("name", DEFAULT_CORE_NAME);

  try (Writer writer = new OutputStreamWriter(FileUtils.openOutputStream(new File(collectionDir, "core.properties")),
      "UTF-8");) {
    props.store(writer, null);
  }

  return tempSolrHome.getAbsolutePath();
}
 
Example #9
Source File: PayloadHelper.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Sets up a RAM-resident Directory, and adds documents (using English.intToEnglish()) with two fields: field and multiField
 * and analyzes them using the PayloadAnalyzer
 * @param similarity The Similarity class to use in the Searcher
 * @param numDocs The num docs to add
 * @return An IndexSearcher
 */
// TODO: randomize
public IndexSearcher setUp(Random random, Similarity similarity, int numDocs) throws IOException {
  Directory directory = new MockDirectoryWrapper(random, new ByteBuffersDirectory());
  PayloadAnalyzer analyzer = new PayloadAnalyzer();

  // TODO randomize this
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
      analyzer).setSimilarity(similarity));
  // writer.infoStream = System.out;
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    doc.add(new TextField(FIELD, English.intToEnglish(i), Field.Store.YES));
    doc.add(new TextField(MULTI_FIELD, English.intToEnglish(i) + "  " + English.intToEnglish(i), Field.Store.YES));
    doc.add(new TextField(NO_PAYLOAD_FIELD, English.intToEnglish(i), Field.Store.YES));
    writer.addDocument(doc);
  }
  writer.forceMerge(1);
  reader = DirectoryReader.open(writer);
  writer.close();

  IndexSearcher searcher = LuceneTestCase.newSearcher(LuceneTestCase.getOnlyLeafReader(reader));
  searcher.setSimilarity(similarity);
  return searcher;
}
 
Example #10
Source File: SimplePrimaryNode.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static IndexWriter initWriter(int id, Random random, Path indexPath, boolean doCheckIndexOnClose) throws IOException {
  Directory dir = SimpleReplicaNode.getDirectory(random, id, indexPath, doCheckIndexOnClose);

  MockAnalyzer analyzer = new MockAnalyzer(random);
  analyzer.setMaxTokenLength(TestUtil.nextInt(random, 1, IndexWriter.MAX_TERM_LENGTH));
  IndexWriterConfig iwc = LuceneTestCase.newIndexWriterConfig(random, analyzer);

  MergePolicy mp = iwc.getMergePolicy();
  //iwc.setInfoStream(new PrintStreamInfoStream(System.out));

  // Force more frequent merging so we stress merge warming:
  if (mp instanceof TieredMergePolicy) {
    TieredMergePolicy tmp = (TieredMergePolicy) mp;
    tmp.setSegmentsPerTier(3);
    tmp.setMaxMergeAtOnce(3);
  } else if (mp instanceof LogMergePolicy) {
    LogMergePolicy lmp = (LogMergePolicy) mp;
    lmp.setMergeFactor(3);
  }

  IndexWriter writer = new IndexWriter(dir, iwc);

  TestUtil.reduceOpenFiles(writer);
  return writer;
}
 
Example #11
Source File: TestFileListWithLineEntityProcessor.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void test() throws Exception {
  File tmpdir = createTempDir(LuceneTestCase.getTestClass().getSimpleName()).toFile();
  createFile(tmpdir, "a.txt", "a line one\na line two\na line three".getBytes(StandardCharsets.UTF_8), false);
  createFile(tmpdir, "b.txt", "b line one\nb line two".getBytes(StandardCharsets.UTF_8), false);
  createFile(tmpdir, "c.txt", "c line one\nc line two\nc line three\nc line four".getBytes(StandardCharsets.UTF_8), false);
  
  String config = generateConfig(tmpdir);
  LocalSolrQueryRequest request = lrf.makeRequest(
      "command", "full-import", "dataConfig", config,
      "clean", "true", "commit", "true", "synchronous", "true", "indent", "true");
  h.query("/dataimport", request);
  
  assertQ(req("*:*"), "//*[@numFound='9']");
  assertQ(req("id:?\\ line\\ one"), "//*[@numFound='3']");
  assertQ(req("id:a\\ line*"), "//*[@numFound='3']");
  assertQ(req("id:b\\ line*"), "//*[@numFound='2']");
  assertQ(req("id:c\\ line*"), "//*[@numFound='4']");    
}
 
Example #12
Source File: TestCoreAdmin.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Test
public void testConfigSet() throws Exception {

  SolrClient client = getSolrAdmin();
  File testDir = createTempDir(LuceneTestCase.getTestClass().getSimpleName()).toFile();
  File newCoreInstanceDir = new File(testDir, "newcore");
  cores.getAllowPaths().add(testDir.toPath()); // Allow the test dir

  CoreAdminRequest.Create req = new CoreAdminRequest.Create();
  req.setCoreName("corewithconfigset");
  req.setInstanceDir(newCoreInstanceDir.getAbsolutePath());
  req.setConfigSet("configset-2");

  CoreAdminResponse response = req.process(client);
  assertThat((String) response.getResponse().get("core"), is("corewithconfigset"));

  try (SolrCore core = cores.getCore("corewithconfigset")) {
    assertThat(core, is(notNullValue()));
  }

}
 
Example #13
Source File: RandomIndexWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void maybeFlushOrCommit() throws IOException {
  LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
  if (docCount++ == flushAt) {
    if (r.nextBoolean()) {
      flushAllBuffersSequentially();
    } else if (r.nextBoolean()) {
      if (LuceneTestCase.VERBOSE) {
        System.out.println("RIW.add/updateDocument: now doing a flush at docCount=" + docCount);
      }
      w.flush();
    } else {
      if (LuceneTestCase.VERBOSE) {
        System.out.println("RIW.add/updateDocument: now doing a commit at docCount=" + docCount);
      }
      w.commit();
    }
    flushAt += TestUtil.nextInt(r, (int) (flushAtFactor * 10), (int) (flushAtFactor * 1000));
    if (flushAtFactor < 2e6) {
      // gradually but exponentially increase time b/w flushes
      flushAtFactor *= 1.05;
    }
  }
}
 
Example #14
Source File: CustomAnalyzerStrField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public CustomAnalyzerStrField() {
  Random r = LuceneTestCase.random();

  // two arg constructor
  Analyzer a2 = new TokenizerChain
    (new KeywordTokenizerFactory(new HashMap<>()),
     r.nextBoolean() ? null : new TokenFilterFactory[0]);
  
  // three arg constructor
  Analyzer a3 = new TokenizerChain
    (r.nextBoolean() ? null : new CharFilterFactory[0],
     new KeywordTokenizerFactory(new HashMap<>()),
     r.nextBoolean() ? null : new TokenFilterFactory[0]);

  if (r.nextBoolean()) {
    indexAnalyzer = a2;
    queryAnalyzer = a3;
  } else {
    queryAnalyzer = a2;
    indexAnalyzer = a3;
  }
}
 
Example #15
Source File: TestByteBuffersDataOutput.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Test
public void testLargeArrayAdd() {
  ByteBuffersDataOutput o = new ByteBuffersDataOutput();
  int MB = 1024 * 1024;
  final byte [] bytes;
  if (LuceneTestCase.TEST_NIGHTLY) {
    bytes = randomBytesOfLength(5 * MB, 15 * MB);
  } else {
    bytes = randomBytesOfLength(MB/2, MB);
  }
  int offset = randomIntBetween(0, 100);
  int len = bytes.length - offset;
  o.writeBytes(bytes, offset, len);
  assertEquals(len, o.size());
  Assert.assertArrayEquals(ArrayUtil.copyOfSubArray(bytes, offset, offset + len), o.toArrayCopy());
}
 
Example #16
Source File: RandomIndexWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Updates a document.
 * @see IndexWriter#updateDocument(Term, Iterable)
 */
public <T extends IndexableField> long updateDocument(Term t, final Iterable<T> doc) throws IOException {
  LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
  final long seqNo;
  if (useSoftDeletes()) {
    if (r.nextInt(5) == 3) {
      seqNo = w.softUpdateDocuments(t, Arrays.asList(doc), new NumericDocValuesField(config.getSoftDeletesField(), 1));
    } else {
      seqNo = w.softUpdateDocument(t, doc, new NumericDocValuesField(config.getSoftDeletesField(), 1));
    }
  } else {
    if (r.nextInt(5) == 3) {
      seqNo = w.updateDocuments(t, Arrays.asList(doc));
    } else {
      seqNo = w.updateDocument(t, doc);
    }
  }
  maybeFlushOrCommit();

  return seqNo;
}
 
Example #17
Source File: RandomIndexWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Close this writer.
 * @see IndexWriter#close()
 */
@Override
public void close() throws IOException {
  boolean success = false;
  try {
    if (w.isClosed() == false) {
      LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    }
    // if someone isn't using getReader() API, we want to be sure to
    // forceMerge since presumably they might open a reader on the dir.
    if (getReaderCalled == false && r.nextInt(8) == 2 && w.isClosed() == false) {
      doRandomForceMerge();
      if (config.getCommitOnClose() == false) {
        // index may have changed, must commit the changes, or otherwise they are discarded by the call to close()
        w.commit();
      }
    }
    success = true;
  } finally {
    if (success) {
      IOUtils.close(w, analyzer);
    } else {
      IOUtils.closeWhileHandlingException(w, analyzer);
    }
  }
}
 
Example #18
Source File: TestDirectoryReaderReopen.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public static void createIndex(Random random, Directory dir, boolean multiSegment) throws IOException {
  IndexWriter w = new IndexWriter(dir, LuceneTestCase.newIndexWriterConfig(random, new MockAnalyzer(random))
      .setMergePolicy(new LogDocMergePolicy()));
  
  for (int i = 0; i < 100; i++) {
    w.addDocument(createDocument(i, 4));
    if (multiSegment && (i % 10) == 0) {
      w.commit();
    }
  }
  
  if (!multiSegment) {
    w.forceMerge(1);
  }
  
  w.close();

  DirectoryReader r = DirectoryReader.open(dir);
  if (multiSegment) {
    assertTrue(r.leaves().size() > 1);
  } else {
    assertTrue(r.leaves().size() == 1);
  }
  r.close();
}
 
Example #19
Source File: TestStressIndexing2.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public static void indexSerial(Random random, Map<String,Document> docs, Directory dir) throws IOException {
  IndexWriter w = new IndexWriter(dir, LuceneTestCase.newIndexWriterConfig(random, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));

  // index all docs in a single thread
  Iterator<Document> iter = docs.values().iterator();
  while (iter.hasNext()) {
    Document d = iter.next();
    ArrayList<IndexableField> fields = new ArrayList<>();
    fields.addAll(d.getFields());
    // put fields in same order each time
    Collections.sort(fields, fieldNameComparator);
    
    Document d1 = new Document();
    for (int i=0; i<fields.size(); i++) {
      d1.add(fields.get(i));
    }
    w.addDocument(d1);
    // System.out.println("indexing "+d1);
  }
  
  w.close();
}
 
Example #20
Source File: RandomPostingsTester.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public static SeedPostings getSeedPostings(String term, long seed, IndexOptions options, boolean allowPayloads) {
  int minDocFreq, maxDocFreq;
  if (term.startsWith("big_")) {
    minDocFreq = LuceneTestCase.RANDOM_MULTIPLIER * 50000;
    maxDocFreq = LuceneTestCase.RANDOM_MULTIPLIER * 70000;
  } else if (term.startsWith("medium_")) {
    minDocFreq = LuceneTestCase.RANDOM_MULTIPLIER * 3000;
    maxDocFreq = LuceneTestCase.RANDOM_MULTIPLIER * 6000;
  } else if (term.startsWith("low_")) {
    minDocFreq = LuceneTestCase.RANDOM_MULTIPLIER;
    maxDocFreq = LuceneTestCase.RANDOM_MULTIPLIER * 40;
  } else {
    minDocFreq = 1;
    maxDocFreq = 3;
  }

  return new SeedPostings(seed, minDocFreq, maxDocFreq, options, allowPayloads);
}
 
Example #21
Source File: TestControlledRealTimeReopenThread.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testEvilSearcherFactory() throws Exception {
  final Directory dir = newDirectory();
  final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  w.commit();

  final IndexReader other = DirectoryReader.open(dir);

  final SearcherFactory theEvilOne = new SearcherFactory() {
    @Override
    public IndexSearcher newSearcher(IndexReader ignored, IndexReader previous) {
      return LuceneTestCase.newSearcher(other);
    }
    };

  expectThrows(IllegalStateException.class, () -> {
    new SearcherManager(w.w, false, false, theEvilOne);
  });

  w.close();
  other.close();
  dir.close();
}
 
Example #22
Source File: TestSearcherManager.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testEvilSearcherFactory() throws Exception {
  final Random random = random();
  final Directory dir = newDirectory();
  final RandomIndexWriter w = new RandomIndexWriter(random, dir);
  w.commit();

  final IndexReader other = DirectoryReader.open(dir);

  final SearcherFactory theEvilOne = new SearcherFactory() {
    @Override
    public IndexSearcher newSearcher(IndexReader ignored, IndexReader previous) {
      return LuceneTestCase.newSearcher(other);
    }
    };

  expectThrows(IllegalStateException.class, () -> {
    new SearcherManager(dir, theEvilOne);
  });
  expectThrows(IllegalStateException.class, () -> {
    new SearcherManager(w.w, random.nextBoolean(), false, theEvilOne);
  });
  w.close();
  other.close();
  dir.close();
}
 
Example #23
Source File: RandomIndexWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c, boolean closeAnalyzer, boolean useSoftDeletes) throws IOException {
  // TODO: this should be solved in a different way; Random should not be shared (!).
  this.r = new Random(r.nextLong());
  if (useSoftDeletes) {
    c.setSoftDeletesField("___soft_deletes");
    softDeletesRatio = 1.d / (double)1 + r.nextInt(10);
  } else {
    softDeletesRatio = 0d;
  }
  w = mockIndexWriter(dir, c, r);
  config = w.getConfig();
  flushAt = TestUtil.nextInt(r, 10, 1000);
  if (closeAnalyzer) {
    analyzer = w.getAnalyzer();
  } else {
    analyzer = null;
  }
  if (LuceneTestCase.VERBOSE) {
    System.out.println("RIW dir=" + dir);
  }

  // Make sure we sometimes test indices that don't get
  // any forced merges:
  doRandomForceMerge = !(c.getMergePolicy() instanceof NoMergePolicy) && r.nextBoolean();
}
 
Example #24
Source File: CoreParserTestIndexData.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
CoreParserTestIndexData(Analyzer analyzer) throws Exception {
  BufferedReader d = new BufferedReader(new InputStreamReader(
      TestCoreParser.class.getResourceAsStream("reuters21578.txt"), StandardCharsets.US_ASCII));
  dir = LuceneTestCase.newDirectory();
  IndexWriter writer = new IndexWriter(dir, LuceneTestCase.newIndexWriterConfig(analyzer));
  String line = d.readLine();
  while (line != null) {
    int endOfDate = line.indexOf('\t');
    String date = line.substring(0, endOfDate).trim();
    String content = line.substring(endOfDate).trim();
    Document doc = new Document();
    doc.add(LuceneTestCase.newTextField("date", date, Field.Store.YES));
    doc.add(LuceneTestCase.newTextField("contents", content, Field.Store.YES));
    doc.add(new IntPoint("date3", Integer.parseInt(date)));
    writer.addDocument(doc);
    line = d.readLine();
  }
  d.close();
  writer.close();
  reader = DirectoryReader.open(dir);
  searcher = LuceneTestCase.newSearcher(reader, false);
}
 
Example #25
Source File: MockDirectoryWrapper.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized IndexInput openInput(String name, IOContext context) throws IOException {
  maybeThrowDeterministicException();
  maybeThrowIOExceptionOnOpen(name);
  maybeYield();
  if (failOnOpenInput) {
    maybeThrowDeterministicException();
  }
  if (!LuceneTestCase.slowFileExists(in, name)) {
    throw randomState.nextBoolean() ? new FileNotFoundException(name + " in dir=" + in) : new NoSuchFileException(name + " in dir=" + in);
  }

  // cannot open a file for input if it's still open for output.
  if (!allowReadingFilesStillOpenForWrite && openFilesForWrite.contains(name)) {
    throw fillOpenTrace(new AccessDeniedException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false);
  }

  IndexInput delegateInput = in.openInput(name, LuceneTestCase.newIOContext(randomState, context));

  final IndexInput ii;
  int randomInt = randomState.nextInt(500);
  if (useSlowOpenClosers && randomInt == 0) {
    if (LuceneTestCase.VERBOSE) {
      System.out.println("MockDirectoryWrapper: using SlowClosingMockIndexInputWrapper for file " + name);
    }
    ii = new SlowClosingMockIndexInputWrapper(this, name, delegateInput);
  } else if (useSlowOpenClosers && randomInt  == 1) { 
    if (LuceneTestCase.VERBOSE) {
      System.out.println("MockDirectoryWrapper: using SlowOpeningMockIndexInputWrapper for file " + name);
    }
    ii = new SlowOpeningMockIndexInputWrapper(this, name, delegateInput);
  } else {
    ii = new MockIndexInputWrapper(this, name, delegateInput, null);
  }
  addFileHandle(ii, name, Handle.Input);
  return ii;
}
 
Example #26
Source File: MockDirectoryWrapper.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
  maybeThrowDeterministicException();
  maybeThrowIOExceptionOnOpen("temp: prefix=" + prefix + " suffix=" + suffix);
  maybeYield();
  if (failOnCreateOutput) {
    maybeThrowDeterministicException();
  }
  if (crashed) {
    throw new IOException("cannot createTempOutput after crash");
  }
  init();
  
  IndexOutput delegateOutput = in.createTempOutput(prefix, suffix, LuceneTestCase.newIOContext(randomState, context));
  String name = delegateOutput.getName();
  if (name.toLowerCase(Locale.ROOT).endsWith(".tmp") == false) {
    throw new IllegalStateException("wrapped directory failed to use .tmp extension: got: " + name);
  }

  unSyncedFiles.add(name);
  createdFiles.add(name);
  final IndexOutput io = new MockIndexOutputWrapper(this, delegateOutput, name);
  addFileHandle(io, name, Handle.Output);
  openFilesForWrite.add(name);
  
  return maybeThrottle(name, io);
}
 
Example #27
Source File: MockIndexOutputWrapper.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void checkDiskFull(byte[] b, int offset, DataInput in, long len) throws IOException {
  long freeSpace = dir.maxSize == 0 ? 0 : dir.maxSize - dir.sizeInBytes();
  long realUsage = 0;

  // Enforce disk full:
  if (dir.maxSize != 0 && freeSpace <= len) {
    // Compute the real disk free.  This will greatly slow
    // down our test but makes it more accurate:
    realUsage = dir.sizeInBytes();
    freeSpace = dir.maxSize - realUsage;
  }

  if (dir.maxSize != 0 && freeSpace <= len) {
    if (freeSpace > 0) {
      realUsage += freeSpace;
      if (b != null) {
        delegate.writeBytes(b, offset, (int) freeSpace);
      } else {
        delegate.copyBytes(in, (int) freeSpace);
      }
    }
    if (realUsage > dir.maxUsedSize) {
      dir.maxUsedSize = realUsage;
    }
    String message = "fake disk full at " + dir.sizeInBytes() + " bytes when writing " + name + " (file length=" + delegate.getFilePointer();
    if (freeSpace > 0) {
      message += "; wrote " + freeSpace + " of " + len + " bytes";
    }
    message += ")";
    if (LuceneTestCase.VERBOSE) {
      System.out.println(Thread.currentThread().getName() + ": MDW: now throw fake disk full");
      new Throwable().printStackTrace(System.out);
    }
    throw new IOException(message);
  }
}
 
Example #28
Source File: MockDirectoryWrapper.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized IndexOutput createOutput(String name, IOContext context) throws IOException {
  maybeThrowDeterministicException();
  maybeThrowIOExceptionOnOpen(name);
  maybeYield();
  if (failOnCreateOutput) {
    maybeThrowDeterministicException();
  }
  if (crashed) {
    throw new IOException("cannot createOutput after crash");
  }
  init();

  if (createdFiles.contains(name)) {
    throw new FileAlreadyExistsException("File \"" + name + "\" was already written to.");
  }

  if (assertNoDeleteOpenFile && openFiles.containsKey(name)) {
    throw new AssertionError("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite");
  }
  
  unSyncedFiles.add(name);
  createdFiles.add(name);
  
  //System.out.println(Thread.currentThread().getName() + ": MDW: create " + name);
  IndexOutput delegateOutput = in.createOutput(name, LuceneTestCase.newIOContext(randomState, context));
  final IndexOutput io = new MockIndexOutputWrapper(this, delegateOutput, name);
  addFileHandle(io, name, Handle.Output);
  openFilesForWrite.add(name);
  return maybeThrottle(name, io);


}
 
Example #29
Source File: TrollingIndexReaderFactory.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public static Trap catchCount(int boundary) {
  return setTrap(new Trap() {
    
    private AtomicInteger count = new AtomicInteger();
  
    @Override
    public String toString() {
      return ""+count.get()+"th tick of "+boundary+" allowed";
    }
    
    private boolean trigered;

    @Override
    protected boolean shouldExit() {
      int now = count.incrementAndGet();
      boolean trigger = now==boundary 
          || (now>boundary && LuceneTestCase.rarely(LuceneTestCase.random()));
      if (trigger) {
        Exception e = new Exception("stack sniffer"); 
        e.fillInStackTrace();
        recordStackTrace(e.getStackTrace());
        trigered = true;
      } 
      return trigger;
    }

    @Override
    public boolean hasCaught() {
      return trigered;
    }
  });
}
 
Example #30
Source File: TestUtil.java    From querqy with Apache License 2.0 5 votes vote down vote up
public static void addNumDocsWithStringField(String fieldname, String value, RandomIndexWriter indexWriter, int num) throws IOException {
    for (int i = 0; i < num; i++) {
        Document doc = new Document();
        doc.add(LuceneTestCase.newStringField(fieldname, value, Field.Store.YES));
        indexWriter.addDocument(doc);
    }
}