Java Code Examples for org.apache.lucene.util.IOUtils

The following examples show how to use org.apache.lucene.util.IOUtils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: lucene-solr   Source File: TestIndexWriterFromReader.java    License: Apache License 2.0 6 votes vote down vote up
public void testInvalidOpenMode() throws Exception {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
  w.addDocument(new Document());
  w.commit();

  DirectoryReader r = DirectoryReader.open(w);
  assertEquals(1, r.maxDoc());
  w.close();

  IndexWriterConfig iwc = newIndexWriterConfig();
  iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
  iwc.setIndexCommit(r.getIndexCommit());
  IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> {
    new IndexWriter(dir, iwc);
  });
  assertEquals("cannot use IndexWriterConfig.setIndexCommit() with OpenMode.CREATE", expected.getMessage());

  IOUtils.close(r, dir);
}
 
Example 2
Source Project: lucene-solr   Source File: TestAddIndexes.java    License: Apache License 2.0 6 votes vote down vote up
public void testIllegalIndexSortChange1() throws Exception {
  Directory dir1 = newDirectory();
  IndexWriterConfig iwc1 = newIndexWriterConfig(new MockAnalyzer(random()));
  iwc1.setIndexSort(new Sort(new SortField("foo", SortField.Type.INT)));
  RandomIndexWriter w1 = new RandomIndexWriter(random(), dir1, iwc1);
  w1.addDocument(new Document());
  w1.commit();
  w1.addDocument(new Document());
  w1.commit();
  // so the index sort is in fact burned into the index:
  w1.forceMerge(1);
  w1.close();

  Directory dir2 = newDirectory();
  IndexWriterConfig iwc2 = newIndexWriterConfig(new MockAnalyzer(random()));
  iwc2.setIndexSort(new Sort(new SortField("foo", SortField.Type.STRING)));
  RandomIndexWriter w2 = new RandomIndexWriter(random(), dir2, iwc2);
  String message = expectThrows(IllegalArgumentException.class, () -> {
      w2.addIndexes(dir1);
    }).getMessage();
  assertEquals("cannot change index sort from <int: \"foo\"> to <string: \"foo\">", message);
  IOUtils.close(dir1, w2, dir2);
}
 
Example 3
Source Project: lucene-solr   Source File: SolrResourceLoader.java    License: Apache License 2.0 6 votes vote down vote up
private static URLClassLoader addURLsToClassLoader(final URLClassLoader oldLoader, List<URL> urls) {
  if (urls.size() == 0) {
    return oldLoader;
  }

  List<URL> allURLs = new ArrayList<>();
  allURLs.addAll(Arrays.asList(oldLoader.getURLs()));
  allURLs.addAll(urls);
  for (URL url : urls) {
    if (log.isDebugEnabled()) {
      log.debug("Adding '{}' to classloader", url);
    }
  }

  ClassLoader oldParent = oldLoader.getParent();
  IOUtils.closeWhileHandlingException(oldLoader);
  return URLClassLoader.newInstance(allURLs.toArray(new URL[allURLs.size()]), oldParent);
}
 
Example 4
Source Project: lucene-solr   Source File: TestLatLonShape.java    License: Apache License 2.0 6 votes vote down vote up
public void testLUCENE8454() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);

  Polygon poly = new Polygon(new double[] {-1.490648725633769E-132d, 90d, 90d, -1.490648725633769E-132d},
      new double[] {0d, 0d, 180d, 0d});

  Document document = new Document();
  addPolygonsToDoc(FIELDNAME, document, poly);
  writer.addDocument(document);

  ///// search //////
  IndexReader reader = writer.getReader();
  writer.close();
  IndexSearcher searcher = newSearcher(reader);

  // search a bbox in the hole
  Query q = LatLonShape.newBoxQuery(FIELDNAME, QueryRelation.DISJOINT,-29.46555603761226d, 0.0d, 8.381903171539307E-8d, 0.9999999403953552d);
  assertEquals(1, searcher.count(q));

  IOUtils.close(reader, dir);
}
 
Example 5
Source Project: lucene-solr   Source File: TestDuelingAnalyzers.java    License: Apache License 2.0 6 votes vote down vote up
public void testLetterAscii() throws Exception {
  Random random = random();
  Analyzer left = new MockAnalyzer(random, jvmLetter, false);
  Analyzer right = new Analyzer() {
    @Override
    protected TokenStreamComponents createComponents(String fieldName) {
      Tokenizer tokenizer = new LetterTokenizer(newAttributeFactory());
      return new TokenStreamComponents(tokenizer, tokenizer);
    }
  };
  for (int i = 0; i < 200; i++) {
    String s = TestUtil.randomSimpleString(random);
    assertEquals(s, left.tokenStream("foo", newStringReader(s)), 
                 right.tokenStream("foo", newStringReader(s)));
  }
  IOUtils.close(left, right);
}
 
Example 6
Source Project: lucene-solr   Source File: TestTaxonomyFacetCounts.java    License: Apache License 2.0 6 votes vote down vote up
public void testDetectMultiValuedField() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir, IndexWriterConfig.OpenMode.CREATE);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  FacetsConfig config = new FacetsConfig();

  Document doc = new Document();
  doc.add(newTextField("field", "text", Field.Store.NO));
  doc.add(new FacetField("a", "path"));
  doc.add(new FacetField("a", "path2"));
  expectThrows(IllegalArgumentException.class, () -> {
    config.build(taxoWriter, doc);
  });

  writer.close();
  IOUtils.close(taxoWriter, dir, taxoDir);
}
 
Example 7
Source Project: lucene-solr   Source File: TestDuelingAnalyzers.java    License: Apache License 2.0 6 votes vote down vote up
public void testLetterAsciiHuge() throws Exception {
  Random random = random();
  int maxLength = 8192; // CharTokenizer.IO_BUFFER_SIZE*2
  MockAnalyzer left = new MockAnalyzer(random, jvmLetter, false);
  left.setMaxTokenLength(255); // match CharTokenizer's max token length
  Analyzer right = new Analyzer() {
    @Override
    protected TokenStreamComponents createComponents(String fieldName) {
      Tokenizer tokenizer = new LetterTokenizer(newAttributeFactory());
      return new TokenStreamComponents(tokenizer, tokenizer);
    }
  };
  int numIterations = atLeast(10);
  for (int i = 0; i < numIterations; i++) {
    String s = TestUtil.randomSimpleString(random, maxLength);
    assertEquals(s, left.tokenStream("foo", newStringReader(s)), 
                 right.tokenStream("foo", newStringReader(s)));
  }
  IOUtils.close(left, right);
}
 
Example 8
Source Project: Elasticsearch   Source File: SuggestUtils.java    License: Apache License 2.0 6 votes vote down vote up
/** NOTE: this method closes the TokenStream, even on exception, which is awkward
 *  because really the caller who called {@link Analyzer#tokenStream} should close it,
 *  but when trying that there are recursion issues when we try to use the same
 *  TokenStrem twice in the same recursion... */
public static int analyze(TokenStream stream, TokenConsumer consumer) throws IOException {
    int numTokens = 0;
    boolean success = false;
    try {
        stream.reset();
        consumer.reset(stream);
        while (stream.incrementToken()) {
            consumer.nextToken();
            numTokens++;
        }
        consumer.end();
    } finally {
        if (success) {
            stream.close();
        } else {
            IOUtils.closeWhileHandlingException(stream);
        }
    }
    return numTokens;
}
 
Example 9
Source Project: lucene-solr   Source File: BlockTermsWriter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void close() throws IOException {
  if (out != null) {
    try {
      final long dirStart = out.getFilePointer();
      
      out.writeVInt(fields.size());
      for(FieldMetaData field : fields) {
        out.writeVInt(field.fieldInfo.number);
        out.writeVLong(field.numTerms);
        out.writeVLong(field.termsStartPointer);
        if (field.fieldInfo.getIndexOptions() != IndexOptions.DOCS) {
          out.writeVLong(field.sumTotalTermFreq);
        }
        out.writeVLong(field.sumDocFreq);
        out.writeVInt(field.docCount);
      }
      writeTrailer(dirStart);
      CodecUtil.writeFooter(out);
    } finally {
      IOUtils.close(out, postingsWriter, termsIndexWriter);
      out = null;
    }
  }
}
 
Example 10
@Override
public void close() throws IOException {
  boolean success = false;
  try {
    if (meta != null) {
      meta.writeVInt(-1); // write EOF marker
    }
    success = true;
  } finally {
    if (success) {
      IOUtils.close(data, meta);
    } else {
      IOUtils.closeWhileHandlingException(data, meta);
    }
  }
}
 
Example 11
Source Project: lucene-solr   Source File: IndexAndTaxonomyRevisionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSegmentsFileLast() throws Exception {
  Directory indexDir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter indexWriter = new IndexWriter(indexDir, conf);
  
  Directory taxoDir = newDirectory();
  SnapshotDirectoryTaxonomyWriter taxoWriter = new SnapshotDirectoryTaxonomyWriter(taxoDir);
  try {
    indexWriter.addDocument(newDocument(taxoWriter));
    indexWriter.commit();
    taxoWriter.commit();
    Revision rev = new IndexAndTaxonomyRevision(indexWriter, taxoWriter);
    Map<String,List<RevisionFile>> sourceFiles = rev.getSourceFiles();
    assertEquals(2, sourceFiles.size());
    for (List<RevisionFile> files : sourceFiles.values()) {
      String lastFile = files.get(files.size() - 1).fileName;
      assertTrue(lastFile.startsWith(IndexFileNames.SEGMENTS));
    }
    indexWriter.close();
  } finally {
    IOUtils.close(indexWriter, taxoWriter, taxoDir, indexDir);
  }
}
 
Example 12
Source Project: lucene-solr   Source File: TestDirectSpellChecker.java    License: Apache License 2.0 6 votes vote down vote up
public void testTransposition() throws Exception {
  DirectSpellChecker spellChecker = new DirectSpellChecker();
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.SIMPLE, true);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, analyzer);

  for (int i = 0; i < 20; i++) {
    Document doc = new Document();
    doc.add(newTextField("numbers", English.intToEnglish(i), Field.Store.NO));
    writer.addDocument(doc);
  }

  IndexReader ir = writer.getReader();

  SuggestWord[] similar = spellChecker.suggestSimilar(new Term(
      "numbers", "fvie"), 1, ir,
      SuggestMode.SUGGEST_WHEN_NOT_IN_INDEX);
  assertEquals(1, similar.length);
  assertEquals("five", similar[0].string);
  
  IOUtils.close(ir, writer, dir, analyzer);
}
 
Example 13
Source Project: lucene-solr   Source File: TestOrdinalMappingLeafReader.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTaxonomyMergeUtils() throws Exception {
  Directory srcIndexDir = newDirectory();
  Directory srcTaxoDir = newDirectory();
  buildIndexWithFacets(srcIndexDir, srcTaxoDir, true);
  
  Directory targetIndexDir = newDirectory();
  Directory targetTaxoDir = newDirectory();
  buildIndexWithFacets(targetIndexDir, targetTaxoDir, false);
  
  IndexWriter destIndexWriter = new IndexWriter(targetIndexDir, newIndexWriterConfig(null));
  DirectoryTaxonomyWriter destTaxoWriter = new DirectoryTaxonomyWriter(targetTaxoDir);
  try {
    TaxonomyMergeUtils.merge(srcIndexDir, srcTaxoDir, new MemoryOrdinalMap(), destIndexWriter, destTaxoWriter, facetConfig);
  } finally {
    IOUtils.close(destIndexWriter, destTaxoWriter);
  }
  verifyResults(targetIndexDir, targetTaxoDir);
  
  IOUtils.close(targetIndexDir, targetTaxoDir, srcIndexDir, srcTaxoDir);
}
 
Example 14
Source Project: lucene-solr   Source File: KoreanTokenizerFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void inform(ResourceLoader loader) throws IOException {
  if (userDictionaryPath != null) {
    try (InputStream stream = loader.openResource(userDictionaryPath)) {
      String encoding = userDictionaryEncoding;
      if (encoding == null) {
        encoding = IOUtils.UTF_8;
      }
      CharsetDecoder decoder = Charset.forName(encoding).newDecoder()
        .onMalformedInput(CodingErrorAction.REPORT)
        .onUnmappableCharacter(CodingErrorAction.REPORT);
      Reader reader = new InputStreamReader(stream, decoder);
      userDictionary = UserDictionary.open(reader);
    }
  } else {
    userDictionary = null;
  }
}
 
Example 15
Source Project: lucene-solr   Source File: RandomIndexWriter.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Close this writer.
 * @see IndexWriter#close()
 */
@Override
public void close() throws IOException {
  boolean success = false;
  try {
    if (w.isClosed() == false) {
      LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, config);
    }
    // if someone isn't using getReader() API, we want to be sure to
    // forceMerge since presumably they might open a reader on the dir.
    if (getReaderCalled == false && r.nextInt(8) == 2 && w.isClosed() == false) {
      doRandomForceMerge();
      if (config.getCommitOnClose() == false) {
        // index may have changed, must commit the changes, or otherwise they are discarded by the call to close()
        w.commit();
      }
    }
    success = true;
  } finally {
    if (success) {
      IOUtils.close(w, analyzer);
    } else {
      IOUtils.closeWhileHandlingException(w, analyzer);
    }
  }
}
 
Example 16
Source Project: lucene-solr   Source File: TestTermAutomatonQuery.java    License: Apache License 2.0 6 votes vote down vote up
public void testOneTermDoesNotExist() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(newTextField("field", "x y z", Field.Store.NO));
  w.addDocument(doc);

  IndexReader r = w.getReader();
  IndexSearcher s = newSearcher(r);

  TokenStream ts = new CannedTokenStream(new Token[] {
      token("a", 1, 1),
      token("x", 1, 1),
    });

  TermAutomatonQuery q = new TokenStreamToTermAutomatonQuery().toQuery("field", ts);
  // System.out.println("DOT: " + q.toDot());
  assertEquals(0, s.search(q, 1).totalHits.value);

  IOUtils.close(w, r, dir);
}
 
Example 17
Source Project: lucene-solr   Source File: MergePolicy.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Sets the merge readers for this merge.
 */
void initMergeReaders(IOUtils.IOFunction<SegmentCommitInfo, MergeReader> readerFactory) throws IOException {
  assert mergeReaders.isEmpty() : "merge readers must be empty";
  assert mergeCompleted.isDone() == false : "merge is already done";
  final ArrayList<MergeReader> readers = new ArrayList<>(segments.size());
  try {
    for (final SegmentCommitInfo info : segments) {
      // Hold onto the "live" reader; we will use this to
      // commit merged deletes
      readers.add(readerFactory.apply(info));
    }
  } finally {
    // ensure we assign this to close them in the case of an exception
    this.mergeReaders = List.copyOf(readers); // we do a copy here to ensure that mergeReaders are an immutable list
  }
}
 
Example 18
Source Project: lucene-solr   Source File: PerfRunData.java    License: Apache License 2.0 6 votes vote down vote up
private Directory createDirectory(boolean eraseIndex, String dirName,
    String dirParam) throws IOException {
  String dirImpl = config.get(dirParam, DEFAULT_DIRECTORY);
  if ("FSDirectory".equals(dirImpl)) {
    Path workDir = Paths.get(config.get("work.dir", "work"));
    Path indexDir = workDir.resolve(dirName);
    if (eraseIndex && Files.exists(indexDir)) {
      IOUtils.rm(indexDir);
    }
    Files.createDirectories(indexDir);
    return FSDirectory.open(indexDir);
  }

  if ("RAMDirectory".equals(dirImpl)) {
    throw new IOException("RAMDirectory has been removed, use ByteBuffersDirectory.");
  }

  if ("ByteBuffersDirectory".equals(dirImpl)) {
    return new ByteBuffersDirectory();
  }

  throw new IOException("Directory type not supported: " + dirImpl);
}
 
Example 19
Source Project: lucene-solr   Source File: TestTaxonomyFacetAssociations.java    License: Apache License 2.0 6 votes vote down vote up
public void testMixedTypesInSameIndexField() throws Exception {
  Directory dir = newDirectory();
  Directory taxoDir = newDirectory();
  
  TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  FacetsConfig config = new FacetsConfig();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);

  Document doc = new Document();
  doc.add(new IntAssociationFacetField(14, "a", "x"));
  doc.add(new FloatAssociationFacetField(55.0f, "b", "y"));
  expectThrows(IllegalArgumentException.class, () -> {
    writer.addDocument(config.build(taxoWriter, doc));
  });
  writer.close();
  IOUtils.close(taxoWriter, dir, taxoDir);
}
 
Example 20
Source Project: lucene-solr   Source File: AnalyzingSuggesterTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testGraphDups() throws Exception {

    final Analyzer analyzer = new MultiCannedAnalyzer(
        new CannedTokenStream(
            token("wifi",1,1),
            token("hotspot",0,2),
            token("network",1,1),
            token("is",1,1),
            token("slow",1,1)),
        new CannedTokenStream(
            token("wi",1,1),
            token("hotspot",0,3),
            token("fi",1,1),
            token("network",1,1),
            token("is",1,1),
            token("fast",1,1)),
        new CannedTokenStream(
            token("wifi",1,1),
            token("hotspot",0,2),
            token("network",1,1)));

    Input keys[] = new Input[] {
        new Input("wifi network is slow", 50),
        new Input("wi fi network is fast", 10),
    };
    //AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer, AnalyzingSuggester.EXACT_FIRST, 256, -1);
    Directory tempDir = getDirectory();
    AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", analyzer);
    suggester.build(new InputArrayIterator(keys));
    List<LookupResult> results = suggester.lookup("wifi network", false, 10);
    if (VERBOSE) {
      System.out.println("Results: " + results);
    }
    assertEquals(2, results.size());
    assertEquals("wifi network is slow", results.get(0).key);
    assertEquals(50, results.get(0).value);
    assertEquals("wi fi network is fast", results.get(1).key);
    assertEquals(10, results.get(1).value);
    IOUtils.close(analyzer, tempDir);
  }
 
Example 21
Source Project: lucene-solr   Source File: TestNumericDocValuesUpdates.java    License: Apache License 2.0 5 votes vote down vote up
public void testIOContext() throws Exception {
  // LUCENE-5591: make sure we pass an IOContext with an approximate
  // segmentSize in FlushInfo
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  // we want a single large enough segment so that a doc-values update writes a large file
  conf.setMergePolicy(NoMergePolicy.INSTANCE);
  conf.setMaxBufferedDocs(Integer.MAX_VALUE); // manually flush
  conf.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
  IndexWriter writer = new IndexWriter(dir, conf);
  for (int i = 0; i < 100; i++) {
    writer.addDocument(doc(i));
  }
  writer.commit();
  writer.close();
  
  NRTCachingDirectory cachingDir = new NRTCachingDirectory(dir, 100, 1/(1024.*1024.));
  conf = newIndexWriterConfig(new MockAnalyzer(random()));
  // we want a single large enough segment so that a doc-values update writes a large file
  conf.setMergePolicy(NoMergePolicy.INSTANCE);
  conf.setMaxBufferedDocs(Integer.MAX_VALUE); // manually flush
  conf.setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH);
  writer = new IndexWriter(cachingDir, conf);
  writer.updateNumericDocValue(new Term("id", "doc-0"), "val", 100L);
  DirectoryReader reader = DirectoryReader.open(writer); // flush
  assertEquals(0, cachingDir.listCachedFiles().length);
  
  IOUtils.close(reader, writer, cachingDir);
}
 
Example 22
Source Project: ltr4l   Source File: AbstractConfigReader.java    License: Apache License 2.0 5 votes vote down vote up
public AbstractConfigReader(String content) throws IOException {
  if(content != null) {
    InputStream is = new ByteArrayInputStream(content.getBytes());
    try (InputStreamReader ir = new InputStreamReader(is, StandardCharsets.UTF_8)){
      ObjectMapper mapper = new ObjectMapper();
      configMap = mapper.readValue(ir, Map.class);
    } finally {
      IOUtils.closeWhileHandlingException(is);
    }
  }
}
 
Example 23
Source Project: lucene-solr   Source File: BaseNormsFormatTestCase.java    License: Apache License 2.0 5 votes vote down vote up
public void testIndependantSparseIterators() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig().setMergePolicy(newLogMergePolicy());
  CannedNormSimilarity sim = new CannedNormSimilarity(new long[] {42, 10, 20});
  conf.setSimilarity(sim);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, conf);
  Document doc = new Document();
  Field indexedField = new TextField("indexed", "a", Field.Store.NO);
  doc.add(indexedField);
  Document emptyDoc = new Document();
  for (int i = 0; i < 3; ++i) {
    writer.addDocument(doc);
    writer.addDocument(emptyDoc);
  }
  writer.forceMerge(1);
  LeafReader r = getOnlyLeafReader(maybeWrapWithMergingReader(writer.getReader()));
  NumericDocValues n1 = r.getNormValues("indexed");
  NumericDocValues n2 = r.getNormValues("indexed");
  assertEquals(0, n1.nextDoc());
  assertEquals(42, n1.longValue());
  assertEquals(2, n1.nextDoc());
  assertEquals(10, n1.longValue());
  assertEquals(0, n2.nextDoc());
  assertEquals(42, n2.longValue());
  assertEquals(2, n2.nextDoc());
  assertEquals(10, n2.longValue());
  assertEquals(4, n2.nextDoc());
  assertEquals(20, n2.longValue());
  assertEquals(4, n1.nextDoc());
  assertEquals(20, n1.longValue());
  assertEquals(DocIdSetIterator.NO_MORE_DOCS, n1.nextDoc());
  assertEquals(DocIdSetIterator.NO_MORE_DOCS, n2.nextDoc());
  IOUtils.close(r, writer, dir);
}
 
Example 24
Source Project: Elasticsearch   Source File: LocalTranslog.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void close() throws IOException {
    if (closed.compareAndSet(false, true)) {
        try (ReleasableLock lock = writeLock.acquire()) {
            IOUtils.close(writeChannel);
            for (FileChannel channel : readChannels.values()) {
                IOUtils.close(channel);
            }
        } finally {
            logger.debug("translog closed");
        }
    }
}
 
Example 25
Source Project: lucene-solr   Source File: TestDocValues.java    License: Apache License 2.0 5 votes vote down vote up
public void testAddNullNumericDocValues() throws IOException {
  Directory dir = newDirectory();
  IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
  Document doc = new Document();
  if (random().nextBoolean()) {
    doc.add(new NumericDocValuesField("foo", null));
  } else {
    doc.add(new BinaryDocValuesField("foo", null));
  }
  IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> iw.addDocument(doc));
  assertEquals("field=\"foo\": null value not allowed", iae.getMessage());
  IOUtils.close(iw, dir);
}
 
Example 26
Source Project: lucene-solr   Source File: UniformSplitTermsReader.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void close() throws IOException {
  try {
    IOUtils.close(blockInput, dictionaryInput, postingsReader);
  } finally {
    // Clear so refs to terms index is GCable even if app hangs onto us.
    fieldToTermsMap.clear();
  }
}
 
Example 27
Source Project: lucene-solr   Source File: HunspellStemFilterFactory.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void inform(ResourceLoader loader) throws IOException {
  String dicts[] = dictionaryFiles.split(",");

  InputStream affix = null;
  List<InputStream> dictionaries = new ArrayList<>();

  try {
    dictionaries = new ArrayList<>();
    for (String file : dicts) {
      dictionaries.add(loader.openResource(file));
    }
    affix = loader.openResource(affixFile);

    Path tempPath = Files.createTempDirectory(Dictionary.getDefaultTempDir(), "Hunspell");
    try (Directory tempDir = FSDirectory.open(tempPath)) {
      this.dictionary = new Dictionary(tempDir, "hunspell", affix, dictionaries, ignoreCase);
    } finally {
      IOUtils.rm(tempPath); 
    }
  } catch (ParseException e) {
    throw new IOException("Unable to load hunspell data! [dictionary=" + dictionaries + ",affix=" + affixFile + "]", e);
  } finally {
    IOUtils.closeWhileHandlingException(affix);
    IOUtils.closeWhileHandlingException(dictionaries);
  }
}
 
Example 28
Source Project: lucene-solr   Source File: VerboseFS.java    License: Apache License 2.0 5 votes vote down vote up
/** Records message, and rethrows exception if not null */
private void sop(String text, Throwable exception) throws IOException {
  if (exception == null) {
    if (infoStream.isEnabled("FS")) {
      infoStream.message("FS", text);
    }
  } else {
    if (infoStream.isEnabled("FS")) {
      infoStream.message("FS", text + " (FAILED: " + exception + ")");
    }
    throw IOUtils.rethrowAlways(exception);
  }
}
 
Example 29
Source Project: lucene-solr   Source File: SolrRequestParsers.java    License: Apache License 2.0 5 votes vote down vote up
public SolrParams parseParamsAndFillStreams(HttpServletRequest req, ArrayList<ContentStream> streams, InputStream in) throws Exception {
  final Map<String,String[]> map = new HashMap<>();

  // also add possible URL parameters and include into the map (parsed using UTF-8):
  final String qs = req.getQueryString();
  if (qs != null) {
    parseQueryString(qs, map);
  }

  // may be -1, so we check again later. But if it's already greater we can stop processing!
  final long totalLength = req.getContentLength();
  final long maxLength = ((long) uploadLimitKB) * 1024L;
  if (totalLength > maxLength) {
    throw new SolrException(ErrorCode.BAD_REQUEST, "application/x-www-form-urlencoded content length (" +
        totalLength + " bytes) exceeds upload limit of " + uploadLimitKB + " KB");
  }

  // get query String from request body, using the charset given in content-type:
  final String cs = ContentStreamBase.getCharsetFromContentType(req.getContentType());
  final Charset charset = (cs == null) ? StandardCharsets.UTF_8 : Charset.forName(cs);

  try {
    // Protect container owned streams from being closed by us, see SOLR-8933
    in = FastInputStream.wrap( in == null ? new CloseShieldInputStream(req.getInputStream()) : in );

    final long bytesRead = parseFormDataContent(in, maxLength, charset, map, false);
    if (bytesRead == 0L && totalLength > 0L) {
      throw getParameterIncompatibilityException();
    }
  } catch (IOException ioe) {
    throw new SolrException(ErrorCode.BAD_REQUEST, ioe);
  } catch (IllegalStateException ise) {
    throw (SolrException) getParameterIncompatibilityException().initCause(ise);
  } finally {
    IOUtils.closeWhileHandlingException(in);
  }

  return new MultiMapSolrParams(map);
}
 
Example 30
Source Project: lucene-solr   Source File: TestNativeFSLockFactory.java    License: Apache License 2.0 5 votes vote down vote up
/** release the lock and test ensureValid fails */
public void testInvalidateLock() throws IOException {
  Directory dir = getDirectory(createTempDir());
  NativeFSLockFactory.NativeFSLock lock =  (NativeFSLockFactory.NativeFSLock) dir.obtainLock("test.lock");
  lock.ensureValid();
  lock.lock.release();
  expectThrows(AlreadyClosedException.class, () -> {
    lock.ensureValid();
  });

  IOUtils.closeWhileHandlingException(lock);
  dir.close();
}