Java Code Examples for org.apache.lucene.util.Version

The following examples show how to use org.apache.lucene.util.Version. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: olat   Source File: IndexWriterWorker.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @param id
 *            Unique index ID. Is used to generate unique directory name.
 * @param tempIndexPath
 *            Absolute directory-path where the temporary index can be generated.
 * @param fullIndexer
 *            Reference to full-index
 */
public IndexWriterWorker(final int id, final File tempIndexDir, final OlatFullIndexer fullIndexer) {
    this.id = id;
    this.indexPartDir = new File(tempIndexDir, "part" + id);
    this.fullIndexer = fullIndexer;
    try {
        final Directory luceneIndexPartDir = FSDirectory.open(indexPartDir);
        indexWriter = new IndexWriter(luceneIndexPartDir, new StandardAnalyzer(Version.LUCENE_30), true, IndexWriter.MaxFieldLength.UNLIMITED);
        indexWriter.setMergeFactor(fullIndexer.getSearchModuleConfig().getIndexerWriterMergeFactor());
        log.info("IndexWriter config MergeFactor=" + indexWriter.getMergeFactor());
        indexWriter.setRAMBufferSizeMB(fullIndexer.getSearchModuleConfig().getIndexerWriterRamBuffer());
        log.info("IndexWriter config RAMBufferSizeMB=" + indexWriter.getRAMBufferSizeMB());
        indexWriter.setUseCompoundFile(false);
    } catch (final IOException e) {
        log.warn("Can not create IndexWriter");
    }
}
 
Example 2
@Test
public void testMultipleWritersOpenOnSameDirectory() throws IOException {
  IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  FastHdfsKeyValueDirectory directory = new FastHdfsKeyValueDirectory(false, _timer, _configuration, new Path(_path,
      "test_multiple"));
  IndexWriter writer1 = new IndexWriter(directory, config.clone());
  addDoc(writer1, getDoc(1));
  IndexWriter writer2 = new IndexWriter(directory, config.clone());
  addDoc(writer2, getDoc(2));
  writer1.close();
  writer2.close();

  DirectoryReader reader = DirectoryReader.open(directory);
  int maxDoc = reader.maxDoc();
  assertEquals(1, maxDoc);
  Document document = reader.document(0);
  assertEquals("2", document.get("id"));
  reader.close();
}
 
Example 3
Source Project: Elasticsearch   Source File: RecoveryFileChunkRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    recoveryId = in.readLong();
    shardId = ShardId.readShardId(in);
    String name = in.readString();
    position = in.readVLong();
    long length = in.readVLong();
    String checksum = in.readOptionalString();
    content = in.readBytesReference();
    Version writtenBy = null;
    String versionString = in.readOptionalString();
    writtenBy = Lucene.parseVersionLenient(versionString, null);
    metaData = new StoreFileMetaData(name, length, checksum, writtenBy);
    lastChunk = in.readBoolean();
    totalTranslogOps = in.readVInt();
    sourceThrottleTimeInNanos = in.readLong();
}
 
Example 4
Source Project: lucene-solr   Source File: TestIndexWriterOnOldIndex.java    License: Apache License 2.0 6 votes vote down vote up
public void testOpenModeAndCreatedVersion() throws IOException {
  assumeTrue("Reenable when 8.0 is released", false);
  InputStream resource = getClass().getResourceAsStream("index.single-empty-doc.8.0.0.zip");
  assertNotNull(resource);
  Path path = createTempDir();
  TestUtil.unzip(resource, path);
  Directory dir = newFSDirectory(path);
  for (OpenMode openMode : OpenMode.values()) {
    Directory tmpDir = newDirectory(dir);
    assertEquals(7 /** 7.0.0 */, SegmentInfos.readLatestCommit(tmpDir).getIndexCreatedVersionMajor());
    IndexWriter w = new IndexWriter(tmpDir, newIndexWriterConfig().setOpenMode(openMode));
    w.commit();
    w.close();
    switch (openMode) {
      case CREATE:
        assertEquals(Version.LATEST.major, SegmentInfos.readLatestCommit(tmpDir).getIndexCreatedVersionMajor());
        break;
      default:
        assertEquals(7 /** 7.0.0 */, SegmentInfos.readLatestCommit(tmpDir).getIndexCreatedVersionMajor());
    }
    tmpDir.close();
  }
  dir.close();
}
 
Example 5
Source Project: lucene-solr   Source File: BaseSegmentInfoFormatTestCase.java    License: Apache License 2.0 6 votes vote down vote up
/** Test versions */
public void testVersions() throws Exception {
  Codec codec = getCodec();
  for (Version v : getVersions()) {
    for (Version minV : new Version[] { v, null}) {
      Directory dir = newDirectory();
      byte id[] = StringHelper.randomId();
      SegmentInfo info = new SegmentInfo(dir, v, minV, "_123", 1, false, codec, 
                                         Collections.<String,String>emptyMap(), id, Collections.emptyMap(), null);
      info.setFiles(Collections.<String>emptySet());
      codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
      SegmentInfo info2 = codec.segmentInfoFormat().read(dir, "_123", id, IOContext.DEFAULT);
      assertEquals(info2.getVersion(), v);
      if (supportsMinVersion()) {
        assertEquals(info2.getMinVersion(), minV);
      } else {
        assertEquals(info2.getMinVersion(), null);
      }
      dir.close();
    }
  }
}
 
Example 6
Source Project: webdsl   Source File: AutoCompleter.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Use a different index as the auto completer index or re-open
 * the existing index if <code>autocompleteIndex</code> is the same value
 * as given in the constructor.
 * @param autocompleteIndexDir the autocomplete directory to use
 * @throws AlreadyClosedException if the Autocompleter is already closed
 * @throws  IOException if autocompleter can not open the directory
 */
// TODO: we should make this final as it is called in the constructor
public void setAutoCompleteIndex(Directory autocompleteIndexDir) throws IOException {
  // this could be the same directory as the current autocompleteIndex
  // modifications to the directory should be synchronized
  synchronized (modifyCurrentIndexLock) {
    ensureOpen();
    if (!IndexReader.indexExists(autocompleteIndexDir)) {
        IndexWriter writer = new IndexWriter(autocompleteIndexDir,
          new IndexWriterConfig(Version.LUCENE_CURRENT,
              new WhitespaceAnalyzer(Version.LUCENE_CURRENT)));
        writer.close();
    }
    swapSearcher(autocompleteIndexDir);
  }
}
 
Example 7
Source Project: dexter   Source File: LuceneHelper.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Opens or creates a lucene index in the given directory
 * 
 * @param wikiIdtToLuceneIdSerialization
 *            - the file containing the serialized mapping between wiki-id
 *            and Lucene documents ids
 * 
 * @param indexPath
 *            - the path of the directory with the Lucene's index
 */
protected LuceneHelper(File wikiIdtToLuceneIdSerialization, File indexPath) {
	logger.info("opening lucene index in folder {}", indexPath);
	config = new IndexWriterConfig(Version.LUCENE_41, ANALYZER);
	this.wikiIdtToLuceneIdSerialization = wikiIdtToLuceneIdSerialization;

	BooleanQuery.setMaxClauseCount(1000);

	try {
		index = FSDirectory.open(indexPath);
		// writer.commit();
	} catch (Exception e) {
		logger.error("opening the index: {}", e.toString());
		System.exit(1);
	}

	summarizer = new ArticleSummarizer();
	writer = getWriter();
	collectionSize = writer.numDocs();
	wikiIdToLuceneId = Collections.emptyMap();
}
 
Example 8
public Solr4QueryParser getLuceneQueryParser(SearchParameters searchParameters, SolrQueryRequest req, FTSQueryParser.RerankPhase rerankPhase)
{
    Analyzer analyzer =  req.getSchema().getQueryAnalyzer();
    Solr4QueryParser parser = new Solr4QueryParser(req, Version.LATEST, searchParameters.getDefaultFieldName(), analyzer, rerankPhase);
    parser.setNamespacePrefixResolver(namespaceDAO);
    parser.setDictionaryService(getDictionaryService(CMISStrictDictionaryService.DEFAULT));
    parser.setTenantService(tenantService);
    parser.setSearchParameters(searchParameters);
    parser.setAllowLeadingWildcard(true);

    Properties props = new CoreDescriptorDecorator(req.getCore().getCoreDescriptor()).getProperties();
    int topTermSpanRewriteLimit = Integer.parseInt(props.getProperty("alfresco.topTermSpanRewriteLimit", "1000"));
    parser.setTopTermSpanRewriteLimit(topTermSpanRewriteLimit);

    return parser;
}
 
Example 9
Source Project: modernmt   Source File: ContextAnalyzerIndex.java    License: Apache License 2.0 6 votes vote down vote up
public ContextAnalyzerIndex(Directory directory, Rescorer rescorer) throws IOException {
    this.indexDirectory = directory;
    this.analyzer = new CorpusAnalyzer();
    this.rescorer = rescorer;

    // Index writer setup
    IndexWriterConfig indexConfig = new IndexWriterConfig(Version.LUCENE_4_10_4, this.analyzer);
    indexConfig.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);
    indexConfig.setSimilarity(new DefaultSimilarity() {

        @Override
        public float lengthNorm(FieldInvertState state) {
            return 1.f;
        }

    });

    this.indexWriter = new IndexWriter(this.indexDirectory, indexConfig);

    // Ensure index exists
    if (!DirectoryReader.indexExists(directory))
        this.indexWriter.commit();
}
 
Example 10
Source Project: incubator-retired-blur   Source File: Blur024CodecTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDocValuesFormat() throws IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new WhitespaceAnalyzer(Version.LUCENE_43));
  conf.setCodec(new Blur024Codec());
  IndexWriter writer = new IndexWriter(directory, conf);

  Document doc = new Document();
  doc.add(new StringField("f", "v", Store.YES));
  doc.add(new SortedDocValuesField("f", new BytesRef("v")));
  writer.addDocument(doc);

  writer.close();

  DirectoryReader reader = DirectoryReader.open(directory);
  AtomicReaderContext context = reader.leaves().get(0);
  AtomicReader atomicReader = context.reader();
  SortedDocValues sortedDocValues = atomicReader.getSortedDocValues("f");
  assertTrue(sortedDocValues.getClass().getName().startsWith(DiskDocValuesProducer.class.getName()));

  reader.close();
}
 
Example 11
Source Project: olat   Source File: SearchInputController.java    License: Apache License 2.0 6 votes vote down vote up
protected Set<String> getHighlightWords(final String searchString) {
    try {
        final Analyzer analyzer = new StandardAnalyzer(Version.LUCENE_CURRENT);
        final TokenStream stream = analyzer.tokenStream("content", new StringReader(searchString));
        final TermAttribute termAtt = stream.addAttribute(TermAttribute.class);
        for (boolean next = stream.incrementToken(); next; next = stream.incrementToken()) {
            final String term = termAtt.term();
            if (log.isDebugEnabled()) {
                log.debug(term);
            }
        }
    } catch (final IOException e) {
        log.error("", e);
    }
    return null;
}
 
Example 12
Source Project: lucene-solr   Source File: TestSegmentTermDocs.java    License: Apache License 2.0 6 votes vote down vote up
public void testTermDocs() throws IOException {
  //After adding the document, we should be able to read it back in
  SegmentReader reader = new SegmentReader(info, Version.LATEST.major, newIOContext(random()));
  assertTrue(reader != null);

  TermsEnum terms = reader.terms(DocHelper.TEXT_FIELD_2_KEY).iterator();
  terms.seekCeil(new BytesRef("field"));
  PostingsEnum termDocs = TestUtil.docs(random(), terms, null, PostingsEnum.FREQS);
  if (termDocs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS)    {
    int docId = termDocs.docID();
    assertTrue(docId == 0);
    int freq = termDocs.freq();
    assertTrue(freq == 3);  
  }
  reader.close();
}
 
Example 13
Source Project: lucene-solr   Source File: SegmentInfo.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Construct a new complete SegmentInfo instance from input.
 * <p>Note: this is public only to allow access from
 * the codecs package.</p>
 */
public SegmentInfo(Directory dir, Version version, Version minVersion, String name, int maxDoc,
                   boolean isCompoundFile, Codec codec, Map<String,String> diagnostics,
                   byte[] id, Map<String,String> attributes, Sort indexSort) {
  assert !(dir instanceof TrackingDirectoryWrapper);
  this.dir = Objects.requireNonNull(dir);
  this.version = Objects.requireNonNull(version);
  this.minVersion = minVersion;
  this.name = Objects.requireNonNull(name);
  this.maxDoc = maxDoc;
  this.isCompoundFile = isCompoundFile;
  this.codec = codec;
  this.diagnostics = Map.copyOf(Objects.requireNonNull(diagnostics));
  this.id = id;
  if (id.length != StringHelper.ID_LENGTH) {
    throw new IllegalArgumentException("invalid id: " + Arrays.toString(id));
  }
  this.attributes = Map.copyOf(Objects.requireNonNull(attributes));
  this.indexSort = indexSort;
}
 
Example 14
Source Project: Lottery   Source File: LuceneContentSvcImpl.java    License: GNU General Public License v2.0 5 votes vote down vote up
@Transactional(readOnly = true)
public void createIndex(Content content, Directory dir) throws IOException {
	boolean exist = IndexReader.indexExists(dir);
	IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(
			Version.LUCENE_30), !exist, IndexWriter.MaxFieldLength.LIMITED);
	try {
		writer.addDocument(LuceneContent.createDocument(content));
	} finally {
		writer.close();
	}
}
 
Example 15
Source Project: webdsl   Source File: AutoCompleter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Removes all terms from the auto complete index.
 * @throws IOException
 * @throws AlreadyClosedException if the Autocompleter is already closed
 */
public void clearIndex() throws IOException {
  synchronized (modifyCurrentIndexLock) {
    ensureOpen();
    final Directory dir = this.autoCompleteIndex;
    final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
        Version.LUCENE_CURRENT,
        new WhitespaceAnalyzer(Version.LUCENE_CURRENT))
        .setOpenMode(OpenMode.CREATE));
    writer.close();
    swapSearcher(dir);
  }
}
 
Example 16
Source Project: exhibitor   Source File: IndexBuilder.java    License: Apache License 2.0 5 votes vote down vote up
public void open() throws Exception
{
    if ( !directory.exists() && !directory.mkdirs() )
    {
        throw new IOException("Could not make: " + directory);
    }

    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_35, new KeywordAnalyzer()).setOpenMode(IndexWriterConfig.OpenMode.CREATE);

    niofsDirectory = new NIOFSDirectory(directory, new SingleInstanceLockFactory());
    writer = new IndexWriter(niofsDirectory, conf);
}
 
Example 17
Source Project: lucene-solr   Source File: TestCustomAnalyzer.java    License: Apache License 2.0 5 votes vote down vote up
public void testIncorrectOrder() throws Exception {
  expectThrows(IllegalStateException.class, () -> {
    CustomAnalyzer.builder()
        .addCharFilter("htmlstrip")
        .withDefaultMatchVersion(Version.LATEST)
        .withTokenizer("whitespace")
        .build();
  });
}
 
Example 18
Source Project: lucene-solr   Source File: IndexSchema.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Constructs a schema using the specified resource name and stream.
 * By default, this follows the normal config path directory searching rules.
 * @see SolrResourceLoader#openResource
 */
public IndexSchema(String name, InputSource is, Version luceneVersion, SolrResourceLoader resourceLoader, Properties substitutableProperties) {
  this(luceneVersion, resourceLoader, substitutableProperties);

  this.resourceName = Objects.requireNonNull(name);
  try {
    readSchema(is);
    loader.inform(loader);
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example 19
Source Project: Elasticsearch   Source File: LengthTokenFilterFactory.java    License: Apache License 2.0 5 votes vote down vote up
@Inject
public LengthTokenFilterFactory(Index index, IndexSettingsService indexSettingsService, @Assisted String name, @Assisted Settings settings) {
    super(index, indexSettingsService.getSettings(), name, settings);
    min = settings.getAsInt("min", 0);
    max = settings.getAsInt("max", Integer.MAX_VALUE);
    if (version.onOrAfter(Version.LUCENE_4_4) && settings.get(ENABLE_POS_INC_KEY) != null) {
        throw new IllegalArgumentException(ENABLE_POS_INC_KEY + " is not supported anymore. Please fix your analysis chain or use"
                + " an older compatibility version (<=4.3) but beware that it might cause highlighting bugs.");
    }
    enablePositionIncrements = version.onOrAfter(Version.LUCENE_4_4) ? true : settings.getAsBoolean(ENABLE_POS_INC_KEY, true);
}
 
Example 20
Source Project: lucene-solr   Source File: TestSegmentMerger.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();
  mergedDir = newDirectory();
  merge1Dir = newDirectory();
  merge2Dir = newDirectory();
  DocHelper.setupDoc(doc1);
  SegmentCommitInfo info1 = DocHelper.writeDoc(random(), merge1Dir, doc1);
  DocHelper.setupDoc(doc2);
  SegmentCommitInfo info2 = DocHelper.writeDoc(random(), merge2Dir, doc2);
  reader1 = new SegmentReader(info1, Version.LATEST.major, newIOContext(random()));
  reader2 = new SegmentReader(info2, Version.LATEST.major, newIOContext(random()));
}
 
Example 21
Source Project: Elasticsearch   Source File: EdgeNGramTokenFilterFactory.java    License: Apache License 2.0 5 votes vote down vote up
EdgeNGramTokenFilterFactory(Index index, Settings indexSettings, String name, Settings settings) {
    super(index, indexSettings, name, settings);
    this.minGram = settings.getAsInt("min_gram", NGramTokenFilter.DEFAULT_MIN_NGRAM_SIZE);
    this.maxGram = settings.getAsInt("max_gram", NGramTokenFilter.DEFAULT_MAX_NGRAM_SIZE);
    this.side = parseSide(settings.get("side", "front"));
    this.esVersion = org.elasticsearch.Version.indexCreated(indexSettings);
}
 
Example 22
Source Project: jesterj   Source File: SolrSchemaUtilTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testLoadDualAnalyzerType() throws Exception {
  SolrSchemaUtil solrSchemaUtil = new SolrSchemaUtil();
  Document schemaDocument = solrSchemaUtil.getSchemaDocument(SOLR_SCHEMA_XML, loader);
  assertNotNull(schemaDocument);

  // note we want to avoid any examples with substitution because substitution is not supported.
  FieldType textField = solrSchemaUtil.getFieldType(schemaDocument, "teststop", Version.LUCENE_7_6_0.toString(), 1.0f, loader);
  assertNotNull(textField);
  assertEquals("teststop", textField.getTypeName());
  assertNotNull(textField.getIndexAnalyzer());
  assertNotNull(textField.getQueryAnalyzer());
  assertNotSame(textField.getIndexAnalyzer(), textField.getQueryAnalyzer());
}
 
Example 23
@Override
public IndexWriter createIndexWriter(Directory directory, ODocument metadata) throws IOException {

  Analyzer analyzer = getAnalyzer(metadata);
  Version version = getLuceneVersion(metadata);
  IndexWriterConfig iwc = new IndexWriterConfig(version, analyzer);
  iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE_OR_APPEND);

  facetManager = new OLuceneFacetManager(this, metadata);

  OLogManager.instance().debug(this, "Creating Lucene index in '%s'...", directory);

  return new IndexWriter(directory, iwc);
}
 
Example 24
@Override
public TokenStream create(TokenStream tokenStream) {
    if (version.onOrAfter(Version.LUCENE_4_4_0)) {
        return new DictionaryCompoundWordTokenFilter(tokenStream, wordList, minWordSize, 
                                                     minSubwordSize, maxSubwordSize, onlyLongestMatch);
    } else {
        return new Lucene43DictionaryCompoundWordTokenFilter(tokenStream, wordList, minWordSize,
                                                             minSubwordSize, maxSubwordSize, onlyLongestMatch);
    }
}
 
Example 25
Source Project: Elasticsearch   Source File: EdgeNGramTokenizerFactory.java    License: Apache License 2.0 5 votes vote down vote up
public EdgeNGramTokenizerFactory(Index index, Settings indexSettings, String name, Settings settings) {
    super(index, indexSettings, name, settings);
    this.minGram = settings.getAsInt("min_gram", NGramTokenizer.DEFAULT_MIN_NGRAM_SIZE);
    this.maxGram = settings.getAsInt("max_gram", NGramTokenizer.DEFAULT_MAX_NGRAM_SIZE);
    this.side = Lucene43EdgeNGramTokenizer.Side.getSide(settings.get("side", Lucene43EdgeNGramTokenizer.DEFAULT_SIDE.getLabel()));
    this.matcher = parseTokenChars(settings.getAsArray("token_chars"));
    this.esVersion = org.elasticsearch.Version.indexCreated(indexSettings);
}
 
Example 26
Source Project: Elasticsearch   Source File: TrimTokenFilterFactory.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TokenStream create(TokenStream tokenStream) {
    if (version.onOrAfter(Version.LUCENE_4_4_0)) {
        return new TrimFilter(tokenStream);
    } else {
        @SuppressWarnings("deprecation")
        final TokenStream filter = new Lucene43TrimFilter(tokenStream, updateOffsets);
        return filter;
    }
}
 
Example 27
Source Project: lucene-solr   Source File: FieldTypePluginLoader.java    License: Apache License 2.0 5 votes vote down vote up
private Version parseConfiguredVersion(String configuredVersion, String pluginClassName) {
  Version version = (configuredVersion != null) ?
          SolrConfig.parseLuceneVersionString(configuredVersion) : schema.getDefaultLuceneMatchVersion();

  if (!version.onOrAfter(Version.LUCENE_8_0_0)) {
    log.warn("{} is using deprecated {}"
        + " emulation. You should at some point declare and reindex to at least 8.0, because "
        + "7.x emulation is deprecated and will be removed in 9.0"
        , pluginClassName
        , version);
  }
  return version;
}
 
Example 28
Source Project: lucene-solr   Source File: TestTieredMergePolicy.java    License: Apache License 2.0 5 votes vote down vote up
public void testForcedMergeWithPending() throws Exception {
  final TieredMergePolicy tmp = new TieredMergePolicy();
  final double maxSegmentSize = 10.0D;
  tmp.setMaxMergedSegmentMB(maxSegmentSize);

  SegmentInfos infos = new SegmentInfos(Version.LATEST.major);
  for (int j = 0; j < 30; ++j) {
    infos.add(makeSegmentCommitInfo("_" + j, 1000, 0, 1.0D, IndexWriter.SOURCE_MERGE));
  }
  final MockMergeContext mergeContext = new MockMergeContext(SegmentCommitInfo::getDelCount);
  mergeContext.setMergingSegments(Collections.singleton(infos.asList().get(0)));
  final int expectedCount = random().nextInt(10) + 3;
  final MergeSpecification specification = tmp.findForcedMerges(infos, expectedCount, segmentsToMerge(infos), mergeContext);
  // Since we have fewer than 30 (the max merge count) segments more than the final size this would have been the final merge
  // so we check that it was prevented.
  assertNull(specification);

  SegmentInfos manySegmentsInfos = new SegmentInfos(Version.LATEST.major);
  final int manySegmentsCount = atLeast(500);
  for (int j = 0; j < manySegmentsCount; ++j) {
    manySegmentsInfos.add(makeSegmentCommitInfo("_" + j, 1000, 0, 0.1D, IndexWriter.SOURCE_MERGE));
  }

  // We set one merge to be ongoing. Since we have more than 30 (the max merge count) times the number of segments
  // of that we want to merge to this is not the final merge and hence the returned specification must not be null.
  mergeContext.setMergingSegments(Collections.singleton(manySegmentsInfos.asList().get(0)));
  final MergeSpecification specificationManySegments =
      tmp.findForcedMerges(manySegmentsInfos, expectedCount, segmentsToMerge(manySegmentsInfos), mergeContext);
  assertMaxSize(specificationManySegments, maxSegmentSize);
  for (OneMerge merge : specificationManySegments.merges) {
    assertEquals("No merges of less than the max merge count are permitted while another merge is in progress",
        merge.segments.size(), tmp.getMaxMergeAtOnceExplicit());
  }
  final int resultingCountManySegments = manySegmentsInfos.size() + specificationManySegments.merges.size()
      - specificationManySegments.merges.stream().mapToInt(spec -> spec.segments.size()).sum();
  assertTrue(resultingCountManySegments >= expectedCount);
}
 
Example 29
private void test(int expected, boolean rowQuery, Collection<String> discoverAuthorizations) throws IOException,
    ParseException {
  DirectoryReader reader = DirectoryReader.open(_dir);
  SuperParser parser = new SuperParser(Version.LUCENE_43, _fieldManager, rowQuery, null, ScoreType.SUPER, new Term(
      BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE));

  Query query = parser.parse("fam.string:value");

  Collection<String> readAuthorizations = null;
  Set<String> discoverableFields = new HashSet<String>();
  discoverableFields.add("rowid");
  discoverableFields.add("recordid");
  discoverableFields.add("family");
  IndexSearcher searcher = new SecureIndexSearcher(reader, getAccessControlFactory(), readAuthorizations,
      discoverAuthorizations, discoverableFields, null);

  TopDocs topDocs = searcher.search(query, 10);
  assertEquals(expected, topDocs.totalHits);
  for (int i = 0; i < expected; i++) {
    int doc = topDocs.scoreDocs[i].doc;
    Document document = searcher.doc(doc);
    List<IndexableField> fields = document.getFields();
    for (IndexableField field : fields) {
      assertTrue(discoverableFields.contains(field.name()));
    }
  }
  reader.close();
}
 
Example 30
Source Project: sql-layer   Source File: Indexer.java    License: GNU Affero General Public License v3.0 5 votes vote down vote up
public Indexer(FullTextIndexShared index, Analyzer analyzer) throws IOException {
    IndexWriterConfig iwc = new IndexWriterConfig(Version.LUCENE_40, analyzer);
    iwc.setMaxBufferedDeleteTerms(1); // The deletion needs to be reflected immediately (on disk)
    this.index = index;
    this.writer = new IndexWriter(index.open(),  iwc);

}