Java Code Examples for org.apache.lucene.index.AtomicReader

The following examples show how to use org.apache.lucene.index.AtomicReader. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: linden   Source File: LindenFieldCacheImpl.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Sets the key to the value for the provided reader;
 * if the key is already set then this doesn't change it.
 */
public void put(AtomicReader reader, CacheKey key, Accountable value) {
  final Object readerKey = reader.getCoreCacheKey();
  synchronized (readerCache) {
    Map<CacheKey, Accountable> innerCache = readerCache.get(readerKey);
    if (innerCache == null) {
      // First time this reader is using FieldCache
      innerCache = new HashMap<>();
      readerCache.put(readerKey, innerCache);
      wrapper.initReader(reader);
    }
    if (innerCache.get(key) == null) {
      innerCache.put(key, value);
    } else {
      // Another thread beat us to it; leave the current
      // value
    }
  }
}
 
Example 2
Source Project: linden   Source File: LindenFieldCacheImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
    throws IOException {
  final Map<String, Integer> uidMap = new HashMap<>();

  Uninvert u = new Uninvert() {
    private String currentValue;

    @Override
    public void visitTerm(BytesRef term) {
      currentValue = term.utf8ToString();
    }

    @Override
    public void visitDoc(int docID) {
      uidMap.put(currentValue, docID);
    }

    @Override
    protected TermsEnum termsEnum(Terms terms) throws IOException {
      return terms.iterator(null);
    }
  };
  u.uninvert(reader, key.field, setDocsWithField);
  return new PerReaderUIDMaps(reader.getContext().ord, uidMap);
}
 
Example 3
Source Project: linden   Source File: CustomCacheWrapper.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Object get(int doc) {
  Map<Integer, Object> innerCache = cache.get(context.reader().getCoreCacheKey());
  if (innerCache == null) {
    synchronized (cache) {
      innerCache = cache.get(context.reader().getCoreCacheKey());
      if (innerCache == null) {
        innerCache = new HashMap<>();
        for (int i = 0; i < context.reader().maxDoc(); ++i) {
          innerCache.put(i, createValue(i));
        }
        LOGGER.info("Create cache for context ord:{}, base:{}, max doc:{}.", context.ord, context.docBase, context.reader().maxDoc());
        cache.put(context.reader().getCoreCacheKey(), innerCache);
        context.reader().addCoreClosedListener(new AtomicReader.CoreClosedListener() {
          @Override
          public void onClose(Object ownerCoreCacheKey) {
            cache.remove(ownerCoreCacheKey);
          }
        });
      }
    }
  }
  return innerCache.get(doc);
}
 
Example 4
Source Project: incubator-retired-blur   Source File: FilterCache.java    License: Apache License 2.0 6 votes vote down vote up
private DocIdSet docIdSetToCache(DocIdSet docIdSet, AtomicReader reader, String segmentName, Directory directory)
    throws IOException {
  if (docIdSet == null) {
    // this is better than returning null, as the nonnull result can be cached
    return DocIdSet.EMPTY_DOCIDSET;
  } else if (docIdSet.isCacheable()) {
    return docIdSet;
  } else {
    final DocIdSetIterator it = docIdSet.iterator();
    // null is allowed to be returned by iterator(),
    // in this case we wrap with the empty set,
    // which is cacheable.
    if (it == null) {
      return DocIdSet.EMPTY_DOCIDSET;
    } else {
      final IndexFileBitSet bits = new IndexFileBitSet(reader.maxDoc(), _id, segmentName, directory);
      if (!bits.exists()) {
        bits.create(it);
      }
      bits.load();
      return bits;
    }
  }
}
 
Example 5
Source Project: incubator-retired-blur   Source File: FacetExecutor.java    License: Apache License 2.0 6 votes vote down vote up
public void addScorers(AtomicReaderContext context, Scorer[] scorers) throws IOException {
  LOG.debug(getPrefix("adding scorers context [{0}] [{1}]"), context, Arrays.asList(scorers));
  if (scorers.length != _length) {
    throw new IOException("Scorer length is not correct expecting [" + _length + "] actual [" + scorers.length + "]");
  }
  Object key = getKey(context);
  Info info = _infoMap.get(key);
  if (info == null) {
    info = new Info(context, scorers, _locks, _instance);
    _infoMap.put(key, info);
  } else {
    AtomicReader reader = context.reader();
    LOG.warn(getPrefix("Info about reader context [{0}] already created, existing Info [{1}] current reader [{2}]."),
        context, info, reader);
  }
}
 
Example 6
Source Project: linden   Source File: IntermediateForm.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This method is used by the index update combiner and process an
 * intermediate form into the current intermediate form. More specifically,
 * the input intermediate forms are a single-document ram index and/or a
 * single delete term.
 * @param form  the input intermediate form
 * @throws IOException
 */
public void process(IntermediateForm form, FacetsConfig facetsConfig) throws IOException {
  if (form.dir.ramBytesUsed() > 0 || form.taxoDir.ramBytesUsed() > 0) {
    if (writer == null) {
      createWriter();
    }

    if (facetsConfig != null) {
      DirectoryTaxonomyWriter.OrdinalMap map = new DirectoryTaxonomyWriter.MemoryOrdinalMap();
      // merge the taxonomies
      taxoWriter.addTaxonomy(form.taxoDir, map);
      int ordinalMap[] = map.getMap();
      DirectoryReader reader = DirectoryReader.open(form.dir);
      try {
        List<AtomicReaderContext> leaves = reader.leaves();
        int numReaders = leaves.size();
        AtomicReader wrappedLeaves[] = new AtomicReader[numReaders];
        for (int i = 0; i < numReaders; i++) {
          wrappedLeaves[i] = new OrdinalMappingAtomicReader(leaves.get(i).reader(), ordinalMap, facetsConfig);
        }
        writer.addIndexes(new MultiReader(wrappedLeaves));
      } finally {
        reader.close();
      }
    } else {
      writer.addIndexes(new Directory[] { form.dir });
    }
    numDocs++;
  }
}
 
Example 7
Source Project: linden   Source File: LindenFieldCacheImpl.java    License: Apache License 2.0 5 votes vote down vote up
private void initReader(AtomicReader reader) {
  if (reader instanceof SegmentReader) {
    reader.addCoreClosedListener(purgeCore);
  } else {
    // we have a slow reader of some sort, try to register a purge event
    // rather than relying on gc:
    Object key = reader.getCoreCacheKey();
    if (key instanceof AtomicReader) {
      ((AtomicReader) key).addReaderClosedListener(purgeReader);
    } else {
      // last chance
      reader.addReaderClosedListener(purgeReader);
    }
  }
}
 
Example 8
Source Project: linden   Source File: LindenFieldCacheImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
    throws IOException {
  int maxDoc = reader.maxDoc();

  final int[][] matrix = new int[maxDoc][];
  BinaryDocValues valuesIn = reader.getBinaryDocValues(key.field);
  if (valuesIn == null) {
    for (int i = 0; i < maxDoc; ++i) {
      matrix[i] = new int[0];
    }
    return new IntList(matrix);
  }
  for (int i = 0; i < maxDoc; ++i) {
    String str = valuesIn.get(i).utf8ToString();
    if (StringUtils.isEmpty(str)) {
      matrix[i] = new int[0];
      continue;
    }
    JSONArray array = JSON.parseArray(str);
    matrix[i] = new int[array.size()];
    for (int j = 0; j < array.size(); ++j) {
      matrix[i][j] = array.getInteger(j);
    }
  }
  return new IntList(matrix);
}
 
Example 9
@Override
public AccessControlReader clone(AtomicReader in) throws IOException {
  try {
    FilterAccessControlReader filterAccessControlReader = (FilterAccessControlReader) super.clone();
    filterAccessControlReader._isClone = true;
    if (_readDocumentVisibilityFilter == null) {
      filterAccessControlReader._noReadAccess = true;
    } else {
      DocIdSet readDocIdSet = _readDocumentVisibilityFilter.getDocIdSet(in.getContext(), in.getLiveDocs());
      if (readDocIdSet == DocIdSet.EMPTY_DOCIDSET || readDocIdSet == null) {
        filterAccessControlReader._noReadAccess = true;
      } else {
        filterAccessControlReader._readBits = readDocIdSet.bits();
        if (filterAccessControlReader._readBits == null) {
          throw new IOException("Read Bits can not be null.");
        }
      }
      filterAccessControlReader._readDocIdSet = readDocIdSet;
    }

    if (_discoverDocumentVisibilityFilter == null) {
      filterAccessControlReader._noDiscoverAccess = true;
    } else {
      DocIdSet discoverDocIdSet = _discoverDocumentVisibilityFilter.getDocIdSet(in.getContext(), in.getLiveDocs());
      if (discoverDocIdSet == DocIdSet.EMPTY_DOCIDSET || discoverDocIdSet == null) {
        filterAccessControlReader._noDiscoverAccess = true;
      } else {
        filterAccessControlReader._discoverBits = discoverDocIdSet.bits();
        if (filterAccessControlReader._discoverBits == null) {
          throw new IOException("Read Bits can not be null.");
        }
      }
      filterAccessControlReader._discoverDocIdSet = discoverDocIdSet;
    }
    return filterAccessControlReader;
  } catch (CloneNotSupportedException e) {
    throw new IOException(e);
  }
}
 
Example 10
Source Project: linden   Source File: LindenFieldCacheImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
    throws IOException {
  int maxDoc = reader.maxDoc();

  final float[][] matrix = new float[maxDoc][];
  BinaryDocValues valuesIn = reader.getBinaryDocValues(key.field);
  if (valuesIn == null) {
    for (int i = 0; i < maxDoc; ++i) {
      matrix[i] = new float[0];
    }
    return new FloatList(matrix);
  }
  for (int i = 0; i < maxDoc; ++i) {
    String str = valuesIn.get(i).utf8ToString();
    if (StringUtils.isEmpty(str)) {
      matrix[i] = new float[0];
      continue;
    }
    JSONArray array = JSON.parseArray(str);
    matrix[i] = new float[array.size()];
    for (int j = 0; j < array.size(); ++j) {
      matrix[i][j] = array.getFloat(j);
    }
  }
  return new FloatList(matrix);
}
 
Example 11
Source Project: linden   Source File: LindenFieldCacheImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
    throws IOException {
  int maxDoc = reader.maxDoc();

  final double[][] matrix = new double[maxDoc][];
  BinaryDocValues valuesIn = reader.getBinaryDocValues(key.field);
  if (valuesIn == null) {
    for (int i = 0; i < maxDoc; ++i) {
      matrix[i] = new double[0];
    }
    return new DoubleList(matrix);
  }
  for (int i = 0; i < maxDoc; ++i) {
    String str = valuesIn.get(i).utf8ToString();
    if (StringUtils.isEmpty(str)) {
      matrix[i] = new double[0];
      continue;
    }
    JSONArray array = JSON.parseArray(str);
    matrix[i] = new double[array.size()];
    for (int j = 0; j < array.size(); ++j) {
      matrix[i][j] = array.getFloat(j);
    }
  }
  return new DoubleList(matrix);
}
 
Example 12
Source Project: linden   Source File: LindenFieldCacheImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
    throws IOException {
  int maxDoc = reader.maxDoc();

  final String[][] matrix = new String[maxDoc][];
  BinaryDocValues valuesIn = reader.getBinaryDocValues(key.field);
  if (valuesIn == null) {
    for (int i = 0; i < maxDoc; ++i) {
      matrix[i] = new String[0];
    }
    return new StringList(matrix);
  }
  for (int i = 0; i < maxDoc; ++i) {
    String str = valuesIn.get(i).utf8ToString();
    if (StringUtils.isEmpty(str)) {
      matrix[i] = new String[0];
      continue;
    }
    JSONArray array = JSON.parseArray(str);
    matrix[i] = new String[array.size()];
    for (int j = 0; j < array.size(); ++j) {
      matrix[i][j] = array.getString(j);
    }
  }
  return new StringList(matrix);
}
 
Example 13
@Override
public DocIdSet getDocIdSet(String fieldName, BytesRef term, AtomicReader reader) {
  Key key = new Key(fieldName, term, reader.getCoreCacheKey());
  DocIdSet docIdSet = _cache.get(key);
  if (docIdSet != null) {
    LOG.debug("Cache hit for key [" + key + "]");
  } else {
    LOG.debug("Cache miss for key [" + key + "]");
  }
  return docIdSet;
}
 
Example 14
private static AtomicReader setDocSize(AtomicReader reader, final int count) {
  return new FilterAtomicReader(reader) {
    @Override
    public Bits getLiveDocs() {
      return new Bits() {
        @Override
        public boolean get(int index) {
          return true;
        }

        @Override
        public int length() {
          return count;
        }
      };
    }

    @Override
    public int numDocs() {
      return count;
    }

    @Override
    public int maxDoc() {
      return count;
    }

    @Override
    public void document(int docID, StoredFieldVisitor visitor) throws IOException {
      // Do nothing
    }
  };
}
 
Example 15
protected void runGisDocValueTest(String s) throws IOException {
  DirectoryReader reader = DirectoryReader.open(_dir);
  AtomicReader atomicReader = reader.leaves().get(0).reader();
  SortedDocValues sortedDocValues = atomicReader.getSortedDocValues("fam.geo");
  BytesRef result = new BytesRef();
  sortedDocValues.get(0, result);
  assertEquals(s, result.utf8ToString());
  System.out.println(result.utf8ToString());
  reader.close();
}
 
Example 16
Source Project: linden   Source File: LindenUtil.java    License: Apache License 2.0 5 votes vote down vote up
private static boolean actualContain(AtomicReader reader, String field, int locDocId) {
  try {
    // index really contains such field of this doc
    return FieldCache.DEFAULT.getDocsWithField(reader, field).get(locDocId);
  } catch (IOException e) {
    return false;
  }
}
 
Example 17
/**
 * Creates a corpus adapter which uses the Lucene index with the given path
 * and searches on the field with the given field name.
 * 
 * @param indexPath
 * @param fieldName
 * @return
 * @throws CorruptIndexException
 * @throws IOException
 */
public static LuceneCorpusAdapter create(String indexPath, String fieldName)
        throws CorruptIndexException, IOException {
    DirectoryReader dirReader = DirectoryReader.open(new NIOFSDirectory(new File(indexPath)));
    List<AtomicReaderContext> leaves = dirReader.leaves();
    AtomicReader reader[] = new AtomicReader[leaves.size()];
    AtomicReaderContext contexts[] = new AtomicReaderContext[leaves.size()];
    for (int i = 0; i < reader.length; i++) {
        contexts[i] = leaves.get(i);
        reader[i] = contexts[i].reader();
    }
    return new LuceneCorpusAdapter(dirReader, reader, contexts, fieldName);
}
 
Example 18
protected LuceneCorpusAdapter(DirectoryReader dirReader, AtomicReader reader[], AtomicReaderContext contexts[],
        String fieldName) {
    this.dirReader = dirReader;
    this.reader = reader;
    this.contexts = contexts;
    this.fieldName = fieldName;
}
 
Example 19
private long getTotalNumberOfRowIds(DirectoryReader reader) throws IOException {
  long total = 0;
  List<AtomicReaderContext> leaves = reader.leaves();
  for (AtomicReaderContext context : leaves) {
    AtomicReader atomicReader = context.reader();
    Terms terms = atomicReader.terms(BlurConstants.ROW_ID);
    long expectedInsertions = terms.size();
    if (expectedInsertions < 0) {
      return -1;
    }
    total += expectedInsertions;
  }
  return total;
}
 
Example 20
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException {
  AtomicReader reader = context.reader();
  List<DocIdSet> list = new ArrayList<DocIdSet>();

  Fields fields = reader.fields();
  Terms terms = fields.terms(_fieldName);
  if (terms == null) {
    // if field is not present then show nothing.
    return DocIdSet.EMPTY_DOCIDSET;
  }
  TermsEnum iterator = terms.iterator(null);
  BytesRef bytesRef;
  DocumentVisibilityEvaluator visibilityEvaluator = new DocumentVisibilityEvaluator(_authorizations);
  while ((bytesRef = iterator.next()) != null) {
    if (isVisible(visibilityEvaluator, bytesRef)) {
      DocIdSet docIdSet = _filterCacheStrategy.getDocIdSet(_fieldName, bytesRef, reader);
      if (docIdSet != null) {
        list.add(docIdSet);
      } else {
        // Do not use acceptDocs because we want the acl cache to be version
        // agnostic.
        DocsEnum docsEnum = iterator.docs(null, null);
        list.add(buildCache(reader, docsEnum, bytesRef));
      }
    }
  }
  return getLogicalOr(list);
}
 
Example 21
public SecureDirectoryReader(DirectoryReader in, final AccessControlReader accessControlReader) {
  super(in, new SubReaderWrapper() {

    @Override
    public AtomicReader wrap(AtomicReader reader) {
      try {
        return new SecureAtomicReader(reader, accessControlReader);
      } catch (IOException e) {
        throw new RuntimeException(e);
      }
    }
  });
  _original = in;
}
 
Example 22
public MergeSortRowIdLookup(IndexReader indexReader) throws IOException {
  if (indexReader instanceof AtomicReader) {
    addAtomicReader((AtomicReader) indexReader);
  } else {
    for (AtomicReaderContext context : indexReader.leaves()) {
      addAtomicReader(context.reader());
    }
  }
}
 
Example 23
private void checkTerms(IndexSearcher searcher, String fieldName) throws IOException {
  IndexReader reader = searcher.getIndexReader();
  for (AtomicReaderContext context : reader.leaves()) {
    AtomicReader atomicReader = context.reader();
    Fields fields = atomicReader.fields();
    Terms terms = fields.terms(fieldName);
    TermsEnum iterator = terms.iterator(null);
    BytesRef bytesRef = iterator.next();
    if (bytesRef != null) {
      System.out.println(bytesRef.utf8ToString());
      fail("There are only restricted terms for this field [" + fieldName + "]");
    }
  }
}
 
Example 24
Source Project: incubator-retired-blur   Source File: BlurUtil.java    License: Apache License 2.0 5 votes vote down vote up
private static void applyColumns(Set<String> alreadyProcessed, OpenBitSet bits,
    Map<String, Set<String>> columnsToFetch, AtomicReader atomicReader, int primeDocRowId, int numberOfDocsInRow,
    Bits liveDocs) throws IOException {
  for (String family : columnsToFetch.keySet()) {
    if (!alreadyProcessed.contains(family)) {
      applyFamily(bits, family, atomicReader, primeDocRowId, numberOfDocsInRow, liveDocs);
      alreadyProcessed.add(family);
    }
  }
}
 
Example 25
Source Project: incubator-retired-blur   Source File: BlurUtil.java    License: Apache License 2.0 5 votes vote down vote up
private static void applyFamilies(Set<String> alreadyProcessed, OpenBitSet bits, Set<String> columnFamiliesToFetch,
    AtomicReader segmentReader, int primeDocRowId, int numberOfDocsInRow, Bits liveDocs) throws IOException {
  for (String family : columnFamiliesToFetch) {
    if (!alreadyProcessed.contains(family)) {
      applyFamily(bits, family, segmentReader, primeDocRowId, numberOfDocsInRow, liveDocs);
      alreadyProcessed.add(family);
    }
  }
}
 
Example 26
Source Project: incubator-retired-blur   Source File: BlurUtil.java    License: Apache License 2.0 5 votes vote down vote up
private static void applyFamily(OpenBitSet bits, String family, AtomicReader atomicReader, int primeDocRowId,
    int numberOfDocsInRow, Bits liveDocs) throws IOException {
  Fields fields = atomicReader.fields();
  Terms terms = fields.terms(BlurConstants.FAMILY);
  TermsEnum iterator = terms.iterator(null);
  BytesRef text = new BytesRef(family);
  int lastDocId = primeDocRowId + numberOfDocsInRow;
  if (iterator.seekExact(text, true)) {
    DocsEnum docs = iterator.docs(liveDocs, null, DocsEnum.FLAG_NONE);
    int doc = primeDocRowId;
    while ((doc = docs.advance(doc)) < lastDocId) {
      bits.set(doc - primeDocRowId);
    }
  }
}
 
Example 27
Source Project: incubator-retired-blur   Source File: TermDocIterable.java    License: Apache License 2.0 5 votes vote down vote up
public TermDocIterable(DocsEnum docsEnum, AtomicReader reader, ResetableDocumentStoredFieldVisitor fieldSelector) {
  if (docsEnum == null) {
    throw new NullPointerException("docsEnum can not be null.");
  }
  this.docsEnum = docsEnum;
  this.reader = reader;
  this.fieldSelector = fieldSelector;
}
 
Example 28
private AtomicReader createIndexReader() throws IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)));
  for (int i = 0; i < BLOCKS; i++) {
    addDocumentBlock(i, COUNT_PER_BLOCK, writer);
  }
  writer.close();
  return SlowCompositeReaderWrapper.wrap(DirectoryReader.open(directory));
}
 
Example 29
@Override
public Boolean execute(IndexContext context) throws IOException, InterruptedException {
  try {
    IndexReader indexReader = context.getIndexReader();
    while (true) {
      long hash = 0;
      for (AtomicReaderContext atomicReaderContext : indexReader.leaves()) {
        AtomicReader reader = atomicReaderContext.reader();
        for (String field : reader.fields()) {
          Terms terms = reader.terms(field);
          BytesRef bytesRef;
          TermsEnum iterator = terms.iterator(null);
          while ((bytesRef = iterator.next()) != null) {
            hash += bytesRef.hashCode();
          }
        }
      }
      System.out.println("hashcode = " + hash);
    }
  } catch (IOException e) {
    e.printStackTrace();
    throw e;
  } catch (Throwable t) {
    t.printStackTrace();
    if (t instanceof InterruptedException) {
      throw t;
    } else if (t instanceof RuntimeException) {
      throw (RuntimeException) t;
    }
    throw new RuntimeException(t);
  }
}
 
Example 30
private AtomicReader getAtomicReader(DirectoryReader reader) throws IOException {
  List<AtomicReaderContext> leaves = reader.leaves();
  if (leaves.size() == 1) {
    return leaves.get(0).reader();
  }
  throw new IOException("Reader [" + reader + "] has more than one segment after optimize.");
}