Java Code Examples for org.apache.lucene.index.IndexReader#CacheHelper

The following examples show how to use org.apache.lucene.index.IndexReader#CacheHelper . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: lucene-solr   File: LRUQueryCache.java    License: Apache License 2.0 6 votes vote down vote up
DocIdSet get(Query key, IndexReader.CacheHelper cacheHelper) {
  assert lock.isHeldByCurrentThread();
  assert key instanceof BoostQuery == false;
  assert key instanceof ConstantScoreQuery == false;
  final IndexReader.CacheKey readerKey = cacheHelper.getKey();
  final LeafCache leafCache = cache.get(readerKey);
  if (leafCache == null) {
    onMiss(readerKey, key);
    return null;
  }
  // this get call moves the query to the most-recently-used position
  final Query singleton = uniqueQueries.get(key);
  if (singleton == null) {
    onMiss(readerKey, key);
    return null;
  }
  final DocIdSet cached = leafCache.get(singleton);
  if (cached == null) {
    onMiss(readerKey, singleton);
  } else {
    onHit(readerKey, singleton);
  }
  return cached;
}
 
Example 2
Source Project: lucene-solr   File: FieldCacheImpl.java    License: Apache License 2.0 6 votes vote down vote up
/** Sets the key to the value for the provided reader;
 *  if the key is already set then this doesn't change it. */
public void put(LeafReader reader, CacheKey key, Accountable value) {
  IndexReader.CacheHelper cacheHelper = reader.getCoreCacheHelper();
  if (cacheHelper == null) {
    throw new IllegalStateException("Cannot cache on " + reader);
  }
  final IndexReader.CacheKey readerKey = cacheHelper.getKey();
  synchronized (readerCache) {
    Map<CacheKey,Accountable> innerCache = readerCache.get(readerKey);
    if (innerCache == null) {
      // First time this reader is using FieldCache
      innerCache = new HashMap<>();
      readerCache.put(readerKey, innerCache);
      wrapper.initReader(reader);
    }
    if (innerCache.get(key) == null) {
      innerCache.put(key, value);
    } else {
      // Another thread beat us to it; leave the current
      // value
    }
  }
}
 
Example 3
private void addReaderClosedListener(IndexReader r) {
  IndexReader.CacheHelper cacheHelper = r.getReaderCacheHelper();
  if (cacheHelper == null) {
    throw new IllegalStateException("StandardDirectoryReader must support caching");
  }
  openReaderCount.incrementAndGet();
  cacheHelper.addClosedListener(new IndexReader.ClosedListener() {
      @Override
      public void onClose(IndexReader.CacheKey cacheKey) {
        onReaderClosed();
      }
    });
}
 
Example 4
private synchronized CachedOrds getCachedOrds(LeafReaderContext context) throws IOException {
  IndexReader.CacheHelper cacheHelper = context.reader().getCoreCacheHelper();
  if (cacheHelper == null) {
    throw new IllegalStateException("Cannot cache ordinals on leaf: " + context.reader());
  }
  Object cacheKey = cacheHelper.getKey();
  CachedOrds ords = ordsCache.get(cacheKey);
  if (ords == null) {
    ords = new CachedOrds(source.getReader(context), context.reader().maxDoc());
    ordsCache.put(cacheKey, ords);
  }

  return ords;
}
 
Example 5
Source Project: lucene-solr   File: LRUQueryCache.java    License: Apache License 2.0 5 votes vote down vote up
private void putIfAbsent(Query query, DocIdSet set, IndexReader.CacheHelper cacheHelper) {
  assert query instanceof BoostQuery == false;
  assert query instanceof ConstantScoreQuery == false;
  // under a lock to make sure that mostRecentlyUsedQueries and cache remain sync'ed
  lock.lock();
  try {
    Query singleton = uniqueQueries.putIfAbsent(query, query);
    if (singleton == null) {
      onQueryCache(query, LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY + QUERY_DEFAULT_RAM_BYTES_USED);
    } else {
      query = singleton;
    }
    final IndexReader.CacheKey key = cacheHelper.getKey();
    LeafCache leafCache = cache.get(key);
    if (leafCache == null) {
      leafCache = new LeafCache(key);
      final LeafCache previous = cache.put(key, leafCache);
      ramBytesUsed += HASHTABLE_RAM_BYTES_PER_ENTRY;
      assert previous == null;
      // we just created a new leaf cache, need to register a close listener
      cacheHelper.addClosedListener(this::clearCoreCacheKey);
    }
    leafCache.putIfAbsent(query, set);
    evictIfNecessary();
  } finally {
    lock.unlock();
  }
}
 
Example 6
Source Project: lucene-solr   File: FieldCacheImpl.java    License: Apache License 2.0 5 votes vote down vote up
private void initReader(LeafReader reader) {
  IndexReader.CacheHelper cacheHelper = reader.getCoreCacheHelper();
  if (cacheHelper == null) {
    throw new IllegalStateException("Cannot cache on " + reader);
  }
  cacheHelper.addClosedListener(purgeCore);
}
 
Example 7
Source Project: lucene-solr   File: FieldCacheImpl.java    License: Apache License 2.0 5 votes vote down vote up
public Object get(LeafReader reader, CacheKey key) throws IOException {
  Map<CacheKey,Accountable> innerCache;
  Accountable value;
  IndexReader.CacheHelper cacheHelper = reader.getCoreCacheHelper();
  if (cacheHelper == null) {
    reader.getCoreCacheHelper();
    throw new IllegalStateException("Cannot cache on " + reader);
  }
  final IndexReader.CacheKey readerKey = cacheHelper.getKey();
  synchronized (readerCache) {
    innerCache = readerCache.get(readerKey);
    if (innerCache == null) {
      // First time this reader is using FieldCache
      innerCache = new HashMap<>();
      readerCache.put(readerKey, innerCache);
      wrapper.initReader(reader);
      value = null;
    } else {
      value = innerCache.get(key);
    }
    if (value == null) {
      value = new CreationPlaceholder();
      innerCache.put(key, value);
    }
  }
  if (value instanceof CreationPlaceholder) {
    synchronized (value) {
      CreationPlaceholder progress = (CreationPlaceholder) value;
      if (progress.value == null) {
        progress.value = createValue(reader, key);
        synchronized (readerCache) {
          innerCache.put(key, progress.value);
        }
      }
      return progress.value;
    }
  }
  return value;
}
 
Example 8
Source Project: lucene-solr   File: SolrCore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Computes fingerprint of a segment and caches it only if all the version in segment are included in the fingerprint.
 * We can't use computeIfAbsent as caching is conditional (as described above)
 * There is chance that two threads may compute fingerprint on the same segment. It might be OK to do so rather than locking entire map.
 *
 * @param searcher   searcher that includes specified LeaderReaderContext
 * @param ctx        LeafReaderContext of a segment to compute fingerprint of
 * @param maxVersion maximum version number to consider for fingerprint computation
 * @return IndexFingerprint of the segment
 * @throws IOException Can throw IOException
 */
public IndexFingerprint getIndexFingerprint(SolrIndexSearcher searcher, LeafReaderContext ctx, long maxVersion)
    throws IOException {
  IndexReader.CacheHelper cacheHelper = ctx.reader().getReaderCacheHelper();
  if (cacheHelper == null) {
    if (log.isDebugEnabled()) {
      log.debug("Cannot cache IndexFingerprint as reader does not support caching. searcher:{} reader:{} readerHash:{} maxVersion:{}", searcher, ctx.reader(), ctx.reader().hashCode(), maxVersion);
    }
    return IndexFingerprint.getFingerprint(searcher, ctx, maxVersion);
  }

  IndexFingerprint f = null;
  f = perSegmentFingerprintCache.get(cacheHelper.getKey());
  // fingerprint is either not cached or
  // if we want fingerprint only up to a version less than maxVersionEncountered in the segment, or
  // documents were deleted from segment for which fingerprint was cached
  //
  if (f == null || (f.getMaxInHash() > maxVersion) || (f.getNumDocs() != ctx.reader().numDocs())) {
    if (log.isDebugEnabled()) {
      log.debug("IndexFingerprint cache miss for searcher:{} reader:{} readerHash:{} maxVersion:{}", searcher, ctx.reader(), ctx.reader().hashCode(), maxVersion);
    }
    f = IndexFingerprint.getFingerprint(searcher, ctx, maxVersion);
    // cache fingerprint for the segment only if all the versions in the segment are included in the fingerprint
    if (f.getMaxVersionEncountered() == f.getMaxInHash()) {
      log.debug("Caching fingerprint for searcher:{} leafReaderContext:{} mavVersion:{}", searcher, ctx, maxVersion);
      perSegmentFingerprintCache.put(cacheHelper.getKey(), f);
    }

  } else {
    if (log.isDebugEnabled()) {
      log.debug("IndexFingerprint cache hit for searcher:{} reader:{} readerHash:{} maxVersion:{}", searcher, ctx.reader(), ctx.reader().hashCode(), maxVersion);
    }
  }
  if (log.isDebugEnabled()) {
    log.debug("Cache Size: {}, Segments Size:{}", perSegmentFingerprintCache.size(), searcher.getTopReaderContext().leaves().size());
  }
  return f;
}
 
Example 9
@Override
public ShapeValues getValues(LeafReaderContext readerContext) throws IOException {
  final ShapeValues targetFuncValues = targetValueSource.getValues(readerContext);
  // The key is a pair of leaf reader with a docId relative to that reader. The value is a Map from field to Shape.
  @SuppressWarnings({"unchecked"})
  final SolrCache<PerSegCacheKey,Shape> cache =
      SolrRequestInfo.getRequestInfo().getReq().getSearcher().getCache(CACHE_KEY_PREFIX + fieldName);
  if (cache == null) {
    return targetFuncValues; // no caching; no configured cache
  }

  return new ShapeValues() {
    int docId = -1;

    @Override
    public Shape value() throws IOException {
      //lookup in cache
      IndexReader.CacheHelper cacheHelper = readerContext.reader().getCoreCacheHelper();
      if (cacheHelper == null) {
        throw new IllegalStateException("Leaf " + readerContext.reader() + " is not suited for caching");
      }
      PerSegCacheKey key = new PerSegCacheKey(cacheHelper.getKey(), docId);
      Shape shape = cache.computeIfAbsent(key, k -> {
        try {
          return targetFuncValues.value();
        } catch (IOException e) {
          return null;
        }
      });
      if (shape != null) {
        //optimize shape on a cache hit if possible. This must be thread-safe and it is.
        if (shape instanceof JtsGeometry) {
          ((JtsGeometry) shape).index(); // TODO would be nice if some day we didn't have to cast
        }
      }
      return shape;
    }

    @Override
    public boolean advanceExact(int doc) throws IOException {
      this.docId = doc;
      return targetFuncValues.advanceExact(doc);
    }

  };

}
 
Example 10
Source Project: crate   File: VersionsAndSeqNoResolver.java    License: Apache License 2.0 4 votes vote down vote up
private static PerThreadIDVersionAndSeqNoLookup[] getLookupState(IndexReader reader, String uidField) throws IOException {
    // We cache on the top level
    // This means cache entries have a shorter lifetime, maybe as low as 1s with the
    // default refresh interval and a steady indexing rate, but on the other hand it
    // proved to be cheaper than having to perform a CHM and a TL get for every segment.
    // See https://github.com/elastic/elasticsearch/pull/19856.
    IndexReader.CacheHelper cacheHelper = reader.getReaderCacheHelper();
    CloseableThreadLocal<PerThreadIDVersionAndSeqNoLookup[]> ctl = LOOKUP_STATES.get(cacheHelper.getKey());
    if (ctl == null) {
        // First time we are seeing this reader's core; make a new CTL:
        ctl = new CloseableThreadLocal<>();
        CloseableThreadLocal<PerThreadIDVersionAndSeqNoLookup[]> other = LOOKUP_STATES.putIfAbsent(cacheHelper.getKey(), ctl);
        if (other == null) {
            // Our CTL won, we must remove it when the reader is closed:
            cacheHelper.addClosedListener(REMOVE_LOOKUP_STATE);
        } else {
            // Another thread beat us to it: just use their CTL:
            ctl = other;
        }
    }

    PerThreadIDVersionAndSeqNoLookup[] lookupState = ctl.get();
    if (lookupState == null) {
        lookupState = new PerThreadIDVersionAndSeqNoLookup[reader.leaves().size()];
        for (LeafReaderContext leaf : reader.leaves()) {
            lookupState[leaf.ord] = new PerThreadIDVersionAndSeqNoLookup(leaf.reader(), uidField);
        }
        ctl.set(lookupState);
    }

    if (lookupState.length != reader.leaves().size()) {
        throw new AssertionError("Mismatched numbers of leaves: " + lookupState.length + " != " + reader.leaves().size());
    }

    if (lookupState.length > 0 && Objects.equals(lookupState[0].uidField, uidField) == false) {
        throw new AssertionError("Index does not consistently use the same uid field: ["
                + uidField + "] != [" + lookupState[0].uidField + "]");
    }

    return lookupState;
}