org.apache.lucene.util.Accountable Java Examples

The following examples show how to use org.apache.lucene.util.Accountable. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AbstractAtomicGeoPointFieldData.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
public static AtomicGeoPointFieldData empty(final int maxDoc) {
    return new AbstractAtomicGeoPointFieldData() {

        @Override
        public long ramBytesUsed() {
            return 0;
        }
        
        @Override
        public Collection<Accountable> getChildResources() {
            return Collections.emptyList();
        }

        @Override
        public void close() {
        }

        @Override
        public MultiGeoPointValues getGeoPointValues() {
            return FieldData.emptyMultiGeoPoints(maxDoc);
        }
    };
}
 
Example #2
Source File: LindenFieldCacheImpl.java    From linden with Apache License 2.0 6 votes vote down vote up
/**
 * Sets the key to the value for the provided reader;
 * if the key is already set then this doesn't change it.
 */
public void put(AtomicReader reader, CacheKey key, Accountable value) {
  final Object readerKey = reader.getCoreCacheKey();
  synchronized (readerCache) {
    Map<CacheKey, Accountable> innerCache = readerCache.get(readerKey);
    if (innerCache == null) {
      // First time this reader is using FieldCache
      innerCache = new HashMap<>();
      readerCache.put(readerKey, innerCache);
      wrapper.initReader(reader);
    }
    if (innerCache.get(key) == null) {
      innerCache.put(key, value);
    } else {
      // Another thread beat us to it; leave the current
      // value
    }
  }
}
 
Example #3
Source File: FieldCacheImpl.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** Sets the key to the value for the provided reader;
 *  if the key is already set then this doesn't change it. */
public void put(LeafReader reader, CacheKey key, Accountable value) {
  IndexReader.CacheHelper cacheHelper = reader.getCoreCacheHelper();
  if (cacheHelper == null) {
    throw new IllegalStateException("Cannot cache on " + reader);
  }
  final IndexReader.CacheKey readerKey = cacheHelper.getKey();
  synchronized (readerCache) {
    Map<CacheKey,Accountable> innerCache = readerCache.get(readerKey);
    if (innerCache == null) {
      // First time this reader is using FieldCache
      innerCache = new HashMap<>();
      readerCache.put(readerKey, innerCache);
      wrapper.initReader(reader);
    }
    if (innerCache.get(key) == null) {
      innerCache.put(key, value);
    } else {
      // Another thread beat us to it; leave the current
      // value
    }
  }
}
 
Example #4
Source File: DirectoryTaxonomyReader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized Collection<Accountable> getChildResources() {
  final List<Accountable> resources = new ArrayList<>();
  long ramBytesUsed = 0;
  for (LeafReaderContext ctx : indexReader.leaves()) {
    ramBytesUsed += ((SegmentReader) ctx.reader()).ramBytesUsed();
  }
  resources.add(Accountables.namedAccountable("indexReader", ramBytesUsed));
  if (taxoArrays != null) {
    resources.add(Accountables.namedAccountable("taxoArrays", taxoArrays));
  }

  synchronized (categoryCache) {
    resources.add(Accountables.namedAccountable("categoryCache", BYTES_PER_CACHE_ENTRY * categoryCache.size()));
  }    

  synchronized (ordinalCache) {
    resources.add(Accountables.namedAccountable("ordinalCache", BYTES_PER_CACHE_ENTRY * ordinalCache.size()));
  }    
  
  return Collections.unmodifiableList(resources);
}
 
Example #5
Source File: FSTCompletionLookup.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  List<Accountable> resources = new ArrayList<>();
  if (normalCompletion != null) {
    resources.add(Accountables.namedAccountable("fst", normalCompletion.getFST()));
  }
  if (higherWeightsCompletion != null && (normalCompletion == null || normalCompletion.getFST() != higherWeightsCompletion.getFST())) {
    resources.add(Accountables.namedAccountable("higher weights fst", higherWeightsCompletion.getFST()));
  }
  return Collections.unmodifiableList(resources);
}
 
Example #6
Source File: LRUQueryCache.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  lock.lock();
  try {
    return Accountables.namedAccountables("segment", cache);
  } finally {
    lock.unlock();
  }
}
 
Example #7
Source File: FieldCacheImpl.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  List<Accountable> resources = new ArrayList<>(3);
  resources.add(Accountables.namedAccountable("term bytes", bytes));
  resources.add(Accountables.namedAccountable("ord -> term", termOrdToBytesOffset));
  resources.add(Accountables.namedAccountable("doc -> ord", docToTermOrd));
  return Collections.unmodifiableList(resources);
}
 
Example #8
Source File: IndicesFieldDataCache.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Inject
public IndicesFieldDataCache(Settings settings, IndicesFieldDataCacheListener indicesFieldDataCacheListener, ThreadPool threadPool) {
    super(settings);
    this.threadPool = threadPool;
    this.indicesFieldDataCacheListener = indicesFieldDataCacheListener;
    final String size = settings.get(INDICES_FIELDDATA_CACHE_SIZE_KEY, "-1");
    final long sizeInBytes = settings.getAsMemory(INDICES_FIELDDATA_CACHE_SIZE_KEY, "-1").bytes();
    CacheBuilder<Key, Accountable> cacheBuilder = CacheBuilder.newBuilder()
            .removalListener(this);
    if (sizeInBytes > 0) {
        cacheBuilder.maximumWeight(sizeInBytes).weigher(new FieldDataWeigher());
    }
    // defaults to 4, but this is a busy map for all indices, increase it a bit by default
    final int concurrencyLevel =  settings.getAsInt(FIELDDATA_CACHE_CONCURRENCY_LEVEL, 16);
    if (concurrencyLevel <= 0) {
        throw new IllegalArgumentException("concurrency_level must be > 0 but was: " + concurrencyLevel);
    }
    cacheBuilder.concurrencyLevel(concurrencyLevel);

    logger.debug("using size [{}] [{}]", size, new ByteSizeValue(sizeInBytes));
    cache = cacheBuilder.build();

    this.cleanInterval = settings.getAsTime(FIELDDATA_CLEAN_INTERVAL_SETTING, TimeValue.timeValueMinutes(1));
    // Start thread that will manage cleaning the field data cache periodically
    threadPool.schedule(this.cleanInterval, ThreadPool.Names.SAME,
            new FieldDataCacheCleaner(this.cache, this.logger, this.threadPool, this.cleanInterval));
}
 
Example #9
Source File: AnalyzingCompletionLookupProvider.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
    if (fst != null) {
        return Collections.singleton(Accountables.namedAccountable("fst", fst));
    } else {
        return Collections.emptyList();
    }
}
 
Example #10
Source File: IndicesFieldDataCache.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public void onRemoval(RemovalNotification<Key, Accountable> notification) {
    Key key = notification.getKey();
    assert key != null && key.listeners != null;
    IndexFieldCache indexCache = key.indexCache;
    final Accountable value = notification.getValue();
    for (IndexFieldDataCache.Listener listener : key.listeners) {
        try {
            listener.onRemoval(key.shardId, indexCache.fieldNames, indexCache.fieldDataType, notification.wasEvicted(), value.ramBytesUsed());
        } catch (Throwable e) {
            // load anyway since listeners should not throw exceptions
            logger.error("Failed to call listener on field data cache unloading", e);
        }
    }
}
 
Example #11
Source File: GeoPointArrayAtomicFieldData.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
    List<Accountable> resources = new ArrayList<>();
    resources.add(Accountables.namedAccountable("indexedPoints", indexedPoint));
    if (set != null) {
        resources.add(Accountables.namedAccountable("missing bitset", set));
    }
    return Collections.unmodifiableList(resources);
}
 
Example #12
Source File: OrdinalMap.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  List<Accountable> resources = new ArrayList<>();
  resources.add(Accountables.namedAccountable("global ord deltas", globalOrdDeltas));
  resources.add(Accountables.namedAccountable("first segments", firstSegments));
  resources.add(Accountables.namedAccountable("segment map", segmentMap));
  // TODO: would be nice to return actual child segment deltas too, but the optimizations are confusing
  return resources;
}
 
Example #13
Source File: GeoPointArrayLegacyAtomicFieldData.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
    List<Accountable> resources = new ArrayList<>();
    resources.add(Accountables.namedAccountable("latitude", lat));
    resources.add(Accountables.namedAccountable("longitude", lon));
    return Collections.unmodifiableList(resources);
}
 
Example #14
Source File: FieldCache.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public CacheEntry(IndexReader.CacheKey readerKey, String fieldName,
                  Class<?> cacheType,
                  Object custom,
                  Accountable value) {
  this.readerKey = readerKey;
  this.fieldName = fieldName;
  this.cacheType = cacheType;
  this.custom = custom;
  this.value = value;
}
 
Example #15
Source File: DefaultSortedSetDocValuesReaderState.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Returns nested resources of this class. 
 * The result should be a point-in-time snapshot (to avoid race conditions).
 * @see Accountables
 */
@Override
public Collection<Accountable> getChildResources() {
  synchronized (cachedOrdMaps) {
    return Accountables.namedAccountables("DefaultSortedSetDocValuesReaderState", cachedOrdMaps);
  }
}
 
Example #16
Source File: Segment.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
Accountable readRamTree(StreamInput in) throws IOException {
    final String name = in.readString();
    final long bytes = in.readVLong();
    int numChildren = in.readVInt();
    if (numChildren == 0) {
        return Accountables.namedAccountable(name, bytes);
    }
    List<Accountable> children = new ArrayList(numChildren);
    while (numChildren-- > 0) {
        children.add(readRamTree(in));
    }
    return Accountables.namedAccountable(name, children, bytes);
}
 
Example #17
Source File: BloomFilteringPostingsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  List<Accountable> resources = new ArrayList<>(Accountables.namedAccountables("field", bloomsByFieldName));
  if (delegateFieldsProducer != null) {
    resources.add(Accountables.namedAccountable("delegate", delegateFieldsProducer));
  }
  return Collections.unmodifiableList(resources);
}
 
Example #18
Source File: SegmentDocValuesProducer.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  final List<Accountable> resources = new ArrayList<>(dvProducers.size());
  for (Accountable producer : dvProducers) {
    resources.add(Accountables.namedAccountable("delegate", producer));
  }
  return Collections.unmodifiableList(resources);
}
 
Example #19
Source File: Engine.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
/** Returns 0 in the case where accountable is null, otherwise returns {@code ramBytesUsed()} */
protected static long guardedRamBytesUsed(Accountable a) {
    if (a == null) {
        return 0;
    }
    return a.ramBytesUsed();
}
 
Example #20
Source File: AnalyzingInfixSuggester.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  List<Accountable> resources = new ArrayList<>();
  try {
    if (searcherMgr != null) {
      SearcherManager mgr;
      IndexSearcher searcher;
      synchronized (searcherMgrLock) {
        mgr = searcherMgr; // acquire & release on same SearcherManager, via local reference
        searcher = mgr.acquire();
      }
      try {
        for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
          LeafReader reader = FilterLeafReader.unwrap(context.reader());
          if (reader instanceof SegmentReader) {
            resources.add(Accountables.namedAccountable("segment", (SegmentReader)reader));
          }
        }
      } finally {
        mgr.release(searcher);
      }
    }
    return Collections.unmodifiableList(resources);
  } catch (IOException ioe) {
    throw new RuntimeException(ioe);
  }
}
 
Example #21
Source File: IndexService.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public void onRemoval(ShardId shardId, Accountable accountable) {
    if (shardId != null) {
        final IndexShard shard = indexService.shard(shardId.id());
        if (shard != null) {
            long ramBytesUsed = accountable != null ? accountable.ramBytesUsed() : 0l;
            shard.shardBitsetFilterCache().onRemoval(ramBytesUsed);
        }
    }
}
 
Example #22
Source File: IndicesSegmentResponse.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
static void toXContent(XContentBuilder builder, Accountable tree) throws IOException {
    builder.startObject();
    builder.field(Fields.DESCRIPTION, tree.toString());
    builder.byteSizeField(Fields.SIZE_IN_BYTES, Fields.SIZE, new ByteSizeValue(tree.ramBytesUsed()));
    Collection<Accountable> children = tree.getChildResources();
    if (children.isEmpty() == false) {
        builder.startArray(Fields.CHILDREN);
        for (Accountable child : children) {
            toXContent(builder, child);
        }
        builder.endArray();
    }
    builder.endObject();
}
 
Example #23
Source File: SimpleTextFieldsReader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  if (fst == null) {
    return Collections.emptyList();
  } else {
    return Collections.singletonList(Accountables.namedAccountable("term cache", fst));
  }
}
 
Example #24
Source File: LindenFieldCacheImpl.java    From linden with Apache License 2.0 5 votes vote down vote up
@Override
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
    throws IOException {
  int maxDoc = reader.maxDoc();

  final long[][] matrix = new long[maxDoc][];
  BinaryDocValues valuesIn = reader.getBinaryDocValues(key.field);
  if (valuesIn == null) {
    for (int i = 0; i < maxDoc; ++i) {
      matrix[i] = new long[0];
    }
    return new LongList(matrix);
  }
  for (int i = 0; i < maxDoc; ++i) {
    String str = valuesIn.get(i).utf8ToString();
    if (StringUtils.isEmpty(str)) {
      matrix[i] = new long[0];
      continue;
    }
    JSONArray array = JSON.parseArray(str);
    matrix[i] = new long[array.size()];
    for (int j = 0; j < array.size(); ++j) {
      matrix[i][j] = array.getInteger(j);
    }
  }
  return new LongList(matrix);
}
 
Example #25
Source File: TaxonomyIndexArrays.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized Collection<Accountable> getChildResources() {
  final List<Accountable> resources = new ArrayList<>();
  resources.add(Accountables.namedAccountable("parents", RamUsageEstimator.shallowSizeOf(parents)));
  if (children != null) {
    resources.add(Accountables.namedAccountable("children", RamUsageEstimator.shallowSizeOf(children)));
  }
  if (siblings != null) {
    resources.add(Accountables.namedAccountable("siblings", RamUsageEstimator.shallowSizeOf(siblings)));
  }
  return Collections.unmodifiableList(resources);
}
 
Example #26
Source File: LindenFieldCacheImpl.java    From linden with Apache License 2.0 5 votes vote down vote up
@Override
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
    throws IOException {
  int maxDoc = reader.maxDoc();

  final double[][] matrix = new double[maxDoc][];
  BinaryDocValues valuesIn = reader.getBinaryDocValues(key.field);
  if (valuesIn == null) {
    for (int i = 0; i < maxDoc; ++i) {
      matrix[i] = new double[0];
    }
    return new DoubleList(matrix);
  }
  for (int i = 0; i < maxDoc; ++i) {
    String str = valuesIn.get(i).utf8ToString();
    if (StringUtils.isEmpty(str)) {
      matrix[i] = new double[0];
      continue;
    }
    JSONArray array = JSON.parseArray(str);
    matrix[i] = new double[array.size()];
    for (int j = 0; j < array.size(); ++j) {
      matrix[i][j] = array.getFloat(j);
    }
  }
  return new DoubleList(matrix);
}
 
Example #27
Source File: LindenFieldCacheImpl.java    From linden with Apache License 2.0 5 votes vote down vote up
@Override
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
    throws IOException {
  int maxDoc = reader.maxDoc();

  final String[][] matrix = new String[maxDoc][];
  BinaryDocValues valuesIn = reader.getBinaryDocValues(key.field);
  if (valuesIn == null) {
    for (int i = 0; i < maxDoc; ++i) {
      matrix[i] = new String[0];
    }
    return new StringList(matrix);
  }
  for (int i = 0; i < maxDoc; ++i) {
    String str = valuesIn.get(i).utf8ToString();
    if (StringUtils.isEmpty(str)) {
      matrix[i] = new String[0];
      continue;
    }
    JSONArray array = JSON.parseArray(str);
    matrix[i] = new String[array.size()];
    for (int j = 0; j < array.size(); ++j) {
      matrix[i][j] = array.getString(j);
    }
  }
  return new StringList(matrix);
}
 
Example #28
Source File: WFSTCompletionLookup.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  if (fst == null) {
    return Collections.emptyList();
  } else {
    return Collections.singleton(Accountables.namedAccountable("fst", fst));
  }
}
 
Example #29
Source File: BlockTermsReader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  List<Accountable> resources = new ArrayList<>();
  if (indexReader != null) {
    resources.add(Accountables.namedAccountable("term index", indexReader));
  }
  if (postingsReader != null) {
    resources.add(Accountables.namedAccountable("delegate", postingsReader));
  }
  return Collections.unmodifiableList(resources);
}
 
Example #30
Source File: CodecReader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  ensureOpen();
  final List<Accountable> resources = new ArrayList<>(6);
  
  // terms/postings
  resources.add(Accountables.namedAccountable("postings", getPostingsReader()));
  
  // norms
  if (getNormsReader() != null) {
    resources.add(Accountables.namedAccountable("norms", getNormsReader()));
  }
  
  // docvalues
  if (getDocValuesReader() != null) {
    resources.add(Accountables.namedAccountable("docvalues", getDocValuesReader()));
  }
  
  // stored fields
  if (getFieldsReader() != null) {
    resources.add(Accountables.namedAccountable("stored fields", getFieldsReader()));
  }

  // term vectors
  if (getTermVectorsReader() != null) {
    resources.add(Accountables.namedAccountable("term vectors", getTermVectorsReader()));
  }

  // points
  if (getPointsReader() != null) {
    resources.add(Accountables.namedAccountable("points", getPointsReader()));
  }
  
  return Collections.unmodifiableList(resources);
}