Java Code Examples for org.apache.lucene.util.Accountable

The following examples show how to use org.apache.lucene.util.Accountable. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
public static AtomicGeoPointFieldData empty(final int maxDoc) {
    return new AbstractAtomicGeoPointFieldData() {

        @Override
        public long ramBytesUsed() {
            return 0;
        }
        
        @Override
        public Collection<Accountable> getChildResources() {
            return Collections.emptyList();
        }

        @Override
        public void close() {
        }

        @Override
        public MultiGeoPointValues getGeoPointValues() {
            return FieldData.emptyMultiGeoPoints(maxDoc);
        }
    };
}
 
Example 2
Source Project: lucene-solr   Source File: DirectoryTaxonomyReader.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized Collection<Accountable> getChildResources() {
  final List<Accountable> resources = new ArrayList<>();
  long ramBytesUsed = 0;
  for (LeafReaderContext ctx : indexReader.leaves()) {
    ramBytesUsed += ((SegmentReader) ctx.reader()).ramBytesUsed();
  }
  resources.add(Accountables.namedAccountable("indexReader", ramBytesUsed));
  if (taxoArrays != null) {
    resources.add(Accountables.namedAccountable("taxoArrays", taxoArrays));
  }

  synchronized (categoryCache) {
    resources.add(Accountables.namedAccountable("categoryCache", BYTES_PER_CACHE_ENTRY * categoryCache.size()));
  }    

  synchronized (ordinalCache) {
    resources.add(Accountables.namedAccountable("ordinalCache", BYTES_PER_CACHE_ENTRY * ordinalCache.size()));
  }    
  
  return Collections.unmodifiableList(resources);
}
 
Example 3
Source Project: lucene-solr   Source File: FieldCacheImpl.java    License: Apache License 2.0 6 votes vote down vote up
/** Sets the key to the value for the provided reader;
 *  if the key is already set then this doesn't change it. */
public void put(LeafReader reader, CacheKey key, Accountable value) {
  IndexReader.CacheHelper cacheHelper = reader.getCoreCacheHelper();
  if (cacheHelper == null) {
    throw new IllegalStateException("Cannot cache on " + reader);
  }
  final IndexReader.CacheKey readerKey = cacheHelper.getKey();
  synchronized (readerCache) {
    Map<CacheKey,Accountable> innerCache = readerCache.get(readerKey);
    if (innerCache == null) {
      // First time this reader is using FieldCache
      innerCache = new HashMap<>();
      readerCache.put(readerKey, innerCache);
      wrapper.initReader(reader);
    }
    if (innerCache.get(key) == null) {
      innerCache.put(key, value);
    } else {
      // Another thread beat us to it; leave the current
      // value
    }
  }
}
 
Example 4
Source Project: linden   Source File: LindenFieldCacheImpl.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Sets the key to the value for the provided reader;
 * if the key is already set then this doesn't change it.
 */
public void put(AtomicReader reader, CacheKey key, Accountable value) {
  final Object readerKey = reader.getCoreCacheKey();
  synchronized (readerCache) {
    Map<CacheKey, Accountable> innerCache = readerCache.get(readerKey);
    if (innerCache == null) {
      // First time this reader is using FieldCache
      innerCache = new HashMap<>();
      readerCache.put(readerKey, innerCache);
      wrapper.initReader(reader);
    }
    if (innerCache.get(key) == null) {
      innerCache.put(key, value);
    } else {
      // Another thread beat us to it; leave the current
      // value
    }
  }
}
 
Example 5
Source Project: lucene-solr   Source File: SegmentDocValuesProducer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  final List<Accountable> resources = new ArrayList<>(dvProducers.size());
  for (Accountable producer : dvProducers) {
    resources.add(Accountables.namedAccountable("delegate", producer));
  }
  return Collections.unmodifiableList(resources);
}
 
Example 6
Source Project: Elasticsearch   Source File: IndicesFieldDataCache.java    License: Apache License 2.0 5 votes vote down vote up
@Inject
public IndicesFieldDataCache(Settings settings, IndicesFieldDataCacheListener indicesFieldDataCacheListener, ThreadPool threadPool) {
    super(settings);
    this.threadPool = threadPool;
    this.indicesFieldDataCacheListener = indicesFieldDataCacheListener;
    final String size = settings.get(INDICES_FIELDDATA_CACHE_SIZE_KEY, "-1");
    final long sizeInBytes = settings.getAsMemory(INDICES_FIELDDATA_CACHE_SIZE_KEY, "-1").bytes();
    CacheBuilder<Key, Accountable> cacheBuilder = CacheBuilder.newBuilder()
            .removalListener(this);
    if (sizeInBytes > 0) {
        cacheBuilder.maximumWeight(sizeInBytes).weigher(new FieldDataWeigher());
    }
    // defaults to 4, but this is a busy map for all indices, increase it a bit by default
    final int concurrencyLevel =  settings.getAsInt(FIELDDATA_CACHE_CONCURRENCY_LEVEL, 16);
    if (concurrencyLevel <= 0) {
        throw new IllegalArgumentException("concurrency_level must be > 0 but was: " + concurrencyLevel);
    }
    cacheBuilder.concurrencyLevel(concurrencyLevel);

    logger.debug("using size [{}] [{}]", size, new ByteSizeValue(sizeInBytes));
    cache = cacheBuilder.build();

    this.cleanInterval = settings.getAsTime(FIELDDATA_CLEAN_INTERVAL_SETTING, TimeValue.timeValueMinutes(1));
    // Start thread that will manage cleaning the field data cache periodically
    threadPool.schedule(this.cleanInterval, ThreadPool.Names.SAME,
            new FieldDataCacheCleaner(this.cache, this.logger, this.threadPool, this.cleanInterval));
}
 
Example 7
Source Project: Elasticsearch   Source File: IndicesFieldDataCache.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void onRemoval(RemovalNotification<Key, Accountable> notification) {
    Key key = notification.getKey();
    assert key != null && key.listeners != null;
    IndexFieldCache indexCache = key.indexCache;
    final Accountable value = notification.getValue();
    for (IndexFieldDataCache.Listener listener : key.listeners) {
        try {
            listener.onRemoval(key.shardId, indexCache.fieldNames, indexCache.fieldDataType, notification.wasEvicted(), value.ramBytesUsed());
        } catch (Throwable e) {
            // load anyway since listeners should not throw exceptions
            logger.error("Failed to call listener on field data cache unloading", e);
        }
    }
}
 
Example 8
Source Project: Elasticsearch   Source File: Completion090PostingsFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
    List<Accountable> resources = new ArrayList<>();
    if (lookupFactory != null) {
        resources.add(Accountables.namedAccountable("lookup", lookupFactory));
    }
    resources.add(Accountables.namedAccountable("delegate", delegateProducer));
    return Collections.unmodifiableList(resources);
}
 
Example 9
@Override
public Collection<Accountable> getChildResources() {
    if (fst != null) {
        return Collections.singleton(Accountables.namedAccountable("fst", fst));
    } else {
        return Collections.emptyList();
    }
}
 
Example 10
Source Project: Elasticsearch   Source File: MultiOrdinals.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
    List<Accountable> resources = new ArrayList<>();
    resources.add(Accountables.namedAccountable("offsets", endOffsets));
    resources.add(Accountables.namedAccountable("ordinals", ords));
    return Collections.unmodifiableCollection(resources);
}
 
Example 11
Source Project: lucene-solr   Source File: LRUQueryCache.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  lock.lock();
  try {
    return Accountables.namedAccountables("segment", cache);
  } finally {
    lock.unlock();
  }
}
 
Example 12
Source Project: Elasticsearch   Source File: AtomicDoubleFieldData.java    License: Apache License 2.0 5 votes vote down vote up
public static AtomicNumericFieldData empty(final int maxDoc) {
    return new AtomicDoubleFieldData(0) {

        @Override
        public SortedNumericDoubleValues getDoubleValues() {
            return FieldData.emptySortedNumericDoubles(maxDoc);
        }
        
        @Override
        public Collection<Accountable> getChildResources() {
            return Collections.emptyList();
        }

    };
}
 
Example 13
Source Project: lucene-solr   Source File: FieldCacheImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  List<Accountable> resources = new ArrayList<>(3);
  resources.add(Accountables.namedAccountable("term bytes", bytes));
  resources.add(Accountables.namedAccountable("ord -> term", termOrdToBytesOffset));
  resources.add(Accountables.namedAccountable("doc -> ord", docToTermOrd));
  return Collections.unmodifiableList(resources);
}
 
Example 14
@Override
public Collection<Accountable> getChildResources() {
    List<Accountable> resources = new ArrayList<>();
    resources.add(Accountables.namedAccountable("indexedPoints", indexedPoint));
    if (set != null) {
        resources.add(Accountables.namedAccountable("missing bitset", set));
    }
    return Collections.unmodifiableList(resources);
}
 
Example 15
Source Project: lucene-solr   Source File: OrdinalMap.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  List<Accountable> resources = new ArrayList<>();
  resources.add(Accountables.namedAccountable("global ord deltas", globalOrdDeltas));
  resources.add(Accountables.namedAccountable("first segments", firstSegments));
  resources.add(Accountables.namedAccountable("segment map", segmentMap));
  // TODO: would be nice to return actual child segment deltas too, but the optimizations are confusing
  return resources;
}
 
Example 16
@Override
public Collection<Accountable> getChildResources() {
    List<Accountable> resources = new ArrayList<>();
    resources.add(Accountables.namedAccountable("latitude", lat));
    resources.add(Accountables.namedAccountable("longitude", lon));
    return Collections.unmodifiableList(resources);
}
 
Example 17
Source Project: lucene-solr   Source File: FieldCache.java    License: Apache License 2.0 5 votes vote down vote up
public CacheEntry(IndexReader.CacheKey readerKey, String fieldName,
                  Class<?> cacheType,
                  Object custom,
                  Accountable value) {
  this.readerKey = readerKey;
  this.fieldName = fieldName;
  this.cacheType = cacheType;
  this.custom = custom;
  this.value = value;
}
 
Example 18
/**
 * Returns nested resources of this class. 
 * The result should be a point-in-time snapshot (to avoid race conditions).
 * @see Accountables
 */
@Override
public Collection<Accountable> getChildResources() {
  synchronized (cachedOrdMaps) {
    return Accountables.namedAccountables("DefaultSortedSetDocValuesReaderState", cachedOrdMaps);
  }
}
 
Example 19
Source Project: Elasticsearch   Source File: Segment.java    License: Apache License 2.0 5 votes vote down vote up
Accountable readRamTree(StreamInput in) throws IOException {
    final String name = in.readString();
    final long bytes = in.readVLong();
    int numChildren = in.readVInt();
    if (numChildren == 0) {
        return Accountables.namedAccountable(name, bytes);
    }
    List<Accountable> children = new ArrayList(numChildren);
    while (numChildren-- > 0) {
        children.add(readRamTree(in));
    }
    return Accountables.namedAccountable(name, children, bytes);
}
 
Example 20
Source Project: lucene-solr   Source File: BloomFilteringPostingsFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  List<Accountable> resources = new ArrayList<>(Accountables.namedAccountables("field", bloomsByFieldName));
  if (delegateFieldsProducer != null) {
    resources.add(Accountables.namedAccountable("delegate", delegateFieldsProducer));
  }
  return Collections.unmodifiableList(resources);
}
 
Example 21
Source Project: lucene-solr   Source File: FSTCompletionLookup.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  List<Accountable> resources = new ArrayList<>();
  if (normalCompletion != null) {
    resources.add(Accountables.namedAccountable("fst", normalCompletion.getFST()));
  }
  if (higherWeightsCompletion != null && (normalCompletion == null || normalCompletion.getFST() != higherWeightsCompletion.getFST())) {
    resources.add(Accountables.namedAccountable("higher weights fst", higherWeightsCompletion.getFST()));
  }
  return Collections.unmodifiableList(resources);
}
 
Example 22
Source Project: Elasticsearch   Source File: Engine.java    License: Apache License 2.0 5 votes vote down vote up
/** Returns 0 in the case where accountable is null, otherwise returns {@code ramBytesUsed()} */
protected static long guardedRamBytesUsed(Accountable a) {
    if (a == null) {
        return 0;
    }
    return a.ramBytesUsed();
}
 
Example 23
Source Project: lucene-solr   Source File: AnalyzingInfixSuggester.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  List<Accountable> resources = new ArrayList<>();
  try {
    if (searcherMgr != null) {
      SearcherManager mgr;
      IndexSearcher searcher;
      synchronized (searcherMgrLock) {
        mgr = searcherMgr; // acquire & release on same SearcherManager, via local reference
        searcher = mgr.acquire();
      }
      try {
        for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
          LeafReader reader = FilterLeafReader.unwrap(context.reader());
          if (reader instanceof SegmentReader) {
            resources.add(Accountables.namedAccountable("segment", (SegmentReader)reader));
          }
        }
      } finally {
        mgr.release(searcher);
      }
    }
    return Collections.unmodifiableList(resources);
  } catch (IOException ioe) {
    throw new RuntimeException(ioe);
  }
}
 
Example 24
Source Project: Elasticsearch   Source File: IndexService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void onRemoval(ShardId shardId, Accountable accountable) {
    if (shardId != null) {
        final IndexShard shard = indexService.shard(shardId.id());
        if (shard != null) {
            long ramBytesUsed = accountable != null ? accountable.ramBytesUsed() : 0l;
            shard.shardBitsetFilterCache().onRemoval(ramBytesUsed);
        }
    }
}
 
Example 25
Source Project: Elasticsearch   Source File: IndicesSegmentResponse.java    License: Apache License 2.0 5 votes vote down vote up
static void toXContent(XContentBuilder builder, Accountable tree) throws IOException {
    builder.startObject();
    builder.field(Fields.DESCRIPTION, tree.toString());
    builder.byteSizeField(Fields.SIZE_IN_BYTES, Fields.SIZE, new ByteSizeValue(tree.ramBytesUsed()));
    Collection<Accountable> children = tree.getChildResources();
    if (children.isEmpty() == false) {
        builder.startArray(Fields.CHILDREN);
        for (Accountable child : children) {
            toXContent(builder, child);
        }
        builder.endArray();
    }
    builder.endObject();
}
 
Example 26
Source Project: lucene-solr   Source File: SimpleTextFieldsReader.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Collection<Accountable> getChildResources() {
  if (fst == null) {
    return Collections.emptyList();
  } else {
    return Collections.singletonList(Accountables.namedAccountable("term cache", fst));
  }
}
 
Example 27
Source Project: linden   Source File: LindenFieldCacheImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
    throws IOException {
  int maxDoc = reader.maxDoc();

  final long[][] matrix = new long[maxDoc][];
  BinaryDocValues valuesIn = reader.getBinaryDocValues(key.field);
  if (valuesIn == null) {
    for (int i = 0; i < maxDoc; ++i) {
      matrix[i] = new long[0];
    }
    return new LongList(matrix);
  }
  for (int i = 0; i < maxDoc; ++i) {
    String str = valuesIn.get(i).utf8ToString();
    if (StringUtils.isEmpty(str)) {
      matrix[i] = new long[0];
      continue;
    }
    JSONArray array = JSON.parseArray(str);
    matrix[i] = new long[array.size()];
    for (int j = 0; j < array.size(); ++j) {
      matrix[i][j] = array.getInteger(j);
    }
  }
  return new LongList(matrix);
}
 
Example 28
Source Project: lucene-solr   Source File: TaxonomyIndexArrays.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized Collection<Accountable> getChildResources() {
  final List<Accountable> resources = new ArrayList<>();
  resources.add(Accountables.namedAccountable("parents", RamUsageEstimator.shallowSizeOf(parents)));
  if (children != null) {
    resources.add(Accountables.namedAccountable("children", RamUsageEstimator.shallowSizeOf(children)));
  }
  if (siblings != null) {
    resources.add(Accountables.namedAccountable("siblings", RamUsageEstimator.shallowSizeOf(siblings)));
  }
  return Collections.unmodifiableList(resources);
}
 
Example 29
Source Project: linden   Source File: LindenFieldCacheImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
    throws IOException {
  int maxDoc = reader.maxDoc();

  final double[][] matrix = new double[maxDoc][];
  BinaryDocValues valuesIn = reader.getBinaryDocValues(key.field);
  if (valuesIn == null) {
    for (int i = 0; i < maxDoc; ++i) {
      matrix[i] = new double[0];
    }
    return new DoubleList(matrix);
  }
  for (int i = 0; i < maxDoc; ++i) {
    String str = valuesIn.get(i).utf8ToString();
    if (StringUtils.isEmpty(str)) {
      matrix[i] = new double[0];
      continue;
    }
    JSONArray array = JSON.parseArray(str);
    matrix[i] = new double[array.size()];
    for (int j = 0; j < array.size(); ++j) {
      matrix[i][j] = array.getFloat(j);
    }
  }
  return new DoubleList(matrix);
}
 
Example 30
Source Project: linden   Source File: LindenFieldCacheImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected Accountable createValue(final AtomicReader reader, CacheKey key, boolean setDocsWithField)
    throws IOException {
  int maxDoc = reader.maxDoc();

  final String[][] matrix = new String[maxDoc][];
  BinaryDocValues valuesIn = reader.getBinaryDocValues(key.field);
  if (valuesIn == null) {
    for (int i = 0; i < maxDoc; ++i) {
      matrix[i] = new String[0];
    }
    return new StringList(matrix);
  }
  for (int i = 0; i < maxDoc; ++i) {
    String str = valuesIn.get(i).utf8ToString();
    if (StringUtils.isEmpty(str)) {
      matrix[i] = new String[0];
      continue;
    }
    JSONArray array = JSON.parseArray(str);
    matrix[i] = new String[array.size()];
    for (int j = 0; j < array.size(); ++j) {
      matrix[i][j] = array.getString(j);
    }
  }
  return new StringList(matrix);
}