org.apache.lucene.util.RamUsageEstimator Java Examples

The following examples show how to use org.apache.lucene.util.RamUsageEstimator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: crate   Author: crate   File: CollectSetAggregation.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Map<Object, Object> iterate(RamAccounting ramAccounting,
                                   MemoryManager memoryManager,
                                   Map<Object, Object> state,
                                   Input... args) throws CircuitBreakingException {
    Object value = args[0].value();
    if (value == null) {
        return state;
    }
    if (state.put(value, PRESENT) == null) {
        ramAccounting.addBytes(
            // values size + 32 bytes for entry, 4 bytes for increased capacity
            RamUsageEstimator.alignObjectSize(innerTypeEstimator.estimateSize(value) + 36L)
        );
    }
    return state;
}
 
Example #2
Source Project: crate   Author: crate   File: CollectSetAggregation.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Map<Object, Long> removeFromAggregatedState(RamAccounting ramAccounting,
                                                   Map<Object, Long> previousAggState,
                                                   Input[] stateToRemove) {
    Object value = stateToRemove[0].value();
    if (value == null) {
        return previousAggState;
    }
    Long numTimesValueSeen = previousAggState.get(value);
    if (numTimesValueSeen == null) {
        return previousAggState;
    }
    if (numTimesValueSeen == 1) {
        previousAggState.remove(value);
        ramAccounting.addBytes(
            // we initially accounted for values size + 32 bytes for entry, 4 bytes for increased capacity
            // and 12 bytes for the array container and the int value it stored
            - RamUsageEstimator.alignObjectSize(innerTypeEstimator.estimateSize(value) + 48L)
        );
    } else {
        previousAggState.put(value, numTimesValueSeen - 1);
    }
    return previousAggState;
}
 
Example #3
Source Project: Elasticsearch   Author: baidu   File: Translog.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Writes all operations in the given iterable to the given output stream including the size of the array
 * use {@link #readOperations(StreamInput)} to read it back.
 */
public static void writeOperations(StreamOutput outStream, List<Operation> toWrite) throws IOException {
    final ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(BigArrays.NON_RECYCLING_INSTANCE);
    try {
        outStream.writeInt(toWrite.size());
        final BufferedChecksumStreamOutput checksumStreamOutput = new BufferedChecksumStreamOutput(out);
        for (Operation op : toWrite) {
            out.reset();
            final long start = out.position();
            out.skip(RamUsageEstimator.NUM_BYTES_INT);
            writeOperationNoSize(checksumStreamOutput, op);
            long end = out.position();
            int operationSize = (int) (out.position() - RamUsageEstimator.NUM_BYTES_INT - start);
            out.seek(start);
            out.writeInt(operationSize);
            out.seek(end);
            ReleasablePagedBytesReference bytes = out.bytes();
            bytes.writeTo(outStream);
        }
    } finally {
        Releasables.close(out.bytes());
    }

}
 
Example #4
Source Project: Elasticsearch   Author: baidu   File: BigByteArray.java    License: Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newBytePage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example #5
Source Project: Elasticsearch   Author: baidu   File: BigIntArray.java    License: Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newIntPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example #6
Source Project: Elasticsearch   Author: baidu   File: BigFloatArray.java    License: Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newIntPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example #7
Source Project: Elasticsearch   Author: baidu   File: BigDoubleArray.java    License: Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newLongPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example #8
Source Project: lucene-solr   Author: apache   File: FSTEnum.java    License: Apache License 2.0 6 votes vote down vote up
private void incr() {
  upto++;
  grow();
  if (arcs.length <= upto) {
    @SuppressWarnings({"rawtypes","unchecked"}) final FST.Arc<T>[] newArcs =
      new FST.Arc[ArrayUtil.oversize(1+upto, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
    System.arraycopy(arcs, 0, newArcs, 0, arcs.length);
    arcs = newArcs;
  }
  if (output.length <= upto) {
    @SuppressWarnings({"rawtypes","unchecked"}) final T[] newOutput =
      (T[]) new Object[ArrayUtil.oversize(1+upto, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
    System.arraycopy(output, 0, newOutput, 0, output.length);
    output = newOutput;
  }
}
 
Example #9
Source Project: crate   Author: crate   File: BigObjectArray.java    License: Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newObjectPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example #10
Source Project: lucene-solr   Author: apache   File: TermsQuery.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @param toField               The field that should contain terms that are specified in the next parameter.
 * @param terms                 The terms that matching documents should have. The terms must be sorted by natural order.
 * @param indexReaderContextId  Refers to the top level index reader used to create the set of terms in the previous parameter.
 */
TermsQuery(String toField, BytesRefHash terms, String fromField, Query fromQuery, Object indexReaderContextId) {
  super(toField);
  this.terms = terms;
  ords = terms.sort();
  this.fromField = fromField;
  this.fromQuery = fromQuery;
  this.indexReaderContextId = indexReaderContextId;

  this.ramBytesUsed = BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(field) +
      RamUsageEstimator.sizeOfObject(fromField) +
      RamUsageEstimator.sizeOfObject(fromQuery, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED) +
      RamUsageEstimator.sizeOfObject(ords) +
      RamUsageEstimator.sizeOfObject(terms);
}
 
Example #11
Source Project: lucene-solr   Author: apache   File: GlobalOrdinalsWithScoreQuery.java    License: Apache License 2.0 6 votes vote down vote up
GlobalOrdinalsWithScoreQuery(GlobalOrdinalsWithScoreCollector collector, ScoreMode scoreMode, String joinField,
                             OrdinalMap globalOrds, Query toQuery, Query fromQuery, int min, int max,
                             Object indexReaderContextId) {
  this.collector = collector;
  this.joinField = joinField;
  this.globalOrds = globalOrds;
  this.toQuery = toQuery;
  this.scoreMode = scoreMode;
  this.fromQuery = fromQuery;
  this.min = min;
  this.max = max;
  this.indexReaderContextId = indexReaderContextId;

  this.ramBytesUsed = BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(this.fromQuery, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED) +
      RamUsageEstimator.sizeOfObject(this.globalOrds) +
      RamUsageEstimator.sizeOfObject(this.joinField) +
      RamUsageEstimator.sizeOfObject(this.toQuery, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED);
}
 
Example #12
Source Project: crate   Author: crate   File: BigIntArray.java    License: Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newIntPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example #13
Source Project: lucene-solr   Author: apache   File: BM25FQuery.java    License: Apache License 2.0 6 votes vote down vote up
private BM25FQuery(BM25Similarity similarity, TreeMap<String, FieldAndWeight> fieldAndWeights, BytesRef[] terms) {
  this.similarity = similarity;
  this.fieldAndWeights = fieldAndWeights;
  this.terms = terms;
  int numFieldTerms = fieldAndWeights.size() * terms.length;
  if (numFieldTerms > IndexSearcher.getMaxClauseCount()) {
    throw new IndexSearcher.TooManyClauses();
  }
  this.fieldTerms = new Term[numFieldTerms];
  Arrays.sort(terms);
  int pos = 0;
  for (String field : fieldAndWeights.keySet()) {
    for (BytesRef term : terms) {
      fieldTerms[pos++] = new Term(field, term);
    }
  }

  this.ramBytesUsed = BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(fieldAndWeights) +
      RamUsageEstimator.sizeOfObject(fieldTerms) +
      RamUsageEstimator.sizeOfObject(terms);
}
 
Example #14
Source Project: junitperf   Author: houbb   File: PerformanceEvaluationTask.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void run() {
    long startTimeNs = System.nanoTime();
    long startMeasurements = startTimeNs + warmUpNs;

    // 堆大小
    long memoryKb = RamUsageEstimator.shallowSizeOf(testInstance);
    statisticsCalculator.setMemory(memoryKb);

    while (isContinue) {
        evaluateStatement(startMeasurements);
    }
}
 
Example #15
Source Project: Elasticsearch   Author: baidu   File: HyperLogLogPlusPlus.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Compute the required precision so that <code>count</code> distinct entries
 * would be counted with linear counting.
 */
public static int precisionFromThreshold(long count) {
    final long hashTableEntries = (long) Math.ceil(count / MAX_LOAD_FACTOR);
    int precision = PackedInts.bitsRequired(hashTableEntries * RamUsageEstimator.NUM_BYTES_INT);
    precision = Math.max(precision, MIN_PRECISION);
    precision = Math.min(precision, MAX_PRECISION);
    return precision;
}
 
Example #16
Source Project: Elasticsearch   Author: baidu   File: GlobalOrdinalsStringTermsAggregator.java    License: Apache License 2.0 5 votes vote down vote up
protected static void copy(BytesRef from, BytesRef to) {
    if (to.bytes.length < from.length) {
        to.bytes = new byte[ArrayUtil.oversize(from.length, RamUsageEstimator.NUM_BYTES_BYTE)];
    }
    to.offset = 0;
    to.length = from.length;
    System.arraycopy(from.bytes, from.offset, to.bytes, 0, from.length);
}
 
Example #17
Source Project: lucene-solr   Author: apache   File: PackedLongValues.java    License: Apache License 2.0 5 votes vote down vote up
/** Build a {@link PackedLongValues} instance that contains values that
 *  have been added to this builder. This operation is destructive. */
public PackedLongValues build() {
  finish();
  pending = null;
  final PackedInts.Reader[] values = ArrayUtil.copyOfSubArray(this.values, 0, valuesOff);
  final long ramBytesUsed = PackedLongValues.BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values);
  return new PackedLongValues(pageShift, pageMask, values, size, ramBytesUsed);
}
 
Example #18
Source Project: Elasticsearch   Author: baidu   File: GeoPointDVAtomicFieldData.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public MultiGeoPointValues getGeoPointValues() {
    return new MultiGeoPointValues() {
        GeoPoint[] points = new GeoPoint[0];
        private int count = 0;

        @Override
        public void setDocument(int docId) {
            values.setDocument(docId);
            count = values.count();
            if (count > points.length) {
                final int previousLength = points.length;
                points = Arrays.copyOf(points, ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
                for (int i = previousLength; i < points.length; ++i) {
                    points[i] = new GeoPoint(Double.NaN, Double.NaN);
                }
            }
            for (int i=0; i<count; ++i) {
                points[i].resetFromIndexHash(values.valueAt(i));
            }
        }

        @Override
        public int count() {
            return count;
        }

        @Override
        public GeoPoint valueAt(int index) {
            return points[index];
        }
    };
}
 
Example #19
Source Project: lucene-solr   Author: apache   File: LTRScoringQuery.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public long ramBytesUsed() {
  return BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(efi) +
      RamUsageEstimator.sizeOfObject(ltrScoringModel) +
      RamUsageEstimator.sizeOfObject(originalQuery, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED);
}
 
Example #20
Source Project: lucene-solr   Author: apache   File: ConcurrentLRUCache.java    License: Apache License 2.0 5 votes vote down vote up
public CacheEntry(K key, V value, long createTime, long lastAccessed) {
  this.key = key;
  this.value = value;
  this.createTime = createTime;
  this.lastAccessed = lastAccessed;
  this.ramBytesUsed =
      BASE_RAM_BYTES_USED +
      RamUsageEstimator.sizeOfObject(key, QUERY_DEFAULT_RAM_BYTES_USED) +
      RamUsageEstimator.sizeOfObject(value, QUERY_DEFAULT_RAM_BYTES_USED);
}
 
Example #21
Source Project: lucene-solr   Author: apache   File: Automaton.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public long ramBytesUsed() {
  // TODO: BitSet RAM usage (isAccept.size()/8) isn't fully accurate...
  return RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.sizeOf(states) + RamUsageEstimator.sizeOf(transitions) +
    RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + (isAccept.size() / 8) + RamUsageEstimator.NUM_BYTES_OBJECT_REF +
    2 * RamUsageEstimator.NUM_BYTES_OBJECT_REF +
    3 * Integer.BYTES +
    1;
}
 
Example #22
Source Project: crate   Author: crate   File: MapSizeEstimatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void test_map_size_estimate_depends_on_actual_instance_size() {
    Map<String, Integer> map = Map.of("x", 10, "y", 20);
    assertThat(
        MapSizeEstimator.INSTANCE.estimateSize(map),
        is(RamUsageEstimator.sizeOfMap(map))
    );
}
 
Example #23
Source Project: crate   Author: crate   File: CollectSetAggregation.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Map<Object, Object> reduce(RamAccounting ramAccounting,
                                  Map<Object, Object> state1,
                                  Map<Object, Object> state2) {
    for (Object newValue : state2.keySet()) {
        if (state1.put(newValue, PRESENT) == null) {
            ramAccounting.addBytes(
                // value size + 32 bytes for entry + 4 bytes for increased capacity
                RamUsageEstimator.alignObjectSize(innerTypeEstimator.estimateSize(newValue) + 36L)
            );
        }
    }
    return state1;
}
 
Example #24
Source Project: lucene-solr   Author: apache   File: RAMOnlyPostingsFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public long ramBytesUsed() {
  long sizeInBytes = 0;
  sizeInBytes +=  (positions!=null) ? RamUsageEstimator.sizeOf(positions) : 0;
  
  if (payloads != null) {
    for(byte[] payload: payloads) {
      sizeInBytes += (payload!=null) ? RamUsageEstimator.sizeOf(payload) : 0;
    }
  }
  return sizeInBytes;
}
 
Example #25
Source Project: Elasticsearch   Author: baidu   File: BigArrays.java    License: Apache License 2.0 5 votes vote down vote up
/** Grow an array to a size that is larger than <code>minSize</code>, preserving content, and potentially reusing part of the provided array. */
public ByteArray grow(ByteArray array, long minSize) {
    if (minSize <= array.size()) {
        return array;
    }
    final long newSize = overSize(minSize, BYTE_PAGE_SIZE, RamUsageEstimator.NUM_BYTES_BYTE);
    return resize(array, newSize);
}
 
Example #26
Source Project: Elasticsearch   Author: baidu   File: BigArrays.java    License: Apache License 2.0 5 votes vote down vote up
/** Grow an array to a size that is larger than <code>minSize</code>, preserving content, and potentially reusing part of the provided array. */
public IntArray grow(IntArray array, long minSize) {
    if (minSize <= array.size()) {
        return array;
    }
    final long newSize = overSize(minSize, INT_PAGE_SIZE, RamUsageEstimator.NUM_BYTES_INT);
    return resize(array, newSize);
}
 
Example #27
Source Project: Elasticsearch   Author: baidu   File: BigArrays.java    License: Apache License 2.0 5 votes vote down vote up
/** Grow an array to a size that is larger than <code>minSize</code>, preserving content, and potentially reusing part of the provided array. */
public LongArray grow(LongArray array, long minSize) {
    if (minSize <= array.size()) {
        return array;
    }
    final long newSize = overSize(minSize, LONG_PAGE_SIZE, RamUsageEstimator.NUM_BYTES_LONG);
    return resize(array, newSize);
}
 
Example #28
Source Project: lucene-solr   Author: apache   File: CaffeineCache.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public V remove(K key) {
  V existing = cache.asMap().remove(key);
  if (existing != null) {
    ramBytes.add(- RamUsageEstimator.sizeOfObject(key, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED));
    ramBytes.add(- RamUsageEstimator.sizeOfObject(existing, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED));
    ramBytes.add(- RamUsageEstimator.LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY);
  }
  return existing;
}
 
Example #29
Source Project: Elasticsearch   Author: baidu   File: BigArrays.java    License: Apache License 2.0 5 votes vote down vote up
/** Grow an array to a size that is larger than <code>minSize</code>, preserving content, and potentially reusing part of the provided array. */
public FloatArray grow(FloatArray array, long minSize) {
    if (minSize <= array.size()) {
        return array;
    }
    final long newSize = overSize(minSize, INT_PAGE_SIZE, RamUsageEstimator.NUM_BYTES_FLOAT);
    return resize(array, newSize);
}
 
Example #30
Source Project: Elasticsearch   Author: baidu   File: BigArrays.java    License: Apache License 2.0 5 votes vote down vote up
/** Grow an array to a size that is larger than <code>minSize</code>, preserving content, and potentially reusing part of the provided array. */
public <T> ObjectArray<T> grow(ObjectArray<T> array, long minSize) {
    if (minSize <= array.size()) {
        return array;
    }
    final long newSize = overSize(minSize, OBJECT_PAGE_SIZE, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
    return resize(array, newSize);
}