Java Code Examples for org.apache.lucene.util.RamUsageEstimator

The following examples show how to use org.apache.lucene.util.RamUsageEstimator. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: crate   Source File: CollectSetAggregation.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Map<Object, Object> iterate(RamAccounting ramAccounting,
                                   MemoryManager memoryManager,
                                   Map<Object, Object> state,
                                   Input... args) throws CircuitBreakingException {
    Object value = args[0].value();
    if (value == null) {
        return state;
    }
    if (state.put(value, PRESENT) == null) {
        ramAccounting.addBytes(
            // values size + 32 bytes for entry, 4 bytes for increased capacity
            RamUsageEstimator.alignObjectSize(innerTypeEstimator.estimateSize(value) + 36L)
        );
    }
    return state;
}
 
Example 2
Source Project: crate   Source File: CollectSetAggregation.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Map<Object, Long> removeFromAggregatedState(RamAccounting ramAccounting,
                                                   Map<Object, Long> previousAggState,
                                                   Input[] stateToRemove) {
    Object value = stateToRemove[0].value();
    if (value == null) {
        return previousAggState;
    }
    Long numTimesValueSeen = previousAggState.get(value);
    if (numTimesValueSeen == null) {
        return previousAggState;
    }
    if (numTimesValueSeen == 1) {
        previousAggState.remove(value);
        ramAccounting.addBytes(
            // we initially accounted for values size + 32 bytes for entry, 4 bytes for increased capacity
            // and 12 bytes for the array container and the int value it stored
            - RamUsageEstimator.alignObjectSize(innerTypeEstimator.estimateSize(value) + 48L)
        );
    } else {
        previousAggState.put(value, numTimesValueSeen - 1);
    }
    return previousAggState;
}
 
Example 3
Source Project: Elasticsearch   Source File: Translog.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Writes all operations in the given iterable to the given output stream including the size of the array
 * use {@link #readOperations(StreamInput)} to read it back.
 */
public static void writeOperations(StreamOutput outStream, List<Operation> toWrite) throws IOException {
    final ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(BigArrays.NON_RECYCLING_INSTANCE);
    try {
        outStream.writeInt(toWrite.size());
        final BufferedChecksumStreamOutput checksumStreamOutput = new BufferedChecksumStreamOutput(out);
        for (Operation op : toWrite) {
            out.reset();
            final long start = out.position();
            out.skip(RamUsageEstimator.NUM_BYTES_INT);
            writeOperationNoSize(checksumStreamOutput, op);
            long end = out.position();
            int operationSize = (int) (out.position() - RamUsageEstimator.NUM_BYTES_INT - start);
            out.seek(start);
            out.writeInt(operationSize);
            out.seek(end);
            ReleasablePagedBytesReference bytes = out.bytes();
            bytes.writeTo(outStream);
        }
    } finally {
        Releasables.close(out.bytes());
    }

}
 
Example 4
Source Project: Elasticsearch   Source File: BigByteArray.java    License: Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newBytePage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example 5
Source Project: Elasticsearch   Source File: BigIntArray.java    License: Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newIntPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example 6
Source Project: Elasticsearch   Source File: BigFloatArray.java    License: Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newIntPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example 7
Source Project: Elasticsearch   Source File: BigDoubleArray.java    License: Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newLongPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example 8
Source Project: lucene-solr   Source File: FSTEnum.java    License: Apache License 2.0 6 votes vote down vote up
private void incr() {
  upto++;
  grow();
  if (arcs.length <= upto) {
    @SuppressWarnings({"rawtypes","unchecked"}) final FST.Arc<T>[] newArcs =
      new FST.Arc[ArrayUtil.oversize(1+upto, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
    System.arraycopy(arcs, 0, newArcs, 0, arcs.length);
    arcs = newArcs;
  }
  if (output.length <= upto) {
    @SuppressWarnings({"rawtypes","unchecked"}) final T[] newOutput =
      (T[]) new Object[ArrayUtil.oversize(1+upto, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
    System.arraycopy(output, 0, newOutput, 0, output.length);
    output = newOutput;
  }
}
 
Example 9
Source Project: crate   Source File: BigObjectArray.java    License: Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newObjectPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example 10
Source Project: lucene-solr   Source File: TermsQuery.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @param toField               The field that should contain terms that are specified in the next parameter.
 * @param terms                 The terms that matching documents should have. The terms must be sorted by natural order.
 * @param indexReaderContextId  Refers to the top level index reader used to create the set of terms in the previous parameter.
 */
TermsQuery(String toField, BytesRefHash terms, String fromField, Query fromQuery, Object indexReaderContextId) {
  super(toField);
  this.terms = terms;
  ords = terms.sort();
  this.fromField = fromField;
  this.fromQuery = fromQuery;
  this.indexReaderContextId = indexReaderContextId;

  this.ramBytesUsed = BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(field) +
      RamUsageEstimator.sizeOfObject(fromField) +
      RamUsageEstimator.sizeOfObject(fromQuery, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED) +
      RamUsageEstimator.sizeOfObject(ords) +
      RamUsageEstimator.sizeOfObject(terms);
}
 
Example 11
Source Project: lucene-solr   Source File: GlobalOrdinalsWithScoreQuery.java    License: Apache License 2.0 6 votes vote down vote up
GlobalOrdinalsWithScoreQuery(GlobalOrdinalsWithScoreCollector collector, ScoreMode scoreMode, String joinField,
                             OrdinalMap globalOrds, Query toQuery, Query fromQuery, int min, int max,
                             Object indexReaderContextId) {
  this.collector = collector;
  this.joinField = joinField;
  this.globalOrds = globalOrds;
  this.toQuery = toQuery;
  this.scoreMode = scoreMode;
  this.fromQuery = fromQuery;
  this.min = min;
  this.max = max;
  this.indexReaderContextId = indexReaderContextId;

  this.ramBytesUsed = BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(this.fromQuery, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED) +
      RamUsageEstimator.sizeOfObject(this.globalOrds) +
      RamUsageEstimator.sizeOfObject(this.joinField) +
      RamUsageEstimator.sizeOfObject(this.toQuery, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED);
}
 
Example 12
Source Project: crate   Source File: BigIntArray.java    License: Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newIntPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example 13
Source Project: lucene-solr   Source File: BM25FQuery.java    License: Apache License 2.0 6 votes vote down vote up
private BM25FQuery(BM25Similarity similarity, TreeMap<String, FieldAndWeight> fieldAndWeights, BytesRef[] terms) {
  this.similarity = similarity;
  this.fieldAndWeights = fieldAndWeights;
  this.terms = terms;
  int numFieldTerms = fieldAndWeights.size() * terms.length;
  if (numFieldTerms > IndexSearcher.getMaxClauseCount()) {
    throw new IndexSearcher.TooManyClauses();
  }
  this.fieldTerms = new Term[numFieldTerms];
  Arrays.sort(terms);
  int pos = 0;
  for (String field : fieldAndWeights.keySet()) {
    for (BytesRef term : terms) {
      fieldTerms[pos++] = new Term(field, term);
    }
  }

  this.ramBytesUsed = BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(fieldAndWeights) +
      RamUsageEstimator.sizeOfObject(fieldTerms) +
      RamUsageEstimator.sizeOfObject(terms);
}
 
Example 14
Source Project: junitperf   Source File: PerformanceEvaluationTask.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void run() {
    long startTimeNs = System.nanoTime();
    long startMeasurements = startTimeNs + warmUpNs;

    // 堆大小
    long memoryKb = RamUsageEstimator.shallowSizeOf(testInstance);
    statisticsCalculator.setMemory(memoryKb);

    while (isContinue) {
        evaluateStatement(startMeasurements);
    }
}
 
Example 15
Source Project: Elasticsearch   Source File: HyperLogLogPlusPlus.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Compute the required precision so that <code>count</code> distinct entries
 * would be counted with linear counting.
 */
public static int precisionFromThreshold(long count) {
    final long hashTableEntries = (long) Math.ceil(count / MAX_LOAD_FACTOR);
    int precision = PackedInts.bitsRequired(hashTableEntries * RamUsageEstimator.NUM_BYTES_INT);
    precision = Math.max(precision, MIN_PRECISION);
    precision = Math.min(precision, MAX_PRECISION);
    return precision;
}
 
Example 16
protected static void copy(BytesRef from, BytesRef to) {
    if (to.bytes.length < from.length) {
        to.bytes = new byte[ArrayUtil.oversize(from.length, RamUsageEstimator.NUM_BYTES_BYTE)];
    }
    to.offset = 0;
    to.length = from.length;
    System.arraycopy(from.bytes, from.offset, to.bytes, 0, from.length);
}
 
Example 17
Source Project: lucene-solr   Source File: PackedLongValues.java    License: Apache License 2.0 5 votes vote down vote up
/** Build a {@link PackedLongValues} instance that contains values that
 *  have been added to this builder. This operation is destructive. */
public PackedLongValues build() {
  finish();
  pending = null;
  final PackedInts.Reader[] values = ArrayUtil.copyOfSubArray(this.values, 0, valuesOff);
  final long ramBytesUsed = PackedLongValues.BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values);
  return new PackedLongValues(pageShift, pageMask, values, size, ramBytesUsed);
}
 
Example 18
Source Project: Elasticsearch   Source File: GeoPointDVAtomicFieldData.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public MultiGeoPointValues getGeoPointValues() {
    return new MultiGeoPointValues() {
        GeoPoint[] points = new GeoPoint[0];
        private int count = 0;

        @Override
        public void setDocument(int docId) {
            values.setDocument(docId);
            count = values.count();
            if (count > points.length) {
                final int previousLength = points.length;
                points = Arrays.copyOf(points, ArrayUtil.oversize(count, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
                for (int i = previousLength; i < points.length; ++i) {
                    points[i] = new GeoPoint(Double.NaN, Double.NaN);
                }
            }
            for (int i=0; i<count; ++i) {
                points[i].resetFromIndexHash(values.valueAt(i));
            }
        }

        @Override
        public int count() {
            return count;
        }

        @Override
        public GeoPoint valueAt(int index) {
            return points[index];
        }
    };
}
 
Example 19
Source Project: lucene-solr   Source File: LTRScoringQuery.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public long ramBytesUsed() {
  return BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(efi) +
      RamUsageEstimator.sizeOfObject(ltrScoringModel) +
      RamUsageEstimator.sizeOfObject(originalQuery, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED);
}
 
Example 20
Source Project: lucene-solr   Source File: ConcurrentLRUCache.java    License: Apache License 2.0 5 votes vote down vote up
public CacheEntry(K key, V value, long createTime, long lastAccessed) {
  this.key = key;
  this.value = value;
  this.createTime = createTime;
  this.lastAccessed = lastAccessed;
  this.ramBytesUsed =
      BASE_RAM_BYTES_USED +
      RamUsageEstimator.sizeOfObject(key, QUERY_DEFAULT_RAM_BYTES_USED) +
      RamUsageEstimator.sizeOfObject(value, QUERY_DEFAULT_RAM_BYTES_USED);
}
 
Example 21
Source Project: lucene-solr   Source File: Automaton.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public long ramBytesUsed() {
  // TODO: BitSet RAM usage (isAccept.size()/8) isn't fully accurate...
  return RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.sizeOf(states) + RamUsageEstimator.sizeOf(transitions) +
    RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + (isAccept.size() / 8) + RamUsageEstimator.NUM_BYTES_OBJECT_REF +
    2 * RamUsageEstimator.NUM_BYTES_OBJECT_REF +
    3 * Integer.BYTES +
    1;
}
 
Example 22
Source Project: crate   Source File: MapSizeEstimatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void test_map_size_estimate_depends_on_actual_instance_size() {
    Map<String, Integer> map = Map.of("x", 10, "y", 20);
    assertThat(
        MapSizeEstimator.INSTANCE.estimateSize(map),
        is(RamUsageEstimator.sizeOfMap(map))
    );
}
 
Example 23
Source Project: crate   Source File: CollectSetAggregation.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Map<Object, Object> reduce(RamAccounting ramAccounting,
                                  Map<Object, Object> state1,
                                  Map<Object, Object> state2) {
    for (Object newValue : state2.keySet()) {
        if (state1.put(newValue, PRESENT) == null) {
            ramAccounting.addBytes(
                // value size + 32 bytes for entry + 4 bytes for increased capacity
                RamUsageEstimator.alignObjectSize(innerTypeEstimator.estimateSize(newValue) + 36L)
            );
        }
    }
    return state1;
}
 
Example 24
Source Project: lucene-solr   Source File: RAMOnlyPostingsFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public long ramBytesUsed() {
  long sizeInBytes = 0;
  sizeInBytes +=  (positions!=null) ? RamUsageEstimator.sizeOf(positions) : 0;
  
  if (payloads != null) {
    for(byte[] payload: payloads) {
      sizeInBytes += (payload!=null) ? RamUsageEstimator.sizeOf(payload) : 0;
    }
  }
  return sizeInBytes;
}
 
Example 25
Source Project: Elasticsearch   Source File: BigArrays.java    License: Apache License 2.0 5 votes vote down vote up
/** Grow an array to a size that is larger than <code>minSize</code>, preserving content, and potentially reusing part of the provided array. */
public ByteArray grow(ByteArray array, long minSize) {
    if (minSize <= array.size()) {
        return array;
    }
    final long newSize = overSize(minSize, BYTE_PAGE_SIZE, RamUsageEstimator.NUM_BYTES_BYTE);
    return resize(array, newSize);
}
 
Example 26
Source Project: Elasticsearch   Source File: BigArrays.java    License: Apache License 2.0 5 votes vote down vote up
/** Grow an array to a size that is larger than <code>minSize</code>, preserving content, and potentially reusing part of the provided array. */
public IntArray grow(IntArray array, long minSize) {
    if (minSize <= array.size()) {
        return array;
    }
    final long newSize = overSize(minSize, INT_PAGE_SIZE, RamUsageEstimator.NUM_BYTES_INT);
    return resize(array, newSize);
}
 
Example 27
Source Project: Elasticsearch   Source File: BigArrays.java    License: Apache License 2.0 5 votes vote down vote up
/** Grow an array to a size that is larger than <code>minSize</code>, preserving content, and potentially reusing part of the provided array. */
public LongArray grow(LongArray array, long minSize) {
    if (minSize <= array.size()) {
        return array;
    }
    final long newSize = overSize(minSize, LONG_PAGE_SIZE, RamUsageEstimator.NUM_BYTES_LONG);
    return resize(array, newSize);
}
 
Example 28
Source Project: lucene-solr   Source File: CaffeineCache.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public V remove(K key) {
  V existing = cache.asMap().remove(key);
  if (existing != null) {
    ramBytes.add(- RamUsageEstimator.sizeOfObject(key, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED));
    ramBytes.add(- RamUsageEstimator.sizeOfObject(existing, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED));
    ramBytes.add(- RamUsageEstimator.LINKED_HASHTABLE_RAM_BYTES_PER_ENTRY);
  }
  return existing;
}
 
Example 29
Source Project: Elasticsearch   Source File: BigArrays.java    License: Apache License 2.0 5 votes vote down vote up
/** Grow an array to a size that is larger than <code>minSize</code>, preserving content, and potentially reusing part of the provided array. */
public FloatArray grow(FloatArray array, long minSize) {
    if (minSize <= array.size()) {
        return array;
    }
    final long newSize = overSize(minSize, INT_PAGE_SIZE, RamUsageEstimator.NUM_BYTES_FLOAT);
    return resize(array, newSize);
}
 
Example 30
Source Project: Elasticsearch   Source File: BigArrays.java    License: Apache License 2.0 5 votes vote down vote up
/** Grow an array to a size that is larger than <code>minSize</code>, preserving content, and potentially reusing part of the provided array. */
public <T> ObjectArray<T> grow(ObjectArray<T> array, long minSize) {
    if (minSize <= array.size()) {
        return array;
    }
    final long newSize = overSize(minSize, OBJECT_PAGE_SIZE, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
    return resize(array, newSize);
}