org.apache.lucene.util.RamUsageEstimator Java Examples

The following examples show how to use org.apache.lucene.util.RamUsageEstimator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BM25FQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private BM25FQuery(BM25Similarity similarity, TreeMap<String, FieldAndWeight> fieldAndWeights, BytesRef[] terms) {
  this.similarity = similarity;
  this.fieldAndWeights = fieldAndWeights;
  this.terms = terms;
  int numFieldTerms = fieldAndWeights.size() * terms.length;
  if (numFieldTerms > IndexSearcher.getMaxClauseCount()) {
    throw new IndexSearcher.TooManyClauses();
  }
  this.fieldTerms = new Term[numFieldTerms];
  Arrays.sort(terms);
  int pos = 0;
  for (String field : fieldAndWeights.keySet()) {
    for (BytesRef term : terms) {
      fieldTerms[pos++] = new Term(field, term);
    }
  }

  this.ramBytesUsed = BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(fieldAndWeights) +
      RamUsageEstimator.sizeOfObject(fieldTerms) +
      RamUsageEstimator.sizeOfObject(terms);
}
 
Example #2
Source File: CollectSetAggregation.java    From crate with Apache License 2.0 6 votes vote down vote up
@Override
public Map<Object, Object> iterate(RamAccounting ramAccounting,
                                   MemoryManager memoryManager,
                                   Map<Object, Object> state,
                                   Input... args) throws CircuitBreakingException {
    Object value = args[0].value();
    if (value == null) {
        return state;
    }
    if (state.put(value, PRESENT) == null) {
        ramAccounting.addBytes(
            // values size + 32 bytes for entry, 4 bytes for increased capacity
            RamUsageEstimator.alignObjectSize(innerTypeEstimator.estimateSize(value) + 36L)
        );
    }
    return state;
}
 
Example #3
Source File: BigDoubleArray.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newLongPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example #4
Source File: CollectSetAggregation.java    From crate with Apache License 2.0 6 votes vote down vote up
@Override
public Map<Object, Long> removeFromAggregatedState(RamAccounting ramAccounting,
                                                   Map<Object, Long> previousAggState,
                                                   Input[] stateToRemove) {
    Object value = stateToRemove[0].value();
    if (value == null) {
        return previousAggState;
    }
    Long numTimesValueSeen = previousAggState.get(value);
    if (numTimesValueSeen == null) {
        return previousAggState;
    }
    if (numTimesValueSeen == 1) {
        previousAggState.remove(value);
        ramAccounting.addBytes(
            // we initially accounted for values size + 32 bytes for entry, 4 bytes for increased capacity
            // and 12 bytes for the array container and the int value it stored
            - RamUsageEstimator.alignObjectSize(innerTypeEstimator.estimateSize(value) + 48L)
        );
    } else {
        previousAggState.put(value, numTimesValueSeen - 1);
    }
    return previousAggState;
}
 
Example #5
Source File: Translog.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/**
 * Writes all operations in the given iterable to the given output stream including the size of the array
 * use {@link #readOperations(StreamInput)} to read it back.
 */
public static void writeOperations(StreamOutput outStream, List<Operation> toWrite) throws IOException {
    final ReleasableBytesStreamOutput out = new ReleasableBytesStreamOutput(BigArrays.NON_RECYCLING_INSTANCE);
    try {
        outStream.writeInt(toWrite.size());
        final BufferedChecksumStreamOutput checksumStreamOutput = new BufferedChecksumStreamOutput(out);
        for (Operation op : toWrite) {
            out.reset();
            final long start = out.position();
            out.skip(RamUsageEstimator.NUM_BYTES_INT);
            writeOperationNoSize(checksumStreamOutput, op);
            long end = out.position();
            int operationSize = (int) (out.position() - RamUsageEstimator.NUM_BYTES_INT - start);
            out.seek(start);
            out.writeInt(operationSize);
            out.seek(end);
            ReleasablePagedBytesReference bytes = out.bytes();
            bytes.writeTo(outStream);
        }
    } finally {
        Releasables.close(out.bytes());
    }

}
 
Example #6
Source File: BigByteArray.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newBytePage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example #7
Source File: BigIntArray.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newIntPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example #8
Source File: BigFloatArray.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newIntPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example #9
Source File: FSTEnum.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void incr() {
  upto++;
  grow();
  if (arcs.length <= upto) {
    @SuppressWarnings({"rawtypes","unchecked"}) final FST.Arc<T>[] newArcs =
      new FST.Arc[ArrayUtil.oversize(1+upto, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
    System.arraycopy(arcs, 0, newArcs, 0, arcs.length);
    arcs = newArcs;
  }
  if (output.length <= upto) {
    @SuppressWarnings({"rawtypes","unchecked"}) final T[] newOutput =
      (T[]) new Object[ArrayUtil.oversize(1+upto, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
    System.arraycopy(output, 0, newOutput, 0, output.length);
    output = newOutput;
  }
}
 
Example #10
Source File: BigObjectArray.java    From crate with Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newObjectPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example #11
Source File: TermsQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * @param toField               The field that should contain terms that are specified in the next parameter.
 * @param terms                 The terms that matching documents should have. The terms must be sorted by natural order.
 * @param indexReaderContextId  Refers to the top level index reader used to create the set of terms in the previous parameter.
 */
TermsQuery(String toField, BytesRefHash terms, String fromField, Query fromQuery, Object indexReaderContextId) {
  super(toField);
  this.terms = terms;
  ords = terms.sort();
  this.fromField = fromField;
  this.fromQuery = fromQuery;
  this.indexReaderContextId = indexReaderContextId;

  this.ramBytesUsed = BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(field) +
      RamUsageEstimator.sizeOfObject(fromField) +
      RamUsageEstimator.sizeOfObject(fromQuery, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED) +
      RamUsageEstimator.sizeOfObject(ords) +
      RamUsageEstimator.sizeOfObject(terms);
}
 
Example #12
Source File: GlobalOrdinalsWithScoreQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
GlobalOrdinalsWithScoreQuery(GlobalOrdinalsWithScoreCollector collector, ScoreMode scoreMode, String joinField,
                             OrdinalMap globalOrds, Query toQuery, Query fromQuery, int min, int max,
                             Object indexReaderContextId) {
  this.collector = collector;
  this.joinField = joinField;
  this.globalOrds = globalOrds;
  this.toQuery = toQuery;
  this.scoreMode = scoreMode;
  this.fromQuery = fromQuery;
  this.min = min;
  this.max = max;
  this.indexReaderContextId = indexReaderContextId;

  this.ramBytesUsed = BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(this.fromQuery, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED) +
      RamUsageEstimator.sizeOfObject(this.globalOrds) +
      RamUsageEstimator.sizeOfObject(this.joinField) +
      RamUsageEstimator.sizeOfObject(this.toQuery, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED);
}
 
Example #13
Source File: BigIntArray.java    From crate with Apache License 2.0 6 votes vote down vote up
/** Change the size of this array. Content between indexes <code>0</code> and <code>min(size(), newSize)</code> will be preserved. */
@Override
public void resize(long newSize) {
    final int numPages = numPages(newSize);
    if (numPages > pages.length) {
        pages = Arrays.copyOf(pages, ArrayUtil.oversize(numPages, RamUsageEstimator.NUM_BYTES_OBJECT_REF));
    }
    for (int i = numPages - 1; i >= 0 && pages[i] == null; --i) {
        pages[i] = newIntPage(i);
    }
    for (int i = numPages; i < pages.length && pages[i] != null; ++i) {
        pages[i] = null;
        releasePage(i);
    }
    this.size = newSize;
}
 
Example #14
Source File: DeltaPackedLongValues.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public DeltaPackedLongValues build() {
  finish();
  pending = null;
  final PackedInts.Reader[] values = ArrayUtil.copyOfSubArray(this.values, 0, valuesOff);
  final long[] mins = ArrayUtil.copyOfSubArray(this.mins, 0, valuesOff);
  final long ramBytesUsed = DeltaPackedLongValues.BASE_RAM_BYTES_USED
      + RamUsageEstimator.sizeOf(values) + RamUsageEstimator.sizeOf(mins);
  return new DeltaPackedLongValues(pageShift, pageMask, values, mins, size, ramBytesUsed);
}
 
Example #15
Source File: BytesStore.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public long ramBytesUsed() {
  long size = BASE_RAM_BYTES_USED;
  for (byte[] block : blocks) {
    size += RamUsageEstimator.sizeOf(block);
  }
  return size;
}
 
Example #16
Source File: GrowableWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public long ramBytesUsed() {
  return RamUsageEstimator.alignObjectSize(
      RamUsageEstimator.NUM_BYTES_OBJECT_HEADER
      + RamUsageEstimator.NUM_BYTES_OBJECT_REF
      + Long.BYTES
      + Float.BYTES)
      + current.ramBytesUsed();
}
 
Example #17
Source File: MonotonicLongValues.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public MonotonicLongValues build() {
  finish();
  pending = null;
  final PackedInts.Reader[] values = ArrayUtil.copyOfSubArray(this.values, 0, valuesOff);
  final long[] mins = ArrayUtil.copyOfSubArray(this.mins, 0, valuesOff);
  final float[] averages = ArrayUtil.copyOfSubArray(this.averages, 0, valuesOff);
  final long ramBytesUsed = MonotonicLongValues.BASE_RAM_BYTES_USED
      + RamUsageEstimator.sizeOf(values) + RamUsageEstimator.sizeOf(mins)
      + RamUsageEstimator.sizeOf(averages);
  return new MonotonicLongValues(pageShift, pageMask, values, mins, averages, size, ramBytesUsed);
}
 
Example #18
Source File: CollectSetAggregation.java    From crate with Apache License 2.0 5 votes vote down vote up
@Nullable
@Override
public Map<Object, Long> newState(RamAccounting ramAccounting,
                                  Version indexVersionCreated,
                                  Version minNodeInCluster,
                                  MemoryManager memoryManager) {
    ramAccounting.addBytes(RamUsageEstimator.alignObjectSize(64L)); // overhead for HashMap: 32 * 0 + 16 * 4 bytes
    return new HashMap<>();
}
 
Example #19
Source File: AnalyzingInfixSuggester.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public long ramBytesUsed() {
  long mem = RamUsageEstimator.shallowSizeOf(this);
  try {
    if (searcherMgr != null) {
      SearcherManager mgr;
      IndexSearcher searcher;
      synchronized (searcherMgrLock) {
        mgr = searcherMgr; // acquire & release on same SearcherManager, via local reference
        searcher = mgr.acquire();
      }
      try {
        for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
          LeafReader reader = FilterLeafReader.unwrap(context.reader());
          if (reader instanceof SegmentReader) {
            mem += ((SegmentReader) context.reader()).ramBytesUsed();
          }
        }
      } finally {
        mgr.release(searcher);
      }
    }
    return mem;
  } catch (IOException ioe) {
    throw new RuntimeException(ioe);
  }
}
 
Example #20
Source File: QueryResultKey.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public QueryResultKey(Query query, List<Query> filters, Sort sort, int nc_flags, int minExactCount) {
  this.query = query;
  this.sort = sort;
  this.filters = filters;
  this.nc_flags = nc_flags;
  this.minExactCount = minExactCount;

  int h = query.hashCode();

  if (filters != null) {
    for (Query filt : filters)
      // NOTE: simple summation used here so keys with the same filters but in
      // different orders get the same hashCode
      h += filt.hashCode();
  }

  sfields = (this.sort !=null) ? this.sort.getSort() : defaultSort;
  long ramSfields = RamUsageEstimator.NUM_BYTES_ARRAY_HEADER;
  for (SortField sf : sfields) {
    h = h*29 + sf.hashCode();
    ramSfields += BASE_SF_RAM_BYTES_USED + RamUsageEstimator.sizeOfObject(sf.getField());
  }
  h = h*31 + minExactCount;

  hc = h;

  ramBytesUsed =
      BASE_RAM_BYTES_USED +
      ramSfields +
      RamUsageEstimator.sizeOfObject(query, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED) +
      RamUsageEstimator.sizeOfObject(filters, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED);
}
 
Example #21
Source File: ContainsPrefixTreeQuery.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public long ramBytesUsed() {
  return RamUsageEstimator.alignObjectSize(
        RamUsageEstimator.NUM_BYTES_OBJECT_REF
      + Integer.BYTES)
      + intSet.ramBytesUsed();
}
 
Example #22
Source File: PointInGeo3DShapeQuery.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public long ramBytesUsed() {
  return BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(field) +
      RamUsageEstimator.sizeOfObject(shape) +
      RamUsageEstimator.sizeOfObject(shapeBounds);
}
 
Example #23
Source File: SortedIntDocSet.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public String toString() {
  return "SortedIntDocSet{" +
      "size=" + size() + "," +
      "ramUsed=" + RamUsageEstimator.humanReadableUnits(ramBytesUsed())+
      '}';
}
 
Example #24
Source File: DocSlice.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Primary constructor for a DocSlice instance.
 *
 * @param offset  starting offset for this range of docs
 * @param len     length of results
 * @param docs    array of docids starting at position 0
 * @param scores  array of scores that corresponds to docs, may be null
 * @param matches total number of matches for the query
 * @param matchesRelation Indicates if {@code matches} is exact or an approximation
 */
public DocSlice(int offset, int len, int[] docs, float[] scores, long matches, float maxScore, TotalHits.Relation matchesRelation) {
  this.offset=offset;
  this.len=len;
  this.docs=docs;
  this.scores=scores;
  this.matches=matches;
  this.maxScore=maxScore;
  this.ramBytesUsed = BASE_RAM_BYTES_USED + (docs == null ? 0 : ((long)docs.length << 2)) + (scores == null ? 0 : ((long)scores.length<<2)+RamUsageEstimator.NUM_BYTES_ARRAY_HEADER);
  this.matchesRelation = matchesRelation;
}
 
Example #25
Source File: ByteBuffersDataOutput.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public long ramBytesUsed() {
  // Return a rough estimation for allocated blocks. Note that we do not make
  // any special distinction for direct memory buffers.
  return RamUsageEstimator.NUM_BYTES_OBJECT_REF * blocks.size() + 
         blocks.stream().mapToLong(buf -> buf.capacity()).sum();
}
 
Example #26
Source File: PerformanceEvaluationTask.java    From junitperf with Apache License 2.0 5 votes vote down vote up
@Override
public void run() {
    long startTimeNs = System.nanoTime();
    long startMeasurements = startTimeNs + warmUpNs;

    // 堆大小
    long memoryKb = RamUsageEstimator.shallowSizeOf(testInstance);
    statisticsCalculator.setMemory(memoryKb);

    while (isContinue) {
        evaluateStatement(startMeasurements);
    }
}
 
Example #27
Source File: AbstractBigArray.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
private static <T> T[] grow(T[] array, int minSize) {
    if (array.length < minSize) {
        final int newLen = ArrayUtil.oversize(minSize, RamUsageEstimator.NUM_BYTES_OBJECT_REF);
        array = Arrays.copyOf(array, newLen);
    }
    return array;
}
 
Example #28
Source File: PackedLongValues.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Build a {@link PackedLongValues} instance that contains values that
 *  have been added to this builder. This operation is destructive. */
public PackedLongValues build() {
  finish();
  pending = null;
  final PackedInts.Reader[] values = ArrayUtil.copyOfSubArray(this.values, 0, valuesOff);
  final long ramBytesUsed = PackedLongValues.BASE_RAM_BYTES_USED + RamUsageEstimator.sizeOf(values);
  return new PackedLongValues(pageShift, pageMask, values, size, ramBytesUsed);
}
 
Example #29
Source File: CollectSetAggregation.java    From crate with Apache License 2.0 5 votes vote down vote up
@Nullable
@Override
public Map<Object, Object> newState(RamAccounting ramAccounting,
                                    Version indexVersionCreated,
                                    Version minNodeInCluster,
                                    MemoryManager memoryManager) {
    ramAccounting.addBytes(RamUsageEstimator.alignObjectSize(64L)); // overhead for HashMap: 32 * 0 + 16 * 4 bytes
    return new HashMap<>();
}
 
Example #30
Source File: BlockTreeTermsReader.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
private FST.Arc<BytesRef> getArc(int ord) {
  if (ord >= arcs.length) {
    @SuppressWarnings({"rawtypes","unchecked"}) final FST.Arc<BytesRef>[] next =
      new FST.Arc[ArrayUtil.oversize(1+ord, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
    System.arraycopy(arcs, 0, next, 0, arcs.length);
    for(int arcOrd=arcs.length;arcOrd<next.length;arcOrd++) {
      next[arcOrd] = new FST.Arc<BytesRef>();
    }
    arcs = next;
  }
  return arcs[ord];
}