Java Code Examples for org.apache.arrow.vector.ValueVector#allocateNew()

@Override
public void allocate(Map<String, ValueVector> vectorMap) throws OutOfMemoryException {
  int estimatedRecordCount;
  if ((reader != null) && (reader.getInput() != null) && (vectorMap.size() > 0)) {
    final OptionManager options = context.getOptions();
    final int listSizeEstimate = (int) options.getOption(ExecConstants.BATCH_LIST_SIZE_ESTIMATE);
    final int varFieldSizeEstimate = (int) options.getOption(ExecConstants.BATCH_VARIABLE_FIELD_SIZE_ESTIMATE);
    final int estimatedRecordSize = BatchSchema.estimateRecordSize(vectorMap, listSizeEstimate, varFieldSizeEstimate);
    if (estimatedRecordSize > 0) {
      estimatedRecordCount = (int) Math.min(reader.getInput().length / estimatedRecordSize, numRowsPerBatch);
    } else {
      estimatedRecordCount = (int) numRowsPerBatch;
    }
  } else {
    estimatedRecordCount = (int) numRowsPerBatch;
  }
  for (final ValueVector v : vectorMap.values()) {
    v.setInitialCapacity(estimatedRecordCount);
    v.allocateNew();
  }
}
 

private void doAlloc() {
  //Allocate vv in the allocationVectors.
  for (ValueVector v : this.allocationVectors) {
    v.allocateNew();
  }

  //Allocate vv for complexWriters.
  if (complexWriters == null) {
    return;
  }

  for (ComplexWriter writer : complexWriters) {
    writer.allocate();
  }

}
 

private void testInterp(Project project, LogicalExpression expr, Table input, Table expected) throws Exception {
  final BufferAllocator childAllocator = getTestAllocator().newChildAllocator("interp", 0, Long.MAX_VALUE);
  try(OperatorContextImpl context = testContext.getNewOperatorContext(childAllocator, project, 1);
      Generator generator = input.toGenerator(context.getAllocator());
      VectorContainer output = new VectorContainer(context.getAllocator());
      ){

    LogicalExpression materializedExpr = ExpressionTreeMaterializer.materializeAndCheckErrors(expr, generator.getOutput().getSchema(), testContext.getFunctionLookupContext());

    ValueVector vector = output.addOrGet(materializedExpr.getCompleteType().toField("out"));
    vector.allocateNew();
    output.buildSchema();
    generator.next(1);
    InterpreterEvaluator.evaluate(generator.getOutput(), context.getFunctionContext(), vector, materializedExpr);
    output.setAllCount(1);
    try(RecordBatchData data = new RecordBatchData(output, context.getAllocator())){
      expected.checkValid(Collections.singletonList(data));
    }

  }

}
 

@Override
public <T extends ValueVector> T addField(Field field, Class<T> clazz) throws SchemaChangeException {
  ValueVector v = fieldVectorMap.get(field.getName().toLowerCase());
  if (v == null || v.getClass() != clazz) {
    // Field does not exist add it to the map
    v = TypeHelper.getNewVector(field, context.getAllocator());
    if (!clazz.isAssignableFrom(v.getClass())) {
      throw new SchemaChangeException(String.format(
        "Class %s was provided, expected %s.", clazz.getSimpleName(), v.getClass().getSimpleName()));
    }
    v.allocateNew();
    fieldVectorMap.put(field.getName().toLowerCase(), v);
  }
  return clazz.cast(v);
}
 

@Override
public void allocate(Map<String, ValueVector> vectorMap) throws OutOfMemoryException {
  for(ValueVector v : vectorMap.values()){
    if(v instanceof FixedWidthVector){
      ((FixedWidthVector) v).allocateNew(context.getTargetBatchSize());
    } else {
      v.allocateNew();
    }
  }
}
 

private void allocateOutgoing() {
  for (final VectorWrapper<?> w : outgoingContainer) {
    final ValueVector v = w.getValueVector();
    if (v instanceof FixedWidthVector) {
      AllocationHelper.allocate(v, context.getTargetBatchSize(), 1);
    } else {
      v.allocateNew();
    }
  }
}
 

public BatchHolder() {

      this.aggrValuesContainer = new VectorContainer();
      boolean success = false;
      try {
        ValueVector vector;

        for (int i = 0; i < materializedValueFields.length; i++) {
          Field outputField = materializedValueFields[i];
          // Create a type-specific ValueVector for this value
          vector = TypeHelper.getNewVector(outputField, allocator);

          // Try to allocate space to store BATCH_SIZE records. Key stored at index i in HashTable has its workspace
          // variables (such as count, sum etc) stored at index i in HashAgg. HashTable and HashAgg both have
          // BatchHolders. Whenever a BatchHolder in HashAgg reaches its capacity, a new BatchHolder is added to
          // HashTable. If HashAgg can't store BATCH_SIZE records in a BatchHolder, it leaves empty slots in current
          // BatchHolder in HashTable, causing the HashTable to be space inefficient. So it is better to allocate space
          // to fit as close to as BATCH_SIZE records.
          if (vector instanceof FixedWidthVector) {
            ((FixedWidthVector) vector).allocateNew(HashTable.BATCH_SIZE);
          } else if (vector instanceof VariableWidthVector) {
            ((VariableWidthVector) vector).allocateNew(((long) HashTable.VARIABLE_WIDTH_VECTOR_SIZE) * HashTable.BATCH_SIZE,
                HashTable.BATCH_SIZE);
          } else if (vector instanceof ObjectVector) {
            ((ObjectVector) vector).allocateNew(HashTable.BATCH_SIZE);
          } else {
            vector.allocateNew();
          }

          capacity = Math.min(capacity, vector.getValueCapacity());

          aggrValuesContainer.add(vector);
        }
        success = true;
      } finally {
        if (!success) {
          aggrValuesContainer.close();
        }
      }
    }
 

private void allocateVectors(int targetRecordCount) {
  boolean memoryAllocated = false;
  double density = lastSuccessfulDensity;
  while (!memoryAllocated) {
    try {
      for (VectorWrapper<?> w : outgoing) {
        final ValueVector v = w.getValueVector();
        if (v instanceof DensityAwareVector) {
          ((DensityAwareVector) v).setInitialCapacity(targetRecordCount, density);
        } else {
          v.setInitialCapacity(targetRecordCount);
        }
        v.allocateNew();
      }
      memoryAllocated = true;
      lastSuccessfulDensity = density;
    } catch (OutOfMemoryException ex) {
      // halve the density and try again
      density = density / 2;
      if (density < 0.01) {
        logger.debug("PriorityQueueCopierTemplate ran out of memory to allocate outgoing batch. " +
            "Records: {}, density: {}", targetRecordCount, density);
        throw ex;
      }
      // try allocating again with lower density
      logger.debug("PriorityQueueCopierTemplate: Ran out of memory. Retrying allocation with lower density.");
    }
  }
}
 

@Override
public void allocate(Map<String, ValueVector> vectorMap) throws OutOfMemoryException {
  for (final ValueVector v : vectorMap.values()) {
    v.allocateNew();
  }
}
 

@Override
public void allocate(Map<String, ValueVector> vectorMap) throws OutOfMemoryException {
  for (final ValueVector v : vectorMap.values()) {
    v.allocateNew();
  }
}
 

@Override
public void allocate(Map<String, ValueVector> vectorMap) throws OutOfMemoryException {
  for (final ValueVector v : vectorMap.values()) {
    v.allocateNew();
  }
}
 

@Override
public void allocate(Map<String, ValueVector> vectorMap) throws OutOfMemoryException {
  for (final ValueVector v : vectorMap.values()) {
    v.allocateNew();
  }
}
 

private void allocateInternal() {
  for (VectorWrapper<?> w : container) {
    ValueVector vv = internal.addOrGet(w.getField());
    vv.allocateNew();
  }
}
 

private void allocateInternal() {
  for (VectorWrapper<?> w : container) {
    ValueVector vv = internal.addOrGet(w.getField());
    vv.allocateNew();
  }
}
 

public BatchHolder(int idx) {

      this.batchIndex = idx;
      SimpleIntVector links = null;
      htContainer = new VectorContainer();
      boolean success = false;
      try {
        for (VectorWrapper<?> w : htContainerOrig) {
          ValueVector vv = TypeHelper.getNewVector(w.getField(), allocator);

          // Capacity for "hashValues" and "links" vectors is BATCH_SIZE records. It is better to allocate space for
          // "key" vectors to store as close to as BATCH_SIZE records. A new BatchHolder is created when either BATCH_SIZE
          // records are inserted or "key" vectors ran out of space. Allocating too less space for "key" vectors will
          // result in unused space in "hashValues" and "links" vectors in the BatchHolder. Also for each new
          // BatchHolder we create a SV4 vector of BATCH_SIZE in HashJoinHelper.
          if (vv instanceof FixedWidthVector) {
            ((FixedWidthVector) vv).allocateNew(BATCH_SIZE);
          } else if (vv instanceof VariableWidthVector) {
            ((VariableWidthVector) vv).allocateNew(VARIABLE_WIDTH_VECTOR_SIZE, BATCH_SIZE);
          } else {
            vv.allocateNew();
          }

          htContainer.add(vv);
        }

        // cache local links reference to support failed creation
        links = allocMetadataVector(HashTable.BATCH_SIZE, EMPTY_SLOT);
        this.links = links;

        this.hashValues = allocMetadataVector(HashTable.BATCH_SIZE, 0);

        success = true;
      } finally {
        if (!success) {
          htContainer.clear();
          if (links != null) {
            links.clear();
          }
        }
      }
    }
 

@Override
public void allocate(Map<String, ValueVector> vectorMap) throws OutOfMemoryException {
  for (final ValueVector v : vectorMap.values()) {
    v.allocateNew();
  }
}