com.carrotsearch.hppc.IntHashSet Java Examples

The following examples show how to use com.carrotsearch.hppc.IntHashSet. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: NodeFetchRequestTest.java    From crate with Apache License 2.0 6 votes vote down vote up
@Test
public void testStreaming() throws Exception {

    IntObjectHashMap<IntContainer> toFetch = new IntObjectHashMap<>();
    IntHashSet docIds = new IntHashSet(3);
    toFetch.put(1, docIds);

    NodeFetchRequest orig = new NodeFetchRequest(UUID.randomUUID(), 1, true, toFetch);

    BytesStreamOutput out = new BytesStreamOutput();
    orig.writeTo(out);

    StreamInput in = out.bytes().streamInput();

    NodeFetchRequest streamed = new NodeFetchRequest(in);

    assertThat(orig.jobId(), is(streamed.jobId()));
    assertThat(orig.fetchPhaseId(), is(streamed.fetchPhaseId()));
    assertThat(orig.isCloseContext(), is(streamed.isCloseContext()));
    assertThat(orig.toFetch().toString(), is(streamed.toFetch().toString()));
}
 
Example #2
Source File: HiveVarcharTruncationReader.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
public void setupProjector(OperatorContext context,
                           Stopwatch javaCodeGenWatch,
                           Stopwatch gandivaCodeGenWatch,
                           VectorContainer incoming,
                           ExpressionEvaluationOptions projectorOptions, VectorContainer projectorOutput) {

  try {
    BatchSchema targetSchema = BatchSchema.newBuilder().addField(field).build();

    ExpressionSplitter varcharSplitter = ProjectOperator.createSplitterWithExpressions(incoming, Collections.singletonList(castExpr),
      null, null, null, context, projectorOptions, projectorOutput, targetSchema);
    varcharSplitter.setupProjector(projectorOutput, javaCodeGenWatch, gandivaCodeGenWatch);
    this.splitter = varcharSplitter;

    IntHashSet transferFieldIds = new IntHashSet();
    List<TransferPair> transfers = Lists.newArrayList();
    ProjectOperator.createSplitterWithExpressions(incoming, Collections.singletonList(noCastExpr),
      transfers, null, transferFieldIds, context, projectorOptions, projectorOutput, targetSchema);
    if (!transfers.isEmpty()) {
      this.transferPair = transfers.get(0);
    }
  }
  catch (Exception e) {
    throw Throwables.propagate(e);
  }
}
 
Example #3
Source File: BaseWriteConfiguration.java    From spliceengine with GNU Affero General Public License v3.0 6 votes vote down vote up
@Override
public WriteResponse processGlobalResult(BulkWriteResult bulkWriteResult) throws Throwable {
    WriteResult writeResult = bulkWriteResult.getGlobalResult();
    if (writeResult.isSuccess())
        return WriteResponse.SUCCESS;
    else if (writeResult.isPartial()) {
        IntObjectHashMap<WriteResult> failedRows = bulkWriteResult.getFailedRows();
        if (failedRows != null && failedRows.size() > 0) {
            return WriteResponse.PARTIAL;
        }
        IntHashSet notRun = bulkWriteResult.getNotRunRows();
        if(notRun!=null && notRun.size()>0)
            return WriteResponse.PARTIAL;
        /*
         * We got a partial result, but didn't specify which rows needed behavior.
         * That's weird, but since we weren't told there would be a problem, we may
         * as well ignore
         */
        return WriteResponse.IGNORE;
    } else if (!writeResult.canRetry())
        throw exceptionFactory.processErrorResult(writeResult);
    else
        return WriteResponse.RETRY;
}
 
Example #4
Source File: BulkWriteResult.java    From spliceengine with GNU Affero General Public License v3.0 6 votes vote down vote up
@Override
public BulkWriteResult read(Kryo kryo, Input input, Class<BulkWriteResult> type) {
		WriteResult globalStatus = kryo.readObject(input,WriteResult.class);
		int notRunSize = input.readInt();
		IntHashSet notRunRows = new IntHashSet(notRunSize);
		for(int i=0;i<notRunSize;i++){
				notRunRows.add(input.readInt());
		}
		int failedSize = input.readInt();
		IntObjectHashMap<WriteResult> failedRows = new IntObjectHashMap<>(failedSize,0.9f);
		for(int i=0;i<failedSize;i++){
				int k = input.readInt();
				WriteResult result = kryo.readObject(input,WriteResult.class);
				failedRows.put(k,result);
		}
		return new BulkWriteResult(globalStatus,notRunRows,failedRows);
}
 
Example #5
Source File: SortedIntDocSet.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Bits getBits() {
  IntHashSet hashSet = new IntHashSet(docs.length);
  for (int doc : docs) {
    hashSet.add(doc);
  }

  return new Bits() {
    @Override
    public boolean get(int index) {
      return hashSet.contains(index);
    }

    @Override
    public int length() {
      return getLength();
    }
  };
}
 
Example #6
Source File: ExpandComponent.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public GroupExpandCollector(SortedDocValues docValues, FixedBitSet groupBits, IntHashSet collapsedSet, int limit, Sort sort) throws IOException {
  int numGroups = collapsedSet.size();
  groups = new LongObjectHashMap<>(numGroups);
  DocIdSetIterator iterator = new BitSetIterator(groupBits, 0); // cost is not useful here
  int group;
  while ((group = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
    groups.put(group, getCollector(limit, sort));
  }

  this.collapsedSet = collapsedSet;
  this.groupBits = groupBits;
  this.docValues = docValues;
  if(docValues instanceof MultiDocValues.MultiSortedDocValues) {
    this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)docValues;
    this.ordinalMap = multiSortedDocValues.mapping;
  }
}
 
Example #7
Source File: TestRangeQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
static boolean sameDocs(String msg, DocList a, DocList b) {
  assertEquals(msg, a.size(), b.size());

  IntHashSet bIds = new IntHashSet(b.size());
  DocIterator bIter = b.iterator();
  while (bIter.hasNext()) {
    bIds.add(bIter.nextDoc());
  }

  DocIterator aIter = a.iterator();
  while (aIter.hasNext()) {
    int doc = aIter.nextDoc();
    assertTrue(msg, bIds.contains(doc));
  }
  return true;
}
 
Example #8
Source File: TitanPartitionGraphTest.java    From titan1withtp3.1 with Apache License 2.0 6 votes vote down vote up
@Test
public void testKeybasedGraphPartitioning() {
    Object[] options = {option(GraphDatabaseConfiguration.IDS_FLUSH), false,
                        option(VertexIDAssigner.PLACEMENT_STRATEGY), PropertyPlacementStrategy.class.getName(),
            option(PropertyPlacementStrategy.PARTITION_KEY), "clusterId"};
    clopen(options);

    int[] groupDegrees = {5,5,5,5,5,5,5,5};
    int numVertices = setupGroupClusters(groupDegrees,CommitMode.PER_VERTEX);

    IntSet partitionIds = new IntHashSet(numVertices); //to track the "spread" of partition ids
    for (int i=0;i<groupDegrees.length;i++) {
        TitanVertex g = getOnlyVertex(tx.query().has("groupid","group"+i));
        int partitionId = -1;
        for (TitanVertex v : g.query().direction(Direction.IN).labels("member").vertices()) {
            if (partitionId<0) partitionId = getPartitionID(v);
            assertEquals(partitionId,getPartitionID(v));
            partitionIds.add(partitionId);
        }
    }
    assertTrue(partitionIds.size()>numPartitions/2); //This is a probabilistic test that might fail
}
 
Example #9
Source File: AddAllTest.java    From hashmapTest with The Unlicense 5 votes vote down vote up
public static void main(String[] args) {
    final long start = System.currentTimeMillis();

    final IntHashSet a = new com.carrotsearch.hppc.IntHashSet();
    for( int i = 10000000; i-- != 0; ) a.add(i);
    IntHashSet b = new com.carrotsearch.hppc.IntHashSet(a.size());
    b.addAll(a);
    b = new com.carrotsearch.hppc.IntHashSet();
    b.addAll(a);

    final long time = System.currentTimeMillis() - start;
    System.out.println( time / 1000.0 );
    System.out.println( b.size() );
}
 
Example #10
Source File: NodeFetchResponseTest.java    From crate with Apache License 2.0 5 votes vote down vote up
@Before
public void setUpStreamBucketsAndStreamer() throws Exception {
    streamers = new IntObjectHashMap<>(1);
    streamers.put(1, new Streamer[]{DataTypes.BOOLEAN.streamer()});

    IntObjectHashMap<IntContainer> toFetch = new IntObjectHashMap<>();
    IntHashSet docIds = new IntHashSet(3);
    toFetch.put(1, docIds);
    StreamBucket.Builder builder = new StreamBucket.Builder(streamers.get(1), RamAccounting.NO_ACCOUNTING);
    builder.add(new RowN(new Object[]{true}));
    fetched = new IntObjectHashMap<>(1);
    fetched.put(1, builder.build());
}
 
Example #11
Source File: PipelineUtils.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
public static Collection<KVPair> doPartialRetry(BulkWrite bulkWrite,BulkWriteResult response,long id) throws Exception{
    IntHashSet notRunRows=response.getNotRunRows();
    IntObjectHashMap<WriteResult> failedRows=response.getFailedRows();
    Collection<KVPair> toRetry=new ArrayList<>(failedRows.size()+notRunRows.size());
    List<String> errorMsgs= Lists.newArrayListWithCapacity(failedRows.size());
    int i=0;
    Collection<KVPair> allWrites=bulkWrite.getMutations();
    for(KVPair kvPair : allWrites){
        if(notRunRows.contains(i))
            toRetry.add(kvPair);
        else{
            WriteResult writeResult=failedRows.get(i);
            if(writeResult!=null){
                errorMsgs.add(writeResult.getErrorMessage());
                if(writeResult.canRetry())
                    toRetry.add(kvPair);
            }
        }
        i++;
    }
    if(LOG.isTraceEnabled()){
        int[] errorCounts=new int[11];
        for(IntObjectCursor<WriteResult> failedCursor : failedRows){
            errorCounts[failedCursor.value.getCode().ordinal()]++;
        }
        SpliceLogUtils.trace(LOG,"[%d] %d failures with types: %s",id,failedRows.size(),Arrays.toString(errorCounts));
    }

    return toRetry;
}
 
Example #12
Source File: IndicesClusterStateService.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
private void applyDeletedShards(final ClusterChangedEvent event) {
    RoutingNodes.RoutingNodeIterator routingNode = event.state().getRoutingNodes().routingNodeIter(event.state().nodes().localNodeId());
    if (routingNode == null) {
        return;
    }
    IntHashSet newShardIds = new IntHashSet();
    for (IndexService indexService : indicesService) {
        String index = indexService.index().name();
        IndexMetaData indexMetaData = event.state().metaData().index(index);
        if (indexMetaData == null) {
            continue;
        }
        // now, go over and delete shards that needs to get deleted
        newShardIds.clear();
        for (ShardRouting shard : routingNode) {
            if (shard.index().equals(index)) {
                newShardIds.add(shard.id());
            }
        }
        for (Integer existingShardId : indexService.shardIds()) {
            if (!newShardIds.contains(existingShardId)) {
                if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
                    if (logger.isDebugEnabled()) {
                        logger.debug("[{}][{}] removing shard (index is closed)", index, existingShardId);
                    }
                    indexService.removeShard(existingShardId, "removing shard (index is closed)");
                } else {
                    // we can just remove the shard, without cleaning it locally, since we will clean it
                    // when all shards are allocated in the IndicesStore
                    if (logger.isDebugEnabled()) {
                        logger.debug("[{}][{}] removing shard (not allocated)", index, existingShardId);
                    }
                    indexService.removeShard(existingShardId, "removing shard (not allocated)");
                }
            }
        }
    }
}
 
Example #13
Source File: TitanPartitionGraphTest.java    From titan1withtp3.1 with Apache License 2.0 5 votes vote down vote up
private void testPartitionSpread(boolean flush, boolean batchCommit) {
    Object[] options = {option(GraphDatabaseConfiguration.IDS_FLUSH), flush};
    clopen(options);

    int[] groupDegrees = {10,15,10,17,10,4,7,20,11};
    int numVertices = setupGroupClusters(groupDegrees,batchCommit?CommitMode.BATCH:CommitMode.PER_VERTEX);

    IntSet partitionIds = new IntHashSet(numVertices); //to track the "spread" of partition ids
    for (int i=0;i<groupDegrees.length;i++) {
        TitanVertex g = getOnlyVertex(tx.query().has("groupid","group"+i));
        assertCount(groupDegrees[i],g.edges(Direction.OUT,"contain"));
        assertCount(groupDegrees[i],g.edges(Direction.IN,"member"));
        assertCount(groupDegrees[i],g.query().direction(Direction.OUT).edges());
        assertCount(groupDegrees[i],g.query().direction(Direction.IN).edges());
        assertCount(groupDegrees[i]*2,g.query().edges());
        for (TitanVertex v : g.query().direction(Direction.IN).labels("member").vertices()) {
            int pid = getPartitionID(v);
            partitionIds.add(pid);
            assertEquals(g, getOnlyElement(v.query().direction(Direction.OUT).labels("member").vertices()));
            VertexList vlist = v.query().direction(Direction.IN).labels("contain").vertexIds();
            assertEquals(1,vlist.size());
            assertEquals(pid,idManager.getPartitionId(vlist.getID(0)));
            assertEquals(g,vlist.get(0));
        }
    }
    if (flush || !batchCommit) { //In these cases we would expect significant spread across partitions
        assertTrue(partitionIds.size()>numPartitions/2); //This is a probabilistic test that might fail
    } else {
        assertEquals(1,partitionIds.size()); //No spread in this case
    }
}
 
Example #14
Source File: ExpandComponent.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public NumericGroupExpandCollector(String field, long nullValue, LongHashSet groupSet, IntHashSet collapsedSet, int limit, Sort sort) throws IOException {
  int numGroups = collapsedSet.size();
  this.nullValue = nullValue;
  groups = new LongObjectHashMap<>(numGroups);
  for (LongCursor cursor : groupSet) {
    groups.put(cursor.value, getCollector(limit, sort));
  }

  this.field = field;
  this.collapsedSet = collapsedSet;
}
 
Example #15
Source File: ProjectOperator.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
public static EvalMode getEvalMode(VectorAccessible incoming, LogicalExpression expr, IntHashSet transferFieldIds){
  // add value vector to transfer if direct reference and this is allowed, otherwise, add to evaluation stack.
  final boolean canDirectTransfer =

    // the expression is a direct read.
    expr instanceof ValueVectorReadExpression

      // we aren't dealing with a selection vector.
      && incoming.getSchema().getSelectionVectorMode() == SelectionVectorMode.NONE

      // the field doesn't have a red path (e.g. a single value out of a list)
      && !((ValueVectorReadExpression) expr).hasReadPath()

      // We aren't already transferring the field.
      && !transferFieldIds.contains(((ValueVectorReadExpression) expr).getFieldId().getFieldIds()[0]);

  if(canDirectTransfer){
    return EvalMode.DIRECT;
  }
  final boolean isComplex =
    expr instanceof FunctionHolderExpr
      && ((FunctionHolderExpr) expr).isComplexWriterFuncHolder();

  if(isComplex){
    return EvalMode.COMPLEX;
  }

  return EvalMode.EVAL;

}
 
Example #16
Source File: MoreCollectors.java    From more-lambdas-java with Artistic License 2.0 4 votes vote down vote up
public static Collector<Integer, ?, IntHashSet> toIntSet() {
    return new CollectorImpl<>(IntHashSet::new, IntHashSet::add, (left, right) -> {
        left.addAll(right);
        return left;
    }, CH_ID);
}
 
Example #17
Source File: BulkWriteResult.java    From spliceengine with GNU Affero General Public License v3.0 4 votes vote down vote up
public BulkWriteResult() {
		notRunRows = new IntHashSet();
		failedRows = new IntObjectHashMap<>();
}
 
Example #18
Source File: BulkWriteResult.java    From spliceengine with GNU Affero General Public License v3.0 4 votes vote down vote up
public BulkWriteResult(WriteContext writeContext, WriteResult globalStatus) {
		notRunRows = new IntHashSet();
		failedRows = new IntObjectHashMap<>();
		this.writeContext = writeContext;
		this.globalStatus = globalStatus;
}
 
Example #19
Source File: BulkWriteResult.java    From spliceengine with GNU Affero General Public License v3.0 4 votes vote down vote up
public BulkWriteResult(WriteResult globalStatus, IntHashSet notRunRows, IntObjectHashMap<WriteResult> failedRows){
		this.notRunRows = notRunRows;
		this.failedRows = failedRows;
		this.globalStatus = globalStatus;
}
 
Example #20
Source File: BulkWriteResult.java    From spliceengine with GNU Affero General Public License v3.0 4 votes vote down vote up
public IntHashSet getNotRunRows() {
		return notRunRows;
}
 
Example #21
Source File: ProjectOperator.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
@Override
public VectorAccessible setup(VectorAccessible incoming) throws Exception {
  this.incoming = incoming;
  this.allocationVectors = Lists.newArrayList();
  final List<NamedExpression> exprs = getExpressionList();
  final List<TransferPair> transfers = new ArrayList<>();

  final ClassGenerator<Projector> cg = context.getClassProducer().createGenerator(Projector
    .TEMPLATE_DEFINITION).getRoot();

  final IntHashSet transferFieldIds = new IntHashSet();

  splitter = createSplitterWithExpressions(incoming, exprs, transfers, cg, transferFieldIds,
    context, projectorOptions, outgoing, null);

  outgoing.buildSchema(SelectionVectorMode.NONE);
  outgoing.setInitialCapacity(context.getTargetBatchSize());
  state = State.CAN_CONSUME;
  initialSchema = outgoing.getSchema();
  splitter.setupProjector(outgoing, javaCodeGenWatch, gandivaCodeGenWatch);
  javaCodeGenWatch.start();
  this.projector = cg.getCodeGenerator().getImplementationClass();
  projector.setup(
    context.getFunctionContext(),
    incoming,
    outgoing,
    transfers,
    new ComplexWriterCreator(){
      @Override
      public ComplexWriter addComplexWriter(String name) {
        VectorAccessibleComplexWriter vc = new VectorAccessibleComplexWriter(outgoing);
        ComplexWriter writer = new ComplexWriterImpl(name, vc);
        complexWriters.add(writer);
        return writer;
      }
    }
  );
  javaCodeGenWatch.stop();
  OperatorStats stats = context.getStats();
  stats.addLongStat(Metric.JAVA_BUILD_TIME, javaCodeGenWatch.elapsed(TimeUnit.MILLISECONDS));
  stats.addLongStat(Metric.GANDIVA_BUILD_TIME, gandivaCodeGenWatch.elapsed(TimeUnit.MILLISECONDS));
  stats.addLongStat(Metric.GANDIVA_EXPRESSIONS, splitter.getNumExprsInGandiva());
  stats.addLongStat(Metric.JAVA_EXPRESSIONS, splitter.getNumExprsInJava());
  stats.addLongStat(Metric.MIXED_EXPRESSIONS, splitter.getNumExprsInBoth());
  stats.addLongStat(Metric.MIXED_SPLITS, splitter.getNumSplitsInBoth());
  stats.setProfileDetails(OperatorProfileDetails
    .newBuilder()
    .addAllSplitInfos(splitter.getSplitInfos())
    .build()
  );
  gandivaCodeGenWatch.reset();
  javaCodeGenWatch.reset();
  return outgoing;
}
 
Example #22
Source File: BulkWriteActionRetryTest.java    From spliceengine with GNU Affero General Public License v3.0 4 votes vote down vote up
@Test
public void testRetryNoRouteToHostException() throws Throwable {
    WriteConfiguration config = new DefaultWriteConfiguration(new Monitor(0,0,10,10L,0),pef);
    WriteResult writeResult = new WriteResult(Code.FAILED, "NoRouteToHostException:No route to host");
    BulkWriteResult bulkWriteResult = new BulkWriteResult(writeResult);
    WriteResponse response = config.processGlobalResult(bulkWriteResult);
    Assert.assertEquals(WriteResponse.RETRY, response);

    writeResult = new WriteResult(Code.FAILED, "FailedServerException:This server is in the failed servers list");
    bulkWriteResult = new BulkWriteResult(writeResult);
    response = config.processGlobalResult(bulkWriteResult);
    Assert.assertEquals(WriteResponse.RETRY, response);

    writeResult = new WriteResult(Code.FAILED, "ServerNotRunningYetException");
    bulkWriteResult = new BulkWriteResult(writeResult);
    response = config.processGlobalResult(bulkWriteResult);
    Assert.assertEquals(WriteResponse.RETRY, response);

    writeResult = new WriteResult(Code.FAILED, "ConnectTimeoutException");
    bulkWriteResult = new BulkWriteResult(writeResult);
    response = config.processGlobalResult(bulkWriteResult);
    Assert.assertEquals(WriteResponse.RETRY, response);

    writeResult = new WriteResult(Code.PARTIAL);
    IntObjectHashMap<WriteResult> failedRows = new IntObjectHashMap<>();
    failedRows.put(1, new WriteResult(Code.FAILED, "NoRouteToHostException:No route to host"));
    bulkWriteResult = new BulkWriteResult(writeResult, new IntHashSet(), failedRows);
    response = config.partialFailure(bulkWriteResult, null);
    Assert.assertEquals(WriteResponse.RETRY, response);


    writeResult = new WriteResult(Code.PARTIAL);
    failedRows = new IntObjectHashMap<>();
    failedRows.put(1, new WriteResult(Code.FAILED, "FailedServerException:This server is in the failed servers list"));
    bulkWriteResult = new BulkWriteResult(writeResult, new IntHashSet(), failedRows);
    response = config.partialFailure(bulkWriteResult, null);
    Assert.assertEquals(WriteResponse.RETRY, response);

    writeResult = new WriteResult(Code.PARTIAL);
    failedRows = new IntObjectHashMap<>();
    failedRows.put(1, new WriteResult(Code.FAILED, "ServerNotRunningYetException"));
    bulkWriteResult = new BulkWriteResult(writeResult, new IntHashSet(), failedRows);
    response = config.partialFailure(bulkWriteResult, null);
    Assert.assertEquals(WriteResponse.RETRY, response);

    writeResult = new WriteResult(Code.PARTIAL);
    failedRows = new IntObjectHashMap<>();
    failedRows.put(1, new WriteResult(Code.FAILED, "ConnectTimeoutException"));
    bulkWriteResult = new BulkWriteResult(writeResult, new IntHashSet(), failedRows);
    response = config.partialFailure(bulkWriteResult, null);
    Assert.assertEquals(WriteResponse.RETRY, response);


    NoRouteToHostException nrthe = new NoRouteToHostException();
    response = config.globalError(nrthe);
    Assert.assertEquals(WriteResponse.RETRY, response);

    FailedServerException failedServerException = new FailedServerException("Failed server");
    response = config.globalError(failedServerException);
    Assert.assertEquals(WriteResponse.RETRY, response);

    ServerNotRunningYetException serverNotRunningYetException = new ServerNotRunningYetException("Server not running");
    response = config.globalError(serverNotRunningYetException);
    Assert.assertEquals(WriteResponse.RETRY, response);

    ConnectTimeoutException connectTimeoutException = new ConnectTimeoutException("connect timeout");
    response = config.globalError(connectTimeoutException);
    Assert.assertEquals(WriteResponse.RETRY, response);
}
 
Example #23
Source File: HiveNonVarcharCoercionReader.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
public void setupProjector(OutputMutator output, VectorContainer projectorOutput,
                           ExpressionEvaluationOptions projectorOptions) {
  createCoercions();
  if (incoming.getSchema() == null || incoming.getSchema().getFieldCount() == 0) {
    return;
  }

  final ClassGenerator<Projector> cg = context.getClassProducer().createGenerator(Projector.TEMPLATE_DEFINITION).getRoot();
  final IntHashSet transferFieldIds = new IntHashSet();
  final List<TransferPair> transfers = Lists.newArrayList();

  List<Integer> decimalFields = new ArrayList<>();
  long numDecimalCoercions = 0;
  int i = 0;
  for (Field f : targetSchema.getFields()) {
    if (MajorTypeHelper.getMajorTypeForField(f).getMinorType().equals(TypeProtos.MinorType.DECIMAL)) {
      decimalFields.add(i);
    }
    i++;
  }

  try {
    splitter = ProjectOperator.createSplitterWithExpressions(incoming, exprs, transfers, cg,
      transferFieldIds, context, projectorOptions, projectorOutput, targetSchema);
    splitter.setupProjector(projectorOutput, javaCodeGenWatch, gandivaCodeGenWatch);

    numDecimalCoercions = decimalFields.stream()
        .filter(id -> !transferFieldIds.contains(id))
        .count();
  } catch (Exception e) {
    throw Throwables.propagate(e);
  }
  javaCodeGenWatch.start();
  this.projector = cg.getCodeGenerator().getImplementationClass();
  this.projector.setup(
    context.getFunctionContext(),
    incoming,
    projectorOutput,
    transfers,
    new Projector.ComplexWriterCreator() {
      @Override
      public BaseWriter.ComplexWriter addComplexWriter(String name) {
        return null;
      }
    }
  );
  javaCodeGenWatch.stop();
  OperatorStats stats = context.getStats();
  stats.addLongStat(ScanOperator.Metric.JAVA_BUILD_TIME, javaCodeGenWatch.elapsed(TimeUnit.NANOSECONDS));
  stats.addLongStat(ScanOperator.Metric.GANDIVA_BUILD_TIME, gandivaCodeGenWatch.elapsed(TimeUnit.NANOSECONDS));
  stats.addLongStat(ScanOperator.Metric.NUM_HIVE_PARQUET_DECIMAL_COERCIONS, numDecimalCoercions);
  gandivaCodeGenWatch.reset();
  javaCodeGenWatch.reset();
}
 
Example #24
Source File: CoercionReader.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up
/**
 * set up projector to write output to given container
 * @param projectorOutput
 */
protected void setupProjector(VectorContainer projectorOutput) {
  if (DEBUG_PRINT) {
    debugPrint(projectorOutput);
  }

  if (incoming.getSchema() == null || incoming.getSchema().getFieldCount() == 0) {
    return;
  }

  final ClassGenerator<Projector> cg = context.getClassProducer().createGenerator(Projector.TEMPLATE_DEFINITION).getRoot();
  final IntHashSet transferFieldIds = new IntHashSet();
  final List<TransferPair> transfers = Lists.newArrayList();

  try {
    splitter = ProjectOperator.createSplitterWithExpressions(incoming, exprs, transfers, cg,
        transferFieldIds, context, projectorOptions, projectorOutput, targetSchema);
    splitter.setupProjector(projectorOutput, javaCodeGenWatch, gandivaCodeGenWatch);
  } catch (Exception e) {
    throw Throwables.propagate(e);
  }
  javaCodeGenWatch.start();
  this.projector = cg.getCodeGenerator().getImplementationClass();
  this.projector.setup(
    context.getFunctionContext(),
    incoming,
    projectorOutput,
    transfers,
    new ComplexWriterCreator() {
      @Override
      public ComplexWriter addComplexWriter(String name) {
        return null;
      }
    }
  );
  javaCodeGenWatch.stop();
  OperatorStats stats = context.getStats();
  stats.addLongStat(ScanOperator.Metric.JAVA_BUILD_TIME, javaCodeGenWatch.elapsed(TimeUnit.NANOSECONDS));
  stats.addLongStat(ScanOperator.Metric.GANDIVA_BUILD_TIME, gandivaCodeGenWatch.elapsed(TimeUnit.NANOSECONDS));
  gandivaCodeGenWatch.reset();
  javaCodeGenWatch.reset();
}