Java Code Examples for org.apache.calcite.plan.RelOptPlanner#getCostFactory()

/**
 * HashToRandomExchange processes M input rows and hash partitions them
 * based on computing a hash value on the distribution fields.
 * If there are N nodes (endpoints), we can assume for costing purposes
 * on average each sender will send M/N rows to 1 destination endpoint.
 * (See DremioCost for symbol notations)
 * Include impact of skewness of distribution : the more keys used, the less likely the distribution will be skewed.
 * The hash cpu cost will be proportional to 1 / #_keys.
 * C =  CPU cost of hashing k fields of M/N rows
 *      + CPU cost of SV remover for M/N rows
 *      + Network cost of sending M/N rows to 1 destination.
 * So, C = (h * 1/k * M/N) + (s * M/N) + (w * M/N)
 * Total cost = N * C
 */
@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
  if (PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    return super.computeSelfCost(planner).multiplyBy(.1);
  }

  RelNode child = this.getInput();
  double inputRows = mq.getRowCount(child);

  int  rowWidth = child.getRowType().getFieldCount() * DremioCost.AVG_FIELD_WIDTH;

  double hashCpuCost = DremioCost.HASH_CPU_COST * inputRows / fields.size();
  double svrCpuCost = DremioCost.SVR_CPU_COST * inputRows;
  double networkCost = DremioCost.BYTE_NETWORK_COST * inputRows * rowWidth;
  Factory costFactory = (Factory)planner.getCostFactory();
  return costFactory.makeCost(inputRows, hashCpuCost + svrCpuCost, 0, networkCost);
}
 

@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery relMetadataQuery) {
  // Elasticsearch does not support contains() in project (only in filter).
  if (hasContains) {
    return planner.getCostFactory().makeInfiniteCost();
  }

  if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    return super.computeSelfCost(planner).multiplyBy(.1);
  }

  // cost is proportional to the number of rows and number of columns being projected
  double rowCount = relMetadataQuery.getRowCount(this);
  // cpu is proportional to the number of columns and row count.  For complex expressions, we also add
  // additional cost for those columns (multiply by DremioCost.PROJECT_CPU_COST).
  double cpuCost = (DremioCost.PROJECT_SIMPLE_CPU_COST * rowCount * simpleFieldCount) + (DremioCost.PROJECT_CPU_COST * (nonSimpleFieldCount > 0 ? rowCount : 0) * nonSimpleFieldCount);
  Factory costFactory = (Factory) planner.getCostFactory();
  return costFactory.makeCost(rowCount, cpuCost, 0, 0);
}
 

@Override
public RelOptCost computeSelfCost(final RelOptPlanner planner, RelMetadataQuery mq) {
  final PlannerSettings settings = PrelUtil.getPlannerSettings(planner);
  final ScanStats stats = this.getGroupScan().getScanStats(settings);
  final int columnCount = this.getRowType().getFieldCount();

  if (PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    return planner.getCostFactory().makeCost(stats.getRecordCount() * columnCount, stats.getCpuCost(), stats.getDiskCost());
  }

  double rowCount = mq.getRowCount(this);
  //double rowCount = stats.getRecordCount();

  // As DRILL-4083 points out, when columnCount == 0, cpuCost becomes zero,
  // which makes the costs of HiveScan and HiveDrillNativeParquetScan the same
  double cpuCost = rowCount * Math.max(columnCount, 1); // For now, assume cpu cost is proportional to row count.

  // If a positive value for CPU cost is given multiply the default CPU cost by given CPU cost.
  if (stats.getCpuCost() > 0) {
    cpuCost *= stats.getCpuCost();
  }

  double ioCost = stats.getDiskCost();
  DrillCostFactory costFactory = (DrillCostFactory)planner.getCostFactory();
  return costFactory.makeCost(rowCount, cpuCost, ioCost, 0);
}
 

/**
 * A SingleMergeExchange processes a total of M rows coming from N
 * sorted input streams (from N senders) and merges them into a single
 * output sorted stream. For costing purposes we can assume each sender
 * is sending M/N rows to a single receiver.
 * (See DremioCost for symbol notations)
 * C =  CPU cost of SV remover for M/N rows
 *     + Network cost of sending M/N rows to 1 destination.
 * So, C = (s * M/N) + (w * M/N)
 * Cost of merging M rows coming from N senders = (M log2 N) * c
 * Total cost = N * C + (M log2 N) * c
 */
@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
  if (PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    return super.computeSelfCost(planner).multiplyBy(.1);
  }
  RelNode child = this.getInput();
  double inputRows = mq.getRowCount(child);
  int  rowWidth = child.getRowType().getFieldCount() * DremioCost.AVG_FIELD_WIDTH;
  double svrCpuCost = DremioCost.SVR_CPU_COST * inputRows;
  double networkCost = DremioCost.BYTE_NETWORK_COST * inputRows * rowWidth;
  int numEndPoints = PrelUtil.getSettings(getCluster()).numEndPoints();
  double mergeCpuCost = DremioCost.COMPARE_CPU_COST * inputRows * (Math.log(numEndPoints)/Math.log(2));
  Factory costFactory = (Factory)planner.getCostFactory();
  return costFactory.makeCost(inputRows, svrCpuCost + mergeCpuCost, 0, networkCost);
}
 

/**
 * A SingleMergeExchange processes a total of M rows coming from N
 * sorted input streams (from N senders) and merges them into a single
 * output sorted stream. For costing purposes we can assume each sender
 * is sending M/N rows to a single receiver.
 * (See DrillCostBase for symbol notations)
 * C =  CPU cost of SV remover for M/N rows
 *     + Network cost of sending M/N rows to 1 destination.
 * So, C = (s * M/N) + (w * M/N)
 * Cost of merging M rows coming from N senders = (M log2 N) * c
 * Total cost = N * C + (M log2 N) * c
 */
@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
  if (PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    return super.computeSelfCost(planner, mq).multiplyBy(.1);
  }
  RelNode child = this.getInput();
  double inputRows = mq.getRowCount(child);
  int  rowWidth = child.getRowType().getFieldCount() * DrillCostBase.AVG_FIELD_WIDTH;
  double svrCpuCost = DrillCostBase.SVR_CPU_COST * inputRows;
  double networkCost = DrillCostBase.BYTE_NETWORK_COST * inputRows * rowWidth;
  int numEndPoints = PrelUtil.getSettings(getCluster()).numEndPoints();
  double mergeCpuCost = DrillCostBase.COMPARE_CPU_COST * inputRows * (Math.log(numEndPoints)/Math.log(2));
  DrillCostFactory costFactory = (DrillCostFactory)planner.getCostFactory();
  return costFactory.makeCost(inputRows, svrCpuCost + mergeCpuCost, 0, networkCost);
}
 

protected  RelOptCost computeCartesianJoinCost(RelOptPlanner planner, RelMetadataQuery mq) {
  final double probeRowCount = mq.getRowCount(this.getLeft());
  final double buildRowCount = mq.getRowCount(this.getRight());

  final DrillCostFactory costFactory = (DrillCostFactory) planner.getCostFactory();

  final double mulFactor = 10000; // This is a magic number,
                                  // just to make sure Cartesian Join is more expensive
                                  // than Non-Cartesian Join.

  final int keySize = 1;  // assume having 1 join key, when estimate join cost.
  final DrillCostBase cost = (DrillCostBase) computeHashJoinCostWithKeySize(planner, keySize, mq).multiplyBy(mulFactor);

  // Cartesian join row count will be product of two inputs. The other factors come from the above estimated DrillCost.
  return costFactory.makeCost(
      buildRowCount * probeRowCount,
      cost.getCpu(),
      cost.getIo(),
      cost.getNetwork(),
      cost.getMemory() );

}
 

@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
  if (PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    return super.computeSelfCost(planner).multiplyBy(.1);
  }
  RelNode child = this.getInput();
  double inputRows = (mq == null)? ROWCOUNT_UNKNOWN : mq.getRowCount(child);

  // int  rowWidth = child.getRowType().getFieldCount() * DrillCostBase.AVG_FIELD_WIDTH;

  /* NOTE: the Exchange costing in general has to be examined in a broader context. A RangePartitionExchange
   * may be added for index plans with RowJeyJoin and Calcite compares the cost of this sub-plan with a
   * full table scan (FTS) sub-plan without an exchange.  The RelSubSet would have Filter-Project-TableScan for
   * the FTS and a RowKeyJoin whose right input is a RangePartitionExchange-IndexScan. Although a final UnionExchange
   * is done for both plans, the intermediate costing of index plan with exchange makes it more expensive than the FTS
   * sub-plan, even though the final cost of the overall FTS would have been more expensive.
   */

  // commenting out following based on comments above
  // double rangePartitionCpuCost = DrillCostBase.RANGE_PARTITION_CPU_COST * inputRows;
  // double svrCpuCost = DrillCostBase.SVR_CPU_COST * inputRows;
  // double networkCost = DrillCostBase.BYTE_NETWORK_COST * inputRows * rowWidth;
  DrillCostFactory costFactory = (DrillCostFactory)planner.getCostFactory();
  return costFactory.makeCost(inputRows, 0, 0, 0 /* see comments above */);
}
 

@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
  if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    return super.computeSelfCost(planner).multiplyBy(.1);
  }
  RelNode child = this.getInput();
  double inputRows = mq.getRowCount(child);

  int numGroupByFields = this.getGroupCount();
  int numAggrFields = this.aggCalls.size();
  double cpuCost = DremioCost.COMPARE_CPU_COST * numGroupByFields * inputRows;
  // add cpu cost for computing the aggregate functions
  cpuCost += DremioCost.FUNC_CPU_COST * numAggrFields * inputRows;
  Factory costFactory = (Factory)planner.getCostFactory();
  return costFactory.makeCost(inputRows, cpuCost, 0 /* disk i/o cost */, 0 /* network cost */);
}
 

@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
  if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    //We use multiplier 0.05 for TopN operator, and 0.1 for Sort, to make TopN a preferred choice.
    return super.computeSelfCost(planner, mq).multiplyBy(.1);
  }

  RelNode child = this.getInput();
  double inputRows = mq.getRowCount(child);
  // int  rowWidth = child.getRowType().getPrecision();
  int numSortFields = this.collation.getFieldCollations().size();
  double cpuCost = DrillCostBase.COMPARE_CPU_COST * numSortFields * inputRows * (Math.log(inputRows)/Math.log(2));
  double diskIOCost = 0; // assume in-memory for now until we enforce operator-level memory constraints

  // TODO: use rowWidth instead of avgFieldWidth * numFields
  // avgFieldWidth * numFields * inputRows
  double numFields = this.getRowType().getFieldCount();
  long fieldWidth = PrelUtil.getPlannerSettings(planner).getOptions()
    .getOption(ExecConstants.AVERAGE_FIELD_WIDTH_KEY).num_val;

  double memCost = fieldWidth * numFields * inputRows;

  DrillCostFactory costFactory = (DrillCostFactory) planner.getCostFactory();
  return costFactory.makeCost(inputRows, cpuCost, diskIOCost, 0, memCost);
}
 

@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
  if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    return super.computeSelfCost(planner).multiplyBy(.1);
  }
  if (joinCategory == JoinCategory.CARTESIAN || joinCategory == JoinCategory.INEQUALITY) {
    return ((Factory)planner.getCostFactory()).makeInfiniteCost();
  }
  double leftRowCount = mq.getRowCount(this.getLeft());
  double rightRowCount = mq.getRowCount(this.getRight());
  // cost of evaluating each leftkey=rightkey join condition
  double joinConditionCost = DremioCost.COMPARE_CPU_COST * this.getLeftKeys().size();
  double cpuCost = joinConditionCost * (leftRowCount + rightRowCount);
  Factory costFactory = (Factory)planner.getCostFactory();
  return costFactory.makeCost(leftRowCount + rightRowCount, cpuCost, 0, 0);
}
 

@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
  if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    return super.computeSelfCost(planner, mq).multiplyBy(.1);
  }
  RelNode child = this.getInput();
  double inputRows = mq.getRowCount(child);

  int numGroupByFields = this.getGroupCount();
  int numAggrFields = this.aggCalls.size();
  double cpuCost = DrillCostBase.COMPARE_CPU_COST * numGroupByFields * inputRows;
  // add cpu cost for computing the aggregate functions
  cpuCost += DrillCostBase.FUNC_CPU_COST * numAggrFields * inputRows;
  DrillCostFactory costFactory = (DrillCostFactory)planner.getCostFactory();
  return costFactory.makeCost(inputRows, cpuCost, 0 /* disk i/o cost */, 0 /* network cost */);
}
 

@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
  if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    return super.computeSelfCost(planner).multiplyBy(.1);
  }
  double leftRowCount = mq.getRowCount(this.getLeft());
  double rightRowCount = mq.getRowCount(this.getRight());
  double nljFactor = PrelUtil.getSettings(getCluster()).getNestedLoopJoinFactor();

  double cpuCost = (leftRowCount * rightRowCount) * nljFactor;

  if (vectorExpression != null) {
    cpuCost *= 0.05;
  }
  Factory costFactory = (Factory) planner.getCostFactory();
  return costFactory.makeCost(leftRowCount * rightRowCount, cpuCost, 0, 0, 0);
}
 

@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
  if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    return super.computeSelfCost(planner, mq).multiplyBy(.1);
  }
  RelNode child = this.getInput();
  double inputRows = mq.getRowCount(child);
  double cpuCost = estimateCpuCost(mq);
  DrillCostFactory costFactory = (DrillCostFactory)planner.getCostFactory();
  return costFactory.makeCost(inputRows, cpuCost, 0, 0);
}
 

@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
  if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    return super.computeSelfCost(planner, mq).multiplyBy(.1);
  }
  // by default, assume cost is proportional to number of rows
  double rowCount = mq.getRowCount(this);
  DrillCostFactory costFactory = (DrillCostFactory)planner.getCostFactory();
  return costFactory.makeCost(rowCount, rowCount, 0, 0).multiplyBy(0.1);
}
 

@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {

  double rowCount = mq.getRowCount(this);
  // Attribute small cost for projecting simple fields. In reality projecting simple columns in not free and
  // this allows projection pushdown/project-merge rules to kick-in thereby eliminating unneeded columns from
  // the projection.
  double cpuCost = DrillCostBase.BASE_CPU_COST * rowCount * this.getRowType().getFieldCount();

  DrillCostBase.DrillCostFactory costFactory = (DrillCostBase.DrillCostFactory) planner.getCostFactory();
  return costFactory.makeCost(rowCount, cpuCost, 0, 0);
}
 

@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
  if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    return super.computeSelfCost(planner).multiplyBy(.1);
  }

  double cpuCost = DremioCost.COMPARE_CPU_COST * fetchSize;
  Factory costFactory = (Factory)planner.getCostFactory();
  return costFactory.makeCost(fetchSize, cpuCost, 0, 0);
}
 

@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
  if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    return super.computeSelfCost(planner, mq).multiplyBy(.1);
  }
  double totalInputRowCount = 0;
  for (int i = 0; i < this.getInputs().size(); i++) {
    totalInputRowCount += mq.getRowCount(this.getInputs().get(i));
  }

  double cpuCost = totalInputRowCount * DrillCostBase.BASE_CPU_COST;
  DrillCostFactory costFactory = (DrillCostFactory)planner.getCostFactory();
  return costFactory.makeCost(totalInputRowCount, cpuCost, 0, 0);
}
 

@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
  if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    return super.computeSelfCost(planner).multiplyBy(.1);
  }
  double rowCount = mq.getRowCount(this.getRight());
  DrillCostFactory costFactory = (DrillCostFactory) planner.getCostFactory();

  // RowKeyJoin operator by itself incurs negligible CPU and I/O cost since it is not doing a real join.
  // The actual cost is attributed to the skip-scan (random I/O). The RK join will hold 1 batch in memory but
  // it is not making any extra copy of either the left or right batches, so the memory cost is 0
  return costFactory.makeCost(rowCount, 0, 0, 0, 0);
}
 

@Override
public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) {
  if(PrelUtil.getSettings(getCluster()).useDefaultCosting()) {
    return super.computeSelfCost(planner).multiplyBy(.1);
  }
  final RelNode child = this.getInput();
  double inputRows = mq.getRowCount(child);

  int numGroupByFields = this.getGroupCount();
  int numAggrFields = this.aggCalls.size();
  // cpu cost of hashing each grouping key
  double cpuCost = DremioCost.HASH_CPU_COST * numGroupByFields * inputRows;
  // add cpu cost for computing the aggregate functions
  cpuCost += DremioCost.FUNC_CPU_COST * numAggrFields * inputRows;
  double diskIOCost = 0; // assume in-memory for now until we enforce operator-level memory constraints

  // TODO: use distinct row count
  // + hash table template stuff
  double factor = PrelUtil.getPlannerSettings(planner).getOptions()
    .getOption(ExecConstants.HASH_AGG_TABLE_FACTOR_KEY).getFloatVal();
  long fieldWidth = PrelUtil.getPlannerSettings(planner).getOptions()
    .getOption(ExecConstants.AVERAGE_FIELD_WIDTH_KEY).getNumVal();

  // table + hashValues + links
  double memCost =
    (
      (fieldWidth * numGroupByFields) +
        IntHolder.WIDTH +
        IntHolder.WIDTH
    ) * inputRows * factor;

  Factory costFactory = (Factory) planner.getCostFactory();
  return costFactory.makeCost(inputRows, cpuCost, diskIOCost, 0 /* network cost */, memCost);
}
 

/**
 *
 * @param planner  : Optimization Planner.
 * @param keySize  : the # of join keys in join condition. Left key size should be equal to right key size.
 * @return         : RelOptCost
 */
private RelOptCost computeHashJoinCostWithKeySize(RelOptPlanner planner, int keySize, RelMetadataQuery relMetadataQuery) {
  /**
   * DRILL-1023, DX-3859:  Need to make sure that join row count is calculated in a reasonable manner.  Calcite's default
   * implementation is leftRowCount * rightRowCount * discountBySelectivity, which is too large (cartesian join).
   * Since we do not support cartesian join, we should just take the maximum of the two join input row counts when
   * computing cost of the join.
   */
  double probeRowCount = relMetadataQuery.getRowCount(this.getLeft());
  double buildRowCount = relMetadataQuery.getRowCount(this.getRight());

  // cpu cost of hashing the join keys for the build side
  double cpuCostBuild = DremioCost.HASH_CPU_COST * keySize * buildRowCount;
  // cpu cost of hashing the join keys for the probe side
  double cpuCostProbe = DremioCost.HASH_CPU_COST * keySize * probeRowCount;
  // cpu cost associated with really large rows
  double cpuCostColCount = getRowType().getFieldCount() * Math.max(probeRowCount, buildRowCount);

  // cpu cost of evaluating each leftkey=rightkey join condition
  double joinConditionCost = DremioCost.COMPARE_CPU_COST * keySize;

  double factor = PrelUtil.getPlannerSettings(planner).getOptions()
      .getOption(ExecConstants.HASH_JOIN_TABLE_FACTOR_KEY).getFloatVal();
  long fieldWidth = PrelUtil.getPlannerSettings(planner).getOptions()
      .getOption(ExecConstants.AVERAGE_FIELD_WIDTH_KEY).getNumVal();

  // table + hashValues + links
  double memCost =
      (
          (fieldWidth * keySize) +
              IntHolder.WIDTH +
              IntHolder.WIDTH
      ) * buildRowCount * factor;

  double cpuCost = joinConditionCost * (probeRowCount) // probe size determine the join condition comparison cost
      + cpuCostBuild + cpuCostProbe + cpuCostColCount;

  Factory costFactory = (Factory) planner.getCostFactory();

  return costFactory.makeCost(buildRowCount + probeRowCount, cpuCost, 0, 0, memCost);
}