Java Code Examples for org.apache.calcite.plan.RelOptUtil#classifyFilters()

The following examples show how to use org.apache.calcite.plan.RelOptUtil#classifyFilters() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: RelMdUtil.java    From Bats with Apache License 2.0 4 votes vote down vote up 
/**
 * Computes the number of distinct rows for a set of keys returned from a
 * join. Also known as NDV (number of distinct values).
 *
 * @param joinRel   RelNode representing the join
 * @param joinType  type of join
 * @param groupKey  keys that the distinct row count will be computed for
 * @param predicate join predicate
 * @param useMaxNdv If true use formula <code>max(left NDV, right NDV)</code>,
 *                  otherwise use <code>left NDV * right NDV</code>.
 * @return number of distinct rows
 */
public static Double getJoinDistinctRowCount(RelMetadataQuery mq,
    RelNode joinRel, JoinRelType joinType, ImmutableBitSet groupKey,
    RexNode predicate, boolean useMaxNdv) {
  Double distRowCount;
  ImmutableBitSet.Builder leftMask = ImmutableBitSet.builder();
  ImmutableBitSet.Builder rightMask = ImmutableBitSet.builder();
  RelNode left = joinRel.getInputs().get(0);
  RelNode right = joinRel.getInputs().get(1);

  RelMdUtil.setLeftRightBitmaps(
      groupKey,
      leftMask,
      rightMask,
      left.getRowType().getFieldCount());

  // determine which filters apply to the left vs right
  RexNode leftPred = null;
  RexNode rightPred = null;
  if (predicate != null) {
    final List<RexNode> leftFilters = new ArrayList<>();
    final List<RexNode> rightFilters = new ArrayList<>();
    final List<RexNode> joinFilters = new ArrayList<>();
    final List<RexNode> predList = RelOptUtil.conjunctions(predicate);

    RelOptUtil.classifyFilters(
        joinRel,
        predList,
        joinType,
        joinType == JoinRelType.INNER,
        !joinType.generatesNullsOnLeft(),
        !joinType.generatesNullsOnRight(),
        joinFilters,
        leftFilters,
        rightFilters);

    RexBuilder rexBuilder = joinRel.getCluster().getRexBuilder();
    leftPred =
        RexUtil.composeConjunction(rexBuilder, leftFilters, true);
    rightPred =
        RexUtil.composeConjunction(rexBuilder, rightFilters, true);
  }

  if (useMaxNdv) {
    distRowCount = Math.max(
        mq.getDistinctRowCount(left, leftMask.build(), leftPred),
        mq.getDistinctRowCount(right, rightMask.build(), rightPred));
  } else {
    distRowCount =
      NumberUtil.multiply(
          mq.getDistinctRowCount(left, leftMask.build(), leftPred),
          mq.getDistinctRowCount(right, rightMask.build(), rightPred));
  }

  return RelMdUtil.numDistinctVals(distRowCount, mq.getRowCount(joinRel));
}
Example 2
Source File: FilterCorrelateRule.java    From Bats with Apache License 2.0 4 votes vote down vote up 
public void onMatch(RelOptRuleCall call) {
  final Filter filter = call.rel(0);
  final Correlate corr = call.rel(1);

  final List<RexNode> aboveFilters =
      RelOptUtil.conjunctions(filter.getCondition());

  final List<RexNode> leftFilters = new ArrayList<>();
  final List<RexNode> rightFilters = new ArrayList<>();

  // Try to push down above filters. These are typically where clause
  // filters. They can be pushed down if they are not on the NULL
  // generating side.
  RelOptUtil.classifyFilters(
      corr,
      aboveFilters,
      JoinRelType.INNER,
      false,
      !corr.getJoinType().toJoinType().generatesNullsOnLeft(),
      !corr.getJoinType().toJoinType().generatesNullsOnRight(),
      aboveFilters,
      leftFilters,
      rightFilters);

  if (leftFilters.isEmpty()
      && rightFilters.isEmpty()) {
    // no filters got pushed
    return;
  }

  // Create Filters on top of the children if any filters were
  // pushed to them.
  final RexBuilder rexBuilder = corr.getCluster().getRexBuilder();
  final RelBuilder relBuilder = call.builder();
  final RelNode leftRel =
      relBuilder.push(corr.getLeft()).filter(leftFilters).build();
  final RelNode rightRel =
      relBuilder.push(corr.getRight()).filter(rightFilters).build();

  // Create the new Correlate
  RelNode newCorrRel =
      corr.copy(corr.getTraitSet(), ImmutableList.of(leftRel, rightRel));

  call.getPlanner().onCopy(corr, newCorrRel);

  if (!leftFilters.isEmpty()) {
    call.getPlanner().onCopy(filter, leftRel);
  }
  if (!rightFilters.isEmpty()) {
    call.getPlanner().onCopy(filter, rightRel);
  }

  // Create a Filter on top of the join if needed
  relBuilder.push(newCorrRel);
  relBuilder.filter(
      RexUtil.fixUp(rexBuilder, aboveFilters,
          RelOptUtil.getFieldTypeList(relBuilder.peek().getRowType())));

  call.transformTo(relBuilder.build());
}
Example 3
Source File: DrillRelMdSelectivity.java    From Bats with Apache License 2.0 4 votes vote down vote up 
private Double getJoinSelectivity(DrillJoinRelBase rel, RelMetadataQuery mq, RexNode predicate) {
  double sel = 1.0;
  // determine which filters apply to the left vs right
  RexNode leftPred, rightPred;
  JoinRelType joinType = rel.getJoinType();
  final RexBuilder rexBuilder = rel.getCluster().getRexBuilder();
  int[] adjustments = new int[rel.getRowType().getFieldCount()];

  if (DrillRelOptUtil.guessRows(rel)) {
    return super.getSelectivity(rel, mq, predicate);
  }

  if (predicate != null) {
    RexNode pred;
    List<RexNode> leftFilters = new ArrayList<>();
    List<RexNode> rightFilters = new ArrayList<>();
    List<RexNode> joinFilters = new ArrayList<>();
    List<RexNode> predList = RelOptUtil.conjunctions(predicate);

    RelOptUtil.classifyFilters(
        rel,
        predList,
        joinType,
        joinType == JoinRelType.INNER,
        !joinType.generatesNullsOnLeft(),
        !joinType.generatesNullsOnRight(),
        joinFilters,
        leftFilters,
        rightFilters);
    leftPred =
        RexUtil.composeConjunction(rexBuilder, leftFilters, true);
    rightPred =
        RexUtil.composeConjunction(rexBuilder, rightFilters, true);
    for (RelNode child : rel.getInputs()) {
      RexNode modifiedPred = null;

      if (child == rel.getLeft()) {
        pred = leftPred;
      } else {
        pred = rightPred;
      }
      if (pred != null) {
        // convert the predicate to reference the types of the children
        modifiedPred =
            pred.accept(new RelOptUtil.RexInputConverter(
            rexBuilder,
            null,
            child.getRowType().getFieldList(),
            adjustments));
      }
      sel *= mq.getSelectivity(child, modifiedPred);
    }
    sel *= RelMdUtil.guessSelectivity(RexUtil.composeConjunction(rexBuilder, joinFilters, true));
  }
  return sel;
}
Example 4
Source File: DrillRelMdDistinctRowCount.java    From Bats with Apache License 2.0 4 votes vote down vote up 
private Double getDistinctRowCountInternal(DrillJoinRelBase joinRel, RelMetadataQuery mq, ImmutableBitSet groupKey,
     RexNode predicate) {
  if (DrillRelOptUtil.guessRows(joinRel)) {
    return super.getDistinctRowCount(joinRel, mq, groupKey, predicate);
  }
  // Assume NDV is unaffected by the join when groupKey comes from one side of the join
  // Alleviates NDV over-estimates
  ImmutableBitSet.Builder leftMask = ImmutableBitSet.builder();
  ImmutableBitSet.Builder rightMask = ImmutableBitSet.builder();
  JoinRelType joinType = joinRel.getJoinType();
  RelNode left = joinRel.getInputs().get(0);
  RelNode right = joinRel.getInputs().get(1);
  RelMdUtil.setLeftRightBitmaps(groupKey, leftMask, rightMask,
      left.getRowType().getFieldCount());
  RexNode leftPred = null;
  RexNode rightPred = null;

  // Identify predicates which can be pushed onto the left and right sides of the join
  if (predicate != null) {
    List<RexNode> leftFilters = new ArrayList<>();
    List<RexNode> rightFilters = new ArrayList<>();
    List<RexNode> joinFilters = new ArrayList<>();
    List<RexNode> predList = RelOptUtil.conjunctions(predicate);
    RelOptUtil.classifyFilters(joinRel, predList, joinType, joinType == JoinRelType.INNER,
        !joinType.generatesNullsOnLeft(), !joinType.generatesNullsOnRight(), joinFilters,
            leftFilters, rightFilters);
    RexBuilder rexBuilder = joinRel.getCluster().getRexBuilder();
    leftPred = RexUtil.composeConjunction(rexBuilder, leftFilters, true);
    rightPred = RexUtil.composeConjunction(rexBuilder, rightFilters, true);
  }

  Double leftDistRowCount = null;
  Double rightDistRowCount = null;
  double distRowCount = 1;
  ImmutableBitSet lmb = leftMask.build();
  ImmutableBitSet rmb = rightMask.build();
  // Get NDV estimates for the left and right side predicates, if applicable
  if (lmb.length() > 0) {
    leftDistRowCount = mq.getDistinctRowCount(left, lmb, leftPred);
    if (leftDistRowCount != null) {
      distRowCount = leftDistRowCount;
    }
  }
  if (rmb.length() > 0) {
    rightDistRowCount = mq.getDistinctRowCount(right, rmb, rightPred);
    if (rightDistRowCount != null) {
      distRowCount = rightDistRowCount;
    }
  }
  // Use max of NDVs from both sides of the join, if applicable
  if (leftDistRowCount != null && rightDistRowCount != null) {
    distRowCount = Math.max(leftDistRowCount, rightDistRowCount);
  }
  return RelMdUtil.numDistinctVals(distRowCount, mq.getRowCount(joinRel));
}
Example 5
Source File: RelMdUtil.java    From calcite with Apache License 2.0 4 votes vote down vote up 
/**
 * Computes the number of distinct rows for a set of keys returned from a
 * join. Also known as NDV (number of distinct values).
 *
 * @param joinRel   RelNode representing the join
 * @param joinType  type of join
 * @param groupKey  keys that the distinct row count will be computed for
 * @param predicate join predicate
 * @param useMaxNdv If true use formula <code>max(left NDV, right NDV)</code>,
 *                  otherwise use <code>left NDV * right NDV</code>.
 * @return number of distinct rows
 */
public static Double getJoinDistinctRowCount(RelMetadataQuery mq,
    RelNode joinRel, JoinRelType joinType, ImmutableBitSet groupKey,
    RexNode predicate, boolean useMaxNdv) {
  if (predicate == null || predicate.isAlwaysTrue()) {
    if (groupKey.isEmpty()) {
      return 1D;
    }
  }
  Join join = (Join) joinRel;
  if (join.isSemiJoin()) {
    return getSemiJoinDistinctRowCount(join, mq, groupKey, predicate);
  }
  Double distRowCount;
  ImmutableBitSet.Builder leftMask = ImmutableBitSet.builder();
  ImmutableBitSet.Builder rightMask = ImmutableBitSet.builder();
  RelNode left = joinRel.getInputs().get(0);
  RelNode right = joinRel.getInputs().get(1);

  RelMdUtil.setLeftRightBitmaps(
      groupKey,
      leftMask,
      rightMask,
      left.getRowType().getFieldCount());

  // determine which filters apply to the left vs right
  RexNode leftPred = null;
  RexNode rightPred = null;
  if (predicate != null) {
    final List<RexNode> leftFilters = new ArrayList<>();
    final List<RexNode> rightFilters = new ArrayList<>();
    final List<RexNode> joinFilters = new ArrayList<>();
    final List<RexNode> predList = RelOptUtil.conjunctions(predicate);

    RelOptUtil.classifyFilters(
        joinRel,
        predList,
        joinType,
        !joinType.isOuterJoin(),
        !joinType.generatesNullsOnLeft(),
        !joinType.generatesNullsOnRight(),
        joinFilters,
        leftFilters,
        rightFilters);

    RexBuilder rexBuilder = joinRel.getCluster().getRexBuilder();
    leftPred =
        RexUtil.composeConjunction(rexBuilder, leftFilters, true);
    rightPred =
        RexUtil.composeConjunction(rexBuilder, rightFilters, true);
  }

  if (useMaxNdv) {
    distRowCount = Math.max(
        mq.getDistinctRowCount(left, leftMask.build(), leftPred),
        mq.getDistinctRowCount(right, rightMask.build(), rightPred));
  } else {
    distRowCount =
      NumberUtil.multiply(
          mq.getDistinctRowCount(left, leftMask.build(), leftPred),
          mq.getDistinctRowCount(right, rightMask.build(), rightPred));
  }

  return RelMdUtil.numDistinctVals(distRowCount, mq.getRowCount(joinRel));
}
Example 6
Source File: FilterCorrelateRule.java    From calcite with Apache License 2.0 4 votes vote down vote up 
public void onMatch(RelOptRuleCall call) {
  final Filter filter = call.rel(0);
  final Correlate corr = call.rel(1);

  final List<RexNode> aboveFilters =
      RelOptUtil.conjunctions(filter.getCondition());

  final List<RexNode> leftFilters = new ArrayList<>();
  final List<RexNode> rightFilters = new ArrayList<>();

  // Try to push down above filters. These are typically where clause
  // filters. They can be pushed down if they are not on the NULL
  // generating side.
  RelOptUtil.classifyFilters(
      corr,
      aboveFilters,
      corr.getJoinType(),
      false,
      true,
      !corr.getJoinType().generatesNullsOnRight(),
      aboveFilters,
      leftFilters,
      rightFilters);

  if (leftFilters.isEmpty()
      && rightFilters.isEmpty()) {
    // no filters got pushed
    return;
  }

  // Create Filters on top of the children if any filters were
  // pushed to them.
  final RexBuilder rexBuilder = corr.getCluster().getRexBuilder();
  final RelBuilder relBuilder = call.builder();
  final RelNode leftRel =
      relBuilder.push(corr.getLeft()).filter(leftFilters).build();
  final RelNode rightRel =
      relBuilder.push(corr.getRight()).filter(rightFilters).build();

  // Create the new Correlate
  RelNode newCorrRel =
      corr.copy(corr.getTraitSet(), ImmutableList.of(leftRel, rightRel));

  call.getPlanner().onCopy(corr, newCorrRel);

  if (!leftFilters.isEmpty()) {
    call.getPlanner().onCopy(filter, leftRel);
  }
  if (!rightFilters.isEmpty()) {
    call.getPlanner().onCopy(filter, rightRel);
  }

  // Create a Filter on top of the join if needed
  relBuilder.push(newCorrRel);
  relBuilder.filter(
      RexUtil.fixUp(rexBuilder, aboveFilters,
          RelOptUtil.getFieldTypeList(relBuilder.peek().getRowType())));

  call.transformTo(relBuilder.build());
}