org.apache.calcite.rel.rules.FilterJoinRule Java Examples

public void registerAbstractRelationalRules() {
  addRule(FilterJoinRule.FILTER_ON_JOIN);
  addRule(FilterJoinRule.JOIN);
  addRule(AbstractConverter.ExpandConversionRule.INSTANCE);
  addRule(JoinCommuteRule.INSTANCE);
  addRule(SemiJoinRule.PROJECT);
  addRule(SemiJoinRule.JOIN);
  if (CalciteSystemProperty.COMMUTE.value()) {
    addRule(JoinAssociateRule.INSTANCE);
  }
  addRule(AggregateRemoveRule.INSTANCE);
  addRule(UnionToDistinctRule.INSTANCE);
  addRule(ProjectRemoveRule.INSTANCE);
  addRule(AggregateJoinTransposeRule.INSTANCE);
  addRule(AggregateProjectMergeRule.INSTANCE);
  addRule(CalcRemoveRule.INSTANCE);
  addRule(SortRemoveRule.INSTANCE);

  // todo: rule which makes Project({OrdinalRef}) disappear
}
 

/**
 * Converts a relational expression to a form where
 * {@link org.apache.calcite.rel.logical.LogicalJoin}s are
 * as close to leaves as possible.
 */
public static RelNode toLeafJoinForm(RelNode rel) {
  final Program program = Programs.hep(
      ImmutableList.of(
          JoinProjectTransposeRule.RIGHT_PROJECT,
          JoinProjectTransposeRule.LEFT_PROJECT,
          FilterJoinRule.FilterIntoJoinRule.FILTER_ON_JOIN,
          ProjectRemoveRule.INSTANCE,
          ProjectMergeRule.INSTANCE),
      false,
      DefaultRelMetadataProvider.INSTANCE);
  if (CalciteSystemProperty.DEBUG.value()) {
    System.out.println(
        RelOptUtil.dumpPlan("before", rel, SqlExplainFormat.TEXT,
            SqlExplainLevel.DIGEST_ATTRIBUTES));
  }
  final RelNode rel2 = program.run(null, rel, null,
      ImmutableList.of(),
      ImmutableList.of());
  if (CalciteSystemProperty.DEBUG.value()) {
    System.out.println(
        RelOptUtil.dumpPlan("after", rel2, SqlExplainFormat.TEXT,
            SqlExplainLevel.DIGEST_ATTRIBUTES));
  }
  return rel2;
}
 

private RelOptPlanner getVolcanoPlanner(RelNode root) {
  final RelBuilderFactory builderFactory =
      RelBuilder.proto(PigRelFactories.ALL_PIG_REL_FACTORIES);
  final RelOptPlanner planner = root.getCluster().getPlanner(); // VolcanoPlanner
  for (RelOptRule r : PigRules.ALL_PIG_OPT_RULES) {
    planner.addRule(r);
  }
  planner.removeRule(FilterAggregateTransposeRule.INSTANCE);
  planner.removeRule(FilterJoinRule.FILTER_ON_JOIN);
  planner.addRule(
      new FilterAggregateTransposeRule(PigFilter.class, builderFactory,
          PigAggregate.class));
  planner.addRule(
      new FilterIntoJoinRule(true, builderFactory, TRUE_PREDICATE));
  planner.setRoot(root);
  return planner;
}
 

/**
 * Converts a relational expression to a form where
 * {@link org.apache.calcite.rel.logical.LogicalJoin}s are
 * as close to leaves as possible.
 */
public static RelNode toLeafJoinForm(RelNode rel) {
  final Program program = Programs.hep(
      ImmutableList.of(
          JoinProjectTransposeRule.RIGHT_PROJECT,
          JoinProjectTransposeRule.LEFT_PROJECT,
          FilterJoinRule.FilterIntoJoinRule.FILTER_ON_JOIN,
          ProjectRemoveRule.INSTANCE,
          ProjectMergeRule.INSTANCE),
      false,
      DefaultRelMetadataProvider.INSTANCE);
  if (CalciteSystemProperty.DEBUG.value()) {
    System.out.println(
        RelOptUtil.dumpPlan("before", rel, SqlExplainFormat.TEXT,
            SqlExplainLevel.DIGEST_ATTRIBUTES));
  }
  final RelNode rel2 = program.run(null, rel, null,
      ImmutableList.of(),
      ImmutableList.of());
  if (CalciteSystemProperty.DEBUG.value()) {
    System.out.println(
        RelOptUtil.dumpPlan("after", rel2, SqlExplainFormat.TEXT,
            SqlExplainLevel.DIGEST_ATTRIBUTES));
  }
  return rel2;
}
 

private RelNode canonicalize(RelNode rel) {
  HepProgram program =
      new HepProgramBuilder()
          .addRuleInstance(FilterProjectTransposeRule.INSTANCE)
          .addRuleInstance(FilterMergeRule.INSTANCE)
          .addRuleInstance(FilterJoinRule.FILTER_ON_JOIN)
          .addRuleInstance(FilterJoinRule.JOIN)
          .addRuleInstance(FilterAggregateTransposeRule.INSTANCE)
          .addRuleInstance(ProjectMergeRule.INSTANCE)
          .addRuleInstance(ProjectRemoveRule.INSTANCE)
          .addRuleInstance(ProjectJoinTransposeRule.INSTANCE)
          .addRuleInstance(ProjectSetOpTransposeRule.INSTANCE)
          .addRuleInstance(FilterToCalcRule.INSTANCE)
          .addRuleInstance(ProjectToCalcRule.INSTANCE)
          .addRuleInstance(FilterCalcMergeRule.INSTANCE)
          .addRuleInstance(ProjectCalcMergeRule.INSTANCE)
          .addRuleInstance(CalcMergeRule.INSTANCE)
          .build();
  final HepPlanner hepPlanner = new HepPlanner(program);
  hepPlanner.setRoot(rel);
  return hepPlanner.findBestExp();
}
 

private RelNode decorrelate(RelNode root) {
    // first adjust count() expression if any
    final RelBuilderFactory f = relBuilderFactory();
    HepProgram program = HepProgram.builder().addRuleInstance(new AdjustProjectForCountAggregateRule(false, f))
            .addRuleInstance(new AdjustProjectForCountAggregateRule(true, f))
            .addRuleInstance(new FilterJoinRule.FilterIntoJoinRule(true, f, FilterJoinRule.TRUE_PREDICATE))
            .addRuleInstance(new FilterProjectTransposeRule(Filter.class, Project.class, true, true, f))
            .addRuleInstance(new FilterCorrelateRule(f)).build();

    HepPlanner planner = createPlanner(program);

    planner.setRoot(root);
    root = planner.findBestExp();

    // Perform decorrelation.
    map.clear();

    final Frame frame = getInvoke(root, null);
    if (frame != null) {
        // has been rewritten; apply rules post-decorrelation
        final HepProgram program2 = HepProgram.builder()
                .addRuleInstance(new FilterJoinRule.FilterIntoJoinRule(true, f, FilterJoinRule.TRUE_PREDICATE))
                .addRuleInstance(new FilterJoinRule.JoinConditionPushRule(f, FilterJoinRule.TRUE_PREDICATE))
                .build();

        final HepPlanner planner2 = createPlanner(program2);
        final RelNode newRoot = frame.r;
        planner2.setRoot(newRoot);
        return planner2.findBestExp();
    }

    return root;
}
 

/** Creates a program that invokes heuristic join-order optimization
 * (via {@link org.apache.calcite.rel.rules.JoinToMultiJoinRule},
 * {@link org.apache.calcite.rel.rules.MultiJoin} and
 * {@link org.apache.calcite.rel.rules.LoptOptimizeJoinRule})
 * if there are 6 or more joins (7 or more relations). */
public static Program heuristicJoinOrder(
    final Iterable<? extends RelOptRule> rules,
    final boolean bushy, final int minJoinCount) {
  return (planner, rel, requiredOutputTraits, materializations, lattices) -> {
    final int joinCount = RelOptUtil.countJoins(rel);
    final Program program;
    if (joinCount < minJoinCount) {
      program = ofRules(rules);
    } else {
      // Create a program that gathers together joins as a MultiJoin.
      final HepProgram hep = new HepProgramBuilder()
          .addRuleInstance(FilterJoinRule.FILTER_ON_JOIN)
          .addMatchOrder(HepMatchOrder.BOTTOM_UP)
          .addRuleInstance(JoinToMultiJoinRule.INSTANCE)
          .build();
      final Program program1 =
          of(hep, false, DefaultRelMetadataProvider.INSTANCE);

      // Create a program that contains a rule to expand a MultiJoin
      // into heuristically ordered joins.
      // We use the rule set passed in, but remove JoinCommuteRule and
      // JoinPushThroughJoinRule, because they cause exhaustive search.
      final List<RelOptRule> list = Lists.newArrayList(rules);
      list.removeAll(
          ImmutableList.of(JoinCommuteRule.INSTANCE,
              JoinAssociateRule.INSTANCE,
              JoinPushThroughJoinRule.LEFT,
              JoinPushThroughJoinRule.RIGHT));
      list.add(bushy
          ? MultiJoinOptimizeBushyRule.INSTANCE
          : LoptOptimizeJoinRule.INSTANCE);
      final Program program2 = ofRules(list);

      program = sequence(program1, program2);
    }
    return program.run(
        planner, rel, requiredOutputTraits, materializations, lattices);
  };
}
 

/** Creates a program that invokes heuristic join-order optimization
 * (via {@link org.apache.calcite.rel.rules.JoinToMultiJoinRule},
 * {@link org.apache.calcite.rel.rules.MultiJoin} and
 * {@link org.apache.calcite.rel.rules.LoptOptimizeJoinRule})
 * if there are 6 or more joins (7 or more relations). */
public static Program heuristicJoinOrder(
    final Iterable<? extends RelOptRule> rules,
    final boolean bushy, final int minJoinCount) {
  return (planner, rel, requiredOutputTraits, materializations, lattices) -> {
    final int joinCount = RelOptUtil.countJoins(rel);
    final Program program;
    if (joinCount < minJoinCount) {
      program = ofRules(rules);
    } else {
      // Create a program that gathers together joins as a MultiJoin.
      final HepProgram hep = new HepProgramBuilder()
          .addRuleInstance(FilterJoinRule.FILTER_ON_JOIN)
          .addMatchOrder(HepMatchOrder.BOTTOM_UP)
          .addRuleInstance(JoinToMultiJoinRule.INSTANCE)
          .build();
      final Program program1 =
          of(hep, false, DefaultRelMetadataProvider.INSTANCE);

      // Create a program that contains a rule to expand a MultiJoin
      // into heuristically ordered joins.
      // We use the rule set passed in, but remove JoinCommuteRule and
      // JoinPushThroughJoinRule, because they cause exhaustive search.
      final List<RelOptRule> list = Lists.newArrayList(rules);
      list.removeAll(
          ImmutableList.of(JoinCommuteRule.INSTANCE,
              JoinAssociateRule.INSTANCE,
              JoinPushThroughJoinRule.LEFT,
              JoinPushThroughJoinRule.RIGHT));
      list.add(bushy
          ? MultiJoinOptimizeBushyRule.INSTANCE
          : LoptOptimizeJoinRule.INSTANCE);
      final Program program2 = ofRules(list);

      program = sequence(program1, program2);
    }
    return program.run(
        planner, rel, requiredOutputTraits, materializations, lattices);
  };
}
 

@Test void testConvertSemiJoin() {
  final String sql = "select * from dept where exists (\n"
      + "  select * from emp\n"
      + "  where emp.deptno = dept.deptno\n"
      + "  and emp.sal > 100)";
  checkConvertMutableRel(
      "Join", // with join type as semi
      sql,
      true,
      ImmutableList.of(
          FilterProjectTransposeRule.INSTANCE,
          FilterJoinRule.FILTER_ON_JOIN,
          ProjectMergeRule.INSTANCE,
          SemiJoinRule.PROJECT));
}
 

@Override
public void register(RelOptPlanner planner) {
    // force clear the query context before traversal relational operators
    OLAPContext.clearThreadLocalContexts();

    // register OLAP rules
    addRules(planner, kylinConfig.getCalciteAddRule());

    planner.addRule(OLAPToEnumerableConverterRule.INSTANCE);
    planner.addRule(OLAPFilterRule.INSTANCE);
    planner.addRule(OLAPProjectRule.INSTANCE);
    planner.addRule(OLAPAggregateRule.INSTANCE);
    planner.addRule(OLAPJoinRule.INSTANCE);
    planner.addRule(OLAPLimitRule.INSTANCE);
    planner.addRule(OLAPSortRule.INSTANCE);
    planner.addRule(OLAPUnionRule.INSTANCE);
    planner.addRule(OLAPWindowRule.INSTANCE);
    planner.addRule(OLAPValuesRule.INSTANCE);

    planner.addRule(AggregateProjectReduceRule.INSTANCE);

    // CalcitePrepareImpl.CONSTANT_REDUCTION_RULES
    if (kylinConfig.isReduceExpressionsRulesEnabled()) {
        planner.addRule(ReduceExpressionsRule.PROJECT_INSTANCE);
        planner.addRule(ReduceExpressionsRule.FILTER_INSTANCE);
        planner.addRule(ReduceExpressionsRule.CALC_INSTANCE);
        planner.addRule(ReduceExpressionsRule.JOIN_INSTANCE);
    }
    // the ValuesReduceRule breaks query test somehow...
    //        planner.addRule(ValuesReduceRule.FILTER_INSTANCE);
    //        planner.addRule(ValuesReduceRule.PROJECT_FILTER_INSTANCE);
    //        planner.addRule(ValuesReduceRule.PROJECT_INSTANCE);

    removeRules(planner, kylinConfig.getCalciteRemoveRule());
    if (!kylinConfig.isEnumerableRulesEnabled()) {
        for (RelOptRule rule : CalcitePrepareImpl.ENUMERABLE_RULES) {
            planner.removeRule(rule);
        }
    }
    // since join is the entry point, we can't push filter past join
    planner.removeRule(FilterJoinRule.FILTER_ON_JOIN);
    planner.removeRule(FilterJoinRule.JOIN);

    // since we don't have statistic of table, the optimization of join is too cost
    planner.removeRule(JoinCommuteRule.INSTANCE);
    planner.removeRule(JoinPushThroughJoinRule.LEFT);
    planner.removeRule(JoinPushThroughJoinRule.RIGHT);

    // keep tree structure like filter -> aggregation -> project -> join/table scan, implementOLAP() rely on this tree pattern
    planner.removeRule(AggregateJoinTransposeRule.INSTANCE);
    planner.removeRule(AggregateProjectMergeRule.INSTANCE);
    planner.removeRule(FilterProjectTransposeRule.INSTANCE);
    planner.removeRule(SortJoinTransposeRule.INSTANCE);
    planner.removeRule(JoinPushExpressionsRule.INSTANCE);
    planner.removeRule(SortUnionTransposeRule.INSTANCE);
    planner.removeRule(JoinUnionTransposeRule.LEFT_UNION);
    planner.removeRule(JoinUnionTransposeRule.RIGHT_UNION);
    planner.removeRule(AggregateUnionTransposeRule.INSTANCE);
    planner.removeRule(DateRangeRules.FILTER_INSTANCE);
    planner.removeRule(SemiJoinRule.JOIN);
    planner.removeRule(SemiJoinRule.PROJECT);
    // distinct count will be split into a separated query that is joined with the left query
    planner.removeRule(AggregateExpandDistinctAggregatesRule.INSTANCE);

    // see Dec 26th email @ http://mail-archives.apache.org/mod_mbox/calcite-dev/201412.mbox/browser
    planner.removeRule(ExpandConversionRule.INSTANCE);
}
 

@Override
public void register(RelOptPlanner planner) {
    // force clear the query context before traversal relational operators
    OLAPContext.clearThreadLocalContexts();

    // register OLAP rules
    addRules(planner, kylinConfig.getCalciteAddRule());

    planner.addRule(OLAPToEnumerableConverterRule.INSTANCE);
    planner.addRule(OLAPFilterRule.INSTANCE);
    planner.addRule(OLAPProjectRule.INSTANCE);
    planner.addRule(OLAPAggregateRule.INSTANCE);
    planner.addRule(OLAPJoinRule.INSTANCE);
    planner.addRule(OLAPLimitRule.INSTANCE);
    planner.addRule(OLAPSortRule.INSTANCE);
    planner.addRule(OLAPUnionRule.INSTANCE);
    planner.addRule(OLAPWindowRule.INSTANCE);
    planner.addRule(OLAPValuesRule.INSTANCE);

    planner.addRule(AggregateProjectReduceRule.INSTANCE);

    // CalcitePrepareImpl.CONSTANT_REDUCTION_RULES
    if (kylinConfig.isReduceExpressionsRulesEnabled()) {
        planner.addRule(ReduceExpressionsRule.PROJECT_INSTANCE);
        planner.addRule(ReduceExpressionsRule.FILTER_INSTANCE);
        planner.addRule(ReduceExpressionsRule.CALC_INSTANCE);
        planner.addRule(ReduceExpressionsRule.JOIN_INSTANCE);
    }
    // the ValuesReduceRule breaks query test somehow...
    //        planner.addRule(ValuesReduceRule.FILTER_INSTANCE);
    //        planner.addRule(ValuesReduceRule.PROJECT_FILTER_INSTANCE);
    //        planner.addRule(ValuesReduceRule.PROJECT_INSTANCE);

    removeRules(planner, kylinConfig.getCalciteRemoveRule());
    if (!kylinConfig.isEnumerableRulesEnabled()) {
        for (RelOptRule rule : CalcitePrepareImpl.ENUMERABLE_RULES) {
            planner.removeRule(rule);
        }
    }
    // since join is the entry point, we can't push filter past join
    planner.removeRule(FilterJoinRule.FILTER_ON_JOIN);
    planner.removeRule(FilterJoinRule.JOIN);

    // since we don't have statistic of table, the optimization of join is too cost
    planner.removeRule(JoinCommuteRule.INSTANCE);
    planner.removeRule(JoinPushThroughJoinRule.LEFT);
    planner.removeRule(JoinPushThroughJoinRule.RIGHT);

    // keep tree structure like filter -> aggregation -> project -> join/table scan, implementOLAP() rely on this tree pattern
    planner.removeRule(AggregateJoinTransposeRule.INSTANCE);
    planner.removeRule(AggregateProjectMergeRule.INSTANCE);
    planner.removeRule(FilterProjectTransposeRule.INSTANCE);
    planner.removeRule(SortJoinTransposeRule.INSTANCE);
    planner.removeRule(JoinPushExpressionsRule.INSTANCE);
    planner.removeRule(SortUnionTransposeRule.INSTANCE);
    planner.removeRule(JoinUnionTransposeRule.LEFT_UNION);
    planner.removeRule(JoinUnionTransposeRule.RIGHT_UNION);
    planner.removeRule(AggregateUnionTransposeRule.INSTANCE);
    planner.removeRule(DateRangeRules.FILTER_INSTANCE);
    planner.removeRule(SemiJoinRule.JOIN);
    planner.removeRule(SemiJoinRule.PROJECT);
    // distinct count will be split into a separated query that is joined with the left query
    planner.removeRule(AggregateExpandDistinctAggregatesRule.INSTANCE);

    // see Dec 26th email @ http://mail-archives.apache.org/mod_mbox/calcite-dev/201412.mbox/browser
    planner.removeRule(ExpandConversionRule.INSTANCE);

    // KYLIN-4464 do not pushdown sort when there is a window function in projection
    planner.removeRule(SortProjectTransposeRule.INSTANCE);
    planner.addRule(KylinSortProjectTransposeRule.INSTANCE);
}
 

protected RelNode decorrelate(RelNode root) {
  // first adjust count() expression if any
  final RelBuilderFactory f = relBuilderFactory();
  HepProgram program = HepProgram.builder()
      .addRuleInstance(new AdjustProjectForCountAggregateRule(false, f))
      .addRuleInstance(new AdjustProjectForCountAggregateRule(true, f))
      .addRuleInstance(
          new FilterJoinRule.FilterIntoJoinRule(true, f,
              FilterJoinRule.TRUE_PREDICATE))
      .addRuleInstance(
          new FilterProjectTransposeRule(Filter.class, Project.class, true,
              true, f))
      .addRuleInstance(new FilterCorrelateRule(f))
      .build();

  HepPlanner planner = createPlanner(program);

  planner.setRoot(root);
  root = planner.findBestExp();

  // Perform decorrelation.
  map.clear();

  final Frame frame = getInvoke(root, null);
  if (frame != null) {
    // has been rewritten; apply rules post-decorrelation
    final HepProgram program2 = HepProgram.builder()
        .addRuleInstance(
            new FilterJoinRule.FilterIntoJoinRule(
                true, f,
                FilterJoinRule.TRUE_PREDICATE))
        .addRuleInstance(
            new FilterJoinRule.JoinConditionPushRule(
                f,
                FilterJoinRule.TRUE_PREDICATE))
        .build();

    final HepPlanner planner2 = createPlanner(program2);
    final RelNode newRoot = frame.r;
    planner2.setRoot(newRoot);
    return planner2.findBestExp();
  }

  return root;
}
 

private static List<RelNode> substitute(
    RelNode root, RelOptMaterialization materialization) {
  // First, if the materialization is in terms of a star table, rewrite
  // the query in terms of the star table.
  if (materialization.starTable != null) {
    RelNode newRoot = RelOptMaterialization.tryUseStar(root,
        materialization.starRelOptTable);
    if (newRoot != null) {
      root = newRoot;
    }
  }

  // Push filters to the bottom, and combine projects on top.
  RelNode target = materialization.queryRel;
  // try to trim unused field in relational expressions.
  root = trimUnusedfields(root);
  target = trimUnusedfields(target);
  HepProgram program =
      new HepProgramBuilder()
          .addRuleInstance(FilterProjectTransposeRule.INSTANCE)
          .addRuleInstance(FilterMergeRule.INSTANCE)
          .addRuleInstance(FilterJoinRule.FILTER_ON_JOIN)
          .addRuleInstance(FilterJoinRule.JOIN)
          .addRuleInstance(FilterAggregateTransposeRule.INSTANCE)
          .addRuleInstance(ProjectMergeRule.INSTANCE)
          .addRuleInstance(ProjectRemoveRule.INSTANCE)
          .addRuleInstance(ProjectJoinTransposeRule.INSTANCE)
          .addRuleInstance(ProjectSetOpTransposeRule.INSTANCE)
          .addRuleInstance(FilterToCalcRule.INSTANCE)
          .addRuleInstance(ProjectToCalcRule.INSTANCE)
          .addRuleInstance(FilterCalcMergeRule.INSTANCE)
          .addRuleInstance(ProjectCalcMergeRule.INSTANCE)
          .addRuleInstance(CalcMergeRule.INSTANCE)
          .build();

  // We must use the same HEP planner for the two optimizations below.
  // Thus different nodes with the same digest will share the same vertex in
  // the plan graph. This is important for the matching process.
  final HepPlanner hepPlanner = new HepPlanner(program);
  hepPlanner.setRoot(target);
  target = hepPlanner.findBestExp();

  hepPlanner.setRoot(root);
  root = hepPlanner.findBestExp();

  return new SubstitutionVisitor(target, root).go(materialization.tableRel);
}