Java Code Examples for org.apache.calcite.rel.type.RelDataType#getField()

The following examples show how to use org.apache.calcite.rel.type.RelDataType#getField() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Bats   File: RuntimeFilterVisitor.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Find a join condition's left input source scan Prel. If we can't find a target scan Prel then this
 * RuntimeFilter can not pushed down to a probe side scan Prel.
 *
 * @param fieldName   left join condition field Name
 * @param leftRelNode left RelNode of a BiRel or the SingleRel
 * @return a left scan Prel which contains the left join condition name or null
 */
private ScanPrel findLeftScanPrel(String fieldName, RelNode leftRelNode) {
  if (leftRelNode instanceof ScanPrel) {
    RelDataType scanRowType = leftRelNode.getRowType();
    RelDataTypeField field = scanRowType.getField(fieldName, true, true);
    if (field != null) {
      //found
      return (ScanPrel) leftRelNode;
    } else {
      return null;
    }
  } else if (leftRelNode instanceof RelSubset) {
    RelNode bestNode = ((RelSubset) leftRelNode).getBest();
    if (bestNode != null) {
      return findLeftScanPrel(fieldName, bestNode);
    } else {
      return null;
    }
  } else {
    List<RelNode> relNodes = leftRelNode.getInputs();
    RelNode leftNode = relNodes.get(0);
    return findLeftScanPrel(fieldName, leftNode);
  }
}
 
Example 2
@Override
public RelNode visit(LogicalProject project) {
  RelNode input = project.getInput().accept(this);
  RelDataType incomingRowType = input.getRowType();
  List<RexNode> newProjects;
  RelDataTypeField modField = incomingRowType.getField(UPDATE_COLUMN, false, false);
  if (modField == null) {
    return project;
  }
  newProjects = FluentIterable.from(project.getProjects())
    .append(new RexInputRef(modField.getIndex(), modField.getType()))
    .toList();
  FieldInfoBuilder fieldInfoBuilder = new FieldInfoBuilder(project.getCluster().getTypeFactory());
  for (RelDataTypeField field : project.getRowType().getFieldList()) {
    fieldInfoBuilder.add(field);
  }
  fieldInfoBuilder.add(UPDATE_COLUMN, modField.getType());
  return new LogicalProject(
    project.getCluster(),
    project.getTraitSet(),
    input,
    newProjects,
    fieldInfoBuilder.build()
  );
}
 
Example 3
@Override
public RelNode visit(LogicalAggregate aggregate) {
  RelNode input = aggregate.getInput().accept(this);
  RelDataType incomingRowType = input.getRowType();
  RelDataTypeField modField = incomingRowType.getField(UPDATE_COLUMN, false, false);
  if (modField == null) {
    return aggregate;
  }

  final AggregateCall aggCall = AggregateCall.create(SqlStdOperatorTable.MAX, false, ImmutableList.of(modField.getIndex()), -1, modField.getType(), UPDATE_COLUMN);
  final List<AggregateCall> aggCalls = FluentIterable.from(aggregate.getAggCallList())
    .append(aggCall)
    .toList();
  return aggregate.copy(
    aggregate.getTraitSet(),
    input,
    aggregate.indicator,
    aggregate.getGroupSet(),
    null,
    aggCalls
  );
}
 
Example 4
Source Project: Bats   File: SqlValidatorUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Gets a map of indexes from the source to fields in the target for the
 * intersecting set of source and target fields.
 *
 * @param sourceFields The source of column names that determine indexes
 * @param targetFields The target fields to be indexed
 */
public static ImmutableMap<Integer, RelDataTypeField> getIndexToFieldMap(
    List<RelDataTypeField> sourceFields,
    RelDataType targetFields) {
  final ImmutableMap.Builder<Integer, RelDataTypeField> output =
      ImmutableMap.builder();
  for (final RelDataTypeField source : sourceFields) {
    final RelDataTypeField target = targetFields.getField(source.getName(), true, false);
    if (target != null) {
      output.put(source.getIndex(), target);
    }
  }
  return output.build();
}
 
Example 5
Source Project: Bats   File: SqlValidatorUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static RelDataType createTypeFromProjection(RelDataType type,
    List<String> columnNameList, RelDataTypeFactory typeFactory,
    boolean caseSensitive) {
  // If the names in columnNameList and type have case-sensitive differences,
  // the resulting type will use those from type. These are presumably more
  // canonical.
  final List<RelDataTypeField> fields =
      new ArrayList<>(columnNameList.size());
  for (String name : columnNameList) {
    RelDataTypeField field = type.getField(name, caseSensitive, false);
    fields.add(type.getFieldList().get(field.getIndex()));
  }
  return typeFactory.createStructType(fields);
}
 
Example 6
Source Project: Bats   File: DrillWriterRel.java    License: Apache License 2.0 5 votes vote down vote up
private List<Integer> resolvePartitionKeys(){
  final List<Integer> keys = Lists.newArrayList();
  final RelDataType inputRowType = getInput().getRowType();
  final List<String> partitionCol = getCreateTableEntry().getPartitionColumns();

  for (final String col : partitionCol) {
    final RelDataTypeField field = inputRowType.getField(col, false, false);
    Preconditions.checkArgument(field != null,
        String.format("partition col %s could not be resolved in table's column lists!", col));
    keys.add(field.getIndex());
  }

  return keys;
}
 
Example 7
Source Project: calcite   File: SqlValidatorUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Gets a map of indexes from the source to fields in the target for the
 * intersecting set of source and target fields.
 *
 * @param sourceFields The source of column names that determine indexes
 * @param targetFields The target fields to be indexed
 */
public static ImmutableMap<Integer, RelDataTypeField> getIndexToFieldMap(
    List<RelDataTypeField> sourceFields,
    RelDataType targetFields) {
  final ImmutableMap.Builder<Integer, RelDataTypeField> output =
      ImmutableMap.builder();
  for (final RelDataTypeField source : sourceFields) {
    final RelDataTypeField target = targetFields.getField(source.getName(), true, false);
    if (target != null) {
      output.put(source.getIndex(), target);
    }
  }
  return output.build();
}
 
Example 8
Source Project: calcite   File: SqlValidatorUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static RelDataType createTypeFromProjection(RelDataType type,
    List<String> columnNameList, RelDataTypeFactory typeFactory,
    boolean caseSensitive) {
  // If the names in columnNameList and type have case-sensitive differences,
  // the resulting type will use those from type. These are presumably more
  // canonical.
  final List<RelDataTypeField> fields =
      new ArrayList<>(columnNameList.size());
  for (String name : columnNameList) {
    RelDataTypeField field = type.getField(name, caseSensitive, false);
    fields.add(type.getFieldList().get(field.getIndex()));
  }
  return typeFactory.createStructType(fields);
}
 
Example 9
Source Project: calcite   File: SqlItemOperator.java    License: Apache License 2.0 5 votes vote down vote up
@Override public RelDataType inferReturnType(SqlOperatorBinding opBinding) {
  final RelDataTypeFactory typeFactory = opBinding.getTypeFactory();
  final RelDataType operandType = opBinding.getOperandType(0);
  switch (operandType.getSqlTypeName()) {
  case ARRAY:
    return typeFactory.createTypeWithNullability(
        operandType.getComponentType(), true);
  case MAP:
    return typeFactory.createTypeWithNullability(operandType.getValueType(),
        true);
  case ROW:
    String fieldName = opBinding.getOperandLiteralValue(1, String.class);
    RelDataTypeField field = operandType.getField(fieldName, false, false);
    if (field == null) {
      throw new AssertionError("Cannot infer type of field '"
          + fieldName + "' within ROW type: " + operandType);
    } else {
      return field.getType();
    }
  case ANY:
  case DYNAMIC_STAR:
    return typeFactory.createTypeWithNullability(
        typeFactory.createSqlType(SqlTypeName.ANY), true);
  default:
    throw new AssertionError();
  }
}
 
Example 10
Source Project: Bats   File: SqlValidatorUtil.java    License: Apache License 2.0 4 votes vote down vote up
@Deprecated // to be removed before 2.0
public static RelDataTypeField lookupField(boolean caseSensitive,
    final RelDataType rowType, String columnName) {
  return rowType.getField(columnName, caseSensitive, false);
}
 
Example 11
Source Project: Bats   File: SqlNameMatchers.java    License: Apache License 2.0 4 votes vote down vote up
public RelDataTypeField field(RelDataType rowType, String fieldName) {
  return rowType.getField(fieldName, caseSensitive, false);
}
 
Example 12
Source Project: Bats   File: RuntimeFilterVisitor.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Generate a possible RuntimeFilter of a HashJoinPrel, left some BF parameters of the generated RuntimeFilter
 * to be set later.
 *
 * @param hashJoinPrel
 * @return null or a partial information RuntimeFilterDef
 */
private RuntimeFilterDef generateRuntimeFilter(HashJoinPrel hashJoinPrel) {
  JoinRelType joinRelType = hashJoinPrel.getJoinType();
  JoinInfo joinInfo = hashJoinPrel.analyzeCondition();
  boolean allowJoin = (joinInfo.isEqui()) && (joinRelType == JoinRelType.INNER || joinRelType == JoinRelType.RIGHT);
  if (!allowJoin) {
    return null;
  }
  //TODO check whether to enable RuntimeFilter according to the NDV percent
  /**
   double threshold = 0.5;
   double percent = leftNDV / rightDNV;
   if (percent > threshold ) {
   return null;
   }
   */

  List<BloomFilterDef> bloomFilterDefs = new ArrayList<>();
  //find the possible left scan node of the left join key
  ScanPrel probeSideScanPrel = null;
  RelNode left = hashJoinPrel.getLeft();
  RelNode right = hashJoinPrel.getRight();
  ExchangePrel exchangePrel = findRightExchangePrel(right);
  if (exchangePrel == null) {
    //Does not support the single fragment mode ,that is the right build side
    //can only be BroadcastExchangePrel or HashToRandomExchangePrel
    return null;
  }
  List<String> leftFields = left.getRowType().getFieldNames();
  List<String> rightFields = right.getRowType().getFieldNames();
  List<Integer> leftKeys = hashJoinPrel.getLeftKeys();
  List<Integer> rightKeys = hashJoinPrel.getRightKeys();
  RelMetadataQuery metadataQuery = left.getCluster().getMetadataQuery();
  int i = 0;
  for (Integer leftKey : leftKeys) {
    String leftFieldName = leftFields.get(leftKey);
    Integer rightKey = rightKeys.get(i++);
    String rightFieldName = rightFields.get(rightKey);

    //This also avoids the left field of the join condition with a function call.
    ScanPrel scanPrel = findLeftScanPrel(leftFieldName, left);
    if (scanPrel != null) {
      boolean encounteredBlockNode = containBlockNode((Prel) left, scanPrel);
      if (encounteredBlockNode) {
        continue;
      }
      //Collect NDV from the Metadata
      RelDataType scanRowType = scanPrel.getRowType();
      RelDataTypeField field = scanRowType.getField(leftFieldName, true, true);
      int index = field.getIndex();
      Double ndv = metadataQuery.getDistinctRowCount(scanPrel, ImmutableBitSet.of(index), null);
      if (ndv == null) {
        //If NDV is not supplied, we use the row count to estimate the ndv.
        ndv = left.estimateRowCount(metadataQuery) * 0.1;
      }
      int bloomFilterSizeInBytes = BloomFilter.optimalNumOfBytes(ndv.longValue(), fpp);
      bloomFilterSizeInBytes = bloomFilterSizeInBytes > bloomFilterMaxSizeInBytesDef ? bloomFilterMaxSizeInBytesDef : bloomFilterSizeInBytes;
      //left the local parameter to be set later.
      BloomFilterDef bloomFilterDef = new BloomFilterDef(bloomFilterSizeInBytes, false, leftFieldName, rightFieldName);
      bloomFilterDef.setLeftNDV(ndv);
      bloomFilterDefs.add(bloomFilterDef);
      toAddRuntimeFilter.add(scanPrel);
      probeSideScanPrel = scanPrel;
    }
  }
  if (bloomFilterDefs.size() > 0) {
    //left sendToForeman parameter to be set later.
    RuntimeFilterDef runtimeFilterDef = new RuntimeFilterDef(true, false, bloomFilterDefs, false, -1);
    probeSideScan2hj.put(probeSideScanPrel, hashJoinPrel);
    return runtimeFilterDef;
  }
  return null;
}
 
Example 13
Source Project: dremio-oss   File: DremioCatalogReader.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public RelDataTypeField field(RelDataType rowType, String columnName) {
  return rowType.getField(columnName, false, false);
}
 
Example 14
@Override
public RelNode visit(LogicalAggregate aggregate) {
  RelNode input = aggregate.getInput().accept(this);

  // Create a new project with null UPDATE_COLUMN below aggregate
  final RelBuilder relBuilder = newCalciteRelBuilderWithoutContext(aggregate.getCluster());
  relBuilder.push(input);
  List<RexNode> nodes = input.getRowType().getFieldList().stream().map(q -> {
    if (UPDATE_COLUMN.equals(q.getName())) {
      return relBuilder.getRexBuilder().makeNullLiteral(q.getType());
    } else{
      return relBuilder.getRexBuilder().makeInputRef(q.getType(), q.getIndex());
    }
  }).collect(Collectors.toList());
  relBuilder.project(nodes, input.getRowType().getFieldNames());

  // create a new aggregate with null UPDATE_COLUMN in groupSet
  RelDataType incomingRowType = relBuilder.peek().getRowType();
  RelDataTypeField modField = incomingRowType.getField(UPDATE_COLUMN, false, false);
  ImmutableBitSet newGroupSet = aggregate.getGroupSet().rebuild().set(modField.getIndex()).build();
  GroupKey groupKey = relBuilder.groupKey(newGroupSet, aggregate.indicator, null);

  final int groupCount = aggregate.getGroupCount();
  final Pointer<Integer> ind = new Pointer<>(groupCount-1);
  final List<String> fieldNames = aggregate.getRowType().getFieldNames();
  final List<AggregateCall> aggCalls = aggregate.getAggCallList().stream().map(q -> {
    ind.value++;
    if (q.getName() == null) {
      return q.rename(fieldNames.get(ind.value));
    }
    return q;
  }).collect(Collectors.toList());

  relBuilder.aggregate(groupKey, aggCalls);

  // create a new project on top to preserve rowType
  Iterable<RexInputRef> projects = FluentIterable.from(aggregate.getRowType().getFieldNames())
    .transform(new Function<String, RexInputRef>() {
      @Override
      public RexInputRef apply(String fieldName) {
        return relBuilder.field(fieldName);
      }
    })
    .append(relBuilder.field(UPDATE_COLUMN));

  relBuilder.project(projects);

  return relBuilder.build();
}
 
Example 15
Source Project: flink   File: MergeTableLikeUtil.java    License: Apache License 2.0 4 votes vote down vote up
private void verifyRowtimeAttribute(
		Map<FeatureOption, MergingStrategy> mergingStrategies,
		SqlIdentifier eventTimeColumnName,
		Map<String, RelDataType> allFieldsTypes) {
	String fullRowtimeExpression = eventTimeColumnName.toString();
	boolean specAlreadyExists = watermarkSpecs.containsKey(fullRowtimeExpression);

	if (specAlreadyExists &&
		mergingStrategies.get(FeatureOption.WATERMARKS) != MergingStrategy.OVERWRITING) {
		throw new ValidationException(String.format(
			"There already exists a watermark spec for column '%s' in the base table. You " +
				"might want to specify EXCLUDING WATERMARKS or OVERWRITING WATERMARKS.",
			fullRowtimeExpression));
	}

	List<String> components = eventTimeColumnName.names;
	if (!allFieldsTypes.containsKey(components.get(0))) {
		throw new ValidationException(
			String.format(
				"The rowtime attribute field '%s' is not defined in the table schema, at %s\n" +
					"Available fields: [%s]",
				fullRowtimeExpression,
				eventTimeColumnName.getParserPosition(),
				allFieldsTypes.keySet().stream().collect(Collectors.joining("', '", "'", "'"))
			));
	}

	if (components.size() > 1) {
		RelDataType componentType = allFieldsTypes.get(components.get(0));
		for (int i = 1; i < components.size(); i++) {
			RelDataTypeField field = componentType.getField(components.get(i), true, false);
			if (field == null) {
				throw new ValidationException(
					String.format(
						"The rowtime attribute field '%s' is not defined in the table schema, at %s\n" +
							"Nested field '%s' was not found in a composite type: %s.",
						fullRowtimeExpression,
						eventTimeColumnName.getComponent(i).getParserPosition(),
						components.get(i),
						FlinkTypeFactory.toLogicalType(allFieldsTypes.get(components.get(0))))
					);
			}
			componentType = field.getType();
		}
	}
}
 
Example 16
Source Project: calcite   File: SqlValidatorUtil.java    License: Apache License 2.0 4 votes vote down vote up
@Deprecated // to be removed before 2.0
public static RelDataTypeField lookupField(boolean caseSensitive,
    final RelDataType rowType, String columnName) {
  return rowType.getField(columnName, caseSensitive, false);
}
 
Example 17
Source Project: calcite   File: SqlNameMatchers.java    License: Apache License 2.0 4 votes vote down vote up
public RelDataTypeField field(RelDataType rowType, String fieldName) {
  return rowType.getField(fieldName, caseSensitive, false);
}
 
Example 18
Source Project: Bats   File: RexBuilder.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Creates an expression accessing a given named field from a record.
 *
 * <p>NOTE: Be careful choosing the value of {@code caseSensitive}.
 * If the field name was supplied by an end-user (e.g. as a column alias in
 * SQL), use your session's case-sensitivity setting.
 * Only hard-code {@code true} if you are sure that the field name is
 * internally generated.
 * Hard-coding {@code false} is almost certainly wrong.</p>
 *
 * @param expr      Expression yielding a record
 * @param fieldName Name of field in record
 * @param caseSensitive Whether match is case-sensitive
 * @return Expression accessing a given named field
 */
public RexNode makeFieldAccess(RexNode expr, String fieldName,
    boolean caseSensitive) {
  final RelDataType type = expr.getType();
  final RelDataTypeField field =
      type.getField(fieldName, caseSensitive, false);
  if (field == null) {
    throw new AssertionError("Type '" + type + "' has no field '"
        + fieldName + "'");
  }
  return makeFieldAccessInternal(expr, field);
}
 
Example 19
Source Project: calcite   File: RexBuilder.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Creates an expression accessing a given named field from a record.
 *
 * <p>NOTE: Be careful choosing the value of {@code caseSensitive}.
 * If the field name was supplied by an end-user (e.g. as a column alias in
 * SQL), use your session's case-sensitivity setting.
 * Only hard-code {@code true} if you are sure that the field name is
 * internally generated.
 * Hard-coding {@code false} is almost certainly wrong.</p>
 *
 * @param expr      Expression yielding a record
 * @param fieldName Name of field in record
 * @param caseSensitive Whether match is case-sensitive
 * @return Expression accessing a given named field
 */
public RexNode makeFieldAccess(RexNode expr, String fieldName,
    boolean caseSensitive) {
  final RelDataType type = expr.getType();
  final RelDataTypeField field =
      type.getField(fieldName, caseSensitive, false);
  if (field == null) {
    throw new AssertionError("Type '" + type + "' has no field '"
        + fieldName + "'");
  }
  return makeFieldAccessInternal(expr, field);
}