org.apache.spark.sql.catalyst.expressions.Expression Java Examples

The following examples show how to use org.apache.spark.sql.catalyst.expressions.Expression. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SparkBenchmarkUtil.java    From iceberg with Apache License 2.0 6 votes vote down vote up
public static UnsafeProjection projection(Schema expectedSchema, Schema actualSchema) {
  StructType struct = SparkSchemaUtil.convert(actualSchema);

  List<AttributeReference> refs = JavaConverters.seqAsJavaListConverter(struct.toAttributes()).asJava();
  List<Attribute> attrs = Lists.newArrayListWithExpectedSize(struct.fields().length);
  List<Expression> exprs = Lists.newArrayListWithExpectedSize(struct.fields().length);

  for (AttributeReference ref : refs) {
    attrs.add(ref.toAttribute());
  }

  for (Types.NestedField field : expectedSchema.columns()) {
    int indexInIterSchema = struct.fieldIndex(field.name());
    exprs.add(refs.get(indexInIterSchema));
  }

  return UnsafeProjection.create(
      JavaConverters.asScalaBufferConverter(exprs).asScala().toSeq(),
      JavaConverters.asScalaBufferConverter(attrs).asScala().toSeq());
}
 
Example #2
Source File: ExpressionTranslatorTest.java    From jpmml-sparkml with GNU Affero General Public License v3.0 6 votes vote down vote up
static
private Expression translateInternal(String sqlStatement){
	StructType schema = new StructType()
		.add("flag", DataTypes.BooleanType)
		.add("x1", DataTypes.DoubleType)
		.add("x2", DataTypes.DoubleType)
		.add("status", DataTypes.IntegerType);

	LogicalPlan logicalPlan = DatasetUtil.createAnalyzedLogicalPlan(ExpressionTranslatorTest.sparkSession, schema, sqlStatement);

	List<Expression> expressions = JavaConversions.seqAsJavaList(logicalPlan.expressions());
	if(expressions.size() != 1){
		throw new IllegalArgumentException();
	}

	return expressions.get(0);
}
 
Example #3
Source File: SparkExpressions.java    From iceberg with Apache License 2.0 5 votes vote down vote up
private static com.netflix.iceberg.expressions.Expression convertIn(Expression expr,
                                                                    Collection<Object> values) {
  if (expr instanceof Attribute) {
    Attribute attr = (Attribute) expr;
    com.netflix.iceberg.expressions.Expression converted = alwaysFalse();
    for (Object item : values) {
      converted = or(converted, equal(attr.name(), item));
    }
    return converted;
  }

  return null;
}
 
Example #4
Source File: SparkExpressions.java    From iceberg with Apache License 2.0 5 votes vote down vote up
private static List<Object> convertLiterals(List<Expression> values) {
  List<Object> converted = Lists.newArrayListWithExpectedSize(values.size());

  for (Expression value : values) {
    if (value instanceof Literal) {
      Literal lit = (Literal) value;
      converted.add(valueFromSpark(lit));
    } else {
      return null;
    }
  }

  return converted;
}
 
Example #5
Source File: SparkExpressions.java    From iceberg with Apache License 2.0 5 votes vote down vote up
private static com.netflix.iceberg.expressions.Expression convert(Operation op,
                                                                  Expression left,
                                                                  Expression right) {
  Pair<Transform, String> attrPair = null;
  Operation leftOperation = null;
  Literal lit = null;

  if (right instanceof Literal) {
    lit = (Literal) right;
    attrPair = convertAttr(left);
    leftOperation = op;
  } else if (left instanceof Literal) {
    lit = (Literal) left;
    attrPair = convertAttr(right);
    leftOperation = op.flipLR();
  }

  if (attrPair != null) {
    switch (attrPair.first()) {
      case IDENTITY:
        return predicate(leftOperation, attrPair.second(), valueFromSpark(lit));
      case YEAR:
        return filter(leftOperation, attrPair.second(), (int) lit.value(),
            SparkExpressions::yearToTimestampMicros);
      case DAY:
        return filter(leftOperation, attrPair.second(), (int) lit.value(),
            SparkExpressions::dayToTimestampMicros);
      default:
    }
  }

  return null;
}
 
Example #6
Source File: SparkExpressions.java    From iceberg with Apache License 2.0 5 votes vote down vote up
private static com.netflix.iceberg.expressions.Expression filter(
    Operation op, String name, int value, Function<Integer, Long> startTsMicros) {
  switch (op) {
    case LT:
      return predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value)));
    case LT_EQ:
      return predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value + 1)));
    case GT:
      return predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value + 1)));
    case GT_EQ:
      return predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value)));
    case EQ:
      return and(
          predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value))),
          predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value + 1)))
      );
    case NOT_EQ:
      return or(
          predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value + 1))),
          predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value)))
      );
    case IN:
    case NOT_IN:
    default:
      throw new IllegalArgumentException("Cannot convert operation to year filter: " + op);
  }
}
 
Example #7
Source File: EncoderHelpers.java    From beam with Apache License 2.0 5 votes vote down vote up
/**
 * Wrap a Beam coder into a Spark Encoder using Catalyst Expression Encoders (which uses java code
 * generation).
 */
public static <T> Encoder<T> fromBeamCoder(Coder<T> coder) {
  Class<? super T> clazz = coder.getEncodedTypeDescriptor().getRawType();
  ClassTag<T> classTag = ClassTag$.MODULE$.apply(clazz);
  List<Expression> serializers =
      Collections.singletonList(
          new EncodeUsingBeamCoder<>(new BoundReference(0, new ObjectType(clazz), true), coder));

  return new ExpressionEncoder<>(
      SchemaHelpers.binarySchema(),
      false,
      JavaConversions.collectionAsScalaIterable(serializers).toSeq(),
      new DecodeUsingBeamCoder<>(
          new Cast(new GetColumnByOrdinal(0, BinaryType), BinaryType), classTag, coder),
      classTag);
}
 
Example #8
Source File: ExpressionTranslator.java    From jpmml-sparkml with GNU Affero General Public License v3.0 5 votes vote down vote up
static
public org.dmg.pmml.Expression translate(Expression expression, boolean compact){
	org.dmg.pmml.Expression pmmlExpression = translateInternal(expression);

	if(compact){
		ExpressionCompactor expressionCompactor = new ExpressionCompactor();

		expressionCompactor.applyTo(pmmlExpression);
	}

	return pmmlExpression;
}
 
Example #9
Source File: ExpressionTranslatorTest.java    From jpmml-sparkml with GNU Affero General Public License v3.0 5 votes vote down vote up
static
public void checkValue(Object expectedValue, String sqlExpression){
	Expression expression = translateInternal("SELECT (" + sqlExpression + ") FROM __THIS__");

	Object sparkValue = expression.eval(InternalRow.empty());

	if(expectedValue instanceof String){
		assertEquals(expectedValue, sparkValue.toString());
	} else

	if(expectedValue instanceof Integer){
		assertEquals(expectedValue, ((Number)sparkValue).intValue());
	} else

	if(expectedValue instanceof Float){
		assertEquals(expectedValue, ((Number)sparkValue).floatValue());
	} else

	if(expectedValue instanceof Double){
		assertEquals(expectedValue, ((Number)sparkValue).doubleValue());
	} else

	{
		assertEquals(expectedValue, sparkValue);
	}

	org.dmg.pmml.Expression pmmlExpression = ExpressionTranslator.translate(expression);

	EvaluationContext context = new VirtualEvaluationContext();
	context.declareAll(Collections.emptyMap());

	FieldValue value = ExpressionUtil.evaluate(pmmlExpression, context);

	Object pmmlValue = FieldValueUtil.getValue(value);
	assertEquals(expectedValue, pmmlValue);
}
 
Example #10
Source File: ExpressionTranslator.java    From jpmml-sparkml with GNU Affero General Public License v3.0 5 votes vote down vote up
static
private String formatMessage(Expression expression){

	if(expression == null){
		return null;
	}

	return "Spark SQL function \'" + expression + "\' (class " + (expression.getClass()).getName() + ") is not supported";
}
 
Example #11
Source File: SparkExpressions.java    From iceberg with Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression notIn(BoundReference<T> ref,
                            com.netflix.iceberg.expressions.Literal<T> lit) {
  throw new UnsupportedOperationException("Not implemented: notIn");
}
 
Example #12
Source File: EncoderHelpers.java    From beam with Apache License 2.0 4 votes vote down vote up
@Override
public Expression child() {
  return child;
}
 
Example #13
Source File: TestFilteredScan.java    From iceberg with Apache License 2.0 4 votes vote down vote up
private void pushFilters(DataSourceReader reader,
                         Expression... expressions) {
  Assert.assertTrue(reader instanceof SupportsPushDownCatalystFilters);
  SupportsPushDownCatalystFilters filterable = (SupportsPushDownCatalystFilters) reader;
  filterable.pushCatalystFilters(expressions);
}
 
Example #14
Source File: ExpressionTranslator.java    From jpmml-sparkml with GNU Affero General Public License v3.0 4 votes vote down vote up
static
public org.dmg.pmml.Expression translate(Expression expression){
	return translate(expression, true);
}
 
Example #15
Source File: ExpressionTranslatorTest.java    From jpmml-sparkml with GNU Affero General Public License v3.0 4 votes vote down vote up
static
private Object evaluate(String sqlExpression){
	Expression expression = translateInternal("SELECT (" + sqlExpression + ") FROM __THIS__");

	return expression.eval(InternalRow.empty());
}
 
Example #16
Source File: ExpressionTranslatorTest.java    From jpmml-sparkml with GNU Affero General Public License v3.0 4 votes vote down vote up
static
private org.dmg.pmml.Expression translate(String sqlExpression){
	Expression expression = translateInternal("SELECT (" + sqlExpression + ") FROM __THIS__");

	return ExpressionTranslator.translate(expression);
}
 
Example #17
Source File: ExpressionTranslatorTest.java    From jpmml-sparkml with GNU Affero General Public License v3.0 4 votes vote down vote up
static
private void checkExpression(org.dmg.pmml.Expression expected, String string){
	org.dmg.pmml.Expression actual = translate(string);

	assertTrue(ReflectionUtil.equals(expected, actual));
}
 
Example #18
Source File: EncoderHelpers.java    From beam with Apache License 2.0 4 votes vote down vote up
public EncodeUsingBeamCoder(Expression child, Coder<T> coder) {
  this.child = child;
  this.coder = coder;
}
 
Example #19
Source File: EncoderHelpers.java    From beam with Apache License 2.0 4 votes vote down vote up
@Override
public Expression child() {
  return child;
}
 
Example #20
Source File: EncoderHelpers.java    From beam with Apache License 2.0 4 votes vote down vote up
public DecodeUsingBeamCoder(Expression child, ClassTag<T> classTag, Coder<T> coder) {
  this.child = child;
  this.classTag = classTag;
  this.coder = coder;
}
 
Example #21
Source File: SparkExpressions.java    From iceberg with Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression in(BoundReference<T> ref,
                         com.netflix.iceberg.expressions.Literal<T> lit) {
  throw new UnsupportedOperationException("Not implemented: in");
}
 
Example #22
Source File: SparkExpressions.java    From iceberg with Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression notEq(BoundReference<T> ref,
                            com.netflix.iceberg.expressions.Literal<T> lit) {
  return column(ref).notEqual(lit.value()).expr();
}
 
Example #23
Source File: SparkExpressions.java    From iceberg with Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression eq(BoundReference<T> ref,
                         com.netflix.iceberg.expressions.Literal<T> lit) {
  return column(ref).equalTo(lit.value()).expr();
}
 
Example #24
Source File: SparkExpressions.java    From iceberg with Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression gtEq(BoundReference<T> ref,
                           com.netflix.iceberg.expressions.Literal<T> lit) {
  return column(ref).geq(lit.value()).expr();
}
 
Example #25
Source File: SparkExpressions.java    From iceberg with Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression gt(BoundReference<T> ref,
                         com.netflix.iceberg.expressions.Literal<T> lit) {
  return column(ref).gt(lit.value()).expr();
}
 
Example #26
Source File: SparkExpressions.java    From iceberg with Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression ltEq(BoundReference<T> ref,
                           com.netflix.iceberg.expressions.Literal<T> lit) {
  return column(ref).leq(lit.value()).expr();
}
 
Example #27
Source File: SparkExpressions.java    From iceberg with Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression lt(BoundReference<T> ref,
                         com.netflix.iceberg.expressions.Literal<T> lit) {
  return column(ref).lt(lit.value()).expr();
}
 
Example #28
Source File: SparkExpressions.java    From iceberg with Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression notNull(BoundReference<T> ref) {
  return column(ref).isNotNull().expr();
}
 
Example #29
Source File: SparkExpressions.java    From iceberg with Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression isNull(BoundReference<T> ref) {
  return column(ref).isNull().expr();
}
 
Example #30
Source File: SparkExpressions.java    From iceberg with Apache License 2.0 4 votes vote down vote up
@Override
public Expression or(Expression left, Expression right) {
  return Or$.MODULE$.apply(left, right);
}