Java Code Examples for org.apache.spark.sql.catalyst.expressions.Expression

The following examples show how to use org.apache.spark.sql.catalyst.expressions.Expression. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: iceberg   Source File: SparkBenchmarkUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static UnsafeProjection projection(Schema expectedSchema, Schema actualSchema) {
  StructType struct = SparkSchemaUtil.convert(actualSchema);

  List<AttributeReference> refs = JavaConverters.seqAsJavaListConverter(struct.toAttributes()).asJava();
  List<Attribute> attrs = Lists.newArrayListWithExpectedSize(struct.fields().length);
  List<Expression> exprs = Lists.newArrayListWithExpectedSize(struct.fields().length);

  for (AttributeReference ref : refs) {
    attrs.add(ref.toAttribute());
  }

  for (Types.NestedField field : expectedSchema.columns()) {
    int indexInIterSchema = struct.fieldIndex(field.name());
    exprs.add(refs.get(indexInIterSchema));
  }

  return UnsafeProjection.create(
      JavaConverters.asScalaBufferConverter(exprs).asScala().toSeq(),
      JavaConverters.asScalaBufferConverter(attrs).asScala().toSeq());
}
 
Example 2
static
private Expression translateInternal(String sqlStatement){
	StructType schema = new StructType()
		.add("flag", DataTypes.BooleanType)
		.add("x1", DataTypes.DoubleType)
		.add("x2", DataTypes.DoubleType)
		.add("status", DataTypes.IntegerType);

	LogicalPlan logicalPlan = DatasetUtil.createAnalyzedLogicalPlan(ExpressionTranslatorTest.sparkSession, schema, sqlStatement);

	List<Expression> expressions = JavaConversions.seqAsJavaList(logicalPlan.expressions());
	if(expressions.size() != 1){
		throw new IllegalArgumentException();
	}

	return expressions.get(0);
}
 
Example 3
Source Project: beam   Source File: EncoderHelpers.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Wrap a Beam coder into a Spark Encoder using Catalyst Expression Encoders (which uses java code
 * generation).
 */
public static <T> Encoder<T> fromBeamCoder(Coder<T> coder) {
  Class<? super T> clazz = coder.getEncodedTypeDescriptor().getRawType();
  ClassTag<T> classTag = ClassTag$.MODULE$.apply(clazz);
  List<Expression> serializers =
      Collections.singletonList(
          new EncodeUsingBeamCoder<>(new BoundReference(0, new ObjectType(clazz), true), coder));

  return new ExpressionEncoder<>(
      SchemaHelpers.binarySchema(),
      false,
      JavaConversions.collectionAsScalaIterable(serializers).toSeq(),
      new DecodeUsingBeamCoder<>(
          new Cast(new GetColumnByOrdinal(0, BinaryType), BinaryType), classTag, coder),
      classTag);
}
 
Example 4
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 5 votes vote down vote up
private static com.netflix.iceberg.expressions.Expression convertIn(Expression expr,
                                                                    Collection<Object> values) {
  if (expr instanceof Attribute) {
    Attribute attr = (Attribute) expr;
    com.netflix.iceberg.expressions.Expression converted = alwaysFalse();
    for (Object item : values) {
      converted = or(converted, equal(attr.name(), item));
    }
    return converted;
  }

  return null;
}
 
Example 5
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 5 votes vote down vote up
private static List<Object> convertLiterals(List<Expression> values) {
  List<Object> converted = Lists.newArrayListWithExpectedSize(values.size());

  for (Expression value : values) {
    if (value instanceof Literal) {
      Literal lit = (Literal) value;
      converted.add(valueFromSpark(lit));
    } else {
      return null;
    }
  }

  return converted;
}
 
Example 6
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 5 votes vote down vote up
private static com.netflix.iceberg.expressions.Expression convert(Operation op,
                                                                  Expression left,
                                                                  Expression right) {
  Pair<Transform, String> attrPair = null;
  Operation leftOperation = null;
  Literal lit = null;

  if (right instanceof Literal) {
    lit = (Literal) right;
    attrPair = convertAttr(left);
    leftOperation = op;
  } else if (left instanceof Literal) {
    lit = (Literal) left;
    attrPair = convertAttr(right);
    leftOperation = op.flipLR();
  }

  if (attrPair != null) {
    switch (attrPair.first()) {
      case IDENTITY:
        return predicate(leftOperation, attrPair.second(), valueFromSpark(lit));
      case YEAR:
        return filter(leftOperation, attrPair.second(), (int) lit.value(),
            SparkExpressions::yearToTimestampMicros);
      case DAY:
        return filter(leftOperation, attrPair.second(), (int) lit.value(),
            SparkExpressions::dayToTimestampMicros);
      default:
    }
  }

  return null;
}
 
Example 7
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 5 votes vote down vote up
private static com.netflix.iceberg.expressions.Expression filter(
    Operation op, String name, int value, Function<Integer, Long> startTsMicros) {
  switch (op) {
    case LT:
      return predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value)));
    case LT_EQ:
      return predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value + 1)));
    case GT:
      return predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value + 1)));
    case GT_EQ:
      return predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value)));
    case EQ:
      return and(
          predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value))),
          predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value + 1)))
      );
    case NOT_EQ:
      return or(
          predicate(Operation.GT_EQ, name, tsLiteral(startTsMicros.apply(value + 1))),
          predicate(Operation.LT, name, tsLiteral(startTsMicros.apply(value)))
      );
    case IN:
    case NOT_IN:
    default:
      throw new IllegalArgumentException("Cannot convert operation to year filter: " + op);
  }
}
 
Example 8
static
public org.dmg.pmml.Expression translate(Expression expression, boolean compact){
	org.dmg.pmml.Expression pmmlExpression = translateInternal(expression);

	if(compact){
		ExpressionCompactor expressionCompactor = new ExpressionCompactor();

		expressionCompactor.applyTo(pmmlExpression);
	}

	return pmmlExpression;
}
 
Example 9
static
private String formatMessage(Expression expression){

	if(expression == null){
		return null;
	}

	return "Spark SQL function \'" + expression + "\' (class " + (expression.getClass()).getName() + ") is not supported";
}
 
Example 10
static
public void checkValue(Object expectedValue, String sqlExpression){
	Expression expression = translateInternal("SELECT (" + sqlExpression + ") FROM __THIS__");

	Object sparkValue = expression.eval(InternalRow.empty());

	if(expectedValue instanceof String){
		assertEquals(expectedValue, sparkValue.toString());
	} else

	if(expectedValue instanceof Integer){
		assertEquals(expectedValue, ((Number)sparkValue).intValue());
	} else

	if(expectedValue instanceof Float){
		assertEquals(expectedValue, ((Number)sparkValue).floatValue());
	} else

	if(expectedValue instanceof Double){
		assertEquals(expectedValue, ((Number)sparkValue).doubleValue());
	} else

	{
		assertEquals(expectedValue, sparkValue);
	}

	org.dmg.pmml.Expression pmmlExpression = ExpressionTranslator.translate(expression);

	EvaluationContext context = new VirtualEvaluationContext();
	context.declareAll(Collections.emptyMap());

	FieldValue value = ExpressionUtil.evaluate(pmmlExpression, context);

	Object pmmlValue = FieldValueUtil.getValue(value);
	assertEquals(expectedValue, pmmlValue);
}
 
Example 11
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 4 votes vote down vote up
public static Expression convert(com.netflix.iceberg.expressions.Expression filter,
                                 Schema schema) {
  return visit(Binder.bind(schema.asStruct(), filter), new ExpressionToSpark(schema));
}
 
Example 12
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Expression alwaysTrue() {
  return functions$.MODULE$.lit(true).expr();
}
 
Example 13
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Expression alwaysFalse() {
  return functions$.MODULE$.lit(false).expr();
}
 
Example 14
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Expression not(Expression child) {
  return Not$.MODULE$.apply(child);
}
 
Example 15
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Expression and(Expression left, Expression right) {
  return And$.MODULE$.apply(left, right);
}
 
Example 16
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Expression or(Expression left, Expression right) {
  return Or$.MODULE$.apply(left, right);
}
 
Example 17
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression isNull(BoundReference<T> ref) {
  return column(ref).isNull().expr();
}
 
Example 18
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression notNull(BoundReference<T> ref) {
  return column(ref).isNotNull().expr();
}
 
Example 19
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression lt(BoundReference<T> ref,
                         com.netflix.iceberg.expressions.Literal<T> lit) {
  return column(ref).lt(lit.value()).expr();
}
 
Example 20
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression ltEq(BoundReference<T> ref,
                           com.netflix.iceberg.expressions.Literal<T> lit) {
  return column(ref).leq(lit.value()).expr();
}
 
Example 21
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression gt(BoundReference<T> ref,
                         com.netflix.iceberg.expressions.Literal<T> lit) {
  return column(ref).gt(lit.value()).expr();
}
 
Example 22
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression gtEq(BoundReference<T> ref,
                           com.netflix.iceberg.expressions.Literal<T> lit) {
  return column(ref).geq(lit.value()).expr();
}
 
Example 23
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression eq(BoundReference<T> ref,
                         com.netflix.iceberg.expressions.Literal<T> lit) {
  return column(ref).equalTo(lit.value()).expr();
}
 
Example 24
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression notEq(BoundReference<T> ref,
                            com.netflix.iceberg.expressions.Literal<T> lit) {
  return column(ref).notEqual(lit.value()).expr();
}
 
Example 25
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression in(BoundReference<T> ref,
                         com.netflix.iceberg.expressions.Literal<T> lit) {
  throw new UnsupportedOperationException("Not implemented: in");
}
 
Example 26
Source Project: iceberg   Source File: SparkExpressions.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public <T> Expression notIn(BoundReference<T> ref,
                            com.netflix.iceberg.expressions.Literal<T> lit) {
  throw new UnsupportedOperationException("Not implemented: notIn");
}
 
Example 27
Source Project: iceberg   Source File: TestFilteredScan.java    License: Apache License 2.0 4 votes vote down vote up
private void pushFilters(DataSourceReader reader,
                         Expression... expressions) {
  Assert.assertTrue(reader instanceof SupportsPushDownCatalystFilters);
  SupportsPushDownCatalystFilters filterable = (SupportsPushDownCatalystFilters) reader;
  filterable.pushCatalystFilters(expressions);
}
 
Example 28
static
public org.dmg.pmml.Expression translate(Expression expression){
	return translate(expression, true);
}
 
Example 29
static
private Object evaluate(String sqlExpression){
	Expression expression = translateInternal("SELECT (" + sqlExpression + ") FROM __THIS__");

	return expression.eval(InternalRow.empty());
}
 
Example 30
static
private org.dmg.pmml.Expression translate(String sqlExpression){
	Expression expression = translateInternal("SELECT (" + sqlExpression + ") FROM __THIS__");

	return ExpressionTranslator.translate(expression);
}