Java Code Examples for org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc

The following examples show how to use org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void testExpressionConversionWithNestedStringAndTimestampColumns() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .val(testDate).timestampCol("timestamp").pred("=", 2)
        .val("test").strCol("strCol").pred("=", 2)
        .pred("and", 2)
        .val(testDate).timestampCol("timestamp").pred("=", 2)
        .val("test").strCol("strCol").pred("=", 2)
        .pred("and", 2)
        .pred("and", 2).build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("(((strCol = 'test') and (timestamp = '1970-01-01 00:02:03.456')) " +
              "and " +
              "((strCol = 'test') and (timestamp = '1970-01-01 00:02:03.456')))",
        filter);
}
 
Example 2
@Test
public void testExpressionConversionWithTwoNotInAndOneIn() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .vals(Arrays.asList("data1", "data2")).strCol("notInCol1").pred("in", 3)
        .pred("not", 1)
        .vals(Arrays.asList("val1", "val2", "val3")).strCol("notInCol2").pred("in", 4)
        .pred("not", 1)
        .pred("and", 2)
        .vals(Arrays.asList("abc", "def", "ghi", "jkl")).strCol("inCol1").pred("in", 5)
        .pred("and", 2)
        .build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("((inCol1) IN ('jkl', 'ghi', 'def', 'abc') and (((notInCol2) NOT IN ('val3', 'val2', 'val1')) and " +
        "((notInCol1) NOT IN ('data2', 'data1'))))", filter);
}
 
Example 3
private List<ObjectPair<Integer, byte[]>> getDropPartitionExpressions(List<Partition> partitions) throws Exception {
  List<ObjectPair<Integer, byte[]>> partExprs = Lists.newArrayList();

  for (int i = 0; i < partitions.size(); i++) {
    Partition partition = partitions.get(i);
    // the catalogTable has only one partition key
    String partitionKey = catalogTable.getPartitionKeys().get(0).getName();
    String partitionValue = partition.getValues().get(0);
    ExprNodeGenericFuncDesc expr = new ExprBuilder(catalogTable.getName())
        .val(partitionValue).strCol(partitionKey).pred("=", 2).build();
    byte[] payload = hiveShims.getSerializeExpression(expr);
    partExprs.add(new ObjectPair<>(i, payload));
  }

  return partExprs;
}
 
Example 4
@Test
public void testListPartitionsByExpression() throws Exception {
  String dbName = testDB.getName();
  Table table = getTestTable();
  ExprBuilder e = new ExprBuilder(table.getName()).val("test").strCol("location").pred("=", 2);
  ExprNodeGenericFuncDesc exprTree = e.build();
  byte[] payload = hiveShims.getSerializeExpression(exprTree);
  List<org.apache.hadoop.hive.metastore.api.Partition> results = new ArrayList<>();

  Partition newPartition = new Partition()
    .withDatabaseName(dbName).withTableName(table.getName())
    .withValues(Lists.newArrayList("newval")).withStorageDescriptor(null);
  when(glueClient.getPartitions(any(GetPartitionsRequest.class)))
          .thenReturn(new GetPartitionsResult().withPartitions(newPartition));
  boolean status = metastoreClient.listPartitionsByExpr(dbName,table.getName(),payload,null,(short)5,results);
  assertFalse(status);
}
 
Example 5
Source Project: multiple-dimension-spread   Source File: HiveExprOrNode.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void addChildNode( final ExprNodeGenericFuncDesc exprNodeDesc ){
  GenericUDF udf = exprNodeDesc.getGenericUDF();
  if( udf instanceof GenericUDFOPAnd ){
    childNodeList.add( new HiveExprAndNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPOr ){
    childNodeList.add( new HiveExprOrNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPNot ){
    childNodeList.add( new HiveExprNotNode( exprNodeDesc.getChildren() ) );
  }
  else{
    childNodeList.add( HiveExprFactory.get( exprNodeDesc , udf , exprNodeDesc.getChildren() ) );
  }
}
 
Example 6
Source Project: multiple-dimension-spread   Source File: HiveExprAndNode.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void addChildNode( final ExprNodeGenericFuncDesc exprNodeDesc ){
  GenericUDF udf = exprNodeDesc.getGenericUDF();
  if( udf instanceof GenericUDFOPAnd ){
    childNodeList.add( new HiveExprAndNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPOr ){
    childNodeList.add( new HiveExprOrNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPNot ){
    childNodeList.add( new HiveExprNotNode( exprNodeDesc.getChildren() ) );
  }
  else{
    childNodeList.add( HiveExprFactory.get( exprNodeDesc , udf , exprNodeDesc.getChildren() ) );
  }
}
 
Example 7
Source Project: multiple-dimension-spread   Source File: HiveExprNotNode.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void addChildNode( final ExprNodeGenericFuncDesc exprNodeDesc ){
  GenericUDF udf = exprNodeDesc.getGenericUDF();
  if( udf instanceof GenericUDFOPAnd ){
    childNodeList.add( new HiveExprAndNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPOr ){
    childNodeList.add( new HiveExprOrNode( exprNodeDesc.getChildren() ) );
  }
  else if( udf instanceof GenericUDFOPNot ){
    childNodeList.add( new HiveExprNotNode( exprNodeDesc.getChildren() ) );
  }
  else{
    childNodeList.add( HiveExprFactory.get( exprNodeDesc , udf , exprNodeDesc.getChildren() ) );
  }
}
 
Example 8
public DecomposedPredicate pushPredicate(Map<String, String> hiveTypeMapping, ExprNodeDesc
    predicate) {
  log.info("Checking predicates for pushdown in DynamoDB query");
  List<IndexSearchCondition> searchConditions = getGenericSearchConditions(hiveTypeMapping,
      predicate);
  log.info("Pushed predicates: " + searchConditions);
  if (searchConditions.isEmpty()) {
    return null;
  } else {
    List<IndexSearchCondition> finalSearchCondition =
        prioritizeSearchConditions(searchConditions);
    IndexPredicateAnalyzer analyzer = new IndexPredicateAnalyzer();
    DecomposedPredicate decomposedPredicate = new DecomposedPredicate();
    decomposedPredicate.pushedPredicate =
        analyzer.translateSearchConditions(finalSearchCondition);
    decomposedPredicate.residualPredicate = (ExprNodeGenericFuncDesc) predicate;
    return decomposedPredicate;
  }
}
 
Example 9
public static String convertHiveExpressionToCatalogExpression(byte[] exprBytes) throws MetaException {
  ExprNodeGenericFuncDesc exprTree = deserializeExpr(exprBytes);
  Set<String> columnNamesInNotInExpression = Sets.newHashSet();
  fieldEscaper(exprTree.getChildren(), exprTree, columnNamesInNotInExpression);
  String expression = rewriteExpressionForNotIn(exprTree.getExprString(), columnNamesInNotInExpression);
  return expression;
}
 
Example 10
private static ExprNodeGenericFuncDesc deserializeExpr(byte[] exprBytes) throws MetaException {
  ExprNodeGenericFuncDesc expr = null;
  try {
    expr = ShimsLoader.getHiveShims().getDeserializeExpression(exprBytes);
  } catch (Exception ex) {
    logger.error("Failed to deserialize the expression", ex);
    throw new MetaException(ex.getMessage());
  }
  if (expr == null) {
    throw new MetaException("Failed to deserialize expression - ExprNodeDesc not present");
  }
  return expr;
}
 
Example 11
private static void addColumnNamesOfNotInExpressionToSet(ExprNodeDesc childNode, ExprNodeDesc parentNode, Set<String> columnsInNotInExpression) {
  if (parentNode != null && childNode != null && parentNode instanceof ExprNodeGenericFuncDesc && childNode instanceof ExprNodeGenericFuncDesc) {
    ExprNodeGenericFuncDesc parentFuncNode = (ExprNodeGenericFuncDesc) parentNode;
    ExprNodeGenericFuncDesc childFuncNode = (ExprNodeGenericFuncDesc) childNode;
    if(parentFuncNode.getGenericUDF() instanceof GenericUDFOPNot && childFuncNode.getGenericUDF() instanceof GenericUDFIn) {
      // The current parent child pair represents a "NOT IN" expression. Add name of the column to the set.
      columnsInNotInExpression.addAll(childFuncNode.getCols());
    }
  }
}
 
Example 12
private ExprBuilder fn(String name, TypeInfo ti, int args) throws Exception {
  List<ExprNodeDesc> children = new ArrayList<>();
  for (int i = 0; i < args; ++i) {
    children.add(stack.pop());
  }
  stack.push(new ExprNodeGenericFuncDesc(ti, FunctionRegistry.getFunctionInfo(name).getGenericUDF(), children));
  return this;
}
 
Example 13
@Test
public void testExpressionConversionByTimestamp() throws Exception {
  Table table = getTestTable();
  ExprBuilder e = new ExprBuilder(table.getName()).val(testDate).timestampCol("timestamp").pred("=", 2);
  ExprNodeGenericFuncDesc exprTree = e.build();
  byte[] payload = hiveShims.getSerializeExpression(exprTree);
  String expression = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("(timestamp = '1970-01-01 00:02:03.456')", expression);
}
 
Example 14
@Test
public void testExpressionConversionWithStringAndTimestampColumn() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .val(testDate).timestampCol("timestamp").pred("=", 2)
        .val("test").strCol("strCol").pred("=", 2)
        .pred("and", 2).build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("((strCol = 'test') and (timestamp = '1970-01-01 00:02:03.456'))", filter);
}
 
Example 15
@Test
public void testExpressionConversionWithNotIn() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .vals(Arrays.asList("val1", "val2", "val3")).strCol("strCol").pred("in", 4)
        .pred("not", 1)
        .build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("((strCol) NOT IN ('val3', 'val2', 'val1'))", filter);
}
 
Example 16
@Test
public void testExpressionConversionWithIn() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .vals(Arrays.asList("val1", "val2", "val3")).strCol("strCol").pred("in", 4)
        .build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("(strCol) IN ('val3', 'val2', 'val1')", filter);
}
 
Example 17
@Test
public void testExpressionConversionWithBetween() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .val("val100").val("val1").strCol("strCol").val(false).pred("between", 4)
        .build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("strCol BETWEEN 'val1' AND 'val100'", filter);
}
 
Example 18
@Test
public void testExpressionConversionWithNotBetween() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .val("val100" ).val("val1").strCol("strCol").val(true).pred("between", 4)
        .build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("strCol NOT BETWEEN 'val1' AND 'val100'", filter);
}
 
Example 19
@Test
public void testExpressionConversionWithMultipleNots() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .val("val100" ).val("val1").strCol("strCol").val(true).pred("between", 4)
        .vals(Arrays.asList("val1", "val2", "val3")).strCol("strCol").pred("in", 4)
        .pred("not", 1)
        .pred("and", 2)
        .build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("(((strCol) NOT IN ('val3', 'val2', 'val1')) and strCol NOT BETWEEN 'val1' AND 'val100')", filter);
}
 
Example 20
@Test
public void testExpressionConversionWithTwoNotInsSameColumn() throws Exception {
  Table table = getTestTable();
  ExprNodeGenericFuncDesc expr = new ExprBuilder(table.getName())
        .vals(Arrays.asList("data1", "data2")).strCol("strCol").pred("in", 3)
        .pred("not", 1)
        .vals(Arrays.asList("val1", "val2", "val3")).strCol("strCol").pred("in", 4)
        .pred("not", 1)
        .pred("or", 2)
        .build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  String filter = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("(((strCol) NOT IN ('val3', 'val2', 'val1')) or ((strCol) NOT IN ('data2', 'data1')))", filter);
}
 
Example 21
@Test
public void testExpressionConversionWithDateCharAndBooleanColumn() throws Exception {
  ExprNodeGenericFuncDesc expr = new ExprBuilder("fooTable")
          // see https://goo.gl/eGEUh2, Hive uses the same method to convert query string value to Date object
          .val(java.sql.Date.valueOf("2017-11-14")).dateCol("dateCol").pred("=", 2)
          .val('c').charCol("charCol").pred("=", 2).pred("and", 2)
          .val(true).booleanCol("booleanCol").pred("=", 2).pred("and", 2).build();
  byte[] payload = hiveShims.getSerializeExpression(expr);

  String catalogExpression = ExpressionHelper.convertHiveExpressionToCatalogExpression(payload);
  assertEquals("((booleanCol = true) and ((charCol = 'c') and (dateCol = '2017-11-14')))",
          catalogExpression);
}
 
Example 22
Source Project: multiple-dimension-spread   Source File: HiveExprOrNode.java    License: Apache License 2.0 5 votes vote down vote up
public HiveExprOrNode( final List<ExprNodeDesc> childExprNodeDesc ){
  for( ExprNodeDesc nodeChild : childExprNodeDesc  ){
    if( nodeChild instanceof ExprNodeGenericFuncDesc ){
      addChildNode( (ExprNodeGenericFuncDesc)nodeChild );
    }
    else if( ( nodeChild instanceof ExprNodeColumnDesc ) || ( nodeChild instanceof ExprNodeFieldDesc ) ){
      childNodeList.add( new BooleanHiveExpr( nodeChild ) );
    }
    else{
      childNodeList.add( new UnsupportHiveExpr() );
    }
  }
}
 
Example 23
Source Project: multiple-dimension-spread   Source File: HiveExprAndNode.java    License: Apache License 2.0 5 votes vote down vote up
public HiveExprAndNode( final List<ExprNodeDesc> childExprNodeDesc ){
  for( ExprNodeDesc nodeChild : childExprNodeDesc  ){
    if( nodeChild instanceof ExprNodeGenericFuncDesc ){
      addChildNode( (ExprNodeGenericFuncDesc)nodeChild );
    }
    else if( ( nodeChild instanceof ExprNodeColumnDesc ) || ( nodeChild instanceof ExprNodeFieldDesc ) ){
      childNodeList.add( new BooleanHiveExpr( nodeChild ) );
    }
    else{
      childNodeList.add( new UnsupportHiveExpr() );
    }
  }
}
 
Example 24
Source Project: multiple-dimension-spread   Source File: HiveExprNotNode.java    License: Apache License 2.0 5 votes vote down vote up
public HiveExprNotNode( final List<ExprNodeDesc> childExprNodeDesc ){
  for( ExprNodeDesc nodeChild : childExprNodeDesc  ){
    if( nodeChild instanceof ExprNodeGenericFuncDesc ){
      addChildNode( (ExprNodeGenericFuncDesc)nodeChild );
    }
    else if( ( nodeChild instanceof ExprNodeColumnDesc ) || ( nodeChild instanceof ExprNodeFieldDesc ) ){
      childNodeList.add( new BooleanHiveExpr( nodeChild ) );
    }
    else{
      childNodeList.add( new UnsupportHiveExpr() );
    }
  }
}
 
Example 25
public static IExtractNode getExtractNode(final ExprNodeDesc target ){
  if( target instanceof ExprNodeGenericFuncDesc ){
    return getExtractNodeFromGenericFunc( (ExprNodeGenericFuncDesc)target );
  }
  else if( target instanceof ExprNodeFieldDesc ){
    return getExtractNodeFromField( (ExprNodeFieldDesc)target  );
  }
  else if( target instanceof ExprNodeColumnDesc ){
    if( ( (ExprNodeColumnDesc)target ).getIsPartitionColOrVirtualCol() ){
      return null;
    }
    return getExtractNodeFromColumn( (ExprNodeColumnDesc)target  );
  }
  return null;
}
 
Example 26
public static IExtractNode getExtractNodeFromGenericFunc( final ExprNodeGenericFuncDesc target ){
  GenericUDF udf = target.getGenericUDF();
  if( ! ( udf instanceof GenericUDFIndex ) ){
    return null;
  }
  return getExtractNodeFromGenericIndex( target , (GenericUDFIndex)udf );
}
 
Example 27
public IExpressionNode createExpressionNode( final List<ExprNodeGenericFuncDesc> filterExprs ){
  HiveExprOrNode hiveOrNode = new HiveExprOrNode();
  for( ExprNodeGenericFuncDesc filterExpr : filterExprs ){
    if( filterExpr != null ){
      hiveOrNode.addChildNode( filterExpr );
    }
  }

  return hiveOrNode.getPushDownFilterNode();
}
 
Example 28
public ExprNodeGenericFuncDesc build() throws Exception {
  if (stack.size() != 1) {
    throw new Exception("Bad test: " + stack.size());
  }
  return (ExprNodeGenericFuncDesc) stack.pop();
}
 
Example 29
private ObjectPair<Integer, byte[]> getDumbExpression() throws Exception {
  ExprNodeGenericFuncDesc expr = new ExprBuilder("fooTable")
      .val("value").strCol("key").pred("=", 2).build();
  byte[] payload = hiveShims.getSerializeExpression(expr);
  return new ObjectPair<>(0, payload);
}
 
Example 30
@Override
public ExprNodeGenericFuncDesc getDeserializeExpression(byte[] exprBytes) {
  return SerializationUtilities.deserializeExpressionFromKryo(exprBytes);
}