org.apache.hadoop.hive.ql.plan.ExprNodeDesc Java Examples

The following examples show how to use org.apache.hadoop.hive.ql.plan.ExprNodeDesc. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DynamoDBFilterPushdown.java    From emr-dynamodb-connector with Apache License 2.0 6 votes vote down vote up
private List<IndexSearchCondition> getGenericSearchConditions(Map<String, String> hiveTypeMapping,
    ExprNodeDesc predicate) {

  IndexPredicateAnalyzer analyzer = new IndexPredicateAnalyzer();

  // DynamoDB does not support filters on columns of types set
  for (Entry<String, String> entry : hiveTypeMapping.entrySet()) {
    if (eligibleHiveTypes.contains(entry.getValue())) {
      analyzer.allowColumnName(entry.getKey());
    }
  }

  for (DynamoDBFilterOperator op : DynamoDBFilterOperator.values()) {
    if (op.getHiveClass() != null) {
      analyzer.addComparisonOp(op.getHiveClass());
    }
  }

  List<IndexSearchCondition> searchConditions = new ArrayList<>();
  analyzer.analyzePredicate(predicate, searchConditions);
  return searchConditions;
}
 
Example #2
Source File: NotNullHiveExpr.java    From multiple-dimension-spread with Apache License 2.0 6 votes vote down vote up
@Override
public IExpressionNode getPushDownFilterNode(){
  if( nodeDescList.size() != 1 ){
    return null;
  }
  ExprNodeDesc columnDesc = nodeDescList.get( 0 );

  if( ! ( columnDesc instanceof ExprNodeColumnDesc ) ){
    return null;
  } 

  IExtractNode extractNode = CreateExtractNodeUtil.getExtractNode( columnDesc ); 
  if( extractNode == null ){
    return null;
  }

  ColumnType targetColumnType = MDSColumnTypeUtil.typeInfoToColumnType( columnDesc.getTypeInfo() );

  return new ExecuterNode( extractNode , new NotNullFilter( targetColumnType ) );
}
 
Example #3
Source File: InHiveExpr.java    From multiple-dimension-spread with Apache License 2.0 6 votes vote down vote up
@Override
public IExpressionNode getPushDownFilterNode(){
  if( nodeDescList.size() < 2 ){
    return null;
  }
  ExprNodeDesc columnDesc = nodeDescList.get( 0 );
  IExtractNode extractNode = CreateExtractNodeUtil.getExtractNode( columnDesc );
  if( extractNode == null ){
    return null;
  }
  IFilter filter = getEqualsExecuter( nodeDescList , 1 );
  if( filter == null ){
    return null;
  }
  return new ExecuterNode( extractNode , filter );
}
 
Example #4
Source File: DynamoDBFilterFactory.java    From emr-dynamodb-connector with Apache License 2.0 6 votes vote down vote up
public DynamoDBFilter getFilter(DynamoDBFilterOperator operator, String columnName, String
    columnType, IndexSearchCondition condition) {
  switch (operator.getType()) {
    case UNARY:
      return getFilter(operator, columnName, columnType);
    case BINARY:
      return getFilter(operator, columnName, columnType, condition.getConstantDesc().getValue()
          .toString());
    case NARY:
      List<ExprNodeDesc> children = ShimsLoader.getHiveShims().getIndexExpression(condition)
          .getChildren();
      String[] values = new String[children.size() - 1];
      // This currently supports IN clause only
      // The first element is column name and rest of the elements are
      // the values it can take
      for (int i = 1; i < children.size(); i++) {
        values[i - 1] = ((ExprNodeConstantDesc) children.get(i)).getValue().toString();
      }
      return getFilter(operator, columnName, columnType, values);
    default:
      throw new RuntimeException("Unknown operator type. Operator: " + operator + " "
          + "OperatorType: " + operator.getType());
  }
}
 
Example #5
Source File: DynamoDBFilterPushdown.java    From emr-dynamodb-connector with Apache License 2.0 6 votes vote down vote up
public DecomposedPredicate pushPredicate(Map<String, String> hiveTypeMapping, ExprNodeDesc
    predicate) {
  log.info("Checking predicates for pushdown in DynamoDB query");
  List<IndexSearchCondition> searchConditions = getGenericSearchConditions(hiveTypeMapping,
      predicate);
  log.info("Pushed predicates: " + searchConditions);
  if (searchConditions.isEmpty()) {
    return null;
  } else {
    List<IndexSearchCondition> finalSearchCondition =
        prioritizeSearchConditions(searchConditions);
    IndexPredicateAnalyzer analyzer = new IndexPredicateAnalyzer();
    DecomposedPredicate decomposedPredicate = new DecomposedPredicate();
    decomposedPredicate.pushedPredicate =
        analyzer.translateSearchConditions(finalSearchCondition);
    decomposedPredicate.residualPredicate = (ExprNodeGenericFuncDesc) predicate;
    return decomposedPredicate;
  }
}
 
Example #6
Source File: NullHiveExpr.java    From multiple-dimension-spread with Apache License 2.0 6 votes vote down vote up
@Override
public IExpressionNode getPushDownFilterNode(){
  if( nodeDescList.size() != 1 ){
    return null;
  }
  ExprNodeDesc columnDesc = nodeDescList.get( 0 );

  if( ! ( columnDesc instanceof ExprNodeColumnDesc ) ){
    return null;
  } 

  IExtractNode extractNode = CreateExtractNodeUtil.getExtractNode( columnDesc );
  if( extractNode == null ){
    return null;
  }

  ColumnType targetColumnType = MDSColumnTypeUtil.typeInfoToColumnType( columnDesc.getTypeInfo() );

  return new ExecuterNode( extractNode , new NullFilter( targetColumnType ) );
}
 
Example #7
Source File: HiveDynamoDBInputFormat.java    From emr-dynamodb-connector with Apache License 2.0 6 votes vote down vote up
private DynamoDBQueryFilter getQueryFilter(JobConf conf, Map<String, String>
    hiveDynamoDBMapping, Map<String, String> hiveTypeMapping) throws IOException {
  if (hiveDynamoDBMapping == null) {
    /*
     * Column mapping may be null when user has mapped a DynamoDB item
     * onto a single hive map<string, string> column.
     */
    return new DynamoDBQueryFilter();
  }

  DynamoDBClient client = new DynamoDBClient(conf);
  String filterExprSerialized = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
  if (filterExprSerialized == null) {
    return new DynamoDBQueryFilter();
  }
  ExprNodeDesc filterExpr =
      ShimsLoader.getHiveShims().deserializeExpression(filterExprSerialized);

  DynamoDBFilterPushdown pushdown = new DynamoDBFilterPushdown();
  List<KeySchemaElement> schema =
      client.describeTable(conf.get(DynamoDBConstants.TABLE_NAME)).getKeySchema();
  DynamoDBQueryFilter queryFilter = pushdown.predicateToDynamoDBFilter(
      schema, hiveDynamoDBMapping, hiveTypeMapping, filterExpr);
  return queryFilter;
}
 
Example #8
Source File: AccumuloPredicateHandler.java    From accumulo-hive-storage-manager with Apache License 2.0 6 votes vote down vote up
/**
 *
 *
 * @param conf JobConf
 * @param desc predicate expression node.
 * @return DecomposedPredicate containing translated search conditions the analyzer can support.
 */
public DecomposedPredicate decompose(JobConf conf, ExprNodeDesc desc) {


    IndexPredicateAnalyzer analyzer = newAnalyzer(conf);
    List<IndexSearchCondition> sConditions = new ArrayList<IndexSearchCondition>();
    ExprNodeDesc residualPredicate = analyzer.analyzePredicate(desc, sConditions);
    if(sConditions.size() == 0){
        if(log.isInfoEnabled())
            log.info("nothing to decompose. Returning");
        return null;
    }
    DecomposedPredicate decomposedPredicate  = new DecomposedPredicate();
    decomposedPredicate.pushedPredicate = analyzer.translateSearchConditions(sConditions);
    decomposedPredicate.residualPredicate = residualPredicate;
    return decomposedPredicate;
}
 
Example #9
Source File: ExpressionHelper.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 6 votes vote down vote up
private static void fieldEscaper(List<ExprNodeDesc> exprNodes, ExprNodeDesc parent, Set<String> columnNamesInNotInExpression) {
  if (exprNodes == null || exprNodes.isEmpty()) {
    return;
  } else {
    for (ExprNodeDesc nodeDesc : exprNodes) {
      String nodeType = nodeDesc.getTypeString().toLowerCase();
      if (QUOTED_TYPES.contains(nodeType)) {
        PrimitiveTypeInfo tInfo = new PrimitiveTypeInfo();
        tInfo.setTypeName(HIVE_STRING_TYPE_NAME);
        nodeDesc.setTypeInfo(tInfo);
      }
      addColumnNamesOfNotInExpressionToSet(nodeDesc, parent, columnNamesInNotInExpression);
      fieldEscaper(nodeDesc.getChildren(), nodeDesc, columnNamesInNotInExpression);
    }
  }
}
 
Example #10
Source File: KuduStorageHandler.java    From HiveKudu-Handler with Apache License 2.0 5 votes vote down vote up
@Override
public DecomposedPredicate decomposePredicate(JobConf jobConf,
                                              Deserializer deserializer, ExprNodeDesc predicate) {
    // TODO: Implement push down to Kudu here.
    DecomposedPredicate decomposedPredicate = new DecomposedPredicate();
    return decomposedPredicate;
}
 
Example #11
Source File: Hive012Binding.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
private void pushFilters(final JobConf jobConf, final TableScanOperator tableScan) {

    final TableScanDesc scanDesc = tableScan.getConf();
    if (scanDesc == null) {
      LOG.debug("Not pushing filters because TableScanDesc is null");
      return;
    }

    // construct column name list for reference by filter push down
    Utilities.setColumnNameList(jobConf, tableScan);

    // push down filters
    final ExprNodeDesc filterExpr = scanDesc.getFilterExpr();
    if (filterExpr == null) {
      LOG.debug("Not pushing filters because FilterExpr is null");
      return;
    }

    final String filterText = filterExpr.getExprString();
    final String filterExprSerialized = Utilities.serializeExpression(filterExpr);
    jobConf.set(
            TableScanDesc.FILTER_TEXT_CONF_STR,
            filterText);
    jobConf.set(
            TableScanDesc.FILTER_EXPR_CONF_STR,
            filterExprSerialized);
  }
 
Example #12
Source File: ExpressionHelper.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
private static void addColumnNamesOfNotInExpressionToSet(ExprNodeDesc childNode, ExprNodeDesc parentNode, Set<String> columnsInNotInExpression) {
  if (parentNode != null && childNode != null && parentNode instanceof ExprNodeGenericFuncDesc && childNode instanceof ExprNodeGenericFuncDesc) {
    ExprNodeGenericFuncDesc parentFuncNode = (ExprNodeGenericFuncDesc) parentNode;
    ExprNodeGenericFuncDesc childFuncNode = (ExprNodeGenericFuncDesc) childNode;
    if(parentFuncNode.getGenericUDF() instanceof GenericUDFOPNot && childFuncNode.getGenericUDF() instanceof GenericUDFIn) {
      // The current parent child pair represents a "NOT IN" expression. Add name of the column to the set.
      columnsInNotInExpression.addAll(childFuncNode.getCols());
    }
  }
}
 
Example #13
Source File: Hive010Binding.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
private void pushFilters(final JobConf jobConf, final TableScanOperator tableScan) {

    final TableScanDesc scanDesc = tableScan.getConf();
    if (scanDesc == null) {
      LOG.debug("Not pushing filters because TableScanDesc is null");
      return;
    }

    // construct column name list for reference by filter push down
    Utilities.setColumnNameList(jobConf, tableScan);

    // push down filters
    final ExprNodeDesc filterExpr = scanDesc.getFilterExpr();
    if (filterExpr == null) {
      LOG.debug("Not pushing filters because FilterExpr is null");
      return;
    }

    final String filterText = filterExpr.getExprString();
    final String filterExprSerialized = Utilities.serializeExpression(filterExpr);
    jobConf.set(
            TableScanDesc.FILTER_TEXT_CONF_STR,
            filterText);
    jobConf.set(
            TableScanDesc.FILTER_EXPR_CONF_STR,
            filterExprSerialized);
  }
 
Example #14
Source File: DynamoDBFilterPushdown.java    From emr-dynamodb-connector with Apache License 2.0 5 votes vote down vote up
public DynamoDBQueryFilter predicateToDynamoDBFilter(List<KeySchemaElement> schema, Map<String,
    String> hiveDynamoDBMapping, Map<String, String> hiveTypeMapping, ExprNodeDesc predicate) {
  List<IndexSearchCondition> searchConditions = getGenericSearchConditions(hiveTypeMapping,
      predicate);

  if (searchConditions.isEmpty()) {
    return null;
  }

  Map<String, DynamoDBFilter> filterMap = new HashMap<>();
  DynamoDBFilterFactory factory = new DynamoDBFilterFactory();

  // The search conditions are supposed to be unique at this point, so not
  // prioritizing them
  for (IndexSearchCondition condition : searchConditions) {
    String hiveColumnName = condition.getColumnDesc().getColumn();
    String dynamoDBColumnName = hiveDynamoDBMapping.get(hiveColumnName);
    DynamoDBFilterOperator op =
        DynamoDBFilterOperator.getFilterOperationFromHiveClass(condition.getComparisonOp());
    DynamoDBFilter filter =
        factory.getFilter(op, dynamoDBColumnName, hiveTypeMapping.get(hiveColumnName), condition);

    if (filterMap.containsKey(dynamoDBColumnName)) {
      // We have special case code for DynamoDB filter BETWEEN because
      // it does not directly map to any Hive predicate
      DynamoDBFilter existingFilter = filterMap.get(dynamoDBColumnName);
      if (isBetweenFilter(op, existingFilter.getOperator())) {
        filterMap.put(dynamoDBColumnName, getBetweenFilter(filter, existingFilter));
      } else {
        throw new RuntimeException("Found two filters for same column: " + dynamoDBColumnName
            + " Filter 1: " + op + " Filter 2: " + existingFilter.getOperator());
      }
    } else {
      filterMap.put(dynamoDBColumnName, filter);
    }
  }

  return getDynamoDBQueryFilter(schema, filterMap);
}
 
Example #15
Source File: AccumuloPredicateHandler.java    From accumulo-hive-storage-manager with Apache License 2.0 5 votes vote down vote up
/**
 *
 * @param conf JobConf
 * @return list of IndexSearchConditions from the filter expression.
 */
public List<IndexSearchCondition> getSearchConditions(JobConf conf) {
    List<IndexSearchCondition> sConditions = Lists.newArrayList();
    String filteredExprSerialized = conf.get(TableScanDesc.FILTER_EXPR_CONF_STR);
    if(filteredExprSerialized == null)
        return sConditions;
    ExprNodeDesc filterExpr = Utilities.deserializeExpression(filteredExprSerialized, conf);
    IndexPredicateAnalyzer analyzer = newAnalyzer(conf);
    ExprNodeDesc residual = analyzer.analyzePredicate(filterExpr, sConditions);
    if(residual != null)
        throw new RuntimeException("Unexpected residual predicate: " + residual.getExprString());
    return sConditions;
}
 
Example #16
Source File: DynamoDBStorageHandler.java    From emr-dynamodb-connector with Apache License 2.0 5 votes vote down vote up
@Override
public DecomposedPredicate decomposePredicate(JobConf jobConf, Deserializer deserializer,
    ExprNodeDesc predicate) {
  if (jobConf.getBoolean(DynamoDBConstants.DYNAMODB_FILTER_PUSHDOWN, true)) {
    return new DynamoDBFilterPushdown()
        .pushPredicate(HiveDynamoDBUtil.extractHiveTypeMapping(jobConf), predicate);
  } else {
    return null;
  }
}
 
Example #17
Source File: AccumuloStorageHandler.java    From accumulo-hive-storage-manager with Apache License 2.0 5 votes vote down vote up
@Override
public DecomposedPredicate decomposePredicate(JobConf conf,
                                              Deserializer deserializer,
                                              ExprNodeDesc desc) {
    if(conf.get(AccumuloSerde.NO_ITERATOR_PUSHDOWN) == null){
        return predicateHandler.decompose(conf, desc);
    } else {
        log.info("Set to ignore iterator. skipping predicate handler");
        return null;
    }
}
 
Example #18
Source File: InHiveExpr.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up
public static IFilter getEqualsExecuter( final List<ExprNodeDesc> nodeDescList , final int start ){
  PrimitiveObjectInspector rootPrimitiveObjectInspector = getPrimitiveObjectInspector( nodeDescList.get( start ) );
  if( rootPrimitiveObjectInspector == null ){
    return null;
  }
  PrimitiveObjectInspector.PrimitiveCategory primitiveCategory = rootPrimitiveObjectInspector.getPrimitiveCategory();
  IFilter filter = null;
  switch( rootPrimitiveObjectInspector.getPrimitiveCategory() ){
    case STRING:
      Set<String> stringDic = new HashSet<String>();
      for( int i = start ; i < nodeDescList.size() ; i++ ){
        PrimitiveObjectInspector primitiveObjectInspector  = getPrimitiveObjectInspector( nodeDescList.get( i ) );
        if( primitiveObjectInspector == null || primitiveObjectInspector.getPrimitiveCategory() != rootPrimitiveObjectInspector.getPrimitiveCategory() ){
          return null;
        }
        stringDic.add( ( (WritableConstantStringObjectInspector)primitiveObjectInspector ).getWritableConstantValue().toString() );
      }
      return new StringDictionaryFilter( stringDic );
    case BYTE:
    case SHORT:
    case INT:
    case LONG:
    case FLOAT:
    case DOUBLE:
    default:
      return null;
  }
}
 
Example #19
Source File: InHiveExpr.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up
public static PrimitiveObjectInspector getPrimitiveObjectInspector( final ExprNodeDesc exprNode ){
  if( ! ( exprNode instanceof ExprNodeConstantDesc ) ){
    return null;
  }
  ExprNodeConstantDesc constDesc = (ExprNodeConstantDesc)exprNode;
  ObjectInspector objectInspector = constDesc.getWritableObjectInspector();
  if( objectInspector.getCategory() != ObjectInspector.Category.PRIMITIVE ){
    return null;
  }
  return (PrimitiveObjectInspector)objectInspector;
}
 
Example #20
Source File: CreateExtractNodeUtil.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up
public static IExtractNode getExtractNode(final ExprNodeDesc target ){
  if( target instanceof ExprNodeGenericFuncDesc ){
    return getExtractNodeFromGenericFunc( (ExprNodeGenericFuncDesc)target );
  }
  else if( target instanceof ExprNodeFieldDesc ){
    return getExtractNodeFromField( (ExprNodeFieldDesc)target  );
  }
  else if( target instanceof ExprNodeColumnDesc ){
    if( ( (ExprNodeColumnDesc)target ).getIsPartitionColOrVirtualCol() ){
      return null;
    }
    return getExtractNodeFromColumn( (ExprNodeColumnDesc)target  );
  }
  return null;
}
 
Example #21
Source File: HiveExprNotNode.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up
public HiveExprNotNode( final List<ExprNodeDesc> childExprNodeDesc ){
  for( ExprNodeDesc nodeChild : childExprNodeDesc  ){
    if( nodeChild instanceof ExprNodeGenericFuncDesc ){
      addChildNode( (ExprNodeGenericFuncDesc)nodeChild );
    }
    else if( ( nodeChild instanceof ExprNodeColumnDesc ) || ( nodeChild instanceof ExprNodeFieldDesc ) ){
      childNodeList.add( new BooleanHiveExpr( nodeChild ) );
    }
    else{
      childNodeList.add( new UnsupportHiveExpr() );
    }
  }
}
 
Example #22
Source File: HiveExprAndNode.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up
public HiveExprAndNode( final List<ExprNodeDesc> childExprNodeDesc ){
  for( ExprNodeDesc nodeChild : childExprNodeDesc  ){
    if( nodeChild instanceof ExprNodeGenericFuncDesc ){
      addChildNode( (ExprNodeGenericFuncDesc)nodeChild );
    }
    else if( ( nodeChild instanceof ExprNodeColumnDesc ) || ( nodeChild instanceof ExprNodeFieldDesc ) ){
      childNodeList.add( new BooleanHiveExpr( nodeChild ) );
    }
    else{
      childNodeList.add( new UnsupportHiveExpr() );
    }
  }
}
 
Example #23
Source File: ExprBuilder.java    From aws-glue-data-catalog-client-for-apache-hive-metastore with Apache License 2.0 5 votes vote down vote up
private ExprBuilder fn(String name, TypeInfo ti, int args) throws Exception {
  List<ExprNodeDesc> children = new ArrayList<>();
  for (int i = 0; i < args; ++i) {
    children.add(stack.pop());
  }
  stack.push(new ExprNodeGenericFuncDesc(ti, FunctionRegistry.getFunctionInfo(name).getGenericUDF(), children));
  return this;
}
 
Example #24
Source File: HiveExprOrNode.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up
public HiveExprOrNode( final List<ExprNodeDesc> childExprNodeDesc ){
  for( ExprNodeDesc nodeChild : childExprNodeDesc  ){
    if( nodeChild instanceof ExprNodeGenericFuncDesc ){
      addChildNode( (ExprNodeGenericFuncDesc)nodeChild );
    }
    else if( ( nodeChild instanceof ExprNodeColumnDesc ) || ( nodeChild instanceof ExprNodeFieldDesc ) ){
      childNodeList.add( new BooleanHiveExpr( nodeChild ) );
    }
    else{
      childNodeList.add( new UnsupportHiveExpr() );
    }
  }
}
 
Example #25
Source File: DynamoDbHive1Shims.java    From emr-dynamodb-connector with Apache License 2.0 4 votes vote down vote up
@Override
public ExprNodeDesc deserializeExpression(String serializedFilterExpr) {
  return Utilities.deserializeExpression(serializedFilterExpr);
}
 
Example #26
Source File: SMStorageHandler.java    From spliceengine with GNU Affero General Public License v3.0 4 votes vote down vote up
@Override
public DecomposedPredicate decomposePredicate(JobConf jobConf,
                                              Deserializer deserializer, ExprNodeDesc predicate) {
    // TODO Auto-generated method stub
    return null;
}
 
Example #27
Source File: JdbcStorageHandler.java    From HiveJdbcStorageHandler with Apache License 2.0 4 votes vote down vote up
/**
 * @see DBConfiguration#INPUT_CONDITIONS_PROPERTY
 */
@Override
public DecomposedPredicate decomposePredicate(JobConf jobConf, Deserializer deserializer, ExprNodeDesc predicate) {
    // TODO Auto-generated method stub
    return null;
}
 
Example #28
Source File: CompareHiveExpr.java    From multiple-dimension-spread with Apache License 2.0 4 votes vote down vote up
public CompareHiveExpr( final List<ExprNodeDesc> nodeDescList , final StringCompareFilterType stringCompareType , final NumberFilterType numberCompareType ){
  this.nodeDescList = nodeDescList;
  this.stringCompareType = stringCompareType;
  this.numberCompareType = numberCompareType;
}
 
Example #29
Source File: NotEqualsHiveExpr.java    From multiple-dimension-spread with Apache License 2.0 4 votes vote down vote up
public NotEqualsHiveExpr( final List<ExprNodeDesc> nodeDescList ){
  super( nodeDescList );
}
 
Example #30
Source File: BetweenHiveExpr.java    From multiple-dimension-spread with Apache License 2.0 4 votes vote down vote up
public BetweenHiveExpr( final List<ExprNodeDesc> nodeDescList ){
  this.nodeDescList = nodeDescList;
}