org.apache.pig.impl.logicalLayer.FrontendException Java Examples

@Test
public void testValidationFailure() throws Exception{
    String input[] = new String[] { "some data" };
    String outputFileName = "test-output.txt";
    boolean sawException = false;
    try {
        Util.createInputFile(pig.getPigContext(),outputFileName, input);
        String query = "a = load '" + inputFileName + "' as (c:chararray, " +
                       "i:int,d:double);" +
                       "store a into '" + outputFileName + "' using PigStorage();";
        Util.buildLp( pig, query );
    } catch (InvocationTargetException e){
        FrontendException pve = (FrontendException)e.getCause();
        pve.printStackTrace();
        // Since output file is present, validation should fail
        // and throw this exception
        assertEquals(6000,pve.getErrorCode());
        assertEquals(PigException.REMOTE_ENVIRONMENT, pve.getErrorSource());
        assertTrue(pve.getCause() instanceof IOException);
        sawException = true;
    } finally {
        assertTrue(sawException);
        Util.deleteFile(pig.getPigContext(), outputFileName);
    }
}
 

/**
 * This method generates parameter value by running specified command
 *
 * @param key - parameter name
 * @param val - string containing command to be executed
 */
public  void processShellCmd(String key, String val, Boolean overwrite)  throws ParameterSubstitutionException, FrontendException {
    if (pigContext != null) {
        BlackAndWhitelistFilter filter = new BlackAndWhitelistFilter(pigContext);
        filter.validate(PigCommandFilter.Command.SH);
    }

    if (param_val.containsKey(key)) {
        if (param_source.get(key).equals(val) || !overwrite) {
            return;
        } else {
            log.warn("Warning : Multiple values found for " + key
                    + ". Using value " + val);
        }
    }

    param_source.put(key, val);

    val = val.substring(1, val.length()-1); //to remove the backticks
    String sub_val = substitute(val);
    sub_val = executeShellCommand(sub_val);
    param_val.put(key, sub_val);
}
 

@Test
public void testBlockErrMessage() throws Throwable {
    PigServer server = new PigServer(cluster.getExecType(), cluster.getProperties());
    PigContext context = server.getPigContext();

    String script = "A = load 'inputdata' using PigStorage() as ( curr_searchQuery );\n" +
            "B = foreach A { domain = CONCAT(curr_searchQuery,\"^www\\.\");\n" +
            "        generate domain; };\n";
    ByteArrayInputStream cmd = new ByteArrayInputStream(script.getBytes());
    InputStreamReader reader = new InputStreamReader(cmd);

    Grunt grunt = new Grunt(new BufferedReader(reader), context);

    try {
        grunt.exec();
    } catch(FrontendException e) {
        e.printStackTrace();
        assertTrue(e.getMessage().contains("Error during parsing. <line 2, column 49>  Unexpected character '\"'"));
    }
}
 

private OperatorSubPlan getSubPlan() throws FrontendException {
    OperatorSubPlan p = null;
    if (currentPlan instanceof OperatorSubPlan) {
        p = new OperatorSubPlan(((OperatorSubPlan)currentPlan).getBasePlan());
    } else {
        p = new OperatorSubPlan(currentPlan);
    }
    Iterator<Operator> iter = currentPlan.getOperators();

    while(iter.hasNext()) {
        Operator op = iter.next();
        if (op instanceof LOForEach) {
            addOperator(op, p);
        }
    }

    return p;
}
 

@Override
public void visit( MapLookupExpression op ) throws FrontendException {
    ExpressionOperator physOp = new POMapLookUp(new OperatorKey(DEFAULT_SCOPE,
            nodeGen.getNextNodeId(DEFAULT_SCOPE)));
    ((POMapLookUp)physOp).setLookUpKey(op.getLookupKey() );
    physOp.setResultType(op.getType());
    physOp.addOriginalLocation(op.getFieldSchema().alias, op.getLocation());
    currentPlan.add(physOp);

    logToPhyMap.put(op, physOp);

    ExpressionOperator from = (ExpressionOperator) logToPhyMap.get(op
            .getMap());
    try {
        currentPlan.connect(from, physOp);
    } catch (PlanException e) {
        int errCode = 2015;
        String msg = "Invalid physical operators in the physical plan" ;
        throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
    }
}
 

/**
 * Tests {@link BlackAndWhitelistValidator}. The logical plan generated
 * contains a filter, and the test must throw a {@link FrontendException} as
 * we set "filter" in the blacklist
 * 
 * @throws Exception
 */
@Test
public void testValidator() throws Exception {
    try {
        // disabling filter
        ctx.getProperties().setProperty(PigConfiguration.PIG_BLACKLIST, "filter");

        LogicalPlan plan = generateLogicalPlan("foo", "bar", ctx.getDfs());

        LogicalRelationalNodeValidator executor = new BlackAndWhitelistValidator(ctx, plan);
        executor.validate();
        fail();
    } catch (Exception e) {
        Util.assertExceptionAndMessage(FrontendException.class, e,
                "filter is disabled. ");
    }
}
 

private void runQuery(String outputPath, String compressionType)
      throws Exception, ExecException, IOException, FrontendException {

   // create a data file
   String filename = TestHelper.createTempFile(data, "");
   PigServer pig = new PigServer(LOCAL);
   filename = filename.replace("\\", "\\\\");
   patternString = patternString.replace("\\", "\\\\");
   String query = "A = LOAD '" + Util.encodeEscape(filename)
         + "' USING PigStorage(',') as (a,b,c);";

   String query2 = "STORE A INTO '" + Util.encodeEscape(outputPath)
         + "' USING org.apache.pig.piggybank.storage.MultiStorage" + "('"
         + Util.encodeEscape(outputPath) + "','0', '" + compressionType + "', '\\t');";

   // Run Pig
   pig.setBatchOn();
   pig.registerQuery(query);
   pig.registerQuery(query2);

   pig.executeBatch();
}
 

/**
 * @param fileSchema the parquet schema from the file
 * @param keyValueMetaData the extra meta data from the files
 * @return the pig schema according to the file
 */
static Schema getPigSchemaFromMultipleFiles(MessageType fileSchema, Map<String, Set<String>> keyValueMetaData) {
  Set<String> pigSchemas = PigMetaData.getPigSchemas(keyValueMetaData);
  if (pigSchemas == null) {
    return pigSchemaConverter.convert(fileSchema);
  }
  Schema mergedPigSchema = null;
  for (String pigSchemaString : pigSchemas) {
    try {
      mergedPigSchema = union(mergedPigSchema, parsePigSchema(pigSchemaString));
    } catch (FrontendException e) {
      throw new ParquetDecodingException("can not merge " + pigSchemaString + " into " + mergedPigSchema, e);
    }
  }
  return mergedPigSchema;
}
 

@Test
public void testPigStorageNoSchema() throws Exception {
    //if the schema file does not exist, and '-schema' option is used
    // it should result in an error
    pigContext.connect();
    String query = "a = LOAD '" + datadir + "originput' using PigStorage('\\t', '-schema') " +
    "as (f1:chararray, f2:int);";
    try{
        pig.registerQuery(query);
        pig.dumpSchema("a");
    }catch(FrontendException ex){
        assertEquals(ex.toString(), 1000, ex.getErrorCode());
        return;
    }
    fail("no exception caught");
}
 

@Override
public Schema outputSchema(Schema input)
{
  try
  {
    if (input.size() != 1)
    {
      throw new RuntimeException("Expected only a single field as input");
    }
    
    if (input.getField(0).type != DataType.BAG)
    {
      throw new RuntimeException("Expected a BAG as input, but found " + DataType.findTypeName(input.getField(0).type));
    }
    
    // get the size of the tuple within the bag
    int innerTupleSize = input.getField(0).schema.getField(0).schema.getFields().size();
    getInstanceProperties().put("tuplesize", innerTupleSize);
  }
  catch (FrontendException e)
  {
    throw new RuntimeException(e);
  }
  return input;
}
 

/***
 *  LODistinct, output schema should be the same as input
 * @param op
 * @throws VisitorException
 */

@Override
public void visit(LODistinct op) throws VisitorException {
    op.resetSchema();

    try {
        // Compute the schema
        op.getSchema() ;
    }
    catch (FrontendException fe) {
        int errCode = 1055;
        String msg = "Problem while reading"
            + " schemas from inputs of Distinct" ;
        msgCollector.collect(msg, MessageType.Error) ;
        throwTypeCheckerException(op, msg, errCode, PigException.INPUT, fe) ;
    }
}
 

private void insertCastsForUDF(UserFuncExpression func, Schema fromSch, Schema toSch, SchemaType toSchType)
throws FrontendException {
    List<FieldSchema> fsLst = fromSch.getFields();
    List<FieldSchema> tsLst = toSch.getFields();
    List<LogicalExpression> args = func.getArguments();
    int i=-1;
    for (FieldSchema fFSch : fsLst) {
        ++i;
        //if we get to the vararg field (if defined) : take it repeatedly
        FieldSchema tFSch = ((toSchType == SchemaType.VARARG) && i >= tsLst.size()) ? 
                tsLst.get(tsLst.size() - 1) : tsLst.get(i);
        if (fFSch.type == tFSch.type) {
            continue;
        }
        insertCast(func, Util.translateFieldSchema(tFSch), args.get(i));
    }
}
 

private void collectUids(LogicalRelationalOperator currentOp, LogicalExpressionPlan exp, Set<Long> uids) throws FrontendException {
    List<Operator> ll = exp.getSinks();
    for(Operator op: ll) {
        if (op instanceof ProjectExpression) {
            if (!((ProjectExpression)op).isRangeOrStarProject()) {
                long uid = ((ProjectExpression)op).getFieldSchema().uid;
                uids.add(uid);
            } else {
                LogicalRelationalOperator ref = ((ProjectExpression)op).findReferent();
                LogicalSchema s = ref.getSchema();
                if (s == null) {
                    throw new SchemaNotDefinedException("Schema not defined for " + ref.getAlias());
                }
                for(LogicalFieldSchema f: s.getFields()) {
                    uids.add(f.uid);
                }
            }
        }
    }
}
 

/***
 * Return concatenated of all fields from all input operators
 * If one of the inputs have no schema then we cannot construct
 * the output schema.
 * @param cs
 * @throws VisitorException
 */
public void visit(LOCross cs) throws VisitorException {
    cs.resetSchema();

    try {
        // Compute the schema
        cs.getSchema() ;
    }
    catch (FrontendException fe) {
        int errCode = 1055;
        String msg = "Problem while reading"
            + " schemas from inputs of Cross" ;
        msgCollector.collect(msg, MessageType.Error) ;
        throwTypeCheckerException(cs, msg, errCode, PigException.INPUT, fe) ;
    }
}
 

@Override
public Schema outputSchema(Schema input)
{
	try {
		if (!(input.size() == 2 || input.size() == 3))
		{
			throw new RuntimeException("Expected input to have two or three fields");
		}

		if (input.getField(1).type != DataType.INTEGER ) {
			throw new RuntimeException("Expected an INT as second input, got: "+input.getField(1).type);
		}

		return new Schema(input.getField(0).schema);
	}

	catch (FrontendException e) {
		e.printStackTrace();
		throw new RuntimeException(e);
	}
}
 

protected ProjExpanderForForeach(
        OperatorPlan p, 
        LOGenerate loGen, 
        Map<Integer, LogicalRelationalOperator> oldPos2Rel,
        Map<ProjectExpression, LogicalRelationalOperator> proj2InpRel,
        LOForEach foreach,
        List<LOInnerLoad> expandedInLoads
)
throws FrontendException {
    super(p, new ReverseDependencyOrderWalker(p));
    this.loGen = loGen;
    this.innerRelPlan = (LogicalPlan) loGen.getPlan();
    this.oldPos2Rel = oldPos2Rel;
    this.proj2InpRel = proj2InpRel;
    this.foreach = foreach;
    this.expandedInLoads = expandedInLoads;
    
}
 

@Override
public boolean isEqual(Operator other) throws FrontendException {
    if (other != null && other instanceof LORank) {
        LORank oR = (LORank)other;
        if (!rankColPlans.equals(oR.rankColPlans))
            return false;
    } else {
        return false;
    }

    return checkEquality((LogicalRelationalOperator)other);
}
 

@Override
public void accept(PlanVisitor v) throws FrontendException {
    if (!(v instanceof LogicalRelationalNodesVisitor)) {
        throw new FrontendException("Expected LogicalPlanVisitor", 2223);
    }
    ((LogicalRelationalNodesVisitor)v).visit(this);
}
 

@Override
public boolean isEqual(Operator other) throws FrontendException {
    if (other != null && other instanceof LOUnion) { 
        return checkEquality((LOUnion)other);
    } else {
        return false;
    }
}
 

/**
 * This is to test the script fails when used with {@link PigServer}, which
 * uses {@link QueryParser} and not the {@link GruntParser}
 */
@Test(expected = FrontendException.class)
public void testWhitelistWithPigServer() throws Exception {
    ctx.getProperties().setProperty(PigConfiguration.PIG_WHITELIST, "load");
    PigServer pigServer = new PigServer(ctx);
    Data data = resetData(pigServer);

    data.set("foo", tuple("a", 1, "b"), tuple("b", 2, "c"),
            tuple("c", 3, "d"));

    pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (f1:chararray,f2:int,f3:chararray);");
    pigServer.registerQuery("B = order A by f1,f2,f3 DESC;");
    pigServer.registerQuery("STORE B INTO 'bar' USING mock.Storage();");
}
 

@Override
public void visit( RegexExpression op ) throws FrontendException {        
    BinaryExpressionOperator exprOp = new PORegexp(new OperatorKey(DEFAULT_SCOPE, nodeGen.getNextNodeId(DEFAULT_SCOPE)));        
    
    attachBinaryExpressionOperator(op, exprOp);
    
    List<Operator> successors = op.getPlan().getSuccessors(op);
    if (successors.get(1) instanceof org.apache.pig.newplan.logical.expression.ConstantExpression) {
        ((PORegexp)exprOp).setConstExpr(true);
    }
}
 

public boolean processDistinct(PODistinct distinct) throws FrontendException {
    SortKeyInfo keyInfos = new SortKeyInfo();
    try {
        keyInfos.insertColumnChainInfo(0,
                (ColumnChainInfo) columnChainInfo.clone(), true);
    } catch (CloneNotSupportedException e) { // We implement Clonable,
                                             // impossible to get here
    }

    // if it is part of main key
    for (SortKeyInfo sortKeyInfo : sortKeyInfos) {
        if (sortKeyInfo.moreSpecificThan(keyInfos)) {
            distinctsToChange.add(distinct);
            return false;
        }
    }

    // if it is part of secondary key
    if (secondarySortKeyInfo != null
            && secondarySortKeyInfo.moreSpecificThan(keyInfos)) {
        distinctsToChange.add(distinct);
        return false;
    }

    // Now set the secondary key
    if (secondarySortKeyInfo == null) {
        distinctsToChange.add(distinct);
        secondarySortKeyInfo = keyInfos;
    }
    return false;
}
 

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    funcList.add(new FuncSpec(this.getClass().getName(), new Schema(new Schema.FieldSchema(null, DataType.DATETIME))));

    return funcList;
}
 

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> l = new ArrayList<FuncSpec>();
    Schema s1 = new Schema(new Schema.FieldSchema(null,DataType.INTEGER));
    l.add(new FuncSpec(this.getClass().getName(), s1));
    return l;
}
 

@Override
public void accept(PlanVisitor v) throws FrontendException {
    if (!(v instanceof LogicalRelationalNodesVisitor)) {
        throw new FrontendException("Expected LogicalPlanVisitor", 2223);
    }
    ((LogicalRelationalNodesVisitor)v).visit(this);
}
 

@Override
public boolean isEqual(Operator other) throws FrontendException {
    if (other != null && other instanceof DivideExpression) {
        DivideExpression ao = (DivideExpression)other;
        return ao.getLhs().isEqual(getLhs()) && ao.getRhs().isEqual(getRhs());
    } else {
        return false;
    }
}
 

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    funcList.add(new FuncSpec(this.getClass().getName(),
        new Schema(new FieldSchema(null, DataType.CHARARRAY))));
    funcList.add(new FuncSpec(IntTest.class.getName(),
        new Schema(new FieldSchema(null, DataType.INTEGER))));
    return funcList;
}
 

/**
 * Helper method to determine if the logical expression plan for a Filter contains
 * non-deterministic operations and should therefore be treated extra carefully
 * during optimization.
 *
 * @param filterPlan
 * @return true of the filter plan contains a non-deterministic UDF
 * @throws FrontendException 
 */
public static boolean planHasNonDeterministicUdf(LogicalExpressionPlan filterPlan)
throws FrontendException {
    Iterator<Operator> it = filterPlan.getOperators();
    while( it.hasNext() ) {
        Operator op = it.next();
        if( op instanceof UserFuncExpression ) {
            if(! ((UserFuncExpression)op).isDeterministic() ){
                return true;
            }
        }
    }
    return false;
}
 

@Override
public List<FuncSpec> getArgToFuncMapping() throws FrontendException {
    List<FuncSpec> funcList = new ArrayList<FuncSpec>();
    funcList.add(new FuncSpec(this.getClass().getName(), new Schema(new Schema.FieldSchema(null, DataType.BYTEARRAY))));
    funcList.add(new FuncSpec(DoubleSignum.class.getName(),  new Schema(new Schema.FieldSchema(null, DataType.DOUBLE))));
    funcList.add(new FuncSpec(FloatSignum.class.getName(),   new Schema(new Schema.FieldSchema(null, DataType.FLOAT))));
    return funcList;
}
 

@Override public void visit(LOInnerLoad load) throws FrontendException {
  // Inner loads are the first operator the post order walker (@PigRelOpWalker) visits first
  // We first look at the plan structure to see if the inner load is for a simple projection,
  // which will not be processed in the nested block
  List<Operator> succesors = load.getPlan().getSuccessors(load);

  // An inner load is for a simple projection if it is a direct input of the @LOGenerate.
  // Nothing need to be done further here.
  if (succesors.size() == 1 && succesors.get(0) instanceof LOGenerate) {
    return;
  }

  // Now get the index of projected column using its alias
  RelDataType inputType = inputRel.getRowType();
  final String colAlias = load.getProjection().getColAlias();
  int index = colAlias != null
                  ? inputType.getFieldNames().indexOf(colAlias)
                  : load.getProjection().getColNum();
  assert index >= 0;

  // The column should have multiset type to serve as input for the inner plan
  assert inputType.getFieldList().get(index).getType().getFamily() instanceof MultisetSqlType;

  // Build a correlated expression from the input row
  final CorrelationId correlId = builder.nextCorrelId();
  final RexNode cor = builder.correl(inputType.getFieldList(), correlId);

  // The project out the column from the correlated expression
  RexNode fieldAccess = builder.getRexBuilder().makeFieldAccess(cor, index);
  builder.push(LogicalValues.createOneRow(builder.getCluster()));
  builder.project(fieldAccess);

  // Flatten the column value so that it can be served as the input relation for the inner plan
  builder.multiSetFlatten();

  // Remember the correlation id, then the walker will walk up successor Pig operators. These
  // operators will be processed in @PigRelOpVisitor until it hits the @LOGenerate operator,
  // which will be processed in this class in visit(LOGenerate)
  corStack.push(correlId);
}