org.apache.pig.PigException Java Examples

@Override
public void accumulate(Tuple b) throws IOException {
    try {
        BigInteger curVal = doTupleWork(b, opProvider);
        if (curVal == null) {
            return;
        }
        if (intermediateVal == null) {
            intermediateVal = getSeed(opProvider.getOp());
        }
        intermediateVal = doWork(intermediateVal, curVal, opProvider.getOp());
    } catch (ExecException ee) {
        throw ee;
    } catch (Exception e) {
        int errCode = 2106;
        throw new ExecException("Error executing function on BigInteger", errCode, PigException.BUG, e);
    }
}
 

protected static Long doTupleWork(Tuple input, KnownOpProvider opProvider) throws ExecException {
    DataBag values = (DataBag)input.get(0);
    // if we were handed an empty bag, return NULL
    // this is in compliance with SQL standard
    if(values.size() == 0) {
        return null;
    }
    Long sofar = AlgebraicLongMathBase.getSeed(opProvider.getOp());
    boolean sawNonNull = false;
    for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
        Tuple t = it.next();
        try {
            // Note: use Number.longValue() instead of casting to Long
            // to allow Integers, too.
            Number n = ((Number)(t.get(0)));
            if (n == null) continue;
            Long d = n.longValue();
            sawNonNull = true;
            sofar = doWork(sofar, d, opProvider.getOp());
        }catch(RuntimeException exp) {
            int errCode = 2103;
            throw new ExecException("Problem doing work on Longs", errCode, PigException.BUG, exp);
        }
    }
    return sawNonNull ? sofar : null;
}
 

@Override
public void visit(NegativeExpression negExp) throws FrontendException {
    byte type = negExp.getExpression().getType() ;
    if (DataType.isNumberType(type)) {
        //do nothing
    }
    else if (type == DataType.BYTEARRAY) {
        // cast bytearray to double
        insertCast(negExp, DataType.DOUBLE, negExp.getExpression());
    }
    else {
        int errCode = 1041;
        String msg = "NEG can be used with numbers or Bytearray only" ;
        msgCollector.collect(msg, MessageType.Error);
        throw new TypeCheckerException(negExp, msg, errCode, PigException.INPUT) ;
    }

}
 

private Pair<POProject,Byte> [] getSortCols(List<PhysicalPlan> plans) throws PlanException, ExecException {
    if(plans!=null){
        @SuppressWarnings("unchecked")
        Pair<POProject,Byte>[] ret = new Pair[plans.size()];
        int i=-1;
        for (PhysicalPlan plan : plans) {
            PhysicalOperator op = plan.getLeaves().get(0);
            POProject proj;
            if (op instanceof POProject) {
                if (((POProject)op).isStar()) return null;
                proj = (POProject)op;
            } else {
                proj = null;
            }
            byte type = op.getResultType();
            ret[++i] = new Pair<POProject, Byte>(proj, type);
        }
        return ret;
    }
    int errCode = 2026;
    String msg = "No expression plan found in POSort.";
    throw new PlanException(msg, errCode, PigException.BUG);
}
 

/**
 * If stop_on_failure is enabled and any job has failed, an ExecException is thrown.
 * @param stop_on_failure whether it's enabled.
 * @throws ExecException If stop_on_failure is enabled and any job is failed
 */
private void checkStopOnFailure(boolean stop_on_failure) throws ExecException{
    if (jc.getFailedJobs().isEmpty())
        return;

    if (stop_on_failure){
        int errCode = 6017;
        StringBuilder msg = new StringBuilder();

        for (int i=0; i<jc.getFailedJobs().size(); i++) {
            Job j = jc.getFailedJobs().get(i);
            msg.append("JobID: " + j.getAssignedJobID() + " Reason: " + j.getMessage());
            if (i!=jc.getFailedJobs().size()-1) {
                msg.append("\n");
            }
        }

        throw new ExecException(msg.toString(), errCode,
                PigException.REMOTE_ENVIRONMENT);
    }
}
 

@Override
public Tuple getNext() throws IOException {
    try {
        if(!reader.nextKeyValue()) {
            return null; 
         }
         final PhoenixPigDBWritable record = reader.getCurrentValue();
        if(record == null) {
            return null;
        }
        final Tuple tuple = TypeUtil.transformToTuple(record,schema.getFields());
        return tuple;
   } catch (InterruptedException e) {
        int errCode = 6018;
        final String errMsg = "Error while reading input";
        throw new ExecException(errMsg, errCode,PigException.REMOTE_ENVIRONMENT, e);
    }
}
 

/**
 * {@link RegexExpression} expects CharArray as input
 * Itself always returns Boolean
 * @param rg
 * @throws FrontendException
 */
@Override
public void visit(RegexExpression rg) throws FrontendException {
    // We allow BYTEARRAY to be converted to CHARARRAY
    if (rg.getLhs().getType() == DataType.BYTEARRAY){
        insertCast(rg, DataType.CHARARRAY, rg.getLhs());
    }
    if (rg.getRhs().getType() == DataType.BYTEARRAY){
        insertCast(rg, DataType.CHARARRAY, rg.getRhs());
    }

    // Other than that if it's not CharArray just say goodbye
    if (rg.getLhs().getType() != DataType.CHARARRAY ||
            rg.getRhs().getType() != DataType.CHARARRAY)
    {
        int errCode = 1037;
        String msg = "Operands of Regex can be CharArray only :" + rg;
        msgCollector.collect(msg, MessageType.Error);
        throw new TypeCheckerException(rg, msg, errCode, PigException.INPUT) ;
    }
}
 

@Override
public void accumulate(Tuple b) throws IOException {
    try {
        String curMin = min(b);
        if (curMin == null) {
            return;
        }
        // check if it lexicographically follows curMax
        if (intermediateMin == null || intermediateMin.compareTo(curMin) < 0) {
            intermediateMin = curMin;
        }            

    } catch (ExecException ee) {
        throw ee;
    } catch (Exception e) {
        int errCode = 2106;
        String msg = "Error while computing max in " + this.getClass().getSimpleName();
        throw new ExecException(msg, errCode, PigException.BUG, e);           
    }
}
 

/**
 * Gets the positions in the schema which are byte arrays
 * @param func
 *
 * @param s -
 *            input schema
 * @throws VisitorException
 */
private List<Integer> getByteArrayPositions(UserFuncExpression func, Schema s)
        throws VisitorException {
    List<Integer> result = new ArrayList<Integer>();
    for (int i = 0; i < s.size(); i++) {
        try {
            FieldSchema fs = s.getField(i);
            if (fs.type == DataType.BYTEARRAY) {
                result.add(i);
            }
        } catch (FrontendException fee) {
            int errCode = 1043;
            String msg = "Unable to retrieve field schema.";
            throw new TypeCheckerException(func, msg, errCode, PigException.INPUT, fee);            }
    }
    return result;
}
 

/**
 * Add given uid, loadFuncSpec to mapping
 * @param uid
 * @param loadFuncSpec
 * @throws VisitorException 
 */
private void addUidLoadFuncToMap(long uid, FuncSpec loadFuncSpec)
throws VisitorException{
    if(loadFuncSpec == null){
        return;
    }
    //ensure that uid always matches to same load func
    FuncSpec curFuncSpec = uid2LoadFuncMap.get(uid);
    if(curFuncSpec == null){
        uid2LoadFuncMap.put(uid, loadFuncSpec);
    }else if(! haveIdenticalCasters(curFuncSpec,loadFuncSpec)){
        String msg = "Bug: uid mapped to two different load functions : " +
        curFuncSpec + " and " + loadFuncSpec;
        throw new VisitorException(msg,2262, PigException.BUG) ;
    }
}
 

private void handlePackage(TezOperator pkgTezOp, POPackage pkg) throws VisitorException {
    // the LocalRearrange(s) must be in the plan of a predecessor tez op
    int lrFound = 0;
    List<TezOperator> preds = this.mPlan.getPredecessors(pkgTezOp);
    for (Iterator<TezOperator> it = preds.iterator(); it.hasNext();) {
        TezOperator predTezOp = it.next();
        if (predTezOp.isVertexGroup()) {
            // Just get one of the inputs to vertex group
            predTezOp = getPlan().getOperator(predTezOp.getVertexGroupMembers().get(0));
        }
        lrFound += patchPackage(predTezOp, pkgTezOp, pkg);
        if(lrFound == pkg.getNumInps()) {
            break;
        }
    }

    if(lrFound != pkg.getNumInps()) {
        int errCode = 2086;
        String msg = "Unexpected problem during optimization. Could not find all LocalRearrange operators.";
        throw new OptimizerException(msg, errCode, PigException.BUG);
    }
}
 

@Test
public void test5() throws Exception {
    // Passing bytearrays
    // Possible matches: (float, float) , (long, double)
    // Throws exception since more than one funcSpec and inp is bytearray
    try {
        pigServer.registerQuery("A = LOAD '" + inputFile + "';");
        pigServer.registerQuery("B = FOREACH A generate $0, " + UDF1.class.getName()
                + "($1,$1);");
        pigServer.openIterator("B");
    } catch (Exception e) {
        PigException pe = LogUtils.getPigException(e);
        String msg = (pe == null ? e.getMessage() : pe.getMessage());
        assertEquals(true, msg.contains("Multiple matching functions"));
    }

}
 

@Override
public void visit( MapLookupExpression op ) throws FrontendException {
    ExpressionOperator physOp = new POMapLookUp(new OperatorKey(DEFAULT_SCOPE,
            nodeGen.getNextNodeId(DEFAULT_SCOPE)));
    ((POMapLookUp)physOp).setLookUpKey(op.getLookupKey() );
    physOp.setResultType(op.getType());
    physOp.addOriginalLocation(op.getFieldSchema().alias, op.getLocation());
    currentPlan.add(physOp);

    logToPhyMap.put(op, physOp);

    ExpressionOperator from = (ExpressionOperator) logToPhyMap.get(op
            .getMap());
    try {
        currentPlan.connect(from, physOp);
    } catch (PlanException e) {
        int errCode = 2015;
        String msg = "Invalid physical operators in the physical plan" ;
        throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
    }
}
 

public static PigException getPigException(Throwable top) {
    Throwable current = top;
    Throwable pigException = top;

    if (current instanceof PigException && 
            (((PigException)current).getErrorCode() != 0) && 
            ((PigException) current).getMarkedAsShowToUser()) {
        return (PigException) current;
    }
    while (current != null && current.getCause() != null){
        current = current.getCause();
        if((current instanceof PigException) && 
                (((PigException)current).getErrorCode() != 0)) {
            pigException = current;
            if (((PigException)pigException).getMarkedAsShowToUser()) {
                break;
            }
        }
    }
    return (pigException instanceof PigException? (PigException)pigException 
    		: null);        
}
 

public void setupRightPipeline(PhysicalPlan rightPipeline) throws FrontendException{

        if(rightPipeline != null){
            if(rightPipeline.getLeaves().size() != 1 || rightPipeline.getRoots().size() != 1){
                int errCode = 2168;
                String errMsg = "Expected physical plan with exactly one root and one leaf.";
                throw new FrontendException(errMsg,errCode,PigException.BUG);
            }

            noInnerPlanOnRightSide = false;
            this.rightPipelineLeaf = rightPipeline.getLeaves().get(0);
            this.rightPipelineRoot = rightPipeline.getRoots().get(0);
            this.rightPipelineRoot.setInputs(null);            
        }
        else
            noInnerPlanOnRightSide = true;
    }
 

public MRCompiler(PhysicalPlan plan,
        PigContext pigContext) throws MRCompilerException {
    super(plan, new DepthFirstWalker<PhysicalOperator, PhysicalPlan>(plan));
    this.plan = plan;
    this.pigContext = pigContext;
    splitsSeen = new HashMap<OperatorKey, MapReduceOper>();
    MRPlan = new MROperPlan();
    nig = NodeIdGenerator.getGenerator();
    udfFinder = new UDFFinder();
    List<PhysicalOperator> roots = plan.getRoots();
    if((roots == null) || (roots.size() <= 0)) {
        int errCode = 2053;
        String msg = "Internal error. Did not find roots in the physical plan.";
        throw new MRCompilerException(msg, errCode, PigException.BUG);
    }
    scope = roots.get(0).getOperatorKey().getScope();
    messageCollector = new CompilationMessageCollector() ;
    phyToMROpMap = new HashMap<PhysicalOperator, MapReduceOper>();

    fileConcatenationThreshold = Integer.parseInt(pigContext.getProperties()
            .getProperty(FILE_CONCATENATION_THRESHOLD, "100"));
    optimisticFileConcatenation = pigContext.getProperties().getProperty(
            OPTIMISTIC_FILE_CONCATENATION, "false").equals("true");
    LOG.info("File concatenation threshold: " + fileConcatenationThreshold
            + " optimistic? " + optimisticFileConcatenation);
}
 

protected static Integer doTupleWork(Tuple input, KnownOpProvider opProvider) throws ExecException {
    DataBag values = (DataBag)input.get(0);
    // if we were handed an empty bag, return NULL
    // this is in compliance with SQL standard
    if(values.size() == 0) {
        return null;
    }
    int sofar = AlgebraicIntMathBase.getSeed(opProvider.getOp());
    boolean sawNonNull = false;
    for (Iterator<Tuple> it = values.iterator(); it.hasNext();) {
        Tuple t = it.next();
        try {
            Integer d = (Integer)(t.get(0));
            if (d == null) continue;
            sawNonNull = true;
            sofar = doWork(sofar, d, opProvider.getOp());
        }catch(RuntimeException exp) {
            int errCode = 2103;
            throw new ExecException("Problem doing work on Doubles", errCode, PigException.BUG, exp);
        }
    }
    return sawNonNull ? sofar : null;
}
 

@Override
public BigDecimal exec(Tuple input) throws IOException {
    try {
        DataBag b = (DataBag)input.get(0);
        Tuple combined = combine(b);

        BigDecimal sum = (BigDecimal)combined.get(0);
        if (sum == null) {
            return null;
        }
        BigDecimal count = (BigDecimal)combined.get(1);

        BigDecimal avg = null;
        if (count.compareTo(BigDecimal.ZERO) > 0) {
            avg = div(sum,count);
        }
        return avg;
    } catch (ExecException ee) {
        throw ee;
    } catch (Exception e) {
        int errCode = 2106;
        String msg = "Error while computing average in " + this.getClass().getSimpleName();
        throw new ExecException(msg, errCode, PigException.BUG, e);
    }
}
 

@Override
public void accumulate(Tuple b) throws IOException {
    try {
        Integer curVal = doTupleWork(b, opProvider);
        if (curVal == null) {
            return;
        }
        if (intermediateVal == null) {
            intermediateVal = getSeed(opProvider.getOp());
        }
        intermediateVal = doWork(intermediateVal, curVal, opProvider.getOp());
    } catch (ExecException ee) {
        throw ee;
    } catch (Exception e) {
        int errCode = 2106;
        throw new ExecException("Error executing function on Doubles", errCode, PigException.BUG, e);
    }
}
 

@Override
public void readFields(DataInput in) throws IOException {
    // Clear our fields, in case we're being reused.
    mFields.clear();

    // Make sure it's a tuple.
    byte b = in.readByte();
    if (b != DataType.TUPLE) {
        int errCode = 2112;
        String msg = "Unexpected data while reading tuple " + "from binary file.";
        throw new ExecException(msg, errCode, PigException.BUG);
    }
    // Read the number of fields
    int sz = in.readInt();
    for (int i = 0; i < sz; i++) {
        try {
            append(DataReaderWriter.readDatum(in));
        } catch (ExecException ee) {
            throw ee;
        }
    }
}
 

@Override
public void accumulate(Tuple b) throws IOException {
    try {
        String curMax = max(b);
        if (curMax == null) {
            return;
        }
        // check if it lexicographically follows curMax
        if (intermediateMax == null || intermediateMax.compareTo(curMax) > 0) {
            intermediateMax = curMax;
        }            

    } catch (ExecException ee) {
        throw ee;
    } catch (Exception e) {
        int errCode = 2106;
        String msg = "Error while computing max in " + this.getClass().getSimpleName();
        throw new ExecException(msg, errCode, PigException.BUG, e);           
    }
}
 

public HPath[] asCollection(String pattern) throws DataStorageException {
    try {
        FileStatus[] paths = this.fs.globStatus(new Path(pattern));

        if (paths == null)
            return new HPath[0];

        List<HPath> hpaths = new ArrayList<HPath>();
        
        for (int i = 0; i < paths.length; ++i) {
            HPath hpath = (HPath)this.asElement(paths[i].getPath().toString());
            if (!hpath.systemElement()) {
                hpaths.add(hpath);
            }
        }

        return hpaths.toArray(new HPath[hpaths.size()]);
    } catch (IOException e) {
        int errCode = 6008;
        String msg = "Failed to obtain glob for " + pattern;
        throw new DataStorageException(msg, errCode, PigException.REMOTE_ENVIRONMENT, e);
    }
}
 

@Override
public Double exec(Tuple input) throws IOException {
    try {
        DataBag b = (DataBag)input.get(0);
        Tuple combined = combine(b);

        Double sum = (Double)combined.get(0);
        if(sum == null) {
            return null;
        }
        double count = (Long)combined.get(1);

        Double avg = null;
        if (count > 0) {
            avg = new Double(sum / count);
        }
        return avg;
    } catch (ExecException ee) {
        throw ee;
    } catch (Exception e) {
        int errCode = 2106;
        String msg = "Error while computing average in " + this.getClass().getSimpleName();
        throw new ExecException(msg, errCode, PigException.BUG, e);
    }
}
 

@Override
public LogicalFieldSchema getFieldSchema() throws FrontendException {
    if (fieldSchema!=null)
        return fieldSchema;
    try {
        fieldSchema =  Util.translateFieldSchema(DataType.determineFieldSchema(val));
    }catch (Exception e) {
        throw new FrontendException(
                "Error determining field schema from object in constant expression",
                1125,
                PigException.INPUT,
                e
        );
    }
    uidOnlyFieldSchema = fieldSchema.mergeUid(uidOnlyFieldSchema);
    if (type != DataType.NULL && fieldSchema.type == DataType.BYTEARRAY && val == null) {
        fieldSchema.type = type;
    }
    return fieldSchema;
}
 

@Override
    public void visit( IsNullExpression op ) throws FrontendException {
        POIsNull pIsNull = new POIsNull(new OperatorKey(DEFAULT_SCOPE, nodeGen
                .getNextNodeId(DEFAULT_SCOPE)));
//        physOp.setAlias(op.getAlias());
        currentPlan.add(pIsNull);

        logToPhyMap.put(op, pIsNull);
        ExpressionOperator from = (ExpressionOperator) logToPhyMap.get(op
                .getExpression());
        pIsNull.setExpr(from);
        pIsNull.setResultType(op.getType());
        pIsNull.setOperandType(op.getExpression().getType());
        try {
            currentPlan.connect(from, pIsNull);
        } catch (PlanException e) {
            int errCode = 2015;
            String msg = "Invalid physical operators in the physical plan" ;
            throw new LogicalToPhysicalTranslatorException(msg, errCode, PigException.BUG, e);
        }
    }
 

@Override
public void accumulate(Tuple b) throws IOException {
    try {
        Double sum = sum(b);
        if(sum == null) {
            return;
        }
        // set default values
        if (intermediateSum == null || intermediateCount == null) {
            intermediateSum = 0.0;
            intermediateCount = 0.0;
        }

        double count = (Long)count(b);

        if (count > 0) {
            intermediateCount += count;
            intermediateSum += sum;
        }
    } catch (ExecException ee) {
        throw ee;
    } catch (Exception e) {
        int errCode = 2106;
        String msg = "Error while computing average in " + this.getClass().getSimpleName();
        throw new ExecException(msg, errCode, PigException.BUG, e);
    }
}
 

/**
 * Get a file to spill contents to.  The file will be registered in the
 * mSpillFiles array.
 * @return stream to write tuples to.
 */
protected DataOutputStream getSpillFile() throws IOException {
    if (mSpillFiles == null) {
        // We want to keep the list as small as possible.
        mSpillFiles = new FileList(1);
    }

    String tmpDirName= System.getProperties().getProperty("java.io.tmpdir") ;
    File tmpDir = new File(tmpDirName);

    // if the directory does not exist, create it.
    if (!tmpDir.exists()){
        log.info("Temporary directory doesn't exists. Trying to create: " + tmpDir.getAbsolutePath());
      // Create the directory and see if it was successful
      if (tmpDir.mkdir()){
        log.info("Successfully created temporary directory: " + tmpDir.getAbsolutePath());
      } else {
          // If execution reaches here, it means that we needed to create the directory but
          // were not successful in doing so.
          //
          // If this directory is created recently then we can simply
          // skip creation. This is to address a rare issue occuring in a cluster despite the
          // the fact that spill() makes call to getSpillFile() in a synchronized
          // block.
          if (tmpDir.exists()) {
            log.info("Temporary directory already exists: " + tmpDir.getAbsolutePath());
          } else {
            int errCode = 2111;
            String msg = "Unable to create temporary directory: " + tmpDir.getAbsolutePath();
            throw new ExecException(msg, errCode, PigException.BUG);
          }
      }
    }

    File f = File.createTempFile("pigbag", null);
    f.deleteOnExit();
    mSpillFiles.add(f);
    return new DataOutputStream(new BufferedOutputStream(
        new FileOutputStream(f)));
}
 

@Override
public Tuple exec(Tuple input) throws IOException {
    try {
        return tfact.newTuple(doTupleWork(input, this));
    } catch (ExecException ee) {
        throw ee;
    } catch (Exception e) {
        int errCode = 2106;
        throw new ExecException("Error executing function on BigDecimal", errCode, PigException.BUG, e);
    }
}
 

public void doInsertBetween(
        E after,
        E newNode,
        E before,
        boolean rewire) throws PlanException {
    checkInPlan(newNode);
    List<E> newNodePreds = getPredecessors(newNode);
    //assuming that the newNode has zero or one predecessor
    E newNodePred = (newNodePreds == null? null: newNodePreds.get(0));
    if (!replaceNode(after, newNode, before, mFromEdges) || !replaceNode(before, newNode, after, mToEdges)) {
        int errCode = 1094;
        String msg = "Attempt to insert between two nodes " +
        "that were not connected.";
        PlanException pe = new PlanException(msg, errCode, PigException.INPUT);
        throw pe;
    }
    mFromEdges.put(newNode, before);
    mToEdges.put(newNode, after);

    if(rewire) {
        if((newNodePred != null) && !(newNodePred.equals(after))) {
            newNodePred.regenerateProjectionMap();
            newNode.rewire(newNodePred, 0, after, true);
        }
        newNode.regenerateProjectionMap();
        IndexHelper<E> indexHelper = new IndexHelper<E>(getPredecessors(newNode));
        before.rewire(after, indexHelper.getIndex(after), newNode, false);
    }
}
 

@Override
public Double exec(Tuple input) throws IOException {
    try {
        return doTupleWork(input, opProvider, DataType.BYTEARRAY);
    } catch (ExecException ee) {
        throw ee;
    } catch (Exception e) {
        int errCode = 2106;
        throw new ExecException("Error executing function on Doubles", errCode, PigException.BUG, e);
    }
}