Java Code Examples for org.apache.pig.impl.PigContext#resolveClassName()

The following examples show how to use org.apache.pig.impl.PigContext#resolveClassName() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Invoker.java    From spork with Apache License 2.0 6 votes vote down vote up 
public Invoker(String fullName, String paramSpecsStr, String isStatic)
throws ClassNotFoundException, FrontendException, SecurityException, NoSuchMethodException {
    String className = fullName.substring(0, fullName.lastIndexOf('.'));
    String methodName = fullName.substring(fullName.lastIndexOf('.')+1);
    Class<?> klazz;
    try {
        klazz = PigContext.resolveClassName(className);
    } catch (IOException e) {
        // the amusing part is that PigContext throws this to wrap one of
        // the exceptions we declare!
        throw new FrontendException(e);
    }
    String[] paramSpecs = "".equals(paramSpecsStr) ? new String[0] : paramSpecsStr.split(" ");
    isStatic_ = "static".equalsIgnoreCase(isStatic) || "true".equals(isStatic);
    paramClasses_ = new Class<?>[paramSpecs.length];
    for (int i = 0; i < paramSpecs.length; i++) {
        paramClasses_[i] = stringToClass(paramSpecs[i]);
    }
    if (!isStatic_) {
        selfClass_ = paramClasses_[0];
    }
    method_ = klazz.getMethod(methodName, (isStatic_ ? paramClasses_ : dropFirstClass(paramClasses_)));
    returnType_ = method_.getGenericReturnType();
}
Example 2
Source File: TezDagBuilder.java    From spork with Apache License 2.0 6 votes vote down vote up 
private void setOutputFormat(org.apache.hadoop.mapreduce.Job job) {
    // the OutputFormat we report to Hadoop is always PigOutputFormat which
    // can be wrapped with LazyOutputFormat provided if it is supported by
    // the Hadoop version and PigConfiguration.PIG_OUTPUT_LAZY is set
    if ("true".equalsIgnoreCase(job.getConfiguration().get(PigConfiguration.PIG_OUTPUT_LAZY))) {
        try {
            Class<?> clazz = PigContext
                    .resolveClassName("org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat");
            Method method = clazz.getMethod("setOutputFormatClass",
                    org.apache.hadoop.mapreduce.Job.class, Class.class);
            method.invoke(null, job, PigOutputFormatTez.class);
        } catch (Exception e) {
            job.setOutputFormatClass(PigOutputFormatTez.class);
            log.warn(PigConfiguration.PIG_OUTPUT_LAZY
                    + " is set but LazyOutputFormat couldn't be loaded. Default PigOutputFormat will be used");
        }
    } else {
        job.setOutputFormatClass(PigOutputFormatTez.class);
    }
}
Example 3
Source File: JobControlCompiler.java    From spork with Apache License 2.0 6 votes vote down vote up 
public static void setOutputFormat(org.apache.hadoop.mapreduce.Job job) {
    // the OutputFormat we report to Hadoop is always PigOutputFormat which
    // can be wrapped with LazyOutputFormat provided if it is supported by
    // the Hadoop version and PigConfiguration.PIG_OUTPUT_LAZY is set
    if ("true".equalsIgnoreCase(job.getConfiguration().get(PigConfiguration.PIG_OUTPUT_LAZY))) {
        try {
            Class<?> clazz = PigContext
                    .resolveClassName("org.apache.hadoop.mapreduce.lib.output.LazyOutputFormat");
            Method method = clazz.getMethod("setOutputFormatClass",
                    org.apache.hadoop.mapreduce.Job.class, Class.class);
            method.invoke(null, job, PigOutputFormat.class);
        } catch (Exception e) {
            job.setOutputFormatClass(PigOutputFormat.class);
            log.warn(PigConfiguration.PIG_OUTPUT_LAZY
                    + " is set but LazyOutputFormat couldn't be loaded. Default PigOutputFormat will be used");
        }
    } else {
        job.setOutputFormatClass(PigOutputFormat.class);
    }
}
Example 4
Source File: UserFuncExpression.java    From spork with Apache License 2.0 5 votes vote down vote up 
public boolean isDeterministic() throws FrontendException{
    Class<?> udfClass;
    try {
        udfClass = PigContext.resolveClassName(getFuncSpec().getClassName());
    }catch(IOException ioe) {
        throw new FrontendException("Cannot instantiate: " + getFuncSpec(), ioe) ;
    }

    if (udfClass.getAnnotation(Nondeterministic.class) == null) {
        return true;
    }
    return false;

}
Example 5
Source File: LogicalPlanBuilder.java    From spork with Apache License 2.0 5 votes vote down vote up 
private void validateFuncSpec(SourceLocation loc, FuncSpec funcSpec, byte ft) throws RecognitionException {
    switch (ft) {
    case FunctionType.COMPARISONFUNC:
    case FunctionType.LOADFUNC:
    case FunctionType.STOREFUNC:
    case FunctionType.STREAMTOPIGFUNC:
    case FunctionType.PIGTOSTREAMFUNC:
        try{
            Class<?> func = PigContext.resolveClassName(funcSpec.getClassName());
            FunctionType.tryCasting(func, ft);
        } catch(Exception ex){
            throw new ParserValidationException(intStream, loc, ex);
        }
    }
}
Example 6
Source File: InvokerGenerator.java    From spork with Apache License 2.0 5 votes vote down vote up 
private Class<?>[] getArgumentClassArray(String[] argumentTypes) {
    Class<?>[] arguments = new Class<?>[argumentTypes.length];
    for (int i = 0; i < argumentTypes.length; i++) {
        try {
            arguments[i]= nameToClassObjectMap.get(argumentTypes[i]);
            if (arguments[i] == null) {
                arguments[i] = PigContext.resolveClassName(argumentTypes[i]);
            }
        } catch (IOException e) {
            throw new RuntimeException("Unable to find class in PigContext: " + argumentTypes[i], e);
        }
    }
    return arguments;
}
Example 7
Source File: TezCompilerUtil.java    From spork with Apache License 2.0 5 votes vote down vote up 
static public void setCustomPartitioner(String customPartitioner, TezOperator tezOp) throws IOException {
    if (customPartitioner != null) {
        for (TezEdgeDescriptor edge : tezOp.inEdges.values()) {
            edge.partitionerClass = PigContext.resolveClassName(customPartitioner);
        }
    }
}
Example 8
Source File: TestPigContextClassCache.java    From spork with Apache License 2.0 5 votes vote down vote up 
@Test
public void testClassCache() throws IOException {
    Class<?> c = PigContext.resolveClassName("TOMAP");
    Assert.assertEquals(TOMAP.class, c);
    
    c = PigContext.resolveClassName("TOMAP");
    Assert.assertEquals(TOMAP.class, c);
    
    c = PigContext.resolveClassName("org.apache.pig.builtin.REPLACE");
    Assert.assertEquals(REPLACE.class, c);
}
Example 9
Source File: AccumulatorOptimizerUtil.java    From spork with Apache License 2.0 4 votes vote down vote up 
/**
 * Check if an operator is qualified to be under POForEach to turn on
 * accumulator. The operator must be in the following list or an
 * <code>POUserFunc</code>.
 *
 * If the operator has sub-operators, they must also belong to this list.
 * <li>ConstantExpression</li>
 * <li>POProject, whose result type is not BAG, or TUPLE and overloaded</li>
 * <li>POMapLookup</li>
 * <li>POCase</li>
 * <li>UnaryExpressionOperator</li>
 * <li>BinaryExpressionOperator</li>
 * <li>POBinCond</li>
 *
 * If the operator is <code>POUserFunc</code>, it must implement
 * <code>Accumulator</code> interface and its inputs pass the check by
 * calling <code>checkUDFInput()</code>
 *
 * @param po the operator to be checked on
 * @return <code>true</code> if it is ok, <code>false</code> if not.
 */
private static boolean check(PhysicalOperator po) {
    if (po instanceof ConstantExpression) {
        return true;
    }

    if (po instanceof POCast) {
        return check(po.getInputs().get(0));
    }

    if (po instanceof POMapLookUp) {
        return check(po.getInputs().get(0));
    }

    if (po instanceof POProject) {
        // POProject can not project data bag
        if (((POProject)po).getResultType() == DataType.BAG) {
            return false;
        }

        // POProject can not overload a data bag
        if (((POProject)po).getResultType() == DataType.TUPLE && ((POProject)po).isOverloaded()) {
            return false;
        }

        return true;
    }

    if (po instanceof UnaryExpressionOperator) {
        return check(((UnaryExpressionOperator)po).getExpr());
    }

    if (po instanceof BinaryExpressionOperator) {
        return check(((BinaryExpressionOperator)po).getLhs()) &&
                check(((BinaryExpressionOperator)po).getRhs());
    }

    if (po instanceof POBinCond) {
        return check(((POBinCond)po).getLhs()) &&
            check(((POBinCond)po).getRhs()) && check(((POBinCond)po).getCond());
    }

    if (po instanceof POUserFunc) {
        String className = ((POUserFunc)po).getFuncSpec().getClassName();
        @SuppressWarnings("rawtypes")
        Class c = null;
        try {
            c = PigContext.resolveClassName(className);
        } catch (Exception e) {
            return false;
        }
        if (!Accumulator.class.isAssignableFrom(c)) {
            return false;
        }

        // check input of UDF
        List<PhysicalOperator> inputs = po.getInputs();
        for (PhysicalOperator p: inputs) {
            if (!checkUDFInput(p)) {
                return false;
            }
        }

        return true;
    }

    return false;
}